Merge remote-tracking branch 'upstream/master' into hcd-template-comments

Signed-off-by: HiFiPhile <[email protected]>

1 files changed, 126 insertions, 77 deletions

diff --git a/tools/build.py b/tools/build.py
index b937a7342..633d2b582 100644..100755
--- a/tools/build.py
+++ b/tools/build.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 import argparse
 import random
 import os
@@ -9,27 +10,42 @@ from multiprocessing import Pool
 
 import build_utils
 
-SUCCEEDED = "\033[32msucceeded\033[0m"
-FAILED = "\033[31mfailed\033[0m"
+STATUS_OK = "\033[32mOK\033[0m"
+STATUS_FAILED = "\033[31mFailed\033[0m"
+STATUS_SKIPPED = "\033[33mSkipped\033[0m"
 
-build_separator = '-' * 106
+RET_OK = 0
+RET_FAILED = 1
+RET_SKIPPED = 2
 
+build_format = '| {:30} | {:40} | {:16} | {:5} |'
+build_separator = '-' * 95
+build_status = [STATUS_OK, STATUS_FAILED, STATUS_SKIPPED]
 
+verbose = False
+
+# -----------------------------
+# Helper
+# -----------------------------
 def run_cmd(cmd):
     #print(cmd)
     r = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    title = 'command error'
+    title = f'Command Error: {cmd}'
     if r.returncode != 0:
         # print build output if failed
-        if os.getenv('CI'):
+        if os.getenv('GITHUB_ACTIONS'):
             print(f"::group::{title}")
             print(r.stdout.decode("utf-8"))
             print(f"::endgroup::")
         else:
             print(title)
             print(r.stdout.decode("utf-8"))
+    elif verbose:
+        print(cmd)
+        print(r.stdout.decode("utf-8"))
     return r
 
+
 def find_family(board):
     bsp_dir = Path("hw/bsp")
     for family_dir in bsp_dir.iterdir():
@@ -52,78 +68,120 @@ def get_examples(family):
     if family == 'espressif':
         all_examples.append('device/board_test')
         all_examples.append('device/video_capture')
+        all_examples.append('host/device_info')
     all_examples.sort()
     return all_examples
 
 
-def build_board_cmake(board, toolchain):
-    start_time = time.monotonic()
+def print_build_result(board, example, status, duration):
+    if isinstance(duration, (int, float)):
+        duration = "{:.2f}s".format(duration)
+    print(build_format.format(board, example, build_status[status], duration))
+
+# -----------------------------
+# CMake
+# -----------------------------
+def cmake_board(board, toolchain, build_flags_on):
     ret = [0, 0, 0]
+    start_time = time.monotonic()
+
+    build_dir = f'cmake-build/cmake-build-{board}'
+    build_flags = ''
+    if len(build_flags_on) > 0:
+        build_flags =  ' '.join(f'-D{flag}=1' for flag in build_flags_on)
+        build_flags = f'-DCFLAGS_CLI="{build_flags}"'
+        build_dir += '-f1_' + '_'.join(build_flags_on)
 
-    build_dir = f"cmake-build/cmake-build-{board}"
     family = find_family(board)
     if family == 'espressif':
         # for espressif, we have to build example individually
         all_examples = get_examples(family)
         for example in all_examples:
-            r = run_cmd(f'cmake examples/{example} -B {build_dir}/{example} -G "Ninja" -DBOARD={board} -DMAX3421_HOST=1')
-            if r.returncode == 0:
-                r = run_cmd(f'cmake --build {build_dir}/{example}')
-            if r.returncode == 0:
-                ret[0] += 1
+            if build_utils.skip_example(example, board):
+                ret[2] += 1
             else:
-                ret[1] += 1
+                rcmd = run_cmd(f'cmake examples/{example} -B {build_dir}/{example} -G "Ninja" '
+                               f'-DBOARD={board} {build_flags}')
+                if rcmd.returncode == 0:
+                    rcmd = run_cmd(f'cmake --build {build_dir}/{example}')
+                ret[0 if rcmd.returncode == 0 else 1] += 1
     else:
-        r = run_cmd(f'cmake examples -B {build_dir} -G "Ninja" -DBOARD={board} -DCMAKE_BUILD_TYPE=MinSizeRel -DTOOLCHAIN={toolchain}')
-        if r.returncode == 0:
-            r = run_cmd(f"cmake --build {build_dir}")
-        if r.returncode == 0:
-            ret[0] += 1
-        else:
-            ret[1] += 1
+        rcmd = run_cmd(f'cmake examples -B {build_dir} -G "Ninja" -DBOARD={board} -DCMAKE_BUILD_TYPE=MinSizeRel '
+                       f'-DTOOLCHAIN={toolchain} {build_flags}')
+        if rcmd.returncode == 0:
+            cmd = f"cmake --build {build_dir}"
+            # circleci docker return $nproc as 36 core, limit parallel according to resource class. Required for IAR, also prevent crashed/killed by docker
+            if os.getenv('CIRCLECI'):
+                resource_class = { 'small': 1, 'medium': 2, 'medium+': 3, 'large': 4 }
+                for rc in resource_class:
+                    if rc in os.getenv('CIRCLE_JOB'):
+                        cmd += f' --parallel {resource_class[rc]}'
+                        break
+            rcmd = run_cmd(cmd)
+        ret[0 if rcmd.returncode == 0 else 1] += 1
 
-    duration = time.monotonic() - start_time
+    example = 'all'
+    print_build_result(board, example, 0 if ret[1] == 0 else 1, time.monotonic() - start_time)
+    return ret
 
-    if ret[1] == 0:
-        status = SUCCEEDED
+
+# -----------------------------
+# Make
+# -----------------------------
+def make_one_example(example, board, make_option):
+    # Check if board is skipped
+    if build_utils.skip_example(example, board):
+        print_build_result(board, example, 2, '-')
+        r = 2
     else:
-        status = FAILED
+        start_time = time.monotonic()
+        # skip -j for circleci
+        if not os.getenv('CIRCLECI'):
+            make_option += ' -j'
+        make_cmd = f"make -C examples/{example} BOARD={board} {make_option}"
+        # run_cmd(f"{make_cmd} clean")
+        build_result = run_cmd(f"{make_cmd} all")
+        r = 0 if build_result.returncode == 0 else 1
+        print_build_result(board, example, r, time.monotonic() - start_time)
 
-    flash_size = "-"
-    sram_size = "-"
-    example = 'all'
-    title = build_utils.build_format.format(example, board, status, "{:.2f}s".format(duration), flash_size, sram_size)
-    print(title)
+    ret = [0, 0, 0]
+    ret[r] = 1
     return ret
 
 
-def build_board_make_all_examples(board, toolchain, all_examples):
+def make_board(board, toolchain):
+    print(build_separator)
+    all_examples = get_examples(find_family(board))
     start_time = time.monotonic()
     ret = [0, 0, 0]
-
     with Pool(processes=os.cpu_count()) as pool:
         pool_args = list((map(lambda e, b=board, o=f"TOOLCHAIN={toolchain}": [e, b, o], all_examples)))
-        r = pool.starmap(build_utils.build_example, pool_args)
+        r = pool.starmap(make_one_example, pool_args)
         # sum all element of same index (column sum)
-        rsum = list(map(sum, list(zip(*r))))
-        ret[0] += rsum[0]
-        ret[1] += rsum[1]
-        ret[2] += rsum[2]
-    duration = time.monotonic() - start_time
-    if ret[1] == 0:
-        status = SUCCEEDED
-    else:
-        status = FAILED
-
-    flash_size = "-"
-    sram_size = "-"
+        ret = list(map(sum, list(zip(*r))))
     example = 'all'
-    title = build_utils.build_format.format(example, board, status, "{:.2f}s".format(duration), flash_size, sram_size)
-    print(title)
+    print_build_result(board, example, 0 if ret[1] == 0 else 1, time.monotonic() - start_time)
+    return ret
+
+
+# -----------------------------
+# Build Family
+# -----------------------------
+def build_boards_list(boards, toolchain, build_system, build_flags_on):
+    ret = [0, 0, 0]
+    for b in boards:
+        r = [0, 0, 0]
+        if build_system == 'cmake':
+            r = cmake_board(b, toolchain, build_flags_on)
+        elif build_system == 'make':
+            r = make_board(b, toolchain)
+        ret[0] += r[0]
+        ret[1] += r[1]
+        ret[2] += r[2]
     return ret
 
 
-def build_family(family, toolchain, build_system, one_per_family, boards):
+def build_family(family, toolchain, build_system, build_flags_on, one_per_family, boards):
     all_boards = []
     for entry in os.scandir(f"hw/bsp/{family}/boards"):
         if entry.is_dir() and entry.name != 'pico_sdk':
@@ -131,7 +189,6 @@ def build_family(family, toolchain, build_system, one_per_family, boards):
     all_boards.sort()
 
     ret = [0, 0, 0]
-
     # If only-one flag is set, select one random board
     if one_per_family:
         for b in boards:
@@ -140,42 +197,40 @@ def build_family(family, toolchain, build_system, one_per_family, boards):
                 return ret
         all_boards = [random.choice(all_boards)]
 
-    # success, failed, skipped
-    all_examples = get_examples(family)
-    for board in all_boards:
-        r = [0, 0, 0]
-        if build_system == 'cmake':
-            r = build_board_cmake(board, toolchain)
-        elif build_system == 'make':
-            r = build_board_make_all_examples(board, toolchain, all_examples)
-        ret[0] += r[0]
-        ret[1] += r[1]
-        ret[2] += r[2]
-
+    ret = build_boards_list(all_boards, toolchain, build_system, build_flags_on)
     return ret
 
 
+# -----------------------------
+# Main
+# -----------------------------
 def main():
+    global verbose
+
     parser = argparse.ArgumentParser()
     parser.add_argument('families', nargs='*', default=[], help='Families to build')
     parser.add_argument('-b', '--board', action='append', default=[], help='Boards to build')
     parser.add_argument('-t', '--toolchain', default='gcc', help='Toolchain to use, default is gcc')
     parser.add_argument('-s', '--build-system', default='cmake', help='Build system to use, default is cmake')
+    parser.add_argument('-f1', '--build-flags-on', action='append', default=[], help='Build flag to pass to build system')
     parser.add_argument('-1', '--one-per-family', action='store_true', default=False, help='Build only one random board inside a family')
+    parser.add_argument('-v', '--verbose', action='store_true', help='Verbose output')
     args = parser.parse_args()
 
     families = args.families
     boards = args.board
     toolchain = args.toolchain
     build_system = args.build_system
+    build_flags_on = args.build_flags_on
     one_per_family = args.one_per_family
+    verbose = args.verbose
 
     if len(families) == 0 and len(boards) == 0:
         print("Please specify families or board to build")
         return 1
 
     print(build_separator)
-    print(build_utils.build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
+    print(build_format.format('Board', 'Example', '\033[39mResult\033[0m', 'Time'))
     total_time = time.monotonic()
     result = [0, 0, 0]
 
@@ -189,28 +244,22 @@ def main():
         all_families = list(families)
     all_families.sort()
 
-    # succeeded, failed
+    # succeeded, failed, skipped
     for f in all_families:
-        fret = build_family(f, toolchain, build_system, one_per_family, boards)
-        result[0] += fret[0]
-        result[1] += fret[1]
-        result[2] += fret[2]
-
-    # build boards
-    for b in boards:
-        r = [0, 0, 0]
-        if build_system == 'cmake':
-            r = build_board_cmake(b, toolchain)
-        elif build_system == 'make':
-            all_examples = get_examples(find_family(b))
-            r = build_board_make_all_examples(b, toolchain, all_examples)
+        r = build_family(f, toolchain, build_system, build_flags_on, one_per_family, boards)
         result[0] += r[0]
         result[1] += r[1]
         result[2] += r[2]
 
+    # build boards
+    r = build_boards_list(boards, toolchain, build_system, build_flags_on)
+    result[0] += r[0]
+    result[1] += r[1]
+    result[2] += r[2]
+
     total_time = time.monotonic() - total_time
     print(build_separator)
-    print(f"Build Summary: {result[0]} {SUCCEEDED}, {result[1]} {FAILED} and took {total_time:.2f}s")
+    print(f"Build Summary: {result[0]} {STATUS_OK}, {result[1]} {STATUS_FAILED} and took {total_time:.2f}s")
     print(build_separator)
     return result[1]