More ci tweak (#2636)

* change concurrency group to ${{ github.workflow }}-${{ github.ref }}
* use argparse for build.py hil_test.py, remove the need to install click
* move ci win/mac to build_cmake.yml
* rename build_family.yml to build_util.yml
* build_util.yml support esp32
* integrate build-espressif into build.yml
* build.py support make with --board option
* add get_deps action
* update hil test to reuse action

3 files changed, 37 insertions, 126 deletions

diff --git a/tools/build.py b/tools/build.py
index 884cd56f4..967f7c95e 100644
--- a/tools/build.py
+++ b/tools/build.py
@@ -1,8 +1,8 @@
+import argparse
 import os
 import sys
 import time
 import subprocess
-import click
 from pathlib import Path
 from multiprocessing import Pool
 
@@ -125,13 +125,20 @@ def build_family(family, toolchain, build_system):
     return ret
 
 
[email protected]()
[email protected]('families', nargs=-1, required=False)
[email protected]('-b', '--board', multiple=True, default=None, help='Boards to build')
[email protected]('-t', '--toolchain', default='gcc', help='Toolchain to use, default is gcc')
[email protected]('-s', '--build-system', default='cmake', help='Build system to use, default is cmake')
-def main(families, board, toolchain, build_system):
-    if len(families) == 0 and len(board) == 0:
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('families', nargs='*', default=[], help='Families to build')
+    parser.add_argument('-b', '--board', action='append', default=[], help='Boards to build')
+    parser.add_argument('-t', '--toolchain', default='gcc', help='Toolchain to use, default is gcc')
+    parser.add_argument('-s', '--build-system', default='cmake', help='Build system to use, default is cmake')
+    args = parser.parse_args()
+
+    families = args.families
+    boards = args.board
+    toolchain = args.toolchain
+    build_system = args.build_system
+
+    if len(families) == 0 and len(boards) == 0:
         print("Please specify families or board to build")
         return 1
 
@@ -159,12 +166,23 @@ def main(families, board, toolchain, build_system):
             total_result[2] += fret[2]
 
     # build board (only cmake)
-    if board is not None:
-        for b in board:
-            r = build_board_cmake(b, toolchain)
-            total_result[0] += r[0]
-            total_result[1] += r[1]
-            total_result[2] += r[2]
+    if boards is not None:
+        for b in boards:
+            if build_system == 'cmake':
+                r = build_board_cmake(b, toolchain)
+                total_result[0] += r[0]
+                total_result[1] += r[1]
+                total_result[2] += r[2]
+            elif build_system == 'make':
+                all_examples = get_examples(find_family(b))
+                with Pool(processes=os.cpu_count()) as pool:
+                    pool_args = list((map(lambda e, bb=b, o=f"TOOLCHAIN={toolchain}": [e, bb, o], all_examples)))
+                    r = pool.starmap(build_utils.build_example, pool_args)
+                    # sum all element of same index (column sum)
+                    rsum = list(map(sum, list(zip(*r))))
+                    total_result[0] += rsum[0]
+                    total_result[1] += rsum[1]
+                    total_result[2] += rsum[2]
 
     total_time = time.monotonic() - total_time
     print(build_separator)
diff --git a/tools/build_esp32.py b/tools/build_esp32.py
deleted file mode 100644
index 951467c23..000000000
--- a/tools/build_esp32.py
+++ /dev/null
@@ -1,106 +0,0 @@
-import os
-import glob
-import sys
-import subprocess
-import time
-
-import build_utils
-
-SUCCEEDED = "\033[32msucceeded\033[0m"
-FAILED = "\033[31mfailed\033[0m"
-SKIPPED = "\033[33mskipped\033[0m"
-
-success_count = 0
-fail_count = 0
-skip_count = 0
-exit_status = 0
-
-total_time = time.monotonic()
-
-build_format = '| {:30} | {:30} | {:18} | {:7} | {:6} | {:6} |'
-build_separator = '-' * 107
-
-def filter_with_input(mylist):
-    if len(sys.argv) > 1:
-        input_args = list(set(mylist).intersection(sys.argv))
-        if len(input_args) > 0:
-            mylist[:] = input_args
-
-
-# Build all examples if not specified
-all_examples = [entry.replace('examples/', '') for entry in glob.glob("examples/*/*_freertos")]
-filter_with_input(all_examples)
-all_examples.append('device/board_test')
-all_examples.sort()
-
-# Build all boards if not specified
-all_boards = []
-for entry in os.scandir("hw/bsp/espressif/boards"):
-    if entry.is_dir():
-        all_boards.append(entry.name)
-filter_with_input(all_boards)
-all_boards.sort()
-
-def build_board(example, board):
-    global success_count, fail_count, skip_count, exit_status
-    start_time = time.monotonic()
-
-    # Check if board is skipped
-    build_dir = f"cmake-build/cmake-build-{board}/{example}"
-
-    # Generate and build
-    r = subprocess.run(f"cmake examples/{example} -B {build_dir} -G \"Ninja\" -DBOARD={board} -DMAX3421_HOST=1",
-                       shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    if r.returncode == 0:
-        r = subprocess.run(f"cmake --build {build_dir}", shell=True, stdout=subprocess.PIPE,
-                           stderr=subprocess.STDOUT)
-    build_duration = time.monotonic() - start_time
-    flash_size = "-"
-    sram_size = "-"
-
-    if r.returncode == 0:
-        success = SUCCEEDED
-        success_count += 1
-        #(flash_size, sram_size) = build_size(example, board)
-    else:
-        exit_status = r.returncode
-        success = FAILED
-        fail_count += 1
-
-    title = build_format.format(example, board, success, "{:.2f}s".format(build_duration), flash_size, sram_size)
-    if os.getenv('CI'):
-        # always print build output if in CI
-        print(f"::group::{title}")
-        print(r.stdout.decode("utf-8"))
-        print(f"::endgroup::")
-    else:
-        # print build output if failed
-        print(title)
-        if r.returncode != 0:
-            print(r.stdout.decode("utf-8"))
-
-
-def build_size(example, board):
-    #elf_file = 'examples/device/{}/_build/{}/{}-firmware.elf'.format(example, board, board)
-    elf_file = 'examples/device/{}/_build/{}/*.elf'.format(example, board)
-    size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8")
-    size_list = size_output.split('\n')[1].split('\t')
-    flash_size = int(size_list[0])
-    sram_size = int(size_list[1]) + int(size_list[2])
-    return (flash_size, sram_size)
-
-
-print(build_separator)
-print(build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
-print(build_separator)
-
-for example in all_examples:
-    for board in all_boards:
-        build_board(example, board)
-
-total_time = time.monotonic() - total_time
-print(build_separator)
-print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(success_count, SUCCEEDED, fail_count, FAILED, skip_count, SKIPPED, total_time))
-print(build_separator)
-
-sys.exit(exit_status)
diff --git a/tools/get_deps.py b/tools/get_deps.py
index 2359b4bd0..20cbe64c7 100644
--- a/tools/get_deps.py
+++ b/tools/get_deps.py
@@ -245,11 +245,10 @@ def main():
     args = parser.parse_args()
 
     families = args.families
-    board = args.board
+    boards = args.board
 
-    if len(families) == 0 and len(board) == 0:
-        print("Please specify family or board to fetch")
-        return
+    if len(families) == 0 and len(boards) == 0:
+        print("Warning: family and board are not specified, only fetching mandatory dependencies.")
 
     status = 0
     deps = list(deps_mandatory.keys())
@@ -258,8 +257,8 @@ def main():
         deps += deps_optional.keys()
     else:
         families = list(families)
-        if board is not None:
-            for b in board:
+        if boards is not None:
+            for b in boards:
                 f = find_family(b)
                 if f is not None:
                     families.append(f)