update circle ci to build make (#2769)

* update build.py script to work with circleci
* build make with circle ci
* build vm for esp only
* nrf imxrt with large resource
* nrf imxrt with large resource
* remove 2 of nrf boards

2 files changed, 94 insertions, 112 deletions

diff --git a/tools/build.py b/tools/build.py
index b937a7342..f9ae68231 100644
--- a/tools/build.py
+++ b/tools/build.py
@@ -9,19 +9,29 @@ from multiprocessing import Pool
 
 import build_utils
 
-SUCCEEDED = "\033[32msucceeded\033[0m"
-FAILED = "\033[31mfailed\033[0m"
+STATUS_OK = "\033[32mOK\033[0m"
+STATUS_FAILED = "\033[31mFailed\033[0m"
+STATUS_SKIPPED = "\033[33mSkipped\033[0m"
 
-build_separator = '-' * 106
+RET_OK = 0
+RET_FAILED = 1
+RET_SKIPPED = 2
 
+build_format = '| {:30} | {:40} | {:16} | {:5} |'
+build_separator = '-' * 95
+build_status = [STATUS_OK, STATUS_FAILED, STATUS_SKIPPED]
 
+
+# -----------------------------
+# Helper
+# -----------------------------
 def run_cmd(cmd):
     #print(cmd)
     r = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    title = 'command error'
+    title = f'Command Error: {cmd}'
     if r.returncode != 0:
         # print build output if failed
-        if os.getenv('CI'):
+        if os.getenv('GITHUB_ACTIONS'):
             print(f"::group::{title}")
             print(r.stdout.decode("utf-8"))
             print(f"::endgroup::")
@@ -30,6 +40,7 @@ def run_cmd(cmd):
             print(r.stdout.decode("utf-8"))
     return r
 
+
 def find_family(board):
     bsp_dir = Path("hw/bsp")
     for family_dir in bsp_dir.iterdir():
@@ -56,70 +67,91 @@ def get_examples(family):
     return all_examples
 
 
-def build_board_cmake(board, toolchain):
-    start_time = time.monotonic()
-    ret = [0, 0, 0]
+def print_build_result(board, example, status, duration):
+    if isinstance(duration, (int, float)):
+        duration = "{:.2f}s".format(duration)
+    print(build_format.format(board, example, build_status[status], duration))
 
+# -----------------------------
+# CMake
+# -----------------------------
+def cmake_board(board, toolchain):
+    ret = [0, 0, 0]
+    start_time = time.monotonic()
     build_dir = f"cmake-build/cmake-build-{board}"
     family = find_family(board)
     if family == 'espressif':
         # for espressif, we have to build example individually
         all_examples = get_examples(family)
         for example in all_examples:
-            r = run_cmd(f'cmake examples/{example} -B {build_dir}/{example} -G "Ninja" -DBOARD={board} -DMAX3421_HOST=1')
-            if r.returncode == 0:
-                r = run_cmd(f'cmake --build {build_dir}/{example}')
-            if r.returncode == 0:
-                ret[0] += 1
-            else:
-                ret[1] += 1
+            rcmd = run_cmd(f'cmake examples/{example} -B {build_dir}/{example} -G "Ninja" -DBOARD={board} -DMAX3421_HOST=1')
+            if rcmd.returncode == 0:
+                rcmd = run_cmd(f'cmake --build {build_dir}/{example}')
+            ret[0 if rcmd.returncode == 0 else 1] += 1
     else:
-        r = run_cmd(f'cmake examples -B {build_dir} -G "Ninja" -DBOARD={board} -DCMAKE_BUILD_TYPE=MinSizeRel -DTOOLCHAIN={toolchain}')
-        if r.returncode == 0:
-            r = run_cmd(f"cmake --build {build_dir}")
-        if r.returncode == 0:
-            ret[0] += 1
-        else:
-            ret[1] += 1
+        rcmd = run_cmd(f'cmake examples -B {build_dir} -G "Ninja" -DBOARD={board} -DCMAKE_BUILD_TYPE=MinSizeRel -DTOOLCHAIN={toolchain}')
+        if rcmd.returncode == 0:
+            rcmd = run_cmd(f"cmake --build {build_dir}")
+        ret[0 if rcmd.returncode == 0 else 1] += 1
 
-    duration = time.monotonic() - start_time
+    example = 'all'
+    print_build_result(board, example, 0 if ret[1] == 0 else 1, time.monotonic() - start_time)
+    return ret
 
-    if ret[1] == 0:
-        status = SUCCEEDED
+
+# -----------------------------
+# Make
+# -----------------------------
+def make_one_example(example, board, make_option):
+    # Check if board is skipped
+    if build_utils.skip_example(example, board):
+        print_build_result(board, example, 2, '-')
+        r = 2
     else:
-        status = FAILED
+        start_time = time.monotonic()
+        # skip -j for circleci
+        if not os.getenv('CIRCLECI'):
+            make_option += ' -j'
+        make_cmd = f"make -C examples/{example} BOARD={board} {make_option}"
+        # run_cmd(f"{make_cmd} clean")
+        build_result = run_cmd(f"{make_cmd} all")
+        r = 0 if build_result.returncode == 0 else 1
+        print_build_result(board, example, r, time.monotonic() - start_time)
 
-    flash_size = "-"
-    sram_size = "-"
-    example = 'all'
-    title = build_utils.build_format.format(example, board, status, "{:.2f}s".format(duration), flash_size, sram_size)
-    print(title)
+    ret = [0, 0, 0]
+    ret[r] = 1
     return ret
 
 
-def build_board_make_all_examples(board, toolchain, all_examples):
+def make_board(board, toolchain):
+    print(build_separator)
+    all_examples = get_examples(find_family(board))
     start_time = time.monotonic()
     ret = [0, 0, 0]
-
     with Pool(processes=os.cpu_count()) as pool:
         pool_args = list((map(lambda e, b=board, o=f"TOOLCHAIN={toolchain}": [e, b, o], all_examples)))
-        r = pool.starmap(build_utils.build_example, pool_args)
+        r = pool.starmap(make_one_example, pool_args)
         # sum all element of same index (column sum)
-        rsum = list(map(sum, list(zip(*r))))
-        ret[0] += rsum[0]
-        ret[1] += rsum[1]
-        ret[2] += rsum[2]
-    duration = time.monotonic() - start_time
-    if ret[1] == 0:
-        status = SUCCEEDED
-    else:
-        status = FAILED
-
-    flash_size = "-"
-    sram_size = "-"
+        ret = list(map(sum, list(zip(*r))))
     example = 'all'
-    title = build_utils.build_format.format(example, board, status, "{:.2f}s".format(duration), flash_size, sram_size)
-    print(title)
+    print_build_result(board, example, 0 if ret[1] == 0 else 1, time.monotonic() - start_time)
+    return ret
+
+
+# -----------------------------
+# Build Family
+# -----------------------------
+def build_boards_list(boards, toolchain, build_system):
+    ret = [0, 0, 0]
+    for b in boards:
+        r = [0, 0, 0]
+        if build_system == 'cmake':
+            r = cmake_board(b, toolchain)
+        elif build_system == 'make':
+            r = make_board(b, toolchain)
+        ret[0] += r[0]
+        ret[1] += r[1]
+        ret[2] += r[2]
     return ret
 
 
@@ -131,7 +163,6 @@ def build_family(family, toolchain, build_system, one_per_family, boards):
     all_boards.sort()
 
     ret = [0, 0, 0]
-
     # If only-one flag is set, select one random board
     if one_per_family:
         for b in boards:
@@ -140,21 +171,13 @@ def build_family(family, toolchain, build_system, one_per_family, boards):
                 return ret
         all_boards = [random.choice(all_boards)]
 
-    # success, failed, skipped
-    all_examples = get_examples(family)
-    for board in all_boards:
-        r = [0, 0, 0]
-        if build_system == 'cmake':
-            r = build_board_cmake(board, toolchain)
-        elif build_system == 'make':
-            r = build_board_make_all_examples(board, toolchain, all_examples)
-        ret[0] += r[0]
-        ret[1] += r[1]
-        ret[2] += r[2]
-
+    ret = build_boards_list(all_boards, toolchain, build_system)
     return ret
 
 
+# -----------------------------
+# Main
+# -----------------------------
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('families', nargs='*', default=[], help='Families to build')
@@ -175,7 +198,7 @@ def main():
         return 1
 
     print(build_separator)
-    print(build_utils.build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
+    print(build_format.format('Board', 'Example', '\033[39mResult\033[0m', 'Time'))
     total_time = time.monotonic()
     result = [0, 0, 0]
 
@@ -189,28 +212,22 @@ def main():
         all_families = list(families)
     all_families.sort()
 
-    # succeeded, failed
+    # succeeded, failed, skipped
     for f in all_families:
-        fret = build_family(f, toolchain, build_system, one_per_family, boards)
-        result[0] += fret[0]
-        result[1] += fret[1]
-        result[2] += fret[2]
-
-    # build boards
-    for b in boards:
-        r = [0, 0, 0]
-        if build_system == 'cmake':
-            r = build_board_cmake(b, toolchain)
-        elif build_system == 'make':
-            all_examples = get_examples(find_family(b))
-            r = build_board_make_all_examples(b, toolchain, all_examples)
+        r = build_family(f, toolchain, build_system, one_per_family, boards)
         result[0] += r[0]
         result[1] += r[1]
         result[2] += r[2]
 
+    # build boards
+    r = build_boards_list(boards, toolchain, build_system)
+    result[0] += r[0]
+    result[1] += r[1]
+    result[2] += r[2]
+
     total_time = time.monotonic() - total_time
     print(build_separator)
-    print(f"Build Summary: {result[0]} {SUCCEEDED}, {result[1]} {FAILED} and took {total_time:.2f}s")
+    print(f"Build Summary: {result[0]} {STATUS_OK}, {result[1]} {STATUS_FAILED} and took {total_time:.2f}s")
     print(build_separator)
     return result[1]
 
diff --git a/tools/build_utils.py b/tools/build_utils.py
index 32aca95dd..5f88db8a2 100644
--- a/tools/build_utils.py
+++ b/tools/build_utils.py
@@ -92,38 +92,3 @@ def build_size(make_cmd):
             return (flash_size, sram_size)
 
     return (0, 0)
-
-
-def build_example(example, board, make_option):
-    start_time = time.monotonic()
-    flash_size = "-"
-    sram_size = "-"
-
-    # succeeded, failed, skipped
-    ret = [0, 0, 0]
-
-    make_cmd = f"make -j -C examples/{example} BOARD={board} {make_option}"
-
-    # Check if board is skipped
-    if skip_example(example, board):
-        status = SKIPPED
-        ret[2] = 1
-        print(build_format.format(example, board, status, '-', flash_size, sram_size))
-    else:
-        build_result = subprocess.run(f"{make_cmd} all", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-
-        if build_result.returncode == 0:
-            status = SUCCEEDED
-            ret[0] = 1
-            (flash_size, sram_size) = build_size(make_cmd)
-        else:
-            status = FAILED
-            ret[1] = 1
-
-        build_duration = time.monotonic() - start_time
-        print(build_format.format(example, board, status, "{:.2f}s".format(build_duration), flash_size, sram_size))
-
-        if build_result.returncode != 0:
-            print(build_result.stdout.decode("utf-8"))
-
-    return ret