From 8fe9022a6e6dea785092f4e0dff8180f1f0d4d86 Mon Sep 17 00:00:00 2001 From: hathach Date: Wed, 29 Jun 2022 14:06:44 +0700 Subject: fix buid_board.py script --- tools/build_utils.py | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) (limited to 'tools/build_utils.py') diff --git a/tools/build_utils.py b/tools/build_utils.py index 299fffa4d..68c2ed98b 100644 --- a/tools/build_utils.py +++ b/tools/build_utils.py @@ -4,26 +4,34 @@ def skip_example(example, board): ex_dir = pathlib.Path('examples/') / example bsp = pathlib.Path("hw/bsp") - board_dir = list(bsp.glob("*/boards/" + board)) - if not board_dir: - # Skip unknown boards - return True - - board_dir = list(board_dir)[0] + # board without family + board_dir = bsp / board - family_dir = board_dir.parent.parent - family = family_dir.name - - # family CMake - family_mk = family_dir / "family.cmake" - - # family.mk - if not family_mk.exists(): - family_mk = family_dir / "family.mk" - - mk_contents = family_mk.read_text() + if board_dir.exists(): + family = "" + mk_contents = "" + else: + # otherwise look into family + board_dir = list(bsp.glob("*/boards/" + board)) + if not board_dir: + # Skip unknown boards + return True + + board_dir = list(board_dir)[0] + + family_dir = board_dir.parent.parent + family = family_dir.name + + # family CMake + family_mk = family_dir / "family.cmake" + + # family.mk + if not family_mk.exists(): + family_mk = family_dir / "family.mk" + + mk_contents = family_mk.read_text() - # Find the mcu + # Find the mcu, first in family mk then board mk if "CFG_TUSB_MCU=OPT_MCU_" not in mk_contents: board_mk = board_dir / "board.cmake" if not board_mk.exists(): -- cgit v1.3.1 From 4da1a9359d7fa9813a01f133f919533184753064 Mon Sep 17 00:00:00 2001 From: hathach Date: Wed, 29 Jun 2022 15:12:01 +0700 Subject: fix build with fomu/fomu --- tools/build_utils.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'tools/build_utils.py') diff --git a/tools/build_utils.py b/tools/build_utils.py index 68c2ed98b..d570d20e3 100644 --- a/tools/build_utils.py +++ b/tools/build_utils.py @@ -3,15 +3,14 @@ import pathlib def skip_example(example, board): ex_dir = pathlib.Path('examples/') / example bsp = pathlib.Path("hw/bsp") - - # board without family - board_dir = bsp / board - if board_dir.exists(): + if (bsp / board / "board.mk").exists(): + # board without family + board_dir = bsp / board family = "" mk_contents = "" else: - # otherwise look into family + # board within family board_dir = list(bsp.glob("*/boards/" + board)) if not board_dir: # Skip unknown boards -- cgit v1.3.1 From 8f9ecace4d9fecb14fbf8e8563d2f19334805d4d Mon Sep 17 00:00:00 2001 From: hathach Date: Wed, 29 Jun 2022 21:06:02 +0700 Subject: update build_board.py to parallel build --- tools/build_board.py | 119 ++++++++++++++++++-------------------------------- tools/build_family.py | 96 ++++++++++++---------------------------- tools/build_utils.py | 66 +++++++++++++++++++++++++--- 3 files changed, 129 insertions(+), 152 deletions(-) (limited to 'tools/build_utils.py') diff --git a/tools/build_board.py b/tools/build_board.py index 4d895329a..62a4ea82f 100644 --- a/tools/build_board.py +++ b/tools/build_board.py @@ -1,8 +1,7 @@ import os -import glob import sys -import subprocess import time +from multiprocessing import Pool import build_utils @@ -10,90 +9,56 @@ SUCCEEDED = "\033[32msucceeded\033[0m" FAILED = "\033[31mfailed\033[0m" SKIPPED = "\033[33mskipped\033[0m" -success_count = 0 -fail_count = 0 -skip_count = 0 -exit_status = 0 - -total_time = time.monotonic() - -build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |' build_separator = '-' * 106 + def filter_with_input(mylist): if len(sys.argv) > 1: input_args = list(set(mylist).intersection(sys.argv)) if len(input_args) > 0: mylist[:] = input_args -# If examples are not specified in arguments, build all -all_examples = [] -for dir1 in os.scandir("examples"): - if dir1.is_dir(): - for entry in os.scandir(dir1.path): - if entry.is_dir(): - all_examples.append(dir1.name + '/' + entry.name) -filter_with_input(all_examples) -all_examples.sort() - -# If boards are not specified in arguments, build all -all_boards = [] -for entry in os.scandir("hw/bsp"): - if entry.is_dir() and os.path.exists(entry.path + "/board.mk"): - all_boards.append(entry.name) -filter_with_input(all_boards) -all_boards.sort() - -def build_board(example, board): - global success_count, fail_count, skip_count, exit_status - start_time = time.monotonic() - flash_size = "-" - sram_size = "-" - - # Check if board is skipped - if build_utils.skip_example(example, board): - success = SKIPPED - skip_count += 1 - print(build_format.format(example, board, success, '-', flash_size, sram_size)) - else: - subprocess.run("make -C examples/{} BOARD={} clean".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - - if build_result.returncode == 0: - success = SUCCEEDED - success_count += 1 - (flash_size, sram_size) = build_size(example, board) - else: - exit_status = build_result.returncode - success = FAILED - fail_count += 1 - build_duration = time.monotonic() - start_time - print(build_format.format(example, board, success, "{:.2f}s".format(build_duration), flash_size, sram_size)) +if __name__ == '__main__': + # If examples are not specified in arguments, build all + all_examples = [] + for dir1 in os.scandir("examples"): + if dir1.is_dir(): + for entry in os.scandir(dir1.path): + if entry.is_dir(): + all_examples.append(dir1.name + '/' + entry.name) + filter_with_input(all_examples) + all_examples.sort() + + # If boards are not specified in arguments, build all + all_boards = [] + for entry in os.scandir("hw/bsp"): + if entry.is_dir() and os.path.exists(entry.path + "/board.mk"): + all_boards.append(entry.name) + filter_with_input(all_boards) + all_boards.sort() - if build_result.returncode != 0: - print(build_result.stdout.decode("utf-8")) - -def build_size(example, board): - #elf_file = 'examples/device/{}/_build/{}/{}-firmware.elf'.format(example, board, board) - elf_file = 'examples/{}/_build/{}/*.elf'.format(example, board) - size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8") - size_list = size_output.split('\n')[1].split('\t') - flash_size = int(size_list[0]) - sram_size = int(size_list[1]) + int(size_list[2]) - return (flash_size, sram_size) - -print(build_separator) -print(build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM')) - -for example in all_examples: print(build_separator) - for board in all_boards: - build_board(example, board) - -total_time = time.monotonic() - total_time -print(build_separator) -print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(success_count, SUCCEEDED, fail_count, FAILED, skip_count, SKIPPED, total_time)) -print(build_separator) + print(build_utils.build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM')) + total_time = time.monotonic() + + # succeeded, failed, skipped + total_result = [0, 0, 0] + for example in all_examples: + print(build_separator) + with Pool(processes=os.cpu_count()) as pool: + pool_args = list((map(lambda b, e=example: [e, b], all_boards))) + result = pool.starmap(build_utils.build_example, pool_args) + # sum all element of same index (column sum) + result = list(map(sum, list(zip(*result)))) + + # add to total result + total_result = list(map(lambda x, y: x + y, total_result, result)) + + total_time = time.monotonic() - total_time + print(build_separator) + print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(total_result[0], SUCCEEDED, total_result[1], + FAILED, total_result[2], SKIPPED, total_time)) + print(build_separator) -sys.exit(exit_status) + sys.exit(total_result[1]) diff --git a/tools/build_family.py b/tools/build_family.py index 57a6c4d17..c6c64d2b3 100644 --- a/tools/build_family.py +++ b/tools/build_family.py @@ -1,7 +1,5 @@ import os -import glob import sys -import subprocess import time from multiprocessing import Pool @@ -11,33 +9,15 @@ SUCCEEDED = "\033[32msucceeded\033[0m" FAILED = "\033[31mfailed\033[0m" SKIPPED = "\033[33mskipped\033[0m" -build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |' build_separator = '-' * 106 + def filter_with_input(mylist): if len(sys.argv) > 1: input_args = list(set(mylist).intersection(sys.argv)) if len(input_args) > 0: mylist[:] = input_args -# If examples are not specified in arguments, build all -all_examples = [] -for dir1 in os.scandir("examples"): - if dir1.is_dir(): - for entry in os.scandir(dir1.path): - if entry.is_dir(): - all_examples.append(dir1.name + '/' + entry.name) -filter_with_input(all_examples) -all_examples.sort() - -# If family are not specified in arguments, build all -all_families = [] -for entry in os.scandir("hw/bsp"): - if entry.is_dir() and os.path.isdir(entry.path + "/boards") and entry.name not in ("esp32s2", "esp32s3"): - all_families.append(entry.name) - -filter_with_input(all_families) -all_families.sort() def build_family(example, family): all_boards = [] @@ -47,70 +27,48 @@ def build_family(example, family): filter_with_input(all_boards) all_boards.sort() - with Pool(processes=len(all_boards)) as pool: + with Pool(processes=os.cpu_count()) as pool: pool_args = list((map(lambda b, e=example: [e, b], all_boards))) - result = pool.starmap(build_board, pool_args) + result = pool.starmap(build_utils.build_example, pool_args) + # sum all element of same index (column sum) return list(map(sum, list(zip(*result)))) - -def build_board(example, board): - start_time = time.monotonic() - flash_size = "-" - sram_size = "-" - - # succeeded, failed, skipped - ret = [0, 0, 0] - - # Check if board is skipped - if build_utils.skip_example(example, board): - status = SKIPPED - ret[2] = 1 - print(build_format.format(example, board, status, '-', flash_size, sram_size)) - else: - #subprocess.run("make -C examples/{} BOARD={} clean".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - - if build_result.returncode == 0: - status = SUCCEEDED - ret[0] = 1 - (flash_size, sram_size) = build_size(example, board) - subprocess.run("make -j -C examples/{} BOARD={} copy-artifact".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - else: - status = FAILED - ret[1] = 1 - - build_duration = time.monotonic() - start_time - print(build_format.format(example, board, status, "{:.2f}s".format(build_duration), flash_size, sram_size)) - if build_result.returncode != 0: - print(build_result.stdout.decode("utf-8")) - - return ret - -def build_size(example, board): - elf_file = 'examples/{}/_build/{}/*.elf'.format(example, board) - size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8") - size_list = size_output.split('\n')[1].split('\t') - flash_size = int(size_list[0]) - sram_size = int(size_list[1]) + int(size_list[2]) - return (flash_size, sram_size) if __name__ == '__main__': - # succeeded, failed, skipped - total_result = [0, 0, 0] + # If examples are not specified in arguments, build all + all_examples = [] + for dir1 in os.scandir("examples"): + if dir1.is_dir(): + for entry in os.scandir(dir1.path): + if entry.is_dir(): + all_examples.append(dir1.name + '/' + entry.name) + filter_with_input(all_examples) + all_examples.sort() + + # If family are not specified in arguments, build all + all_families = [] + for entry in os.scandir("hw/bsp"): + if entry.is_dir() and os.path.isdir(entry.path + "/boards") and entry.name not in ("esp32s2", "esp32s3"): + all_families.append(entry.name) + filter_with_input(all_families) + all_families.sort() print(build_separator) - print(build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM')) + print(build_utils.build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM')) total_time = time.monotonic() + # succeeded, failed, skipped + total_result = [0, 0, 0] for example in all_examples: print(build_separator) for family in all_families: fret = build_family(example, family) - total_result = list(map(lambda x, y: x+y, total_result, fret)) + total_result = list(map(lambda x, y: x + y, total_result, fret)) total_time = time.monotonic() - total_time print(build_separator) - print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(total_result[0], SUCCEEDED, total_result[1], FAILED, total_result[2], SKIPPED, total_time)) + print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(total_result[0], SUCCEEDED, total_result[1], + FAILED, total_result[2], SKIPPED, total_time)) print(build_separator) sys.exit(total_result[1]) diff --git a/tools/build_utils.py b/tools/build_utils.py index d570d20e3..f457c7986 100644 --- a/tools/build_utils.py +++ b/tools/build_utils.py @@ -1,9 +1,18 @@ +import subprocess import pathlib +import time + +build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |' + +SUCCEEDED = "\033[32msucceeded\033[0m" +FAILED = "\033[31mfailed\033[0m" +SKIPPED = "\033[33mskipped\033[0m" + def skip_example(example, board): ex_dir = pathlib.Path('examples/') / example bsp = pathlib.Path("hw/bsp") - + if (bsp / board / "board.mk").exists(): # board without family board_dir = bsp / board @@ -15,19 +24,19 @@ def skip_example(example, board): if not board_dir: # Skip unknown boards return True - + board_dir = list(board_dir)[0] - + family_dir = board_dir.parent.parent family = family_dir.name - + # family CMake family_mk = family_dir / "family.cmake" - + # family.mk if not family_mk.exists(): family_mk = family_dir / "family.mk" - + mk_contents = family_mk.read_text() # Find the mcu, first in family mk then board mk @@ -66,3 +75,48 @@ def skip_example(example, board): "family:" + family in onlys) return False + + +def build_example(example, board): + start_time = time.monotonic() + flash_size = "-" + sram_size = "-" + + # succeeded, failed, skipped + ret = [0, 0, 0] + + # Check if board is skipped + if skip_example(example, board): + status = SKIPPED + ret[2] = 1 + print(build_format.format(example, board, status, '-', flash_size, sram_size)) + else: + build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + if build_result.returncode == 0: + status = SUCCEEDED + ret[0] = 1 + (flash_size, sram_size) = build_size(example, board) + subprocess.run("make -j -C examples/{} BOARD={} copy-artifact".format(example, board), shell=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + else: + status = FAILED + ret[1] = 1 + + build_duration = time.monotonic() - start_time + print(build_format.format(example, board, status, "{:.2f}s".format(build_duration), flash_size, sram_size)) + + if build_result.returncode != 0: + print(build_result.stdout.decode("utf-8")) + + return ret + + +def build_size(example, board): + elf_file = 'examples/{}/_build/{}/*.elf'.format(example, board) + size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8") + size_list = size_output.split('\n')[1].split('\t') + flash_size = int(size_list[0]) + sram_size = int(size_list[1]) + int(size_list[2]) + return (flash_size, sram_size) -- cgit v1.3.1