From 993f7b6a2c0b325683ebb4f7dc5b0039d92c1eab Mon Sep 17 00:00:00 2001 From: hathach Date: Tue, 10 May 2022 13:15:07 +0700 Subject: include dual examples to ci --- tools/build_board.py | 11 +++++------ tools/build_family.py | 11 +++++------ 2 files changed, 10 insertions(+), 12 deletions(-) (limited to 'tools') diff --git a/tools/build_board.py b/tools/build_board.py index b2a80c680..4d895329a 100644 --- a/tools/build_board.py +++ b/tools/build_board.py @@ -28,12 +28,11 @@ def filter_with_input(mylist): # If examples are not specified in arguments, build all all_examples = [] -for entry in os.scandir("examples/device"): - if entry.is_dir(): - all_examples.append("device/" + entry.name) -for entry in os.scandir("examples/host"): - if entry.is_dir(): - all_examples.append("host/" + entry.name) +for dir1 in os.scandir("examples"): + if dir1.is_dir(): + for entry in os.scandir(dir1.path): + if entry.is_dir(): + all_examples.append(dir1.name + '/' + entry.name) filter_with_input(all_examples) all_examples.sort() diff --git a/tools/build_family.py b/tools/build_family.py index 4094d07db..095ca4ff7 100644 --- a/tools/build_family.py +++ b/tools/build_family.py @@ -28,12 +28,11 @@ def filter_with_input(mylist): # If examples are not specified in arguments, build all all_examples = [] -for entry in os.scandir("examples/device"): - if entry.is_dir(): - all_examples.append("device/" + entry.name) -for entry in os.scandir("examples/host"): - if entry.is_dir(): - all_examples.append("host/" + entry.name) +for dir1 in os.scandir("examples"): + if dir1.is_dir(): + for entry in os.scandir(dir1.path): + if entry.is_dir(): + all_examples.append(dir1.name + '/' + entry.name) filter_with_input(all_examples) all_examples.sort() -- cgit v1.3.1 From 8fe9022a6e6dea785092f4e0dff8180f1f0d4d86 Mon Sep 17 00:00:00 2001 From: hathach Date: Wed, 29 Jun 2022 14:06:44 +0700 Subject: fix buid_board.py script --- hw/bsp/ea4088qs/ea4088qs.c | 4 +-- hw/bsp/frdm_k32l2b/board.mk | 2 +- hw/bsp/frdm_kl25z/board.mk | 2 +- hw/bsp/kuiic/kuiic.c | 2 +- hw/bsp/lpcxpresso11u37/board.mk | 2 +- hw/bsp/lpcxpresso1347/board.mk | 2 +- hw/bsp/nutiny_nuc121s/board.mk | 3 +++ hw/bsp/nutiny_nuc125s/board.mk | 3 +++ hw/bsp/nutiny_nuc126v/board.mk | 3 +++ hw/bsp/nutiny_sdk_nuc505/board.mk | 3 +++ hw/bsp/same70_qmtech/board.mk | 2 +- hw/bsp/same70_xplained/board.mk | 2 +- hw/bsp/samg55xplained/board.mk | 2 +- hw/bsp/spresense/board.mk | 3 ++- hw/bsp/stm32l0538disco/board.mk | 2 +- src/portable/synopsys/dwc2/dwc2_stm32.h | 3 +++ tools/build_family.py | 2 +- tools/build_utils.py | 44 +++++++++++++++++++-------------- 18 files changed, 55 insertions(+), 31 deletions(-) (limited to 'tools') diff --git a/hw/bsp/ea4088qs/ea4088qs.c b/hw/bsp/ea4088qs/ea4088qs.c index fa905b2f9..7150ed393 100644 --- a/hw/bsp/ea4088qs/ea4088qs.c +++ b/hw/bsp/ea4088qs/ea4088qs.c @@ -33,11 +33,11 @@ void USB_IRQHandler(void) { #if CFG_TUD_ENABLED - tuh_int_handler(0); + tud_int_handler(0); #endif #if CFG_TUH_ENABLED - tud_int_handler(0); + tuh_int_handler(0); #endif } diff --git a/hw/bsp/frdm_k32l2b/board.mk b/hw/bsp/frdm_k32l2b/board.mk index 56df553da..a737eb360 100644 --- a/hw/bsp/frdm_k32l2b/board.mk +++ b/hw/bsp/frdm_k32l2b/board.mk @@ -9,7 +9,7 @@ CFLAGS += \ -DCFG_TUSB_MCU=OPT_MCU_K32L2BXX # mcu driver cause following warnings -CFLAGS += -Wno-error=unused-parameter +CFLAGS += -Wno-error=unused-parameter -Wno-error=redundant-decls MCU_DIR = $(SDK_DIR)/devices/K32L2B31A diff --git a/hw/bsp/frdm_kl25z/board.mk b/hw/bsp/frdm_kl25z/board.mk index 3585b8b03..6a72d516b 100644 --- a/hw/bsp/frdm_kl25z/board.mk +++ b/hw/bsp/frdm_kl25z/board.mk @@ -14,7 +14,7 @@ LDFLAGS += \ -Wl,--defsym,__heap_size__=0 # mcu driver cause following warnings -CFLAGS += -Wno-error=unused-parameter -Wno-error=format +CFLAGS += -Wno-error=unused-parameter -Wno-error=format -Wno-error=redundant-decls MCU_DIR = $(SDK_DIR)/devices/MKL25Z4 diff --git a/hw/bsp/kuiic/kuiic.c b/hw/bsp/kuiic/kuiic.c index 737ef3f5c..7aaa9e03c 100644 --- a/hw/bsp/kuiic/kuiic.c +++ b/hw/bsp/kuiic/kuiic.c @@ -45,7 +45,7 @@ * Variables ******************************************************************************/ /* System clock frequency. */ -extern uint32_t SystemCoreClock; +// extern uint32_t SystemCoreClock; /******************************************************************************* * Variables for BOARD_BootClockRUN configuration diff --git a/hw/bsp/lpcxpresso11u37/board.mk b/hw/bsp/lpcxpresso11u37/board.mk index b736eebe1..be6d2ed52 100644 --- a/hw/bsp/lpcxpresso11u37/board.mk +++ b/hw/bsp/lpcxpresso11u37/board.mk @@ -15,7 +15,7 @@ CFLAGS += \ -DCFG_TUSB_MEM_ALIGN='__attribute__((aligned(64)))' # mcu driver cause following warnings -CFLAGS += -Wno-error=strict-prototypes -Wno-error=unused-parameter +CFLAGS += -Wno-error=strict-prototypes -Wno-error=unused-parameter -Wno-error=redundant-decls MCU_DIR = hw/mcu/nxp/lpcopen/lpc11uxx/lpc_chip_11uxx diff --git a/hw/bsp/lpcxpresso1347/board.mk b/hw/bsp/lpcxpresso1347/board.mk index 62135c274..bf9c97c82 100644 --- a/hw/bsp/lpcxpresso1347/board.mk +++ b/hw/bsp/lpcxpresso1347/board.mk @@ -15,7 +15,7 @@ CFLAGS += \ -DCFG_TUSB_MEM_ALIGN='__attribute__((aligned(64)))' # startup.c and lpc_types.h cause following errors -CFLAGS += -Wno-error=strict-prototypes +CFLAGS += -Wno-error=strict-prototypes -Wno-error=redundant-decls MCU_DIR = hw/mcu/nxp/lpcopen/lpc13xx/lpc_chip_13xx diff --git a/hw/bsp/nutiny_nuc121s/board.mk b/hw/bsp/nutiny_nuc121s/board.mk index ff1d5aa3a..ad2ee1ea0 100644 --- a/hw/bsp/nutiny_nuc121s/board.mk +++ b/hw/bsp/nutiny_nuc121s/board.mk @@ -10,6 +10,9 @@ CFLAGS += \ -DCFG_EXAMPLE_MSC_READONLY \ -DCFG_TUSB_MCU=OPT_MCU_NUC121 +# mcu driver cause following warnings +CFLAGS += -Wno-error=redundant-decls + # All source paths should be relative to the top level. LD_FILE = hw/bsp/$(BOARD)/nuc121_flash.ld diff --git a/hw/bsp/nutiny_nuc125s/board.mk b/hw/bsp/nutiny_nuc125s/board.mk index bb56e42ab..000c8cd95 100644 --- a/hw/bsp/nutiny_nuc125s/board.mk +++ b/hw/bsp/nutiny_nuc125s/board.mk @@ -10,6 +10,9 @@ CFLAGS += \ -DCFG_EXAMPLE_MSC_READONLY \ -DCFG_TUSB_MCU=OPT_MCU_NUC121 +# mcu driver cause following warnings +CFLAGS += -Wno-error=redundant-decls + # All source paths should be relative to the top level. LD_FILE = hw/bsp/$(BOARD)/nuc125_flash.ld diff --git a/hw/bsp/nutiny_nuc126v/board.mk b/hw/bsp/nutiny_nuc126v/board.mk index 4f0ebf201..0dcd897cb 100644 --- a/hw/bsp/nutiny_nuc126v/board.mk +++ b/hw/bsp/nutiny_nuc126v/board.mk @@ -11,6 +11,9 @@ CFLAGS += \ -D__CORTEX_SC=0 \ -DCFG_TUSB_MCU=OPT_MCU_NUC126 +# mcu driver cause following warnings +CFLAGS += -Wno-error=redundant-decls + # All source paths should be relative to the top level. LD_FILE = hw/bsp/$(BOARD)/nuc126_flash.ld diff --git a/hw/bsp/nutiny_sdk_nuc505/board.mk b/hw/bsp/nutiny_sdk_nuc505/board.mk index e8514347e..f27577e36 100644 --- a/hw/bsp/nutiny_sdk_nuc505/board.mk +++ b/hw/bsp/nutiny_sdk_nuc505/board.mk @@ -9,6 +9,9 @@ CFLAGS += \ -mfpu=fpv4-sp-d16 \ -DCFG_TUSB_MCU=OPT_MCU_NUC505 +# mcu driver cause following warnings +CFLAGS += -Wno-error=redundant-decls + # All source paths should be relative to the top level. LD_FILE = hw/bsp/$(BOARD)/nuc505_flashtoram.ld diff --git a/hw/bsp/same70_qmtech/board.mk b/hw/bsp/same70_qmtech/board.mk index ba7088e44..2aa09f5dd 100644 --- a/hw/bsp/same70_qmtech/board.mk +++ b/hw/bsp/same70_qmtech/board.mk @@ -11,7 +11,7 @@ CFLAGS += \ -DCFG_TUSB_MCU=OPT_MCU_SAMX7X # suppress following warnings from mcu driver -CFLAGS += -Wno-error=unused-parameter -Wno-error=cast-align -Wno-error=cast-qual +CFLAGS += -Wno-error=unused-parameter -Wno-error=cast-align -Wno-error=cast-qual -Wno-error=redundant-decls ASF_DIR = hw/mcu/microchip/same70 diff --git a/hw/bsp/same70_xplained/board.mk b/hw/bsp/same70_xplained/board.mk index cb2decf50..cbc51e6b0 100644 --- a/hw/bsp/same70_xplained/board.mk +++ b/hw/bsp/same70_xplained/board.mk @@ -11,7 +11,7 @@ CFLAGS += \ -DCFG_TUSB_MCU=OPT_MCU_SAMX7X # suppress following warnings from mcu driver -CFLAGS += -Wno-error=unused-parameter -Wno-error=cast-align -Wno-error=cast-qual +CFLAGS += -Wno-error=unused-parameter -Wno-error=cast-align -Wno-error=cast-qual -Wno-error=redundant-decls ASF_DIR = hw/mcu/microchip/same70 diff --git a/hw/bsp/samg55xplained/board.mk b/hw/bsp/samg55xplained/board.mk index deff6944c..d0d0ade01 100644 --- a/hw/bsp/samg55xplained/board.mk +++ b/hw/bsp/samg55xplained/board.mk @@ -12,7 +12,7 @@ CFLAGS += \ -DCFG_TUSB_MCU=OPT_MCU_SAMG # suppress following warnings from mcu driver -CFLAGS += -Wno-error=undef -Wno-error=cast-qual -Wno-error=null-dereference +CFLAGS += -Wno-error=undef -Wno-error=cast-qual -Wno-error=null-dereference -Wno-error=redundant-decls ASF_DIR = hw/mcu/microchip/samg55 diff --git a/hw/bsp/spresense/board.mk b/hw/bsp/spresense/board.mk index ba291e817..78d7f6a66 100644 --- a/hw/bsp/spresense/board.mk +++ b/hw/bsp/spresense/board.mk @@ -34,8 +34,9 @@ CFLAGS += \ -Wno-error=unused-parameter \ -DCFG_TUSB_MCU=OPT_MCU_CXD56 \ +# suppress following warnings from mcu driver # lwip/src/core/raw.c:334:43: error: declaration of 'recv' shadows a global declaration -CFLAGS += -Wno-error=shadow +CFLAGS += -Wno-error=shadow -Wno-error=redundant-decls SPRESENSE_SDK = $(TOP)/hw/mcu/sony/cxd56/spresense-exported-sdk diff --git a/hw/bsp/stm32l0538disco/board.mk b/hw/bsp/stm32l0538disco/board.mk index e19101d64..69f09075a 100644 --- a/hw/bsp/stm32l0538disco/board.mk +++ b/hw/bsp/stm32l0538disco/board.mk @@ -17,7 +17,7 @@ CFLAGS += \ -DCFG_TUSB_MCU=OPT_MCU_STM32L0 # mcu driver cause following warnings -CFLAGS += -Wno-error=unused-parameter -Wno-error=maybe-uninitialized +CFLAGS += -Wno-error=unused-parameter -Wno-error=maybe-uninitialized -Wno-error=redundant-decls # All source paths should be relative to the top level. LD_FILE = hw/bsp/$(BOARD)/STM32L053C8Tx_FLASH.ld diff --git a/src/portable/synopsys/dwc2/dwc2_stm32.h b/src/portable/synopsys/dwc2/dwc2_stm32.h index 1d849541e..1187e0d6e 100644 --- a/src/portable/synopsys/dwc2/dwc2_stm32.h +++ b/src/portable/synopsys/dwc2/dwc2_stm32.h @@ -43,6 +43,9 @@ #define EP_MAX_FS USB_OTG_FS_MAX_IN_ENDPOINTS #define EP_FIFO_SIZE_FS USB_OTG_FS_TOTAL_FIFO_SIZE + #define EP_MAX_HS USB_OTG_HS_MAX_IN_ENDPOINTS + #define EP_FIFO_SIZE_HS USB_OTG_HS_TOTAL_FIFO_SIZE + #elif CFG_TUSB_MCU == OPT_MCU_STM32F4 #include "stm32f4xx.h" #define EP_MAX_FS USB_OTG_FS_MAX_IN_ENDPOINTS diff --git a/tools/build_family.py b/tools/build_family.py index 095ca4ff7..680411eab 100644 --- a/tools/build_family.py +++ b/tools/build_family.py @@ -48,7 +48,7 @@ all_families.sort() def build_family(example, family): all_boards = [] for entry in os.scandir("hw/bsp/{}/boards".format(family)): - if entry.is_dir(): + if entry.is_dir() and entry.name != 'pico_sdk': all_boards.append(entry.name) filter_with_input(all_boards) all_boards.sort() diff --git a/tools/build_utils.py b/tools/build_utils.py index 299fffa4d..68c2ed98b 100644 --- a/tools/build_utils.py +++ b/tools/build_utils.py @@ -4,26 +4,34 @@ def skip_example(example, board): ex_dir = pathlib.Path('examples/') / example bsp = pathlib.Path("hw/bsp") - board_dir = list(bsp.glob("*/boards/" + board)) - if not board_dir: - # Skip unknown boards - return True - - board_dir = list(board_dir)[0] + # board without family + board_dir = bsp / board - family_dir = board_dir.parent.parent - family = family_dir.name - - # family CMake - family_mk = family_dir / "family.cmake" - - # family.mk - if not family_mk.exists(): - family_mk = family_dir / "family.mk" - - mk_contents = family_mk.read_text() + if board_dir.exists(): + family = "" + mk_contents = "" + else: + # otherwise look into family + board_dir = list(bsp.glob("*/boards/" + board)) + if not board_dir: + # Skip unknown boards + return True + + board_dir = list(board_dir)[0] + + family_dir = board_dir.parent.parent + family = family_dir.name + + # family CMake + family_mk = family_dir / "family.cmake" + + # family.mk + if not family_mk.exists(): + family_mk = family_dir / "family.mk" + + mk_contents = family_mk.read_text() - # Find the mcu + # Find the mcu, first in family mk then board mk if "CFG_TUSB_MCU=OPT_MCU_" not in mk_contents: board_mk = board_dir / "board.cmake" if not board_mk.exists(): -- cgit v1.3.1 From 4da1a9359d7fa9813a01f133f919533184753064 Mon Sep 17 00:00:00 2001 From: hathach Date: Wed, 29 Jun 2022 15:12:01 +0700 Subject: fix build with fomu/fomu --- tools/build_utils.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'tools') diff --git a/tools/build_utils.py b/tools/build_utils.py index 68c2ed98b..d570d20e3 100644 --- a/tools/build_utils.py +++ b/tools/build_utils.py @@ -3,15 +3,14 @@ import pathlib def skip_example(example, board): ex_dir = pathlib.Path('examples/') / example bsp = pathlib.Path("hw/bsp") - - # board without family - board_dir = bsp / board - if board_dir.exists(): + if (bsp / board / "board.mk").exists(): + # board without family + board_dir = bsp / board family = "" mk_contents = "" else: - # otherwise look into family + # board within family board_dir = list(bsp.glob("*/boards/" + board)) if not board_dir: # Skip unknown boards -- cgit v1.3.1 From 8d07ea7ce560f2a938c573a0f0487ad3776402c4 Mon Sep 17 00:00:00 2001 From: hathach Date: Wed, 29 Jun 2022 16:39:19 +0700 Subject: build_family.py in parallel --- tools/build_family.py | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) (limited to 'tools') diff --git a/tools/build_family.py b/tools/build_family.py index 680411eab..5f03bfc87 100644 --- a/tools/build_family.py +++ b/tools/build_family.py @@ -3,6 +3,7 @@ import glob import sys import subprocess import time +from multiprocessing import Process import build_utils @@ -52,9 +53,15 @@ def build_family(example, family): all_boards.append(entry.name) filter_with_input(all_boards) all_boards.sort() - + + plist = [] for board in all_boards: - build_board(example, board) + p = Process(target=build_board, args=(example, board)) + plist.append(p) + p.start() + + for p in plist: + p.join() def build_board(example, board): global success_count, fail_count, skip_count, exit_status @@ -88,7 +95,6 @@ def build_board(example, board): print(build_result.stdout.decode("utf-8")) def build_size(example, board): - #elf_file = 'examples/device/{}/_build/{}/{}-firmware.elf'.format(example, board, board) elf_file = 'examples/{}/_build/{}/*.elf'.format(example, board) size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8") size_list = size_output.split('\n')[1].split('\t') @@ -96,17 +102,18 @@ def build_size(example, board): sram_size = int(size_list[1]) + int(size_list[2]) return (flash_size, sram_size) -print(build_separator) -print(build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM')) - -for example in all_examples: +if __name__ == '__main__': print(build_separator) - for family in all_families: - build_family(example, family) + print(build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM')) -total_time = time.monotonic() - total_time -print(build_separator) -print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(success_count, SUCCEEDED, fail_count, FAILED, skip_count, SKIPPED, total_time)) -print(build_separator) + for example in all_examples: + print(build_separator) + for family in all_families: + build_family(example, family) + + total_time = time.monotonic() - total_time + print(build_separator) + print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(success_count, SUCCEEDED, fail_count, FAILED, skip_count, SKIPPED, total_time)) + print(build_separator) -sys.exit(exit_status) + sys.exit(exit_status) -- cgit v1.3.1 From d5d5a6437cd73d2ad7b35e73b3ffc9d65aaa9ad3 Mon Sep 17 00:00:00 2001 From: hathach Date: Wed, 29 Jun 2022 18:23:45 +0700 Subject: more parallel ci --- tools/build_family.py | 55 ++++++++++++++++++++++++--------------------------- 1 file changed, 26 insertions(+), 29 deletions(-) (limited to 'tools') diff --git a/tools/build_family.py b/tools/build_family.py index 5f03bfc87..57a6c4d17 100644 --- a/tools/build_family.py +++ b/tools/build_family.py @@ -3,7 +3,7 @@ import glob import sys import subprocess import time -from multiprocessing import Process +from multiprocessing import Pool import build_utils @@ -11,13 +11,6 @@ SUCCEEDED = "\033[32msucceeded\033[0m" FAILED = "\033[31mfailed\033[0m" SKIPPED = "\033[33mskipped\033[0m" -success_count = 0 -fail_count = 0 -skip_count = 0 -exit_status = 0 - -total_time = time.monotonic() - build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |' build_separator = '-' * 106 @@ -54,46 +47,45 @@ def build_family(example, family): filter_with_input(all_boards) all_boards.sort() - plist = [] - for board in all_boards: - p = Process(target=build_board, args=(example, board)) - plist.append(p) - p.start() - - for p in plist: - p.join() + with Pool(processes=len(all_boards)) as pool: + pool_args = list((map(lambda b, e=example: [e, b], all_boards))) + result = pool.starmap(build_board, pool_args) + return list(map(sum, list(zip(*result)))) def build_board(example, board): - global success_count, fail_count, skip_count, exit_status start_time = time.monotonic() flash_size = "-" sram_size = "-" + # succeeded, failed, skipped + ret = [0, 0, 0] + # Check if board is skipped if build_utils.skip_example(example, board): - success = SKIPPED - skip_count += 1 - print(build_format.format(example, board, success, '-', flash_size, sram_size)) + status = SKIPPED + ret[2] = 1 + print(build_format.format(example, board, status, '-', flash_size, sram_size)) else: #subprocess.run("make -C examples/{} BOARD={} clean".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) if build_result.returncode == 0: - success = SUCCEEDED - success_count += 1 + status = SUCCEEDED + ret[0] = 1 (flash_size, sram_size) = build_size(example, board) subprocess.run("make -j -C examples/{} BOARD={} copy-artifact".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) else: - exit_status = build_result.returncode - success = FAILED - fail_count += 1 + status = FAILED + ret[1] = 1 build_duration = time.monotonic() - start_time - print(build_format.format(example, board, success, "{:.2f}s".format(build_duration), flash_size, sram_size)) + print(build_format.format(example, board, status, "{:.2f}s".format(build_duration), flash_size, sram_size)) if build_result.returncode != 0: print(build_result.stdout.decode("utf-8")) + return ret + def build_size(example, board): elf_file = 'examples/{}/_build/{}/*.elf'.format(example, board) size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8") @@ -103,17 +95,22 @@ def build_size(example, board): return (flash_size, sram_size) if __name__ == '__main__': + # succeeded, failed, skipped + total_result = [0, 0, 0] + print(build_separator) print(build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM')) + total_time = time.monotonic() for example in all_examples: print(build_separator) for family in all_families: - build_family(example, family) + fret = build_family(example, family) + total_result = list(map(lambda x, y: x+y, total_result, fret)) total_time = time.monotonic() - total_time print(build_separator) - print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(success_count, SUCCEEDED, fail_count, FAILED, skip_count, SKIPPED, total_time)) + print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(total_result[0], SUCCEEDED, total_result[1], FAILED, total_result[2], SKIPPED, total_time)) print(build_separator) - sys.exit(exit_status) + sys.exit(total_result[1]) -- cgit v1.3.1 From 8f9ecace4d9fecb14fbf8e8563d2f19334805d4d Mon Sep 17 00:00:00 2001 From: hathach Date: Wed, 29 Jun 2022 21:06:02 +0700 Subject: update build_board.py to parallel build --- tools/build_board.py | 119 ++++++++++++++++++-------------------------------- tools/build_family.py | 96 ++++++++++++---------------------------- tools/build_utils.py | 66 +++++++++++++++++++++++++--- 3 files changed, 129 insertions(+), 152 deletions(-) (limited to 'tools') diff --git a/tools/build_board.py b/tools/build_board.py index 4d895329a..62a4ea82f 100644 --- a/tools/build_board.py +++ b/tools/build_board.py @@ -1,8 +1,7 @@ import os -import glob import sys -import subprocess import time +from multiprocessing import Pool import build_utils @@ -10,90 +9,56 @@ SUCCEEDED = "\033[32msucceeded\033[0m" FAILED = "\033[31mfailed\033[0m" SKIPPED = "\033[33mskipped\033[0m" -success_count = 0 -fail_count = 0 -skip_count = 0 -exit_status = 0 - -total_time = time.monotonic() - -build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |' build_separator = '-' * 106 + def filter_with_input(mylist): if len(sys.argv) > 1: input_args = list(set(mylist).intersection(sys.argv)) if len(input_args) > 0: mylist[:] = input_args -# If examples are not specified in arguments, build all -all_examples = [] -for dir1 in os.scandir("examples"): - if dir1.is_dir(): - for entry in os.scandir(dir1.path): - if entry.is_dir(): - all_examples.append(dir1.name + '/' + entry.name) -filter_with_input(all_examples) -all_examples.sort() - -# If boards are not specified in arguments, build all -all_boards = [] -for entry in os.scandir("hw/bsp"): - if entry.is_dir() and os.path.exists(entry.path + "/board.mk"): - all_boards.append(entry.name) -filter_with_input(all_boards) -all_boards.sort() - -def build_board(example, board): - global success_count, fail_count, skip_count, exit_status - start_time = time.monotonic() - flash_size = "-" - sram_size = "-" - - # Check if board is skipped - if build_utils.skip_example(example, board): - success = SKIPPED - skip_count += 1 - print(build_format.format(example, board, success, '-', flash_size, sram_size)) - else: - subprocess.run("make -C examples/{} BOARD={} clean".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - - if build_result.returncode == 0: - success = SUCCEEDED - success_count += 1 - (flash_size, sram_size) = build_size(example, board) - else: - exit_status = build_result.returncode - success = FAILED - fail_count += 1 - build_duration = time.monotonic() - start_time - print(build_format.format(example, board, success, "{:.2f}s".format(build_duration), flash_size, sram_size)) +if __name__ == '__main__': + # If examples are not specified in arguments, build all + all_examples = [] + for dir1 in os.scandir("examples"): + if dir1.is_dir(): + for entry in os.scandir(dir1.path): + if entry.is_dir(): + all_examples.append(dir1.name + '/' + entry.name) + filter_with_input(all_examples) + all_examples.sort() + + # If boards are not specified in arguments, build all + all_boards = [] + for entry in os.scandir("hw/bsp"): + if entry.is_dir() and os.path.exists(entry.path + "/board.mk"): + all_boards.append(entry.name) + filter_with_input(all_boards) + all_boards.sort() - if build_result.returncode != 0: - print(build_result.stdout.decode("utf-8")) - -def build_size(example, board): - #elf_file = 'examples/device/{}/_build/{}/{}-firmware.elf'.format(example, board, board) - elf_file = 'examples/{}/_build/{}/*.elf'.format(example, board) - size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8") - size_list = size_output.split('\n')[1].split('\t') - flash_size = int(size_list[0]) - sram_size = int(size_list[1]) + int(size_list[2]) - return (flash_size, sram_size) - -print(build_separator) -print(build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM')) - -for example in all_examples: print(build_separator) - for board in all_boards: - build_board(example, board) - -total_time = time.monotonic() - total_time -print(build_separator) -print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(success_count, SUCCEEDED, fail_count, FAILED, skip_count, SKIPPED, total_time)) -print(build_separator) + print(build_utils.build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM')) + total_time = time.monotonic() + + # succeeded, failed, skipped + total_result = [0, 0, 0] + for example in all_examples: + print(build_separator) + with Pool(processes=os.cpu_count()) as pool: + pool_args = list((map(lambda b, e=example: [e, b], all_boards))) + result = pool.starmap(build_utils.build_example, pool_args) + # sum all element of same index (column sum) + result = list(map(sum, list(zip(*result)))) + + # add to total result + total_result = list(map(lambda x, y: x + y, total_result, result)) + + total_time = time.monotonic() - total_time + print(build_separator) + print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(total_result[0], SUCCEEDED, total_result[1], + FAILED, total_result[2], SKIPPED, total_time)) + print(build_separator) -sys.exit(exit_status) + sys.exit(total_result[1]) diff --git a/tools/build_family.py b/tools/build_family.py index 57a6c4d17..c6c64d2b3 100644 --- a/tools/build_family.py +++ b/tools/build_family.py @@ -1,7 +1,5 @@ import os -import glob import sys -import subprocess import time from multiprocessing import Pool @@ -11,33 +9,15 @@ SUCCEEDED = "\033[32msucceeded\033[0m" FAILED = "\033[31mfailed\033[0m" SKIPPED = "\033[33mskipped\033[0m" -build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |' build_separator = '-' * 106 + def filter_with_input(mylist): if len(sys.argv) > 1: input_args = list(set(mylist).intersection(sys.argv)) if len(input_args) > 0: mylist[:] = input_args -# If examples are not specified in arguments, build all -all_examples = [] -for dir1 in os.scandir("examples"): - if dir1.is_dir(): - for entry in os.scandir(dir1.path): - if entry.is_dir(): - all_examples.append(dir1.name + '/' + entry.name) -filter_with_input(all_examples) -all_examples.sort() - -# If family are not specified in arguments, build all -all_families = [] -for entry in os.scandir("hw/bsp"): - if entry.is_dir() and os.path.isdir(entry.path + "/boards") and entry.name not in ("esp32s2", "esp32s3"): - all_families.append(entry.name) - -filter_with_input(all_families) -all_families.sort() def build_family(example, family): all_boards = [] @@ -47,70 +27,48 @@ def build_family(example, family): filter_with_input(all_boards) all_boards.sort() - with Pool(processes=len(all_boards)) as pool: + with Pool(processes=os.cpu_count()) as pool: pool_args = list((map(lambda b, e=example: [e, b], all_boards))) - result = pool.starmap(build_board, pool_args) + result = pool.starmap(build_utils.build_example, pool_args) + # sum all element of same index (column sum) return list(map(sum, list(zip(*result)))) - -def build_board(example, board): - start_time = time.monotonic() - flash_size = "-" - sram_size = "-" - - # succeeded, failed, skipped - ret = [0, 0, 0] - - # Check if board is skipped - if build_utils.skip_example(example, board): - status = SKIPPED - ret[2] = 1 - print(build_format.format(example, board, status, '-', flash_size, sram_size)) - else: - #subprocess.run("make -C examples/{} BOARD={} clean".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - - if build_result.returncode == 0: - status = SUCCEEDED - ret[0] = 1 - (flash_size, sram_size) = build_size(example, board) - subprocess.run("make -j -C examples/{} BOARD={} copy-artifact".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - else: - status = FAILED - ret[1] = 1 - - build_duration = time.monotonic() - start_time - print(build_format.format(example, board, status, "{:.2f}s".format(build_duration), flash_size, sram_size)) - if build_result.returncode != 0: - print(build_result.stdout.decode("utf-8")) - - return ret - -def build_size(example, board): - elf_file = 'examples/{}/_build/{}/*.elf'.format(example, board) - size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8") - size_list = size_output.split('\n')[1].split('\t') - flash_size = int(size_list[0]) - sram_size = int(size_list[1]) + int(size_list[2]) - return (flash_size, sram_size) if __name__ == '__main__': - # succeeded, failed, skipped - total_result = [0, 0, 0] + # If examples are not specified in arguments, build all + all_examples = [] + for dir1 in os.scandir("examples"): + if dir1.is_dir(): + for entry in os.scandir(dir1.path): + if entry.is_dir(): + all_examples.append(dir1.name + '/' + entry.name) + filter_with_input(all_examples) + all_examples.sort() + + # If family are not specified in arguments, build all + all_families = [] + for entry in os.scandir("hw/bsp"): + if entry.is_dir() and os.path.isdir(entry.path + "/boards") and entry.name not in ("esp32s2", "esp32s3"): + all_families.append(entry.name) + filter_with_input(all_families) + all_families.sort() print(build_separator) - print(build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM')) + print(build_utils.build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM')) total_time = time.monotonic() + # succeeded, failed, skipped + total_result = [0, 0, 0] for example in all_examples: print(build_separator) for family in all_families: fret = build_family(example, family) - total_result = list(map(lambda x, y: x+y, total_result, fret)) + total_result = list(map(lambda x, y: x + y, total_result, fret)) total_time = time.monotonic() - total_time print(build_separator) - print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(total_result[0], SUCCEEDED, total_result[1], FAILED, total_result[2], SKIPPED, total_time)) + print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(total_result[0], SUCCEEDED, total_result[1], + FAILED, total_result[2], SKIPPED, total_time)) print(build_separator) sys.exit(total_result[1]) diff --git a/tools/build_utils.py b/tools/build_utils.py index d570d20e3..f457c7986 100644 --- a/tools/build_utils.py +++ b/tools/build_utils.py @@ -1,9 +1,18 @@ +import subprocess import pathlib +import time + +build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |' + +SUCCEEDED = "\033[32msucceeded\033[0m" +FAILED = "\033[31mfailed\033[0m" +SKIPPED = "\033[33mskipped\033[0m" + def skip_example(example, board): ex_dir = pathlib.Path('examples/') / example bsp = pathlib.Path("hw/bsp") - + if (bsp / board / "board.mk").exists(): # board without family board_dir = bsp / board @@ -15,19 +24,19 @@ def skip_example(example, board): if not board_dir: # Skip unknown boards return True - + board_dir = list(board_dir)[0] - + family_dir = board_dir.parent.parent family = family_dir.name - + # family CMake family_mk = family_dir / "family.cmake" - + # family.mk if not family_mk.exists(): family_mk = family_dir / "family.mk" - + mk_contents = family_mk.read_text() # Find the mcu, first in family mk then board mk @@ -66,3 +75,48 @@ def skip_example(example, board): "family:" + family in onlys) return False + + +def build_example(example, board): + start_time = time.monotonic() + flash_size = "-" + sram_size = "-" + + # succeeded, failed, skipped + ret = [0, 0, 0] + + # Check if board is skipped + if skip_example(example, board): + status = SKIPPED + ret[2] = 1 + print(build_format.format(example, board, status, '-', flash_size, sram_size)) + else: + build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + + if build_result.returncode == 0: + status = SUCCEEDED + ret[0] = 1 + (flash_size, sram_size) = build_size(example, board) + subprocess.run("make -j -C examples/{} BOARD={} copy-artifact".format(example, board), shell=True, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + else: + status = FAILED + ret[1] = 1 + + build_duration = time.monotonic() - start_time + print(build_format.format(example, board, status, "{:.2f}s".format(build_duration), flash_size, sram_size)) + + if build_result.returncode != 0: + print(build_result.stdout.decode("utf-8")) + + return ret + + +def build_size(example, board): + elf_file = 'examples/{}/_build/{}/*.elf'.format(example, board) + size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8") + size_list = size_output.split('\n')[1].split('\t') + flash_size = int(size_list[0]) + sram_size = int(size_list[1]) + int(size_list[2]) + return (flash_size, sram_size) -- cgit v1.3.1 From 5323472afd1f35d0e62bd58bb3293bcfce36483b Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 1 Jul 2022 16:37:34 +0700 Subject: update get-deps for ci --- .github/workflows/build_arm.yml | 2 +- tools/build_board.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/.github/workflows/build_arm.yml b/.github/workflows/build_arm.yml index 7cc1b10f9..935d57e06 100644 --- a/.github/workflows/build_arm.yml +++ b/.github/workflows/build_arm.yml @@ -94,7 +94,7 @@ jobs: - name: Get Dependencies run: | b=`find hw/bsp/${{ matrix.family }}/boards -depth -maxdepth 1 -type d -name '[^.]?*' -printf %f -quit` - make BOARD={b} get-deps + make -C examples/device/board_test BOARD=${b} get-deps - name: Build run: python3 tools/build_family.py ${{ matrix.family }} diff --git a/tools/build_board.py b/tools/build_board.py index 62a4ea82f..8d10ef820 100644 --- a/tools/build_board.py +++ b/tools/build_board.py @@ -1,6 +1,7 @@ import os import sys import time +import subprocess from multiprocessing import Pool import build_utils @@ -38,6 +39,10 @@ if __name__ == '__main__': filter_with_input(all_boards) all_boards.sort() + # Get dependencies + for b in all_boards: + subprocess.run("make -C examples/device/board_test BOARD={} get-deps".format(b), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(build_separator) print(build_utils.build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM')) total_time = time.monotonic() -- cgit v1.3.1 From 53db23142a89def1ad2f866863a559a931eeb035 Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 1 Jul 2022 17:23:14 +0700 Subject: add get-dependencies.py --- .github/workflows/build_aarch64.yml | 3 +++ .github/workflows/build_arm.yml | 15 ++++++--------- .github/workflows/build_msp430.yml | 3 +++ .github/workflows/build_renesas.yml | 3 +++ .github/workflows/build_riscv.yml | 3 +++ hw/bsp/rp2040/family.mk | 2 ++ tools/get_dependencies.py | 25 +++++++++++++++++++++++++ 7 files changed, 45 insertions(+), 9 deletions(-) create mode 100644 tools/get_dependencies.py (limited to 'tools') diff --git a/.github/workflows/build_aarch64.yml b/.github/workflows/build_aarch64.yml index 1720ba592..b4ea8a0eb 100644 --- a/.github/workflows/build_aarch64.yml +++ b/.github/workflows/build_aarch64.yml @@ -55,6 +55,9 @@ jobs: - name: Set Toolchain Path run: echo >> $GITHUB_PATH `echo ~/cache/toolchain/*/bin` + - name: Get Dependencies + run: python3 tools/get_dependencies.py ${{ matrix.family }} + - name: Build run: python3 tools/build_family.py ${{ matrix.family }} diff --git a/.github/workflows/build_arm.yml b/.github/workflows/build_arm.yml index 935d57e06..4297ba895 100644 --- a/.github/workflows/build_arm.yml +++ b/.github/workflows/build_arm.yml @@ -89,12 +89,9 @@ jobs: run: | git clone --depth 1 -b develop https://github.com/raspberrypi/pico-sdk ~/pico-sdk echo >> $GITHUB_ENV PICO_SDK_PATH=~/pico-sdk - git submodule update --init hw/mcu/raspberry_pi/Pico-PIO-USB - name: Get Dependencies - run: | - b=`find hw/bsp/${{ matrix.family }}/boards -depth -maxdepth 1 -type d -name '[^.]?*' -printf %f -quit` - make -C examples/device/board_test BOARD=${b} get-deps + run: python3 tools/get_dependencies.py ${{ matrix.family }} - name: Build run: python3 tools/build_family.py ${{ matrix.family }} @@ -127,16 +124,16 @@ jobs: - name: Setup Python uses: actions/setup-python@v3 + - name: Install ARM GCC + uses: carlosperate/arm-none-eabi-gcc-action@v1 + with: + release: '11.2-2022.02' + - name: Checkout TinyUSB uses: actions/checkout@v3 - name: Checkout common submodules in lib run: git submodule update --init lib/FreeRTOS-Kernel lib/lwip - - name: Install ARM GCC - uses: carlosperate/arm-none-eabi-gcc-action@v1 - with: - release: '11.2-2022.02' - - name: Build run: python3 tools/build_board.py ${{ matrix.example }} diff --git a/.github/workflows/build_msp430.yml b/.github/workflows/build_msp430.yml index 6a468ab04..ea93f09a0 100644 --- a/.github/workflows/build_msp430.yml +++ b/.github/workflows/build_msp430.yml @@ -52,6 +52,9 @@ jobs: - name: Set Toolchain Path run: echo >> $GITHUB_PATH `echo ~/cache/toolchain/*/bin` + - name: Get Dependencies + run: python3 tools/get_dependencies.py ${{ matrix.family }} + - name: Build run: python3 tools/build_family.py ${{ matrix.family }} diff --git a/.github/workflows/build_renesas.yml b/.github/workflows/build_renesas.yml index 50618ff6b..2563d3549 100644 --- a/.github/workflows/build_renesas.yml +++ b/.github/workflows/build_renesas.yml @@ -53,6 +53,9 @@ jobs: - name: Set Toolchain Path run: echo >> $GITHUB_PATH `echo ~/cache/toolchain/*/bin` + - name: Get Dependencies + run: python3 tools/get_dependencies.py ${{ matrix.family }} + - name: Build run: python3 tools/build_family.py ${{ matrix.family }} diff --git a/.github/workflows/build_riscv.yml b/.github/workflows/build_riscv.yml index 2d670138d..90dc35206 100644 --- a/.github/workflows/build_riscv.yml +++ b/.github/workflows/build_riscv.yml @@ -53,6 +53,9 @@ jobs: - name: Set Toolchain Path run: echo >> $GITHUB_PATH `echo ~/cache/toolchain/*/bin` + - name: Get Dependencies + run: python3 tools/get_dependencies.py ${{ matrix.family }} + - name: Build run: python3 tools/build_family.py ${{ matrix.family }} diff --git a/hw/bsp/rp2040/family.mk b/hw/bsp/rp2040/family.mk index 5db784b14..cf6b53793 100644 --- a/hw/bsp/rp2040/family.mk +++ b/hw/bsp/rp2040/family.mk @@ -1,6 +1,8 @@ JLINK_DEVICE = rp2040_m0_0 PYOCD_TARGET = rp2040 +DEPS_SUBMODULES += hw/mcu/raspberry_pi/Pico-PIO-USB + ifeq ($(DEBUG), 1) CMAKE_DEFSYM += -DCMAKE_BUILD_TYPE=Debug endif diff --git a/tools/get_dependencies.py b/tools/get_dependencies.py new file mode 100644 index 000000000..e7d3e0a76 --- /dev/null +++ b/tools/get_dependencies.py @@ -0,0 +1,25 @@ +import os +import sys +import subprocess + + +# dependency lookup (ABC sorted) +# deps = { +# 'LPC11UXX' : [ [] ] +# } + + +def get_family_dep(family): + for entry in os.scandir("hw/bsp/{}/boards".format(family)): + if entry.is_dir(): + result = subprocess.run("make -C examples/device/board_test BOARD={} get-deps".format(entry.name), + shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + print(result.stdout.decode("utf-8")) + return result.returncode + +status = 0 +all_family = sys.argv[1:] +for f in all_family: + status += get_family_dep(f) + +sys.exit(status) \ No newline at end of file -- cgit v1.3.1