Merge branch 'master' into fork/roma-jam/feature/add_tusb_teardown

4 files changed, 190 insertions, 38 deletions

diff --git a/tools/build.py b/tools/build.py
index 633d2b582..ce4d0ef1a 100755
--- a/tools/build.py
+++ b/tools/build.py
@@ -23,6 +23,7 @@ build_separator = '-' * 95
 build_status = [STATUS_OK, STATUS_FAILED, STATUS_SKIPPED]
 
 verbose = False
+parallel_jobs = os.cpu_count()
 
 # -----------------------------
 # Helper
@@ -81,7 +82,7 @@ def print_build_result(board, example, status, duration):
 # -----------------------------
 # CMake
 # -----------------------------
-def cmake_board(board, toolchain, build_flags_on):
+def cmake_board(board, build_args, build_flags_on):
     ret = [0, 0, 0]
     start_time = time.monotonic()
 
@@ -100,23 +101,25 @@ def cmake_board(board, toolchain, build_flags_on):
             if build_utils.skip_example(example, board):
                 ret[2] += 1
             else:
-                rcmd = run_cmd(f'cmake examples/{example} -B {build_dir}/{example} -G "Ninja" '
-                               f'-DBOARD={board} {build_flags}')
-                if rcmd.returncode == 0:
-                    rcmd = run_cmd(f'cmake --build {build_dir}/{example}')
+                rcmd = run_cmd(f'idf.py -C examples/{example} -B {build_dir}/{example} -G Ninja '
+                               f'-DBOARD={board} {build_flags} build')
                 ret[0 if rcmd.returncode == 0 else 1] += 1
     else:
-        rcmd = run_cmd(f'cmake examples -B {build_dir} -G "Ninja" -DBOARD={board} -DCMAKE_BUILD_TYPE=MinSizeRel '
-                       f'-DTOOLCHAIN={toolchain} {build_flags}')
+        rcmd = run_cmd(f'cmake examples -B {build_dir} -G Ninja -DBOARD={board} -DCMAKE_BUILD_TYPE=MinSizeRel '
+                       f'{build_args} {build_flags}')
         if rcmd.returncode == 0:
             cmd = f"cmake --build {build_dir}"
-            # circleci docker return $nproc as 36 core, limit parallel according to resource class. Required for IAR, also prevent crashed/killed by docker
+            njobs = parallel_jobs
+
+            # circleci docker return $nproc as 36 core, limit parallel according to resource class.
+            # Required for IAR, also prevent crashed/killed by docker
             if os.getenv('CIRCLECI'):
                 resource_class = { 'small': 1, 'medium': 2, 'medium+': 3, 'large': 4 }
                 for rc in resource_class:
                     if rc in os.getenv('CIRCLE_JOB'):
-                        cmd += f' --parallel {resource_class[rc]}'
+                        njobs = resource_class[rc]
                         break
+            cmd += f' --parallel {njobs}'
             rcmd = run_cmd(cmd)
         ret[0 if rcmd.returncode == 0 else 1] += 1
 
@@ -149,42 +152,54 @@ def make_one_example(example, board, make_option):
     return ret
 
 
-def make_board(board, toolchain):
+def make_board(board, build_args):
     print(build_separator)
-    all_examples = get_examples(find_family(board))
+    family = find_family(board);
+    all_examples = get_examples(family)
     start_time = time.monotonic()
     ret = [0, 0, 0]
-    with Pool(processes=os.cpu_count()) as pool:
-        pool_args = list((map(lambda e, b=board, o=f"TOOLCHAIN={toolchain}": [e, b, o], all_examples)))
-        r = pool.starmap(make_one_example, pool_args)
-        # sum all element of same index (column sum)
-        ret = list(map(sum, list(zip(*r))))
-    example = 'all'
-    print_build_result(board, example, 0 if ret[1] == 0 else 1, time.monotonic() - start_time)
+    if family == 'espressif' or family == 'rp2040':
+        # espressif and rp2040 do not support make, use cmake instead
+        final_status = 2
+    else:
+        with Pool(processes=os.cpu_count()) as pool:
+            pool_args = list((map(lambda e, b=board, o=f"{build_args}": [e, b, o], all_examples)))
+            r = pool.starmap(make_one_example, pool_args)
+            # sum all element of same index (column sum)
+            ret = list(map(sum, list(zip(*r))))
+        final_status = 0 if ret[1] == 0 else 1
+    print_build_result(board, 'all', final_status, time.monotonic() - start_time)
     return ret
 
 
 # -----------------------------
 # Build Family
 # -----------------------------
-def build_boards_list(boards, toolchain, build_system, build_flags_on):
+def build_boards_list(boards, build_defines, build_system, build_flags_on):
     ret = [0, 0, 0]
     for b in boards:
         r = [0, 0, 0]
         if build_system == 'cmake':
-            r = cmake_board(b, toolchain, build_flags_on)
+            build_args = ' '.join(f'-D{d}' for d in build_defines)
+            r = cmake_board(b, build_args, build_flags_on)
         elif build_system == 'make':
-            r = make_board(b, toolchain)
+            build_args = ' '.join(f'{d}' for d in build_defines)
+            r = make_board(b, build_args)
         ret[0] += r[0]
         ret[1] += r[1]
         ret[2] += r[2]
     return ret
 
 
-def build_family(family, toolchain, build_system, build_flags_on, one_per_family, boards):
+def build_family(family, build_defines, build_system, build_flags_on, one_per_family, boards):
+    skip_ci = ['pico_sdk']
+    if os.getenv('GITHUB_ACTIONS') or os.getenv('CIRCLECI'):
+        skip_ci_file = Path(f"hw/bsp/{family}/skip_ci.txt")
+        if skip_ci_file.exists():
+            skip_ci = skip_ci_file.read_text().split()
     all_boards = []
     for entry in os.scandir(f"hw/bsp/{family}/boards"):
-        if entry.is_dir() and entry.name != 'pico_sdk':
+        if entry.is_dir() and not entry.name in skip_ci:
             all_boards.append(entry.name)
     all_boards.sort()
 
@@ -197,7 +212,7 @@ def build_family(family, toolchain, build_system, build_flags_on, one_per_family
                 return ret
         all_boards = [random.choice(all_boards)]
 
-    ret = build_boards_list(all_boards, toolchain, build_system, build_flags_on)
+    ret = build_boards_list(all_boards, build_defines, build_system, build_flags_on)
     return ret
 
 
@@ -206,14 +221,17 @@ def build_family(family, toolchain, build_system, build_flags_on, one_per_family
 # -----------------------------
 def main():
     global verbose
+    global parallel_jobs
 
     parser = argparse.ArgumentParser()
     parser.add_argument('families', nargs='*', default=[], help='Families to build')
     parser.add_argument('-b', '--board', action='append', default=[], help='Boards to build')
     parser.add_argument('-t', '--toolchain', default='gcc', help='Toolchain to use, default is gcc')
     parser.add_argument('-s', '--build-system', default='cmake', help='Build system to use, default is cmake')
+    parser.add_argument('-D', '--define-symbol', action='append', default=[], help='Define to pass to build system')
     parser.add_argument('-f1', '--build-flags-on', action='append', default=[], help='Build flag to pass to build system')
     parser.add_argument('-1', '--one-per-family', action='store_true', default=False, help='Build only one random board inside a family')
+    parser.add_argument('-j', '--jobs', type=int, default=os.cpu_count(), help='Number of jobs to run in parallel')
     parser.add_argument('-v', '--verbose', action='store_true', help='Verbose output')
     args = parser.parse_args()
 
@@ -221,9 +239,13 @@ def main():
     boards = args.board
     toolchain = args.toolchain
     build_system = args.build_system
+    build_defines = args.define_symbol
     build_flags_on = args.build_flags_on
     one_per_family = args.one_per_family
     verbose = args.verbose
+    parallel_jobs = args.jobs
+
+    build_defines.append(f'TOOLCHAIN={toolchain}')
 
     if len(families) == 0 and len(boards) == 0:
         print("Please specify families or board to build")
@@ -246,13 +268,13 @@ def main():
 
     # succeeded, failed, skipped
     for f in all_families:
-        r = build_family(f, toolchain, build_system, build_flags_on, one_per_family, boards)
+        r = build_family(f, build_defines, build_system, build_flags_on, one_per_family, boards)
         result[0] += r[0]
         result[1] += r[1]
         result[2] += r[2]
 
     # build boards
-    r = build_boards_list(boards, toolchain, build_system, build_flags_on)
+    r = build_boards_list(boards, build_defines, build_system, build_flags_on)
     result[0] += r[0]
     result[1] += r[1]
     result[2] += r[2]
diff --git a/tools/build_utils.py b/tools/build_utils.py
index 2998f940d..d80ceea7c 100755
--- a/tools/build_utils.py
+++ b/tools/build_utils.py
@@ -26,6 +26,8 @@ def skip_example(example, board):
 
     # family.mk
     family_mk = family_dir / "family.mk"
+    if not family_mk.exists():
+        family_mk = family_dir / "family.cmake"
     mk_contents = family_mk.read_text()
 
     # Find the mcu, first in family mk then board mk
diff --git a/tools/gen_presets.py b/tools/gen_presets.py
new file mode 100755
index 000000000..94b8d16b0
--- /dev/null
+++ b/tools/gen_presets.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+import os
+import json
+from pathlib import Path
+
+def main():
+    board_list = []
+
+    # Find all board.cmake files
+    for root, dirs, files in os.walk("hw/bsp"):
+        for file in files:
+            if file == "board.cmake":
+                board_list.append(os.path.basename(root))
+
+    print('Generating presets for the following boards:')
+    print(board_list)
+
+    # Generate the presets
+    presets = {}
+    presets['version'] = 6
+
+    # Configure presets
+    presets['configurePresets'] = [
+        {"name": "default",
+         "hidden": True,
+         "description": r"Configure preset for the ${presetName} board",
+         "generator": "Ninja Multi-Config",
+         "binaryDir": r"${sourceDir}/build/${presetName}",
+         "cacheVariables": {
+             "CMAKE_DEFAULT_BUILD_TYPE": "RelWithDebInfo",
+             "BOARD": r"${presetName}"
+         }}]
+
+    presets['configurePresets'].extend(
+        sorted(
+            [
+                {
+                    'name': board,
+                    'inherits': 'default'
+                }
+                for board in board_list
+            ], key=lambda x: x['name']
+        )
+    )
+
+    # Build presets
+    # no inheritance since 'name' doesn't support macro expansion
+    presets['buildPresets'] = sorted(
+        [
+            {
+                'name': board,
+                'description': "Build preset for the " + board + " board",
+                'configurePreset': board
+            }
+            for board in board_list
+        ], key=lambda x: x['name']
+    )
+
+    # Workflow presets
+    presets['workflowPresets'] = sorted(
+        [
+            {
+                "name": board,
+                "steps": [
+                    {
+                        "type": "configure",
+                        "name": board
+                    },
+                    {
+                        "type": "build",
+                        "name": board
+                    }
+                ]
+            }
+            for board in board_list
+        ], key=lambda x: x['name']
+    )
+
+    path_boardpresets = "hw/bsp/BoardPresets.json"
+    with open(path_boardpresets, "w") as f:
+        f.write('{}\n'.format(json.dumps(presets, indent=2)))
+
+    # Generate presets for examples
+    presets = {
+        "version": 6,
+        "include": [
+        ]
+    }
+
+    example_list = []
+    for root, dirs, files in os.walk("examples"):
+        for file in files:
+            # Filter out ESP-IDF CMakeLists.txt in src folder
+            if file == "CMakeLists.txt" and os.path.basename(root) != 'src':
+                presets['include'] = [os.path.relpath(path_boardpresets, root).replace(os.sep, '/')]
+                with open(os.path.join(root, 'CMakePresets.json'), 'w') as f:
+                    f.write('{}\n'.format(json.dumps(presets, indent=2)))
+                example_list.append(os.path.basename(root))
+
+    print('Generating presets for the following examples:')
+    print(example_list)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tools/get_deps.py b/tools/get_deps.py
index c8459c1f1..4a980b255 100755
--- a/tools/get_deps.py
+++ b/tools/get_deps.py
@@ -25,9 +25,9 @@ deps_optional = {
     'hw/mcu/allwinner': ['https://github.com/hathach/allwinner_driver.git',
                          '8e5e89e8e132c0fd90e72d5422e5d3d68232b756',
                          'fc100s'],
-    'hw/mcu/analog/max32' : ['https://github.com/analogdevicesinc/msdk.git',
+    'hw/mcu/analog/msdk' : ['https://github.com/analogdevicesinc/msdk.git',
                              'b20b398d3e5e2007594e54a74ba3d2a2e50ddd75',
-                             'max32650 max32666 max32690 max78002'],
+                             'maxim'],
     'hw/mcu/bridgetek/ft9xx/ft90x-sdk': ['https://github.com/BRTSG-FOSS/ft90x-sdk.git',
                                          '91060164afe239fcb394122e8bf9eb24d3194eb1',
                                          'brtmm90x'],
@@ -55,11 +55,11 @@ deps_optional = {
     'hw/mcu/nxp/lpcopen': ['https://github.com/hathach/nxp_lpcopen.git',
                            'b41cf930e65c734d8ec6de04f1d57d46787c76ae',
                            'lpc11 lpc13 lpc15 lpc17 lpc18 lpc40 lpc43'],
-    'hw/mcu/nxp/mcux-sdk': ['https://github.com/hathach/mcux-sdk.git',
-                            '144f1eb7ea8c06512e12f12b27383601c0272410',
+    'hw/mcu/nxp/mcux-sdk': ['https://github.com/nxp-mcuxpresso/mcux-sdk',
+                            'a1bdae309a14ec95a4f64a96d3315a4f89c397c6',
                             'kinetis_k kinetis_k32l2 kinetis_kl lpc51 lpc54 lpc55 mcx imxrt'],
     'hw/mcu/raspberry_pi/Pico-PIO-USB': ['https://github.com/sekigon-gonnoc/Pico-PIO-USB.git',
-                                         'fe9133fc513b82cc3dc62c67cb51f2339cf29ef7',
+                                         '3c1eec341a5232640e4c00628b889b641af34b28',
                                          'rp2040'],
     'hw/mcu/renesas/fsp': ['https://github.com/renesas/fsp.git',
                            'edcc97d684b6f716728a60d7a6fea049d9870bd6',
@@ -103,6 +103,9 @@ deps_optional = {
     'hw/mcu/st/cmsis_device_h7': ['https://github.com/STMicroelectronics/cmsis_device_h7.git',
                                   '60dc2c913203dc8629dc233d4384dcc41c91e77f',
                                   'stm32h7'],
+    'hw/mcu/st/cmsis_device_h7rs': ['https://github.com/STMicroelectronics/cmsis_device_h7rs.git',
+                                  '832649d1fd09bd901e9f68e979522e5c209ebf20',
+                                  'stm32h7rs'],
     'hw/mcu/st/cmsis_device_h5': ['https://github.com/STMicroelectronics/cmsis_device_h5.git',
                                   'cd2d1d579743de57b88ccaf61a968b9c05848ffc',
                                   'stm32h5'],
@@ -118,15 +121,21 @@ deps_optional = {
     'hw/mcu/st/cmsis_device_l5': ['https://github.com/STMicroelectronics/cmsis_device_l5.git',
                                   'd922865fc0326a102c26211c44b8e42f52c1e53d',
                                   'stm32l5'],
+    'hw/mcu/st/cmsis_device_n6': ['https://github.com/STMicroelectronics/cmsis-device-n6.git',
+                                  'f818b00f775444e8d19ef6cad822534c345e054f',
+                                  'stm32n6'],
     'hw/mcu/st/cmsis_device_u5': ['https://github.com/STMicroelectronics/cmsis_device_u5.git',
                                   '5ad9797c54ec3e55eff770fc9b3cd4a1aefc1309',
                                   'stm32u5'],
     'hw/mcu/st/cmsis_device_wb': ['https://github.com/STMicroelectronics/cmsis_device_wb.git',
-                                  '9c5d1920dd9fabbe2548e10561d63db829bb744f',
+                                  'd6a7fa2e7de084f5e5e47f2ab88b022fe9b50e5a',
                                   'stm32wb'],
     'hw/mcu/st/stm32-mfxstm32l152': ['https://github.com/STMicroelectronics/stm32-mfxstm32l152.git',
                                      '7f4389efee9c6a655b55e5df3fceef5586b35f9b',
                                      'stm32h7'],
+    'hw/mcu/st/stm32-tcpp0203': ['https://github.com/STMicroelectronics/stm32-tcpp0203.git',
+                                 '9918655bff176ac3046ccf378b5c7bbbc6a38d15',
+                                 'stm32h7rs stm32n6'],
     'hw/mcu/st/stm32c0xx_hal_driver': ['https://github.com/STMicroelectronics/stm32c0xx_hal_driver.git',
                                        '41253e2f1d7ae4a4d0c379cf63f5bcf71fcf8eb3',
                                        'stm32c0'],
@@ -157,6 +166,9 @@ deps_optional = {
     'hw/mcu/st/stm32h7xx_hal_driver': ['https://github.com/STMicroelectronics/stm32h7xx_hal_driver.git',
                                        'd8461b980b59b1625207d8c4f2ce0a9c2a7a3b04',
                                        'stm32h7'],
+    'hw/mcu/st/stm32h7rsxx_hal_driver': ['https://github.com/STMicroelectronics/stm32h7rsxx-hal-driver.git',
+                                       '7ca2e07ca21bc66b53654e845b4c85c884343b60',
+                                       'stm32h7rs'],
     'hw/mcu/st/stm32h5xx_hal_driver': ['https://github.com/STMicroelectronics/stm32h5xx_hal_driver.git',
                                        '2cf77de584196d619cec1b4586c3b9e2820a254e',
                                        'stm32h5'],
@@ -172,6 +184,9 @@ deps_optional = {
     'hw/mcu/st/stm32l5xx_hal_driver': ['https://github.com/STMicroelectronics/stm32l5xx_hal_driver.git',
                                        '675c32a75df37f39d50d61f51cb0dcf53f07e1cb',
                                        'stm32l5'],
+    'hw/mcu/st/stm32n6xx_hal_driver': ['https://github.com/STMicroelectronics/stm32n6xx-hal-driver.git',
+                                       '49f9989d10cf6817d4b07ac01848956b46bd0fd6',
+                                       'stm32n6'],
     'hw/mcu/st/stm32u5xx_hal_driver': ['https://github.com/STMicroelectronics/stm32u5xx_hal_driver.git',
                                        '4d93097a67928e9377e655ddd14622adc31b9770',
                                        'stm32u5'],
@@ -198,7 +213,7 @@ deps_optional = {
                     'imxrt kinetis_k32l2 kinetis_kl lpc51 lpc54 lpc55 mcx mm32 msp432e4 nrf saml2x '
                     'lpc11 lpc13 lpc15 lpc17 lpc18 lpc40 lpc43 '
                     'stm32c0 stm32f0 stm32f1 stm32f2 stm32f3 stm32f4 stm32f7 stm32g0 stm32g4 stm32h5 '
-                    'stm32h7 stm32l0 stm32l1 stm32l4 stm32l5 stm32u5 stm32wb '
+                    'stm32h7 stm32h7rs stm32l0 stm32l1 stm32l4 stm32l5 stm32n6 stm32u5 stm32wb '
                     'sam3x samd11 samd21 samd51 samd5x_e5x same5x same7x saml2x samg '
                     'tm4c '],
     'lib/CMSIS_6': ['https://github.com/ARM-software/CMSIS_6.git',
@@ -217,7 +232,12 @@ TOP = Path(__file__).parent.parent.resolve()
 
 
 def run_cmd(cmd):
-    return subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    r = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    title = f'Command Error: {cmd}'
+    if r.returncode != 0:
+        print(title)
+        print(r.stdout.decode("utf-8"))
+    return r
 
 
 def get_a_dep(d):
@@ -238,11 +258,13 @@ def get_a_dep(d):
         p.mkdir(parents=True)
         run_cmd(f"{git_cmd} init")
         run_cmd(f"{git_cmd} remote add origin {url}")
+        head = None
+    else:
+        # Check if commit is already fetched
+        result = run_cmd(f"{git_cmd} rev-parse HEAD")
+        head = result.stdout.decode("utf-8").splitlines()[0]
+        run_cmd(f"{git_cmd} reset --hard")
 
-    # Check if commit is already fetched
-    result = run_cmd(f"{git_cmd} rev-parse HEAD")
-    head = result.stdout.decode("utf-8").splitlines()[0]
-    run_cmd(f"{git_cmd} reset --hard")
     if commit != head:
         run_cmd(f"{git_cmd} fetch --depth 1 origin {commit}")
         run_cmd(f"{git_cmd} checkout FETCH_HEAD")
@@ -264,6 +286,7 @@ def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('families', nargs='*', default=[], help='Families to fetch')
     parser.add_argument('-b', '--board', action='append', default=[], help='Boards to fetch')
+    parser.add_argument('-D', '--define', action='append', default=[], help='Have no effect')
     parser.add_argument('-f1', '--build-flags-on', action='append', default=[], help='Have no effect')
     parser.add_argument('--print', action='store_true', help='Print commit hash only')
     args = parser.parse_args()