Merge remote-tracking branch 'upstream/master' into hcd-template-comments

Signed-off-by: HiFiPhile <[email protected]>

9 files changed, 388 insertions, 162 deletions

diff --git a/tools/build.py b/tools/build.py
index b937a7342..633d2b582 100644..100755
--- a/tools/build.py
+++ b/tools/build.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 import argparse
 import random
 import os
@@ -9,27 +10,42 @@ from multiprocessing import Pool
 
 import build_utils
 
-SUCCEEDED = "\033[32msucceeded\033[0m"
-FAILED = "\033[31mfailed\033[0m"
+STATUS_OK = "\033[32mOK\033[0m"
+STATUS_FAILED = "\033[31mFailed\033[0m"
+STATUS_SKIPPED = "\033[33mSkipped\033[0m"
 
-build_separator = '-' * 106
+RET_OK = 0
+RET_FAILED = 1
+RET_SKIPPED = 2
 
+build_format = '| {:30} | {:40} | {:16} | {:5} |'
+build_separator = '-' * 95
+build_status = [STATUS_OK, STATUS_FAILED, STATUS_SKIPPED]
 
+verbose = False
+
+# -----------------------------
+# Helper
+# -----------------------------
 def run_cmd(cmd):
     #print(cmd)
     r = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    title = 'command error'
+    title = f'Command Error: {cmd}'
     if r.returncode != 0:
         # print build output if failed
-        if os.getenv('CI'):
+        if os.getenv('GITHUB_ACTIONS'):
             print(f"::group::{title}")
             print(r.stdout.decode("utf-8"))
             print(f"::endgroup::")
         else:
             print(title)
             print(r.stdout.decode("utf-8"))
+    elif verbose:
+        print(cmd)
+        print(r.stdout.decode("utf-8"))
     return r
 
+
 def find_family(board):
     bsp_dir = Path("hw/bsp")
     for family_dir in bsp_dir.iterdir():
@@ -52,78 +68,120 @@ def get_examples(family):
     if family == 'espressif':
         all_examples.append('device/board_test')
         all_examples.append('device/video_capture')
+        all_examples.append('host/device_info')
     all_examples.sort()
     return all_examples
 
 
-def build_board_cmake(board, toolchain):
-    start_time = time.monotonic()
+def print_build_result(board, example, status, duration):
+    if isinstance(duration, (int, float)):
+        duration = "{:.2f}s".format(duration)
+    print(build_format.format(board, example, build_status[status], duration))
+
+# -----------------------------
+# CMake
+# -----------------------------
+def cmake_board(board, toolchain, build_flags_on):
     ret = [0, 0, 0]
+    start_time = time.monotonic()
+
+    build_dir = f'cmake-build/cmake-build-{board}'
+    build_flags = ''
+    if len(build_flags_on) > 0:
+        build_flags =  ' '.join(f'-D{flag}=1' for flag in build_flags_on)
+        build_flags = f'-DCFLAGS_CLI="{build_flags}"'
+        build_dir += '-f1_' + '_'.join(build_flags_on)
 
-    build_dir = f"cmake-build/cmake-build-{board}"
     family = find_family(board)
     if family == 'espressif':
         # for espressif, we have to build example individually
         all_examples = get_examples(family)
         for example in all_examples:
-            r = run_cmd(f'cmake examples/{example} -B {build_dir}/{example} -G "Ninja" -DBOARD={board} -DMAX3421_HOST=1')
-            if r.returncode == 0:
-                r = run_cmd(f'cmake --build {build_dir}/{example}')
-            if r.returncode == 0:
-                ret[0] += 1
+            if build_utils.skip_example(example, board):
+                ret[2] += 1
             else:
-                ret[1] += 1
+                rcmd = run_cmd(f'cmake examples/{example} -B {build_dir}/{example} -G "Ninja" '
+                               f'-DBOARD={board} {build_flags}')
+                if rcmd.returncode == 0:
+                    rcmd = run_cmd(f'cmake --build {build_dir}/{example}')
+                ret[0 if rcmd.returncode == 0 else 1] += 1
     else:
-        r = run_cmd(f'cmake examples -B {build_dir} -G "Ninja" -DBOARD={board} -DCMAKE_BUILD_TYPE=MinSizeRel -DTOOLCHAIN={toolchain}')
-        if r.returncode == 0:
-            r = run_cmd(f"cmake --build {build_dir}")
-        if r.returncode == 0:
-            ret[0] += 1
-        else:
-            ret[1] += 1
+        rcmd = run_cmd(f'cmake examples -B {build_dir} -G "Ninja" -DBOARD={board} -DCMAKE_BUILD_TYPE=MinSizeRel '
+                       f'-DTOOLCHAIN={toolchain} {build_flags}')
+        if rcmd.returncode == 0:
+            cmd = f"cmake --build {build_dir}"
+            # circleci docker return $nproc as 36 core, limit parallel according to resource class. Required for IAR, also prevent crashed/killed by docker
+            if os.getenv('CIRCLECI'):
+                resource_class = { 'small': 1, 'medium': 2, 'medium+': 3, 'large': 4 }
+                for rc in resource_class:
+                    if rc in os.getenv('CIRCLE_JOB'):
+                        cmd += f' --parallel {resource_class[rc]}'
+                        break
+            rcmd = run_cmd(cmd)
+        ret[0 if rcmd.returncode == 0 else 1] += 1
 
-    duration = time.monotonic() - start_time
+    example = 'all'
+    print_build_result(board, example, 0 if ret[1] == 0 else 1, time.monotonic() - start_time)
+    return ret
 
-    if ret[1] == 0:
-        status = SUCCEEDED
+
+# -----------------------------
+# Make
+# -----------------------------
+def make_one_example(example, board, make_option):
+    # Check if board is skipped
+    if build_utils.skip_example(example, board):
+        print_build_result(board, example, 2, '-')
+        r = 2
     else:
-        status = FAILED
+        start_time = time.monotonic()
+        # skip -j for circleci
+        if not os.getenv('CIRCLECI'):
+            make_option += ' -j'
+        make_cmd = f"make -C examples/{example} BOARD={board} {make_option}"
+        # run_cmd(f"{make_cmd} clean")
+        build_result = run_cmd(f"{make_cmd} all")
+        r = 0 if build_result.returncode == 0 else 1
+        print_build_result(board, example, r, time.monotonic() - start_time)
 
-    flash_size = "-"
-    sram_size = "-"
-    example = 'all'
-    title = build_utils.build_format.format(example, board, status, "{:.2f}s".format(duration), flash_size, sram_size)
-    print(title)
+    ret = [0, 0, 0]
+    ret[r] = 1
     return ret
 
 
-def build_board_make_all_examples(board, toolchain, all_examples):
+def make_board(board, toolchain):
+    print(build_separator)
+    all_examples = get_examples(find_family(board))
     start_time = time.monotonic()
     ret = [0, 0, 0]
-
     with Pool(processes=os.cpu_count()) as pool:
         pool_args = list((map(lambda e, b=board, o=f"TOOLCHAIN={toolchain}": [e, b, o], all_examples)))
-        r = pool.starmap(build_utils.build_example, pool_args)
+        r = pool.starmap(make_one_example, pool_args)
         # sum all element of same index (column sum)
-        rsum = list(map(sum, list(zip(*r))))
-        ret[0] += rsum[0]
-        ret[1] += rsum[1]
-        ret[2] += rsum[2]
-    duration = time.monotonic() - start_time
-    if ret[1] == 0:
-        status = SUCCEEDED
-    else:
-        status = FAILED
-
-    flash_size = "-"
-    sram_size = "-"
+        ret = list(map(sum, list(zip(*r))))
     example = 'all'
-    title = build_utils.build_format.format(example, board, status, "{:.2f}s".format(duration), flash_size, sram_size)
-    print(title)
+    print_build_result(board, example, 0 if ret[1] == 0 else 1, time.monotonic() - start_time)
+    return ret
+
+
+# -----------------------------
+# Build Family
+# -----------------------------
+def build_boards_list(boards, toolchain, build_system, build_flags_on):
+    ret = [0, 0, 0]
+    for b in boards:
+        r = [0, 0, 0]
+        if build_system == 'cmake':
+            r = cmake_board(b, toolchain, build_flags_on)
+        elif build_system == 'make':
+            r = make_board(b, toolchain)
+        ret[0] += r[0]
+        ret[1] += r[1]
+        ret[2] += r[2]
     return ret
 
 
-def build_family(family, toolchain, build_system, one_per_family, boards):
+def build_family(family, toolchain, build_system, build_flags_on, one_per_family, boards):
     all_boards = []
     for entry in os.scandir(f"hw/bsp/{family}/boards"):
         if entry.is_dir() and entry.name != 'pico_sdk':
@@ -131,7 +189,6 @@ def build_family(family, toolchain, build_system, one_per_family, boards):
     all_boards.sort()
 
     ret = [0, 0, 0]
-
     # If only-one flag is set, select one random board
     if one_per_family:
         for b in boards:
@@ -140,42 +197,40 @@ def build_family(family, toolchain, build_system, one_per_family, boards):
                 return ret
         all_boards = [random.choice(all_boards)]
 
-    # success, failed, skipped
-    all_examples = get_examples(family)
-    for board in all_boards:
-        r = [0, 0, 0]
-        if build_system == 'cmake':
-            r = build_board_cmake(board, toolchain)
-        elif build_system == 'make':
-            r = build_board_make_all_examples(board, toolchain, all_examples)
-        ret[0] += r[0]
-        ret[1] += r[1]
-        ret[2] += r[2]
-
+    ret = build_boards_list(all_boards, toolchain, build_system, build_flags_on)
     return ret
 
 
+# -----------------------------
+# Main
+# -----------------------------
 def main():
+    global verbose
+
     parser = argparse.ArgumentParser()
     parser.add_argument('families', nargs='*', default=[], help='Families to build')
     parser.add_argument('-b', '--board', action='append', default=[], help='Boards to build')
     parser.add_argument('-t', '--toolchain', default='gcc', help='Toolchain to use, default is gcc')
     parser.add_argument('-s', '--build-system', default='cmake', help='Build system to use, default is cmake')
+    parser.add_argument('-f1', '--build-flags-on', action='append', default=[], help='Build flag to pass to build system')
     parser.add_argument('-1', '--one-per-family', action='store_true', default=False, help='Build only one random board inside a family')
+    parser.add_argument('-v', '--verbose', action='store_true', help='Verbose output')
     args = parser.parse_args()
 
     families = args.families
     boards = args.board
     toolchain = args.toolchain
     build_system = args.build_system
+    build_flags_on = args.build_flags_on
     one_per_family = args.one_per_family
+    verbose = args.verbose
 
     if len(families) == 0 and len(boards) == 0:
         print("Please specify families or board to build")
         return 1
 
     print(build_separator)
-    print(build_utils.build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
+    print(build_format.format('Board', 'Example', '\033[39mResult\033[0m', 'Time'))
     total_time = time.monotonic()
     result = [0, 0, 0]
 
@@ -189,28 +244,22 @@ def main():
         all_families = list(families)
     all_families.sort()
 
-    # succeeded, failed
+    # succeeded, failed, skipped
     for f in all_families:
-        fret = build_family(f, toolchain, build_system, one_per_family, boards)
-        result[0] += fret[0]
-        result[1] += fret[1]
-        result[2] += fret[2]
-
-    # build boards
-    for b in boards:
-        r = [0, 0, 0]
-        if build_system == 'cmake':
-            r = build_board_cmake(b, toolchain)
-        elif build_system == 'make':
-            all_examples = get_examples(find_family(b))
-            r = build_board_make_all_examples(b, toolchain, all_examples)
+        r = build_family(f, toolchain, build_system, build_flags_on, one_per_family, boards)
         result[0] += r[0]
         result[1] += r[1]
         result[2] += r[2]
 
+    # build boards
+    r = build_boards_list(boards, toolchain, build_system, build_flags_on)
+    result[0] += r[0]
+    result[1] += r[1]
+    result[2] += r[2]
+
     total_time = time.monotonic() - total_time
     print(build_separator)
-    print(f"Build Summary: {result[0]} {SUCCEEDED}, {result[1]} {FAILED} and took {total_time:.2f}s")
+    print(f"Build Summary: {result[0]} {STATUS_OK}, {result[1]} {STATUS_FAILED} and took {total_time:.2f}s")
     print(build_separator)
     return result[1]
 
diff --git a/tools/build_utils.py b/tools/build_utils.py
index 32aca95dd..2998f940d 100644..100755
--- a/tools/build_utils.py
+++ b/tools/build_utils.py
@@ -1,6 +1,7 @@
+#!/usr/bin/env python3
 import subprocess
 import pathlib
-import time
+import re
 
 build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |'
 
@@ -13,54 +14,54 @@ def skip_example(example, board):
     ex_dir = pathlib.Path('examples/') / example
     bsp = pathlib.Path("hw/bsp")
 
-    if (bsp / board / "board.mk").exists():
-        # board without family
-        board_dir = bsp / board
-        family = ""
-        mk_contents = ""
-    else:
-        # board within family
-        board_dir = list(bsp.glob("*/boards/" + board))
-        if not board_dir:
-            # Skip unknown boards
-            return True
-
-        board_dir = list(board_dir)[0]
+    # board within family
+    board_dir = list(bsp.glob("*/boards/" + board))
+    if not board_dir:
+        # Skip unknown boards
+        return True
 
-        family_dir = board_dir.parent.parent
-        family = family_dir.name
+    board_dir = list(board_dir)[0]
+    family_dir = board_dir.parent.parent
+    family = family_dir.name
 
-        # family.mk
-        family_mk = family_dir / "family.mk"
-        mk_contents = family_mk.read_text()
+    # family.mk
+    family_mk = family_dir / "family.mk"
+    mk_contents = family_mk.read_text()
 
     # Find the mcu, first in family mk then board mk
     if "CFG_TUSB_MCU=OPT_MCU_" not in mk_contents:
-        board_mk = board_dir / "board.cmake"
+        board_mk = board_dir / "board.mk"
         if not board_mk.exists():
-            board_mk = board_dir / "board.mk"
-
+            board_mk = board_dir / "board.cmake"
         mk_contents = board_mk.read_text()
 
     mcu = "NONE"
-    for token in mk_contents.split():
-        if "CFG_TUSB_MCU=OPT_MCU_" in token:
-            # Strip " because cmake files has them.
-            token = token.strip("\"")
-            _, opt_mcu = token.split("=")
-            mcu = opt_mcu[len("OPT_MCU_"):]
-            break
-        if "esp32s2" in token:
-            mcu = "ESP32S2"
-            break
-        if "esp32s3" in token:
-            mcu = "ESP32S3"
-            break
+    if family == "espressif":
+        for line in mk_contents.splitlines():
+            match = re.search(r'set\(IDF_TARGET\s+"([^"]+)"\)', line)
+            if match:
+                mcu = match.group(1).upper()
+                break
+    else:
+        for token in mk_contents.split():
+            if "CFG_TUSB_MCU=OPT_MCU_" in token:
+                # Strip " because cmake files has them.
+                token = token.strip("\"")
+                _, opt_mcu = token.split("=")
+                mcu = opt_mcu[len("OPT_MCU_"):]
+            if mcu != "NONE":
+                break
 
     # Skip all OPT_MCU_NONE these are WIP port
     if mcu == "NONE":
         return True
 
+    max3421_enabled = False
+    for line in mk_contents.splitlines():
+        if "MAX3421_HOST=1" in line or 'MAX3421_HOST 1' in line:
+            max3421_enabled = True
+            break
+
     skip_file = ex_dir / "skip.txt"
     only_file = ex_dir / "only.txt"
 
@@ -74,6 +75,7 @@ def skip_example(example, board):
     if only_file.exists():
         onlys = only_file.read_text().split()
         if not ("mcu:" + mcu in onlys or
+                ("mcu:MAX3421" in onlys and max3421_enabled) or
                 "board:" + board in onlys or
                 "family:" + family in onlys):
             return True
@@ -92,38 +94,3 @@ def build_size(make_cmd):
             return (flash_size, sram_size)
 
     return (0, 0)
-
-
-def build_example(example, board, make_option):
-    start_time = time.monotonic()
-    flash_size = "-"
-    sram_size = "-"
-
-    # succeeded, failed, skipped
-    ret = [0, 0, 0]
-
-    make_cmd = f"make -j -C examples/{example} BOARD={board} {make_option}"
-
-    # Check if board is skipped
-    if skip_example(example, board):
-        status = SKIPPED
-        ret[2] = 1
-        print(build_format.format(example, board, status, '-', flash_size, sram_size))
-    else:
-        build_result = subprocess.run(f"{make_cmd} all", shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-
-        if build_result.returncode == 0:
-            status = SUCCEEDED
-            ret[0] = 1
-            (flash_size, sram_size) = build_size(make_cmd)
-        else:
-            status = FAILED
-            ret[1] = 1
-
-        build_duration = time.monotonic() - start_time
-        print(build_format.format(example, board, status, "{:.2f}s".format(build_duration), flash_size, sram_size))
-
-        if build_result.returncode != 0:
-            print(build_result.stdout.decode("utf-8"))
-
-    return ret
diff --git a/tools/gen_doc.py b/tools/gen_doc.py
index 668c77ef6..ab07bc116 100644..100755
--- a/tools/gen_doc.py
+++ b/tools/gen_doc.py
@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+import re
 import pandas as pd
 from tabulate import tabulate
 from pathlib import Path
@@ -10,7 +12,6 @@ TOP = Path(__file__).parent.parent.resolve()
 # -----------------------------------------
 # Dependencies
 # -----------------------------------------
-
 def gen_deps_doc():
     deps_rst = Path(TOP) / "docs/reference/dependencies.rst"
     df = pd.DataFrame.from_dict(deps_all, orient='index', columns=['Repo', 'Commit', 'Required by'])
@@ -31,5 +32,86 @@ MCU low-level peripheral driver and external libraries for building TinyUSB exam
         f.write(outstr)
 
 
+# -----------------------------------------
+# Dependencies
+# -----------------------------------------
+def extract_metadata(file_path):
+    metadata = {}
+    try:
+        with open(file_path, 'r') as file:
+            content = file.read()
+            # Match metadata block
+            match = re.search(r'/\*\s*metadata:(.*?)\*/', content, re.DOTALL)
+            if match:
+                block = match.group(1)
+                # Extract key-value pairs
+                for line in block.splitlines():
+                    key_value = re.match(r'\s*(\w+):\s*(.+)', line)
+                    if key_value:
+                        key, value = key_value.groups()
+                        metadata[key] = value.strip()
+    except FileNotFoundError:
+        pass
+    return metadata
+
+
+def gen_boards_doc():
+    # 'Manufacturer' : { 'Board' }
+    vendor_data = {}
+    # 'Board' : [ 'Name', 'Family', 'url', 'note' ]
+    all_boards = {}
+    #  extract metadata from family.c
+    for family_dir in sorted((Path(TOP) / "hw/bsp").iterdir()):
+        if family_dir.is_dir():
+            family_c = family_dir / "family.c"
+            if not family_c.exists():
+                family_c = family_dir / "boards/family.c"
+            f_meta = extract_metadata(family_c)
+            if not f_meta:
+                continue
+            manuf = f_meta.get('manufacturer', '')
+            if manuf not in vendor_data:
+                vendor_data[manuf] = {}
+            # extract metadata from board.h
+            for board_dir in sorted((family_dir / "boards").iterdir()):
+                if board_dir.is_dir():
+                    b_meta = extract_metadata(board_dir / "board.h")
+                    if not b_meta:
+                        continue
+                    b_entry = [
+                        b_meta.get('name', ''),
+                        family_dir.name,
+                        b_meta.get('url', ''),
+                        b_meta.get('note', '')
+                    ]
+                    vendor_data[manuf][board_dir.name] = b_entry
+    boards_rst = Path(TOP) / "docs/reference/boards.rst"
+    with boards_rst.open('w') as f:
+        title = f"""\
+****************
+Supported Boards
+****************
+
+The board support code is only used for self-contained examples and testing. It is not used when TinyUSB is part of a larger project.
+It is responsible for getting the MCU started and the USB peripheral clocked with minimal of on-board devices
+
+-  One LED : for status
+-  One Button : to get input from user
+-  One UART : needed for logging with LOGGER=uart, maybe required for host/dual examples
+
+Following boards are supported"""
+        f.write(title)
+        for manuf, boards in sorted(vendor_data.items()):
+            f.write(f"\n\n{manuf}\n")
+            f.write(f"{'-' * len(manuf)}\n\n")
+            df = pd.DataFrame.from_dict(boards, orient='index', columns=['Name', 'Family', 'URL', 'Note'])
+            df = df.rename_axis("Board")
+            f.write(tabulate(df, headers="keys", tablefmt='rst'))
+
+
+# -----------------------------------------
+# Main
+# -----------------------------------------
 if __name__ == "__main__":
     gen_deps_doc()
+    gen_boards_doc()
diff --git a/tools/gen_presets.py b/tools/gen_presets.py
new file mode 100755
index 000000000..94b8d16b0
--- /dev/null
+++ b/tools/gen_presets.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python3
+import os
+import json
+from pathlib import Path
+
+def main():
+    board_list = []
+
+    # Find all board.cmake files
+    for root, dirs, files in os.walk("hw/bsp"):
+        for file in files:
+            if file == "board.cmake":
+                board_list.append(os.path.basename(root))
+
+    print('Generating presets for the following boards:')
+    print(board_list)
+
+    # Generate the presets
+    presets = {}
+    presets['version'] = 6
+
+    # Configure presets
+    presets['configurePresets'] = [
+        {"name": "default",
+         "hidden": True,
+         "description": r"Configure preset for the ${presetName} board",
+         "generator": "Ninja Multi-Config",
+         "binaryDir": r"${sourceDir}/build/${presetName}",
+         "cacheVariables": {
+             "CMAKE_DEFAULT_BUILD_TYPE": "RelWithDebInfo",
+             "BOARD": r"${presetName}"
+         }}]
+
+    presets['configurePresets'].extend(
+        sorted(
+            [
+                {
+                    'name': board,
+                    'inherits': 'default'
+                }
+                for board in board_list
+            ], key=lambda x: x['name']
+        )
+    )
+
+    # Build presets
+    # no inheritance since 'name' doesn't support macro expansion
+    presets['buildPresets'] = sorted(
+        [
+            {
+                'name': board,
+                'description': "Build preset for the " + board + " board",
+                'configurePreset': board
+            }
+            for board in board_list
+        ], key=lambda x: x['name']
+    )
+
+    # Workflow presets
+    presets['workflowPresets'] = sorted(
+        [
+            {
+                "name": board,
+                "steps": [
+                    {
+                        "type": "configure",
+                        "name": board
+                    },
+                    {
+                        "type": "build",
+                        "name": board
+                    }
+                ]
+            }
+            for board in board_list
+        ], key=lambda x: x['name']
+    )
+
+    path_boardpresets = "hw/bsp/BoardPresets.json"
+    with open(path_boardpresets, "w") as f:
+        f.write('{}\n'.format(json.dumps(presets, indent=2)))
+
+    # Generate presets for examples
+    presets = {
+        "version": 6,
+        "include": [
+        ]
+    }
+
+    example_list = []
+    for root, dirs, files in os.walk("examples"):
+        for file in files:
+            # Filter out ESP-IDF CMakeLists.txt in src folder
+            if file == "CMakeLists.txt" and os.path.basename(root) != 'src':
+                presets['include'] = [os.path.relpath(path_boardpresets, root).replace(os.sep, '/')]
+                with open(os.path.join(root, 'CMakePresets.json'), 'w') as f:
+                    f.write('{}\n'.format(json.dumps(presets, indent=2)))
+                example_list.append(os.path.basename(root))
+
+    print('Generating presets for the following examples:')
+    print(example_list)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tools/get_deps.py b/tools/get_deps.py
index b639ed6d6..ba9dc23ce 100644..100755
--- a/tools/get_deps.py
+++ b/tools/get_deps.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 import argparse
 import sys
 import subprocess
@@ -24,6 +25,9 @@ deps_optional = {
     'hw/mcu/allwinner': ['https://github.com/hathach/allwinner_driver.git',
                          '8e5e89e8e132c0fd90e72d5422e5d3d68232b756',
                          'fc100s'],
+    'hw/mcu/analog/max32' : ['https://github.com/analogdevicesinc/msdk.git',
+                             'b20b398d3e5e2007594e54a74ba3d2a2e50ddd75',
+                             'max32650 max32666 max32690 max78002'],
     'hw/mcu/bridgetek/ft9xx/ft90x-sdk': ['https://github.com/BRTSG-FOSS/ft90x-sdk.git',
                                          '91060164afe239fcb394122e8bf9eb24d3194eb1',
                                          'brtmm90x'],
@@ -54,11 +58,11 @@ deps_optional = {
     'hw/mcu/nxp/mcux-sdk': ['https://github.com/hathach/mcux-sdk.git',
                             '144f1eb7ea8c06512e12f12b27383601c0272410',
                             'kinetis_k kinetis_k32l2 kinetis_kl lpc51 lpc54 lpc55 mcx imxrt'],
-    'hw/mcu/raspberry_pi/Pico-PIO-USB': ['https://github.com/sekigon-gonnoc/Pico-PIO-USB.git',
-                                         '7902e9fa8ed4a271d8d1d5e7e50516c2292b7bc2',
+    'hw/mcu/raspberry_pi/Pico-PIO-USB': ['https://github.com/hathach/Pico-PIO-USB.git',
+                                         '810653f66adadba3e0e4b4b56d5167ac4f7fdbf7',
                                          'rp2040'],
     'hw/mcu/renesas/fsp': ['https://github.com/renesas/fsp.git',
-                           'd52e5a6a59b7c638da860c2bb309b6e78e752ff8',
+                           'edcc97d684b6f716728a60d7a6fea049d9870bd6',
                            'ra'],
     'hw/mcu/renesas/rx': ['https://github.com/kkitayam/rx_device.git',
                           '706b4e0cf485605c32351e2f90f5698267996023',
@@ -69,6 +73,9 @@ deps_optional = {
     'hw/mcu/sony/cxd56/spresense-exported-sdk': ['https://github.com/sonydevworld/spresense-exported-sdk.git',
                                                  '2ec2a1538362696118dc3fdf56f33dacaf8f4067',
                                                  'spresense'],
+    'hw/mcu/st/cmsis_device_c0': ['https://github.com/STMicroelectronics/cmsis_device_c0.git',
+                                  'fb56b1b70c73b74eacda2a4bcc36886444364ab3',
+                                  'stm32c0'],
     'hw/mcu/st/cmsis_device_f0': ['https://github.com/STMicroelectronics/cmsis_device_f0.git',
                                   '2fc25ee22264bc27034358be0bd400b893ef837e',
                                   'stm32f0'],
@@ -117,6 +124,12 @@ deps_optional = {
     'hw/mcu/st/cmsis_device_wb': ['https://github.com/STMicroelectronics/cmsis_device_wb.git',
                                   '9c5d1920dd9fabbe2548e10561d63db829bb744f',
                                   'stm32wb'],
+    'hw/mcu/st/stm32-mfxstm32l152': ['https://github.com/STMicroelectronics/stm32-mfxstm32l152.git',
+                                     '7f4389efee9c6a655b55e5df3fceef5586b35f9b',
+                                     'stm32h7'],
+    'hw/mcu/st/stm32c0xx_hal_driver': ['https://github.com/STMicroelectronics/stm32c0xx_hal_driver.git',
+                                       '41253e2f1d7ae4a4d0c379cf63f5bcf71fcf8eb3',
+                                       'stm32c0'],
     'hw/mcu/st/stm32f0xx_hal_driver': ['https://github.com/STMicroelectronics/stm32f0xx_hal_driver.git',
                                        '0e95cd88657030f640a11e690a8a5186c7712ea5',
                                        'stm32f0'],
@@ -181,13 +194,16 @@ deps_optional = {
                             '77c4095087e5ed2c548ec9058e655d0b8757663b',
                             'ch32f20x'],
     'lib/CMSIS_5': ['https://github.com/ARM-software/CMSIS_5.git',
-                    '20285262657d1b482d132d20d755c8c330d55c1f',
-                    'imxrt kinetis_k32l2 kinetis_kl lpc51 lpc54 lpc55 mcx mm32 msp432e4 nrf ra saml2x'
-                    'lpc11 lpc13 lpc15 lpc17 lpc18 lpc40 lpc43'
-                    'stm32f0 stm32f1 stm32f2 stm32f3 stm32f4 stm32f7 stm32g0 stm32g4 stm32h5'
-                    'stm32h7 stm32l0 stm32l1 stm32l4 stm32l5 stm32u5 stm32wb'
-                    'sam3x samd11 samd21 samd51 samd5x_e5x same5x same7x saml2x samg'
-                    'tm4c'],
+                    '2b7495b8535bdcb306dac29b9ded4cfb679d7e5c',
+                    'imxrt kinetis_k32l2 kinetis_kl lpc51 lpc54 lpc55 mcx mm32 msp432e4 nrf saml2x '
+                    'lpc11 lpc13 lpc15 lpc17 lpc18 lpc40 lpc43 '
+                    'stm32c0 stm32f0 stm32f1 stm32f2 stm32f3 stm32f4 stm32f7 stm32g0 stm32g4 stm32h5 '
+                    'stm32h7 stm32l0 stm32l1 stm32l4 stm32l5 stm32u5 stm32wb '
+                    'sam3x samd11 samd21 samd51 samd5x_e5x same5x same7x saml2x samg '
+                    'tm4c '],
+    'lib/CMSIS_6': ['https://github.com/ARM-software/CMSIS_6.git',
+                    'b0bbb0423b278ca632cfe1474eb227961d835fd2',
+                    'ra'],
     'lib/sct_neopixel': ['https://github.com/gsteiert/sct_neopixel.git',
                          'e73e04ca63495672d955f9268e003cffe168fcd8',
                          'lpc55'],
@@ -201,7 +217,12 @@ TOP = Path(__file__).parent.parent.resolve()
 
 
 def run_cmd(cmd):
-    return subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    r = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    title = f'Command Error: {cmd}'
+    if r.returncode != 0:
+        print(title)
+        print(r.stdout.decode("utf-8"))
+    return r
 
 
 def get_a_dep(d):
@@ -248,6 +269,7 @@ def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('families', nargs='*', default=[], help='Families to fetch')
     parser.add_argument('-b', '--board', action='append', default=[], help='Boards to fetch')
+    parser.add_argument('-f1', '--build-flags-on', action='append', default=[], help='Have no effect')
     parser.add_argument('--print', action='store_true', help='Print commit hash only')
     args = parser.parse_args()
 
diff --git a/tools/iar_gen.py b/tools/iar_gen.py
index ebcfa1423..8d45659db 100644..100755
--- a/tools/iar_gen.py
+++ b/tools/iar_gen.py
@@ -1,4 +1,4 @@
-#!/usr/bin/python3
+#!/usr/bin/env python3
 
 import os
 import sys
diff --git a/tools/make_release.py b/tools/make_release.py
index 126e07292..c1caf3300 100644..100755
--- a/tools/make_release.py
+++ b/tools/make_release.py
@@ -1,7 +1,8 @@
+#!/usr/bin/env python3
 import re
 import gen_doc
 
-version = '0.16.0'
+version = '0.18.0'
 
 print('version {}'.format(version))
 ver_id = version.split('.')
diff --git a/tools/mksunxi.py b/tools/mksunxi.py
index fd8557cfc..fd8557cfc 100644..100755
--- a/tools/mksunxi.py
+++ b/tools/mksunxi.py
diff --git a/tools/pcapng_to_corpus.py b/tools/pcapng_to_corpus.py
index 9c31365eb..3089f0bb6 100755
--- a/tools/pcapng_to_corpus.py
+++ b/tools/pcapng_to_corpus.py
@@ -1,4 +1,4 @@
-#!/bin/python3
+#!/usr/bin/env python3
 import argparse
 import pcapng
 import zipfile