Merge branch 'master' into copilot/upgrade-net-lwip-webserver-descriptors

1 files changed, 350 insertions, 0 deletions

diff --git a/tools/metrics_compare_base.py b/tools/metrics_compare_base.py
new file mode 100644
index 000000000..799a96800
--- /dev/null
+++ b/tools/metrics_compare_base.py
@@ -0,0 +1,350 @@
+#!/usr/bin/env python3
+"""Build base branch (master) and current tree, then compare code size metrics.
+
+Creates cmake-metrics/<board>/{base,build} directories for each board.
+With --combined, also writes cmake-metrics/_combined/metrics_compare.md aggregating
+all boards into a single comparison.
+
+Usage:
+  python tools/metrics_compare_base.py -b raspberry_pi_pico
+  python tools/metrics_compare_base.py -b raspberry_pi_pico -b raspberry_pi_pico2
+  python tools/metrics_compare_base.py -b raspberry_pi_pico -f portable/raspberrypi
+  python tools/metrics_compare_base.py -b raspberry_pi_pico -e device/cdc_msc
+  python tools/metrics_compare_base.py -b raspberry_pi_pico -e device/cdc_msc --bloaty
+  python tools/metrics_compare_base.py --ci                          # first board of each arm-gcc family, combined
+  python tools/metrics_compare_base.py -b pico -b pico2 --combined   # aggregate listed boards
+"""
+import argparse
+import glob
+import json
+import os
+import re
+import shlex
+import subprocess
+import sys
+
+TINYUSB_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+METRICS_DIR = os.path.join(TINYUSB_ROOT, 'cmake-metrics')
+
+def tinyusb_src_filter(checkout_dir):
+    """Return a path-substring filter that uniquely matches TinyUSB stack source files
+    in `checkout_dir`. The substring is the absolute path to the checkout's `src/`
+    dir — collision-free with vendored deps (pico-sdk, lwip, FreeRTOS, etc.) which
+    live at unrelated paths."""
+    return os.path.realpath(os.path.join(checkout_dir, 'src')) + os.sep
+
+verbose = False
+
+
+def run(cmd, **kwargs):
+    """Run a command. cmd must be a list (no shell=True). On `timeout=`-induced
+    TimeoutExpired, return a CompletedProcess with rc=124 instead of letting the
+    exception propagate, so the caller can fall through to error reporting and
+    worktree cleanup rather than crashing with a traceback."""
+    if not isinstance(cmd, list):
+        raise TypeError('run() requires a list, got str — fix the caller')
+    if verbose:
+        print(f'  $ {" ".join(shlex.quote(str(c)) for c in cmd)}')
+    try:
+        return subprocess.run(cmd, capture_output=True, text=True, **kwargs)
+    except subprocess.TimeoutExpired as e:
+        msg = f'Command timed out after {e.timeout}s: {" ".join(shlex.quote(str(c)) for c in cmd)}'
+        stderr = (e.stderr or '') + ('\n' if e.stderr else '') + msg
+        return subprocess.CompletedProcess(cmd, 124, stdout=(e.stdout or ''), stderr=stderr)
+
+
+def symlink_deps(main_root, worktree_dir):
+    """Symlink dependency directories (fetched by tools/get_deps.py) from the main
+    checkout into the temporary worktree. Without this, the base build fails because
+    the worktree doesn't have the untracked deps."""
+    def link_subdirs(rel_parent):
+        src_parent = os.path.join(main_root, rel_parent)
+        dst_parent = os.path.join(worktree_dir, rel_parent)
+        if not os.path.isdir(src_parent):
+            return
+        os.makedirs(dst_parent, exist_ok=True)
+        for entry in os.listdir(src_parent):
+            src = os.path.join(src_parent, entry)
+            dst = os.path.join(dst_parent, entry)
+            if os.path.isdir(src) and not os.path.exists(dst):
+                os.symlink(src, dst)
+
+    # lib/* and tools/* deps (e.g. lib/lwip, tools/linkermap)
+    link_subdirs('lib')
+    link_subdirs('tools')
+    # hw/mcu/<vendor>/<dep> (e.g. hw/mcu/raspberry_pi/Pico-PIO-USB)
+    hw_mcu = os.path.join(main_root, 'hw', 'mcu')
+    if os.path.isdir(hw_mcu):
+        for vendor in os.listdir(hw_mcu):
+            link_subdirs(os.path.join('hw', 'mcu', vendor))
+
+
+def ci_first_boards():
+    """Return the first board (alphabetical) of each arm-gcc CI family."""
+    matrix_py = os.path.join(TINYUSB_ROOT, '.github', 'workflows', 'ci_set_matrix.py')
+    if not os.path.isfile(matrix_py):
+        return []
+    ret = run([sys.executable, matrix_py])
+    if ret.returncode != 0:
+        return []
+    try:
+        data = json.loads(ret.stdout)
+    except json.JSONDecodeError:
+        return []
+    families = data.get('arm-gcc', [])
+    boards = []
+    bsp_root = os.path.join(TINYUSB_ROOT, 'hw', 'bsp')
+    for family in families:
+        family_boards = sorted(
+            d for d in os.listdir(os.path.join(bsp_root, family, 'boards'))
+            if os.path.isdir(os.path.join(bsp_root, family, 'boards', d))
+        ) if os.path.isdir(os.path.join(bsp_root, family, 'boards')) else []
+        if family_boards:
+            boards.append(family_boards[0])
+    return boards
+
+
+def build_board(src_dir, build_dir, board, example=None):
+    """Configure and build examples for a board. Returns True on success.
+
+    When `example` is given, only that target is built (`cmake --build --target NAME`),
+    keeping single-example workflows fast.
+    """
+    os.makedirs(build_dir, exist_ok=True)
+    ret = run(['cmake', '-B', build_dir, '-G', 'Ninja',
+               f'-DBOARD={board}', '-DCMAKE_BUILD_TYPE=MinSizeRel',
+               os.path.join(src_dir, 'examples')])
+    if ret.returncode != 0:
+        print(f'  Error configuring {board}: {ret.stderr}')
+        return False
+    cmd = ['cmake', '--build', build_dir]
+    if example:
+        cmd += ['--target', os.path.basename(example)]
+    ret = run(cmd, timeout=600)
+    if ret.returncode != 0:
+        print(f'  Error building {board}: {ret.stderr}')
+        return False
+    return True
+
+
+def generate_metrics(build_dir, out_basename, filters, example=None):
+    """Run metrics.py combine on .map.json files. Returns metrics json path or None.
+
+    `filters` is a list of substrings; metrics.py keeps a compile unit if its path
+    contains any of them.
+    """
+    if example:
+        patterns = glob.glob(f'{build_dir}/{example}/*.map.json')
+    else:
+        patterns = glob.glob(f'{build_dir}/**/*.map.json', recursive=True)
+    if not patterns:
+        print(f'  Error: no .map.json files in {build_dir}' + (f' for {example}' if example else ''))
+        return None
+
+    metrics_py = os.path.join(TINYUSB_ROOT, 'tools', 'metrics.py')
+    cmd = [sys.executable, metrics_py, 'combine']
+    for f in filters:
+        cmd += ['-f', f]
+    cmd += ['-j', '-q', '-o', out_basename, *patterns]
+    ret = run(cmd)
+    if ret.returncode != 0:
+        print(f'  Error: {ret.stderr}')
+        return None
+    return f'{out_basename}.json'
+
+
+def main():
+    global verbose
+
+    parser = argparse.ArgumentParser(description='Compare code size metrics with base branch')
+    parser.add_argument('-b', '--board', action='append', default=[],
+                        help='Board name (repeatable). Required unless --ci is given.')
+    parser.add_argument('-f', '--filter', action='append', default=None,
+                        help='Path-substring filter (repeatable). When given, '
+                             'overrides the default and is applied to BOTH base and '
+                             'current builds. Default: each side\'s own absolute '
+                             '<checkout>/src/ path, which uniquely matches TinyUSB '
+                             'stack code without colliding with vendored deps.')
+    parser.add_argument('--base-branch', default='master',
+                        help='Base branch to compare against (default: master)')
+    parser.add_argument('-e', '--example', action='append', default=None,
+                        help='Compare specific example (repeatable, e.g. -e device/cdc_msc -e host/cdc_msc_hid)')
+    parser.add_argument('--bloaty', action='store_true',
+                        help='Use bloaty for detailed section/symbol diff (requires -e)')
+    parser.add_argument('--ci', action='store_true',
+                        help='Add the first board of every arm-gcc CI family. Implies --combined.')
+    parser.add_argument('--combined', action='store_true',
+                        help='Aggregate map.json files across all boards into one comparison '
+                             '(in cmake-metrics/_combined/), instead of (or in addition to) per-board.')
+    parser.add_argument('-v', '--verbose', action='store_true',
+                        help='Print build commands')
+    args = parser.parse_args()
+    verbose = args.verbose
+
+    if args.bloaty and not args.example:
+        parser.error('--bloaty requires -e/--example')
+
+    if args.ci:
+        args.combined = True
+        ci_boards = ci_first_boards()
+        if not ci_boards:
+            parser.error('--ci: failed to derive boards from .github/workflows/ci_set_matrix.py')
+        # Append, dedup, preserve order
+        seen = set(args.board)
+        for b in ci_boards:
+            if b not in seen:
+                args.board.append(b)
+                seen.add(b)
+
+    if not args.board:
+        parser.error('at least one -b BOARD is required (or pass --ci)')
+
+    metrics_py = os.path.join(TINYUSB_ROOT, 'tools', 'metrics.py')
+    worktree_dir = os.path.join(METRICS_DIR, '_worktree')
+
+    # Per-side filters: when no override is given, each build uses its own
+    # absolute <checkout>/src/ path so we only match TinyUSB stack code from that
+    # checkout (and never vendored-dep `src/` like pico-sdk/src/...).
+    if args.filter:
+        base_filters = cur_filters = list(args.filter)
+    else:
+        base_filters = [tinyusb_src_filter(worktree_dir)]
+        cur_filters = [tinyusb_src_filter(TINYUSB_ROOT)]
+
+    # Step 1: Create worktree for base branch
+    print(f'[1/5] Setting up {args.base_branch} worktree...')
+    if os.path.isdir(worktree_dir):
+        run(['git', '-C', TINYUSB_ROOT, 'worktree', 'remove', '--force', worktree_dir])
+    # --detach: check out the ref at a detached HEAD instead of trying to claim the
+    # branch. Lets us add a worktree of `master` even if master is already checked
+    # out elsewhere (main repo, another worktree).
+    ret = run(['git', '-C', TINYUSB_ROOT, 'worktree', 'add', '--detach',
+               worktree_dir, args.base_branch])
+    if ret.returncode != 0:
+        print(f'Error creating worktree: {ret.stderr}')
+        sys.exit(1)
+
+    # Symlink dependency dirs (lib/*, hw/mcu/*/*, tools/*) so the worktree builds.
+    symlink_deps(TINYUSB_ROOT, worktree_dir)
+
+    try:
+        examples = args.example or [None]
+        # For --combined: track every (base_build, cur_build) pair so we can aggregate at the end.
+        built_pairs = []
+
+        for board in args.board:
+            print(f'\n=== {board} ===')
+            board_dir = os.path.join(METRICS_DIR, board)
+            base_build = os.path.join(board_dir, 'base')
+            cur_build = os.path.join(board_dir, 'build')
+
+            # Build only the requested examples (or all if -e not given). Single-example
+            # mode used to build everything and filter at metric time — that was wasted work.
+            board_failed = False
+            for example in examples:
+                build_label = f' --target {os.path.basename(example)}' if example else ''
+                print(f'[2/5] Building {args.base_branch} for {board}{build_label}...')
+                if not build_board(worktree_dir, base_build, board, example):
+                    board_failed = True
+                    break
+                print(f'[3/5] Building current for {board}{build_label}...')
+                if not build_board(TINYUSB_ROOT, cur_build, board, example):
+                    board_failed = True
+                    break
+            if board_failed:
+                continue
+
+            built_pairs.append((board, base_build, cur_build))
+
+            for example in examples:
+                suffix = f'_{example.replace("/", "_")}' if example else ''
+                label = f' ({example})' if example else ''
+
+                # Step 4: Generate metrics
+                print(f'[4/5] Generating metrics for {board}{label}...')
+                base_json = generate_metrics(base_build, os.path.join(board_dir, f'base_metrics{suffix}'),
+                                             base_filters, example)
+                cur_json = generate_metrics(cur_build, os.path.join(board_dir, f'build_metrics{suffix}'),
+                                            cur_filters, example)
+                if not base_json or not cur_json:
+                    continue
+
+                # Step 5: Compare
+                out_base = os.path.join(board_dir, f'metrics_compare{suffix}')
+                print(f'[5/5] Comparing {board}{label}...')
+                ret = run([sys.executable, metrics_py, 'compare', '-m', '-o', out_base, base_json, cur_json])
+                print(ret.stdout)
+
+                # Optional: bloaty diff
+                if args.bloaty and example:
+                    elf_name = os.path.basename(example)
+                    base_elf = os.path.join(base_build, example, f'{elf_name}.elf')
+                    cur_elf = os.path.join(cur_build, example, f'{elf_name}.elf')
+                    if os.path.exists(base_elf) and os.path.exists(cur_elf):
+                        # Bloaty expects one regex; OR-join all filters (current side
+                        # for the new ELF, base side for the base ELF).
+                        bloaty_regex = '(' + '|'.join(
+                            re.escape(f) for f in (cur_filters + base_filters)
+                        ) + ')'
+                        bloaty_common = ['bloaty', '--domain=vm', f'--source-filter={bloaty_regex}']
+                        print(f'--- bloaty sections ---')
+                        ret = run(bloaty_common + ['-d', 'compileunits,sections', cur_elf, '--', base_elf])
+                        print(ret.stdout)
+                        print(f'--- bloaty symbols ---')
+                        ret = run(bloaty_common + ['-d', 'compileunits,symbols', '-s', 'vm',
+                                                    cur_elf, '--', base_elf])
+                        print(ret.stdout)
+                    else:
+                        print(f'  bloaty: ELF not found')
+
+        # Optional combined comparison across all boards.
+        # Aggregates the per-board metrics JSONs (not raw map.json globs) so the argv
+        # stays small even with --ci spanning many boards.
+        if args.combined and built_pairs:
+            combined_dir = os.path.join(METRICS_DIR, '_combined')
+            os.makedirs(combined_dir, exist_ok=True)
+
+            # Use the no-suffix per-board JSONs (whole-board metrics). Combined mode
+            # is meant for board-level sweeps; -e/--example combinations skip combined.
+            base_jsons, cur_jsons = [], []
+            for board, _, _ in built_pairs:
+                bj = os.path.join(METRICS_DIR, board, 'base_metrics.json')
+                cj = os.path.join(METRICS_DIR, board, 'build_metrics.json')
+                if os.path.isfile(bj) and os.path.isfile(cj):
+                    base_jsons.append(bj)
+                    cur_jsons.append(cj)
+
+            if not base_jsons or not cur_jsons:
+                print('  combined: no per-board metrics found (did you pass -e? skip --combined with -e)')
+            else:
+                print(f'\n=== combined ({len(base_jsons)} boards) ===')
+                base_out = os.path.join(combined_dir, 'base_metrics')
+                cur_out = os.path.join(combined_dir, 'build_metrics')
+
+                # Per-board JSONs are already filtered to TinyUSB-only files; combine
+                # without re-filtering so we don't accidentally drop entries.
+                def _combine(out_basename, inputs):
+                    cmd = [sys.executable, metrics_py, 'combine',
+                           '-j', '-q', '-o', out_basename, *inputs]
+                    return run(cmd)
+
+                ret = _combine(base_out, base_jsons)
+                if ret.returncode != 0:
+                    print(f'  combined base error: {ret.stderr}')
+                else:
+                    ret = _combine(cur_out, cur_jsons)
+                    if ret.returncode != 0:
+                        print(f'  combined current error: {ret.stderr}')
+                    else:
+                        out_combined = os.path.join(combined_dir, 'metrics_compare')
+                        ret = run([sys.executable, metrics_py, 'compare', '-m',
+                                   '-o', out_combined, f'{base_out}.json', f'{cur_out}.json'])
+                        print(ret.stdout)
+                        print(f'  combined report: {out_combined}.md')
+    finally:
+        print(f'\nCleaning up worktree...')
+        run(['git', '-C', TINYUSB_ROOT, 'worktree', 'remove', '--force', worktree_dir])
+
+
+if __name__ == '__main__':
+    main()