diff options
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/metrics_compare_base.py | 350 |
1 files changed, 350 insertions, 0 deletions
diff --git a/tools/metrics_compare_base.py b/tools/metrics_compare_base.py new file mode 100644 index 000000000..799a96800 --- /dev/null +++ b/tools/metrics_compare_base.py @@ -0,0 +1,350 @@ +#!/usr/bin/env python3 +"""Build base branch (master) and current tree, then compare code size metrics. + +Creates cmake-metrics/<board>/{base,build} directories for each board. +With --combined, also writes cmake-metrics/_combined/metrics_compare.md aggregating +all boards into a single comparison. + +Usage: + python tools/metrics_compare_base.py -b raspberry_pi_pico + python tools/metrics_compare_base.py -b raspberry_pi_pico -b raspberry_pi_pico2 + python tools/metrics_compare_base.py -b raspberry_pi_pico -f portable/raspberrypi + python tools/metrics_compare_base.py -b raspberry_pi_pico -e device/cdc_msc + python tools/metrics_compare_base.py -b raspberry_pi_pico -e device/cdc_msc --bloaty + python tools/metrics_compare_base.py --ci # first board of each arm-gcc family, combined + python tools/metrics_compare_base.py -b pico -b pico2 --combined # aggregate listed boards +""" +import argparse +import glob +import json +import os +import re +import shlex +import subprocess +import sys + +TINYUSB_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +METRICS_DIR = os.path.join(TINYUSB_ROOT, 'cmake-metrics') + +def tinyusb_src_filter(checkout_dir): + """Return a path-substring filter that uniquely matches TinyUSB stack source files + in `checkout_dir`. The substring is the absolute path to the checkout's `src/` + dir — collision-free with vendored deps (pico-sdk, lwip, FreeRTOS, etc.) which + live at unrelated paths.""" + return os.path.realpath(os.path.join(checkout_dir, 'src')) + os.sep + +verbose = False + + +def run(cmd, **kwargs): + """Run a command. cmd must be a list (no shell=True). On `timeout=`-induced + TimeoutExpired, return a CompletedProcess with rc=124 instead of letting the + exception propagate, so the caller can fall through to error reporting and + worktree cleanup rather than crashing with a traceback.""" + if not isinstance(cmd, list): + raise TypeError('run() requires a list, got str — fix the caller') + if verbose: + print(f' $ {" ".join(shlex.quote(str(c)) for c in cmd)}') + try: + return subprocess.run(cmd, capture_output=True, text=True, **kwargs) + except subprocess.TimeoutExpired as e: + msg = f'Command timed out after {e.timeout}s: {" ".join(shlex.quote(str(c)) for c in cmd)}' + stderr = (e.stderr or '') + ('\n' if e.stderr else '') + msg + return subprocess.CompletedProcess(cmd, 124, stdout=(e.stdout or ''), stderr=stderr) + + +def symlink_deps(main_root, worktree_dir): + """Symlink dependency directories (fetched by tools/get_deps.py) from the main + checkout into the temporary worktree. Without this, the base build fails because + the worktree doesn't have the untracked deps.""" + def link_subdirs(rel_parent): + src_parent = os.path.join(main_root, rel_parent) + dst_parent = os.path.join(worktree_dir, rel_parent) + if not os.path.isdir(src_parent): + return + os.makedirs(dst_parent, exist_ok=True) + for entry in os.listdir(src_parent): + src = os.path.join(src_parent, entry) + dst = os.path.join(dst_parent, entry) + if os.path.isdir(src) and not os.path.exists(dst): + os.symlink(src, dst) + + # lib/* and tools/* deps (e.g. lib/lwip, tools/linkermap) + link_subdirs('lib') + link_subdirs('tools') + # hw/mcu/<vendor>/<dep> (e.g. hw/mcu/raspberry_pi/Pico-PIO-USB) + hw_mcu = os.path.join(main_root, 'hw', 'mcu') + if os.path.isdir(hw_mcu): + for vendor in os.listdir(hw_mcu): + link_subdirs(os.path.join('hw', 'mcu', vendor)) + + +def ci_first_boards(): + """Return the first board (alphabetical) of each arm-gcc CI family.""" + matrix_py = os.path.join(TINYUSB_ROOT, '.github', 'workflows', 'ci_set_matrix.py') + if not os.path.isfile(matrix_py): + return [] + ret = run([sys.executable, matrix_py]) + if ret.returncode != 0: + return [] + try: + data = json.loads(ret.stdout) + except json.JSONDecodeError: + return [] + families = data.get('arm-gcc', []) + boards = [] + bsp_root = os.path.join(TINYUSB_ROOT, 'hw', 'bsp') + for family in families: + family_boards = sorted( + d for d in os.listdir(os.path.join(bsp_root, family, 'boards')) + if os.path.isdir(os.path.join(bsp_root, family, 'boards', d)) + ) if os.path.isdir(os.path.join(bsp_root, family, 'boards')) else [] + if family_boards: + boards.append(family_boards[0]) + return boards + + +def build_board(src_dir, build_dir, board, example=None): + """Configure and build examples for a board. Returns True on success. + + When `example` is given, only that target is built (`cmake --build --target NAME`), + keeping single-example workflows fast. + """ + os.makedirs(build_dir, exist_ok=True) + ret = run(['cmake', '-B', build_dir, '-G', 'Ninja', + f'-DBOARD={board}', '-DCMAKE_BUILD_TYPE=MinSizeRel', + os.path.join(src_dir, 'examples')]) + if ret.returncode != 0: + print(f' Error configuring {board}: {ret.stderr}') + return False + cmd = ['cmake', '--build', build_dir] + if example: + cmd += ['--target', os.path.basename(example)] + ret = run(cmd, timeout=600) + if ret.returncode != 0: + print(f' Error building {board}: {ret.stderr}') + return False + return True + + +def generate_metrics(build_dir, out_basename, filters, example=None): + """Run metrics.py combine on .map.json files. Returns metrics json path or None. + + `filters` is a list of substrings; metrics.py keeps a compile unit if its path + contains any of them. + """ + if example: + patterns = glob.glob(f'{build_dir}/{example}/*.map.json') + else: + patterns = glob.glob(f'{build_dir}/**/*.map.json', recursive=True) + if not patterns: + print(f' Error: no .map.json files in {build_dir}' + (f' for {example}' if example else '')) + return None + + metrics_py = os.path.join(TINYUSB_ROOT, 'tools', 'metrics.py') + cmd = [sys.executable, metrics_py, 'combine'] + for f in filters: + cmd += ['-f', f] + cmd += ['-j', '-q', '-o', out_basename, *patterns] + ret = run(cmd) + if ret.returncode != 0: + print(f' Error: {ret.stderr}') + return None + return f'{out_basename}.json' + + +def main(): + global verbose + + parser = argparse.ArgumentParser(description='Compare code size metrics with base branch') + parser.add_argument('-b', '--board', action='append', default=[], + help='Board name (repeatable). Required unless --ci is given.') + parser.add_argument('-f', '--filter', action='append', default=None, + help='Path-substring filter (repeatable). When given, ' + 'overrides the default and is applied to BOTH base and ' + 'current builds. Default: each side\'s own absolute ' + '<checkout>/src/ path, which uniquely matches TinyUSB ' + 'stack code without colliding with vendored deps.') + parser.add_argument('--base-branch', default='master', + help='Base branch to compare against (default: master)') + parser.add_argument('-e', '--example', action='append', default=None, + help='Compare specific example (repeatable, e.g. -e device/cdc_msc -e host/cdc_msc_hid)') + parser.add_argument('--bloaty', action='store_true', + help='Use bloaty for detailed section/symbol diff (requires -e)') + parser.add_argument('--ci', action='store_true', + help='Add the first board of every arm-gcc CI family. Implies --combined.') + parser.add_argument('--combined', action='store_true', + help='Aggregate map.json files across all boards into one comparison ' + '(in cmake-metrics/_combined/), instead of (or in addition to) per-board.') + parser.add_argument('-v', '--verbose', action='store_true', + help='Print build commands') + args = parser.parse_args() + verbose = args.verbose + + if args.bloaty and not args.example: + parser.error('--bloaty requires -e/--example') + + if args.ci: + args.combined = True + ci_boards = ci_first_boards() + if not ci_boards: + parser.error('--ci: failed to derive boards from .github/workflows/ci_set_matrix.py') + # Append, dedup, preserve order + seen = set(args.board) + for b in ci_boards: + if b not in seen: + args.board.append(b) + seen.add(b) + + if not args.board: + parser.error('at least one -b BOARD is required (or pass --ci)') + + metrics_py = os.path.join(TINYUSB_ROOT, 'tools', 'metrics.py') + worktree_dir = os.path.join(METRICS_DIR, '_worktree') + + # Per-side filters: when no override is given, each build uses its own + # absolute <checkout>/src/ path so we only match TinyUSB stack code from that + # checkout (and never vendored-dep `src/` like pico-sdk/src/...). + if args.filter: + base_filters = cur_filters = list(args.filter) + else: + base_filters = [tinyusb_src_filter(worktree_dir)] + cur_filters = [tinyusb_src_filter(TINYUSB_ROOT)] + + # Step 1: Create worktree for base branch + print(f'[1/5] Setting up {args.base_branch} worktree...') + if os.path.isdir(worktree_dir): + run(['git', '-C', TINYUSB_ROOT, 'worktree', 'remove', '--force', worktree_dir]) + # --detach: check out the ref at a detached HEAD instead of trying to claim the + # branch. Lets us add a worktree of `master` even if master is already checked + # out elsewhere (main repo, another worktree). + ret = run(['git', '-C', TINYUSB_ROOT, 'worktree', 'add', '--detach', + worktree_dir, args.base_branch]) + if ret.returncode != 0: + print(f'Error creating worktree: {ret.stderr}') + sys.exit(1) + + # Symlink dependency dirs (lib/*, hw/mcu/*/*, tools/*) so the worktree builds. + symlink_deps(TINYUSB_ROOT, worktree_dir) + + try: + examples = args.example or [None] + # For --combined: track every (base_build, cur_build) pair so we can aggregate at the end. + built_pairs = [] + + for board in args.board: + print(f'\n=== {board} ===') + board_dir = os.path.join(METRICS_DIR, board) + base_build = os.path.join(board_dir, 'base') + cur_build = os.path.join(board_dir, 'build') + + # Build only the requested examples (or all if -e not given). Single-example + # mode used to build everything and filter at metric time — that was wasted work. + board_failed = False + for example in examples: + build_label = f' --target {os.path.basename(example)}' if example else '' + print(f'[2/5] Building {args.base_branch} for {board}{build_label}...') + if not build_board(worktree_dir, base_build, board, example): + board_failed = True + break + print(f'[3/5] Building current for {board}{build_label}...') + if not build_board(TINYUSB_ROOT, cur_build, board, example): + board_failed = True + break + if board_failed: + continue + + built_pairs.append((board, base_build, cur_build)) + + for example in examples: + suffix = f'_{example.replace("/", "_")}' if example else '' + label = f' ({example})' if example else '' + + # Step 4: Generate metrics + print(f'[4/5] Generating metrics for {board}{label}...') + base_json = generate_metrics(base_build, os.path.join(board_dir, f'base_metrics{suffix}'), + base_filters, example) + cur_json = generate_metrics(cur_build, os.path.join(board_dir, f'build_metrics{suffix}'), + cur_filters, example) + if not base_json or not cur_json: + continue + + # Step 5: Compare + out_base = os.path.join(board_dir, f'metrics_compare{suffix}') + print(f'[5/5] Comparing {board}{label}...') + ret = run([sys.executable, metrics_py, 'compare', '-m', '-o', out_base, base_json, cur_json]) + print(ret.stdout) + + # Optional: bloaty diff + if args.bloaty and example: + elf_name = os.path.basename(example) + base_elf = os.path.join(base_build, example, f'{elf_name}.elf') + cur_elf = os.path.join(cur_build, example, f'{elf_name}.elf') + if os.path.exists(base_elf) and os.path.exists(cur_elf): + # Bloaty expects one regex; OR-join all filters (current side + # for the new ELF, base side for the base ELF). + bloaty_regex = '(' + '|'.join( + re.escape(f) for f in (cur_filters + base_filters) + ) + ')' + bloaty_common = ['bloaty', '--domain=vm', f'--source-filter={bloaty_regex}'] + print(f'--- bloaty sections ---') + ret = run(bloaty_common + ['-d', 'compileunits,sections', cur_elf, '--', base_elf]) + print(ret.stdout) + print(f'--- bloaty symbols ---') + ret = run(bloaty_common + ['-d', 'compileunits,symbols', '-s', 'vm', + cur_elf, '--', base_elf]) + print(ret.stdout) + else: + print(f' bloaty: ELF not found') + + # Optional combined comparison across all boards. + # Aggregates the per-board metrics JSONs (not raw map.json globs) so the argv + # stays small even with --ci spanning many boards. + if args.combined and built_pairs: + combined_dir = os.path.join(METRICS_DIR, '_combined') + os.makedirs(combined_dir, exist_ok=True) + + # Use the no-suffix per-board JSONs (whole-board metrics). Combined mode + # is meant for board-level sweeps; -e/--example combinations skip combined. + base_jsons, cur_jsons = [], [] + for board, _, _ in built_pairs: + bj = os.path.join(METRICS_DIR, board, 'base_metrics.json') + cj = os.path.join(METRICS_DIR, board, 'build_metrics.json') + if os.path.isfile(bj) and os.path.isfile(cj): + base_jsons.append(bj) + cur_jsons.append(cj) + + if not base_jsons or not cur_jsons: + print(' combined: no per-board metrics found (did you pass -e? skip --combined with -e)') + else: + print(f'\n=== combined ({len(base_jsons)} boards) ===') + base_out = os.path.join(combined_dir, 'base_metrics') + cur_out = os.path.join(combined_dir, 'build_metrics') + + # Per-board JSONs are already filtered to TinyUSB-only files; combine + # without re-filtering so we don't accidentally drop entries. + def _combine(out_basename, inputs): + cmd = [sys.executable, metrics_py, 'combine', + '-j', '-q', '-o', out_basename, *inputs] + return run(cmd) + + ret = _combine(base_out, base_jsons) + if ret.returncode != 0: + print(f' combined base error: {ret.stderr}') + else: + ret = _combine(cur_out, cur_jsons) + if ret.returncode != 0: + print(f' combined current error: {ret.stderr}') + else: + out_combined = os.path.join(combined_dir, 'metrics_compare') + ret = run([sys.executable, metrics_py, 'compare', '-m', + '-o', out_combined, f'{base_out}.json', f'{cur_out}.json']) + print(ret.stdout) + print(f' combined report: {out_combined}.md') + finally: + print(f'\nCleaning up worktree...') + run(['git', '-C', TINYUSB_ROOT, 'worktree', 'remove', '--force', worktree_dir]) + + +if __name__ == '__main__': + main() |