diff options
Diffstat (limited to '.github/scripts/metrics_pair_compare.py')
| -rwxr-xr-x | .github/scripts/metrics_pair_compare.py | 130 |
1 files changed, 130 insertions, 0 deletions
diff --git a/.github/scripts/metrics_pair_compare.py b/.github/scripts/metrics_pair_compare.py new file mode 100755 index 000000000..50107cf72 --- /dev/null +++ b/.github/scripts/metrics_pair_compare.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Board+example-matched code-size compare for PR-scoped builds. + +The averaged metrics baseline (metrics-tinyusb) spans every family and example; +a scoped PR builds a subset, so comparing against it is apples-to-oranges. This +compares the intersection of (board, example) pairs present on BOTH sides, +averaged over exactly those pairs, and names what was dropped. See +docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md #code-metrics. +""" +import argparse +import glob +import json +import os +import sys +import tempfile + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'tools')) +import metrics + +# dropped (board, example) pairs named in the PR comment before it truncates +DROPPED_SHOWN = 20 + + +def board_family(board, repo_root): + hits = glob.glob(os.path.join(repo_root, 'hw/bsp/*/boards', board)) + return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None + + +def collect(root, repo_root): + """{(board, 'role/example'): [file entries]} from every + **/cmake-build-<board>/metrics_by_example.json under root. + + Keyed on the BOARD, not its family. The two sides are built by + `--one-first`, which returns all_boards[0] for a family with no + ci_preferred_boards entry - so a PR that adds hw/bsp/<family>/boards/a_new_board + shifts which board is built, and a family key would file the base run's sizes and + the PR run's sizes under the same name and publish the difference between two + unrelated MCUs as this PR's code-size impact. On the board key that mismatch lands + in `dropped` (reported as not compared), which is the truth.""" + pairs = {} + pat = os.path.join(root, '**', 'metrics_by_example.json') + for f in sorted(glob.glob(pat, recursive=True)): + board = os.path.basename(os.path.dirname(f)) + if not board.startswith('cmake-build-'): + print(f'pair_compare: {f} not under a cmake-build-<board> dir, skipping', file=sys.stderr) + continue + board = board[len('cmake-build-'):] + if not board_family(board, repo_root): + # unknown board: the name is still a usable key, but say so - it means the + # artifact came from a tree whose hw/bsp does not match this checkout + print(f'pair_compare: no family for board {board}', file=sys.stderr) + # parse into a LOCAL dict and merge only once the whole file came out clean: + # a file that blows up half way through must drop WHOLE, or the entries read + # before the malformation stay in the comparison while stderr says the file + # was skipped, and a silently truncated table gets published as the verdict + try: + one = {} + for ex, ent in json.load(open(f)).items(): + one.setdefault((board, ex), []).extend(ent.get('files', [])) + except (OSError, ValueError, AttributeError, TypeError) as e: + print(f'pair_compare: unreadable {f} ({e}), skipping', file=sys.stderr) + continue + for k, v in one.items(): + pairs.setdefault(k, []).extend(v) + return pairs + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument('--base-dir', required=True) + ap.add_argument('--new-dir', required=True) + ap.add_argument('--out', default='metrics_compare') + a = ap.parse_args() + repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + base = collect(a.base_dir, repo_root) + new = collect(a.new_dir, repo_root) + common = sorted(set(base) & set(new)) + dropped = sorted(set(base) ^ set(new)) + + if not common: + with open(a.out + '.md', 'w') as f: + if new and not base: + # interim state: master has not uploaded a per-example baseline yet. + # Blaming the PR's scoping for that sends people hunting the wrong bug + f.write('_No per-example baseline from the base branch yet (the first ' + 'master push after this feature merges uploads it); comparison ' + 'will appear on the next push._\n') + else: + f.write('_Code-size comparison skipped: no (board, example) pair was ' + 'built on both the base branch and this PR._\n') + return + + def synth(pairs, path): + with open(path, 'w') as f: + json.dump({'files': [e for k in common for e in pairs[k]]}, f) + + with tempfile.TemporaryDirectory() as td: + b, n = os.path.join(td, 'base.json'), os.path.join(td, 'new.json') + synth(base, b) + synth(new, n) + comparison = metrics.compare_files(b, n, ['tinyusb/src']) + if comparison is None: + with open(a.out + '.md', 'w') as f: + f.write('_Code-size comparison failed to produce data._\n') + return + metrics.write_compare_markdown(comparison, a.out + '.md', 'name+') + + with open(a.out + '.md', 'a') as f: + boards = sorted({k[0] for k in common}) + f.write(f'\n_Scoped compare: {len(common)} (board, example) pairs across ' + f'{", ".join(boards)}._\n') + if dropped: + # GitHub caps a comment at 65,536 chars and this footer rides inside the + # sticky code-metrics comment: a broad scoped PR drops hundreds of pairs, + # and the raw list alone reached ~65KB and reddened the whole job. Only a + # summary goes in the comment; the full list goes to the job log. + names = [f'{board}:{ex}' for board, ex in dropped] + print('pair_compare: not compared (missing on one side): ' + + ', '.join(names), file=sys.stderr) + more = len(names) - DROPPED_SHOWN + f.write(f'_Not compared (missing on one side): {len(names)} pairs - ' + + ', '.join(names[:DROPPED_SHOWN]) + + (f', ... and {more} more (see the code-metrics job log)' + if more > 0 else '') + + '._\n') + + +if __name__ == '__main__': + main() |