1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
|
#!/usr/bin/env python3
"""Board+example-matched code-size compare for PR-scoped builds.
The averaged metrics baseline (metrics-tinyusb) spans every family and example;
a scoped PR builds a subset, so comparing against it is apples-to-oranges. This
compares the intersection of (board, example) pairs present on BOTH sides,
averaged over exactly those pairs, and names what was dropped. See
docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md #code-metrics.
"""
import argparse
import glob
import json
import os
import sys
import tempfile
sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'tools'))
import metrics
# dropped (board, example) pairs named in the PR comment before it truncates
DROPPED_SHOWN = 20
def board_family(board, repo_root):
hits = glob.glob(os.path.join(repo_root, 'hw/bsp/*/boards', board))
return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None
def collect(root, repo_root):
"""{(board, 'role/example'): [file entries]} from every
**/cmake-build-<board>/metrics_by_example.json under root.
Keyed on the BOARD, not its family. The two sides are built by
`--one-first`, which returns all_boards[0] for a family with no
ci_preferred_boards entry - so a PR that adds hw/bsp/<family>/boards/a_new_board
shifts which board is built, and a family key would file the base run's sizes and
the PR run's sizes under the same name and publish the difference between two
unrelated MCUs as this PR's code-size impact. On the board key that mismatch lands
in `dropped` (reported as not compared), which is the truth."""
pairs = {}
pat = os.path.join(root, '**', 'metrics_by_example.json')
for f in sorted(glob.glob(pat, recursive=True)):
board = os.path.basename(os.path.dirname(f))
if not board.startswith('cmake-build-'):
print(f'pair_compare: {f} not under a cmake-build-<board> dir, skipping', file=sys.stderr)
continue
board = board[len('cmake-build-'):]
if not board_family(board, repo_root):
# unknown board: the name is still a usable key, but say so - it means the
# artifact came from a tree whose hw/bsp does not match this checkout
print(f'pair_compare: no family for board {board}', file=sys.stderr)
# parse into a LOCAL dict and merge only once the whole file came out clean:
# a file that blows up half way through must drop WHOLE, or the entries read
# before the malformation stay in the comparison while stderr says the file
# was skipped, and a silently truncated table gets published as the verdict
try:
one = {}
for ex, ent in json.load(open(f)).items():
one.setdefault((board, ex), []).extend(ent.get('files', []))
except (OSError, ValueError, AttributeError, TypeError) as e:
print(f'pair_compare: unreadable {f} ({e}), skipping', file=sys.stderr)
continue
for k, v in one.items():
pairs.setdefault(k, []).extend(v)
return pairs
def main():
ap = argparse.ArgumentParser(description=__doc__)
ap.add_argument('--base-dir', required=True)
ap.add_argument('--new-dir', required=True)
ap.add_argument('--out', default='metrics_compare')
a = ap.parse_args()
repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
base = collect(a.base_dir, repo_root)
new = collect(a.new_dir, repo_root)
common = sorted(set(base) & set(new))
dropped = sorted(set(base) ^ set(new))
if not common:
with open(a.out + '.md', 'w') as f:
if new and not base:
# interim state: master has not uploaded a per-example baseline yet.
# Blaming the PR's scoping for that sends people hunting the wrong bug
f.write('_No per-example baseline from the base branch yet (the first '
'master push after this feature merges uploads it); comparison '
'will appear on the next push._\n')
else:
f.write('_Code-size comparison skipped: no (board, example) pair was '
'built on both the base branch and this PR._\n')
return
def synth(pairs, path):
with open(path, 'w') as f:
json.dump({'files': [e for k in common for e in pairs[k]]}, f)
with tempfile.TemporaryDirectory() as td:
b, n = os.path.join(td, 'base.json'), os.path.join(td, 'new.json')
synth(base, b)
synth(new, n)
comparison = metrics.compare_files(b, n, ['tinyusb/src'])
if comparison is None:
with open(a.out + '.md', 'w') as f:
f.write('_Code-size comparison failed to produce data._\n')
return
metrics.write_compare_markdown(comparison, a.out + '.md', 'name+')
with open(a.out + '.md', 'a') as f:
boards = sorted({k[0] for k in common})
f.write(f'\n_Scoped compare: {len(common)} (board, example) pairs across '
f'{", ".join(boards)}._\n')
if dropped:
# GitHub caps a comment at 65,536 chars and this footer rides inside the
# sticky code-metrics comment: a broad scoped PR drops hundreds of pairs,
# and the raw list alone reached ~65KB and reddened the whole job. Only a
# summary goes in the comment; the full list goes to the job log.
names = [f'{board}:{ex}' for board, ex in dropped]
print('pair_compare: not compared (missing on one side): '
+ ', '.join(names), file=sys.stderr)
more = len(names) - DROPPED_SHOWN
f.write(f'_Not compared (missing on one side): {len(names)} pairs - '
+ ', '.join(names[:DROPPED_SHOWN])
+ (f', ... and {more} more (see the code-metrics job log)'
if more > 0 else '')
+ '._\n')
if __name__ == '__main__':
main()
|
