summaryrefslogtreecommitdiff
path: root/.github/scripts
diff options
context:
space:
mode:
Diffstat (limited to '.github/scripts')
-rwxr-xr-x.github/scripts/ci_set_matrix.py88
-rw-r--r--.github/scripts/hil_ci_set_matrix.py49
-rwxr-xr-x.github/scripts/metrics_pair_compare.py130
3 files changed, 256 insertions, 11 deletions
diff --git a/.github/scripts/ci_set_matrix.py b/.github/scripts/ci_set_matrix.py
index 50ada5964..79f466893 100755
--- a/.github/scripts/ci_set_matrix.py
+++ b/.github/scripts/ci_set_matrix.py
@@ -1,5 +1,9 @@
#!/usr/bin/env python3
+import argparse
import json
+import os
+import subprocess
+import sys
# toolchain, url
toolchain_list = [
@@ -97,15 +101,87 @@ family_list = {
}
-def set_matrix_json():
+def set_matrix_json(select=None):
+ sel_fams = None
+ if select:
+ # every shape check is explicit: this runs AFTER main()'s fail-open handler, so
+ # an AttributeError on e.g. {"build": ["stm32f4"]} would red the step instead
+ # of falling back to the full matrix - the outcome that handler exists to prevent
+ b = select.get('build') if isinstance(select, dict) else None
+ if not isinstance(b, dict):
+ b = {}
+ if b.get('full') is False:
+ fams = b.get('families')
+ if not (isinstance(fams, list) and all(isinstance(f, str) for f in fams)):
+ # key ABSENT (or not a list of names) is an unusable selection, not
+ # "nothing selected": scoping every toolchain to [] would build zero
+ # families and report a vacuous green. An explicit families: [] stays a
+ # legitimate nothing-selected.
+ print('ci_set_matrix: UNSCOPED - build.full is false but the families '
+ 'list is unusable, emitting the full matrix', file=sys.stderr)
+ else:
+ sel_fams = set(fams)
matrix = {}
for toolchain in toolchain_list:
- filtered_families = [family for family, supported_toolchain in family_list.items() if
- toolchain in supported_toolchain]
- matrix[toolchain] = filtered_families
-
+ fams = [family for family, tc in family_list.items() if toolchain in tc]
+ if sel_fams is not None:
+ fams = [f for f in fams if f in sel_fams]
+ matrix[toolchain] = fams
+ if sel_fams:
+ # a family this file does not list builds on no toolchain, so it contributes no
+ # leg. hw/bsp holds several CI has never built (efm32, py32f0, same7x, ...) plus
+ # espressif, whose boards hil-build-esp builds by name.
+ unbuilt = sorted(f for f in sel_fams if f not in family_list)
+ if unbuilt and not any(matrix.values()):
+ # NONE of the selected families is buildable here, so every leg would skip
+ # and the PR would go green from a build job that ran no compiler. That is
+ # an unusable selection, not "nothing selected": say UNSCOPED - which
+ # build.yml and .circleci/config.yml both grep for - and emit the full
+ # matrix. An explicit families: [] is still a legitimate nothing-selected,
+ # and a PARTIAL miss still scopes to the families that do build.
+ print(f'ci_set_matrix: UNSCOPED - no selected family is built by any '
+ f'toolchain here ({", ".join(unbuilt)}), emitting the full matrix',
+ file=sys.stderr)
+ return set_matrix_json(None)
+ if unbuilt:
+ print(f'ci_set_matrix: selected families built by no toolchain here: '
+ f'{", ".join(unbuilt)}', file=sys.stderr)
print(json.dumps(matrix))
+def main():
+ parser = argparse.ArgumentParser()
+ group = parser.add_mutually_exclusive_group()
+ group.add_argument('--select', help='tools/ci_select.py JSON; scopes families when build.full is false')
+ # a whole selection as one argv/env value can exceed the exec limits on a big
+ # diff, which fails the calling step BEFORE it can fall open; callers that
+ # already have the selection on disk pass the path instead
+ group.add_argument('--select-file', help='file holding the same JSON as --select')
+ group.add_argument('--base', help='git ref: run tools/ci_select.py --base REF and scope from it')
+ args = parser.parse_args()
+
+ select = None
+ try:
+ if args.select:
+ select = json.loads(args.select)
+ elif args.select_file:
+ with open(args.select_file) as f:
+ select = json.load(f)
+ elif args.base:
+ root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+ r = subprocess.run([sys.executable, os.path.join(root, 'tools', 'ci_select.py'),
+ '--base', args.base],
+ capture_output=True, text=True, cwd=root, check=True)
+ select = json.loads(r.stdout)
+ except Exception as e: # fail-open: an unusable selection must never turn into a red job
+ # UNSCOPED is the marker build.yml greps for: it must then drop the build extras
+ # (example map, family regex) too, or a full build gets labelled and filtered as
+ # a scoped one. Keep the token on every fall-open path.
+ print(f'ci_set_matrix: UNSCOPED - selection unusable ({e}), emitting the full '
+ f'matrix', file=sys.stderr)
+ select = None
+ set_matrix_json(select)
+
+
if __name__ == '__main__':
- set_matrix_json()
+ main()
diff --git a/.github/scripts/hil_ci_set_matrix.py b/.github/scripts/hil_ci_set_matrix.py
index 65f50788e..bf50061dd 100644
--- a/.github/scripts/hil_ci_set_matrix.py
+++ b/.github/scripts/hil_ci_set_matrix.py
@@ -1,6 +1,7 @@
import argparse
import json
import os
+import sys
def _resolve_config_path(config_file):
@@ -19,13 +20,47 @@ def _resolve_config_path(config_file):
def main():
parser = argparse.ArgumentParser()
parser.add_argument('config_files', nargs='+', help='Configuration JSON file(s)')
- parser.add_argument('--select', help='hil_select.py JSON; scopes boards when full=false')
+ g = parser.add_mutually_exclusive_group()
+ g.add_argument('--select', help='ci_select.py JSON; scopes boards when full=false')
+ # a whole selection as one argv can exceed MAX_ARG_STRLEN on a big diff, which
+ # would fail the step instead of falling open; callers that already have the
+ # selection on disk pass the path instead
+ g.add_argument('--select-file', help='file holding the same JSON as --select')
args = parser.parse_args()
+ raw = args.select
+ sel = None
+ try:
+ if args.select_file:
+ with open(args.select_file) as f:
+ raw = f.read()
+ if raw:
+ sel = json.loads(raw)
+ if sel is not None and not isinstance(sel, dict):
+ raise ValueError(f'selection is {type(sel).__name__}, not an object')
+ except Exception as e: # fail-open: an unusable selection must never red the job
+ print(f'hil_ci_set_matrix: selection unusable ({e}) - full roster',
+ file=sys.stderr)
+ sel = None
+
selected = None
- sel = json.loads(args.select) if args.select else None
if sel and not sel.get('full'):
- selected = set(sel.get('boards', {}))
+ # key ABSENT is an unusable selection, not "nothing selected" - same reading as
+ # ci_set_matrix.py. Filtering every board out would skip every hil-build leg and,
+ # through needs:, both rig jobs: an all-green PR with zero hardware coverage.
+ # An explicit boards: {} stays a legitimate nothing-selected.
+ if not isinstance(sel.get('boards'), dict):
+ print('hil_ci_set_matrix: selection has full false but no usable boards '
+ 'map - full roster', file=sys.stderr)
+ sel = None # ALL of it is unusable, hil_examples included: keeping
+ # the -e lists would build a few examples per board
+ # while the rig, unfiltered, runs that board's whole
+ # test list - flash failures on the fail-open path
+ else:
+ selected = set(sel['boards'])
+ ex_map = (sel or {}).get('hil_examples') or {}
+ if not isinstance(ex_map, dict):
+ ex_map = {}
# Toolchain buckets must match the toolchains instantiated by the hil-build
# job in .github/workflows/build.yml. Keep all keys present (even if empty)
@@ -68,8 +103,12 @@ def main():
f'hil-build-esp jobs in .github/workflows/build.yml')
build_board = f'-b {name}'
- if 'build' in board and 'args' in board['build']:
- build_board += ' ' + ' '.join(f'-D{a}' for a in board['build']['args'])
+
+ # PR selection: build only the examples this board will run (its test
+ # list plus device/board_test, the parking firmware) - tools/build.py -e.
+ # Absent key (hand runs, full non-PR builds) keeps --target all.
+ for ex in ex_map.get(name, []):
+ build_board += f' -e {ex}'
# Each variant builds into cmake-build-<variant.name> with its own cmake
# -D defines and raw CFLAGS. No 'variant' -> a single build named after
diff --git a/.github/scripts/metrics_pair_compare.py b/.github/scripts/metrics_pair_compare.py
new file mode 100755
index 000000000..50107cf72
--- /dev/null
+++ b/.github/scripts/metrics_pair_compare.py
@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""Board+example-matched code-size compare for PR-scoped builds.
+
+The averaged metrics baseline (metrics-tinyusb) spans every family and example;
+a scoped PR builds a subset, so comparing against it is apples-to-oranges. This
+compares the intersection of (board, example) pairs present on BOTH sides,
+averaged over exactly those pairs, and names what was dropped. See
+docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md #code-metrics.
+"""
+import argparse
+import glob
+import json
+import os
+import sys
+import tempfile
+
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'tools'))
+import metrics
+
+# dropped (board, example) pairs named in the PR comment before it truncates
+DROPPED_SHOWN = 20
+
+
+def board_family(board, repo_root):
+ hits = glob.glob(os.path.join(repo_root, 'hw/bsp/*/boards', board))
+ return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None
+
+
+def collect(root, repo_root):
+ """{(board, 'role/example'): [file entries]} from every
+ **/cmake-build-<board>/metrics_by_example.json under root.
+
+ Keyed on the BOARD, not its family. The two sides are built by
+ `--one-first`, which returns all_boards[0] for a family with no
+ ci_preferred_boards entry - so a PR that adds hw/bsp/<family>/boards/a_new_board
+ shifts which board is built, and a family key would file the base run's sizes and
+ the PR run's sizes under the same name and publish the difference between two
+ unrelated MCUs as this PR's code-size impact. On the board key that mismatch lands
+ in `dropped` (reported as not compared), which is the truth."""
+ pairs = {}
+ pat = os.path.join(root, '**', 'metrics_by_example.json')
+ for f in sorted(glob.glob(pat, recursive=True)):
+ board = os.path.basename(os.path.dirname(f))
+ if not board.startswith('cmake-build-'):
+ print(f'pair_compare: {f} not under a cmake-build-<board> dir, skipping', file=sys.stderr)
+ continue
+ board = board[len('cmake-build-'):]
+ if not board_family(board, repo_root):
+ # unknown board: the name is still a usable key, but say so - it means the
+ # artifact came from a tree whose hw/bsp does not match this checkout
+ print(f'pair_compare: no family for board {board}', file=sys.stderr)
+ # parse into a LOCAL dict and merge only once the whole file came out clean:
+ # a file that blows up half way through must drop WHOLE, or the entries read
+ # before the malformation stay in the comparison while stderr says the file
+ # was skipped, and a silently truncated table gets published as the verdict
+ try:
+ one = {}
+ for ex, ent in json.load(open(f)).items():
+ one.setdefault((board, ex), []).extend(ent.get('files', []))
+ except (OSError, ValueError, AttributeError, TypeError) as e:
+ print(f'pair_compare: unreadable {f} ({e}), skipping', file=sys.stderr)
+ continue
+ for k, v in one.items():
+ pairs.setdefault(k, []).extend(v)
+ return pairs
+
+
+def main():
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument('--base-dir', required=True)
+ ap.add_argument('--new-dir', required=True)
+ ap.add_argument('--out', default='metrics_compare')
+ a = ap.parse_args()
+ repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+ base = collect(a.base_dir, repo_root)
+ new = collect(a.new_dir, repo_root)
+ common = sorted(set(base) & set(new))
+ dropped = sorted(set(base) ^ set(new))
+
+ if not common:
+ with open(a.out + '.md', 'w') as f:
+ if new and not base:
+ # interim state: master has not uploaded a per-example baseline yet.
+ # Blaming the PR's scoping for that sends people hunting the wrong bug
+ f.write('_No per-example baseline from the base branch yet (the first '
+ 'master push after this feature merges uploads it); comparison '
+ 'will appear on the next push._\n')
+ else:
+ f.write('_Code-size comparison skipped: no (board, example) pair was '
+ 'built on both the base branch and this PR._\n')
+ return
+
+ def synth(pairs, path):
+ with open(path, 'w') as f:
+ json.dump({'files': [e for k in common for e in pairs[k]]}, f)
+
+ with tempfile.TemporaryDirectory() as td:
+ b, n = os.path.join(td, 'base.json'), os.path.join(td, 'new.json')
+ synth(base, b)
+ synth(new, n)
+ comparison = metrics.compare_files(b, n, ['tinyusb/src'])
+ if comparison is None:
+ with open(a.out + '.md', 'w') as f:
+ f.write('_Code-size comparison failed to produce data._\n')
+ return
+ metrics.write_compare_markdown(comparison, a.out + '.md', 'name+')
+
+ with open(a.out + '.md', 'a') as f:
+ boards = sorted({k[0] for k in common})
+ f.write(f'\n_Scoped compare: {len(common)} (board, example) pairs across '
+ f'{", ".join(boards)}._\n')
+ if dropped:
+ # GitHub caps a comment at 65,536 chars and this footer rides inside the
+ # sticky code-metrics comment: a broad scoped PR drops hundreds of pairs,
+ # and the raw list alone reached ~65KB and reddened the whole job. Only a
+ # summary goes in the comment; the full list goes to the job log.
+ names = [f'{board}:{ex}' for board, ex in dropped]
+ print('pair_compare: not compared (missing on one side): '
+ + ', '.join(names), file=sys.stderr)
+ more = len(names) - DROPPED_SHOWN
+ f.write(f'_Not compared (missing on one side): {len(names)} pairs - '
+ + ', '.join(names[:DROPPED_SHOWN])
+ + (f', ... and {more} more (see the code-metrics job log)'
+ if more > 0 else '')
+ + '._\n')
+
+
+if __name__ == '__main__':
+ main()
generated by cgit v1.3.1 (git 2.54.0) at 2026-08-22 20:59:31 +0000