From 04d0f71984117b8c72349f4584bd9e26a37b129c Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 21 Aug 2026 11:07:27 +0700 Subject: ci: scope the build matrix and the HIL run to what a PR affects Every PR built all 74 legs (2494 example builds on GHA cmake alone) and flashed all 30 rig boards, whatever it touched. One classifier now walks the PR diff twice and answers three questions: which families to build, which examples per family, and which boards run which tests. Fail-open throughout - anything no rule classifies, any exception, any unusable output falls back to the full matrix, and a master push always builds everything. test/hil/helper/hil_select.py moves to tools/ci_select.py: it is no longer HIL-only, and tools/ is where the build side can import it. test_hil_select.py follows it as test_ci_select.py. Rules (docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md holds the full table): a port selects the families whose family.cmake references it, and its role - a dcd change skips host examples and vice versa; a class selects only the examples whose tusb_config.h enables its CFG_TU[DH]_ macro, following cross-class includes; an example selects itself; hw/bsp selects its family or board; hw/mcu and lib select whoever references them. CMake is the reference for all of it - make follows whatever cmake decides, family.mk is never scanned. Empty means empty (maintainer ruling): a rule that classifies a path to nothing selects nothing. Ports no family references, classes no config enables, libs no example builds and hw/mcu paths that resolve nowhere are all real - nothing compiles them, so nothing can validate them, and the master-push build is the net. Structural tests pin each such case with an explicit allowlist, so the day one stops being empty it fails pre-commit instead of silently narrowing CI. Per-example builds: build.py grows a repeatable -e, resolved against the targets CMake actually registered and batched into one `cmake --build --target a b c`. build_utils mirrors CMake's family_filter (the whole FAMILY_MCUS list, ${...} and string(TOUPPER ...) resolved) for the cmake side, while the make side keeps master's algorithm verbatim - the two build systems answer differently and a shared answer breaks lpc54's make link. hil-build gains this even on a full selection: 1702 example builds become 515. Transport: the selection travels as a file, never an argv or env var - a mass-sweep diff selects 261 KB against a 128 KiB exec limit, and E2BIG would fail the step before its own fallback could run. CircleCI carries the example map inside the generated config (pipeline parameters cap at 512 chars), swapped into the parameter defaults by sentinel match, and drops the scoping wholesale if that rewrite fails. Every PR-derived value written to $GITHUB_ENV/$GITHUB_OUTPUT is character-screened. Code metrics follow the scoping: metrics.py emits per-example totals, and metrics_pair_compare compares the (board, example) pairs present on both sides instead of a scoped run against a full-matrix average. The selector's own suite gates it in both providers: a selector that exits 0 with valid-but-wrong JSON is the one failure fail-open cannot catch, so a red suite means the full matrix. --- .github/scripts/ci_set_matrix.py | 78 +++++++++++++++++-- .github/scripts/hil_ci_set_matrix.py | 47 +++++++++++- .github/scripts/metrics_pair_compare.py | 130 ++++++++++++++++++++++++++++++++ 3 files changed, 246 insertions(+), 9 deletions(-) create mode 100755 .github/scripts/metrics_pair_compare.py (limited to '.github/scripts') diff --git a/.github/scripts/ci_set_matrix.py b/.github/scripts/ci_set_matrix.py index 50ada5964..ee3609bed 100755 --- a/.github/scripts/ci_set_matrix.py +++ b/.github/scripts/ci_set_matrix.py @@ -1,5 +1,9 @@ #!/usr/bin/env python3 +import argparse import json +import os +import subprocess +import sys # toolchain, url toolchain_list = [ @@ -97,15 +101,77 @@ family_list = { } -def set_matrix_json(): +def set_matrix_json(select=None): + sel_fams = None + if select: + # every shape check is explicit: this runs AFTER main()'s fail-open handler, so + # an AttributeError on e.g. {"build": ["stm32f4"]} would red the step instead + # of falling back to the full matrix - the outcome that handler exists to prevent + b = select.get('build') if isinstance(select, dict) else None + if not isinstance(b, dict): + b = {} + if b.get('full') is False: + fams = b.get('families') + if not (isinstance(fams, list) and all(isinstance(f, str) for f in fams)): + # key ABSENT (or not a list of names) is an unusable selection, not + # "nothing selected": scoping every toolchain to [] would build zero + # families and report a vacuous green. An explicit families: [] stays a + # legitimate nothing-selected. + print('ci_set_matrix: UNSCOPED - build.full is false but the families ' + 'list is unusable, emitting the full matrix', file=sys.stderr) + else: + sel_fams = set(fams) matrix = {} for toolchain in toolchain_list: - filtered_families = [family for family, supported_toolchain in family_list.items() if - toolchain in supported_toolchain] - matrix[toolchain] = filtered_families - + fams = [family for family, tc in family_list.items() if toolchain in tc] + if sel_fams is not None: + fams = [f for f in fams if f in sel_fams] + matrix[toolchain] = fams + if sel_fams is not None: + # a family this file does not list builds on no toolchain, so the selection maps + # to an empty matrix and every leg skips - which looks exactly like a working + # scoped run. Say so: hw/bsp holds several families CI has never built + # (efm32, py32f0, ...) and espressif, whose boards are built by hil-build-esp + unbuilt = sorted(f for f in sel_fams if f not in family_list) + if unbuilt: + print(f'ci_set_matrix: selected families built by no toolchain here: ' + f'{", ".join(unbuilt)}', file=sys.stderr) print(json.dumps(matrix)) +def main(): + parser = argparse.ArgumentParser() + group = parser.add_mutually_exclusive_group() + group.add_argument('--select', help='tools/ci_select.py JSON; scopes families when build.full is false') + # a whole selection as one argv/env value can exceed the exec limits on a big + # diff, which fails the calling step BEFORE it can fall open; callers that + # already have the selection on disk pass the path instead + group.add_argument('--select-file', help='file holding the same JSON as --select') + group.add_argument('--base', help='git ref: run tools/ci_select.py --base REF and scope from it') + args = parser.parse_args() + + select = None + try: + if args.select: + select = json.loads(args.select) + elif args.select_file: + with open(args.select_file) as f: + select = json.load(f) + elif args.base: + root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + r = subprocess.run([sys.executable, os.path.join(root, 'tools', 'ci_select.py'), + '--base', args.base], + capture_output=True, text=True, cwd=root, check=True) + select = json.loads(r.stdout) + except Exception as e: # fail-open: an unusable selection must never turn into a red job + # UNSCOPED is the marker build.yml greps for: it must then drop the build extras + # (example map, family regex) too, or a full build gets labelled and filtered as + # a scoped one. Keep the token on every fall-open path. + print(f'ci_set_matrix: UNSCOPED - selection unusable ({e}), emitting the full ' + f'matrix', file=sys.stderr) + select = None + set_matrix_json(select) + + if __name__ == '__main__': - set_matrix_json() + main() diff --git a/.github/scripts/hil_ci_set_matrix.py b/.github/scripts/hil_ci_set_matrix.py index 65f50788e..396c4175a 100644 --- a/.github/scripts/hil_ci_set_matrix.py +++ b/.github/scripts/hil_ci_set_matrix.py @@ -1,6 +1,7 @@ import argparse import json import os +import sys def _resolve_config_path(config_file): @@ -19,13 +20,47 @@ def _resolve_config_path(config_file): def main(): parser = argparse.ArgumentParser() parser.add_argument('config_files', nargs='+', help='Configuration JSON file(s)') - parser.add_argument('--select', help='hil_select.py JSON; scopes boards when full=false') + g = parser.add_mutually_exclusive_group() + g.add_argument('--select', help='ci_select.py JSON; scopes boards when full=false') + # a whole selection as one argv can exceed MAX_ARG_STRLEN on a big diff, which + # would fail the step instead of falling open; callers that already have the + # selection on disk pass the path instead + g.add_argument('--select-file', help='file holding the same JSON as --select') args = parser.parse_args() + raw = args.select + sel = None + try: + if args.select_file: + with open(args.select_file) as f: + raw = f.read() + if raw: + sel = json.loads(raw) + if sel is not None and not isinstance(sel, dict): + raise ValueError(f'selection is {type(sel).__name__}, not an object') + except Exception as e: # fail-open: an unusable selection must never red the job + print(f'hil_ci_set_matrix: selection unusable ({e}) - full roster', + file=sys.stderr) + sel = None + selected = None - sel = json.loads(args.select) if args.select else None if sel and not sel.get('full'): - selected = set(sel.get('boards', {})) + # key ABSENT is an unusable selection, not "nothing selected" - same reading as + # ci_set_matrix.py. Filtering every board out would skip every hil-build leg and, + # through needs:, both rig jobs: an all-green PR with zero hardware coverage. + # An explicit boards: {} stays a legitimate nothing-selected. + if not isinstance(sel.get('boards'), dict): + print('hil_ci_set_matrix: selection has full false but no usable boards ' + 'map - full roster', file=sys.stderr) + sel = None # ALL of it is unusable, hil_examples included: keeping + # the -e lists would build a few examples per board + # while the rig, unfiltered, runs that board's whole + # test list - flash failures on the fail-open path + else: + selected = set(sel['boards']) + ex_map = (sel or {}).get('hil_examples') or {} + if not isinstance(ex_map, dict): + ex_map = {} # Toolchain buckets must match the toolchains instantiated by the hil-build # job in .github/workflows/build.yml. Keep all keys present (even if empty) @@ -71,6 +106,12 @@ def main(): if 'build' in board and 'args' in board['build']: build_board += ' ' + ' '.join(f'-D{a}' for a in board['build']['args']) + # PR selection: build only the examples this board will run (its test + # list plus device/board_test, the parking firmware) - tools/build.py -e. + # Absent key (hand runs, full non-PR builds) keeps --target all. + for ex in ex_map.get(name, []): + build_board += f' -e {ex}' + # Each variant builds into cmake-build- with its own cmake # -D defines and raw CFLAGS. No 'variant' -> a single build named after # the board. diff --git a/.github/scripts/metrics_pair_compare.py b/.github/scripts/metrics_pair_compare.py new file mode 100755 index 000000000..50107cf72 --- /dev/null +++ b/.github/scripts/metrics_pair_compare.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Board+example-matched code-size compare for PR-scoped builds. + +The averaged metrics baseline (metrics-tinyusb) spans every family and example; +a scoped PR builds a subset, so comparing against it is apples-to-oranges. This +compares the intersection of (board, example) pairs present on BOTH sides, +averaged over exactly those pairs, and names what was dropped. See +docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md #code-metrics. +""" +import argparse +import glob +import json +import os +import sys +import tempfile + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'tools')) +import metrics + +# dropped (board, example) pairs named in the PR comment before it truncates +DROPPED_SHOWN = 20 + + +def board_family(board, repo_root): + hits = glob.glob(os.path.join(repo_root, 'hw/bsp/*/boards', board)) + return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None + + +def collect(root, repo_root): + """{(board, 'role/example'): [file entries]} from every + **/cmake-build-/metrics_by_example.json under root. + + Keyed on the BOARD, not its family. The two sides are built by + `--one-first`, which returns all_boards[0] for a family with no + ci_preferred_boards entry - so a PR that adds hw/bsp//boards/a_new_board + shifts which board is built, and a family key would file the base run's sizes and + the PR run's sizes under the same name and publish the difference between two + unrelated MCUs as this PR's code-size impact. On the board key that mismatch lands + in `dropped` (reported as not compared), which is the truth.""" + pairs = {} + pat = os.path.join(root, '**', 'metrics_by_example.json') + for f in sorted(glob.glob(pat, recursive=True)): + board = os.path.basename(os.path.dirname(f)) + if not board.startswith('cmake-build-'): + print(f'pair_compare: {f} not under a cmake-build- dir, skipping', file=sys.stderr) + continue + board = board[len('cmake-build-'):] + if not board_family(board, repo_root): + # unknown board: the name is still a usable key, but say so - it means the + # artifact came from a tree whose hw/bsp does not match this checkout + print(f'pair_compare: no family for board {board}', file=sys.stderr) + # parse into a LOCAL dict and merge only once the whole file came out clean: + # a file that blows up half way through must drop WHOLE, or the entries read + # before the malformation stay in the comparison while stderr says the file + # was skipped, and a silently truncated table gets published as the verdict + try: + one = {} + for ex, ent in json.load(open(f)).items(): + one.setdefault((board, ex), []).extend(ent.get('files', [])) + except (OSError, ValueError, AttributeError, TypeError) as e: + print(f'pair_compare: unreadable {f} ({e}), skipping', file=sys.stderr) + continue + for k, v in one.items(): + pairs.setdefault(k, []).extend(v) + return pairs + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument('--base-dir', required=True) + ap.add_argument('--new-dir', required=True) + ap.add_argument('--out', default='metrics_compare') + a = ap.parse_args() + repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + base = collect(a.base_dir, repo_root) + new = collect(a.new_dir, repo_root) + common = sorted(set(base) & set(new)) + dropped = sorted(set(base) ^ set(new)) + + if not common: + with open(a.out + '.md', 'w') as f: + if new and not base: + # interim state: master has not uploaded a per-example baseline yet. + # Blaming the PR's scoping for that sends people hunting the wrong bug + f.write('_No per-example baseline from the base branch yet (the first ' + 'master push after this feature merges uploads it); comparison ' + 'will appear on the next push._\n') + else: + f.write('_Code-size comparison skipped: no (board, example) pair was ' + 'built on both the base branch and this PR._\n') + return + + def synth(pairs, path): + with open(path, 'w') as f: + json.dump({'files': [e for k in common for e in pairs[k]]}, f) + + with tempfile.TemporaryDirectory() as td: + b, n = os.path.join(td, 'base.json'), os.path.join(td, 'new.json') + synth(base, b) + synth(new, n) + comparison = metrics.compare_files(b, n, ['tinyusb/src']) + if comparison is None: + with open(a.out + '.md', 'w') as f: + f.write('_Code-size comparison failed to produce data._\n') + return + metrics.write_compare_markdown(comparison, a.out + '.md', 'name+') + + with open(a.out + '.md', 'a') as f: + boards = sorted({k[0] for k in common}) + f.write(f'\n_Scoped compare: {len(common)} (board, example) pairs across ' + f'{", ".join(boards)}._\n') + if dropped: + # GitHub caps a comment at 65,536 chars and this footer rides inside the + # sticky code-metrics comment: a broad scoped PR drops hundreds of pairs, + # and the raw list alone reached ~65KB and reddened the whole job. Only a + # summary goes in the comment; the full list goes to the job log. + names = [f'{board}:{ex}' for board, ex in dropped] + print('pair_compare: not compared (missing on one side): ' + + ', '.join(names), file=sys.stderr) + more = len(names) - DROPPED_SHOWN + f.write(f'_Not compared (missing on one side): {len(names)} pairs - ' + + ', '.join(names[:DROPPED_SHOWN]) + + (f', ... and {more} more (see the code-metrics job log)' + if more > 0 else '') + + '._\n') + + +if __name__ == '__main__': + main() -- cgit v1.3.1 From a408a8e9af4a043202f79a2b8e20d229093148e5 Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 21 Aug 2026 14:23:40 +0700 Subject: hil: express a board's always-on defines as a variant, dropping build.args The roster had two ways to pass a cmake -D to a board's build: `build.args`, applied to every variant, and `variant[].defines`, applied to one. They did the same thing, and only metro_m4_express used the first - for MAX3421_HOST=1, which is what makes it the one rig board that compiles hcd_max3421.c. A board whose define is always on now carries a single variant named after itself, which is exactly the shape `board.get('variant') or [{'name': name, 'flags': ''}]` already synthesises everywhere - so the build dir, the HIL report row and the variant-boundary handling are unchanged. raspberry_pi_pico has used that shape for its flags all along. Removes the BuildCfg type and the parallel code path from all four consumers: hil_test.build_board, hil_pool_check's two builders, hil_ci_set_matrix and ci_select.board_options. Verified: the hil-build matrix entry is byte-identical (`-b metro_m4_express -DMAX3421_HOST=1`), hil_test's build command is unchanged, ci_select still selects the board for a max3421 diff with MAX3421_HOST in its options, and a real build of dual/host_info_to_device_cdc and host/cdc_msc_hid on that board still compiles hcd_max3421.c. --- .github/scripts/hil_ci_set_matrix.py | 2 -- test/hil/helper/hil_pool_check.py | 4 +--- test/hil/hil_test.py | 14 ++++---------- test/hil/test/test_ci_select.py | 10 +++++----- test/hil/tinyusb.json | 13 ++++++++----- tools/ci_select.py | 10 ++++++---- 6 files changed, 24 insertions(+), 29 deletions(-) (limited to '.github/scripts') diff --git a/.github/scripts/hil_ci_set_matrix.py b/.github/scripts/hil_ci_set_matrix.py index 396c4175a..bf50061dd 100644 --- a/.github/scripts/hil_ci_set_matrix.py +++ b/.github/scripts/hil_ci_set_matrix.py @@ -103,8 +103,6 @@ def main(): f'hil-build-esp jobs in .github/workflows/build.yml') build_board = f'-b {name}' - if 'build' in board and 'args' in board['build']: - build_board += ' ' + ' '.join(f'-D{a}' for a in board['build']['args']) # PR selection: build only the examples this board will run (its test # list plus device/board_test, the parking firmware) - tools/build.py -e. diff --git a/test/hil/helper/hil_pool_check.py b/test/hil/helper/hil_pool_check.py index d926bbe3d..179a417ed 100644 --- a/test/hil/helper/hil_pool_check.py +++ b/test/hil/helper/hil_pool_check.py @@ -433,7 +433,7 @@ def build_example(board: dict, variant: str, example: str) -> int: cmd = ['idf.py', '-C', f'examples/{example}', '-B', f'cmake-build/cmake-build-{vcfg["name"]}/{example}', '-G', 'Ninja', f'-DBOARD={name}', 'build'] - for d in board.get('build', {}).get('args', []) + vcfg.get('defines', []): + for d in vcfg.get('defines', []): cmd.insert(-1, f'-D{d}') if vcfg.get('flags'): cmd.insert(-1, f'-DCFLAGS_CLI={vcfg["flags"]}') @@ -446,8 +446,6 @@ def build_example(board: dict, variant: str, example: str) -> int: cmd = [sys.executable, str(hil_util.TINYUSB_ROOT / 'tools' / 'build.py'), '-b', name, '-T', Path(example).name, '-j', str(max(1, (os.cpu_count() or _jobs) // _jobs))] - for d in board.get('build', {}).get('args', []): - cmd += ['-D', d] if vcfg['name'] != name: cmd += ['--build-name', vcfg['name']] for d in vcfg.get('defines', []): diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py index 174251343..fcd7c7e6f 100755 --- a/test/hil/hil_test.py +++ b/test/hil/hil_test.py @@ -194,10 +194,6 @@ class TestsCfg(TypedDict, total=False): dev_attached: list[AttachedDevCfg] -class BuildCfg(TypedDict, total=False): - args: list[str] - - class VariantCfg(TypedDict, total=False): name: str # build dir (cmake-build-) and HIL report row flags: str # raw CFLAGS, e.g. "-DCFG_TUD_DWC2_DMA_ENABLE=1" @@ -209,7 +205,9 @@ class Board(TypedDict): uid: str tests: TestsCfg flasher: FlasherCfg - build: NotRequired[BuildCfg] + # every build knob lives here, including a board's always-on defines: a board that + # needs one carries a single variant named after itself (metro_m4_express / + # MAX3421_HOST=1), which is exactly what the `or [...]` default below synthesises variant: NotRequired[list[VariantCfg]] toolchain: NotRequired[str] # CI build bucket override, e.g. "riscv-gcc" (consumed by hil_ci_set_matrix.py) @@ -1670,21 +1668,17 @@ def test_example(board: Board, variant: str, example: str) -> tuple[int, str, st def build_board(board: Board) -> tuple[str, int]: """Build firmware for this board via tools/build.py. - Honors board config's variant list and build.args defines. + Honors board config's variant list (name, defines, flags). Output goes to cmake-build/cmake-build-/ (tools/build.py layout). Unbounded on purpose: --build is a local convenience (no CI workflow passes it), so the developer watching the build is the timeout.""" name = board['name'] - bcfg = cast(BuildCfg, board.get('build', {})) - extra_defs = bcfg.get('args', []) variants = board.get('variant') or [{'name': name, 'flags': ''}] failed = 0 for v in variants: cmd = [sys.executable, str(hil_util.TINYUSB_ROOT / 'tools' / 'build.py'), '-b', name] - for d in extra_defs: - cmd += ['-D', d] if v['name'] != name: cmd += ['--build-name', v['name']] for d in v.get('defines', []): diff --git a/test/hil/test/test_ci_select.py b/test/hil/test/test_ci_select.py index 031e8e287..f5c64ac1c 100644 --- a/test/hil/test/test_ci_select.py +++ b/test/hil/test/test_ci_select.py @@ -329,7 +329,7 @@ class TestOptionGatedPort(unittest.TestCase): # host-side option board (max3421 as host controller), off any max3421 family OPT_ROSTER = [('test/hil/opt.json', [ {'name': 'fake_dual_board', 'uid': 'o1', 'flasher': {'name': 'jlink'}, - 'build': {'args': ['MAX3421_HOST=1']}, + 'variant': [{'name': 'fake_dual_board', 'defines': ['MAX3421_HOST=1']}], 'tests': {'device': True, 'host': False, 'dual': True}}, {'name': 'fake_host_board', 'uid': 'o2', 'flasher': {'name': 'jlink'}, 'variant': [{'name': 'fake_host_board', 'flags': '-DMAX3421_HOST=1'}], @@ -346,10 +346,10 @@ class TestOptionGatedPort(unittest.TestCase): for board in boards: self.assertIn(board, s['boards']) - def test_option_selects_via_args_defines_and_flags(self): + def test_option_selects_via_defines_and_flags(self): s = ci_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, self.OPT_ROSTER) self.assertFalse(s['full']) - self.assertIn('fake_dual_board', s['boards']) # build.args + self.assertIn('fake_dual_board', s['boards']) # variant defines self.assertIn('fake_host_board', s['boards']) # variant flags self.assertNotIn('fake_off_board', s['boards']) # variant defines, but =0 @@ -1762,7 +1762,7 @@ class TestBuildPyExampleFilter(unittest.TestCase): {'tinyusb_metrics', 'cdc_msc', 'cdc_msc-membrowse-upload'}) def test_build_defines_reach_the_example_filter(self): - # metro_m4_express gets MAX3421_HOST=1 from the roster build args, never + # metro_m4_express gets MAX3421_HOST=1 from its roster variant, never # from its BSP: without threading them through, -e drops the rig's only # MAX3421 dual firmware that --target all used to build self.assertIsNone(self.build.resolve_example_target_groups( @@ -1934,7 +1934,7 @@ class TestSkipExampleMirrorsFamilyFilter(unittest.TestCase): def test_build_define_enables_max3421_only_list(self): # family_support.cmake:940 appends MAX3421 to FAMILY_MCUS when # MAX3421_HOST=1; on metro_m4_express that define comes from the roster - # build args, so skip_example has to be told about it + # variant defines, so skip_example has to be told about it ex = 'dual/host_info_to_device_cdc' self.assertTrue(self.build_utils.skip_example(ex, 'metro_m4_express')) self.assertFalse(self.build_utils.skip_example(ex, 'metro_m4_express', diff --git a/test/hil/tinyusb.json b/test/hil/tinyusb.json index 6f552f126..8fd4683a4 100644 --- a/test/hil/tinyusb.json +++ b/test/hil/tinyusb.json @@ -157,11 +157,14 @@ { "name": "metro_m4_express", "uid": "9995AD485337433231202020FF100A34", - "build": { - "args": [ - "MAX3421_HOST=1" - ] - }, + "variant": [ + { + "name": "metro_m4_express", + "defines": [ + "MAX3421_HOST=1" + ] + } + ], "tests": { "device": true, "host": false, diff --git a/tools/ci_select.py b/tools/ci_select.py index cd63899c1..ced3bbbc0 100755 --- a/tools/ci_select.py +++ b/tools/ci_select.py @@ -188,10 +188,12 @@ def bsp_board_options(board_name: str, repo_root: str) -> frozenset: def board_options(board: dict, repo_root: str) -> set: - """Build options a board has truthy: the roster entry's build.args plus each - variant's defines (NAME=VALUE) and raw CFLAGS (-DNAME=VALUE), plus whatever its - own board.cmake sets (a board can enable a gated port without the roster saying so).""" - toks = list(board.get('build', {}).get('args', [])) + """Build options a board has truthy: each variant's defines (NAME=VALUE) and raw + CFLAGS (-DNAME=VALUE), plus whatever its own board.cmake sets (a board can enable a + gated port without the roster saying so). A board whose option is always on carries + a single variant named after itself - metro_m4_express and MAX3421_HOST=1, which is + what makes it the one rig board that compiles hcd_max3421.c.""" + toks = [] for v in board.get('variant', []): toks += list(v.get('defines', [])) toks += v.get('flags', '').split() -- cgit v1.3.1 From f17be6770b601a32bdfcc1459e8851becf7fea84 Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 21 Aug 2026 16:08:25 +0700 Subject: ci_set_matrix: fall open when no selected family builds anywhere family_list maps a family to the toolchains that build it, and seven hw/bsp families are in neither: cxd56, efm32, espressif, f1c100s, pic32mz, py32f0, same7x. Scoping to one of them intersected to nothing, so every toolchain key was [], every cmake leg skipped on `if: inputs.build-args != '[]'`, code-metrics took its no-metrics branch, and the PR went green from a build job that ran no compiler. The only signal was a stderr line nothing greps for. Not a coverage regression - master gave the same diff no compile coverage either, since none of the other families compiles same7x's board.h. What is new is that the gap used to be masked by the full matrix and is now the whole answer, and that green now means "ran no compiler" rather than "compiled 64 families". A selection whose families ALL miss is now unusable rather than empty: it prints UNSCOPED, which build.yml and .circleci/config.yml already grep to drop the build extras with it, and emits the full matrix. The two neighbouring cases keep their own answers - an explicit families: [] is still a legitimate nothing-selected, and a partial miss still scopes to the families that do build, noting the rest. The contract test pinned an exact count of fall-open markers, which this would have broken; it now pins the invariant (every message that emits the full matrix carries the marker) and was checked to still fail when a marker is removed. Also corrects the drift guard's note about espressif: hil-build-esp builds its boards by name, but that job is gated on repository_owner, so on a fork an espressif-only PR builds nowhere. --- .github/scripts/ci_set_matrix.py | 20 +++++++++++++++----- test/hil/test/test_ci_metrics.py | 13 +++++++++++-- test/hil/test/test_ci_select.py | 36 ++++++++++++++++++++++++++++++------ 3 files changed, 56 insertions(+), 13 deletions(-) (limited to '.github/scripts') diff --git a/.github/scripts/ci_set_matrix.py b/.github/scripts/ci_set_matrix.py index ee3609bed..79f466893 100755 --- a/.github/scripts/ci_set_matrix.py +++ b/.github/scripts/ci_set_matrix.py @@ -127,12 +127,22 @@ def set_matrix_json(select=None): if sel_fams is not None: fams = [f for f in fams if f in sel_fams] matrix[toolchain] = fams - if sel_fams is not None: - # a family this file does not list builds on no toolchain, so the selection maps - # to an empty matrix and every leg skips - which looks exactly like a working - # scoped run. Say so: hw/bsp holds several families CI has never built - # (efm32, py32f0, ...) and espressif, whose boards are built by hil-build-esp + if sel_fams: + # a family this file does not list builds on no toolchain, so it contributes no + # leg. hw/bsp holds several CI has never built (efm32, py32f0, same7x, ...) plus + # espressif, whose boards hil-build-esp builds by name. unbuilt = sorted(f for f in sel_fams if f not in family_list) + if unbuilt and not any(matrix.values()): + # NONE of the selected families is buildable here, so every leg would skip + # and the PR would go green from a build job that ran no compiler. That is + # an unusable selection, not "nothing selected": say UNSCOPED - which + # build.yml and .circleci/config.yml both grep for - and emit the full + # matrix. An explicit families: [] is still a legitimate nothing-selected, + # and a PARTIAL miss still scopes to the families that do build. + print(f'ci_set_matrix: UNSCOPED - no selected family is built by any ' + f'toolchain here ({", ".join(unbuilt)}), emitting the full matrix', + file=sys.stderr) + return set_matrix_json(None) if unbuilt: print(f'ci_set_matrix: selected families built by no toolchain here: ' f'{", ".join(unbuilt)}', file=sys.stderr) diff --git a/test/hil/test/test_ci_metrics.py b/test/hil/test/test_ci_metrics.py index 6c236e827..a76b6e3a0 100644 --- a/test/hil/test/test_ci_metrics.py +++ b/test/hil/test/test_ci_metrics.py @@ -435,8 +435,17 @@ class TestWorkflowSelectionHandOff(unittest.TestCase): self.assertIn('BUILD_SELECT_FILE', self.build) scripts = os.path.join(os.path.dirname(CIRCLECI), '.github', 'scripts') matrix = open(os.path.join(scripts, 'ci_set_matrix.py')).read() - self.assertEqual(matrix.count('ci_set_matrix: UNSCOPED'), 2, - 'every fall-open path must print the marker build.yml greps for') + # count-independent: pin the INVARIANT, not the number of fall-open paths - + # every message that emits the full matrix must carry the marker, and a purely + # informational note (a partial family miss) must not claim to have done so. + # Adjacent string literals are joined first, since these messages wrap. + import re as _re + flat = _re.sub(r"['\"]\s*\n\s*f?['\"]", '', matrix) + hits = [m.start() for m in _re.finditer('emitting the full ', flat)] + self.assertGreaterEqual(len(hits), 2, 'fall-open messages not found') + for i in hits: + self.assertIn('UNSCOPED', flat[max(0, i - 200):i], + 'a fall-open path without the marker build.yml greps for') def test_membrowse_upload_sees_the_same_board_as_the_build(self): # $EX_ARGS is passed for the BOARD it selects: --one-first picks a board that can diff --git a/test/hil/test/test_ci_select.py b/test/hil/test/test_ci_select.py index f5c64ac1c..8f1841531 100644 --- a/test/hil/test/test_ci_select.py +++ b/test/hil/test/test_ci_select.py @@ -777,12 +777,15 @@ class TestOrphanInvariant(unittest.TestCase): for v in vendors: self.assertTrue(ci_select.mcu_families(v + '/x.c', REPO), f'{v}: resolves to no family') - # hw/bsp families ci_set_matrix's family_list does not map to any toolchain. Before - # scoping these were harmless - the matrix was always every family in family_list, - # so a PR touching one of them still compiled the other 64. Now the selection - # intersects to nothing and every leg skips, so a family landing here by accident is - # a silent hole. espressif is deliberate: its boards are built by hil-build-esp, - # keyed on board name rather than family. + # hw/bsp families ci_set_matrix's family_list does not map to any toolchain. Master + # gave a PR touching one of these no compile coverage either - none of the other 64 + # families compiles same7x's board.h - so this is not new. What IS new is that the + # gap used to be masked by a full matrix and is now the whole answer, which is why + # ci_set_matrix treats a selection that intersects family_list to NOTHING as + # unusable (UNSCOPED -> full matrix) rather than emitting an all-empty one. + # espressif is here because hil-build-esp builds its boards by name rather than by + # family - though only on hathach/tinyusb: that job is gated on repository_owner, + # so on a fork an espressif-only PR builds nowhere. UNBUILT_FAMILIES = {'cxd56', 'efm32', 'espressif', 'f1c100s', 'pic32mz', 'py32f0', 'same7x'} @@ -1505,6 +1508,27 @@ class TestCiSetMatrix(unittest.TestCase): self.assertEqual(json.loads(r.stdout), base) self.assertIn('ci_set_matrix: UNSCOPED', r.stderr) # build.yml greps this + def test_families_no_toolchain_builds_falls_open(self): + # hw/bsp/same7x is real but in no toolchain's list, so scoping to it emits an + # all-empty matrix: every leg skips and the PR goes green from a build job that + # ran no compiler. Unusable, not "nothing selected" - and the marker matters, + # because that is what build.yml and CircleCI grep to drop the build extras too. + base = json.loads(self.run_matrix().stdout) + r = self.run_matrix('--select', + json.dumps({'build': {'full': False, 'families': ['same7x']}})) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertEqual(json.loads(r.stdout), base) + self.assertIn('ci_set_matrix: UNSCOPED', r.stderr) + + def test_a_partial_toolchain_miss_still_scopes(self): + # one buildable family is real coverage: scope to it and just note the other + r = self.run_matrix('--select', json.dumps( + {'build': {'full': False, 'families': ['stm32f4', 'same7x']}})) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertEqual(json.loads(r.stdout)['arm-gcc'], ['stm32f4']) + self.assertNotIn('UNSCOPED', r.stderr) + self.assertIn('same7x', r.stderr) + def test_explicit_empty_families_selects_nothing(self): # an explicit [] IS a legitimate answer (a diff that builds nothing) r = self.run_matrix('--select', -- cgit v1.3.1