From 04d0f71984117b8c72349f4584bd9e26a37b129c Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 21 Aug 2026 11:07:27 +0700 Subject: ci: scope the build matrix and the HIL run to what a PR affects Every PR built all 74 legs (2494 example builds on GHA cmake alone) and flashed all 30 rig boards, whatever it touched. One classifier now walks the PR diff twice and answers three questions: which families to build, which examples per family, and which boards run which tests. Fail-open throughout - anything no rule classifies, any exception, any unusable output falls back to the full matrix, and a master push always builds everything. test/hil/helper/hil_select.py moves to tools/ci_select.py: it is no longer HIL-only, and tools/ is where the build side can import it. test_hil_select.py follows it as test_ci_select.py. Rules (docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md holds the full table): a port selects the families whose family.cmake references it, and its role - a dcd change skips host examples and vice versa; a class selects only the examples whose tusb_config.h enables its CFG_TU[DH]_ macro, following cross-class includes; an example selects itself; hw/bsp selects its family or board; hw/mcu and lib select whoever references them. CMake is the reference for all of it - make follows whatever cmake decides, family.mk is never scanned. Empty means empty (maintainer ruling): a rule that classifies a path to nothing selects nothing. Ports no family references, classes no config enables, libs no example builds and hw/mcu paths that resolve nowhere are all real - nothing compiles them, so nothing can validate them, and the master-push build is the net. Structural tests pin each such case with an explicit allowlist, so the day one stops being empty it fails pre-commit instead of silently narrowing CI. Per-example builds: build.py grows a repeatable -e, resolved against the targets CMake actually registered and batched into one `cmake --build --target a b c`. build_utils mirrors CMake's family_filter (the whole FAMILY_MCUS list, ${...} and string(TOUPPER ...) resolved) for the cmake side, while the make side keeps master's algorithm verbatim - the two build systems answer differently and a shared answer breaks lpc54's make link. hil-build gains this even on a full selection: 1702 example builds become 515. Transport: the selection travels as a file, never an argv or env var - a mass-sweep diff selects 261 KB against a 128 KiB exec limit, and E2BIG would fail the step before its own fallback could run. CircleCI carries the example map inside the generated config (pipeline parameters cap at 512 chars), swapped into the parameter defaults by sentinel match, and drops the scoping wholesale if that rewrite fails. Every PR-derived value written to $GITHUB_ENV/$GITHUB_OUTPUT is character-screened. Code metrics follow the scoping: metrics.py emits per-example totals, and metrics_pair_compare compares the (board, example) pairs present on both sides instead of a scoped run against a full-matrix average. The selector's own suite gates it in both providers: a selector that exits 0 with valid-but-wrong JSON is the one failure fail-open cannot catch, so a red suite means the full matrix. --- .github/scripts/ci_set_matrix.py | 78 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 72 insertions(+), 6 deletions(-) (limited to '.github/scripts/ci_set_matrix.py') diff --git a/.github/scripts/ci_set_matrix.py b/.github/scripts/ci_set_matrix.py index 50ada5964..ee3609bed 100755 --- a/.github/scripts/ci_set_matrix.py +++ b/.github/scripts/ci_set_matrix.py @@ -1,5 +1,9 @@ #!/usr/bin/env python3 +import argparse import json +import os +import subprocess +import sys # toolchain, url toolchain_list = [ @@ -97,15 +101,77 @@ family_list = { } -def set_matrix_json(): +def set_matrix_json(select=None): + sel_fams = None + if select: + # every shape check is explicit: this runs AFTER main()'s fail-open handler, so + # an AttributeError on e.g. {"build": ["stm32f4"]} would red the step instead + # of falling back to the full matrix - the outcome that handler exists to prevent + b = select.get('build') if isinstance(select, dict) else None + if not isinstance(b, dict): + b = {} + if b.get('full') is False: + fams = b.get('families') + if not (isinstance(fams, list) and all(isinstance(f, str) for f in fams)): + # key ABSENT (or not a list of names) is an unusable selection, not + # "nothing selected": scoping every toolchain to [] would build zero + # families and report a vacuous green. An explicit families: [] stays a + # legitimate nothing-selected. + print('ci_set_matrix: UNSCOPED - build.full is false but the families ' + 'list is unusable, emitting the full matrix', file=sys.stderr) + else: + sel_fams = set(fams) matrix = {} for toolchain in toolchain_list: - filtered_families = [family for family, supported_toolchain in family_list.items() if - toolchain in supported_toolchain] - matrix[toolchain] = filtered_families - + fams = [family for family, tc in family_list.items() if toolchain in tc] + if sel_fams is not None: + fams = [f for f in fams if f in sel_fams] + matrix[toolchain] = fams + if sel_fams is not None: + # a family this file does not list builds on no toolchain, so the selection maps + # to an empty matrix and every leg skips - which looks exactly like a working + # scoped run. Say so: hw/bsp holds several families CI has never built + # (efm32, py32f0, ...) and espressif, whose boards are built by hil-build-esp + unbuilt = sorted(f for f in sel_fams if f not in family_list) + if unbuilt: + print(f'ci_set_matrix: selected families built by no toolchain here: ' + f'{", ".join(unbuilt)}', file=sys.stderr) print(json.dumps(matrix)) +def main(): + parser = argparse.ArgumentParser() + group = parser.add_mutually_exclusive_group() + group.add_argument('--select', help='tools/ci_select.py JSON; scopes families when build.full is false') + # a whole selection as one argv/env value can exceed the exec limits on a big + # diff, which fails the calling step BEFORE it can fall open; callers that + # already have the selection on disk pass the path instead + group.add_argument('--select-file', help='file holding the same JSON as --select') + group.add_argument('--base', help='git ref: run tools/ci_select.py --base REF and scope from it') + args = parser.parse_args() + + select = None + try: + if args.select: + select = json.loads(args.select) + elif args.select_file: + with open(args.select_file) as f: + select = json.load(f) + elif args.base: + root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + r = subprocess.run([sys.executable, os.path.join(root, 'tools', 'ci_select.py'), + '--base', args.base], + capture_output=True, text=True, cwd=root, check=True) + select = json.loads(r.stdout) + except Exception as e: # fail-open: an unusable selection must never turn into a red job + # UNSCOPED is the marker build.yml greps for: it must then drop the build extras + # (example map, family regex) too, or a full build gets labelled and filtered as + # a scoped one. Keep the token on every fall-open path. + print(f'ci_set_matrix: UNSCOPED - selection unusable ({e}), emitting the full ' + f'matrix', file=sys.stderr) + select = None + set_matrix_json(select) + + if __name__ == '__main__': - set_matrix_json() + main() -- cgit v1.3.1 From f17be6770b601a32bdfcc1459e8851becf7fea84 Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 21 Aug 2026 16:08:25 +0700 Subject: ci_set_matrix: fall open when no selected family builds anywhere family_list maps a family to the toolchains that build it, and seven hw/bsp families are in neither: cxd56, efm32, espressif, f1c100s, pic32mz, py32f0, same7x. Scoping to one of them intersected to nothing, so every toolchain key was [], every cmake leg skipped on `if: inputs.build-args != '[]'`, code-metrics took its no-metrics branch, and the PR went green from a build job that ran no compiler. The only signal was a stderr line nothing greps for. Not a coverage regression - master gave the same diff no compile coverage either, since none of the other families compiles same7x's board.h. What is new is that the gap used to be masked by the full matrix and is now the whole answer, and that green now means "ran no compiler" rather than "compiled 64 families". A selection whose families ALL miss is now unusable rather than empty: it prints UNSCOPED, which build.yml and .circleci/config.yml already grep to drop the build extras with it, and emits the full matrix. The two neighbouring cases keep their own answers - an explicit families: [] is still a legitimate nothing-selected, and a partial miss still scopes to the families that do build, noting the rest. The contract test pinned an exact count of fall-open markers, which this would have broken; it now pins the invariant (every message that emits the full matrix carries the marker) and was checked to still fail when a marker is removed. Also corrects the drift guard's note about espressif: hil-build-esp builds its boards by name, but that job is gated on repository_owner, so on a fork an espressif-only PR builds nowhere. --- .github/scripts/ci_set_matrix.py | 20 +++++++++++++++----- test/hil/test/test_ci_metrics.py | 13 +++++++++++-- test/hil/test/test_ci_select.py | 36 ++++++++++++++++++++++++++++++------ 3 files changed, 56 insertions(+), 13 deletions(-) (limited to '.github/scripts/ci_set_matrix.py') diff --git a/.github/scripts/ci_set_matrix.py b/.github/scripts/ci_set_matrix.py index ee3609bed..79f466893 100755 --- a/.github/scripts/ci_set_matrix.py +++ b/.github/scripts/ci_set_matrix.py @@ -127,12 +127,22 @@ def set_matrix_json(select=None): if sel_fams is not None: fams = [f for f in fams if f in sel_fams] matrix[toolchain] = fams - if sel_fams is not None: - # a family this file does not list builds on no toolchain, so the selection maps - # to an empty matrix and every leg skips - which looks exactly like a working - # scoped run. Say so: hw/bsp holds several families CI has never built - # (efm32, py32f0, ...) and espressif, whose boards are built by hil-build-esp + if sel_fams: + # a family this file does not list builds on no toolchain, so it contributes no + # leg. hw/bsp holds several CI has never built (efm32, py32f0, same7x, ...) plus + # espressif, whose boards hil-build-esp builds by name. unbuilt = sorted(f for f in sel_fams if f not in family_list) + if unbuilt and not any(matrix.values()): + # NONE of the selected families is buildable here, so every leg would skip + # and the PR would go green from a build job that ran no compiler. That is + # an unusable selection, not "nothing selected": say UNSCOPED - which + # build.yml and .circleci/config.yml both grep for - and emit the full + # matrix. An explicit families: [] is still a legitimate nothing-selected, + # and a PARTIAL miss still scopes to the families that do build. + print(f'ci_set_matrix: UNSCOPED - no selected family is built by any ' + f'toolchain here ({", ".join(unbuilt)}), emitting the full matrix', + file=sys.stderr) + return set_matrix_json(None) if unbuilt: print(f'ci_set_matrix: selected families built by no toolchain here: ' f'{", ".join(unbuilt)}', file=sys.stderr) diff --git a/test/hil/test/test_ci_metrics.py b/test/hil/test/test_ci_metrics.py index 6c236e827..a76b6e3a0 100644 --- a/test/hil/test/test_ci_metrics.py +++ b/test/hil/test/test_ci_metrics.py @@ -435,8 +435,17 @@ class TestWorkflowSelectionHandOff(unittest.TestCase): self.assertIn('BUILD_SELECT_FILE', self.build) scripts = os.path.join(os.path.dirname(CIRCLECI), '.github', 'scripts') matrix = open(os.path.join(scripts, 'ci_set_matrix.py')).read() - self.assertEqual(matrix.count('ci_set_matrix: UNSCOPED'), 2, - 'every fall-open path must print the marker build.yml greps for') + # count-independent: pin the INVARIANT, not the number of fall-open paths - + # every message that emits the full matrix must carry the marker, and a purely + # informational note (a partial family miss) must not claim to have done so. + # Adjacent string literals are joined first, since these messages wrap. + import re as _re + flat = _re.sub(r"['\"]\s*\n\s*f?['\"]", '', matrix) + hits = [m.start() for m in _re.finditer('emitting the full ', flat)] + self.assertGreaterEqual(len(hits), 2, 'fall-open messages not found') + for i in hits: + self.assertIn('UNSCOPED', flat[max(0, i - 200):i], + 'a fall-open path without the marker build.yml greps for') def test_membrowse_upload_sees_the_same_board_as_the_build(self): # $EX_ARGS is passed for the BOARD it selects: --one-first picks a board that can diff --git a/test/hil/test/test_ci_select.py b/test/hil/test/test_ci_select.py index f5c64ac1c..8f1841531 100644 --- a/test/hil/test/test_ci_select.py +++ b/test/hil/test/test_ci_select.py @@ -777,12 +777,15 @@ class TestOrphanInvariant(unittest.TestCase): for v in vendors: self.assertTrue(ci_select.mcu_families(v + '/x.c', REPO), f'{v}: resolves to no family') - # hw/bsp families ci_set_matrix's family_list does not map to any toolchain. Before - # scoping these were harmless - the matrix was always every family in family_list, - # so a PR touching one of them still compiled the other 64. Now the selection - # intersects to nothing and every leg skips, so a family landing here by accident is - # a silent hole. espressif is deliberate: its boards are built by hil-build-esp, - # keyed on board name rather than family. + # hw/bsp families ci_set_matrix's family_list does not map to any toolchain. Master + # gave a PR touching one of these no compile coverage either - none of the other 64 + # families compiles same7x's board.h - so this is not new. What IS new is that the + # gap used to be masked by a full matrix and is now the whole answer, which is why + # ci_set_matrix treats a selection that intersects family_list to NOTHING as + # unusable (UNSCOPED -> full matrix) rather than emitting an all-empty one. + # espressif is here because hil-build-esp builds its boards by name rather than by + # family - though only on hathach/tinyusb: that job is gated on repository_owner, + # so on a fork an espressif-only PR builds nowhere. UNBUILT_FAMILIES = {'cxd56', 'efm32', 'espressif', 'f1c100s', 'pic32mz', 'py32f0', 'same7x'} @@ -1505,6 +1508,27 @@ class TestCiSetMatrix(unittest.TestCase): self.assertEqual(json.loads(r.stdout), base) self.assertIn('ci_set_matrix: UNSCOPED', r.stderr) # build.yml greps this + def test_families_no_toolchain_builds_falls_open(self): + # hw/bsp/same7x is real but in no toolchain's list, so scoping to it emits an + # all-empty matrix: every leg skips and the PR goes green from a build job that + # ran no compiler. Unusable, not "nothing selected" - and the marker matters, + # because that is what build.yml and CircleCI grep to drop the build extras too. + base = json.loads(self.run_matrix().stdout) + r = self.run_matrix('--select', + json.dumps({'build': {'full': False, 'families': ['same7x']}})) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertEqual(json.loads(r.stdout), base) + self.assertIn('ci_set_matrix: UNSCOPED', r.stderr) + + def test_a_partial_toolchain_miss_still_scopes(self): + # one buildable family is real coverage: scope to it and just note the other + r = self.run_matrix('--select', json.dumps( + {'build': {'full': False, 'families': ['stm32f4', 'same7x']}})) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertEqual(json.loads(r.stdout)['arm-gcc'], ['stm32f4']) + self.assertNotIn('UNSCOPED', r.stderr) + self.assertIn('same7x', r.stderr) + def test_explicit_empty_families_selects_nothing(self): # an explicit [] IS a legitimate answer (a diff that builds nothing) r = self.run_matrix('--select', -- cgit v1.3.1 From da255b1d2db10b8f31332a779b2a526f579acee1 Mon Sep 17 00:00:00 2001 From: Ha Thach Date: Tue, 25 Aug 2026 09:46:42 +0700 Subject: ci: an empty selection must build nothing, plus selector follow-ups (#3845) ci: an empty selection must build nothing, plus selector follow-ups A PR whose build axis legitimately selected nothing rebuilt everything. build.yml reads .build.families twice - as a |-joined regex, and implicitly as "is anything selected" - but tested only -z "$FAMILY_REGEX", which an empty list and a charset-rejected one both satisfy while meaning opposite things. ci_set_matrix had already returned the correct all-empty matrix; the fall-open branch discarded it. #3842 and #3840 each spent 74 cmake legs on it. Branch on the two cases instead, rename FAM_* to FAMILY_*, and cover the block with a test that extracts it from build.yml and executes it - it had no test at all, which is how this shipped through two merges. Follow-ups to the same machinery: glob.escape the repo root at five sites, so a checkout path containing [ or * stops failing closed; drop the ci-full label, read after the matrix was already computed and so never functional; delete 13 mcu:MKL25ZXX / mcu:SAME5X skip tokens matching no board; carry the rule table in the module docstring, guarded against drift; and pin six selection behaviours a mutation pass proved untested. Cut the selector's cost 1.8x (26.0s -> 14.6s) with 0 divergences over 260 paths, and stop scoping the membrowse upload by the PR example filter. --- .github/scripts/ci_set_matrix.py | 8 +- .github/workflows/build.yml | 35 ++-- .github/workflows/build_util.yml | 13 +- docs/reference/hil_boards.md | 2 +- .../2026-08-19-ci-build-family-filter-design.md | 13 ++ examples/device/audio_4_channel_mic/skip.txt | 1 - .../device/audio_4_channel_mic_freertos/skip.txt | 1 - examples/device/audio_test/skip.txt | 1 - examples/device/audio_test_freertos/skip.txt | 1 - examples/device/audio_test_multi_rate/skip.txt | 1 - examples/device/cdc_msc_freertos/skip.txt | 1 - examples/device/cdc_uac2/skip.txt | 1 - examples/device/hid_composite_freertos/skip.txt | 1 - examples/device/midi_test_freertos/skip.txt | 1 - examples/device/msc_dual_lun/skip.txt | 1 - examples/device/uac2_headset/skip.txt | 1 - examples/device/uac2_speaker_fb/skip.txt | 1 - test/hil/test/test_ci_metrics.py | 126 ++++++++++++- test/hil/test/test_ci_select.py | 204 ++++++++++++++++++++- tools/build.py | 4 +- tools/ci_select.py | 130 ++++++++++--- 21 files changed, 466 insertions(+), 81 deletions(-) (limited to '.github/scripts/ci_set_matrix.py') diff --git a/.github/scripts/ci_set_matrix.py b/.github/scripts/ci_set_matrix.py index 79f466893..409e6dbc1 100755 --- a/.github/scripts/ci_set_matrix.py +++ b/.github/scripts/ci_set_matrix.py @@ -131,7 +131,13 @@ def set_matrix_json(select=None): # a family this file does not list builds on no toolchain, so it contributes no # leg. hw/bsp holds several CI has never built (efm32, py32f0, same7x, ...) plus # espressif, whose boards hil-build-esp builds by name. - unbuilt = sorted(f for f in sel_fams if f not in family_list) + # espressif is not a gap: its examples need the ESP-IDF environment + # (CLAUDE.md: `. "$IDF_PATH/export.sh"` before any build), which the cmake legs + # do not have - that is why it is commented out of family_list above. Its + # coverage comes from hil-build-esp, which builds those boards BY NAME in an IDF + # container, so an espressif-only PR is already validated and falling open to the + # full matrix would add 74 legs, none of which can compile espressif. + unbuilt = sorted(f for f in sel_fams if f not in family_list and f != 'espressif') if unbuilt and not any(matrix.values()): # NONE of the selected families is buildable here, so every leg would skip # and the PR would go green from a build job that ran no compiler. That is diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 39a4e7afd..c26fe5cf8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -68,14 +68,9 @@ jobs: with: fetch-depth: 0 - # The `ci-full` PR label turns the scoping off for one PR: no selection file is - # written, so both matrices and every rig job fall back to the unscoped behaviour. - # An escape hatch is the point - a selector bug under-selects SILENTLY, and without - # a label the only routes back to a full matrix are accidental (touch an - # unclassified path, or break the selector badly enough that it falls open). - name: CI selection (PR only) id: hil-select - if: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ci-full') + if: github.event_name == 'pull_request' env: BASE_REF: ${{ github.base_ref }} run: | @@ -179,29 +174,43 @@ jobs: # treats false like null, so .build.full is compared explicitly. EXAMPLE_MAP='{}' BUILD_FILTERED='false' - FAM_REGEX='' + FAMILY_REGEX='' if [ -n "$BUILD_SELECT_FILE" ]; then EXAMPLE_MAP=$(jq -c '.build.family_examples // {}' "$BUILD_SELECT_FILE") || EXAMPLE_MAP='{}' BUILD_FILTERED=$(jq -r 'if (.build? | type) == "object" and .build.full == false then "true" else "false" end' "$BUILD_SELECT_FILE") || BUILD_FILTERED='false' if [ "$BUILD_FILTERED" = "true" ]; then - FAM_REGEX=$(jq -r '.build.families | join("|")' "$BUILD_SELECT_FILE") || FAM_REGEX='' + FAMILY_COUNT=$(jq -r '.build.families | length' "$BUILD_SELECT_FILE") || FAMILY_COUNT=0 + FAMILY_REGEX=$(jq -r '.build.families | join("|")' "$BUILD_SELECT_FILE") || FAMILY_REGEX='' # family names come from hw/bsp dir names, which rule 6 reads straight out # of the PR's diff path - and this is interpolated raw into a # `name_is_regexp` artifact pattern, so a regex metacharacter there would # silently match another family's baseline - case "$FAM_REGEX" in + FAMILY_REJECTED=0 + case "$FAMILY_REGEX" in *[!-A-Za-z0-9_\|]*) echo "::warning::unexpected characters in the family list - dropping the scoping" - FAM_REGEX='' ;; + FAMILY_REGEX=''; FAMILY_REJECTED=1 ;; esac - if [ -z "$FAM_REGEX" ]; then - # all three drop together, as CircleCI's fall-open does. Resetting only + # An EMPTY families list and a REJECTED one both leave FAMILY_REGEX empty and + # mean opposite things, so branch on which happened. Testing `-z` alone sent + # every nothing-selected PR down the fall-open path: a docs/.gitignore diff + # (#3842) and a test/hil-only diff (#3840) each rebuilt all 74 cmake legs + # after the selector had correctly chosen none. + if [ "$FAMILY_REJECTED" = "1" ]; then + # unusable: fall open, and all three drop together. Resetting only # build_filtered leaves the build scoped while code-metrics takes the # UNSCOPED branch, diffing a 1-family run against the full averaged # baseline and publishing that as the PR's code-size impact. BUILD_FILTERED='false' EXAMPLE_MAP='{}' MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py) + elif [ "$FAMILY_COUNT" = "0" ]; then + # legitimate nothing-selected. MATRIX_JSON already holds the all-empty + # matrix ci_set_matrix produced from this selection - keep it, so every + # leg skips. Nothing is built, so there is nothing to compare a baseline + # against: build_filtered goes false to keep code-metrics off the scoped + # path, and EXAMPLE_MAP stays '{}' (family_examples is empty anyway). + BUILD_FILTERED='false' fi fi fi @@ -210,7 +219,7 @@ jobs: echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT echo "example_map=$EXAMPLE_MAP" >> $GITHUB_OUTPUT echo "build_filtered=$BUILD_FILTERED" >> $GITHUB_OUTPUT - echo "build_families_regex=$FAM_REGEX" >> $GITHUB_OUTPUT + echo "build_families_regex=$FAMILY_REGEX" >> $GITHUB_OUTPUT # HIL matrix (merged from tinyusb + hifiphile configs), scoped on PRs. # Scoping is best-effort too: fall back to the unscoped (full) matrix. diff --git a/.github/workflows/build_util.yml b/.github/workflows/build_util.yml index 52999616d..407ed1e71 100644 --- a/.github/workflows/build_util.yml +++ b/.github/workflows/build_util.yml @@ -126,16 +126,11 @@ jobs: MEMBROWSE_API_KEY: ${{ secrets.MEMBROWSE_API_KEY }} run: | # if code-changed is false --> there is no elf -> membrowse target upload with --identical flag - # $EX_ARGS is passed for the BOARD it picks, not to scope the targets: - # --one-first now chooses a board that can build the -e set (tools/build.py), - # so omitting it here would configure a DIFFERENT, empty build dir and upload - # --identical for a board that was never compiled. The target list is not - # scoped by it - `examples-membrowse-upload` is not `all`, so it passes - # through as the aggregate, which has no DEPENDS (hw/bsp/family_support.cmake): - # it rebuilds nothing and still records every example, --identical for the - # ones without an elf. + # deliberately unscoped by $EX_ARGS: keeps the size history on a stable board + # per family, at the cost of an --identical-only upload where that board is not + # the one the Build step picked (test_ci_metrics pins which families those are) BUILD_PY_ARGS="-s ${{ inputs.build-system }} ${{ steps.setup-toolchain.outputs.build_option }} ${{ inputs.build-options }}" - python tools/build.py $BUILD_PY_ARGS --target examples-membrowse-upload -j 1 ${{ matrix.arg }} $EX_ARGS + python tools/build.py $BUILD_PY_ARGS --target examples-membrowse-upload -j 1 ${{ matrix.arg }} shell: bash - name: Upload Artifacts for Metrics diff --git a/docs/reference/hil_boards.md b/docs/reference/hil_boards.md index e8f364646..678f7f0ed 100644 --- a/docs/reference/hil_boards.md +++ b/docs/reference/hil_boards.md @@ -12,7 +12,7 @@ | espressif_s3_devkitm | device, host | esptool | espressif_s3_devkitm, espressif_s3_devkitm-DMA | Use TS3USB30 mux to test both device and host | | feather_nrf52840_express | device | jlink | | | | max32666fthr | device | openocd | | | -| metro_m4_express | device, dual | jlink | | pl23x; audio_test_freertos skipped: samd51 iso-IN capture fails (arecord EIO) | +| metro_m4_express | device, dual | jlink | metro_m4_express | pl23x; audio_test_freertos skipped: samd51 iso-IN capture fails (arecord EIO) | | lpcxpresso11u37 | device | jlink | | | | lpcxpresso55s28 | device | jlink | | | | ra4m1_ek | device | jlink | | | diff --git a/docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md b/docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md index b10f5b4ae..524568aeb 100644 --- a/docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md +++ b/docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md @@ -146,6 +146,19 @@ the existing `test_hil_util.BottomLayer` structural tests. Fail-open survives where it belongs: an *unclassified* path or any exception widens to `ALL` on every axis. +### A class no example enables selects nothing + +`src/class/bth` is the live instance: no example's `tusb_config.h` sets `CFG_TUD_BTH`, so +rules 8-10 resolve to no examples and a bth-only PR builds nothing and runs nothing. That is +the empty-means-empty ruling applied to classes, and it is deliberate — nothing compiles the +file, so nothing can validate it, and the master-push build is the net. + +Worth stating plainly because the exposure changed: GHA used to rebuild everything for such +a PR by accident, through the empty-`families` bug in `build.yml`. With that fixed, both +providers now correctly build nothing, so `tud_bt_*` can be broken by a green PR. +`TestClassesWithNoEnablingExample` pins the set to `{bth}` so a second class cannot enter +this state unnoticed. + ### Why `hw/mcu/**` is rule 7 and not "full" `hw/mcu` is overwhelmingly dependency territory — `tools/get_deps.py` has 87 entries under it, diff --git a/examples/device/audio_4_channel_mic/skip.txt b/examples/device/audio_4_channel_mic/skip.txt index 3ca433c08..e5e74cd60 100644 --- a/examples/device/audio_4_channel_mic/skip.txt +++ b/examples/device/audio_4_channel_mic/skip.txt @@ -1,5 +1,4 @@ mcu:SAMD11 -mcu:SAME5X mcu:SAMG family:broadcom_64bit family:espressif diff --git a/examples/device/audio_4_channel_mic_freertos/skip.txt b/examples/device/audio_4_channel_mic_freertos/skip.txt index 1fd6b4b8a..cfde51051 100644 --- a/examples/device/audio_4_channel_mic_freertos/skip.txt +++ b/examples/device/audio_4_channel_mic_freertos/skip.txt @@ -7,7 +7,6 @@ mcu:CXD56 mcu:F1C100S mcu:GD32VF103 mcu:MCXA15 -mcu:MKL25ZXX mcu:MSP430x5xx mcu:FT90X mcu:SAMD11 diff --git a/examples/device/audio_test/skip.txt b/examples/device/audio_test/skip.txt index 42394bb11..862c91c6f 100644 --- a/examples/device/audio_test/skip.txt +++ b/examples/device/audio_test/skip.txt @@ -1,5 +1,4 @@ mcu:SAMD11 -mcu:SAME5X mcu:SAMG family:espressif mcu:CH583 diff --git a/examples/device/audio_test_freertos/skip.txt b/examples/device/audio_test_freertos/skip.txt index 660bacd25..3d8d43286 100644 --- a/examples/device/audio_test_freertos/skip.txt +++ b/examples/device/audio_test_freertos/skip.txt @@ -7,7 +7,6 @@ mcu:CXD56 mcu:F1C100S mcu:GD32VF103 mcu:MCXA15 -mcu:MKL25ZXX mcu:MSP430x5xx mcu:FT90X mcu:SAMD11 diff --git a/examples/device/audio_test_multi_rate/skip.txt b/examples/device/audio_test_multi_rate/skip.txt index 42394bb11..862c91c6f 100644 --- a/examples/device/audio_test_multi_rate/skip.txt +++ b/examples/device/audio_test_multi_rate/skip.txt @@ -1,5 +1,4 @@ mcu:SAMD11 -mcu:SAME5X mcu:SAMG family:espressif mcu:CH583 diff --git a/examples/device/cdc_msc_freertos/skip.txt b/examples/device/cdc_msc_freertos/skip.txt index 48781de84..095e350c9 100644 --- a/examples/device/cdc_msc_freertos/skip.txt +++ b/examples/device/cdc_msc_freertos/skip.txt @@ -7,7 +7,6 @@ mcu:CXD56 mcu:F1C100S mcu:GD32VF103 mcu:MCXA15 -mcu:MKL25ZXX mcu:MSP430x5xx mcu:FT90X mcu:SAMD11 diff --git a/examples/device/cdc_uac2/skip.txt b/examples/device/cdc_uac2/skip.txt index db1d5b80b..3159cb176 100644 --- a/examples/device/cdc_uac2/skip.txt +++ b/examples/device/cdc_uac2/skip.txt @@ -2,7 +2,6 @@ mcu:LPC11UXX mcu:LPC13XX mcu:NUC121 mcu:SAMD11 -mcu:SAME5X mcu:SAMG board:stm32l052dap52 family:espressif diff --git a/examples/device/hid_composite_freertos/skip.txt b/examples/device/hid_composite_freertos/skip.txt index 97d8e168b..0e8415d3b 100644 --- a/examples/device/hid_composite_freertos/skip.txt +++ b/examples/device/hid_composite_freertos/skip.txt @@ -7,7 +7,6 @@ mcu:CXD56 mcu:F1C100S mcu:GD32VF103 mcu:MCXA15 -mcu:MKL25ZXX mcu:MSP430x5xx mcu:FT90X mcu:SAMD11 diff --git a/examples/device/midi_test_freertos/skip.txt b/examples/device/midi_test_freertos/skip.txt index 97d8e168b..0e8415d3b 100644 --- a/examples/device/midi_test_freertos/skip.txt +++ b/examples/device/midi_test_freertos/skip.txt @@ -7,7 +7,6 @@ mcu:CXD56 mcu:F1C100S mcu:GD32VF103 mcu:MCXA15 -mcu:MKL25ZXX mcu:MSP430x5xx mcu:FT90X mcu:SAMD11 diff --git a/examples/device/msc_dual_lun/skip.txt b/examples/device/msc_dual_lun/skip.txt index a9e3a99b1..833fd072c 100644 --- a/examples/device/msc_dual_lun/skip.txt +++ b/examples/device/msc_dual_lun/skip.txt @@ -1,3 +1,2 @@ mcu:SAMD11 -mcu:MKL25ZXX family:espressif diff --git a/examples/device/uac2_headset/skip.txt b/examples/device/uac2_headset/skip.txt index db1d5b80b..3159cb176 100644 --- a/examples/device/uac2_headset/skip.txt +++ b/examples/device/uac2_headset/skip.txt @@ -2,7 +2,6 @@ mcu:LPC11UXX mcu:LPC13XX mcu:NUC121 mcu:SAMD11 -mcu:SAME5X mcu:SAMG board:stm32l052dap52 family:espressif diff --git a/examples/device/uac2_speaker_fb/skip.txt b/examples/device/uac2_speaker_fb/skip.txt index 0c7339c65..88df3e549 100644 --- a/examples/device/uac2_speaker_fb/skip.txt +++ b/examples/device/uac2_speaker_fb/skip.txt @@ -2,7 +2,6 @@ mcu:LPC11UXX mcu:LPC13XX mcu:NUC121 mcu:SAMD11 -mcu:SAME5X mcu:SAMG board:stm32l052dap52 family:broadcom_64bit diff --git a/test/hil/test/test_ci_metrics.py b/test/hil/test/test_ci_metrics.py index a76b6e3a0..aac251824 100644 --- a/test/hil/test/test_ci_metrics.py +++ b/test/hil/test/test_ci_metrics.py @@ -447,16 +447,126 @@ class TestWorkflowSelectionHandOff(unittest.TestCase): self.assertIn('UNSCOPED', flat[max(0, i - 200):i], 'a fall-open path without the marker build.yml greps for') - def test_membrowse_upload_sees_the_same_board_as_the_build(self): - # $EX_ARGS is passed for the BOARD it selects: --one-first picks a board that can - # build the -e set, so without it membrowse configures a different, empty build - # dir and uploads --identical for a board that was never compiled. It does NOT - # scope the targets - `examples-membrowse-upload` is not `all`, so it passes - # through as the aggregate, which has no DEPENDS and still records every example. + def _run_extras_block(self, sel): + """Extract the build-extras shell block from build.yml and run it for real. + Nothing else exercises it, which is why the empty/rejected conflation shipped.""" + import re as _re, shlex, subprocess, tempfile, json as _json + repo = os.path.dirname(CIRCLECI) + i = self.build.index("EXAMPLE_MAP='{}'\n BUILD_FILTERED='false'") + i = self.build.rindex('\n', 0, i) + 1 + j = self.build.index(' echo "matrix=$MATRIX_JSON"', i) + block = _re.sub(r'^ {10}', '', self.build[i:j], flags=_re.M) + with tempfile.TemporaryDirectory() as d: + selp = os.path.join(d, 'sel.json') + with open(selp, 'w') as fh: + _json.dump(sel, fh) + matrix = subprocess.run( + [sys.executable, os.path.join(repo, '.github/scripts/ci_set_matrix.py'), + '--select-file', selp], capture_output=True, text=True, cwd=repo).stdout.strip() + self.assertTrue(matrix, 'ci_set_matrix produced nothing') + sh = os.path.join(d, 'probe.sh') + with open(sh, 'w') as fh: + # shlex.quote, not hand-rolled quoting: a TMPDIR with a space in it + # made this fail for a reason that had nothing to do with the block + fh.write('BUILD_SELECT_FILE=' + shlex.quote(selp) + '\n') + fh.write('MATRIX_JSON=' + shlex.quote(matrix) + '\n') + fh.write(block) + # sentinel + newline separated: the block itself writes ::warning:: to + # stdout, and '|' would collide with the regex's own separator + fh.write('\nprintf "@@R@@\\n%s\\n%s\\n%s" "$MATRIX_JSON" "$BUILD_FILTERED" "$FAMILY_REGEX"\n') + r = subprocess.run(['bash', sh], capture_output=True, text=True, cwd=repo) + self.assertEqual(r.returncode, 0, r.stderr) + mj, filtered, regex = r.stdout.split('@@R@@\n', 1)[1].split('\n', 2) + return sum(len(v) for v in _json.loads(mj).values()), filtered, regex + + def test_an_empty_family_list_is_not_treated_as_unusable(self): + """.build.families is read twice - as a count and as a `|`-joined regex. An EMPTY + list and one REJECTED by the charset guard both leave the regex empty and mean + opposite things, so the block has to branch on which happened. + + Testing `-z "$FAMILY_REGEX"` alone sent every nothing-selected PR down the + fall-open path and discarded the correct all-empty matrix: #3842 (docs + + .gitignore) and #3840 (test/hil only) each rebuilt all 74 cmake legs after the + selector had correctly chosen none.""" + legs, filtered, regex = self._run_extras_block( + {'build': {'full': False, 'families': [], 'family_examples': {}}}) + self.assertEqual(legs, 0, 'an empty families list must keep the all-empty matrix') + self.assertEqual(filtered, 'false', 'nothing was built, so nothing to compare') + self.assertEqual(regex, '') + + def test_a_real_family_list_stays_scoped(self): + legs, filtered, regex = self._run_extras_block( + {'build': {'full': False, 'families': ['stm32f4', 'rp2040'], + 'family_examples': {}}}) + self.assertGreater(legs, 0) + self.assertEqual(filtered, 'true') + self.assertEqual(regex, 'stm32f4|rp2040') + + def test_a_regex_metacharacter_in_a_family_name_falls_open(self): + # the name is interpolated raw into a name_is_regexp artifact pattern, so a + # metacharacter would match another family's baseline - reject and widen + legs, filtered, regex = self._run_extras_block( + {'build': {'full': False, 'families': ['stm32f4.*'], 'family_examples': {}}}) + self.assertGreater(legs, 100, 'a rejected family list must fall open to full') + self.assertEqual(filtered, 'false') + self.assertEqual(regex, '') + + def test_membrowse_upload_is_not_scoped_by_the_pr_filter(self): + # by decision, the upload runs unfiltered so the size history stays keyed on the + # family's preferred board whatever the PR touched. $EX_ARGS would not have + # scoped the targets either way - `examples-membrowse-upload` is not `all`, so + # resolve_example_target_groups passes it through as the aggregate - but it DID + # move the board, because --one-first picks one that can build the -e set. + # + # The accepted cost: on a family whose preferred board cannot build that set, + # the upload lands on a board the Build step never compiled and every example + # goes up --identical. test_the_upload_board_can_diverge_from_the_built_board + # keeps that consequence measured rather than assumed. line = [l for l in self.util.splitlines() if '--target examples-membrowse-upload' in l][0] - self.assertIn('$EX_ARGS', line) - self.assertNotIn('-e ', line.replace('$EX_ARGS', '')) + self.assertNotIn('$EX_ARGS', line) + self.assertNotIn('-e ', line) + + def test_the_upload_board_can_diverge_from_the_built_board(self): + """Pins the SIZE of what the removal gave up, so it cannot grow unnoticed. + + --one-first with no -e returns preferred_list[0]; with one it returns the first + preferred board that can build it. Where those differ, the Membrowse Upload step + configures a build dir the Build step never wrote.""" + sys.path.insert(0, os.path.join(REPO, 'tools')) + import build as build_py + roles = ('device', 'host', 'dual') + exs = sorted(f'{r}/{n}' for r in roles + for n in os.listdir(os.path.join(REPO, 'examples', r)) + if os.path.isdir(os.path.join(REPO, 'examples', r, n))) + fams = sorted(d for d in os.listdir(os.path.join(REPO, 'hw/bsp')) + if os.path.isdir(os.path.join(REPO, 'hw/bsp', d, 'boards'))) + cwd = os.getcwd() + os.chdir(REPO) + try: + diverging = set() + for fam in fams: + try: + base = build_py.get_family_boards(fam, False, True, None, 'cmake', ()) + except Exception: + continue + if not base: + continue + for e in exs: + try: + one = build_py.get_family_boards(fam, False, True, [e], 'cmake', ()) + except Exception: + continue + if one and one[0] != base[0]: + diverging.add(fam) + break + finally: + os.chdir(cwd) + self.assertEqual(diverging, {'imxrt', 'lpc11', 'lpc18', 'lpc54', 'mcx', 'rp2040', + 'rx', 'samd11', 'stm32l0', 'stm32l4', 'tm4c'}, + 'the set of families whose membrowse upload can land on an ' + 'uncompiled board changed; re-check whether dropping $EX_ARGS ' + 'from the upload step is still the right trade') if __name__ == '__main__': diff --git a/test/hil/test/test_ci_select.py b/test/hil/test/test_ci_select.py index a19392bde..dc10f769a 100644 --- a/test/hil/test/test_ci_select.py +++ b/test/hil/test/test_ci_select.py @@ -302,7 +302,11 @@ class TestArgsEmission(unittest.TestCase): out = j.loads(r.stdout) self.assertFalse(out['full']) self.assertIn('tinyusb.json', out['args']) - self.assertTrue(any('cdc_device' in line for line in out['reasons'])) + # reasons are a stderr diagnostic, deliberately NOT in the payload: they were + # 97% of a 9.8 MB JSON on a dep bump, and every consumer re-parses that file + self.assertNotIn('reasons', out, 'reasons must not ride in the machine-read JSON') + self.assertNotIn('reasons', out['build']) + self.assertIn('cdc_device', r.stderr) # A core-class diff must select boards THROUGH THE CLI: the in-process tests # inject their own repo root, so only this subprocess path catches a broken # repo_root derivation -- which once made every repo-relative glob match @@ -938,6 +942,181 @@ class TestClassesWithNoEnablingExample(unittest.TestCase): 'both axes, so nothing compiles it until the next master push') +class TestExampleMapOmitsFullFamilies(unittest.TestCase): + """A family whose selection is ALREADY everything it can build carries no -e list. + + Sixth of the same shape as the class below, found the same way: a perf rewrite of + _prune_buildable dropped the `set(kept) != set(buildable)` test and all 216 tests + stayed green. The build outcome is identical either way -- build.py applies the same + skip_example the pruner just did -- so nothing compiled differently and only the + payload grew (22 families x 33 examples on one dcd_dwc2.c diff). That is exactly the + kind of drift no build failure ever reports.""" + + def test_a_device_only_port_diff_still_omits_families_it_cannot_narrow(self): + # dcd_dwc2.c selects device+dual examples only, but a family whose host examples + # are all unbuildable anyway ends up wanting its entire buildable set + b = ci_select.classify_build(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO) + self.assertFalse(b['full']) + self.assertTrue(b['families']) + omitted = [f for f in b['families'] if f not in b['family_examples']] + self.assertTrue(omitted, 'no family omitted its -e list; the "already everything ' + 'this family builds" case stopped being detected') + for fam in omitted: + self.assertNotIn(fam, b['family_examples']) + + def test_a_family_that_can_build_more_than_the_diff_wants_keeps_its_list(self): + # the other direction: one example selects itself and nothing else, so every + # family it lands on must carry an explicit -e or CI builds all 46 + b = ci_select.classify_build(['examples/device/cdc_msc/src/main.c'], REPO) + self.assertFalse(b['full']) + for fam in b['families']: + self.assertEqual(b['family_examples'].get(fam), ['device/cdc_msc'], fam) + + +class TestSelectionBehavioursThatHadNoTest(unittest.TestCase): + """Five behaviours a reviewer's mutation pass proved were unpinned: break each one + and the whole suite stayed green. Each test here fails against its mutant. + + They are grouped because they share a shape - every one is a small expression whose + removal silently NARROWS the selection, which is the failure direction that merges a + regression rather than wasting a runner.""" + + def test_build_defines_reach_the_prefilter(self): + # mutant: `defines = ()` in build.py's build_boards_list. metro_m4_express gets + # MAX3421_HOST=1 from its roster variant, never from its BSP, so without the + # defines the -e prefilter drops the rig's only MAX3421 firmware and hil-tinyusb + # has nothing to flash. + import build as build_py, build_utils, inspect + src = inspect.getsource(build_py.build_boards_list) + self.assertIn('defines = tuple(sorted(build_defines))', src, + 'the -D tokens must reach cmake_board/skip_example') + old = os.getcwd() + os.chdir(REPO) + try: + ex, board = 'dual/host_info_to_device_cdc', 'metro_m4_express' + self.assertTrue(build_utils.skip_example(ex, board), + 'without the define this example is correctly skipped') + self.assertFalse(build_utils.skip_example(ex, board, ('MAX3421_HOST=1',)), + 'with it, it must build - that is what the roster passes') + finally: + os.chdir(old) + + def test_one_first_prefers_a_board_that_can_build_the_filter(self): + # mutant: buildable() -> True, i.e. back to all_boards[0]. lpc54's first board + # skips every msc_file_explorer example, so the leg would compile nothing. + import build as build_py + old_env, old = os.environ.get('GITHUB_ACTIONS'), os.getcwd() + os.environ['GITHUB_ACTIONS'] = 'true' + os.chdir(REPO) + try: + unfiltered = build_py.get_family_boards('lpc54', False, True) + filtered = build_py.get_family_boards('lpc54', False, True, + ['host/msc_file_explorer']) + self.assertEqual(unfiltered, ['lpcxpresso54114'], 'unfiltered pick must not move') + self.assertNotEqual(filtered, unfiltered, + 'the -e pick must avoid a board that skips the whole filter') + import build_utils + self.assertFalse(build_utils.skip_example('host/msc_file_explorer', filtered[0]), + f'{filtered[0]} must actually build the filtered example') + finally: + os.chdir(old) + if old_env is None: + os.environ.pop('GITHUB_ACTIONS', None) + else: + os.environ['GITHUB_ACTIONS'] = old_env + + def test_a_class_file_selects_its_own_macro_not_just_the_directory(self): + # mutant: delete the _CLS_STEM_RE block. src/class/midi holds MIDI 1.0 AND 2.0; + # examples/device/midi2_device is the only example enabling CFG_TUD_MIDI2 and the + # only one that compiles midi2_device.c, but the directory macro alone misses it. + got = ci_select._build_class_examples('midi', 'midi2_device.c', {'device'}, REPO) + self.assertIn('device/midi2_device', got, + 'a midi2 change must select the example that compiles it') + host = ci_select._build_class_examples('midi', 'midi2_host.c', {'host'}, REPO) + self.assertIn('host/midi2_host', host) + # and the plain midi files must NOT drag midi2 in + plain = ci_select._build_class_examples('midi', 'midi_device.c', {'device'}, REPO) + self.assertNotIn('device/midi2_device', plain) + + def test_a_port_change_selects_the_dual_examples(self): + # mutant: drop `+ ('dual',)`. A dcd/hcd change must build the dual examples - + # they exercise both stacks on one board, so a dwc2 break lands there first. + s = ci_select.classify_build(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO) + duals = {e for exs in s['family_examples'].values() for e in exs + if e.startswith('dual/')} + self.assertTrue(duals, 'a dcd change selected no dual example') + + def test_the_selector_answers_the_same_with_and_without_ci_env(self): + # mutant: drop ci=True from _prune_buildable. ci_skip_boards/ci_preferred_boards + # only apply when GITHUB_ACTIONS/CIRCLECI is set, so without the pin a laptop and + # a runner disagree - and /pre-pr would report a family list CI will not build. + files = ['examples/host/cdc_msc_hid_freertos/src/main.c'] + old = os.environ.get('GITHUB_ACTIONS') + os.environ.pop('GITHUB_ACTIONS', None) + try: + local = ci_select.classify_build(files, REPO)['families'] + os.environ['GITHUB_ACTIONS'] = 'true' + import importlib + importlib.reload(ci_select) + runner = ci_select.classify_build(files, REPO)['families'] + finally: + if old is None: + os.environ.pop('GITHUB_ACTIONS', None) + else: + os.environ['GITHUB_ACTIONS'] = old + import importlib + importlib.reload(ci_select) + self.assertEqual(local, runner, 'the selector must not depend on the CI env vars') + + +class TestRuleTableIsCarbonOfTheSpec(unittest.TestCase): + """ci_select's module docstring carries the rule table so a reader landing in the + code does not have to open the spec to learn what rule 6 is. Both are maintained by + hand, so this pins them cell-for-cell: edit one without the other and this fails. + + It also pins the table against the CODE - every rule id the docstring claims must + appear as a `# rule N` marker on a branch of _classify_build_one, so a row cannot be + documented without a branch, or a branch renumbered without the table.""" + + @staticmethod + def _rows(text): + import re as _re + out = [] + for l in text.splitlines(): + if not l.startswith('| '): + continue + c = [x.strip() for x in l.strip().strip('|').split('|')] + if len(c) == 5 and _re.fullmatch(r'\d+[a-z]?', c[0]): + out.append(c) + return out + + def test_docstring_table_matches_the_spec(self): + spec = open(os.path.join( + REPO, 'docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md')).read() + doc, spec_rows = self._rows(ci_select.__doc__), self._rows(spec) + self.assertTrue(spec_rows, 'no rule table found in the spec') + self.assertEqual([r[0] for r in doc], [r[0] for r in spec_rows], + 'rule ids differ between ci_select.__doc__ and the spec') + for d, s in zip(doc, spec_rows): + self.assertEqual(d, s, f'rule {d[0]} differs between the docstring and the spec') + + def test_every_documented_rule_has_a_branch(self): + import re as _re + src = open(os.path.join(REPO, 'tools/ci_select.py')).read() + marked = set() + # handles `# rule 6`, `# rules 1, 1b` and `# rules 8-10` + for m in _re.finditer(r'#\s*rules?\s+([0-9a-z, -]+)', src): + for tok in _re.split(r',\s*', m.group(1).strip()): + rng = _re.fullmatch(r'(\d+)\s*-\s*(\d+)', tok.strip()) + if rng: + marked.update(str(n) for n in range(int(rng.group(1)), int(rng.group(2)) + 1)) + elif _re.fullmatch(r'\d+[a-z]?', tok.strip()): + marked.add(tok.strip()) + documented = {r[0] for r in self._rows(ci_select.__doc__)} + missing = sorted(documented - marked, key=lambda s: (int(_re.match(r'\d+', s).group()), s)) + self.assertEqual(missing, [], f'documented rules with no `# rule N` branch marker: {missing}') + + class TestNoTrackedFileIsUnclassified(unittest.TestCase): """Rule 17 (unclassified -> full on both axes) is the fail-open net for paths nobody anticipated. It must stay that way - a wrong `full` costs runner minutes and is @@ -1477,10 +1656,17 @@ class TestBuildPostFilter(unittest.TestCase): # the accepted net for a break outside its #if guard). src/class/bth is the # live instance of this state today; TestClassesWithNoEnablingExample pins the # whole set, so a new one cannot appear unnoticed. - s = ci_select.classify_build(['src/class/vendor/vendor_host.c'], REPO) + # src/class/bth/bth_device.c, a file that EXISTS: the old assertion named + # src/class/vendor/vendor_host.c, deleted by the same branch, so any made-up + # path reached the same branch and the test passed vacuously. + real = os.path.join(REPO, 'src/class/bth/bth_device.c') + self.assertTrue(os.path.isfile(real), 'the case needs a file that exists') + s = ci_select.classify_build(['src/class/bth/bth_device.c'], REPO) self.assertFalse(s['full']) self.assertEqual(s['families'], []) self.assertTrue(any('no contribution' in r for r in s['reasons']), s['reasons']) + # and the reason must name the class, not just any empty answer + self.assertTrue(any('bth' in r for r in s['reasons']), s['reasons']) def test_class_source_with_examples_still_scopes(self): s = ci_select.classify_build(['src/class/cdc/cdc_device.c'], REPO) @@ -2040,14 +2226,14 @@ class TestMcuTokensResolve(unittest.TestCase): # produce, or a rename nobody followed through. `family:samd21` was one of these # until the nine examples/host/*/only.txt files were corrected to samd2x_l2x. # - # The `mcu:` entries are NOT all harmless. MIMXRT10XX/MIMXRT11XX and LPC177X_8X sit - # beside a live token in the same file, so they gate nothing either way. MKL25ZXX - # (device/msc_dual_lun) and SAME5X (device/audio_test) do not: those skips are dead, - # and both examples are built today on the boards their skip file meant to exclude - - # successfully, which is why nobody noticed. Correcting them REMOVES working build - # coverage, so it is a maintainer call, not a drive-by fix. + # The remaining `mcu:` entries sit beside a live token in the same file, so they gate + # nothing either way. MKL25ZXX (7 files) and SAME5X (1) were dead too, but unlike + # these they were the ONLY token for their board - the examples were already being + # built on the very boards those lines meant to exclude. Dropping them is a no-op for + # the build (verified per example) and was chosen over re-pointing, which would have + # removed working coverage. UNREACHABLE_TOKENS = { - 'mcu': {'LPC177X_8X', 'MIMXRT10XX', 'MIMXRT11XX', 'MKL25ZXX', 'SAME5X', 'STM32U3'}, + 'mcu': {'LPC177X_8X', 'MIMXRT10XX', 'MIMXRT11XX', 'STM32U3'}, 'family': set(), 'board': set(), } diff --git a/tools/build.py b/tools/build.py index eeefca22d..0bb366e3d 100755 --- a/tools/build.py +++ b/tools/build.py @@ -356,11 +356,11 @@ def get_family_boards(family, one_random, one_first, examples=None, build_system # the WHOLE preferred list, in order - stopping at entry one would abandon a # curated list for the raw alphabetical order the moment its first board cannot # build the filter, which also moves the board the metrics baseline is keyed on + # the whole preferred list, in order. Unreachable-when-unfiltered: with + # examples is None, buildable() is True and the loop returns on entry one. for b in preferred_list: if buildable(b): return [b] - if preferred_list and examples is None: - return [preferred_list[0]] candidates = [b for b in all_boards if buildable(b)] or all_boards if one_first: return [candidates[0]] diff --git a/tools/ci_select.py b/tools/ci_select.py index 89a0d214c..53fbcd3a1 100755 --- a/tools/ci_select.py +++ b/tools/ci_select.py @@ -13,6 +13,38 @@ JSON: full, boards (name -> 'all' | [tests]), families (bsp families the diff touches, including ones with no rig board - build-only consumers such as /pre-pr sample from these), args (hil_test.py args per config) and args_flasher (the same args split by each board's flasher, for CI legs that split one rig by flasher). + +THE RULE TABLE. First match wins; answers union per family (build) and per board +(HIL). A CARBON COPY of the table in the design spec above - edit both, or +TestRuleTableIsCarbonOfTheSpec fails. `FAM` = the families whose family.cmake +references the changed path (CMake only; make follows it). `DEV`/`HOST`/`DUAL`/ +`TYPEC`/`ALL` are the example role sets. The Build families column is PRE-PRUNE: +_prune_buildable then intersects each family with what it can actually build. + +| # | Changed path | Build families | Build examples | HIL boards → tests | +| 1 | `docs/`, `.claude/`, `*.md`, `*.rst`, `LICENSE` | — | — | — | +| 1b | `.gitignore`, `.clang-format`, `.idea/**`, `test/{fuzz,unit-test}/**`, non-build `.github/**`, packaging manifests | — | — | — | +| 2 | `test/hil/**` | — | — | all boards → all tests | +| 2b | `tools/metrics.py`, `.github/scripts/metrics_*.py` | `ALL` (unchanged — `tinyusb_metrics` runs `metrics.py` as a build target) | `ALL` | — (nothing on the rig runs it) | +| 3 | `src/portable//dcd_*`, `*_device.[ch]` | `FAM` | `DEV`+`DUAL` | `FAM`'s device-role boards → device+dual tests | +| 4 | `src/portable//hcd_*`, `*_host.[ch]` | `FAM` | `HOST`+`DUAL` | `FAM`'s host-role boards → host+dual tests | +| 5 | `src/portable//**` (anything else) | `FAM` | `ALL` | `FAM`'s boards → all their tests | +| 5b | `src/portable//**` where `FAM` is empty | — | — | — (empty resolves to nothing on BOTH axes) | +| 6 | `hw/bsp//**` | that family | `ALL` | that family's boards → all tests (a `boards//` path narrows to that board) | +| 7 | `hw/mcu//**` | `FAM` — empty resolves to nothing (maintainer ruling) | `ALL` | `FAM`'s boards → all tests; empty resolves to nothing (maintainer ruling) ⚠ *see below* | +| 8 | `src/class//*_device.[ch]` | `ALL` | examples enabling `CFG_TUD_` | device-role boards → HIL tests enabling `CFG_TUD_` | +| 9 | `src/class//*_host.[ch]` | `ALL` | examples enabling `CFG_TUH_` | host-role boards → HIL tests enabling `CFG_TUH_` | +| 10 | `src/class//**` (shared header) | `ALL` | either, **plus include-edge classes** | both roles → same, plus include-edge classes | +| 11 | `src/device/**` | `ALL` | `DEV`+`DUAL` | device-role boards → device+dual tests | +| 12 | `src/host/**` | `ALL` | `HOST`+`DUAL` | host-role boards → host+dual tests | +| 12b | `src/typec/**` | `ALL` | examples enabling `CFG_TUC_ENABLED` | — (no rig board runs a typec test) | +| 13 | `examples///**` | `ALL` | just `` | if `` is a HIL test: all boards → that test; else nothing | +| 14 | `examples/device/board_test/**` | `ALL` | just `board_test` | all boards → all tests (HIL parking firmware) | +| 15 | `examples/build_system/**`, `examples/CMakeLists.txt`, `examples//CMakeLists.txt` | `ALL` | `ALL` | all boards → all tests | +| 16 | `src/common/`, `src/osal/`, `src/tusb.[ch]`, `src/tusb_option.h`, `tools/{build,build_utils,ci_select}.py`, `tools/cmake/**`, `src/CMakeLists.txt`, `src/tinyusb.mk`, `hw/bsp/{family_support.{cmake,mk},family_rules.mk,zephyr_board_aliases.cmake,board.c,board_api.h,ansi_escape.h}`, `.github/**`, `.circleci/**` | `ALL` | `ALL` | all boards → all tests | +| 16a | `lib//**` | `ALL` | examples whose own `CMakeLists.txt`/`Makefile` names `lib/` | those examples that are HIL tests, on all boards; empty resolves to nothing | +| 16b | `tools/get_deps.py` | families whose `deps_mandatory`/`deps_optional` entries changed | `ALL` | those families' boards → all tests; a logic change, an `'all'` entry, no base content or a changed token naming no family → full | +| 17 | anything unclassified (no tracked file reaches this — TestNoTrackedFileIsUnclassified) | `ALL` | `ALL` | all boards → all tests (fail-open) | """ import argparse import ast @@ -53,7 +85,10 @@ def _read(path: str) -> str: _NONCODE_RE = re.compile( - r'^(docs/|\.claude/|.*\.(md|rst)$|LICENSE)') + # LICENSE is anchored and LICENSES/ named separately: a bare `LICENSE` alternative + # also swallowed anything merely STARTING with it (a future LICENSE_extra.c), + # which is the silent-under-selection direction + r'^(docs/|\.claude/|.*\.(md|rst)$|LICENSE$|LICENSES/)') # Repo metadata and tooling that no CI build reads. Enumerated rather than left to # rule 17, which widens BOTH axes: a PR touching only .gitignore and a README was # creating 74 cmake legs (each a runner doing checkout + toolchain + get_deps before @@ -152,10 +187,20 @@ def board_tests(board: dict) -> list: return [x for x in run if x not in t.get('skip', [])] + +def _rg(repo_root: str, *parts: str) -> str: + """A glob pattern rooted at repo_root, with the ROOT escaped and the parts left as + patterns. The root is a filesystem path, not a pattern: a checkout at + /w/pr[1]/tinyusb (a worktree named after a PR, a CI workspace with brackets) makes + an unescaped '[1]' a character class that matches nothing, and every lookup below + then resolves to zero - families=0 instead of 30, i.e. the selector fails CLOSED + and the whole matrix compiles nothing while reporting green.""" + return os.path.join(glob.escape(repo_root), *parts) + # cached: called per changed file x roster board, and the tree doesn't change mid-run @functools.lru_cache(maxsize=None) def board_family(board_name: str, repo_root: str): - hits = glob.glob(os.path.join(repo_root, 'hw/bsp/*/boards', board_name)) + hits = glob.glob(_rg(repo_root, 'hw/bsp/*/boards', board_name)) return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None @@ -263,10 +308,10 @@ def _family_file_texts(repo_root: str) -> tuple: CMakeLists.txt, read once. path_families is called per distinct directory in the diff and its own cache only helps repeats: a 6,000-file hw/mcu dep bump re-read these 84 files 99,892 times (2.2 s) before this.""" - bsp_root = os.path.join(repo_root, 'hw/bsp') + bsp_root = os.path.join(repo_root, 'hw/bsp') # escaped by _rg below out = [] - for f in sorted(glob.glob(os.path.join(bsp_root, '*/family.cmake')) + - glob.glob(os.path.join(bsp_root, '*/components/*/CMakeLists.txt'))): + for f in sorted(glob.glob(_rg(bsp_root, '*/family.cmake')) + + glob.glob(_rg(bsp_root, '*/components/*/CMakeLists.txt'))): try: out.append((os.path.relpath(f, bsp_root).split(os.sep, 1)[0], _read(f))) except OSError: @@ -386,7 +431,7 @@ def class_include_edges(repo_root: str) -> dict: Derived from the actual #include lines rather than a hand-written table so it cannot rot when a class picks up or drops a cross-class include.""" edges = {} - for f in sorted(glob.glob(os.path.join(repo_root, 'src/class/*/*.[ch]'))): + for f in sorted(glob.glob(_rg(repo_root, 'src/class/*/*.[ch]'))): cls = os.path.basename(os.path.dirname(f)) try: text = _read(f) @@ -470,11 +515,23 @@ def _class_roles(base: str) -> set: return {'device', 'host'} -def _config_enables(cfg_path: str, macros) -> bool: +@functools.lru_cache(maxsize=None) +def _config_text(cfg_path: str) -> str: + """An example's tusb_config.h, read once. Every class path re-asks the same 46 + configs on both axes, so the reads go up with the diff: 4,240 of the same 46 files + for a diff touching all of src/class (0.48s -> 0.13s), and they cannot change + mid-run. Cached here rather than on _config_enables so the macros argument stays an + ordinary list at every call site.""" try: with open(cfg_path, encoding='utf-8', errors='replace') as f: - text = f.read() + return f.read() except OSError: + return '' + + +def _config_enables(cfg_path: str, macros) -> bool: + text = _config_text(cfg_path) + if not text: return False for m in macros: for value in re.findall(_DEF_VALUE.format(m), text, re.M): @@ -511,10 +568,13 @@ def lib_examples(lib_name: str, repo_root: str) -> set: pat = re.compile(re.escape('lib/' + lib_name) + r'(?=[/\s"\')}]|$)', re.M) out = set() for ex in all_examples(repo_root): - for f in sorted(glob.glob(os.path.join(repo_root, 'examples', ex, '**', '*'), + # the two filenames directly: '**/*' enumerated 489 entries per lib against a + # clean tree to use 107, and grows without bound once `make BOARD=... all` has + # written examples///_build/ - which is where /pre-pr runs + for f in sorted(glob.glob(_rg(repo_root, 'examples', ex, '**', 'CMakeLists.txt'), + recursive=True) + + glob.glob(_rg(repo_root, 'examples', ex, '**', 'Makefile'), recursive=True)): - if os.path.basename(f) not in ('CMakeLists.txt', 'Makefile'): - continue try: text = _read(f) except OSError: @@ -580,7 +640,7 @@ def _classify_one(path, repo_root, roster_boards, extras: set, s: _Sel, if _NONCODE_RE.match(path) or _META_RE.match(path): s.reasons.append(f'{path}: non-code, no contribution') return - if _METRICS_RE.match(path): + if _METRICS_RE.match(path): # rule 2b s.reasons.append(f'{path}: build-size metrics tooling, no HIL contribution') return if _FULL_RE.match(path): @@ -903,7 +963,13 @@ def main(): print(f'ci_select[build]: {r}', file=sys.stderr) for r in s['reasons']: print(f'ci_select: {r}', file=sys.stderr) - print(json.dumps(s)) + # reasons go to stderr ONLY - they are a human diagnostic and no consumer reads them + # back. They are also ~97% of the payload (a whole-tree diff: 453 KB -> 12 KB), which + # build.yml re-parses with ci_set_matrix, hil_ci_set_matrix, an inline python and + # three jq calls. The in-process dicts still carry them, for the log and the tests. + out = {k: v for k, v in s.items() if k != 'reasons'} + out['build'] = {k: v for k, v in s['build'].items() if k != 'reasons'} + print(json.dumps(out)) # ------------------------------------------------------------- @@ -925,7 +991,7 @@ def all_examples(repo_root: str) -> tuple: """Every examples// with a CMakeLists.txt, as 'role/name'.""" out = [] for role in _EX_ROLES: - for d in sorted(glob.glob(os.path.join(repo_root, 'examples', role, '*/'))): + for d in sorted(glob.glob(_rg(repo_root, 'examples', role, '*/'))): if os.path.isfile(os.path.join(d, 'CMakeLists.txt')): out.append(f'{role}/{os.path.basename(d.rstrip(os.sep))}') return tuple(out) @@ -979,13 +1045,13 @@ class _BSel: def _classify_build_one(path, repo_root, s: _BSel, get_deps_families=None): base = os.path.basename(path) - if _NONCODE_RE.match(path) or _META_RE.match(path): # rule 1 + if _NONCODE_RE.match(path) or _META_RE.match(path): # rules 1, 1b s.reasons.append(f'{path}: non-code, no build contribution') return if re.match(r'test/hil/', path): # rule 2 s.reasons.append(f'{path}: HIL harness, no build contribution') return - if path == GET_DEPS_PATH: # get_deps rule + if path == GET_DEPS_PATH: # rule 16b if get_deps_families is None: s.force_full(f'{path}: dep changes not resolvable -> full build matrix') return @@ -1002,6 +1068,7 @@ def _classify_build_one(path, repo_root, s: _BSel, get_deps_families=None): roles = _port_roles(base) exs = 'all' if roles == {'device', 'host'} else \ role_examples(repo_root, tuple(roles) + ('dual',)) + # rule 5b: fams empty -> s.add iterates nothing -> no contribution s.add(fams, exs, f'{path}: port {port} -> families {sorted(fams)}') return if re.match(r'hw/bsp/[^/]+/', path): # rule 6 @@ -1064,7 +1131,7 @@ def _classify_build_one(path, repo_root, s: _BSel, get_deps_families=None): s.add(all_bsp_families(repo_root), exs, f'{path}: typec -> {sorted(exs)}') return m = re.match(r'lib/([^/]+)/', path) - if m: # lib rule + if m: # rule 16a lib = m.group(1) exs = lib_examples(lib, repo_root) if not exs: @@ -1150,11 +1217,25 @@ def _prune_buildable(fams, fam_ex, repo_root): # for anything else spins up CI's most expensive leg to skip every example # it was given. Identical to the unfiltered list on all 81 other families. pool = set(build_py.get_examples(fam)) + + # asked per example instead of materialising the family's whole buildable + # list: skip_example is by far the hottest call in the selector, and every + # question below short-circuits (one cdc_device.c diff: 6,883 calls -> 1,889) + def can_build(ex): + # EITHER build system: this one list gates CircleCI's make legs too, and + # the two answer differently (build_utils.skip_example) + return ex in pool and any( + not build_utils.skip_example(ex, b) or + not build_utils.skip_example(ex, b, (), 'make') for b in boards) + + want = fam_ex.get(fam) try: - buildable = [e for e in allex if e in pool and - any(not build_utils.skip_example(e, b) or - not build_utils.skip_example(e, b, (), 'make') - for b in boards)] + if want is None: + kept = None if any(can_build(e) for e in allex) else [] + else: + kept = [e for e in want if can_build(e)] + if kept and not any(can_build(e) for e in allex if e not in want): + kept = None # already everything the family can build except OSError as e: # a family mid-bring-up (boards/ but no family.cmake/family.mk yet) # reads as unbuildable to the scrape; keep it rather than tracebacking @@ -1162,13 +1243,10 @@ def _prune_buildable(fams, fam_ex, repo_root): reasons.append(f'{fam}: mcu scrape unreadable ({e}), kept unfiltered') out_fams.append(fam) continue - want = fam_ex.get(fam) - have = set(buildable) - kept = buildable if want is None else [e for e in want if e in have] - if not kept: + if kept == []: continue # this diff builds nothing for this family out_fams.append(fam) - if set(kept) != set(buildable): + if kept is not None: out_ex[fam] = kept return out_fams, out_ex, reasons -- cgit v1.3.1