diff options
| -rw-r--r-- | docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md | 10 | ||||
| -rw-r--r-- | test/hil/test/test_ci_metrics.py | 18 | ||||
| -rw-r--r-- | test/hil/test/test_ci_select.py | 47 | ||||
| -rwxr-xr-x | tools/ci_select.py | 10 |
4 files changed, 75 insertions, 10 deletions
diff --git a/docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md b/docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md index 524568aeb..799c83c23 100644 --- a/docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md +++ b/docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md @@ -49,8 +49,8 @@ never inflates one axis with another's breadth. | # | Changed path | Build families | Build examples | HIL boards → tests | | --- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------- | ----------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | | 1 | `docs/`, `.claude/`, `*.md`, `*.rst`, `LICENSE` | — | — | — | -| 1b | `.gitignore`, `.clang-format`, `.idea/**`, `test/{fuzz,unit-test}/**`, non-build `.github/**`, packaging manifests | — | — | — | -| 2 | `test/hil/**` | — | — | all boards → all tests | +| 1b | `.gitignore`, `.clang-format`, `.idea/**`, `test/{fuzz,unit-test}/**`, `test/hil/test/**`, non-build `.github/**`, packaging manifests | — | — | — | +| 2 | `test/hil/**` (not `test/hil/test/**`) | — | — | all boards → all tests | | 2b | `tools/metrics.py`, `.github/scripts/metrics_*.py` | `ALL` (unchanged — `tinyusb_metrics` runs `metrics.py` as a build target) | `ALL` | — (nothing on the rig runs it) | | 3 | `src/portable/<port>/dcd_*`, `*_device.[ch]` | `FAM` | `DEV`+`DUAL` | `FAM`'s device-role boards → device+dual tests | | 4 | `src/portable/<port>/hcd_*`, `*_host.[ch]` | `FAM` | `HOST`+`DUAL` | `FAM`'s host-role boards → host+dual tests | @@ -75,6 +75,12 @@ never inflates one axis with another's breadth. **Rule 2 is deliberately asymmetric.** A `test/hil/**` change is invisible to the family matrix but is exactly what the rig exercises, so it builds nothing and runs everything. +`test/hil/test/**` is carved out to rule 1b: it holds the harness's own unit tests, which +nothing on the rig runs (pre-commit does, and `build.yml` runs `test_ci_select.py` as the +gate before trusting a selection). A bare `test/hil/` prefix was booking the full 27-board +rig for diffs that cannot reach it. The carve-out is a claim about that directory's +contents, so a test pins its file list: add anything the rig reads and it fails. + **Rule 7 is the one HIL-side behaviour change in this design.** Today `hw/mcu/` sits in `hil_select`'s `_FULL_RE` and forces the full HIL matrix. Since the build axis now resolves those paths to a family through the same scan, forcing full on the rig is inconsistent. The diff --git a/test/hil/test/test_ci_metrics.py b/test/hil/test/test_ci_metrics.py index aac251824..6f1511913 100644 --- a/test/hil/test/test_ci_metrics.py +++ b/test/hil/test/test_ci_metrics.py @@ -532,7 +532,12 @@ class TestWorkflowSelectionHandOff(unittest.TestCase): --one-first with no -e returns preferred_list[0]; with one it returns the first preferred board that can build it. Where those differ, the Membrowse Upload step - configures a build dir the Build step never wrote.""" + configures a build dir the Build step never wrote. + + ci=True unconditionally, as _prune_buildable does and for the same reason: the + answer must be the runner's, not the developer's. The CI skip lists are off by + default locally, which moves the pick on three families - this test asserted the + local set and went red on its first CI run.""" sys.path.insert(0, os.path.join(REPO, 'tools')) import build as build_py roles = ('device', 'host', 'dual') @@ -547,14 +552,16 @@ class TestWorkflowSelectionHandOff(unittest.TestCase): diverging = set() for fam in fams: try: - base = build_py.get_family_boards(fam, False, True, None, 'cmake', ()) + base = build_py.get_family_boards(fam, False, True, None, 'cmake', + (), ci=True) except Exception: continue if not base: continue for e in exs: try: - one = build_py.get_family_boards(fam, False, True, [e], 'cmake', ()) + one = build_py.get_family_boards(fam, False, True, [e], 'cmake', + (), ci=True) except Exception: continue if one and one[0] != base[0]: @@ -562,8 +569,9 @@ class TestWorkflowSelectionHandOff(unittest.TestCase): break finally: os.chdir(cwd) - self.assertEqual(diverging, {'imxrt', 'lpc11', 'lpc18', 'lpc54', 'mcx', 'rp2040', - 'rx', 'samd11', 'stm32l0', 'stm32l4', 'tm4c'}, + self.assertEqual(diverging, {'imxrt', 'lpc11', 'lpc18', 'lpc54', 'mcx', 'rx', + 'samd11', 'samd2x_l2x', 'samd5x_e5x', 'stm32l0', + 'stm32l4', 'tm4c'}, 'the set of families whose membrowse upload can land on an ' 'uncompiled board changed; re-check whether dropping $EX_ARGS ' 'from the upload step is still the right trade') diff --git a/test/hil/test/test_ci_select.py b/test/hil/test/test_ci_select.py index dc10f769a..c34bccd1f 100644 --- a/test/hil/test/test_ci_select.py +++ b/test/hil/test/test_ci_select.py @@ -942,6 +942,53 @@ class TestClassesWithNoEnablingExample(unittest.TestCase): 'both axes, so nothing compiles it until the next master push') +class TestTheHarnessTestsAreNotTheHarness(unittest.TestCase): + """test/hil/test/ selects nothing; test/hil/ itself still selects everything. + + Rule 2 is a bare `test/hil/` prefix, so the harness's own unit tests were booking + the full 27-board rig - ~11 minutes of exclusive hardware for a diff that cannot + reach it. Nothing on the rig runs them: pre-commit does, and build.yml runs + test_ci_select.py as the gate before trusting a selection at all. + + The carve-out is only safe while that directory holds nothing rig-affecting, which + is what the second test pins.""" + + def test_the_harness_own_tests_select_nothing_on_either_axis(self): + for p in ('test/hil/test/test_ci_select.py', 'test/hil/test/test_ci_metrics.py', + 'test/hil/test/test_hil_bounded.py', 'test/hil/test/stubs/pymtp.py'): + s = ci_select.classify([p], REPO, ROSTERS) + self.assertFalse(s['full'], p) + self.assertFalse(s['boards'], p) + b = ci_select.classify_build([p], REPO) + self.assertFalse(b['full'], p) + self.assertFalse(b['families'], p) + + def test_the_harness_itself_still_takes_the_whole_rig(self): + # the thing rule 2 exists for: these decide what the rig does, so they cannot be + # trusted to narrow their own blast radius + for p in ('test/hil/hil_test.py', 'test/hil/tinyusb.json', + 'test/hil/helper/hil_ci_set_matrix.py'): + s = ci_select.classify([p], REPO, ROSTERS) + self.assertTrue(s['full'], f'{p} must still force the full rig') + + def test_nothing_rig_affecting_has_moved_into_the_carve_out(self): + """The carve-out is a claim about that directory's contents; pin them. + + A new file there that the rig DOES read would silently stop selecting the rig. + Listing them costs one line per file and makes that a failing test instead.""" + out = subprocess.run(['git', 'ls-files', 'test/hil/test'], cwd=REPO, + capture_output=True, text=True, check=True) + self.assertEqual(sorted(out.stdout.split()), [ + 'test/hil/test/stubs/pymtp.py', + 'test/hil/test/test_ci_metrics.py', + 'test/hil/test/test_ci_select.py', + 'test/hil/test/test_hil_bounded.py', + 'test/hil/test/test_hil_health.py', + 'test/hil/test/test_hil_util.py', + ], 'test/hil/test/ gained or lost a file; it is carved out of rule 2, so confirm ' + 'the rig still does not read anything in there before updating this list') + + class TestExampleMapOmitsFullFamilies(unittest.TestCase): """A family whose selection is ALREADY everything it can build carries no -e list. diff --git a/tools/ci_select.py b/tools/ci_select.py index 53fbcd3a1..ca9d54c27 100755 --- a/tools/ci_select.py +++ b/tools/ci_select.py @@ -23,8 +23,8 @@ _prune_buildable then intersects each family with what it can actually build. | # | Changed path | Build families | Build examples | HIL boards → tests | | 1 | `docs/`, `.claude/`, `*.md`, `*.rst`, `LICENSE` | — | — | — | -| 1b | `.gitignore`, `.clang-format`, `.idea/**`, `test/{fuzz,unit-test}/**`, non-build `.github/**`, packaging manifests | — | — | — | -| 2 | `test/hil/**` | — | — | all boards → all tests | +| 1b | `.gitignore`, `.clang-format`, `.idea/**`, `test/{fuzz,unit-test}/**`, `test/hil/test/**`, non-build `.github/**`, packaging manifests | — | — | — | +| 2 | `test/hil/**` (not `test/hil/test/**`) | — | — | all boards → all tests | | 2b | `tools/metrics.py`, `.github/scripts/metrics_*.py` | `ALL` (unchanged — `tinyusb_metrics` runs `metrics.py` as a build target) | `ALL` | — (nothing on the rig runs it) | | 3 | `src/portable/<port>/dcd_*`, `*_device.[ch]` | `FAM` | `DEV`+`DUAL` | `FAM`'s device-role boards → device+dual tests | | 4 | `src/portable/<port>/hcd_*`, `*_host.[ch]` | `FAM` | `HOST`+`DUAL` | `FAM`'s host-role boards → host+dual tests | @@ -108,7 +108,11 @@ _META_RE = re.compile( r'version\.yml$|SConscript$|' r'.*CMakePresets\.json$|hw/bsp/BoardPresets\.json$|examples/west\.yml$|' r'.*/[0-9]+-tinyusb[^/]*\.rules$|tools/usb_drivers/|tools/codespell/|' - r'test/(fuzz|unit-test)/|' + # test/hil/test/ holds the harness's own unit tests, not the harness: nothing on + # the rig runs them (pre-commit does, and build.yml runs test_ci_select.py as the + # gate before trusting a selection), so they cannot change what the rig does. + # The harness itself stays under _FULL_RE's test/hil/ prefix. + r'test/(fuzz|unit-test)/|test/hil/test/|' # .github, minus the build machinery named in _FULL_RE r'\.github/(FUNDING\.yml$|labeler\.yml$|membrowse_pr_message\.j2$|ISSUE_TEMPLATE/|' r'workflows/(cifuzz|claude|claude-code-review|labeler|membrowse-comment|' |