From b610ff039bafa1040c19d6a11cb04adcb22936e5 Mon Sep 17 00:00:00 2001 From: Ha Thach Date: Tue, 25 Aug 2026 10:35:45 +0700 Subject: ci_select: fix the membrowse test's env dependence, and stop HIL unit tests taking the rig (#3846) test_the_upload_board_can_diverge_from_the_built_board called get_family_boards without ci=True, so it pinned the developer's set, not the runner's: the CI skip lists move the one-first pick on three families. It held locally and went red on its first CI run. Pass ci=True, as _prune_buildable already does, and pin the runner's twelve. Rule 2 is a bare test/hil/ prefix, so the harness's own unit tests booked the full 27-board rig for diffs that cannot reach it. Carve test/hil/test/** out to rule 1b, beside test/{fuzz,unit-test}/**; the harness itself is untouched. A test pins that directory's file list, so anything added there that the rig does read fails rather than silently skipping hardware. Rule table updated in the spec and its carbon in the docstring. --- .../2026-08-19-ci-build-family-filter-design.md | 10 ++++- test/hil/test/test_ci_metrics.py | 18 ++++++--- test/hil/test/test_ci_select.py | 47 ++++++++++++++++++++++ tools/ci_select.py | 10 +++-- 4 files changed, 75 insertions(+), 10 deletions(-) diff --git a/docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md b/docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md index 524568aeb..799c83c23 100644 --- a/docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md +++ b/docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md @@ -49,8 +49,8 @@ never inflates one axis with another's breadth. | # | Changed path | Build families | Build examples | HIL boards → tests | | --- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------- | ----------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | | 1 | `docs/`, `.claude/`, `*.md`, `*.rst`, `LICENSE` | — | — | — | -| 1b | `.gitignore`, `.clang-format`, `.idea/**`, `test/{fuzz,unit-test}/**`, non-build `.github/**`, packaging manifests | — | — | — | -| 2 | `test/hil/**` | — | — | all boards → all tests | +| 1b | `.gitignore`, `.clang-format`, `.idea/**`, `test/{fuzz,unit-test}/**`, `test/hil/test/**`, non-build `.github/**`, packaging manifests | — | — | — | +| 2 | `test/hil/**` (not `test/hil/test/**`) | — | — | all boards → all tests | | 2b | `tools/metrics.py`, `.github/scripts/metrics_*.py` | `ALL` (unchanged — `tinyusb_metrics` runs `metrics.py` as a build target) | `ALL` | — (nothing on the rig runs it) | | 3 | `src/portable//dcd_*`, `*_device.[ch]` | `FAM` | `DEV`+`DUAL` | `FAM`'s device-role boards → device+dual tests | | 4 | `src/portable//hcd_*`, `*_host.[ch]` | `FAM` | `HOST`+`DUAL` | `FAM`'s host-role boards → host+dual tests | @@ -75,6 +75,12 @@ never inflates one axis with another's breadth. **Rule 2 is deliberately asymmetric.** A `test/hil/**` change is invisible to the family matrix but is exactly what the rig exercises, so it builds nothing and runs everything. +`test/hil/test/**` is carved out to rule 1b: it holds the harness's own unit tests, which +nothing on the rig runs (pre-commit does, and `build.yml` runs `test_ci_select.py` as the +gate before trusting a selection). A bare `test/hil/` prefix was booking the full 27-board +rig for diffs that cannot reach it. The carve-out is a claim about that directory's +contents, so a test pins its file list: add anything the rig reads and it fails. + **Rule 7 is the one HIL-side behaviour change in this design.** Today `hw/mcu/` sits in `hil_select`'s `_FULL_RE` and forces the full HIL matrix. Since the build axis now resolves those paths to a family through the same scan, forcing full on the rig is inconsistent. The diff --git a/test/hil/test/test_ci_metrics.py b/test/hil/test/test_ci_metrics.py index aac251824..6f1511913 100644 --- a/test/hil/test/test_ci_metrics.py +++ b/test/hil/test/test_ci_metrics.py @@ -532,7 +532,12 @@ class TestWorkflowSelectionHandOff(unittest.TestCase): --one-first with no -e returns preferred_list[0]; with one it returns the first preferred board that can build it. Where those differ, the Membrowse Upload step - configures a build dir the Build step never wrote.""" + configures a build dir the Build step never wrote. + + ci=True unconditionally, as _prune_buildable does and for the same reason: the + answer must be the runner's, not the developer's. The CI skip lists are off by + default locally, which moves the pick on three families - this test asserted the + local set and went red on its first CI run.""" sys.path.insert(0, os.path.join(REPO, 'tools')) import build as build_py roles = ('device', 'host', 'dual') @@ -547,14 +552,16 @@ class TestWorkflowSelectionHandOff(unittest.TestCase): diverging = set() for fam in fams: try: - base = build_py.get_family_boards(fam, False, True, None, 'cmake', ()) + base = build_py.get_family_boards(fam, False, True, None, 'cmake', + (), ci=True) except Exception: continue if not base: continue for e in exs: try: - one = build_py.get_family_boards(fam, False, True, [e], 'cmake', ()) + one = build_py.get_family_boards(fam, False, True, [e], 'cmake', + (), ci=True) except Exception: continue if one and one[0] != base[0]: @@ -562,8 +569,9 @@ class TestWorkflowSelectionHandOff(unittest.TestCase): break finally: os.chdir(cwd) - self.assertEqual(diverging, {'imxrt', 'lpc11', 'lpc18', 'lpc54', 'mcx', 'rp2040', - 'rx', 'samd11', 'stm32l0', 'stm32l4', 'tm4c'}, + self.assertEqual(diverging, {'imxrt', 'lpc11', 'lpc18', 'lpc54', 'mcx', 'rx', + 'samd11', 'samd2x_l2x', 'samd5x_e5x', 'stm32l0', + 'stm32l4', 'tm4c'}, 'the set of families whose membrowse upload can land on an ' 'uncompiled board changed; re-check whether dropping $EX_ARGS ' 'from the upload step is still the right trade') diff --git a/test/hil/test/test_ci_select.py b/test/hil/test/test_ci_select.py index dc10f769a..c34bccd1f 100644 --- a/test/hil/test/test_ci_select.py +++ b/test/hil/test/test_ci_select.py @@ -942,6 +942,53 @@ class TestClassesWithNoEnablingExample(unittest.TestCase): 'both axes, so nothing compiles it until the next master push') +class TestTheHarnessTestsAreNotTheHarness(unittest.TestCase): + """test/hil/test/ selects nothing; test/hil/ itself still selects everything. + + Rule 2 is a bare `test/hil/` prefix, so the harness's own unit tests were booking + the full 27-board rig - ~11 minutes of exclusive hardware for a diff that cannot + reach it. Nothing on the rig runs them: pre-commit does, and build.yml runs + test_ci_select.py as the gate before trusting a selection at all. + + The carve-out is only safe while that directory holds nothing rig-affecting, which + is what the second test pins.""" + + def test_the_harness_own_tests_select_nothing_on_either_axis(self): + for p in ('test/hil/test/test_ci_select.py', 'test/hil/test/test_ci_metrics.py', + 'test/hil/test/test_hil_bounded.py', 'test/hil/test/stubs/pymtp.py'): + s = ci_select.classify([p], REPO, ROSTERS) + self.assertFalse(s['full'], p) + self.assertFalse(s['boards'], p) + b = ci_select.classify_build([p], REPO) + self.assertFalse(b['full'], p) + self.assertFalse(b['families'], p) + + def test_the_harness_itself_still_takes_the_whole_rig(self): + # the thing rule 2 exists for: these decide what the rig does, so they cannot be + # trusted to narrow their own blast radius + for p in ('test/hil/hil_test.py', 'test/hil/tinyusb.json', + 'test/hil/helper/hil_ci_set_matrix.py'): + s = ci_select.classify([p], REPO, ROSTERS) + self.assertTrue(s['full'], f'{p} must still force the full rig') + + def test_nothing_rig_affecting_has_moved_into_the_carve_out(self): + """The carve-out is a claim about that directory's contents; pin them. + + A new file there that the rig DOES read would silently stop selecting the rig. + Listing them costs one line per file and makes that a failing test instead.""" + out = subprocess.run(['git', 'ls-files', 'test/hil/test'], cwd=REPO, + capture_output=True, text=True, check=True) + self.assertEqual(sorted(out.stdout.split()), [ + 'test/hil/test/stubs/pymtp.py', + 'test/hil/test/test_ci_metrics.py', + 'test/hil/test/test_ci_select.py', + 'test/hil/test/test_hil_bounded.py', + 'test/hil/test/test_hil_health.py', + 'test/hil/test/test_hil_util.py', + ], 'test/hil/test/ gained or lost a file; it is carved out of rule 2, so confirm ' + 'the rig still does not read anything in there before updating this list') + + class TestExampleMapOmitsFullFamilies(unittest.TestCase): """A family whose selection is ALREADY everything it can build carries no -e list. diff --git a/tools/ci_select.py b/tools/ci_select.py index 53fbcd3a1..ca9d54c27 100755 --- a/tools/ci_select.py +++ b/tools/ci_select.py @@ -23,8 +23,8 @@ _prune_buildable then intersects each family with what it can actually build. | # | Changed path | Build families | Build examples | HIL boards → tests | | 1 | `docs/`, `.claude/`, `*.md`, `*.rst`, `LICENSE` | — | — | — | -| 1b | `.gitignore`, `.clang-format`, `.idea/**`, `test/{fuzz,unit-test}/**`, non-build `.github/**`, packaging manifests | — | — | — | -| 2 | `test/hil/**` | — | — | all boards → all tests | +| 1b | `.gitignore`, `.clang-format`, `.idea/**`, `test/{fuzz,unit-test}/**`, `test/hil/test/**`, non-build `.github/**`, packaging manifests | — | — | — | +| 2 | `test/hil/**` (not `test/hil/test/**`) | — | — | all boards → all tests | | 2b | `tools/metrics.py`, `.github/scripts/metrics_*.py` | `ALL` (unchanged — `tinyusb_metrics` runs `metrics.py` as a build target) | `ALL` | — (nothing on the rig runs it) | | 3 | `src/portable//dcd_*`, `*_device.[ch]` | `FAM` | `DEV`+`DUAL` | `FAM`'s device-role boards → device+dual tests | | 4 | `src/portable//hcd_*`, `*_host.[ch]` | `FAM` | `HOST`+`DUAL` | `FAM`'s host-role boards → host+dual tests | @@ -108,7 +108,11 @@ _META_RE = re.compile( r'version\.yml$|SConscript$|' r'.*CMakePresets\.json$|hw/bsp/BoardPresets\.json$|examples/west\.yml$|' r'.*/[0-9]+-tinyusb[^/]*\.rules$|tools/usb_drivers/|tools/codespell/|' - r'test/(fuzz|unit-test)/|' + # test/hil/test/ holds the harness's own unit tests, not the harness: nothing on + # the rig runs them (pre-commit does, and build.yml runs test_ci_select.py as the + # gate before trusting a selection), so they cannot change what the rig does. + # The harness itself stays under _FULL_RE's test/hil/ prefix. + r'test/(fuzz|unit-test)/|test/hil/test/|' # .github, minus the build machinery named in _FULL_RE r'\.github/(FUNDING\.yml$|labeler\.yml$|membrowse_pr_message\.j2$|ISSUE_TEMPLATE/|' r'workflows/(cifuzz|claude|claude-code-review|labeler|membrowse-comment|' -- cgit v1.3.1