diff options
Diffstat (limited to 'test')
| -rw-r--r-- | test/hil/test/test_ci_metrics.py | 18 | ||||
| -rw-r--r-- | test/hil/test/test_ci_select.py | 47 |
2 files changed, 60 insertions, 5 deletions
diff --git a/test/hil/test/test_ci_metrics.py b/test/hil/test/test_ci_metrics.py index aac251824..6f1511913 100644 --- a/test/hil/test/test_ci_metrics.py +++ b/test/hil/test/test_ci_metrics.py @@ -532,7 +532,12 @@ class TestWorkflowSelectionHandOff(unittest.TestCase): --one-first with no -e returns preferred_list[0]; with one it returns the first preferred board that can build it. Where those differ, the Membrowse Upload step - configures a build dir the Build step never wrote.""" + configures a build dir the Build step never wrote. + + ci=True unconditionally, as _prune_buildable does and for the same reason: the + answer must be the runner's, not the developer's. The CI skip lists are off by + default locally, which moves the pick on three families - this test asserted the + local set and went red on its first CI run.""" sys.path.insert(0, os.path.join(REPO, 'tools')) import build as build_py roles = ('device', 'host', 'dual') @@ -547,14 +552,16 @@ class TestWorkflowSelectionHandOff(unittest.TestCase): diverging = set() for fam in fams: try: - base = build_py.get_family_boards(fam, False, True, None, 'cmake', ()) + base = build_py.get_family_boards(fam, False, True, None, 'cmake', + (), ci=True) except Exception: continue if not base: continue for e in exs: try: - one = build_py.get_family_boards(fam, False, True, [e], 'cmake', ()) + one = build_py.get_family_boards(fam, False, True, [e], 'cmake', + (), ci=True) except Exception: continue if one and one[0] != base[0]: @@ -562,8 +569,9 @@ class TestWorkflowSelectionHandOff(unittest.TestCase): break finally: os.chdir(cwd) - self.assertEqual(diverging, {'imxrt', 'lpc11', 'lpc18', 'lpc54', 'mcx', 'rp2040', - 'rx', 'samd11', 'stm32l0', 'stm32l4', 'tm4c'}, + self.assertEqual(diverging, {'imxrt', 'lpc11', 'lpc18', 'lpc54', 'mcx', 'rx', + 'samd11', 'samd2x_l2x', 'samd5x_e5x', 'stm32l0', + 'stm32l4', 'tm4c'}, 'the set of families whose membrowse upload can land on an ' 'uncompiled board changed; re-check whether dropping $EX_ARGS ' 'from the upload step is still the right trade') diff --git a/test/hil/test/test_ci_select.py b/test/hil/test/test_ci_select.py index dc10f769a..c34bccd1f 100644 --- a/test/hil/test/test_ci_select.py +++ b/test/hil/test/test_ci_select.py @@ -942,6 +942,53 @@ class TestClassesWithNoEnablingExample(unittest.TestCase): 'both axes, so nothing compiles it until the next master push') +class TestTheHarnessTestsAreNotTheHarness(unittest.TestCase): + """test/hil/test/ selects nothing; test/hil/ itself still selects everything. + + Rule 2 is a bare `test/hil/` prefix, so the harness's own unit tests were booking + the full 27-board rig - ~11 minutes of exclusive hardware for a diff that cannot + reach it. Nothing on the rig runs them: pre-commit does, and build.yml runs + test_ci_select.py as the gate before trusting a selection at all. + + The carve-out is only safe while that directory holds nothing rig-affecting, which + is what the second test pins.""" + + def test_the_harness_own_tests_select_nothing_on_either_axis(self): + for p in ('test/hil/test/test_ci_select.py', 'test/hil/test/test_ci_metrics.py', + 'test/hil/test/test_hil_bounded.py', 'test/hil/test/stubs/pymtp.py'): + s = ci_select.classify([p], REPO, ROSTERS) + self.assertFalse(s['full'], p) + self.assertFalse(s['boards'], p) + b = ci_select.classify_build([p], REPO) + self.assertFalse(b['full'], p) + self.assertFalse(b['families'], p) + + def test_the_harness_itself_still_takes_the_whole_rig(self): + # the thing rule 2 exists for: these decide what the rig does, so they cannot be + # trusted to narrow their own blast radius + for p in ('test/hil/hil_test.py', 'test/hil/tinyusb.json', + 'test/hil/helper/hil_ci_set_matrix.py'): + s = ci_select.classify([p], REPO, ROSTERS) + self.assertTrue(s['full'], f'{p} must still force the full rig') + + def test_nothing_rig_affecting_has_moved_into_the_carve_out(self): + """The carve-out is a claim about that directory's contents; pin them. + + A new file there that the rig DOES read would silently stop selecting the rig. + Listing them costs one line per file and makes that a failing test instead.""" + out = subprocess.run(['git', 'ls-files', 'test/hil/test'], cwd=REPO, + capture_output=True, text=True, check=True) + self.assertEqual(sorted(out.stdout.split()), [ + 'test/hil/test/stubs/pymtp.py', + 'test/hil/test/test_ci_metrics.py', + 'test/hil/test/test_ci_select.py', + 'test/hil/test/test_hil_bounded.py', + 'test/hil/test/test_hil_health.py', + 'test/hil/test/test_hil_util.py', + ], 'test/hil/test/ gained or lost a file; it is carved out of rule 2, so confirm ' + 'the rig still does not read anything in there before updating this list') + + class TestExampleMapOmitsFullFamilies(unittest.TestCase): """A family whose selection is ALREADY everything it can build carries no -e list. |