diff options
| -rwxr-xr-x | .github/scripts/ci_set_matrix.py | 20 | ||||
| -rw-r--r-- | test/hil/test/test_ci_metrics.py | 13 | ||||
| -rw-r--r-- | test/hil/test/test_ci_select.py | 36 |
3 files changed, 56 insertions, 13 deletions
diff --git a/.github/scripts/ci_set_matrix.py b/.github/scripts/ci_set_matrix.py index ee3609bed..79f466893 100755 --- a/.github/scripts/ci_set_matrix.py +++ b/.github/scripts/ci_set_matrix.py @@ -127,12 +127,22 @@ def set_matrix_json(select=None): if sel_fams is not None: fams = [f for f in fams if f in sel_fams] matrix[toolchain] = fams - if sel_fams is not None: - # a family this file does not list builds on no toolchain, so the selection maps - # to an empty matrix and every leg skips - which looks exactly like a working - # scoped run. Say so: hw/bsp holds several families CI has never built - # (efm32, py32f0, ...) and espressif, whose boards are built by hil-build-esp + if sel_fams: + # a family this file does not list builds on no toolchain, so it contributes no + # leg. hw/bsp holds several CI has never built (efm32, py32f0, same7x, ...) plus + # espressif, whose boards hil-build-esp builds by name. unbuilt = sorted(f for f in sel_fams if f not in family_list) + if unbuilt and not any(matrix.values()): + # NONE of the selected families is buildable here, so every leg would skip + # and the PR would go green from a build job that ran no compiler. That is + # an unusable selection, not "nothing selected": say UNSCOPED - which + # build.yml and .circleci/config.yml both grep for - and emit the full + # matrix. An explicit families: [] is still a legitimate nothing-selected, + # and a PARTIAL miss still scopes to the families that do build. + print(f'ci_set_matrix: UNSCOPED - no selected family is built by any ' + f'toolchain here ({", ".join(unbuilt)}), emitting the full matrix', + file=sys.stderr) + return set_matrix_json(None) if unbuilt: print(f'ci_set_matrix: selected families built by no toolchain here: ' f'{", ".join(unbuilt)}', file=sys.stderr) diff --git a/test/hil/test/test_ci_metrics.py b/test/hil/test/test_ci_metrics.py index 6c236e827..a76b6e3a0 100644 --- a/test/hil/test/test_ci_metrics.py +++ b/test/hil/test/test_ci_metrics.py @@ -435,8 +435,17 @@ class TestWorkflowSelectionHandOff(unittest.TestCase): self.assertIn('BUILD_SELECT_FILE', self.build) scripts = os.path.join(os.path.dirname(CIRCLECI), '.github', 'scripts') matrix = open(os.path.join(scripts, 'ci_set_matrix.py')).read() - self.assertEqual(matrix.count('ci_set_matrix: UNSCOPED'), 2, - 'every fall-open path must print the marker build.yml greps for') + # count-independent: pin the INVARIANT, not the number of fall-open paths - + # every message that emits the full matrix must carry the marker, and a purely + # informational note (a partial family miss) must not claim to have done so. + # Adjacent string literals are joined first, since these messages wrap. + import re as _re + flat = _re.sub(r"['\"]\s*\n\s*f?['\"]", '', matrix) + hits = [m.start() for m in _re.finditer('emitting the full ', flat)] + self.assertGreaterEqual(len(hits), 2, 'fall-open messages not found') + for i in hits: + self.assertIn('UNSCOPED', flat[max(0, i - 200):i], + 'a fall-open path without the marker build.yml greps for') def test_membrowse_upload_sees_the_same_board_as_the_build(self): # $EX_ARGS is passed for the BOARD it selects: --one-first picks a board that can diff --git a/test/hil/test/test_ci_select.py b/test/hil/test/test_ci_select.py index f5c64ac1c..8f1841531 100644 --- a/test/hil/test/test_ci_select.py +++ b/test/hil/test/test_ci_select.py @@ -777,12 +777,15 @@ class TestOrphanInvariant(unittest.TestCase): for v in vendors: self.assertTrue(ci_select.mcu_families(v + '/x.c', REPO), f'{v}: resolves to no family') - # hw/bsp families ci_set_matrix's family_list does not map to any toolchain. Before - # scoping these were harmless - the matrix was always every family in family_list, - # so a PR touching one of them still compiled the other 64. Now the selection - # intersects to nothing and every leg skips, so a family landing here by accident is - # a silent hole. espressif is deliberate: its boards are built by hil-build-esp, - # keyed on board name rather than family. + # hw/bsp families ci_set_matrix's family_list does not map to any toolchain. Master + # gave a PR touching one of these no compile coverage either - none of the other 64 + # families compiles same7x's board.h - so this is not new. What IS new is that the + # gap used to be masked by a full matrix and is now the whole answer, which is why + # ci_set_matrix treats a selection that intersects family_list to NOTHING as + # unusable (UNSCOPED -> full matrix) rather than emitting an all-empty one. + # espressif is here because hil-build-esp builds its boards by name rather than by + # family - though only on hathach/tinyusb: that job is gated on repository_owner, + # so on a fork an espressif-only PR builds nowhere. UNBUILT_FAMILIES = {'cxd56', 'efm32', 'espressif', 'f1c100s', 'pic32mz', 'py32f0', 'same7x'} @@ -1505,6 +1508,27 @@ class TestCiSetMatrix(unittest.TestCase): self.assertEqual(json.loads(r.stdout), base) self.assertIn('ci_set_matrix: UNSCOPED', r.stderr) # build.yml greps this + def test_families_no_toolchain_builds_falls_open(self): + # hw/bsp/same7x is real but in no toolchain's list, so scoping to it emits an + # all-empty matrix: every leg skips and the PR goes green from a build job that + # ran no compiler. Unusable, not "nothing selected" - and the marker matters, + # because that is what build.yml and CircleCI grep to drop the build extras too. + base = json.loads(self.run_matrix().stdout) + r = self.run_matrix('--select', + json.dumps({'build': {'full': False, 'families': ['same7x']}})) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertEqual(json.loads(r.stdout), base) + self.assertIn('ci_set_matrix: UNSCOPED', r.stderr) + + def test_a_partial_toolchain_miss_still_scopes(self): + # one buildable family is real coverage: scope to it and just note the other + r = self.run_matrix('--select', json.dumps( + {'build': {'full': False, 'families': ['stm32f4', 'same7x']}})) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertEqual(json.loads(r.stdout)['arm-gcc'], ['stm32f4']) + self.assertNotIn('UNSCOPED', r.stderr) + self.assertIn('same7x', r.stderr) + def test_explicit_empty_families_selects_nothing(self): # an explicit [] IS a legitimate answer (a diff that builds nothing) r = self.run_matrix('--select', |