Merge pull request #3841 from hathach/build-filterHEADmaster

ci: scope the build matrix and HIL run to what a PR affects

3 files changed, 256 insertions, 11 deletions

diff --git a/.github/scripts/ci_set_matrix.py b/.github/scripts/ci_set_matrix.py
index 50ada5964..79f466893 100755
--- a/.github/scripts/ci_set_matrix.py
+++ b/.github/scripts/ci_set_matrix.py
@@ -1,5 +1,9 @@
 #!/usr/bin/env python3
+import argparse
 import json
+import os
+import subprocess
+import sys
 
 # toolchain, url
 toolchain_list = [
@@ -97,15 +101,87 @@ family_list = {
 }
 
 
-def set_matrix_json():
+def set_matrix_json(select=None):
+    sel_fams = None
+    if select:
+        # every shape check is explicit: this runs AFTER main()'s fail-open handler, so
+        # an AttributeError on e.g. {"build": ["stm32f4"]} would red the step instead
+        # of falling back to the full matrix - the outcome that handler exists to prevent
+        b = select.get('build') if isinstance(select, dict) else None
+        if not isinstance(b, dict):
+            b = {}
+        if b.get('full') is False:
+            fams = b.get('families')
+            if not (isinstance(fams, list) and all(isinstance(f, str) for f in fams)):
+                # key ABSENT (or not a list of names) is an unusable selection, not
+                # "nothing selected": scoping every toolchain to [] would build zero
+                # families and report a vacuous green. An explicit families: [] stays a
+                # legitimate nothing-selected.
+                print('ci_set_matrix: UNSCOPED - build.full is false but the families '
+                      'list is unusable, emitting the full matrix', file=sys.stderr)
+            else:
+                sel_fams = set(fams)
     matrix = {}
     for toolchain in toolchain_list:
-        filtered_families = [family for family, supported_toolchain in family_list.items() if
-                             toolchain in supported_toolchain]
-        matrix[toolchain] = filtered_families
-
+        fams = [family for family, tc in family_list.items() if toolchain in tc]
+        if sel_fams is not None:
+            fams = [f for f in fams if f in sel_fams]
+        matrix[toolchain] = fams
+    if sel_fams:
+        # a family this file does not list builds on no toolchain, so it contributes no
+        # leg. hw/bsp holds several CI has never built (efm32, py32f0, same7x, ...) plus
+        # espressif, whose boards hil-build-esp builds by name.
+        unbuilt = sorted(f for f in sel_fams if f not in family_list)
+        if unbuilt and not any(matrix.values()):
+            # NONE of the selected families is buildable here, so every leg would skip
+            # and the PR would go green from a build job that ran no compiler. That is
+            # an unusable selection, not "nothing selected": say UNSCOPED - which
+            # build.yml and .circleci/config.yml both grep for - and emit the full
+            # matrix. An explicit families: [] is still a legitimate nothing-selected,
+            # and a PARTIAL miss still scopes to the families that do build.
+            print(f'ci_set_matrix: UNSCOPED - no selected family is built by any '
+                  f'toolchain here ({", ".join(unbuilt)}), emitting the full matrix',
+                  file=sys.stderr)
+            return set_matrix_json(None)
+        if unbuilt:
+            print(f'ci_set_matrix: selected families built by no toolchain here: '
+                  f'{", ".join(unbuilt)}', file=sys.stderr)
     print(json.dumps(matrix))
 
 
+def main():
+    parser = argparse.ArgumentParser()
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument('--select', help='tools/ci_select.py JSON; scopes families when build.full is false')
+    # a whole selection as one argv/env value can exceed the exec limits on a big
+    # diff, which fails the calling step BEFORE it can fall open; callers that
+    # already have the selection on disk pass the path instead
+    group.add_argument('--select-file', help='file holding the same JSON as --select')
+    group.add_argument('--base', help='git ref: run tools/ci_select.py --base REF and scope from it')
+    args = parser.parse_args()
+
+    select = None
+    try:
+        if args.select:
+            select = json.loads(args.select)
+        elif args.select_file:
+            with open(args.select_file) as f:
+                select = json.load(f)
+        elif args.base:
+            root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+            r = subprocess.run([sys.executable, os.path.join(root, 'tools', 'ci_select.py'),
+                                '--base', args.base],
+                               capture_output=True, text=True, cwd=root, check=True)
+            select = json.loads(r.stdout)
+    except Exception as e:  # fail-open: an unusable selection must never turn into a red job
+        # UNSCOPED is the marker build.yml greps for: it must then drop the build extras
+        # (example map, family regex) too, or a full build gets labelled and filtered as
+        # a scoped one. Keep the token on every fall-open path.
+        print(f'ci_set_matrix: UNSCOPED - selection unusable ({e}), emitting the full '
+              f'matrix', file=sys.stderr)
+        select = None
+    set_matrix_json(select)
+
+
 if __name__ == '__main__':
-    set_matrix_json()
+    main()
diff --git a/.github/scripts/hil_ci_set_matrix.py b/.github/scripts/hil_ci_set_matrix.py
index 65f50788e..bf50061dd 100644
--- a/.github/scripts/hil_ci_set_matrix.py
+++ b/.github/scripts/hil_ci_set_matrix.py
@@ -1,6 +1,7 @@
 import argparse
 import json
 import os
+import sys
 
 
 def _resolve_config_path(config_file):
@@ -19,13 +20,47 @@ def _resolve_config_path(config_file):
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('config_files', nargs='+', help='Configuration JSON file(s)')
-    parser.add_argument('--select', help='hil_select.py JSON; scopes boards when full=false')
+    g = parser.add_mutually_exclusive_group()
+    g.add_argument('--select', help='ci_select.py JSON; scopes boards when full=false')
+    # a whole selection as one argv can exceed MAX_ARG_STRLEN on a big diff, which
+    # would fail the step instead of falling open; callers that already have the
+    # selection on disk pass the path instead
+    g.add_argument('--select-file', help='file holding the same JSON as --select')
     args = parser.parse_args()
 
+    raw = args.select
+    sel = None
+    try:
+        if args.select_file:
+            with open(args.select_file) as f:
+                raw = f.read()
+        if raw:
+            sel = json.loads(raw)
+        if sel is not None and not isinstance(sel, dict):
+            raise ValueError(f'selection is {type(sel).__name__}, not an object')
+    except Exception as e:  # fail-open: an unusable selection must never red the job
+        print(f'hil_ci_set_matrix: selection unusable ({e}) - full roster',
+              file=sys.stderr)
+        sel = None
+
     selected = None
-    sel = json.loads(args.select) if args.select else None
     if sel and not sel.get('full'):
-        selected = set(sel.get('boards', {}))
+        # key ABSENT is an unusable selection, not "nothing selected" - same reading as
+        # ci_set_matrix.py. Filtering every board out would skip every hil-build leg and,
+        # through needs:, both rig jobs: an all-green PR with zero hardware coverage.
+        # An explicit boards: {} stays a legitimate nothing-selected.
+        if not isinstance(sel.get('boards'), dict):
+            print('hil_ci_set_matrix: selection has full false but no usable boards '
+                  'map - full roster', file=sys.stderr)
+            sel = None          # ALL of it is unusable, hil_examples included: keeping
+                                # the -e lists would build a few examples per board
+                                # while the rig, unfiltered, runs that board's whole
+                                # test list - flash failures on the fail-open path
+        else:
+            selected = set(sel['boards'])
+    ex_map = (sel or {}).get('hil_examples') or {}
+    if not isinstance(ex_map, dict):
+        ex_map = {}
 
     # Toolchain buckets must match the toolchains instantiated by the hil-build
     # job in .github/workflows/build.yml. Keep all keys present (even if empty)
@@ -68,8 +103,12 @@ def main():
                     f'hil-build-esp jobs in .github/workflows/build.yml')
 
             build_board = f'-b {name}'
-            if 'build' in board and 'args' in board['build']:
-                build_board += ' ' + ' '.join(f'-D{a}' for a in board['build']['args'])
+
+            # PR selection: build only the examples this board will run (its test
+            # list plus device/board_test, the parking firmware) - tools/build.py -e.
+            # Absent key (hand runs, full non-PR builds) keeps --target all.
+            for ex in ex_map.get(name, []):
+                build_board += f' -e {ex}'
 
             # Each variant builds into cmake-build-<variant.name> with its own cmake
             # -D defines and raw CFLAGS. No 'variant' -> a single build named after
diff --git a/.github/scripts/metrics_pair_compare.py b/.github/scripts/metrics_pair_compare.py
new file mode 100755
index 000000000..50107cf72
--- /dev/null
+++ b/.github/scripts/metrics_pair_compare.py
@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""Board+example-matched code-size compare for PR-scoped builds.
+
+The averaged metrics baseline (metrics-tinyusb) spans every family and example;
+a scoped PR builds a subset, so comparing against it is apples-to-oranges. This
+compares the intersection of (board, example) pairs present on BOTH sides,
+averaged over exactly those pairs, and names what was dropped. See
+docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md #code-metrics.
+"""
+import argparse
+import glob
+import json
+import os
+import sys
+import tempfile
+
+sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'tools'))
+import metrics
+
+# dropped (board, example) pairs named in the PR comment before it truncates
+DROPPED_SHOWN = 20
+
+
+def board_family(board, repo_root):
+    hits = glob.glob(os.path.join(repo_root, 'hw/bsp/*/boards', board))
+    return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None
+
+
+def collect(root, repo_root):
+    """{(board, 'role/example'): [file entries]} from every
+    **/cmake-build-<board>/metrics_by_example.json under root.
+
+    Keyed on the BOARD, not its family. The two sides are built by
+    `--one-first`, which returns all_boards[0] for a family with no
+    ci_preferred_boards entry - so a PR that adds hw/bsp/<family>/boards/a_new_board
+    shifts which board is built, and a family key would file the base run's sizes and
+    the PR run's sizes under the same name and publish the difference between two
+    unrelated MCUs as this PR's code-size impact. On the board key that mismatch lands
+    in `dropped` (reported as not compared), which is the truth."""
+    pairs = {}
+    pat = os.path.join(root, '**', 'metrics_by_example.json')
+    for f in sorted(glob.glob(pat, recursive=True)):
+        board = os.path.basename(os.path.dirname(f))
+        if not board.startswith('cmake-build-'):
+            print(f'pair_compare: {f} not under a cmake-build-<board> dir, skipping', file=sys.stderr)
+            continue
+        board = board[len('cmake-build-'):]
+        if not board_family(board, repo_root):
+            # unknown board: the name is still a usable key, but say so - it means the
+            # artifact came from a tree whose hw/bsp does not match this checkout
+            print(f'pair_compare: no family for board {board}', file=sys.stderr)
+        # parse into a LOCAL dict and merge only once the whole file came out clean:
+        # a file that blows up half way through must drop WHOLE, or the entries read
+        # before the malformation stay in the comparison while stderr says the file
+        # was skipped, and a silently truncated table gets published as the verdict
+        try:
+            one = {}
+            for ex, ent in json.load(open(f)).items():
+                one.setdefault((board, ex), []).extend(ent.get('files', []))
+        except (OSError, ValueError, AttributeError, TypeError) as e:
+            print(f'pair_compare: unreadable {f} ({e}), skipping', file=sys.stderr)
+            continue
+        for k, v in one.items():
+            pairs.setdefault(k, []).extend(v)
+    return pairs
+
+
+def main():
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument('--base-dir', required=True)
+    ap.add_argument('--new-dir', required=True)
+    ap.add_argument('--out', default='metrics_compare')
+    a = ap.parse_args()
+    repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+    base = collect(a.base_dir, repo_root)
+    new = collect(a.new_dir, repo_root)
+    common = sorted(set(base) & set(new))
+    dropped = sorted(set(base) ^ set(new))
+
+    if not common:
+        with open(a.out + '.md', 'w') as f:
+            if new and not base:
+                # interim state: master has not uploaded a per-example baseline yet.
+                # Blaming the PR's scoping for that sends people hunting the wrong bug
+                f.write('_No per-example baseline from the base branch yet (the first '
+                        'master push after this feature merges uploads it); comparison '
+                        'will appear on the next push._\n')
+            else:
+                f.write('_Code-size comparison skipped: no (board, example) pair was '
+                        'built on both the base branch and this PR._\n')
+        return
+
+    def synth(pairs, path):
+        with open(path, 'w') as f:
+            json.dump({'files': [e for k in common for e in pairs[k]]}, f)
+
+    with tempfile.TemporaryDirectory() as td:
+        b, n = os.path.join(td, 'base.json'), os.path.join(td, 'new.json')
+        synth(base, b)
+        synth(new, n)
+        comparison = metrics.compare_files(b, n, ['tinyusb/src'])
+        if comparison is None:
+            with open(a.out + '.md', 'w') as f:
+                f.write('_Code-size comparison failed to produce data._\n')
+            return
+        metrics.write_compare_markdown(comparison, a.out + '.md', 'name+')
+
+    with open(a.out + '.md', 'a') as f:
+        boards = sorted({k[0] for k in common})
+        f.write(f'\n_Scoped compare: {len(common)} (board, example) pairs across '
+                f'{", ".join(boards)}._\n')
+        if dropped:
+            # GitHub caps a comment at 65,536 chars and this footer rides inside the
+            # sticky code-metrics comment: a broad scoped PR drops hundreds of pairs,
+            # and the raw list alone reached ~65KB and reddened the whole job. Only a
+            # summary goes in the comment; the full list goes to the job log.
+            names = [f'{board}:{ex}' for board, ex in dropped]
+            print('pair_compare: not compared (missing on one side): '
+                  + ', '.join(names), file=sys.stderr)
+            more = len(names) - DROPPED_SHOWN
+            f.write(f'_Not compared (missing on one side): {len(names)} pairs - '
+                    + ', '.join(names[:DROPPED_SHOWN])
+                    + (f', ... and {more} more (see the code-metrics job log)'
+                       if more > 0 else '')
+                    + '._\n')
+
+
+if __name__ == '__main__':
+    main()