summaryrefslogtreecommitdiff
path: root/test
diff options
context:
space:
mode:
authorHa Thach <[email protected]>2026-07-30 02:29:32 +0700
committerGitHub <[email protected]>2026-07-30 02:29:32 +0700
commiteef5af86aa26fe3d72e41156a586a6ed3ffce9f8 (patch)
treefa5ebe3a8d46738338eaee2d4a80717a7a10f464 /test
parent5d8afbb23248e9d1d5d1fdf434c9c58cc54ba5b7 (diff)
hil, ci: scope HIL builds and tests to the boards a PR affects (#3797)
hil, ci: scope HIL builds and tests to the boards a PR affects Add test/hil/hil_select.py, a stdlib-only selector that maps a PR diff to the rig boards, tests and BSP families a change can affect, and wire it into CI so pull requests build and run only those. A port change picks its families' boards, a class change picks the examples enabling that class, and device/host changes prune the other role. Anything unclassified — infra, an unmapped port, a selector error — falls back to the full matrix, and push/schedule runs are untouched. Move the shared example lists to hil_examples.py; 54 hardware-free tests cover the rules.
Diffstat (limited to 'test')
-rw-r--r--test/hil/hil_ci.sh1
-rw-r--r--test/hil/hil_ci_set_matrix.py15
-rw-r--r--test/hil/hil_examples.py37
-rwxr-xr-xtest/hil/hil_select.py520
-rwxr-xr-xtest/hil/hil_test.py102
-rw-r--r--test/hil/test_hil_select.py542
6 files changed, 1173 insertions, 44 deletions
diff --git a/test/hil/hil_ci.sh b/test/hil/hil_ci.sh
index 3384b4e2e..ef93bcb49 100644
--- a/test/hil/hil_ci.sh
+++ b/test/hil/hil_ci.sh
@@ -55,6 +55,7 @@ scp -q "$ROOT_DIR/test/hil/hil_test.py" \
"$ROOT_DIR/test/hil/hil_flash.py" \
"$ROOT_DIR/test/hil/hil_lock.py" \
"$ROOT_DIR/test/hil/usbtest.py" \
+ "$ROOT_DIR/test/hil/hil_examples.py" \
"$ROOT_DIR/test/hil/pymtp.py" \
"$CONFIG" \
"$REMOTE:$REMOTE_DIR/test/hil/"
diff --git a/test/hil/hil_ci_set_matrix.py b/test/hil/hil_ci_set_matrix.py
index 13f7f1882..bca989bd1 100644
--- a/test/hil/hil_ci_set_matrix.py
+++ b/test/hil/hil_ci_set_matrix.py
@@ -17,8 +17,14 @@ def _resolve_config_path(config_file):
def main():
parser = argparse.ArgumentParser()
parser.add_argument('config_files', nargs='+', help='Configuration JSON file(s)')
+ parser.add_argument('--select', help='hil_select.py JSON; scopes boards when full=false')
args = parser.parse_args()
+ selected = None
+ sel = json.loads(args.select) if args.select else None
+ if sel and not sel.get('full'):
+ selected = set(sel.get('boards', {}))
+
# Toolchain buckets must match the toolchains instantiated by the hil-build
# job in .github/workflows/build.yml. Keep all keys present (even if empty)
# so `fromJSON(hil_json)[toolchain]` always resolves to a list.
@@ -40,6 +46,8 @@ def main():
config = json.load(f)
for board in config['boards']:
+ if selected is not None and board['name'] not in selected:
+ continue
name = board['name']
flasher = board['flasher']
# esptool boards must build under esp-idf; others default to arm-gcc
@@ -49,6 +57,13 @@ def main():
toolchain = 'esp-idf'
else:
toolchain = board.get('toolchain', 'arm-gcc')
+ if toolchain not in matrix:
+ # a board in no bucket would never be built, and the bare KeyError
+ # below would only say so as a traceback from the set-matrix job
+ raise SystemExit(
+ f'{name}: toolchain {toolchain!r} is not a build bucket '
+ f'({", ".join(matrix)}); add it here and to the hil-build / '
+ f'hil-build-esp jobs in .github/workflows/build.yml')
build_board = f'-b {name}'
if 'build' in board and 'args' in board['build']:
diff --git a/test/hil/hil_examples.py b/test/hil/hil_examples.py
new file mode 100644
index 000000000..4c8b6918b
--- /dev/null
+++ b/test/hil/hil_examples.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# HIL example test lists, shared by hil_test.py (runner) and hil_select.py
+# (PR-diff selector). Stdlib-only: hil_select runs on bare CI runners.
+
+# The per-board run order is shuffled (see test_board).
+# Every example carries a unique hardcoded idProduct (see its usb_descriptors.c)
+
+# device tests
+device_tests = [
+ 'device/cdc_dual_ports',
+ 'device/cdc_msc',
+ 'device/dfu',
+ 'device/cdc_msc_throughput',
+ 'device/audio_test_freertos',
+ 'device/dfu_runtime',
+ 'device/cdc_msc_freertos',
+ 'device/hid_boot_interface',
+ 'device/msc_dual_lun',
+ 'device/hid_generic_inout',
+ 'device/printer_to_cdc',
+ 'device/midi_test',
+ 'device/mtp',
+ 'device/usbtest', # cafe:4010, unique PID; runs the Linux testusb tier-4 battery via usbtest.py
+ # 'device/net_lwip_webserver', # disabled for PR #3605: USB net iface enum is flaky on the CI HIL host
+]
+
+dual_tests = [
+ 'dual/host_info_to_device_cdc',
+]
+
+host_test = [
+ 'host/cdc_msc_hid',
+ 'host/msc_file_explorer',
+ 'host/msc_file_explorer_freertos',
+ 'host/device_info',
+]
diff --git a/test/hil/hil_select.py b/test/hil/hil_select.py
new file mode 100755
index 000000000..3ac3f1fdb
--- /dev/null
+++ b/test/hil/hil_select.py
@@ -0,0 +1,520 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+"""PR-diff -> HIL selection: which rig boards and which tests a change can affect.
+
+Stdlib-only (runs on bare CI runners; never imports hil_test/hil_flash/hil_lock).
+Fail-open: any file no rule classifies forces the full matrix. See
+docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md.
+
+JSON: full, boards (name -> 'all' | [tests]), families (bsp families the diff
+touches, including ones with no rig board - build-only consumers such as /pre-pr
+sample from these), args (hil_test.py args per config) and args_flasher (the same
+args split by each board's flasher, for CI legs that split one rig by flasher).
+"""
+import argparse
+import functools
+import glob
+import json
+import os
+import re
+import subprocess
+import sys
+
+from hil_examples import device_tests, dual_tests, host_test
+
+ALL_TESTS = {'device': device_tests, 'dual': dual_tests, 'host': host_test}
+
+# class dir -> config macro suffix exceptions (rule 3); dfu is per-file, handled inline
+NET_MACROS = ('ECM_RNDIS', 'NCM')
+
+_NONCODE_RE = re.compile(
+ r'^(docs/|\.claude/|.*\.(md|rst)$|LICENSE)')
+_FULL_RE = re.compile(
+ r'^(src/common/|src/osal/|src/tusb\.c$|src/tusb\.h$|src/tusb_option\.h$|'
+ r'test/hil/|\.github/workflows/build.*\.yml$|\.github/actions/|'
+ r'tools/build\.py$|tools/get_deps\.py$|tools/cmake/|hw/mcu/|lib/|'
+ r'hw/bsp/(family_support\.cmake|board_api\.h|board\.c|ansi_escape\.h)$|'
+ r'examples/build_system/|examples/CMakeLists\.txt$|'
+ # board_test is HIL infrastructure, not a test: hil_test.py flashes it to park
+ # every board (variant boundary + end-of-board teardown), so every board depends on it
+ r'examples/device/board_test/)')
+
+# --no-renames: with rename detection git reports only a rename's destination, so code
+# moved out of an HIL-relevant path would be classified by its new path alone
+GIT_DIFF_ARGV = ['git', 'diff', '--no-renames', '--name-only']
+
+
+def test_role(test: str) -> str:
+ return test.split('/', 1)[0] # 'device' | 'dual' | 'host'
+
+
+def board_roles(board: dict) -> set:
+ t = board.get('tests', {})
+ roles = set()
+ if t.get('device'):
+ roles.add('device')
+ if t.get('host'):
+ roles.add('host')
+ if t.get('dual'):
+ roles.update(('device', 'host'))
+ for only in t.get('only', []):
+ r = test_role(only)
+ roles.update(('device', 'host') if r == 'dual' else (r,))
+ return roles
+
+
+def board_tests(board: dict) -> list:
+ """Every test this board would run today (mirrors hil_test.test_board's default)."""
+ t = board.get('tests', {})
+ if 'only' in t:
+ run = list(t['only'])
+ else:
+ run = []
+ if t.get('device'):
+ run += device_tests
+ if t.get('dual'):
+ run += dual_tests
+ if t.get('host'):
+ run += host_test
+ return [x for x in run if x not in t.get('skip', [])]
+
+
+# cached: called per changed file x roster board, and the tree doesn't change mid-run
[email protected]_cache(maxsize=None)
+def board_family(board_name: str, repo_root: str):
+ hits = glob.glob(os.path.join(repo_root, 'hw/bsp/*/boards', board_name))
+ return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None
+
+
+# `if (OPTION STREQUAL "1")` guards in family_support.cmake, and the option tokens
+# a roster entry passes to the build (NAME=VALUE / -DNAME=VALUE)
+_CM_IF_RE = re.compile(r'if\s*\(')
+_CM_ELSE_RE = re.compile(r'else(if)?\s*\(')
+_CM_ENDIF_RE = re.compile(r'endif\s*\(')
+_CM_OPT_RE = re.compile(r'if\s*\(\s*\$?\{?([A-Za-z_]\w*)\}?\s+STREQUAL\s+"?1"?\s*\)')
+_CM_PORT_RE = re.compile(r'src/portable/((?:[^/\s]+/)?[^/\s]+)/')
+_FALSY = ('', '0', 'off', 'false', 'no')
+
+
[email protected]_cache(maxsize=None)
+def port_option_gates(repo_root: str) -> dict:
+ """port dir -> build options that compile it regardless of the board's family
+ file, e.g. {'analog/max3421': {'MAX3421_HOST'}} from family_support.cmake."""
+ gates = {}
+ try:
+ text = open(os.path.join(repo_root, 'hw/bsp/family_support.cmake')).read()
+ except OSError:
+ return gates
+ stack = [] # one entry per open if(): its option, or None
+ for line in text.splitlines():
+ line = line.strip()
+ if _CM_IF_RE.match(line):
+ m = _CM_OPT_RE.match(line)
+ stack.append(m.group(1) if m else None)
+ elif _CM_ELSE_RE.match(line):
+ if stack:
+ stack[-1] = None # the guard doesn't hold in this branch
+ elif _CM_ENDIF_RE.match(line):
+ if stack:
+ stack.pop()
+ opts = {o for o in stack if o}
+ m = _CM_PORT_RE.search(line)
+ if opts and m:
+ gates.setdefault(m.group(1), set()).update(opts)
+ return gates
+
+
+_CM_SET_RE = re.compile(r'set\s*\(\s*([A-Za-z_]\w*)\s+([^)\s]+)\s*\)')
+
+
+# cached: called per changed portable file x roster board
[email protected]_cache(maxsize=None)
+def bsp_board_options(board_name: str, repo_root: str) -> frozenset:
+ """Build options a board turns on in its own BSP: `set(<OPT> <value>)` in
+ hw/bsp/<family>/boards/<board>/board.cmake, e.g. MAX3421_HOST on the espressif
+ and rp2040 max3421 boards. CMake only - HIL CI builds nothing with Make, so a
+ board.mk-only option (e.g. nrf5340dk's MAX3421_HOST) compiles no port here."""
+ fam = board_family(board_name, repo_root)
+ if not fam:
+ return frozenset()
+ path = os.path.join(repo_root, 'hw/bsp', fam, 'boards', board_name, 'board.cmake')
+ try:
+ text = open(path).read()
+ except OSError:
+ return frozenset()
+ out = set()
+ for line in text.splitlines():
+ line = line.strip()
+ if line.startswith('#'):
+ continue
+ m = _CM_SET_RE.match(line)
+ if m and m.group(2).strip('"').lower() not in _FALSY:
+ out.add(m.group(1))
+ return frozenset(out)
+
+
+def board_options(board: dict, repo_root: str) -> set:
+ """Build options a board has truthy: the roster entry's build.args plus each
+ variant's defines (NAME=VALUE) and raw CFLAGS (-DNAME=VALUE), plus whatever its
+ own board.cmake sets (a board can enable a gated port without the roster saying so)."""
+ toks = list(board.get('build', {}).get('args', []))
+ for v in board.get('variant', []):
+ toks += list(v.get('defines', []))
+ toks += v.get('flags', '').split()
+ out = set(bsp_board_options(board['name'], repo_root))
+ for t in toks:
+ name, _, val = (t[2:] if t.startswith('-D') else t).partition('=')
+ if name and val.strip().strip('"').lower() not in _FALSY:
+ out.add(name.strip())
+ return out
+
+
[email protected]_cache(maxsize=None)
+def port_families(port_dir: str, repo_root: str) -> set:
+ """Board families that compile this src/portable dir. CMake only: HIL CI builds
+ every board with CMake, so a port wired up in family.mk alone is compiled for no
+ HIL board and must not select one. family.cmake lists portable sources directly
+ for most families; espressif instead references them from a nested component
+ CMakeLists.txt (hw/bsp/espressif/components/tinyusb_src/CMakeLists.txt)."""
+ fams = set()
+ bsp_root = os.path.join(repo_root, 'hw/bsp')
+ # trailing '/' so a port dir is not a prefix of a sibling: bare 'microchip/pic'
+ # would otherwise match '.../microchip/pic32mz/...' and inherit its families
+ needle = port_dir + '/'
+ for f in glob.glob(os.path.join(bsp_root, '*/family.cmake')) + \
+ glob.glob(os.path.join(bsp_root, '*/components/*/CMakeLists.txt')):
+ try:
+ if needle in open(f).read():
+ fam = os.path.relpath(f, bsp_root).split(os.sep, 1)[0]
+ fams.add(fam)
+ except OSError:
+ pass
+ return fams
+
+
+_CLS_INC_RE = re.compile(r'#\s*include\s*[<"]class/([^/"<>]+)/([^"<>]+)[">]')
+
+
[email protected]_cache(maxsize=None)
+def class_include_edges(repo_root: str) -> dict:
+ """'<class>/<header>' -> the other class dirs that include it. A class header
+ pulled in by a second class ships in every firmware enabling that second class:
+ src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, and
+ net_device.h includes class/cdc/cdc.h. The class rule derives macros from the
+ directory name alone, so without this edge a change to the included header
+ selects only its own class's examples - and on a board that skips those (e.g.
+ metro_m4_express skips audio_test_freertos), nothing at all.
+
+ Derived from the actual #include lines rather than a hand-written table so it
+ cannot rot when a class picks up or drops a cross-class include."""
+ edges = {}
+ for f in sorted(glob.glob(os.path.join(repo_root, 'src/class/*/*.[ch]'))):
+ cls = os.path.basename(os.path.dirname(f))
+ try:
+ text = open(f).read()
+ except OSError:
+ continue
+ for inc_cls, inc_hdr in _CLS_INC_RE.findall(text):
+ if inc_cls != cls:
+ edges.setdefault(f'{inc_cls}/{inc_hdr}', set()).add(cls)
+ return edges
+
+
+def class_macros(cls: str, base: str, prefix: str) -> list:
+ """Config macros that compile a class dir's code, for role prefix TUD/TUH.
+ `base` refines dfu only (it splits DFU from DFU_RUNTIME per file); pass '' for
+ a class reached through an include edge, where the widest set is correct."""
+ if cls == 'net':
+ return [f'CFG_{prefix}_{m}' for m in NET_MACROS]
+ if cls == 'dfu':
+ if base.startswith('dfu_rt'):
+ return [f'CFG_{prefix}_DFU_RUNTIME']
+ if base.startswith('dfu_device') or base.startswith('dfu_host'):
+ return [f'CFG_{prefix}_DFU']
+ return [f'CFG_{prefix}_DFU', f'CFG_{prefix}_DFU_RUNTIME']
+ return [f'CFG_{prefix}_{cls.upper()}']
+
+
+def _config_enables(cfg_path: str, macros) -> bool:
+ try:
+ text = open(cfg_path).read()
+ except OSError:
+ return False
+ return any(re.search(rf'#define\s+{m}\s+\(?\s*0*[1-9]', text) for m in macros)
+
+
+def roster_only_tests(all_boards) -> set:
+ """Test paths that only appear in a roster board's tests.only list (e.g.
+ espressif boards), not in the shared device/dual/host_test lists."""
+ out = set()
+ for b in all_boards:
+ out.update(b.get('tests', {}).get('only', []))
+ return out
+
+
+def class_examples(macros, role: str, repo_root: str, extra_tests: set) -> set:
+ """Tests (from role's + dual lists, plus roster-only-list tests of that role)
+ whose example config enables any macro."""
+ pool = role_tests({role}, extra_tests)
+ out = set()
+ for test in pool:
+ cfg = os.path.join(repo_root, 'examples', test, 'src', 'tusb_config.h')
+ if _config_enables(cfg, macros):
+ out.add(test)
+ return out
+
+
+def role_tests(roles: set, extras: set) -> set:
+ """Every test for the given role(s): each role's own list + dual tests,
+ plus roster-only-list tests (extras) matching those roles or 'dual'."""
+ pool = set(dual_tests)
+ for r in roles:
+ pool |= set(ALL_TESTS[r])
+ pool |= {t for t in extras if test_role(t) in roles or test_role(t) == 'dual'}
+ return pool
+
+
+class _Sel:
+ """Accumulates contributions. board->set(tests) plus 'all-board' markers."""
+ def __init__(self):
+ self.full = False
+ self.by_board = {} # name -> set of tests, or 'all'
+ self.roles = set() # roles touched by any contribution
+ self.families = set() # bsp families touched (incl. off-rig ones: build-only consumers)
+ self.reasons = []
+
+ def add(self, boards, tests, reason):
+ """tests: 'all' or iterable of test paths."""
+ self.reasons.append(reason)
+ for b in boards:
+ cur = self.by_board.get(b)
+ if tests == 'all' or cur == 'all':
+ self.by_board[b] = 'all'
+ else:
+ self.by_board[b] = (cur or set()) | set(tests)
+
+ def force_full(self, reason):
+ self.full = True
+ self.reasons.append(reason)
+
+
+def _classify_one(path, repo_root, roster_boards, extras: set, s: _Sel):
+ base = os.path.basename(path)
+ if _NONCODE_RE.match(path):
+ s.reasons.append(f'{path}: non-code, no contribution')
+ return
+ if _FULL_RE.match(path):
+ s.force_full(f'{path}: core/infra -> full matrix')
+ return
+
+ m = re.match(r'src/portable/((?:[^/]+/)?[^/]+)/', path)
+ if m:
+ port = m.group(1)
+ if re.match(r'(dcd_|.*_device)', base):
+ roles = {'device'}
+ elif re.match(r'(hcd_|.*_host)', base):
+ roles = {'host'}
+ else:
+ roles = {'device', 'host'}
+ fams = port_families(port, repo_root)
+ if not fams:
+ # no family references this port: either a new/renamed port dir or a
+ # family.cmake layout the scan misses - widen instead of contributing nothing
+ s.force_full(f'{path}: port {port} maps to no board family -> full matrix')
+ return
+ s.families.update(fams)
+ # a board can also pull the port in through a build option (e.g. MAX3421_HOST=1
+ # from the roster on metro_m4_express, or from its own board.cmake), which its
+ # family file never names
+ gates = port_option_gates(repo_root).get(port, set())
+ boards = [b['name'] for b in roster_boards
+ if (board_family(b['name'], repo_root) in fams or
+ (gates and board_options(b, repo_root) & gates)) and (board_roles(b) & roles)]
+ tests = role_tests(roles, extras)
+ s.roles.update(roles)
+ why = f'{path}: port {port} -> families {sorted(fams)}'
+ if gates:
+ why += f' + option {sorted(gates)}'
+ s.add(boards, tests, f'{why} -> boards {boards} ({"/".join(sorted(roles))})')
+ return
+
+ m = re.match(r'src/class/([^/]+)/', path)
+ if m:
+ cls = m.group(1)
+ if re.search(r'_device\.[ch]$', base):
+ roles = {'device'}
+ elif re.search(r'_host\.[ch]$', base):
+ roles = {'host'}
+ else:
+ roles = {'device', 'host'}
+ # this file's own class, plus any class whose headers include it
+ via = sorted(class_include_edges(repo_root).get(f'{cls}/{base}', ()))
+
+ def macros(prefix):
+ return (class_macros(cls, base, prefix) +
+ [m2 for c in via for m2 in class_macros(c, '', prefix)])
+ tests = set()
+ if 'device' in roles:
+ tests |= class_examples(macros('TUD'), 'device', repo_root, extras)
+ if 'host' in roles:
+ tests |= class_examples(macros('TUH'), 'host', repo_root, extras)
+ boards = [b['name'] for b in roster_boards if board_roles(b) & roles]
+ s.roles.update(roles)
+ why = f'{path}: class {cls}' + (f' (+ included by {via})' if via else '')
+ s.add(boards, tests, f'{why} -> {sorted(tests)} ({"/".join(sorted(roles))})')
+ return
+
+ m = re.match(r'src/(device|host)/', path)
+ if m:
+ role = m.group(1)
+ boards = [b['name'] for b in roster_boards if role in board_roles(b)]
+ s.roles.add(role)
+ s.add(boards, role_tests({role}, extras), f'{path}: core {role} stack -> all {role} tests')
+ return
+
+ m = re.match(r'hw/bsp/([^/]+)/(?:boards/([^/]+)/)?', path)
+ if m:
+ fam, brd = m.group(1), m.group(2)
+ s.families.add(fam)
+ if brd:
+ boards = [b['name'] for b in roster_boards if b['name'] == brd]
+ why = f'{path}: bsp board {brd}'
+ else:
+ boards = [b['name'] for b in roster_boards
+ if board_family(b['name'], repo_root) == fam]
+ why = f'{path}: bsp family {fam}'
+ s.roles.update(('device', 'host'))
+ s.add(boards, 'all', f'{why} -> boards {boards}')
+ return
+
+ m = re.match(r'examples/(device|host|dual)/([^/]+)/', path)
+ if m:
+ test = f'{m.group(1)}/{m.group(2)}'
+ known = any(test in pool for pool in ALL_TESTS.values()) or test in extras
+ if known:
+ boards = [b['name'] for b in roster_boards]
+ role = test_role(test)
+ s.roles.update(('device', 'host') if role == 'dual' else (role,))
+ s.add(boards, [test], f'{path}: example -> {test} on all boards')
+ else:
+ s.reasons.append(f'{path}: example not in HIL lists, no contribution')
+ return
+
+ s.force_full(f'{path}: unclassified -> full matrix')
+
+
+def classify(changed_files, repo_root, rosters):
+ all_boards = []
+ seen = set()
+ for _, boards in rosters:
+ for b in boards:
+ if b['name'] not in seen:
+ seen.add(b['name'])
+ all_boards.append(b)
+
+ extras = roster_only_tests(all_boards)
+ s = _Sel()
+ # no early exit once full: keep classifying so `families` still reports every
+ # family the diff touches (build-only consumers need it). Nothing after the first
+ # force_full can change full/boards/args - the full branch below ignores by_board.
+ for path in changed_files:
+ _classify_one(path, repo_root, all_boards, extras, s)
+
+ if s.full:
+ return {'full': True, 'boards': {b['name']: 'all' for b in all_boards},
+ 'families': sorted(s.families), 'reasons': s.reasons}
+
+ # role pruning: single-role selections drop the other role's tests and boards
+ by_name = {b['name']: b for b in all_boards}
+ out = {}
+ for name, tests in s.by_board.items():
+ allowed = board_tests(by_name[name])
+ if tests == 'all':
+ kept = list(allowed)
+ else:
+ kept = [t for t in allowed if t in tests]
+ if s.roles and s.roles != {'device', 'host'}:
+ role = next(iter(s.roles))
+ kept = [t for t in kept if test_role(t) in (role, 'dual')]
+ if kept:
+ out[name] = 'all' if set(kept) == set(allowed) else sorted(kept)
+ return {'full': False, 'boards': out, 'families': sorted(s.families),
+ 'reasons': s.reasons}
+
+
+def _board_args(name, chosen) -> list:
+ parts = [f'-b {name}']
+ if chosen != 'all':
+ parts.append(f'-bt {name}:{",".join(chosen)}')
+ return parts
+
+
+def selection_args(sel, rosters):
+ """hil_test.py args per config. Empty means either 'full matrix' or 'nothing
+ selected' - callers must read sel['full'] to tell them apart."""
+ args = {}
+ for cfg_path, boards in rosters:
+ parts = []
+ if not sel['full']:
+ for b in boards:
+ chosen = sel['boards'].get(b['name'])
+ if chosen is not None:
+ parts += _board_args(b['name'], chosen)
+ args[os.path.basename(cfg_path)] = ' '.join(parts)
+ return args
+
+
+def selection_args_by_flasher(sel, rosters):
+ """{config: {flasher name: args}}. CI runs one rig as several jobs split by
+ flasher (esptool vs the rest); each must gate on its own subset, otherwise the
+ other leg runs a filter matching zero boards and reports a vacuous green."""
+ out = {}
+ for cfg_path, boards in rosters:
+ per = {}
+ if not sel['full']:
+ for b in boards:
+ chosen = sel['boards'].get(b['name'])
+ if chosen is None:
+ continue
+ per.setdefault(b.get('flasher', {}).get('name', ''), []).extend(
+ _board_args(b['name'], chosen))
+ out[os.path.basename(cfg_path)] = {f: ' '.join(p) for f, p in per.items()}
+ return out
+
+
+def changed_files_from_git(base, repo_root):
+ mb = subprocess.run(['git', 'merge-base', 'HEAD', base], cwd=repo_root,
+ capture_output=True, text=True, check=True).stdout.strip()
+ diff = subprocess.run(GIT_DIFF_ARGV + [f'{mb}..HEAD'], cwd=repo_root,
+ capture_output=True, text=True, check=True).stdout
+ return [l for l in diff.splitlines() if l.strip()]
+
+
+def main():
+ ap = argparse.ArgumentParser(description=__doc__)
+ g = ap.add_mutually_exclusive_group(required=True)
+ g.add_argument('--base', help='git ref to diff against (merge-base..HEAD)')
+ g.add_argument('--diff-file', help='newline-separated changed-file list')
+ ap.add_argument('configs', nargs='+', help='rig roster JSON file(s)')
+ a = ap.parse_args()
+
+ repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+ rosters = []
+ for c in a.configs:
+ with open(c) as f:
+ rosters.append((c, json.load(f)['boards']))
+
+ files = (open(a.diff_file).read().splitlines() if a.diff_file
+ else changed_files_from_git(a.base, repo_root))
+ files = [f for f in files if f.strip()]
+
+ s = classify(files, repo_root, rosters)
+ s['args'] = selection_args(s, rosters)
+ s['args_flasher'] = selection_args_by_flasher(s, rosters)
+ for r in s['reasons']:
+ print(f'hil_select: {r}', file=sys.stderr)
+ print(json.dumps(s))
+
+
+if __name__ == '__main__':
+ main()
diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py
index 71e85f55f..96d52e601 100755
--- a/test/hil/hil_test.py
+++ b/test/hil/hil_test.py
@@ -59,6 +59,7 @@ from multiprocessing import TimeoutError as MpTimeoutError
import hil_flash
import hil_lock
+from hil_examples import device_tests, dual_tests, host_test
# Raw Lock/Semaphore objects passed via Pool initargs are inheritable only under the fork
# start method (spawn/forkserver pickle them and fail at Pool creation) — pin it so a
@@ -1351,39 +1352,6 @@ def test_device_usbtest(board):
# Main
# -------------------------------------------------------------
-# The per-board run order is shuffled (see test_board).
-# Every example carries a unique hardcoded idProduct (see its usb_descriptors.c)
-
-# device tests
-device_tests = [
- 'device/cdc_dual_ports',
- 'device/cdc_msc',
- 'device/dfu',
- 'device/cdc_msc_throughput',
- 'device/audio_test_freertos',
- 'device/dfu_runtime',
- 'device/cdc_msc_freertos',
- 'device/hid_boot_interface',
- 'device/msc_dual_lun',
- 'device/hid_generic_inout',
- 'device/printer_to_cdc',
- 'device/midi_test',
- 'device/mtp',
- 'device/usbtest', # cafe:4010, unique PID; runs the Linux testusb tier-4 battery via usbtest.py
- # 'device/net_lwip_webserver', # disabled for PR #3605: USB net iface enum is flaky on the CI HIL host
-]
-
-dual_tests = [
- 'dual/host_info_to_device_cdc',
-]
-
-host_test = [
- 'host/cdc_msc_hid',
- 'host/msc_file_explorer',
- 'host/msc_file_explorer_freertos',
- 'host/device_info',
-]
-
def test_example(board: Board, variant: str, example: str) -> tuple[int, str, str | None]:
"""
@@ -1517,6 +1485,10 @@ def build_board(board: Board) -> tuple[str, int]:
return name, failed
+# pseudo-test column for a variant boundary the park-flash could not clear (see below)
+BOUNDARY_CELL = 'same-PID boundary'
+
+
def test_board(board: Board) -> tuple[str, int, list[str], list, float]:
name = board['name']
flasher = board['flasher']
@@ -1568,6 +1540,7 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]:
err_count = 0
failed_tests = []
+ board_wide_fail = False # re-run the whole board, not a subset of its tests
rows = [] # list of (row_label, {example: status}, duration) — one row per build variant
# a -t/-bt filtered run times only a subset; report no duration so an accumulate
# re-run keeps the previous full-run value
@@ -1587,10 +1560,36 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]:
random.Random(f'{shuffle_seed}:{name}:{vname}').shuffle(run_list)
if run_list[0] == prev_last:
run_list[0], run_list[-1] = run_list[-1], run_list[0]
+ cells = {}
+ if run_list and run_list[0] == prev_last and not skip_flash:
+ # Same example (same PID) still repeats across the boundary: a one-test
+ # list (the common case for a -bt scoped run) leaves nothing to swap
+ # with. Park on board_test first - it disables the board's USB, so the
+ # PID goes away and the next flash must re-enumerate to be seen.
+ t_park = time.monotonic()
+ park_ec, park_status, _ = test_example(board, vname, 'device/board_test')
+ if park_ec or park_status == 'skip':
+ # Boundary not cleared: the previous variant's device may still be
+ # enumerated under the same PID, so this variant's tests could pass
+ # against its firmware. Skip them - a false green proves nothing and
+ # is worse than a gap - and record the boundary itself as the failure
+ # (a visible ❌ cell, mirroring the board-lock row above) so the report
+ # matches the exit code instead of rendering all-green.
+ why = 'no board_test binary' if park_status == 'skip' else 'park flash failed'
+ log_line(f'{vname:40} {"same-PID boundary":30} {STATUS_FAILED}: not cleared ({why}); '
+ f'skipping {len(run_list)} test(s) on this variant')
+ err_count += 1
+ cells[BOUNDARY_CELL] = 'fail'
+ # blaming run_list[0] would re-run an innocent test that then passes,
+ # leaving the boundary unretested; re-run the whole board instead
+ board_wide_fail = True
+ # leave prev_last alone: the board still holds the previous variant's
+ # firmware, so the next variant must attempt the park again
+ run_list = []
+ t_board += time.monotonic() - t_park # park is teardown, not board cost
if run_list:
prev_last = run_list[-1]
t_variant = time.monotonic()
- cells = {}
for test in run_list:
ec, status, metric = test_example(board, vname, test)
err_count += ec
@@ -1609,7 +1608,7 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]:
if not skip_flash:
test_example(board, variants[0]['name'], 'device/board_test')
- return name, err_count, sorted(set(failed_tests)), rows, t_total
+ return name, err_count, [] if board_wide_fail else sorted(set(failed_tests)), rows, t_total
finally:
if _lock_fh:
try:
@@ -1704,11 +1703,13 @@ def render_matrix(rows_all: list) -> str:
return summary + '\n\n' + '\n'.join([header, sep] + body)
-def accumulate_report(mret: list, report_dir: Path, fresh: bool) -> str:
+def accumulate_report(mret: list, report_dir: Path, fresh: bool, scope: str = '') -> str:
"""Merge this run's results into hil_report.json in report_dir, then (re)write
- the markdown matrix to hil_report.md. `fresh` (a full run, no --accumulate/-bt)
+ the markdown matrix to hil_report.md. `fresh` (a first run, no --accumulate)
starts a new report; otherwise a re-run accumulates so boards/tests that
- already passed are preserved while re-run cells are updated. Returns the md."""
+ already passed are preserved while re-run cells are updated. `scope` names the
+ board filter, if any, so a scoped table is not mistaken for a full one.
+ Returns the md."""
acc = {} # ordered {row_label: [cells dict, duration str|None]}
jpath = report_dir / REPORT_JSON
if not fresh and jpath.is_file():
@@ -1736,6 +1737,10 @@ def accumulate_report(mret: list, report_dir: Path, fresh: bool) -> str:
del acc[name]
for row_label, cells, dur in rows:
row = acc.setdefault(row_label, [{}, None])
+ # the boundary cell is only ever written on failure, so a re-run of this
+ # variant that cleared the boundary must drop the previous attempt's ❌
+ if BOUNDARY_CELL not in cells:
+ row[0].pop(BOUNDARY_CELL, None)
row[0].update(cells)
if dur is not None:
row[1] = dur
@@ -1745,6 +1750,10 @@ def accumulate_report(mret: list, report_dir: Path, fresh: bool) -> str:
for k, (c, d) in acc.items()]}, indent=2) + '\n')
md = render_matrix([(k, c, d) for k, (c, d) in acc.items()])
+ if scope:
+ # a scoped run's small table is otherwise indistinguishable from a full one,
+ # and it replaces the previous full table in the sticky PR comment
+ md = f'_Scoped run: {scope}. Boards/tests not listed were not run._\n\n' + md
(report_dir / REPORT_MD).write_text(md + '\n', encoding='utf-8')
return md
@@ -1831,13 +1840,14 @@ def main() -> None:
# HIL report sidecar (hil_report.json/.md) and the .failed re-run spec live in
# report_dir (CI keys it by run id, so it persists across run attempts but is
- # private to one run). A full run starts fresh; a re-run (--accumulate / -bt,
- # i.e. the .failed file) merges so already-passed boards/tests are preserved.
- # Clear prior state up front on a fresh run so a crash mid-run can't leave a
- # stale report or re-run spec to be consumed by a retry.
+ # private to one run). A full run starts fresh; a re-run (--accumulate, which
+ # the generated .failed spec always starts with) merges so already-passed
+ # boards/tests are preserved. Clear prior state up front on a fresh run so a
+ # crash mid-run can't leave a stale report or re-run spec for a retry.
+ # -bt alone is not a re-run marker: PR-scoped first attempts pass -bt too.
report_dir = Path(os.environ.get('HIL_REPORT_DIR', '.'))
failed_fname = report_dir / (config_file.name + '.failed')
- fresh = not (args.accumulate or args.board_test)
+ fresh = not args.accumulate
if fresh:
report_dir.mkdir(parents=True, exist_ok=True)
for f in (REPORT_JSON, REPORT_MD):
@@ -1935,7 +1945,11 @@ def main() -> None:
print(f'warning: cannot persist controller hints to {CONTROLLER_CACHE}: {e}')
# board x test result matrix -> hil_report.md (accumulates across re-runs) + stdout
- report = accumulate_report(mret, report_dir, fresh)
+ # -b/-bt in play means a filtered run (PR selection or a re-run spec): say so in the
+ # report, which otherwise looks exactly like a full run that happened to be small
+ scoped = sorted(set(args.board) | set(board_test))
+ scope = f'{len(scoped)} board(s) — {", ".join(scoped)}' if scoped else ''
+ report = accumulate_report(mret, report_dir, fresh, scope)
print()
print(report)
print(f'\nReport written to {(report_dir / REPORT_MD).resolve()}')
diff --git a/test/hil/test_hil_select.py b/test/hil/test_hil_select.py
new file mode 100644
index 000000000..6a2bf6210
--- /dev/null
+++ b/test/hil/test_hil_select.py
@@ -0,0 +1,542 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+# Unit tests for hil_select.py — pure logic, no hardware, no git. Run directly:
+# python3 test/hil/test_hil_select.py
+import glob
+import json
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+import hil_select
+from hil_examples import device_tests, dual_tests, host_test
+
+REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+def real_rosters():
+ """The actual rig rosters, for regression tests that need real-world data
+ (a specific board/family/only-list) rather than the synthetic ROSTER above."""
+ rosters = []
+ for name in ('tinyusb.json', 'hfp.json'):
+ path = os.path.join(REPO, 'test/hil', name)
+ with open(path) as f:
+ rosters.append((f'test/hil/{name}', json.load(f)['boards']))
+ return rosters
+
+
+def on_roster(tc, *names):
+ """The subset of `names` currently in the live rig rosters, skipping the test
+ when none are. Parking/unparking a board is routine rig maintenance and must not
+ fail this suite: CI runs it right before the selector and treats a failure as
+ 'selector unusable', dropping PR scoping and annotating the run."""
+ have = {b['name'] for _, boards in real_rosters() for b in boards}
+ got = [n for n in names if n in have]
+ if not got:
+ tc.skipTest(f'not in the rig roster: {", ".join(names)}')
+ return got
+
+
+ROSTER = [
+ # device-only, rp2040 family
+ {'name': 'raspberry_pi_pico', 'uid': 'u1', 'flasher': {'name': 'openocd'},
+ 'tests': {'device': True, 'host': True, 'dual': True}},
+ # device-only, stm32f4 family
+ {'name': 'stm32f407disco', 'uid': 'u2', 'flasher': {'name': 'jlink'},
+ 'tests': {'device': True, 'host': False, 'dual': False}},
+ # host-only board
+ {'name': 'raspberry_pi_pico2', 'uid': 'u3', 'flasher': {'name': 'openocd'},
+ 'tests': {'device': False, 'host': True, 'dual': False}},
+ # only-list board (espressif-style), flashed by the CI leg that splits on esptool
+ {'name': 'espressif_s3_devkitm', 'uid': 'u4', 'flasher': {'name': 'esptool'},
+ 'tests': {'only': ['device/cdc_msc_freertos', 'host/device_info']}},
+]
+ROSTERS = [('test/hil/tinyusb.json', ROSTER)]
+
+
+def sel(files):
+ return hil_select.classify(files, REPO, ROSTERS)
+
+
+class TestPortRule(unittest.TestCase):
+ def test_dcd_rp2040_selects_pico_family_only(self):
+ s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c'])
+ self.assertFalse(s['full'])
+ self.assertIn('raspberry_pi_pico', s['boards'])
+ self.assertNotIn('stm32f407disco', s['boards'])
+ self.assertNotIn('espressif_s3_devkitm', s['boards'])
+ # device role: no host tests in pico's list
+ self.assertTrue(all(not t.startswith('host/') for t in s['boards']['raspberry_pi_pico']))
+ # host-only boards drop out entirely on a device-role change
+ self.assertNotIn('raspberry_pi_pico2', s['boards'])
+
+ def test_shared_port_file_is_both_roles(self):
+ s = sel(['src/portable/synopsys/dwc2/dwc2_common.c'])
+ self.assertFalse(s['full'])
+ self.assertNotIn('raspberry_pi_pico', s['boards']) # rp2040 is not a dwc2 family
+ self.assertIn('stm32f407disco', s['boards']) # stm32f4 is
+
+
+class TestCoreRoleRule(unittest.TestCase):
+ def test_usbd_selects_all_device_tests_everywhere(self):
+ s = sel(['src/device/usbd.c'])
+ self.assertFalse(s['full'])
+ self.assertNotIn('raspberry_pi_pico2', s['boards']) # host-only board dropped
+ pico = s['boards']['raspberry_pi_pico']
+ self.assertTrue(set(device_tests).issubset(set(pico)))
+ self.assertTrue(set(dual_tests).issubset(set(pico))) # dual survives device role
+ self.assertTrue(all(not t.startswith('host/') for t in pico))
+ # only-list board: selection intersects its only-list
+ esp = s['boards']['espressif_s3_devkitm']
+ self.assertEqual(esp, ['device/cdc_msc_freertos'])
+
+ def test_host_change_drops_device(self):
+ s = sel(['src/host/usbh.c'])
+ self.assertFalse(s['full'])
+ self.assertIn('raspberry_pi_pico2', s['boards'])
+ self.assertNotIn('stm32f407disco', s['boards']) # device-only board dropped
+
+
+class TestClassRule(unittest.TestCase):
+ def test_cdc_device_selects_cdc_examples_only(self):
+ s = sel(['src/class/cdc/cdc_device.c'])
+ self.assertFalse(s['full'])
+ pico = s['boards']['raspberry_pi_pico']
+ self.assertIn('device/cdc_msc', pico)
+ self.assertIn('device/cdc_dual_ports', pico)
+ self.assertNotIn('device/msc_dual_lun', pico) # CFG_TUD_CDC 0 there
+ self.assertNotIn('device/usbtest', pico) # CFG_TUD_CDC 0 there
+ self.assertTrue(all(not t.startswith('host/') for t in pico))
+
+ def test_msc_host_selects_host_side(self):
+ s = sel(['src/class/msc/msc_host.c'])
+ self.assertFalse(s['full'])
+ self.assertNotIn('stm32f407disco', s['boards']) # device-only board
+ pico2 = s['boards']['raspberry_pi_pico2']
+ self.assertIn('host/msc_file_explorer', pico2)
+ self.assertTrue(all(not t.startswith('device/') for t in pico2))
+
+
+class TestClassIncludeEdges(unittest.TestCase):
+ """A class header another class includes reaches that class's examples too.
+ src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, so
+ midi_test's firmware contains audio.h - but the class rule derives macros from
+ the directory name alone, so an audio.h change used to select only
+ device/audio_test_freertos. On boards that skip that example the per-board
+ intersection emptied and an audio.h-only PR ran ZERO HIL on them."""
+ def test_edges_derived_from_includes(self):
+ edges = hil_select.class_include_edges(REPO)
+ self.assertEqual(edges.get('audio/audio.h'), {'midi'})
+ self.assertEqual(edges.get('cdc/cdc.h'), {'net'})
+
+ def test_audio_header_selects_midi_example(self):
+ s = hil_select.classify(['src/class/audio/audio.h'], REPO, real_rosters())
+ self.assertFalse(s['full'])
+ # every board that runs device/midi_test at all must run it here (boards with
+ # a tests.only list, e.g. espressif, run the freertos examples instead)
+ by_name = {b['name']: b for _, bs in real_rosters() for b in bs}
+ checked = 0
+ for name, tests in s['boards'].items():
+ if 'device/midi_test' in hil_select.board_tests(by_name[name]):
+ self.assertIn('device/midi_test', tests, name)
+ checked += 1
+ self.assertTrue(checked)
+
+ def test_audio_header_reaches_boards_that_skip_audio(self):
+ # both skip device/audio_test_freertos: without the midi edge their
+ # intersection is empty and they drop out of the selection entirely
+ boards = on_roster(self, 'metro_m4_express', 'nrf54lm20dk')
+ s = hil_select.classify(['src/class/audio/audio.h'], REPO, real_rosters())
+ for board in boards:
+ self.assertEqual(s['boards'].get(board), ['device/midi_test'], board)
+
+ def test_edge_is_per_header_not_per_class(self):
+ # midi includes audio.h, not audio_device.h: an audio_device change must
+ # not drag midi's examples in
+ s = hil_select.classify(['src/class/audio/audio_device.c'], REPO, real_rosters())
+ self.assertFalse(s['full'])
+ for tests in s['boards'].values():
+ if tests != 'all':
+ self.assertNotIn('device/midi_test', tests)
+
+
+class TestFallbackRules(unittest.TestCase):
+ def test_unknown_tool_is_full(self):
+ s = sel(['tools/random_new_script.py'])
+ self.assertTrue(s['full'])
+
+ def test_docs_only_is_empty_not_full(self):
+ s = sel(['docs/info/contributing.rst', 'README.rst'])
+ self.assertFalse(s['full'])
+ self.assertEqual(s['boards'], {})
+
+ def test_bsp_family_selects_family_boards(self):
+ s = sel(['hw/bsp/rp2040/family.cmake'])
+ self.assertFalse(s['full'])
+ self.assertIn('raspberry_pi_pico', s['boards'])
+ self.assertEqual(s['boards']['raspberry_pi_pico'], 'all')
+ self.assertNotIn('stm32f407disco', s['boards'])
+
+ def test_bsp_board_narrows_to_board(self):
+ s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h'])
+ self.assertFalse(s['full'])
+ self.assertEqual(list(s['boards'].keys()), ['raspberry_pi_pico'])
+
+ def test_example_change_selects_that_example(self):
+ s = sel(['examples/device/cdc_msc/src/main.c'])
+ self.assertFalse(s['full'])
+ self.assertEqual(s['boards']['raspberry_pi_pico'], ['device/cdc_msc'])
+
+ def test_core_common_is_full(self):
+ for f in ['src/tusb.c', 'src/common/tusb_fifo.c', 'src/osal/osal_freertos.h']:
+ self.assertTrue(sel([f])['full'], f)
+
+ def test_board_test_example_is_full(self):
+ # board_test is the park/teardown firmware hil_test.py flashes on every board,
+ # not an unlisted example: a regression there must not skip the whole rig
+ for f in ['examples/device/board_test/src/main.c',
+ 'examples/device/board_test/CMakeLists.txt']:
+ self.assertTrue(sel([f])['full'], f)
+
+ def test_harness_is_full(self):
+ for f in ['test/hil/hil_test.py', '.github/workflows/build.yml', 'hw/mcu/nxp/x.c', 'lib/foo/x.c']:
+ self.assertTrue(sel([f])['full'], f)
+
+ def test_mixed_roles_no_pruning(self):
+ s = sel(['src/device/usbd.c', 'src/host/usbh.c'])
+ self.assertFalse(s['full'])
+ self.assertIn('raspberry_pi_pico2', s['boards'])
+ self.assertIn('stm32f407disco', s['boards'])
+
+ def test_cmakelists_and_requirements_are_full(self):
+ for f in ['src/CMakeLists.txt', 'examples/CMakeLists.txt',
+ 'examples/device/CMakeLists.txt', 'test/hil/requirements.txt']:
+ self.assertTrue(sel([f])['full'], f)
+
+ def test_docs_txt_is_noncode(self):
+ s = sel(['docs/info/changelog.txt'])
+ self.assertFalse(s['full'])
+ self.assertEqual(s['boards'], {})
+
+
+class TestArgsEmission(unittest.TestCase):
+ def test_args_for_scoped_selection(self):
+ s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c'])
+ args = hil_select.selection_args(s, ROSTERS)
+ a = args['tinyusb.json']
+ self.assertIn('-b raspberry_pi_pico', a)
+ self.assertNotIn('stm32f407disco', a)
+ self.assertIn('-bt raspberry_pi_pico:', a) # device-only subset of a device+host board
+
+ def test_args_full_is_empty(self):
+ s = sel(['tools/random_new_script.py'])
+ self.assertEqual(hil_select.selection_args(s, ROSTERS), {'tinyusb.json': ''})
+
+ def test_args_all_board_gets_bare_b(self):
+ s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h'])
+ a = hil_select.selection_args(s, ROSTERS)['tinyusb.json']
+ self.assertIn('-b raspberry_pi_pico', a)
+ self.assertNotIn('-bt', a)
+
+ def test_args_by_flasher_splits_esp_from_the_rest(self):
+ s = sel(['src/device/usbd.c'])
+ per = hil_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json']
+ self.assertIn('espressif_s3_devkitm', per['esptool'])
+ self.assertIn('raspberry_pi_pico', per['openocd'])
+ self.assertNotIn('espressif_s3_devkitm', per.get('openocd', '') + per.get('jlink', ''))
+
+ def test_args_by_flasher_omits_a_flasher_with_no_selected_board(self):
+ # the esp CI leg must see no args at all here, not a filter matching zero boards
+ s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h'])
+ per = hil_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json']
+ self.assertEqual(per, {'openocd': '-b raspberry_pi_pico'})
+
+ def test_args_by_flasher_full_is_empty(self):
+ s = sel(['tools/random_new_script.py'])
+ self.assertEqual(hil_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}})
+
+ def test_cli_diff_file(self):
+ import subprocess, tempfile, json as j
+ with tempfile.NamedTemporaryFile('w', suffix='.txt', delete=False) as f:
+ f.write('src/class/cdc/cdc_device.c\n')
+ path = f.name
+ r = subprocess.run([sys.executable, os.path.join(REPO, 'test/hil/hil_select.py'),
+ '--diff-file', path, os.path.join(REPO, 'test/hil/tinyusb.json')],
+ capture_output=True, text=True)
+ self.assertEqual(r.returncode, 0, r.stderr)
+ out = j.loads(r.stdout)
+ self.assertFalse(out['full'])
+ self.assertIn('tinyusb.json', out['args'])
+ self.assertTrue(any('cdc_device' in line for line in out['reasons']))
+ os.unlink(path)
+
+
+class TestRealRosterPortFamilies(unittest.TestCase):
+ """Regression for port_families() missing espressif's dwc2 reference, which
+ lives in a component CMakeLists.txt rather than family.cmake/family.mk."""
+ def test_dwc2_change_selects_espressif_boards(self):
+ boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev')
+ s = hil_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters())
+ self.assertFalse(s['full'])
+ for board in boards:
+ self.assertIn(board, s['boards'])
+
+
+class TestOptionGatedPort(unittest.TestCase):
+ """Regression: family_support.cmake compiles some ports from a build option
+ (MAX3421_HOST=1 -> hcd_max3421.c), so a board's family file never names them."""
+ # host-side option board (max3421 as host controller), off any max3421 family
+ OPT_ROSTER = [('test/hil/opt.json', [
+ {'name': 'fake_dual_board', 'uid': 'o1', 'flasher': {'name': 'jlink'},
+ 'build': {'args': ['MAX3421_HOST=1']},
+ 'tests': {'device': True, 'host': False, 'dual': True}},
+ {'name': 'fake_host_board', 'uid': 'o2', 'flasher': {'name': 'jlink'},
+ 'variant': [{'name': 'fake_host_board', 'flags': '-DMAX3421_HOST=1'}],
+ 'tests': {'device': False, 'host': True, 'dual': False}},
+ {'name': 'fake_off_board', 'uid': 'o3', 'flasher': {'name': 'jlink'},
+ 'variant': [{'name': 'fake_off_board', 'defines': ['MAX3421_HOST=0']}],
+ 'tests': {'device': True, 'host': True, 'dual': True}},
+ ])]
+
+ def test_real_roster_max3421_selects_option_board(self):
+ boards = on_roster(self, 'metro_m4_express')
+ s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, real_rosters())
+ self.assertFalse(s['full'])
+ for board in boards:
+ self.assertIn(board, s['boards'])
+
+ def test_option_selects_via_args_defines_and_flags(self):
+ s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, self.OPT_ROSTER)
+ self.assertFalse(s['full'])
+ self.assertIn('fake_dual_board', s['boards']) # build.args
+ self.assertIn('fake_host_board', s['boards']) # variant flags
+ self.assertNotIn('fake_off_board', s['boards']) # variant defines, but =0
+
+ def test_device_role_port_does_not_pull_host_only_option_board(self):
+ s = hil_select.classify(['src/portable/analog/max3421/dcd_max3421.c'], REPO, self.OPT_ROSTER)
+ self.assertFalse(s['full'])
+ self.assertNotIn('fake_host_board', s['boards']) # host-only board, device change
+ self.assertIn('fake_dual_board', s['boards']) # device-capable option board
+
+ def test_gates_parsed_from_family_support(self):
+ self.assertEqual(hil_select.port_option_gates(REPO).get('analog/max3421'),
+ {'MAX3421_HOST'})
+
+ def test_board_cmake_option_counts(self):
+ """A board can enable a gated port in its own BSP rather than via the roster
+ (hw/bsp/espressif/boards/*/board.cmake -> set(MAX3421_HOST 1)); board_options()
+ must see those too, or such a board joining the roster is silently dropped."""
+ self.assertIn('MAX3421_HOST',
+ hil_select.bsp_board_options('adafruit_feather_esp32s3', REPO))
+ self.assertIn('CFG_TUH_RPI_PIO_USB',
+ hil_select.bsp_board_options('adafruit_fruit_jam', REPO))
+ # commented-out `# set(MAX3421_HOST 1)` must not count
+ self.assertNotIn('MAX3421_HOST',
+ hil_select.bsp_board_options('feather_nrf52840_express', REPO))
+
+ def test_board_cmake_option_selects_off_family_board(self):
+ # adafruit_feather_esp32s3 is not on any rig roster; stand it in as one to
+ # prove the BSP-sourced option alone pulls a max3421 change onto the board
+ roster = [('test/hil/opt.json', [
+ {'name': 'adafruit_feather_esp32s3', 'uid': 'o1', 'flasher': {'name': 'esptool'},
+ 'tests': {'device': False, 'host': True, 'dual': False}}])]
+ s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster)
+ self.assertFalse(s['full'])
+ self.assertIn('adafruit_feather_esp32s3', s['boards'])
+
+ def test_board_mk_option_is_ignored(self):
+ """Make-only options must not select: HIL CI builds with CMake exclusively, so
+ hw/bsp/nrf/boards/nrf5340dk/board.mk's MAX3421_HOST compiles nothing here."""
+ roster = [('test/hil/opt.json', [
+ {'name': 'nrf5340dk', 'uid': 'o1', 'flasher': {'name': 'jlink'},
+ 'tests': {'device': False, 'host': True, 'dual': False}}])]
+ s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster)
+ self.assertFalse(s['full'])
+ self.assertEqual(s['boards'], {})
+
+
+class TestPortFamiliesCmakeOnly(unittest.TestCase):
+ """port_families() is CMake-only (HIL CI never builds with Make) and matches on
+ 'port_dir/' so a port dir is not a prefix of a sibling."""
+ def test_make_only_family_is_not_a_family(self):
+ # hw/bsp/pic32mz has family.mk but no family.cmake
+ self.assertEqual(hil_select.port_families('microchip/pic32mz', REPO), set())
+
+ def test_prefix_port_does_not_inherit_sibling_families(self):
+ # bare-substring matching let 'microchip/pic' match '.../microchip/pic32mz/...'
+ self.assertEqual(hil_select.port_families('microchip/pic', REPO), set())
+
+ def test_make_only_port_forces_full(self):
+ s = sel(['src/portable/microchip/pic32mz/dcd_pic32mz.c'])
+ self.assertTrue(s['full'])
+ self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons'])
+
+ def test_cmake_families_still_found(self):
+ self.assertEqual(hil_select.port_families('raspberrypi/rp2040', REPO), {'rp2040'})
+ self.assertIn('stm32f4', hil_select.port_families('synopsys/dwc2', REPO))
+
+
+class TestPortFamiliesCoverage(unittest.TestCase):
+ """Systematic guard: every real dcd_*/hcd_* port directory should map to at
+ least one board family, so a future family.cmake/CMakeLists.txt layout that
+ port_families() doesn't scan fails loudly instead of silently dropping boards
+ (as espressif's dwc2 reference did - see TestRealRosterPortFamilies)."""
+ # Ports with no board family: not a bug, just not wired into any rig board.
+ # Add here (with a reason) only if port_families() legitimately can't find one.
+ # A port listed here force-fulls (fail-open), so it is never under-selected.
+ NO_FAMILY = {
+ 'template', # reference/example port, not built by any board
+ # hw/bsp/pic32mz has family.mk only (no family.cmake), and port_families()
+ # is CMake-only because HIL CI builds every board with CMake - so this port
+ # is compiled for no HIL board.
+ 'microchip/pic32mz',
+ 'microchip/pic', # same: only ever referenced from pic32mz's family.mk
+ }
+
+ @staticmethod
+ def _dcd_hcd_ports():
+ portable_root = os.path.join(REPO, 'src/portable')
+ ports = []
+ for entry in sorted(os.listdir(portable_root)):
+ d = os.path.join(portable_root, entry)
+ if not os.path.isdir(d):
+ continue
+ if glob.glob(os.path.join(d, 'dcd_*.c')) or glob.glob(os.path.join(d, 'hcd_*.c')):
+ ports.append(entry)
+ continue
+ for sub in sorted(os.listdir(d)):
+ sd = os.path.join(d, sub)
+ if os.path.isdir(sd) and (glob.glob(os.path.join(sd, 'dcd_*.c')) or
+ glob.glob(os.path.join(sd, 'hcd_*.c'))):
+ ports.append(f'{entry}/{sub}')
+ return ports
+
+ def test_every_port_maps_to_a_family(self):
+ ports = self._dcd_hcd_ports()
+ self.assertTrue(ports) # sanity: the scan itself found something
+ for port in ports:
+ if port in self.NO_FAMILY:
+ continue
+ fams = hil_select.port_families(port, REPO)
+ self.assertTrue(fams, f'{port}: no family references this port '
+ f'(port_families() scan gap, or add to NO_FAMILY)')
+
+
+class TestRealRosterOnlyListTests(unittest.TestCase):
+ """Regression for roster-only-list tests (e.g. espressif's hid_composite_freertos)
+ being invisible to the selector because it only knew the shared hil_examples lists."""
+ def test_only_list_example_change_selects_it(self):
+ boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev')
+ s = hil_select.classify(['examples/device/hid_composite_freertos/src/main.c'], REPO, real_rosters())
+ self.assertFalse(s['full'])
+ for board in boards:
+ self.assertEqual(s['boards'][board], ['device/hid_composite_freertos'])
+
+ def test_class_change_includes_only_list_boards(self):
+ boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev')
+ s = hil_select.classify(['src/class/hid/hid_device.c'], REPO, real_rosters())
+ self.assertFalse(s['full'])
+ for board in boards:
+ self.assertIn(board, s['boards'])
+
+
+class TestPortAndCoreRoleUseExtras(unittest.TestCase):
+ """Regression: the port rule and core-role rule must thread the roster-only
+ test universe (extras) the same way the class rule already does, so a DCD
+ or device-stack change doesn't silently drop espressif's only-list tests
+ (e.g. hid_composite_freertos) that aren't in the shared device_tests list."""
+ def test_dcd_change_includes_only_list_test(self):
+ boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev')
+ s = hil_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters())
+ self.assertFalse(s['full'])
+ for board in boards:
+ tests = s['boards'][board]
+ self.assertIn('device/hid_composite_freertos', tests)
+ self.assertIn('device/cdc_msc_freertos', tests)
+ self.assertIn('device/audio_test_freertos', tests)
+ self.assertIn('device/usbtest', tests)
+
+ def test_core_device_change_includes_only_list_test(self):
+ boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev')
+ s = hil_select.classify(['src/device/usbd.c'], REPO, real_rosters())
+ self.assertFalse(s['full'])
+ for board in boards:
+ tests = s['boards'][board]
+ self.assertIn('device/hid_composite_freertos', tests)
+ self.assertIn('device/cdc_msc_freertos', tests)
+ self.assertIn('device/audio_test_freertos', tests)
+ self.assertIn('device/usbtest', tests)
+
+ def test_host_change_does_not_leak_device_only_list_test(self):
+ s = hil_select.classify(['src/host/usbh.c'], REPO, real_rosters())
+ self.assertFalse(s['full'])
+ for board, tests in s['boards'].items():
+ if tests == 'all':
+ continue
+ self.assertNotIn('device/hid_composite_freertos', tests, board)
+
+
+class TestFamilies(unittest.TestCase):
+ """`families` exists for consumers that build (not just test) the diff: most
+ families have no rig board, so `boards` alone would compile nothing for them."""
+ def test_off_rig_port_still_reports_family(self):
+ s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c'])
+ self.assertFalse(s['full'])
+ self.assertEqual(s['boards'], {}) # no same7x board on the rig
+ self.assertEqual(s['families'], ['same7x'])
+
+ def test_port_families_are_reported(self):
+ s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c'])
+ self.assertIn('rp2040', s['families'])
+
+ def test_bsp_family_and_board_report_family(self):
+ self.assertEqual(sel(['hw/bsp/rp2040/family.cmake'])['families'], ['rp2040'])
+ self.assertEqual(sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h'])['families'],
+ ['rp2040'])
+
+ def test_docs_only_has_no_families(self):
+ self.assertEqual(sel(['docs/info/contributing.rst'])['families'], [])
+
+ def test_full_selection_still_reports_families(self):
+ """A full-matrix file must not hide the families of the other changed files:
+ consumers that build from `families` (e.g. /pre-pr) ignore `boards` when full."""
+ s = sel(['src/common/tusb_fifo.c', 'src/portable/microchip/samx7x/dcd_samx7x.c'])
+ self.assertTrue(s['full'])
+ self.assertIn('same7x', s['families'])
+ # full stays full: every roster board, and no args to narrow the run
+ self.assertEqual(set(s['boards']), {b['name'] for b in ROSTER})
+ self.assertTrue(all(v == 'all' for v in s['boards'].values()))
+ self.assertEqual(hil_select.selection_args(s, ROSTERS), {'tinyusb.json': ''})
+ self.assertEqual(hil_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}})
+
+ def test_family_order_does_not_matter(self):
+ # same as above with the full-matrix file last (was the only order that worked)
+ s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c', 'src/common/tusb_fifo.c'])
+ self.assertTrue(s['full'])
+ self.assertIn('same7x', s['families'])
+
+
+class TestGitDiffArgv(unittest.TestCase):
+ def test_diff_disables_rename_detection(self):
+ """Without --no-renames git reports only a rename's destination, so moving an
+ HIL-relevant file to a non-code path would be classified as non-code only."""
+ self.assertIn('--no-renames', hil_select.GIT_DIFF_ARGV)
+
+
+class TestPortWithoutFamilyIsFull(unittest.TestCase):
+ """A port dir no family file references must widen (full matrix), not silently
+ contribute zero boards — the fail-open contract."""
+ def test_unreferenced_port_forces_full(self):
+ orig = hil_select.port_families
+ hil_select.port_families = lambda port_dir, repo_root: set()
+ try:
+ s = sel(['src/portable/vendor/newip/dcd_newip.c'])
+ finally:
+ hil_select.port_families = orig
+ self.assertTrue(s['full'])
+ self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons'])
+
+
+if __name__ == '__main__':
+ unittest.main(verbosity=1)