diff options
| author | Ha Thach <[email protected]> | 2026-07-30 02:29:32 +0700 |
|---|---|---|
| committer | GitHub <[email protected]> | 2026-07-30 02:29:32 +0700 |
| commit | eef5af86aa26fe3d72e41156a586a6ed3ffce9f8 (patch) | |
| tree | fa5ebe3a8d46738338eaee2d4a80717a7a10f464 /test | |
| parent | 5d8afbb23248e9d1d5d1fdf434c9c58cc54ba5b7 (diff) | |
hil, ci: scope HIL builds and tests to the boards a PR affects (#3797)
hil, ci: scope HIL builds and tests to the boards a PR affects
Add test/hil/hil_select.py, a stdlib-only selector that maps a PR diff to the
rig boards, tests and BSP families a change can affect, and wire it into CI so
pull requests build and run only those. A port change picks its families' boards,
a class change picks the examples enabling that class, and device/host changes
prune the other role. Anything unclassified — infra, an unmapped port, a selector
error — falls back to the full matrix, and push/schedule runs are untouched.
Move the shared example lists to hil_examples.py; 54 hardware-free tests cover
the rules.
Diffstat (limited to 'test')
| -rw-r--r-- | test/hil/hil_ci.sh | 1 | ||||
| -rw-r--r-- | test/hil/hil_ci_set_matrix.py | 15 | ||||
| -rw-r--r-- | test/hil/hil_examples.py | 37 | ||||
| -rwxr-xr-x | test/hil/hil_select.py | 520 | ||||
| -rwxr-xr-x | test/hil/hil_test.py | 102 | ||||
| -rw-r--r-- | test/hil/test_hil_select.py | 542 |
6 files changed, 1173 insertions, 44 deletions
diff --git a/test/hil/hil_ci.sh b/test/hil/hil_ci.sh index 3384b4e2e..ef93bcb49 100644 --- a/test/hil/hil_ci.sh +++ b/test/hil/hil_ci.sh @@ -55,6 +55,7 @@ scp -q "$ROOT_DIR/test/hil/hil_test.py" \ "$ROOT_DIR/test/hil/hil_flash.py" \ "$ROOT_DIR/test/hil/hil_lock.py" \ "$ROOT_DIR/test/hil/usbtest.py" \ + "$ROOT_DIR/test/hil/hil_examples.py" \ "$ROOT_DIR/test/hil/pymtp.py" \ "$CONFIG" \ "$REMOTE:$REMOTE_DIR/test/hil/" diff --git a/test/hil/hil_ci_set_matrix.py b/test/hil/hil_ci_set_matrix.py index 13f7f1882..bca989bd1 100644 --- a/test/hil/hil_ci_set_matrix.py +++ b/test/hil/hil_ci_set_matrix.py @@ -17,8 +17,14 @@ def _resolve_config_path(config_file): def main(): parser = argparse.ArgumentParser() parser.add_argument('config_files', nargs='+', help='Configuration JSON file(s)') + parser.add_argument('--select', help='hil_select.py JSON; scopes boards when full=false') args = parser.parse_args() + selected = None + sel = json.loads(args.select) if args.select else None + if sel and not sel.get('full'): + selected = set(sel.get('boards', {})) + # Toolchain buckets must match the toolchains instantiated by the hil-build # job in .github/workflows/build.yml. Keep all keys present (even if empty) # so `fromJSON(hil_json)[toolchain]` always resolves to a list. @@ -40,6 +46,8 @@ def main(): config = json.load(f) for board in config['boards']: + if selected is not None and board['name'] not in selected: + continue name = board['name'] flasher = board['flasher'] # esptool boards must build under esp-idf; others default to arm-gcc @@ -49,6 +57,13 @@ def main(): toolchain = 'esp-idf' else: toolchain = board.get('toolchain', 'arm-gcc') + if toolchain not in matrix: + # a board in no bucket would never be built, and the bare KeyError + # below would only say so as a traceback from the set-matrix job + raise SystemExit( + f'{name}: toolchain {toolchain!r} is not a build bucket ' + f'({", ".join(matrix)}); add it here and to the hil-build / ' + f'hil-build-esp jobs in .github/workflows/build.yml') build_board = f'-b {name}' if 'build' in board and 'args' in board['build']: diff --git a/test/hil/hil_examples.py b/test/hil/hil_examples.py new file mode 100644 index 000000000..4c8b6918b --- /dev/null +++ b/test/hil/hil_examples.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# HIL example test lists, shared by hil_test.py (runner) and hil_select.py +# (PR-diff selector). Stdlib-only: hil_select runs on bare CI runners. + +# The per-board run order is shuffled (see test_board). +# Every example carries a unique hardcoded idProduct (see its usb_descriptors.c) + +# device tests +device_tests = [ + 'device/cdc_dual_ports', + 'device/cdc_msc', + 'device/dfu', + 'device/cdc_msc_throughput', + 'device/audio_test_freertos', + 'device/dfu_runtime', + 'device/cdc_msc_freertos', + 'device/hid_boot_interface', + 'device/msc_dual_lun', + 'device/hid_generic_inout', + 'device/printer_to_cdc', + 'device/midi_test', + 'device/mtp', + 'device/usbtest', # cafe:4010, unique PID; runs the Linux testusb tier-4 battery via usbtest.py + # 'device/net_lwip_webserver', # disabled for PR #3605: USB net iface enum is flaky on the CI HIL host +] + +dual_tests = [ + 'dual/host_info_to_device_cdc', +] + +host_test = [ + 'host/cdc_msc_hid', + 'host/msc_file_explorer', + 'host/msc_file_explorer_freertos', + 'host/device_info', +] diff --git a/test/hil/hil_select.py b/test/hil/hil_select.py new file mode 100755 index 000000000..3ac3f1fdb --- /dev/null +++ b/test/hil/hil_select.py @@ -0,0 +1,520 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""PR-diff -> HIL selection: which rig boards and which tests a change can affect. + +Stdlib-only (runs on bare CI runners; never imports hil_test/hil_flash/hil_lock). +Fail-open: any file no rule classifies forces the full matrix. See +docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md. + +JSON: full, boards (name -> 'all' | [tests]), families (bsp families the diff +touches, including ones with no rig board - build-only consumers such as /pre-pr +sample from these), args (hil_test.py args per config) and args_flasher (the same +args split by each board's flasher, for CI legs that split one rig by flasher). +""" +import argparse +import functools +import glob +import json +import os +import re +import subprocess +import sys + +from hil_examples import device_tests, dual_tests, host_test + +ALL_TESTS = {'device': device_tests, 'dual': dual_tests, 'host': host_test} + +# class dir -> config macro suffix exceptions (rule 3); dfu is per-file, handled inline +NET_MACROS = ('ECM_RNDIS', 'NCM') + +_NONCODE_RE = re.compile( + r'^(docs/|\.claude/|.*\.(md|rst)$|LICENSE)') +_FULL_RE = re.compile( + r'^(src/common/|src/osal/|src/tusb\.c$|src/tusb\.h$|src/tusb_option\.h$|' + r'test/hil/|\.github/workflows/build.*\.yml$|\.github/actions/|' + r'tools/build\.py$|tools/get_deps\.py$|tools/cmake/|hw/mcu/|lib/|' + r'hw/bsp/(family_support\.cmake|board_api\.h|board\.c|ansi_escape\.h)$|' + r'examples/build_system/|examples/CMakeLists\.txt$|' + # board_test is HIL infrastructure, not a test: hil_test.py flashes it to park + # every board (variant boundary + end-of-board teardown), so every board depends on it + r'examples/device/board_test/)') + +# --no-renames: with rename detection git reports only a rename's destination, so code +# moved out of an HIL-relevant path would be classified by its new path alone +GIT_DIFF_ARGV = ['git', 'diff', '--no-renames', '--name-only'] + + +def test_role(test: str) -> str: + return test.split('/', 1)[0] # 'device' | 'dual' | 'host' + + +def board_roles(board: dict) -> set: + t = board.get('tests', {}) + roles = set() + if t.get('device'): + roles.add('device') + if t.get('host'): + roles.add('host') + if t.get('dual'): + roles.update(('device', 'host')) + for only in t.get('only', []): + r = test_role(only) + roles.update(('device', 'host') if r == 'dual' else (r,)) + return roles + + +def board_tests(board: dict) -> list: + """Every test this board would run today (mirrors hil_test.test_board's default).""" + t = board.get('tests', {}) + if 'only' in t: + run = list(t['only']) + else: + run = [] + if t.get('device'): + run += device_tests + if t.get('dual'): + run += dual_tests + if t.get('host'): + run += host_test + return [x for x in run if x not in t.get('skip', [])] + + +# cached: called per changed file x roster board, and the tree doesn't change mid-run [email protected]_cache(maxsize=None) +def board_family(board_name: str, repo_root: str): + hits = glob.glob(os.path.join(repo_root, 'hw/bsp/*/boards', board_name)) + return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None + + +# `if (OPTION STREQUAL "1")` guards in family_support.cmake, and the option tokens +# a roster entry passes to the build (NAME=VALUE / -DNAME=VALUE) +_CM_IF_RE = re.compile(r'if\s*\(') +_CM_ELSE_RE = re.compile(r'else(if)?\s*\(') +_CM_ENDIF_RE = re.compile(r'endif\s*\(') +_CM_OPT_RE = re.compile(r'if\s*\(\s*\$?\{?([A-Za-z_]\w*)\}?\s+STREQUAL\s+"?1"?\s*\)') +_CM_PORT_RE = re.compile(r'src/portable/((?:[^/\s]+/)?[^/\s]+)/') +_FALSY = ('', '0', 'off', 'false', 'no') + + [email protected]_cache(maxsize=None) +def port_option_gates(repo_root: str) -> dict: + """port dir -> build options that compile it regardless of the board's family + file, e.g. {'analog/max3421': {'MAX3421_HOST'}} from family_support.cmake.""" + gates = {} + try: + text = open(os.path.join(repo_root, 'hw/bsp/family_support.cmake')).read() + except OSError: + return gates + stack = [] # one entry per open if(): its option, or None + for line in text.splitlines(): + line = line.strip() + if _CM_IF_RE.match(line): + m = _CM_OPT_RE.match(line) + stack.append(m.group(1) if m else None) + elif _CM_ELSE_RE.match(line): + if stack: + stack[-1] = None # the guard doesn't hold in this branch + elif _CM_ENDIF_RE.match(line): + if stack: + stack.pop() + opts = {o for o in stack if o} + m = _CM_PORT_RE.search(line) + if opts and m: + gates.setdefault(m.group(1), set()).update(opts) + return gates + + +_CM_SET_RE = re.compile(r'set\s*\(\s*([A-Za-z_]\w*)\s+([^)\s]+)\s*\)') + + +# cached: called per changed portable file x roster board [email protected]_cache(maxsize=None) +def bsp_board_options(board_name: str, repo_root: str) -> frozenset: + """Build options a board turns on in its own BSP: `set(<OPT> <value>)` in + hw/bsp/<family>/boards/<board>/board.cmake, e.g. MAX3421_HOST on the espressif + and rp2040 max3421 boards. CMake only - HIL CI builds nothing with Make, so a + board.mk-only option (e.g. nrf5340dk's MAX3421_HOST) compiles no port here.""" + fam = board_family(board_name, repo_root) + if not fam: + return frozenset() + path = os.path.join(repo_root, 'hw/bsp', fam, 'boards', board_name, 'board.cmake') + try: + text = open(path).read() + except OSError: + return frozenset() + out = set() + for line in text.splitlines(): + line = line.strip() + if line.startswith('#'): + continue + m = _CM_SET_RE.match(line) + if m and m.group(2).strip('"').lower() not in _FALSY: + out.add(m.group(1)) + return frozenset(out) + + +def board_options(board: dict, repo_root: str) -> set: + """Build options a board has truthy: the roster entry's build.args plus each + variant's defines (NAME=VALUE) and raw CFLAGS (-DNAME=VALUE), plus whatever its + own board.cmake sets (a board can enable a gated port without the roster saying so).""" + toks = list(board.get('build', {}).get('args', [])) + for v in board.get('variant', []): + toks += list(v.get('defines', [])) + toks += v.get('flags', '').split() + out = set(bsp_board_options(board['name'], repo_root)) + for t in toks: + name, _, val = (t[2:] if t.startswith('-D') else t).partition('=') + if name and val.strip().strip('"').lower() not in _FALSY: + out.add(name.strip()) + return out + + [email protected]_cache(maxsize=None) +def port_families(port_dir: str, repo_root: str) -> set: + """Board families that compile this src/portable dir. CMake only: HIL CI builds + every board with CMake, so a port wired up in family.mk alone is compiled for no + HIL board and must not select one. family.cmake lists portable sources directly + for most families; espressif instead references them from a nested component + CMakeLists.txt (hw/bsp/espressif/components/tinyusb_src/CMakeLists.txt).""" + fams = set() + bsp_root = os.path.join(repo_root, 'hw/bsp') + # trailing '/' so a port dir is not a prefix of a sibling: bare 'microchip/pic' + # would otherwise match '.../microchip/pic32mz/...' and inherit its families + needle = port_dir + '/' + for f in glob.glob(os.path.join(bsp_root, '*/family.cmake')) + \ + glob.glob(os.path.join(bsp_root, '*/components/*/CMakeLists.txt')): + try: + if needle in open(f).read(): + fam = os.path.relpath(f, bsp_root).split(os.sep, 1)[0] + fams.add(fam) + except OSError: + pass + return fams + + +_CLS_INC_RE = re.compile(r'#\s*include\s*[<"]class/([^/"<>]+)/([^"<>]+)[">]') + + [email protected]_cache(maxsize=None) +def class_include_edges(repo_root: str) -> dict: + """'<class>/<header>' -> the other class dirs that include it. A class header + pulled in by a second class ships in every firmware enabling that second class: + src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, and + net_device.h includes class/cdc/cdc.h. The class rule derives macros from the + directory name alone, so without this edge a change to the included header + selects only its own class's examples - and on a board that skips those (e.g. + metro_m4_express skips audio_test_freertos), nothing at all. + + Derived from the actual #include lines rather than a hand-written table so it + cannot rot when a class picks up or drops a cross-class include.""" + edges = {} + for f in sorted(glob.glob(os.path.join(repo_root, 'src/class/*/*.[ch]'))): + cls = os.path.basename(os.path.dirname(f)) + try: + text = open(f).read() + except OSError: + continue + for inc_cls, inc_hdr in _CLS_INC_RE.findall(text): + if inc_cls != cls: + edges.setdefault(f'{inc_cls}/{inc_hdr}', set()).add(cls) + return edges + + +def class_macros(cls: str, base: str, prefix: str) -> list: + """Config macros that compile a class dir's code, for role prefix TUD/TUH. + `base` refines dfu only (it splits DFU from DFU_RUNTIME per file); pass '' for + a class reached through an include edge, where the widest set is correct.""" + if cls == 'net': + return [f'CFG_{prefix}_{m}' for m in NET_MACROS] + if cls == 'dfu': + if base.startswith('dfu_rt'): + return [f'CFG_{prefix}_DFU_RUNTIME'] + if base.startswith('dfu_device') or base.startswith('dfu_host'): + return [f'CFG_{prefix}_DFU'] + return [f'CFG_{prefix}_DFU', f'CFG_{prefix}_DFU_RUNTIME'] + return [f'CFG_{prefix}_{cls.upper()}'] + + +def _config_enables(cfg_path: str, macros) -> bool: + try: + text = open(cfg_path).read() + except OSError: + return False + return any(re.search(rf'#define\s+{m}\s+\(?\s*0*[1-9]', text) for m in macros) + + +def roster_only_tests(all_boards) -> set: + """Test paths that only appear in a roster board's tests.only list (e.g. + espressif boards), not in the shared device/dual/host_test lists.""" + out = set() + for b in all_boards: + out.update(b.get('tests', {}).get('only', [])) + return out + + +def class_examples(macros, role: str, repo_root: str, extra_tests: set) -> set: + """Tests (from role's + dual lists, plus roster-only-list tests of that role) + whose example config enables any macro.""" + pool = role_tests({role}, extra_tests) + out = set() + for test in pool: + cfg = os.path.join(repo_root, 'examples', test, 'src', 'tusb_config.h') + if _config_enables(cfg, macros): + out.add(test) + return out + + +def role_tests(roles: set, extras: set) -> set: + """Every test for the given role(s): each role's own list + dual tests, + plus roster-only-list tests (extras) matching those roles or 'dual'.""" + pool = set(dual_tests) + for r in roles: + pool |= set(ALL_TESTS[r]) + pool |= {t for t in extras if test_role(t) in roles or test_role(t) == 'dual'} + return pool + + +class _Sel: + """Accumulates contributions. board->set(tests) plus 'all-board' markers.""" + def __init__(self): + self.full = False + self.by_board = {} # name -> set of tests, or 'all' + self.roles = set() # roles touched by any contribution + self.families = set() # bsp families touched (incl. off-rig ones: build-only consumers) + self.reasons = [] + + def add(self, boards, tests, reason): + """tests: 'all' or iterable of test paths.""" + self.reasons.append(reason) + for b in boards: + cur = self.by_board.get(b) + if tests == 'all' or cur == 'all': + self.by_board[b] = 'all' + else: + self.by_board[b] = (cur or set()) | set(tests) + + def force_full(self, reason): + self.full = True + self.reasons.append(reason) + + +def _classify_one(path, repo_root, roster_boards, extras: set, s: _Sel): + base = os.path.basename(path) + if _NONCODE_RE.match(path): + s.reasons.append(f'{path}: non-code, no contribution') + return + if _FULL_RE.match(path): + s.force_full(f'{path}: core/infra -> full matrix') + return + + m = re.match(r'src/portable/((?:[^/]+/)?[^/]+)/', path) + if m: + port = m.group(1) + if re.match(r'(dcd_|.*_device)', base): + roles = {'device'} + elif re.match(r'(hcd_|.*_host)', base): + roles = {'host'} + else: + roles = {'device', 'host'} + fams = port_families(port, repo_root) + if not fams: + # no family references this port: either a new/renamed port dir or a + # family.cmake layout the scan misses - widen instead of contributing nothing + s.force_full(f'{path}: port {port} maps to no board family -> full matrix') + return + s.families.update(fams) + # a board can also pull the port in through a build option (e.g. MAX3421_HOST=1 + # from the roster on metro_m4_express, or from its own board.cmake), which its + # family file never names + gates = port_option_gates(repo_root).get(port, set()) + boards = [b['name'] for b in roster_boards + if (board_family(b['name'], repo_root) in fams or + (gates and board_options(b, repo_root) & gates)) and (board_roles(b) & roles)] + tests = role_tests(roles, extras) + s.roles.update(roles) + why = f'{path}: port {port} -> families {sorted(fams)}' + if gates: + why += f' + option {sorted(gates)}' + s.add(boards, tests, f'{why} -> boards {boards} ({"/".join(sorted(roles))})') + return + + m = re.match(r'src/class/([^/]+)/', path) + if m: + cls = m.group(1) + if re.search(r'_device\.[ch]$', base): + roles = {'device'} + elif re.search(r'_host\.[ch]$', base): + roles = {'host'} + else: + roles = {'device', 'host'} + # this file's own class, plus any class whose headers include it + via = sorted(class_include_edges(repo_root).get(f'{cls}/{base}', ())) + + def macros(prefix): + return (class_macros(cls, base, prefix) + + [m2 for c in via for m2 in class_macros(c, '', prefix)]) + tests = set() + if 'device' in roles: + tests |= class_examples(macros('TUD'), 'device', repo_root, extras) + if 'host' in roles: + tests |= class_examples(macros('TUH'), 'host', repo_root, extras) + boards = [b['name'] for b in roster_boards if board_roles(b) & roles] + s.roles.update(roles) + why = f'{path}: class {cls}' + (f' (+ included by {via})' if via else '') + s.add(boards, tests, f'{why} -> {sorted(tests)} ({"/".join(sorted(roles))})') + return + + m = re.match(r'src/(device|host)/', path) + if m: + role = m.group(1) + boards = [b['name'] for b in roster_boards if role in board_roles(b)] + s.roles.add(role) + s.add(boards, role_tests({role}, extras), f'{path}: core {role} stack -> all {role} tests') + return + + m = re.match(r'hw/bsp/([^/]+)/(?:boards/([^/]+)/)?', path) + if m: + fam, brd = m.group(1), m.group(2) + s.families.add(fam) + if brd: + boards = [b['name'] for b in roster_boards if b['name'] == brd] + why = f'{path}: bsp board {brd}' + else: + boards = [b['name'] for b in roster_boards + if board_family(b['name'], repo_root) == fam] + why = f'{path}: bsp family {fam}' + s.roles.update(('device', 'host')) + s.add(boards, 'all', f'{why} -> boards {boards}') + return + + m = re.match(r'examples/(device|host|dual)/([^/]+)/', path) + if m: + test = f'{m.group(1)}/{m.group(2)}' + known = any(test in pool for pool in ALL_TESTS.values()) or test in extras + if known: + boards = [b['name'] for b in roster_boards] + role = test_role(test) + s.roles.update(('device', 'host') if role == 'dual' else (role,)) + s.add(boards, [test], f'{path}: example -> {test} on all boards') + else: + s.reasons.append(f'{path}: example not in HIL lists, no contribution') + return + + s.force_full(f'{path}: unclassified -> full matrix') + + +def classify(changed_files, repo_root, rosters): + all_boards = [] + seen = set() + for _, boards in rosters: + for b in boards: + if b['name'] not in seen: + seen.add(b['name']) + all_boards.append(b) + + extras = roster_only_tests(all_boards) + s = _Sel() + # no early exit once full: keep classifying so `families` still reports every + # family the diff touches (build-only consumers need it). Nothing after the first + # force_full can change full/boards/args - the full branch below ignores by_board. + for path in changed_files: + _classify_one(path, repo_root, all_boards, extras, s) + + if s.full: + return {'full': True, 'boards': {b['name']: 'all' for b in all_boards}, + 'families': sorted(s.families), 'reasons': s.reasons} + + # role pruning: single-role selections drop the other role's tests and boards + by_name = {b['name']: b for b in all_boards} + out = {} + for name, tests in s.by_board.items(): + allowed = board_tests(by_name[name]) + if tests == 'all': + kept = list(allowed) + else: + kept = [t for t in allowed if t in tests] + if s.roles and s.roles != {'device', 'host'}: + role = next(iter(s.roles)) + kept = [t for t in kept if test_role(t) in (role, 'dual')] + if kept: + out[name] = 'all' if set(kept) == set(allowed) else sorted(kept) + return {'full': False, 'boards': out, 'families': sorted(s.families), + 'reasons': s.reasons} + + +def _board_args(name, chosen) -> list: + parts = [f'-b {name}'] + if chosen != 'all': + parts.append(f'-bt {name}:{",".join(chosen)}') + return parts + + +def selection_args(sel, rosters): + """hil_test.py args per config. Empty means either 'full matrix' or 'nothing + selected' - callers must read sel['full'] to tell them apart.""" + args = {} + for cfg_path, boards in rosters: + parts = [] + if not sel['full']: + for b in boards: + chosen = sel['boards'].get(b['name']) + if chosen is not None: + parts += _board_args(b['name'], chosen) + args[os.path.basename(cfg_path)] = ' '.join(parts) + return args + + +def selection_args_by_flasher(sel, rosters): + """{config: {flasher name: args}}. CI runs one rig as several jobs split by + flasher (esptool vs the rest); each must gate on its own subset, otherwise the + other leg runs a filter matching zero boards and reports a vacuous green.""" + out = {} + for cfg_path, boards in rosters: + per = {} + if not sel['full']: + for b in boards: + chosen = sel['boards'].get(b['name']) + if chosen is None: + continue + per.setdefault(b.get('flasher', {}).get('name', ''), []).extend( + _board_args(b['name'], chosen)) + out[os.path.basename(cfg_path)] = {f: ' '.join(p) for f, p in per.items()} + return out + + +def changed_files_from_git(base, repo_root): + mb = subprocess.run(['git', 'merge-base', 'HEAD', base], cwd=repo_root, + capture_output=True, text=True, check=True).stdout.strip() + diff = subprocess.run(GIT_DIFF_ARGV + [f'{mb}..HEAD'], cwd=repo_root, + capture_output=True, text=True, check=True).stdout + return [l for l in diff.splitlines() if l.strip()] + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + g = ap.add_mutually_exclusive_group(required=True) + g.add_argument('--base', help='git ref to diff against (merge-base..HEAD)') + g.add_argument('--diff-file', help='newline-separated changed-file list') + ap.add_argument('configs', nargs='+', help='rig roster JSON file(s)') + a = ap.parse_args() + + repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + rosters = [] + for c in a.configs: + with open(c) as f: + rosters.append((c, json.load(f)['boards'])) + + files = (open(a.diff_file).read().splitlines() if a.diff_file + else changed_files_from_git(a.base, repo_root)) + files = [f for f in files if f.strip()] + + s = classify(files, repo_root, rosters) + s['args'] = selection_args(s, rosters) + s['args_flasher'] = selection_args_by_flasher(s, rosters) + for r in s['reasons']: + print(f'hil_select: {r}', file=sys.stderr) + print(json.dumps(s)) + + +if __name__ == '__main__': + main() diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py index 71e85f55f..96d52e601 100755 --- a/test/hil/hil_test.py +++ b/test/hil/hil_test.py @@ -59,6 +59,7 @@ from multiprocessing import TimeoutError as MpTimeoutError import hil_flash import hil_lock +from hil_examples import device_tests, dual_tests, host_test # Raw Lock/Semaphore objects passed via Pool initargs are inheritable only under the fork # start method (spawn/forkserver pickle them and fail at Pool creation) — pin it so a @@ -1351,39 +1352,6 @@ def test_device_usbtest(board): # Main # ------------------------------------------------------------- -# The per-board run order is shuffled (see test_board). -# Every example carries a unique hardcoded idProduct (see its usb_descriptors.c) - -# device tests -device_tests = [ - 'device/cdc_dual_ports', - 'device/cdc_msc', - 'device/dfu', - 'device/cdc_msc_throughput', - 'device/audio_test_freertos', - 'device/dfu_runtime', - 'device/cdc_msc_freertos', - 'device/hid_boot_interface', - 'device/msc_dual_lun', - 'device/hid_generic_inout', - 'device/printer_to_cdc', - 'device/midi_test', - 'device/mtp', - 'device/usbtest', # cafe:4010, unique PID; runs the Linux testusb tier-4 battery via usbtest.py - # 'device/net_lwip_webserver', # disabled for PR #3605: USB net iface enum is flaky on the CI HIL host -] - -dual_tests = [ - 'dual/host_info_to_device_cdc', -] - -host_test = [ - 'host/cdc_msc_hid', - 'host/msc_file_explorer', - 'host/msc_file_explorer_freertos', - 'host/device_info', -] - def test_example(board: Board, variant: str, example: str) -> tuple[int, str, str | None]: """ @@ -1517,6 +1485,10 @@ def build_board(board: Board) -> tuple[str, int]: return name, failed +# pseudo-test column for a variant boundary the park-flash could not clear (see below) +BOUNDARY_CELL = 'same-PID boundary' + + def test_board(board: Board) -> tuple[str, int, list[str], list, float]: name = board['name'] flasher = board['flasher'] @@ -1568,6 +1540,7 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: err_count = 0 failed_tests = [] + board_wide_fail = False # re-run the whole board, not a subset of its tests rows = [] # list of (row_label, {example: status}, duration) — one row per build variant # a -t/-bt filtered run times only a subset; report no duration so an accumulate # re-run keeps the previous full-run value @@ -1587,10 +1560,36 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: random.Random(f'{shuffle_seed}:{name}:{vname}').shuffle(run_list) if run_list[0] == prev_last: run_list[0], run_list[-1] = run_list[-1], run_list[0] + cells = {} + if run_list and run_list[0] == prev_last and not skip_flash: + # Same example (same PID) still repeats across the boundary: a one-test + # list (the common case for a -bt scoped run) leaves nothing to swap + # with. Park on board_test first - it disables the board's USB, so the + # PID goes away and the next flash must re-enumerate to be seen. + t_park = time.monotonic() + park_ec, park_status, _ = test_example(board, vname, 'device/board_test') + if park_ec or park_status == 'skip': + # Boundary not cleared: the previous variant's device may still be + # enumerated under the same PID, so this variant's tests could pass + # against its firmware. Skip them - a false green proves nothing and + # is worse than a gap - and record the boundary itself as the failure + # (a visible ❌ cell, mirroring the board-lock row above) so the report + # matches the exit code instead of rendering all-green. + why = 'no board_test binary' if park_status == 'skip' else 'park flash failed' + log_line(f'{vname:40} {"same-PID boundary":30} {STATUS_FAILED}: not cleared ({why}); ' + f'skipping {len(run_list)} test(s) on this variant') + err_count += 1 + cells[BOUNDARY_CELL] = 'fail' + # blaming run_list[0] would re-run an innocent test that then passes, + # leaving the boundary unretested; re-run the whole board instead + board_wide_fail = True + # leave prev_last alone: the board still holds the previous variant's + # firmware, so the next variant must attempt the park again + run_list = [] + t_board += time.monotonic() - t_park # park is teardown, not board cost if run_list: prev_last = run_list[-1] t_variant = time.monotonic() - cells = {} for test in run_list: ec, status, metric = test_example(board, vname, test) err_count += ec @@ -1609,7 +1608,7 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: if not skip_flash: test_example(board, variants[0]['name'], 'device/board_test') - return name, err_count, sorted(set(failed_tests)), rows, t_total + return name, err_count, [] if board_wide_fail else sorted(set(failed_tests)), rows, t_total finally: if _lock_fh: try: @@ -1704,11 +1703,13 @@ def render_matrix(rows_all: list) -> str: return summary + '\n\n' + '\n'.join([header, sep] + body) -def accumulate_report(mret: list, report_dir: Path, fresh: bool) -> str: +def accumulate_report(mret: list, report_dir: Path, fresh: bool, scope: str = '') -> str: """Merge this run's results into hil_report.json in report_dir, then (re)write - the markdown matrix to hil_report.md. `fresh` (a full run, no --accumulate/-bt) + the markdown matrix to hil_report.md. `fresh` (a first run, no --accumulate) starts a new report; otherwise a re-run accumulates so boards/tests that - already passed are preserved while re-run cells are updated. Returns the md.""" + already passed are preserved while re-run cells are updated. `scope` names the + board filter, if any, so a scoped table is not mistaken for a full one. + Returns the md.""" acc = {} # ordered {row_label: [cells dict, duration str|None]} jpath = report_dir / REPORT_JSON if not fresh and jpath.is_file(): @@ -1736,6 +1737,10 @@ def accumulate_report(mret: list, report_dir: Path, fresh: bool) -> str: del acc[name] for row_label, cells, dur in rows: row = acc.setdefault(row_label, [{}, None]) + # the boundary cell is only ever written on failure, so a re-run of this + # variant that cleared the boundary must drop the previous attempt's ❌ + if BOUNDARY_CELL not in cells: + row[0].pop(BOUNDARY_CELL, None) row[0].update(cells) if dur is not None: row[1] = dur @@ -1745,6 +1750,10 @@ def accumulate_report(mret: list, report_dir: Path, fresh: bool) -> str: for k, (c, d) in acc.items()]}, indent=2) + '\n') md = render_matrix([(k, c, d) for k, (c, d) in acc.items()]) + if scope: + # a scoped run's small table is otherwise indistinguishable from a full one, + # and it replaces the previous full table in the sticky PR comment + md = f'_Scoped run: {scope}. Boards/tests not listed were not run._\n\n' + md (report_dir / REPORT_MD).write_text(md + '\n', encoding='utf-8') return md @@ -1831,13 +1840,14 @@ def main() -> None: # HIL report sidecar (hil_report.json/.md) and the .failed re-run spec live in # report_dir (CI keys it by run id, so it persists across run attempts but is - # private to one run). A full run starts fresh; a re-run (--accumulate / -bt, - # i.e. the .failed file) merges so already-passed boards/tests are preserved. - # Clear prior state up front on a fresh run so a crash mid-run can't leave a - # stale report or re-run spec to be consumed by a retry. + # private to one run). A full run starts fresh; a re-run (--accumulate, which + # the generated .failed spec always starts with) merges so already-passed + # boards/tests are preserved. Clear prior state up front on a fresh run so a + # crash mid-run can't leave a stale report or re-run spec for a retry. + # -bt alone is not a re-run marker: PR-scoped first attempts pass -bt too. report_dir = Path(os.environ.get('HIL_REPORT_DIR', '.')) failed_fname = report_dir / (config_file.name + '.failed') - fresh = not (args.accumulate or args.board_test) + fresh = not args.accumulate if fresh: report_dir.mkdir(parents=True, exist_ok=True) for f in (REPORT_JSON, REPORT_MD): @@ -1935,7 +1945,11 @@ def main() -> None: print(f'warning: cannot persist controller hints to {CONTROLLER_CACHE}: {e}') # board x test result matrix -> hil_report.md (accumulates across re-runs) + stdout - report = accumulate_report(mret, report_dir, fresh) + # -b/-bt in play means a filtered run (PR selection or a re-run spec): say so in the + # report, which otherwise looks exactly like a full run that happened to be small + scoped = sorted(set(args.board) | set(board_test)) + scope = f'{len(scoped)} board(s) — {", ".join(scoped)}' if scoped else '' + report = accumulate_report(mret, report_dir, fresh, scope) print() print(report) print(f'\nReport written to {(report_dir / REPORT_MD).resolve()}') diff --git a/test/hil/test_hil_select.py b/test/hil/test_hil_select.py new file mode 100644 index 000000000..6a2bf6210 --- /dev/null +++ b/test/hil/test_hil_select.py @@ -0,0 +1,542 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests for hil_select.py — pure logic, no hardware, no git. Run directly: +# python3 test/hil/test_hil_select.py +import glob +import json +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import hil_select +from hil_examples import device_tests, dual_tests, host_test + +REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +def real_rosters(): + """The actual rig rosters, for regression tests that need real-world data + (a specific board/family/only-list) rather than the synthetic ROSTER above.""" + rosters = [] + for name in ('tinyusb.json', 'hfp.json'): + path = os.path.join(REPO, 'test/hil', name) + with open(path) as f: + rosters.append((f'test/hil/{name}', json.load(f)['boards'])) + return rosters + + +def on_roster(tc, *names): + """The subset of `names` currently in the live rig rosters, skipping the test + when none are. Parking/unparking a board is routine rig maintenance and must not + fail this suite: CI runs it right before the selector and treats a failure as + 'selector unusable', dropping PR scoping and annotating the run.""" + have = {b['name'] for _, boards in real_rosters() for b in boards} + got = [n for n in names if n in have] + if not got: + tc.skipTest(f'not in the rig roster: {", ".join(names)}') + return got + + +ROSTER = [ + # device-only, rp2040 family + {'name': 'raspberry_pi_pico', 'uid': 'u1', 'flasher': {'name': 'openocd'}, + 'tests': {'device': True, 'host': True, 'dual': True}}, + # device-only, stm32f4 family + {'name': 'stm32f407disco', 'uid': 'u2', 'flasher': {'name': 'jlink'}, + 'tests': {'device': True, 'host': False, 'dual': False}}, + # host-only board + {'name': 'raspberry_pi_pico2', 'uid': 'u3', 'flasher': {'name': 'openocd'}, + 'tests': {'device': False, 'host': True, 'dual': False}}, + # only-list board (espressif-style), flashed by the CI leg that splits on esptool + {'name': 'espressif_s3_devkitm', 'uid': 'u4', 'flasher': {'name': 'esptool'}, + 'tests': {'only': ['device/cdc_msc_freertos', 'host/device_info']}}, +] +ROSTERS = [('test/hil/tinyusb.json', ROSTER)] + + +def sel(files): + return hil_select.classify(files, REPO, ROSTERS) + + +class TestPortRule(unittest.TestCase): + def test_dcd_rp2040_selects_pico_family_only(self): + s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico', s['boards']) + self.assertNotIn('stm32f407disco', s['boards']) + self.assertNotIn('espressif_s3_devkitm', s['boards']) + # device role: no host tests in pico's list + self.assertTrue(all(not t.startswith('host/') for t in s['boards']['raspberry_pi_pico'])) + # host-only boards drop out entirely on a device-role change + self.assertNotIn('raspberry_pi_pico2', s['boards']) + + def test_shared_port_file_is_both_roles(self): + s = sel(['src/portable/synopsys/dwc2/dwc2_common.c']) + self.assertFalse(s['full']) + self.assertNotIn('raspberry_pi_pico', s['boards']) # rp2040 is not a dwc2 family + self.assertIn('stm32f407disco', s['boards']) # stm32f4 is + + +class TestCoreRoleRule(unittest.TestCase): + def test_usbd_selects_all_device_tests_everywhere(self): + s = sel(['src/device/usbd.c']) + self.assertFalse(s['full']) + self.assertNotIn('raspberry_pi_pico2', s['boards']) # host-only board dropped + pico = s['boards']['raspberry_pi_pico'] + self.assertTrue(set(device_tests).issubset(set(pico))) + self.assertTrue(set(dual_tests).issubset(set(pico))) # dual survives device role + self.assertTrue(all(not t.startswith('host/') for t in pico)) + # only-list board: selection intersects its only-list + esp = s['boards']['espressif_s3_devkitm'] + self.assertEqual(esp, ['device/cdc_msc_freertos']) + + def test_host_change_drops_device(self): + s = sel(['src/host/usbh.c']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico2', s['boards']) + self.assertNotIn('stm32f407disco', s['boards']) # device-only board dropped + + +class TestClassRule(unittest.TestCase): + def test_cdc_device_selects_cdc_examples_only(self): + s = sel(['src/class/cdc/cdc_device.c']) + self.assertFalse(s['full']) + pico = s['boards']['raspberry_pi_pico'] + self.assertIn('device/cdc_msc', pico) + self.assertIn('device/cdc_dual_ports', pico) + self.assertNotIn('device/msc_dual_lun', pico) # CFG_TUD_CDC 0 there + self.assertNotIn('device/usbtest', pico) # CFG_TUD_CDC 0 there + self.assertTrue(all(not t.startswith('host/') for t in pico)) + + def test_msc_host_selects_host_side(self): + s = sel(['src/class/msc/msc_host.c']) + self.assertFalse(s['full']) + self.assertNotIn('stm32f407disco', s['boards']) # device-only board + pico2 = s['boards']['raspberry_pi_pico2'] + self.assertIn('host/msc_file_explorer', pico2) + self.assertTrue(all(not t.startswith('device/') for t in pico2)) + + +class TestClassIncludeEdges(unittest.TestCase): + """A class header another class includes reaches that class's examples too. + src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, so + midi_test's firmware contains audio.h - but the class rule derives macros from + the directory name alone, so an audio.h change used to select only + device/audio_test_freertos. On boards that skip that example the per-board + intersection emptied and an audio.h-only PR ran ZERO HIL on them.""" + def test_edges_derived_from_includes(self): + edges = hil_select.class_include_edges(REPO) + self.assertEqual(edges.get('audio/audio.h'), {'midi'}) + self.assertEqual(edges.get('cdc/cdc.h'), {'net'}) + + def test_audio_header_selects_midi_example(self): + s = hil_select.classify(['src/class/audio/audio.h'], REPO, real_rosters()) + self.assertFalse(s['full']) + # every board that runs device/midi_test at all must run it here (boards with + # a tests.only list, e.g. espressif, run the freertos examples instead) + by_name = {b['name']: b for _, bs in real_rosters() for b in bs} + checked = 0 + for name, tests in s['boards'].items(): + if 'device/midi_test' in hil_select.board_tests(by_name[name]): + self.assertIn('device/midi_test', tests, name) + checked += 1 + self.assertTrue(checked) + + def test_audio_header_reaches_boards_that_skip_audio(self): + # both skip device/audio_test_freertos: without the midi edge their + # intersection is empty and they drop out of the selection entirely + boards = on_roster(self, 'metro_m4_express', 'nrf54lm20dk') + s = hil_select.classify(['src/class/audio/audio.h'], REPO, real_rosters()) + for board in boards: + self.assertEqual(s['boards'].get(board), ['device/midi_test'], board) + + def test_edge_is_per_header_not_per_class(self): + # midi includes audio.h, not audio_device.h: an audio_device change must + # not drag midi's examples in + s = hil_select.classify(['src/class/audio/audio_device.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for tests in s['boards'].values(): + if tests != 'all': + self.assertNotIn('device/midi_test', tests) + + +class TestFallbackRules(unittest.TestCase): + def test_unknown_tool_is_full(self): + s = sel(['tools/random_new_script.py']) + self.assertTrue(s['full']) + + def test_docs_only_is_empty_not_full(self): + s = sel(['docs/info/contributing.rst', 'README.rst']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + def test_bsp_family_selects_family_boards(self): + s = sel(['hw/bsp/rp2040/family.cmake']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico', s['boards']) + self.assertEqual(s['boards']['raspberry_pi_pico'], 'all') + self.assertNotIn('stm32f407disco', s['boards']) + + def test_bsp_board_narrows_to_board(self): + s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) + self.assertFalse(s['full']) + self.assertEqual(list(s['boards'].keys()), ['raspberry_pi_pico']) + + def test_example_change_selects_that_example(self): + s = sel(['examples/device/cdc_msc/src/main.c']) + self.assertFalse(s['full']) + self.assertEqual(s['boards']['raspberry_pi_pico'], ['device/cdc_msc']) + + def test_core_common_is_full(self): + for f in ['src/tusb.c', 'src/common/tusb_fifo.c', 'src/osal/osal_freertos.h']: + self.assertTrue(sel([f])['full'], f) + + def test_board_test_example_is_full(self): + # board_test is the park/teardown firmware hil_test.py flashes on every board, + # not an unlisted example: a regression there must not skip the whole rig + for f in ['examples/device/board_test/src/main.c', + 'examples/device/board_test/CMakeLists.txt']: + self.assertTrue(sel([f])['full'], f) + + def test_harness_is_full(self): + for f in ['test/hil/hil_test.py', '.github/workflows/build.yml', 'hw/mcu/nxp/x.c', 'lib/foo/x.c']: + self.assertTrue(sel([f])['full'], f) + + def test_mixed_roles_no_pruning(self): + s = sel(['src/device/usbd.c', 'src/host/usbh.c']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico2', s['boards']) + self.assertIn('stm32f407disco', s['boards']) + + def test_cmakelists_and_requirements_are_full(self): + for f in ['src/CMakeLists.txt', 'examples/CMakeLists.txt', + 'examples/device/CMakeLists.txt', 'test/hil/requirements.txt']: + self.assertTrue(sel([f])['full'], f) + + def test_docs_txt_is_noncode(self): + s = sel(['docs/info/changelog.txt']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + +class TestArgsEmission(unittest.TestCase): + def test_args_for_scoped_selection(self): + s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + args = hil_select.selection_args(s, ROSTERS) + a = args['tinyusb.json'] + self.assertIn('-b raspberry_pi_pico', a) + self.assertNotIn('stm32f407disco', a) + self.assertIn('-bt raspberry_pi_pico:', a) # device-only subset of a device+host board + + def test_args_full_is_empty(self): + s = sel(['tools/random_new_script.py']) + self.assertEqual(hil_select.selection_args(s, ROSTERS), {'tinyusb.json': ''}) + + def test_args_all_board_gets_bare_b(self): + s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) + a = hil_select.selection_args(s, ROSTERS)['tinyusb.json'] + self.assertIn('-b raspberry_pi_pico', a) + self.assertNotIn('-bt', a) + + def test_args_by_flasher_splits_esp_from_the_rest(self): + s = sel(['src/device/usbd.c']) + per = hil_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json'] + self.assertIn('espressif_s3_devkitm', per['esptool']) + self.assertIn('raspberry_pi_pico', per['openocd']) + self.assertNotIn('espressif_s3_devkitm', per.get('openocd', '') + per.get('jlink', '')) + + def test_args_by_flasher_omits_a_flasher_with_no_selected_board(self): + # the esp CI leg must see no args at all here, not a filter matching zero boards + s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) + per = hil_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json'] + self.assertEqual(per, {'openocd': '-b raspberry_pi_pico'}) + + def test_args_by_flasher_full_is_empty(self): + s = sel(['tools/random_new_script.py']) + self.assertEqual(hil_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}}) + + def test_cli_diff_file(self): + import subprocess, tempfile, json as j + with tempfile.NamedTemporaryFile('w', suffix='.txt', delete=False) as f: + f.write('src/class/cdc/cdc_device.c\n') + path = f.name + r = subprocess.run([sys.executable, os.path.join(REPO, 'test/hil/hil_select.py'), + '--diff-file', path, os.path.join(REPO, 'test/hil/tinyusb.json')], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + out = j.loads(r.stdout) + self.assertFalse(out['full']) + self.assertIn('tinyusb.json', out['args']) + self.assertTrue(any('cdc_device' in line for line in out['reasons'])) + os.unlink(path) + + +class TestRealRosterPortFamilies(unittest.TestCase): + """Regression for port_families() missing espressif's dwc2 reference, which + lives in a component CMakeLists.txt rather than family.cmake/family.mk.""" + def test_dwc2_change_selects_espressif_boards(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = hil_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertIn(board, s['boards']) + + +class TestOptionGatedPort(unittest.TestCase): + """Regression: family_support.cmake compiles some ports from a build option + (MAX3421_HOST=1 -> hcd_max3421.c), so a board's family file never names them.""" + # host-side option board (max3421 as host controller), off any max3421 family + OPT_ROSTER = [('test/hil/opt.json', [ + {'name': 'fake_dual_board', 'uid': 'o1', 'flasher': {'name': 'jlink'}, + 'build': {'args': ['MAX3421_HOST=1']}, + 'tests': {'device': True, 'host': False, 'dual': True}}, + {'name': 'fake_host_board', 'uid': 'o2', 'flasher': {'name': 'jlink'}, + 'variant': [{'name': 'fake_host_board', 'flags': '-DMAX3421_HOST=1'}], + 'tests': {'device': False, 'host': True, 'dual': False}}, + {'name': 'fake_off_board', 'uid': 'o3', 'flasher': {'name': 'jlink'}, + 'variant': [{'name': 'fake_off_board', 'defines': ['MAX3421_HOST=0']}], + 'tests': {'device': True, 'host': True, 'dual': True}}, + ])] + + def test_real_roster_max3421_selects_option_board(self): + boards = on_roster(self, 'metro_m4_express') + s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertIn(board, s['boards']) + + def test_option_selects_via_args_defines_and_flags(self): + s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, self.OPT_ROSTER) + self.assertFalse(s['full']) + self.assertIn('fake_dual_board', s['boards']) # build.args + self.assertIn('fake_host_board', s['boards']) # variant flags + self.assertNotIn('fake_off_board', s['boards']) # variant defines, but =0 + + def test_device_role_port_does_not_pull_host_only_option_board(self): + s = hil_select.classify(['src/portable/analog/max3421/dcd_max3421.c'], REPO, self.OPT_ROSTER) + self.assertFalse(s['full']) + self.assertNotIn('fake_host_board', s['boards']) # host-only board, device change + self.assertIn('fake_dual_board', s['boards']) # device-capable option board + + def test_gates_parsed_from_family_support(self): + self.assertEqual(hil_select.port_option_gates(REPO).get('analog/max3421'), + {'MAX3421_HOST'}) + + def test_board_cmake_option_counts(self): + """A board can enable a gated port in its own BSP rather than via the roster + (hw/bsp/espressif/boards/*/board.cmake -> set(MAX3421_HOST 1)); board_options() + must see those too, or such a board joining the roster is silently dropped.""" + self.assertIn('MAX3421_HOST', + hil_select.bsp_board_options('adafruit_feather_esp32s3', REPO)) + self.assertIn('CFG_TUH_RPI_PIO_USB', + hil_select.bsp_board_options('adafruit_fruit_jam', REPO)) + # commented-out `# set(MAX3421_HOST 1)` must not count + self.assertNotIn('MAX3421_HOST', + hil_select.bsp_board_options('feather_nrf52840_express', REPO)) + + def test_board_cmake_option_selects_off_family_board(self): + # adafruit_feather_esp32s3 is not on any rig roster; stand it in as one to + # prove the BSP-sourced option alone pulls a max3421 change onto the board + roster = [('test/hil/opt.json', [ + {'name': 'adafruit_feather_esp32s3', 'uid': 'o1', 'flasher': {'name': 'esptool'}, + 'tests': {'device': False, 'host': True, 'dual': False}}])] + s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster) + self.assertFalse(s['full']) + self.assertIn('adafruit_feather_esp32s3', s['boards']) + + def test_board_mk_option_is_ignored(self): + """Make-only options must not select: HIL CI builds with CMake exclusively, so + hw/bsp/nrf/boards/nrf5340dk/board.mk's MAX3421_HOST compiles nothing here.""" + roster = [('test/hil/opt.json', [ + {'name': 'nrf5340dk', 'uid': 'o1', 'flasher': {'name': 'jlink'}, + 'tests': {'device': False, 'host': True, 'dual': False}}])] + s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + +class TestPortFamiliesCmakeOnly(unittest.TestCase): + """port_families() is CMake-only (HIL CI never builds with Make) and matches on + 'port_dir/' so a port dir is not a prefix of a sibling.""" + def test_make_only_family_is_not_a_family(self): + # hw/bsp/pic32mz has family.mk but no family.cmake + self.assertEqual(hil_select.port_families('microchip/pic32mz', REPO), set()) + + def test_prefix_port_does_not_inherit_sibling_families(self): + # bare-substring matching let 'microchip/pic' match '.../microchip/pic32mz/...' + self.assertEqual(hil_select.port_families('microchip/pic', REPO), set()) + + def test_make_only_port_forces_full(self): + s = sel(['src/portable/microchip/pic32mz/dcd_pic32mz.c']) + self.assertTrue(s['full']) + self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons']) + + def test_cmake_families_still_found(self): + self.assertEqual(hil_select.port_families('raspberrypi/rp2040', REPO), {'rp2040'}) + self.assertIn('stm32f4', hil_select.port_families('synopsys/dwc2', REPO)) + + +class TestPortFamiliesCoverage(unittest.TestCase): + """Systematic guard: every real dcd_*/hcd_* port directory should map to at + least one board family, so a future family.cmake/CMakeLists.txt layout that + port_families() doesn't scan fails loudly instead of silently dropping boards + (as espressif's dwc2 reference did - see TestRealRosterPortFamilies).""" + # Ports with no board family: not a bug, just not wired into any rig board. + # Add here (with a reason) only if port_families() legitimately can't find one. + # A port listed here force-fulls (fail-open), so it is never under-selected. + NO_FAMILY = { + 'template', # reference/example port, not built by any board + # hw/bsp/pic32mz has family.mk only (no family.cmake), and port_families() + # is CMake-only because HIL CI builds every board with CMake - so this port + # is compiled for no HIL board. + 'microchip/pic32mz', + 'microchip/pic', # same: only ever referenced from pic32mz's family.mk + } + + @staticmethod + def _dcd_hcd_ports(): + portable_root = os.path.join(REPO, 'src/portable') + ports = [] + for entry in sorted(os.listdir(portable_root)): + d = os.path.join(portable_root, entry) + if not os.path.isdir(d): + continue + if glob.glob(os.path.join(d, 'dcd_*.c')) or glob.glob(os.path.join(d, 'hcd_*.c')): + ports.append(entry) + continue + for sub in sorted(os.listdir(d)): + sd = os.path.join(d, sub) + if os.path.isdir(sd) and (glob.glob(os.path.join(sd, 'dcd_*.c')) or + glob.glob(os.path.join(sd, 'hcd_*.c'))): + ports.append(f'{entry}/{sub}') + return ports + + def test_every_port_maps_to_a_family(self): + ports = self._dcd_hcd_ports() + self.assertTrue(ports) # sanity: the scan itself found something + for port in ports: + if port in self.NO_FAMILY: + continue + fams = hil_select.port_families(port, REPO) + self.assertTrue(fams, f'{port}: no family references this port ' + f'(port_families() scan gap, or add to NO_FAMILY)') + + +class TestRealRosterOnlyListTests(unittest.TestCase): + """Regression for roster-only-list tests (e.g. espressif's hid_composite_freertos) + being invisible to the selector because it only knew the shared hil_examples lists.""" + def test_only_list_example_change_selects_it(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = hil_select.classify(['examples/device/hid_composite_freertos/src/main.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertEqual(s['boards'][board], ['device/hid_composite_freertos']) + + def test_class_change_includes_only_list_boards(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = hil_select.classify(['src/class/hid/hid_device.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertIn(board, s['boards']) + + +class TestPortAndCoreRoleUseExtras(unittest.TestCase): + """Regression: the port rule and core-role rule must thread the roster-only + test universe (extras) the same way the class rule already does, so a DCD + or device-stack change doesn't silently drop espressif's only-list tests + (e.g. hid_composite_freertos) that aren't in the shared device_tests list.""" + def test_dcd_change_includes_only_list_test(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = hil_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + tests = s['boards'][board] + self.assertIn('device/hid_composite_freertos', tests) + self.assertIn('device/cdc_msc_freertos', tests) + self.assertIn('device/audio_test_freertos', tests) + self.assertIn('device/usbtest', tests) + + def test_core_device_change_includes_only_list_test(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = hil_select.classify(['src/device/usbd.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + tests = s['boards'][board] + self.assertIn('device/hid_composite_freertos', tests) + self.assertIn('device/cdc_msc_freertos', tests) + self.assertIn('device/audio_test_freertos', tests) + self.assertIn('device/usbtest', tests) + + def test_host_change_does_not_leak_device_only_list_test(self): + s = hil_select.classify(['src/host/usbh.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board, tests in s['boards'].items(): + if tests == 'all': + continue + self.assertNotIn('device/hid_composite_freertos', tests, board) + + +class TestFamilies(unittest.TestCase): + """`families` exists for consumers that build (not just test) the diff: most + families have no rig board, so `boards` alone would compile nothing for them.""" + def test_off_rig_port_still_reports_family(self): + s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) # no same7x board on the rig + self.assertEqual(s['families'], ['same7x']) + + def test_port_families_are_reported(self): + s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + self.assertIn('rp2040', s['families']) + + def test_bsp_family_and_board_report_family(self): + self.assertEqual(sel(['hw/bsp/rp2040/family.cmake'])['families'], ['rp2040']) + self.assertEqual(sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h'])['families'], + ['rp2040']) + + def test_docs_only_has_no_families(self): + self.assertEqual(sel(['docs/info/contributing.rst'])['families'], []) + + def test_full_selection_still_reports_families(self): + """A full-matrix file must not hide the families of the other changed files: + consumers that build from `families` (e.g. /pre-pr) ignore `boards` when full.""" + s = sel(['src/common/tusb_fifo.c', 'src/portable/microchip/samx7x/dcd_samx7x.c']) + self.assertTrue(s['full']) + self.assertIn('same7x', s['families']) + # full stays full: every roster board, and no args to narrow the run + self.assertEqual(set(s['boards']), {b['name'] for b in ROSTER}) + self.assertTrue(all(v == 'all' for v in s['boards'].values())) + self.assertEqual(hil_select.selection_args(s, ROSTERS), {'tinyusb.json': ''}) + self.assertEqual(hil_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}}) + + def test_family_order_does_not_matter(self): + # same as above with the full-matrix file last (was the only order that worked) + s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c', 'src/common/tusb_fifo.c']) + self.assertTrue(s['full']) + self.assertIn('same7x', s['families']) + + +class TestGitDiffArgv(unittest.TestCase): + def test_diff_disables_rename_detection(self): + """Without --no-renames git reports only a rename's destination, so moving an + HIL-relevant file to a non-code path would be classified as non-code only.""" + self.assertIn('--no-renames', hil_select.GIT_DIFF_ARGV) + + +class TestPortWithoutFamilyIsFull(unittest.TestCase): + """A port dir no family file references must widen (full matrix), not silently + contribute zero boards — the fail-open contract.""" + def test_unreferenced_port_forces_full(self): + orig = hil_select.port_families + hil_select.port_families = lambda port_dir, repo_root: set() + try: + s = sel(['src/portable/vendor/newip/dcd_newip.c']) + finally: + hil_select.port_families = orig + self.assertTrue(s['full']) + self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons']) + + +if __name__ == '__main__': + unittest.main(verbosity=1) |
