diff options
Diffstat (limited to 'test/hil')
28 files changed, 12509 insertions, 2558 deletions
diff --git a/test/hil/helper/__init__.py b/test/hil/helper/__init__.py new file mode 100644 index 000000000..a080a2f55 --- /dev/null +++ b/test/hil/helper/__init__.py @@ -0,0 +1,4 @@ +# Marks helper/ as a REGULAR package. Without this it is only a PEP 420 namespace portion, +# and a regular package named `helper` anywhere on sys.path wins over it even though +# test/hil is sys.path[0] -- one transitive pip install would break every HIL entry point +# at import. `helper` is a real distribution name on PyPI. diff --git a/test/hil/helper/hil_health.py b/test/hil/helper/hil_health.py new file mode 100644 index 000000000..d78d0f220 --- /dev/null +++ b/test/hil/helper/hil_health.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Shutting a wedged HIL run down: kill what the workers spawned. + +A device whose usbfs node is held by a D-state process cannot be freed -- SIGKILL is not +delivered in uninterruptible sleep -- so the goal is never to fix the rig from here. It is +to free the runner's single job slot and leave a report naming what survived, instead of +letting the job sit until GitHub cancels it with nothing to show. + +Deliberately shallow. We SIGKILL the process groups the workers spawned, wait a grace, +and report whoever is still alive; we do not re-scan groups, prove pid ownership or +escalate through sudo. A root-owned survivor is named in the report for hil_pool_check +and the usb-kernel-recover skill to deal with -- signalling a pid we cannot prove is ours +is the worse failure, and the job ceiling backstops whatever this misses. + +Everything here is stdlib-only and reads /proc unprivileged (dmesg is restricted on the +rig), which keeps it importable -- and testable -- on a bare runner. +""" +import os +import signal +import threading +import time +from pathlib import Path + +PROC = Path('/proc') + + +# How long to let a SIGKILL land before calling a process a survivor. Generous enough to +# cover scheduling delay on a loaded rig, short enough that a fleet-wide sweep stays quick. +CONFIRM_KILL_GRACE = 2.0 + + +def _p(*args, **kwargs) -> None: + # These run on the free-the-runner path, where stdout can already be a dead pipe (a + # dropped ssh session). An unguarded print would raise BrokenPipeError out of + # hil_test's inner finally, skipping shutdown_pool AND the report writing. + try: + print(*args, **kwargs) + except (OSError, ValueError): + # ValueError, not just OSError: printing to a CLOSED stream raises + # "ValueError: I/O operation on closed file", and both hil_pool_check and + # hil_test redirect stdout into a StringIO that can be closed under us. Escaping + # here skips shutdown_pool/kill_pool_children/os._exit -- stranding the runner, + # the exact failure this wrapper exists to prevent. + pass + + +def _state(pid_dir: Path) -> str: + """The state letter from /proc/<pid>/stat. comm can contain ')', so the field is + located from the right rather than by splitting.""" + # bytes, not read_text(): read_text decodes with the LOCALE encoding, so under LANG=C + # (systemd services, self-hosted runners) a non-ASCII comm raises UnicodeDecodeError + # and the entry silently vanishes from the scan. + stat = (pid_dir / 'stat').read_bytes() + return chr(stat[stat.rindex(b')') + 2]) + + +def _pids(): + """/proc pid entries. Yields nothing rather than raising if /proc is unreadable.""" + try: + entries = list(PROC.iterdir()) + except OSError: + return + for entry in entries: + if entry.name.isdigit(): + yield entry + + +def d_state_note() -> str: + """Pids in uninterruptible sleep, for the report. Never aborts, never blocks. + + A D-state process at start-up is NOT a fault on its own -- a healthy in-flight testusb + looks exactly like this, and the rig supports a dev run alongside CI. It is a hint for + whoever reads a red cell below. Diagnosis proper is hil_pool_check and the + usb-kernel-recover skill; this is one line, not a probe.""" + stuck = [] + for d in PROC.glob('[0-9]*'): + try: + if _state(d) == 'D': + stuck.append(d.name) + except (OSError, ValueError, IndexError): + pass # raced with exit, or /proc is restricted: not our problem here + if not stuck: + return '' + return (f'{len(stuck)} process(es) in D state when this run started: ' + f'{sorted(stuck)[:10]}') + + +def shutdown_pool(pool, grace: float = 30) -> bool: + """terminate() a worker Pool without ever blocking forever. + + multiprocessing joins its workers unbounded (util.py _exit_function terminate()s the + daemonic ones, then calls p.join() -- no timeout -- on every remaining active child, + CPython 3.13.5), and a worker in uninterruptible sleep never + reaps -- so terminate() itself hangs, taking the runner's only job slot with it. False + when the pool refuses to die within `grace` (the caller must then abandon it); a + terminate() that *raises* counts as failure too, the pool being just as alive.""" + outcome = {} + + def _term(): + try: + pool.terminate() + outcome['ok'] = True + except BaseException as e: # noqa: BLE001 - any failure means the pool is still up + # Say what happened: Pool._terminate_pool really can raise (CPython: + # AssertionError 'Cannot have cache with result_handler not alive'), and a + # swallowed one is indistinguishable from an unkillable D-state worker. + outcome['err'] = e + _p(f'warning: Pool.terminate() raised {type(e).__name__}: {e}', flush=True) + + t = threading.Thread(target=_term, daemon=True) + t.start() + t.join(grace) + # Decide on the thread, not the dict: _term may set outcome['ok'] after join(grace) + # expired, reporting a merely-slow terminate as success on one read and abandoned on + # another. Still inside terminate() == not shut down. + if t.is_alive(): + return False + return outcome.get('ok', False) + + +def child_procs(pids) -> dict: + """{ancestor pid in `pids`: [(descendant pid, its pgid), ...]}, from ONE walk of /proc. + + DESCENDANTS, not direct children: a worker's usbtest.py spawns its recovery flasher + through run_cmd (own session), so it is a GRANDCHILD that a direct-child sweep misses + and a kill mid-recovery would orphan on the probe. pgid comes back too because the two + kinds of child need different signals (see kill_pool_children).""" + wanted = set(pids) + by_parent: dict = {} # ppid -> [(pid, pgid), ...] for EVERY process + for entry in _pids(): + try: + stat = (entry / 'stat').read_bytes() + except OSError: + continue # exited between the scan and the read, or not readable + # comm (field 2) is parenthesised and may contain spaces and ')' -- so split only + # what follows the LAST ')': state, ppid, pgrp, ... + try: + fields = stat[stat.rindex(b')') + 2:].split() + ppid, pgid = int(fields[1]), int(fields[2]) + except (ValueError, IndexError): + continue # truncated or unparsable stat line + by_parent.setdefault(ppid, []).append((int(entry.name), pgid)) + out: dict = {} + for root in wanted: + todo = list(by_parent.get(root, [])) + while todo: + pid, pgid = todo.pop() + out.setdefault(root, []).append((pid, pgid)) + todo += by_parent.get(pid, []) + return out + + +def _pool_procs(pool, extra) -> list: + """The pool's worker Process objects, plus each extra's own process. + + Manager() runs in its own child process and inherits the same descriptors as the + workers, so leaving it behind defeats the point: os._exit skips its finalizer.""" + procs = list(getattr(pool, '_pool', []) or []) + for e in extra: + procs.append(getattr(e, '_process', e)) + return procs + + + + +def kill_worker_children(pool, *extra) -> int: + """SIGKILL what the pool's workers spawned; returns how many SURVIVED. + + For the TIMEOUT path only. On the normal path each worker has already run + kill_own_children() and retired (maxtasksperchild=1), so this walks fresh idle workers + and finds nothing -- measured: 4 tasks, zero overlap with the pool at sweep time. + + Call it BEFORE shutdown_pool(): terminate() reaps the (interruptible) worker and its + flasher is reparented to init, erasing the ppid link this matches on. Signalling the + worker's own group instead cannot work -- a forked pool worker inherits OUR group + (CPython 3.13.5 multiprocessing never setsid/setpgid) and run_cmd gives every flasher + a session of its own. + + TWO passes because our SIGKILL can fail an in-flight flash and the worker then retries + in a fresh session, which one /proc snapshot misses. `seen` stops a pid signalled in + pass 1 being confirmed twice. + """ + seen: set = set() + total = 0 + for i in range(2): + if i: + time.sleep(0.5) + procs = _pool_procs(pool, extra) + total += _kill_kids( + child_procs(getattr(p, 'pid', None) for p in procs if p is not None), seen) + return total + + +def kill_own_children() -> int: + """SIGKILL what THIS process spawned. Returns how many survived. + + For the worker to call before it returns. maxtasksperchild=1 retires it the moment the + task ends, reparenting its children to init, so main()'s sweep walks fresh idle workers + and finds nothing (measured over 4 tasks: zero overlap, sweep 0, 4 strays alive). + Inside the worker the ppid link is still live. + """ + return _kill_kids(child_procs([os.getpid()]), set()) + + +def _kill_kids(kids: dict, seen: set) -> int: + """SIGKILL every pid in a ppid-tree snapshot; return how many survived. + + Every pid here is a DESCENDANT of a process we own, so it is ours by construction -- no + argv identity check, because we never signal anything we did not discover through our + own ppid tree. + """ + try: + own = os.getpgid(0) + except OSError: + own = None # cannot tell our own group apart: never killpg, signal pids only + touched: list = [] + for children in kids.values(): + for cpid, cpgid in children: + if cpid in seen: + continue # a previous pass already signalled it + seen.add(cpid) + try: + if own is not None and cpgid != own: + # A run_cmd child: its own session, so one killpg also reaps what it + # spawned. Recorded because killpg cannot report a partial kill. + os.killpg(cpgid, signal.SIGKILL) + else: + # Shares our group (a plain subprocess.run), so killpg would take + # down the whole run -- it is signalled by pid in _kill_and_confirm. + pass + touched.append(cpid) + except PermissionError: + # NOT "already gone": the signal did not land, so this pid MUST still be + # confirmed, or the one case this handler exists for (an all-root session: + # the sudo wrapper died, its root members did not) is the one case that + # never reaches the report. + touched.append(cpid) + except ProcessLookupError: + pass # already gone + except OSError: + pass + # Both paths need confirming: a killpg'd flasher and a same-group mtype blocked on a + # wedged device are both in D state, and os.kill reported success on either. + denied = _kill_and_confirm(touched) + if denied: + _p(f'warning: could not kill {sorted(denied)}; they still hold whatever they ' + f'had open (probe, usbfs node) into the next job', flush=True) + # SURVIVORS, not the count we signalled: the caller needs to know the rig is dirty for + # the next job, and a killpg is counted once per child sharing the group anyway. + return len(denied) + + +def _kill_and_confirm(pids) -> list: + """SIGKILL every pid, then return those STILL alive after ONE grace window. + + SIGKILL is QUEUED, not delivered, for a task in uninterruptible sleep -- and testusb + waits in a plain wait_for_completion() with no timeout (v6.12.96 usbtest.c:1404; + usb_sg_wait, message.c:765), so that is the normal state of a healthy in-flight case + too. os.kill returning success proves nothing; only the recheck does. It is also + asynchronous, so probing immediately reports a process we just killed as a survivor + (measured: 11 of 20 plain `sleep`s with no grace). + + Signal all, then poll the set against ONE shared deadline: per-pid windows made this + scale with stray count, minutes on a convoy. A pid we cannot signal is reported, never + sudo-killed. + """ + pending = [] + for pid in pids: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + continue # already gone + except OSError: + pass # EPERM (root-owned): it stays, and the poll below reports it + pending.append(pid) + + deadline = time.monotonic() + CONFIRM_KILL_GRACE + while True: + alive = [] + for pid in pending: + try: + os.kill(pid, 0) + except ProcessLookupError: + continue # ESRCH: genuinely gone + except OSError: + pass # EPERM: it exists; the state check decides + try: + # a ZOMBIE answers kill(pid, 0) too: dead, merely unreaped. Not a survivor. + if _state(PROC / str(pid)) == 'Z': + continue + except (OSError, ValueError, IndexError): + continue # unreadable: assume gone rather than cry wolf + alive.append(pid) + pending = alive + if not pending or time.monotonic() >= deadline: + return pending # outlasted SIGKILL: D state, or not ours to kill + time.sleep(0.02) + + +def kill_pool_children(pool, *extra) -> int: + """SIGKILL the pool's worker processes themselves. Returns how many are STILL ALIVE + after the grace -- not how many were signalled. + + Survivors, not signals: the caller turns this number into "power-cycle the host", so + counting signals would send someone to a hypervisor over workers that all died. + + Call after a shutdown_pool() that returned False, and after kill_worker_children(). + A D-state worker ignores SIGKILL, but every other worker dies and drops the inherited + descriptors -- a survivor holds the runner's stdout pipe open and the runner waits for + EOF even after we exit, so the early exit would not free the job slot.""" + killed_procs: list = [] + for proc in _pool_procs(pool, extra): + try: + # Process.kill(), never a raw pid: multiprocessing's _send_signal re-checks + # `self.returncode is None` first, so once shutdown_pool's thread has reaped a + # worker this is a no-op instead of signalling a pid the OS may have recycled. + # os.pidfd_open(proc.pid) is worse: it skips that guard entirely. + if proc is None or not proc.is_alive(): + continue + proc.kill() + killed_procs.append(proc) + except (OSError, AttributeError, ValueError): + continue # already reaped, never started, or not a real process + # Re-check the Process objects, never the pids collected a moment ago: shutdown_pool's + # thread is STILL join()ing workers, so a pid killed here can be reaped and RECYCLED + # before _kill_and_confirm signals it -- and on EPERM that escalates to `sudo -n kill + # -9 <stale pid>`, killing an unrelated ROOT process as the last act before os._exit. + killed_pids = [] + for proc in killed_procs: + try: + if proc.is_alive() and proc.pid is not None: + killed_pids.append(proc.pid) + except (OSError, AttributeError, ValueError): + continue + # SIGKILL is asynchronous and a D-state task ignores it: only a confirmed survivor + # justifies the caller's power-cycle wording + return len(_kill_and_confirm(killed_pids)) if killed_pids else 0 diff --git a/test/hil/hil_lock.py b/test/hil/helper/hil_lock.py index e570da16a..91f05ca86 100755 --- a/test/hil/hil_lock.py +++ b/test/hil/helper/hil_lock.py @@ -10,7 +10,6 @@ batteries per host controller; they have no CLI meaning. The CLI below """ import argparse import fcntl -import glob import json import os import re @@ -19,6 +18,9 @@ import signal import sys import time +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # helper/ scripts import via the test/hil root +from helper import hil_util + BOARD_LOCK_DIR = '/tmp/tinyusb-hil-locks' CI_REASON = 'hil_test.py' # release-protected holder tag (release refuses to kill it) PROTECTED_REASONS = {CI_REASON, 'pool_check'} # cmd_release refuses to SIGTERM these holders @@ -44,10 +46,9 @@ def flock_nb(board: str): def write_record(fh, reason: str) -> bool: - """Holder record; the flock itself is already held. Returns False on a write - failure — acquire_board_lock stays best-effort (the flock is the authority), - but cmd_hold aborts on it like board_lock.py did (a hold whose record is - missing is invisible to status/release).""" + """Holder record; the flock itself is already held. Returns False on a write failure: + acquire_board_lock stays best-effort (the flock is the authority), but cmd_hold aborts + -- a hold whose record is missing is invisible to status/release.""" try: fh.truncate(0) fh.seek(0) @@ -118,22 +119,29 @@ def acquire_board_lock(board_name, reason=CI_REASON): return fh -# Per-host-controller concurrency (see controller_of/controller_slot below): a usbtest battery -# saturates its DUT's host controller, so batteries and flashes are budgeted per controller. -# - uPD720201 cards need their latest firmware (>= 2.0.2.6; RAM-uploaded, reloads every -# power cycle): ROM firmware dies under battery + re-enumeration churn, and usbtest.py -# refuses the unlink-stress cases on it. -# - widths (profiled 2026-07-13/14): wall time 22.2/14.3/12.5/10.8 min at usbtest width -# 1/2/3/4, plateau after; flash width beyond 8 only adds flasher-hub contention; -# battery case failures start at 12/8 (bandwidth stretch on shared leaf-hub uplinks). -# - a marginal DUT port bouncing during concurrent batteries can wedge/kill a uPD720201 -# ("xHCI host not responding to stop endpoint command"): fix the port/cable or pull -# the board, don't lower the widths (2026-07-16: every death traced to one board's port). -FLASH_PARALLEL = int(os.getenv('HIL_FLASH_PARALLEL', '8')) -USBTEST_PARALLEL = int(os.getenv('HIL_USBTEST_PARALLEL', '4')) +# Per-host-controller concurrency (see controller_of/controller_slot below): a usbtest +# battery saturates its DUT's host controller, so batteries and flashes are budgeted per +# controller. The 4/2 defaults trade ~3.5 min on the usbtest leg for bandwidth margin on +# the shared leaf-hub uplinks, where battery case failures were observed from 12/8 +# (profiled 2026-07-13/14: 22.2/14.3/12.5/10.8 min at usbtest width 1/2/3/4, plateau +# after). Raise per run via HIL_FLASH_PARALLEL/HIL_USBTEST_PARALLEL. +# - uPD720201 cards need firmware >= 2.0.2.6 (RAM-uploaded, reloads every power cycle): +# the ROM firmware dies under battery + re-enumeration churn. +# - a marginal DUT port bouncing during concurrent batteries can kill a uPD720201 ("xHCI +# host not responding to stop endpoint command"): fix the port/cable or pull the board +# -- lowering the widths does not fix a bad port (2026-07-16, every death). +FLASH_PARALLEL = hil_util.pos_int_env('HIL_FLASH_PARALLEL', 4) +USBTEST_PARALLEL = hil_util.pos_int_env('HIL_USBTEST_PARALLEL', 2) CONTROLLER_SLOTS = 12 # lock slots; controllers are assigned to slots on first sight -usbtest_sems = None # CONTROLLER_SLOTS semaphores: per-slot usbtest-battery permits -flash_sems = None # CONTROLLER_SLOTS semaphores: per-slot flash permits +# Bound on ONE permit wait. Generous: a real queue behind a slow board is normal, +# and this only has to beat the pool guard so a leaked permit cannot consume it. +PERMIT_TIMEOUT = hil_util.pos_int_env('HIL_PERMIT_TIMEOUT', 900) +# CONTROLLER_SLOTS + 1 entries each, built by make_permit_sems: UNKNOWN_SLOT indexes the +# extra one. Sized to CONTROLLER_SLOTS instead, the first unresolved board IndexErrors +# inside a pool worker -- which now surfaces through drain_pool as a worker-raise (the +# finished boards survive), but still loses this board and aborts the run. +usbtest_sems = None # per-slot usbtest-battery permits +flash_sems = None # per-slot flash permits controller_map = None # shared dict: 'pci:<addr>' -> slot, 'uid:<uid>' -> pci addr cache controller_meta = None # guards slot assignment in controller_map controller_hints = {} # static uid -> pci from the last run's cache (read-only per worker) @@ -155,29 +163,33 @@ def init_scheduling(b_sems, f_sems, cmap, cmeta, hints, log_fn=None): # Per-controller scheduling # ------------------------------------------------------------- def controller_of(uid: str): - """Resolve a DUT uid to its root host controller's PCI address, or None if the device - is not enumerated (e.g. parked in board_test firmware with USB off). Successful - resolutions are cached — cabling does not change mid-run. Dual-port parts (e.g. - CH32V307 usbhs/usbfs variants) share one uid and one cache entry: budgeting is only - exact when both ports sit on the same controller (true on this rig).""" + """Resolve a DUT uid to its root host controller's PCI address, or None when it cannot + be resolved — the device is not enumerated (e.g. parked in board_test firmware with USB + off), or sysfs would not answer. Successful resolutions are cached — cabling does not + change mid-run. Dual-port parts (e.g. CH32V307 usbhs/usbfs variants) share one uid and + one cache entry: budgeting is only exact when both ports sit on the same controller + (true on this rig).""" if controller_map is None: return None cached = controller_map.get(f'uid:{uid}') if cached: return cached - for f in glob.glob('/sys/bus/usb/devices/*/serial'): - d = os.path.dirname(f) + # vid='cafe' first: the target is always a TinyUSB DUT, and the VID is a lock-free + # descriptor field. Without it this reads every probe's and hub's `serial` -- the one + # attribute served under device_lock -- so a wedged peer would block us here. + devs = hil_util.usb_scan(vid='cafe', serial=uid) + for dev in devs: + busnum = hil_util.read_sysfs(os.path.join(dev['dir'], 'busnum')) + if busnum is None: + continue try: - if open(f).read().strip().lower() != uid.lower(): - continue - bus = int(open(os.path.join(d, 'busnum')).read()) - root = os.path.realpath(f'/sys/bus/usb/devices/usb{bus}') - m = re.findall(r'[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]', root) - if m: - controller_map[f'uid:{uid}'] = m[-1] - return m[-1] - except (OSError, ValueError): + root = os.path.realpath(f'/sys/bus/usb/devices/usb{int(busnum)}') + except ValueError: continue + m = re.findall(r'[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]', root) + if m: + controller_map[f'uid:{uid}'] = m[-1] + return m[-1] return None @@ -196,39 +208,73 @@ def controller_slot(pci: str) -> int: return slot +# Unresolved boards budget in a slot of their OWN, one past the real ones, and that slot +# holds exactly ONE permit whatever the per-controller width is. Neither neighbour works: +# a permit on every slot (the old fail-closed rule) serialized the whole fleet the moment +# one board could not be resolved, while a full private budget let unknown boards run a second +# controller's worth of batteries on top of the resolved ones -- doubling the load on +# whichever physical controller they actually sit on, which is the saturation the +# uPD720201 deaths above are attributed to. Width 1 caps the over-subscription at +1. +UNKNOWN_SLOT = CONTROLLER_SLOTS + + +def make_permit_sems(semaphore, width: int) -> list: + """One semaphore per controller slot at `width`, plus the unknown bucket at 1.""" + return [semaphore(width) for _ in range(CONTROLLER_SLOTS)] + [semaphore(1)] + + class controller_permit: - """Context manager: one permit from `sems` on the board's controller slot. If the - controller is unknown, fail closed: take one permit from EVERY slot, in order, so the - operation respects the budget wherever it might land. `warn_unknown` logs that fallback - (used by usbtest, where the device is expected to be enumerated by the caller).""" + """Context manager: one permit from `sems` on the board's controller slot. An + unresolved controller budgets in UNKNOWN_SLOT, which admits one at a time: unresolved + boards serialize against each other, never against the whole rig, and never add a + second full budget to a controller. `warn_unknown` logs that fallback (used by + usbtest, where the device is expected to be enumerated by the caller).""" def __init__(self, sems, uid: str, warn_unknown: bool = False): self.sems = sems self.slots = None self.uid = uid + # what __enter__ actually ACQUIRED. Not the same as self.slots: a bounded acquire + # that times out is skipped on purpose, and releasing it anyway would add a permit + # that was never taken -- multiprocessing semaphores are unbounded, so the width + # grows for the rest of the run, on the controller throttle that exists to keep + # concurrent batteries from killing the uPD720201 xHCI. + self.taken: list = [] if sems is None: return - pci = controller_of(uid) - if pci is None and not warn_unknown: - # last-run cabling hint, flash budgeting only: a mis-budgeted flash is harmless, - # but a battery must never trust a stale hint (it could stack two batteries on - # one controller). In practice only a board's first flash lands here - batteries - # assert enumeration before taking their permit. - pci = controller_hints.get(uid) + # Hint FIRST for flash budgeting: a mis-budgeted flash is harmless, and the board + # is usually parked in board_test with USB off at this point, so controller_of + # cannot resolve it anyway -- it just walks the whole bus to say so, once per + # flash permit (~14 examples x ~21 boards a leg), each walk spawning a bounded + # reader per device. usbtest still resolves for real (warn_unknown), and by then + # the DUT is enumerated, so that walk succeeds and caches. + pci = None if warn_unknown else controller_hints.get(uid) + if pci is None: + pci = controller_of(uid) if pci is None and warn_unknown: log(f'warning: cannot resolve {uid} to a host controller; ' - 'taking a permit on every slot (over-serialized)') - self.slots = [controller_slot(pci)] if pci else list(range(CONTROLLER_SLOTS)) + f'budgeting it in the unknown bucket') + self.slots = [controller_slot(pci) if pci else UNKNOWN_SLOT] def __enter__(self): if self.slots: t0 = time.monotonic() - taken = [] + taken = self.taken = [] try: for s in self.slots: - self.sems[s].acquire() + # BOUNDED. multiprocessing semaphores are NOT released when a holder + # dies, and the pool sweep SIGKILLs workers -- so a permit lost that + # way would block every later worker on this controller forever, and + # boards unrelated to the wedge would burn the whole pool guard. On + # expiry proceed over-subscribed and say so: a slower controller is a + # far better failure than a hung run. + if not self.sems[s].acquire(timeout=PERMIT_TIMEOUT): + log(f'warning: waited {PERMIT_TIMEOUT}s for a permit on slot {s} ' + f'(uid {self.uid}); a holder probably died without releasing ' + f'it -- proceeding over-subscribed') + continue taken.append(s) - # stays inside the try: if this raises (e.g. broken stdout), the permits - # must be released - a failed __enter__ never gets its __exit__ + # inside the try: a failed __enter__ never gets its __exit__, so a raise + # here (e.g. broken stdout) must still release the permits if PROFILE and time.monotonic() - t0 > 1.0: log(f'[prof] permit wait {time.monotonic() - t0:.1f}s ' f'(uid {self.uid}, slots {self.slots})') @@ -240,8 +286,9 @@ class controller_permit: def __exit__(self, *exc): if self.slots: - for s in reversed(self.slots): + for s in reversed(self.taken): self.sems[s].release() + self.taken = [] return False @@ -288,12 +335,10 @@ def is_locked(board: str) -> bool: def cmd_hold(boards, reason): os.makedirs(BOARD_LOCK_DIR, exist_ok=True) - # No pre-check: the holder's own LOCK_NB flock is the only authority — a - # recorded pid may be stale or recycled (e.g. a live hil_test.py worker - # that already released this board's flock but not its record). - # The holder signals success through this pipe. A generic is_locked() - # poll would be fooled by a RIVAL invocation's flock — only the holder - # itself knows whether it won every board. + # No pre-check: the holder's own LOCK_NB flock is the only authority, since a recorded + # pid may be stale or recycled. The holder signals success through this pipe because a + # generic is_locked() poll would be fooled by a RIVAL invocation's flock — only the + # holder knows whether it won every board. r_fd, w_fd = os.pipe() pid = os.fork() if pid > 0: @@ -317,12 +362,12 @@ def cmd_hold(boards, reason): os._exit(0) # holder (grandchild): acquire all flocks, signal the parent, sleep until killed os.close(r_fd) - # Keep the success pipe clear of fds 0-2: invoked with stdio closed, - # os.pipe() can land there and the dup2 loop below would clobber it. + # Keep the success pipe clear of fds 0-2: invoked with stdio closed, os.pipe() can + # land there and the dup2 loop below would clobber it. if w_fd <= 2: w_fd = fcntl.fcntl(w_fd, fcntl.F_DUPFD, 3) - # Detach stdio: a `hold` whose output is captured must see EOF when the - # front-end exits — the immortal holder must not keep that pipe open. + # Detach stdio: a `hold` whose output is captured must see EOF when the front-end + # exits — the immortal holder must not keep that pipe open. devnull = os.open(os.devnull, os.O_RDWR) for std_fd in (0, 1, 2): os.dup2(devnull, std_fd) @@ -345,8 +390,8 @@ def cmd_hold(boards, reason): os.close(w_fd) def _bow_out(*_): - # clear the records before dying so read_record/status stay truthful - # (the kernel drops the flocks themselves on exit either way) + # clear the records before dying so read_record/status stay truthful (the kernel + # drops the flocks themselves on exit either way) for h in handles: clear_record(h) os._exit(0) @@ -368,8 +413,8 @@ def cmd_release(boards): try: fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) except OSError: - # flock genuinely held — never SIGTERM on a mere pid record: the - # pid may be recycled, or a live worker that already moved on. + # flock genuinely held — never SIGTERM on a mere pid record: the pid may be + # recycled, or a live worker that already moved on. fh.close() info = read_record(b) or {} pid = info.get('pid') @@ -449,9 +494,9 @@ def main(): p_hold.add_argument('boards', nargs='*') p_hold.add_argument('--all', action='store_true') p_hold.add_argument('--config', - default=os.path.join(os.path.dirname(os.path.abspath(__file__)), + default=os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'tinyusb.json'), - help='board roster JSON (default: tinyusb.json beside this script)') + help='board roster JSON (default: tinyusb.json in test/hil, one level above this script)') p_hold.add_argument('--reason', required=True) p_rel = sub.add_parser('release') p_rel.add_argument('boards', nargs='*') diff --git a/test/hil/hil_pool_check.py b/test/hil/helper/hil_pool_check.py index 98f24288a..d98b92bd4 100644 --- a/test/hil/hil_pool_check.py +++ b/test/hil/helper/hil_pool_check.py @@ -16,7 +16,7 @@ reported, never waited on or bypassed). Config is picked by hostname unless given: ci -> tinyusb.json, tusb (hifiphile rig) -> hfp.json, anything else is a dev PC -> local.json. -Lives in test/hil/ beside hil_lock.py and hil_flash.py, which it imports; board +Lives in test/hil/helper/ beside hil_lock.py; imports it and hil_flash; board recovery uses the repo's .claude/skills/usb-kernel-recover/scripts/usb_recover.sh. """ @@ -29,19 +29,17 @@ import re import shlex import shutil import socket -import subprocess import sys import threading import time from concurrent.futures import ThreadPoolExecutor from pathlib import Path -REPO_ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(Path(__file__).resolve().parent)) # for import-as-module callers - -import hil_lock +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # hil_flash + the helper package import hil_flash +from helper import hil_lock, hil_util +REPO_ROOT = hil_util.TINYUSB_ROOT USB_RECOVER = REPO_ROOT / '.claude' / 'skills' / 'usb-kernel-recover' / 'scripts' / 'usb_recover.sh' SEEN_CACHE = Path.home() / '.cache' / 'tinyusb-hil' / 'pool_seen.json' CONFIG_BY_HOST = {'ci': 'tinyusb.json', 'tusb': 'hfp.json'} # anything else: dev PC -> local.json @@ -56,6 +54,7 @@ ENUM_WAIT_RETRY = 8 # s, uid wait after a recovery reset/re-flash SERIAL_WAIT = 6 # s, host-board serial-output wait print_mutex = threading.Lock() +_STRANDED_WARNED = False # scan_usb's caveat: once per process, not once per poll t0 = time.monotonic() @@ -66,20 +65,33 @@ def say(msg: str) -> None: def scan_usb() -> dict: """busport -> {'serial', 'vidpid', 'ino'} for every enumerated USB device. Only - <bus>-<port>[.<port>...] dirs match (root hubs, named 'usbN' with no dash, are - excluded: their fabricated PCI-address 'serial' and slow autosuspend-wake read - cost 6-7s/scan on this rig). Keyed by busport, not serial: a serial can be - shared by two different devices (e.g. an Espressif USB-Serial-JTAG bridge and - the cafe TinyUSB device it flashes derive both from the same MAC) — collapsing - them into one dict slot would silently drop whichever lost the race.""" + <bus>-<port>[.<port>...] dirs match; root hubs ('usbN', no dash) are excluded because + their 'serial' is a fabricated PCI address, and including them measured 6-7s/scan slower + (an observation; NOT an autosuspend wake -- that read is cached and does no I/O). + Keyed by busport, not serial: two devices can share a serial (an Espressif + USB-Serial-JTAG bridge and the cafe device it flashes both derive it from the same + MAC), and one dict slot would silently drop whichever lost the race.""" found = {} - for f in glob.glob('/sys/bus/usb/devices/*-*/serial'): - d = os.path.dirname(f) - busport = os.path.basename(d) + # usb_scan's `serial` read is bounded by default (see hil_util.read_sysfs) -- this tool + # has no pool guard behind it and is run exactly when a device is suspected wedged. A + # device that will not answer is simply absent from the table; the footer says so. + devs = hil_util.usb_scan() + # ONCE per process, at SCAN time, not only in the footer: this tool prints rows as it + # goes over minutes, so a board dropped from the scan says "probe MISSING" within + # seconds while the only qualification would arrive after the final counts -- and an + # operator acting on the streaming output, or a run cut short by ^C, never sees it. + global _STRANDED_WARNED + if hil_util.sysfs_stranded() and not _STRANDED_WARNED: + _STRANDED_WARNED = True + say('WARNING: a bounded sysfs read gave up; rows below that say a probe or board ' + 'is missing may be this scan losing sight of healthy hardware. Find the ' + 'wedged device (usb-kernel-recover) and re-run.') + for dev in devs: try: - sn = open(f).read().strip().lower() - vidpid = f'{open(d + "/idVendor").read().strip()}:{open(d + "/idProduct").read().strip()}' - found[busport] = {'serial': sn, 'vidpid': vidpid, 'ino': os.stat(d + '/').st_ino} + found[dev['busport']] = { + 'serial': dev['serial'].lower(), + 'vidpid': f"{dev['vid']}:{dev['pid']}", + 'ino': os.stat(dev['dir'] + '/').st_ino} except OSError: continue return found @@ -112,7 +124,7 @@ def find_usb(uid: str, devs: dict | None = None): def find_device(uid: str, pid: str | None): """Board-online check: TinyUSB device (idVendor cafe) with this uid, optionally - PID-pinned. VID cafe keeps an Espressif USB-Serial-JTAG (303a) sharing the MAC + PID-pinned. VID cafe keeps an Espressif USB-Serial-JTAG (303a) that shares the MAC serial from false-passing.""" for busport, dev in scan_usb().items(): if (dev['serial'] == uid.lower() and dev['vidpid'].startswith('cafe:') @@ -134,22 +146,20 @@ def wait_device(uid: str, pid: str | None, old_ino, budget: float): def lock_board(name: str): - """Nonblocking flock per hil_lock.py protocol. Returns handle, or a str with - the holder's info when the board is locked elsewhere. Board locks are ALWAYS - respected: a held board is reported as locked and skipped — never waited on, - and there is deliberately no bypass here.""" + """Nonblocking flock per hil_lock.py protocol. Returns the handle, or a str with the + holder's info when the board is locked elsewhere. Board locks are ALWAYS respected: a + held board is reported and skipped, never waited on, and there is no bypass here.""" os.makedirs(hil_lock.BOARD_LOCK_DIR, exist_ok=True) try: fh = hil_lock.flock_nb(name) except OSError: - # NB: conflates a held flock with open() failures (EACCES/EROFS/ENOSPC) — - # benign while everything on the rig runs as one uid; a cross-uid setup - # would need flock_nb to distinguish the two + # NB: conflates a held flock with open() failures (EACCES/EROFS/ENOSPC) — benign + # while everything on the rig runs as one uid info = hil_lock.read_record(name) return json.dumps(info) if info else 'unknown holder' if not hil_lock.write_record(fh, 'pool_check'): - # an invisible lock (flock held, no record) is worse than no lock: status - # can't show us and release can't recognize the protected holder — bail out + # an invisible lock (flock held, no record) is worse than no lock: status cannot + # show us and release cannot recognize the protected holder hil_lock.clear_record(fh) fh.close() return 'ERROR: holder record write failed (lock dir unwritable?)' @@ -165,24 +175,27 @@ def can_recover() -> bool: if not USB_RECOVER.is_file(): return False try: - r = subprocess.run(['sudo', '-n', 'true'], capture_output=True) - except OSError: # sudo not installed (bare dev PC/container): recovery off, not fatal + # run_cmd, not subprocess.run: run's post-timeout reap is an UNBOUNDED wait(), and + # our kill bounces off a setuid-root sudo with EPERM, leaving communicate() on a + # pipe that never closes. run_cmd killpgs, escalates through sudo, reaps bounded. + r = hil_util.run_cmd('sudo -n true', timeout=10, quiet=True) + except OSError: # sudo not installed return False return r.returncode == 0 def recover_probe(uid: str, busport: str) -> bool: - """Soft-replug an enumerated-but-wedged probe: deauthorize+reauthorize (no VBUS - cut, touches only this device). Success = the probe re-enumerated (new sysfs - inode), not the helper's exit code (observed to flake while the toggle worked). - J-Links respond with a full disconnect and can stay off the bus for >8 s.""" + """Soft-replug an enumerated-but-wedged probe: deauthorize+reauthorize (no VBUS cut, + touches only this device). Success = the probe re-enumerated (new sysfs inode), not the + helper's exit code, which flakes while the toggle works. J-Links respond with a full + disconnect and can stay off the bus for >8 s.""" pre = find_usb(uid) - try: - # bounded: the sysfs authorized store can block in D state on a wedged - # device, and this runs while the board's (release-protected) flock is held - subprocess.run(['sudo', '-n', str(USB_RECOVER), 'authorized', busport], - capture_output=True, text=True, timeout=30) - except subprocess.TimeoutExpired: + # Bounded through run_cmd (same reason as can_recover): the sysfs authorized store can + # block in D state on a wedged device, and this runs while the board's release- + # PROTECTED flock is held -- a hang here would lock the board until the host reboots. + cmd = ' '.join(shlex.quote(a) for a in + ['sudo', '-n', str(USB_RECOVER), 'authorized', busport]) + if hil_util.run_cmd(cmd, timeout=30, quiet=True).returncode == 124: return False deadline = time.monotonic() + 20 while time.monotonic() < deadline: @@ -271,31 +284,28 @@ def get_expected_pid(example: str) -> str | None: def call_flasher(fn, *fn_args) -> tuple[int, str]: - """Run a hil_flash flash_*/reset_* backend, normalizing raises to a failure: - several backends raise instead of returning nonzero (get_serial_dev - RuntimeError when a bridge's /dev/serial/by-id node vanishes, config.env - FileNotFoundError, .jlink script OSError) and an exception must not skip the - caller's retry/recovery ladder. Returns (returncode, error line).""" + """Run a hil_flash flash_*/reset_* backend, normalizing raises to a failure: several + backends raise instead of returning nonzero (get_serial_dev when a bridge's + /dev/serial/by-id node vanishes, a missing config.env, a .jlink script OSError), and an + exception must not skip the caller's retry/recovery ladder. Returns (rc, error line).""" try: ret = fn(*fn_args) if ret.returncode == 0: return 0, '' - err = flash_error_line(hil_flash.cmd_stdout_text(ret.stdout)) + err = flash_error_line(hil_util.cmd_stdout_text(ret.stdout)) return ret.returncode, err or f'rc={ret.returncode}' except Exception as e: return -1, repr(e)[:90] def flash(board: dict, fw, allow_recovery: bool, probe_port: str, note: list) -> bool: - """Flash the resolved firmware with one retry; on repeated failure soft-replug - the probe and always make one final flash attempt afterward, regardless of - whether the replug is confirmed — some probes (WCH-Link, ST-Link, CP210x, - picoprobe) leave their sysfs kobject intact across an authorized toggle - instead of dropping off the bus. Returns True on success. + """Flash the resolved firmware with one retry; on repeated failure soft-replug the + probe and always make one final attempt afterward, confirmed replug or not — some + probes (WCH-Link, ST-Link, CP210x, picoprobe) keep their sysfs kobject across an + authorized toggle instead of dropping off the bus. Returns True on success. - `fw` comes from pick_example: a re-resolve here would use the global search - policy and miss a firmware ensure_fw just built into cmake-build/ under an - exclusive -B.""" + `fw` comes from pick_example: a re-resolve here would use the global search policy and + miss a firmware ensure_fw just built into cmake-build/ under an exclusive -B.""" fn = getattr(hil_flash, f'flash_{board["flasher"]["name"].lower()}') for attempt in range(3): if attempt == 2: @@ -303,9 +313,9 @@ def flash(board: dict, fw, allow_recovery: bool, probe_port: str, note: list) -> return False cur = find_usb(board['flasher']['uid']) if cur is None: - # probe gone from the bus: its old busport may now hold an UNRELATED - # device (bus renumbering) and the helper only checks occupancy, so - # toggling would deauthorize an innocent fixture — skip the toggle + # probe gone from the bus: its old busport may now hold an UNRELATED device + # (bus renumbering) and the helper only checks occupancy, so toggling would + # deauthorize an innocent fixture note.append('probe vanished before toggle') else: say(f'{board["name"]:26} recovery: replugging probe {cur[0]} (authorized toggle)') @@ -318,7 +328,8 @@ def flash(board: dict, fw, allow_recovery: bool, probe_port: str, note: list) -> if rc == 0: return True if rc == 127: # flasher binary missing: retries/probe recovery can't fix env - note.append(f'flasher tool missing ({err}) — esptool needs the ESP-IDF env (get-idf)' + note.append(f'flasher tool missing ({err}) — esptool needs the ESP-IDF env ' + f'(. "$IDF_PATH/export.sh")' if board['flasher']['name'].lower() == 'esptool' else f'flasher tool missing: {err}') return False @@ -349,27 +360,64 @@ def check_host_serial(board: dict, do_reset: bool = True, want_hello: bool = Fal do_reset=False listens to the firmware as-is: used right after a flash whose own reset already started it — a second openocd/JLink session back-to-back on - the same probe can fail transiently and leave the target halted.""" + the same probe can fail transiently and leave the target halted. + + "logger": "rtt" boards have no VCOM: the same check runs over the probe's RTT + console instead. The reset happens BEFORE the console opens (it owns the probe), + which also zeroes the .bss ring — so pre-reset backlog cannot count as life, and + without a reset Commander delivers the boot burst the preceding flash left.""" + if board.get('logger') == 'rtt': + if do_reset: + # a failed reset leaves the previous run's ring intact: attaching anyway would + # score stale output as life, so bail to host_alive's board_test reflash ladder + rc, err = call_flasher(getattr(hil_flash, f'reset_{board["flasher"]["name"].lower()}'), board) + if rc: + say(f'{board["name"]:26} reset failed: {err}') + return None + try: + ser = hil_util.JlinkRtt(board, timeout=0.3) + except hil_util.RttError as e: + say(f'{board["name"]:26} no RTT console: {e}') + return None + try: + data = b'' + deadline = time.monotonic() + SERIAL_WAIT + while time.monotonic() < deadline: + ser.write(b'U') + data += ser.read(256) + # JLinkExe's banner arrives whether or not the target is alive -- + # judged unfiltered it scores a dead board 'alive'. Same shared filter + # as test_host_device_info; complete_only drops a trailing partial + # line, so a banner FRAGMENT split by this read boundary cannot count + # as target output either. + td = hil_util.strip_banner(data, complete_only=True) + if want_hello: + if b'Hello from TinyUSB' in td: + return td + elif td and not boardtest_output(td): + return td + return hil_util.strip_banner(data) + except hil_util.RttError: + return None # console died mid-poll (server exited, probe dropped) + finally: + ser.close() import serial try: - port = hil_flash.get_serial_dev(board['flasher']['uid'], None, None, 0) + port = hil_util.get_serial_dev(board['flasher']['uid'], None, None, 0) ser = serial.Serial(port, baudrate=115200, timeout=0.3, write_timeout=1) except Exception as e: say(f'{board["name"]:26} no flasher serial port: {e}') return None try: - # flush BEFORE issuing the reset: pyserial's open-time flush is long past, - # so this drops the pre-reset CDC backlog (which must not count as life) - # while keeping the board's post-reset boot banner, which prints while the - # reset tool is still tearing down and would be eaten by a post-reset flush + # flush BEFORE the reset: this drops the pre-reset CDC backlog (which must not + # count as life) while keeping the post-reset boot banner, which prints while the + # reset tool is still tearing down and a post-reset flush would eat ser.reset_input_buffer() if do_reset: getattr(hil_flash, f'reset_{board["flasher"]["name"].lower()}')(board) - # collect the WHOLE window and judge content, not the first chunk: the - # probe's CDC bridge has its own FIFO, so stale pre-flash output (e.g. - # board_test hellos) can arrive after our host-side flush and must not - # decide the verdict alone. Early-exit once non-board_test output proves - # a real example is talking. + # judge the WHOLE window, not the first chunk: the probe's CDC bridge has its own + # FIFO, so stale pre-flash output (e.g. board_test hellos) can arrive after our + # host-side flush and must not decide the verdict alone. data = b'' deadline = time.monotonic() + SERIAL_WAIT while time.monotonic() < deadline: @@ -380,9 +428,9 @@ def check_host_serial(board: dict, do_reset: bool = True, want_hello: bool = Fal pass except serial.SerialException: return None # port dropped mid-poll (bridge re-enumerating) - # early-exit on the caller's positive signal: fresh board_test hello - # (park verification) vs any non-board_test output (example liveness); - # stale bridge-FIFO backlog of the OTHER kind must not end the window + # early-exit on the caller's positive signal (board_test hello for park + # verification, any non-board_test output for example liveness): stale + # bridge-FIFO backlog of the OTHER kind must not end the window if want_hello: if b'Hello from TinyUSB' in data: return data @@ -408,16 +456,13 @@ def boardtest_output(data: bytes) -> bool: def build_example(board: dict, variant: str, example: str) -> int: """Build one example for this board: tools/build.py (same invocation shape as - hil_test.build_board: -T target, -D per build.args, variant defines/flags, - --build-name), or idf.py directly for espressif (tools/build.py's esp branch - ignores -T and builds everything; variant flags travel as -DCFLAGS_CLI, the - same channel tools/build.py uses). Bounded and process-group-killed via - run_cmd; 600 s: a first configure+build of an SDK-heavy family (pico, nrf, - esp) exceeds the old 300. Builds normally run pre-lock (pick_example / the - pre-park ensure), so a board flock is not held here except on rare recovery - paths. Per-build compile parallelism is capped at cpu/-j so -j concurrent - builds cannot swamp sibling workers' verification windows. Returns the - build's returncode (127 = ESP-IDF env missing).""" + hil_test.build_board), or idf.py directly for espressif (tools/build.py's esp branch + ignores -T and builds everything; variant flags travel as -DCFLAGS_CLI, the channel + tools/build.py uses). Bounded and process-group-killed via run_cmd; 600 s covers a + first configure+build of an SDK-heavy family (pico, nrf, esp). Builds normally run + pre-lock, so a board flock is not held here except on rare recovery paths. Per-build + compile parallelism is capped at cpu/-j so -j concurrent builds cannot swamp sibling + workers' verification windows. Returns the returncode (127 = ESP-IDF env missing).""" name = board['name'] variants = board.get('variant') or [{'name': name}] vcfg = next((v for v in variants if v['name'] == variant), variants[0]) @@ -428,7 +473,7 @@ def build_example(board: dict, variant: str, example: str) -> int: cmd = ['idf.py', '-C', f'examples/{example}', '-B', f'cmake-build/cmake-build-{vcfg["name"]}/{example}', '-G', 'Ninja', f'-DBOARD={name}', 'build'] - for d in board.get('build', {}).get('args', []) + vcfg.get('defines', []): + for d in vcfg.get('defines', []): cmd.insert(-1, f'-D{d}') if vcfg.get('flags'): cmd.insert(-1, f'-DCFLAGS_CLI={vcfg["flags"]}') @@ -436,13 +481,11 @@ def build_example(board: dict, variant: str, example: str) -> int: # SOURCE tree (idf.py -B relocates only the build dir), so concurrent esp # builds of one example for different targets corrupt each other's solve with _esp_lock, _build_sem: - return hil_flash.run_cmd(shlex.join(cmd), cwd=str(hil_flash.TINYUSB_ROOT), - timeout=600).returncode - cmd = [sys.executable, str(hil_flash.TINYUSB_ROOT / 'tools' / 'build.py'), + return hil_util.run_cmd(shlex.join(cmd), cwd=str(hil_util.TINYUSB_ROOT), + timeout=600).returncode + cmd = [sys.executable, str(hil_util.TINYUSB_ROOT / 'tools' / 'build.py'), '-b', name, '-T', Path(example).name, '-j', str(max(1, (os.cpu_count() or _jobs) // _jobs))] - for d in board.get('build', {}).get('args', []): - cmd += ['-D', d] if vcfg['name'] != name: cmd += ['--build-name', vcfg['name']] for d in vcfg.get('defines', []): @@ -450,8 +493,8 @@ def build_example(board: dict, variant: str, example: str) -> int: for tok in vcfg.get('flags', '').split(): cmd += [f'--cflag={tok}'] with _build_sem: - return hil_flash.run_cmd(shlex.join(cmd), cwd=str(hil_flash.TINYUSB_ROOT), - timeout=600).returncode + return hil_util.run_cmd(shlex.join(cmd), cwd=str(hil_util.TINYUSB_ROOT), + timeout=600).returncode _deps_lock = threading.Lock() # one get_deps at a time (it also drains _build_sem) @@ -463,15 +506,13 @@ _builds: dict = {} # (variant, example) -> (fw|None, reason): one at def ensure_fw(board: dict, variant: str, example: str, note: list): - """Firmware for `example`, building it when absent — never skip a board for - lack of a build (--no-build opts out). One retry with deps fetched and the - CMake caches dropped when the first build fails (fresh checkouts lack the - family deps; a cache configured in a broken env poisons every later attempt). - Returns the firmware path, or None with the failure noted. Call BEFORE - taking the board lock: builds are long. One build attempt per - (variant, example) per run, success or failure — memoized in _builds, so a - repeat call (park, under the held flock) resolves instantly even when an - exclusive -B hides the fresh cmake-build/ artifact from the global search.""" + """Firmware for `example`, building it when absent — never skip a board for lack of a + build (--no-build opts out). One retry with deps fetched and the CMake caches dropped + when the first build fails (fresh checkouts lack the family deps; a cache configured + in a broken env poisons every later attempt). Returns the firmware path, or None with + the failure noted. Call BEFORE taking the board lock: builds are long. One attempt per + (variant, example) per run, memoized in _builds, so a repeat call (park, under the + held flock) resolves instantly even when an exclusive -B hides the fresh artifact.""" fw = hil_flash.find_firmware(variant, example, flasher=board['flasher']['name']) if fw: return fw @@ -485,31 +526,30 @@ def ensure_fw(board: dict, variant: str, example: str, note: list): rc = build_example(board, variant, example) if rc == 127 and board['flasher']['name'].lower() == 'esptool': _builds[key] = (None, 'no-env') - note.append(f'cannot build {base}: ESP-IDF env missing (get-idf)') + note.append(f'cannot build {base}: ESP-IDF env missing ' + f'(. "$IDF_PATH/export.sh")') return None if rc == 124: # hung build: a deps/cache retry cannot cure it, don't double the stall _builds[key] = (None, 'timeout') note.append(f'build timeout: {base}') return None if rc != 0: - # retry once with deps fetched and the CMake caches dropped (cache only — - # a tree wipe would destroy every other example's firmware). get_deps - # git-resets already-present shared deps (lib/fatfs's ffconf.h dance), so - # it must exclude every in-flight build, not just other get_deps calls: - # it drains ALL build slots before running. + # retry once with deps fetched and the CMake caches dropped (cache only — a tree + # wipe would destroy every other example's firmware). get_deps git-resets shared + # deps that are already present, so it drains ALL build slots first. with _deps_lock: for _ in range(_jobs): _build_sem.acquire() try: - r = hil_flash.run_cmd(shlex.join([sys.executable, str(hil_flash.TINYUSB_ROOT / 'tools' / 'get_deps.py'), - '-b', board['name']]), - cwd=str(hil_flash.TINYUSB_ROOT), timeout=600) + r = hil_util.run_cmd(shlex.join([sys.executable, str(hil_util.TINYUSB_ROOT / 'tools' / 'get_deps.py'), + '-b', board['name']]), + cwd=str(hil_util.TINYUSB_ROOT), timeout=600) finally: for _ in range(_jobs): _build_sem.release() if r.returncode != 0: note.append('get_deps failed') - bd = hil_flash.TINYUSB_ROOT / 'cmake-build' / f'cmake-build-{variant}' + bd = hil_util.TINYUSB_ROOT / 'cmake-build' / f'cmake-build-{variant}' # esp configures one level deeper (<variant>/<example>/): wipe both layouts for d in (bd, bd / example): shutil.rmtree(d / 'CMakeFiles', ignore_errors=True) @@ -519,9 +559,8 @@ def ensure_fw(board: dict, variant: str, example: str, note: list): _builds[key] = (None, 'fail') note.append(f'build failed: {base}') return None - # tools/build.py and the idf.py invocation above always write to cmake-build/: - # look there too even when an explicit -B narrowed the global search — this is - # OUR fresh build, not a stale-candidate fallback + # both build paths write to cmake-build/, so look there even when an explicit -B + # narrowed the global search — this is OUR fresh build, not a stale fallback fw = hil_flash.find_firmware(variant, example, roots=[hil_flash.build_dir, 'cmake-build'], flasher=board['flasher']['name']) @@ -673,26 +712,24 @@ def check_board(board: dict, args, allow_recovery: bool, seen: dict) -> dict: else 'probe never seen by pool_check') say(f'{name:26} probe MISSING ({board["flasher"]["name"]} {board["flasher"]["uid"]})') - # existing firmware only here; a missing build is built on the spot further - # down (after a lock peek), except in scan/no-build modes — and never for a - # missing probe (nothing could be flashed anyway) + # existing firmware only; a missing build is built further down (after a lock peek), + # except in scan/no-build modes and never for a missing probe example, kind, variant, fw = pick_example(board, note, build_missing=False) if kind == 'host': note.append('host-only board') if args.scan_only: hit = find_device(board['uid'], None) - # report the BOARD's usb state, not just the probe's: the enumerated device - # (with busport), off-bus (normal when parked in board_test), or n/a for - # host-only boards whose uid never enumerates + # report the BOARD's usb state too: enumerated (with busport), off-bus (normal + # when parked in board_test), or n/a for host-only boards if hit: row['device'] = f'✅ {hit[1]} @{hit[0]}' elif kind == 'host': row['device'] = '– n/a (host-only)' else: row['device'] = '⚫ off bus (parked?)' - # scan verifies probe presence only: that check DID run, so probe present - # is ok; a missing probe means no firmware could be delivered → flash-failed + # scan verifies probe presence only, so probe present is ok; a missing probe means + # no firmware could be delivered → flash-failed row['status'] = 'ok' if probe else 'flash-failed' if probe: say(f'{name:26} probe ✅ {probe[0]}' + (f' device {hit[1]}' if hit else '')) @@ -710,9 +747,9 @@ def check_board(board: dict, args, allow_recovery: bool, seen: dict) -> dict: and hil_flash.find_firmware(bt_variant, 'device/board_test', flasher=board['flasher']['name']) is None) if need_example or need_bt: - # builds are long and run BEFORE locking (park must never hold the flock - # through a build); peek the lock first so minutes of building are not - # wasted on — or a rebuilt tree swapped under — a board CI holds right now + # builds are long and run BEFORE locking (park must never hold the flock through + # one); peek first so minutes of building are not wasted on — or a rebuilt tree + # swapped under — a board CI holds right now peek = lock_board(name) if isinstance(peek, str): if peek.startswith('ERROR:'): # environment failure, not a held lock @@ -728,8 +765,8 @@ def check_board(board: dict, args, allow_recovery: bool, seen: dict) -> dict: if need_example: example, kind, variant, fw = pick_example(board, note, build_missing=True) if need_bt and (example is not None or kind == 'host'): - # skip the park-image build when the example build already failed on a - # device board: the row returns before any flash/park could use it + # skip the park build when the example build already failed on a device board: + # the row returns before any flash/park could use it ensure_board_test(board, bt_variant, note) if example is None: @@ -740,9 +777,8 @@ def check_board(board: dict, args, allow_recovery: bool, seen: dict) -> dict: row['status'] = 'flash-failed' say(f'{name:26} probe ✅ {probe[0]} (no firmware to flash)') return row - # host-only board: aliveness is still checkable without flashing — reset and - # listen to whatever firmware is on it (the parked board_test echoes and - # prints a periodic hello on the flasher UART) + # host-only board: aliveness is still checkable without flashing — reset and listen + # to whatever is on it (parked board_test echoes and hellos on the flasher UART) lk = lock_board(name) if isinstance(lk, str): @@ -783,9 +819,8 @@ def check_board(board: dict, args, allow_recovery: bool, seen: dict) -> dict: say(f'{name:26} {row["flash"]} {row["device"]}') return row finally: - # teardown for EVERY path that attempted a flash (a failed programmer op - # can still have erased/half-written the target): re-park while the - # board lock is still held + # teardown for EVERY path that attempted a flash (a failed programmer op can + # still have erased/half-written the target), while the lock is still held if not args.no_park: park_board(board, kind, row, note) finally: @@ -801,8 +836,8 @@ def park_board(board: dict, kind: str, row: dict, note: list) -> None: 'failed' verify verdict — that is the more diagnostic signal), with one exception: an espressif board without the ESP-IDF env cannot build board_test — noted, not a board fault.""" - # capture BEFORE the park flash: uid-disappearance only verifies the park if - # the device was on the bus to begin with (a fast park drops it immediately) + # capture BEFORE the park flash: uid-disappearance only verifies the park if the + # device was on the bus to begin with on_bus_before = kind != 'host' and find_device(board['uid'], None) is not None variant = resolve_variant(board, 'device/board_test', note) fw = ensure_board_test(board, variant, note) @@ -826,9 +861,9 @@ def park_board(board: dict, kind: str, row: dict, note: list) -> None: row['status'] = 'flash-failed' return if kind == 'host': - # no second reset (the park flash's own reset already started board_test); - # POSITIVE marker: its hello must appear — stale example output may still - # drain from the probe bridge's FIFO alongside it and is not disqualifying + # no second reset (the park flash's own reset started board_test); POSITIVE + # marker: its hello must appear, and stale bridge-FIFO output alongside it is not + # disqualifying data = check_host_serial(board, do_reset=False, want_hello=True) if not (data and b'Hello from TinyUSB' in data): note.append('park unverified: no board_test output') @@ -836,8 +871,8 @@ def park_board(board: dict, kind: str, row: dict, note: list) -> None: row['status'] = 'flash-failed' return if not on_bus_before: - # board never enumerated this run: uid-disappearance can't distinguish a - # verified park from a silent no-op — say so instead of passing vacuously + # never enumerated this run: uid-disappearance cannot tell a verified park from a + # silent no-op — say so instead of passing vacuously note.append('park unverified (device already off bus)') return deadline = time.monotonic() + 6 @@ -898,9 +933,8 @@ def controller_summary() -> list[str]: def main() -> None: - # toolchain/flasher CLIs live in the user bin dirs (arm-none-eabi-gcc + esptool - # in ~/.local/bin, STM32_Programmer_CLI in ~/bin) which non-login shells may - # lack — same PATH shim hil_ci.sh applies on the remote side + # toolchain/flasher CLIs live in the user bin dirs, which non-login shells may lack -- + # the same PATH shim hil_ci.sh applies on the remote side for d in (Path.home() / 'bin', Path.home() / '.local' / 'bin'): if d.is_dir() and str(d) not in os.environ.get('PATH', '').split(os.pathsep): os.environ['PATH'] = f'{d}{os.pathsep}{os.environ.get("PATH", "")}' @@ -917,8 +951,8 @@ def main() -> None: help='do not build missing firmware (default: build the light example on the spot)') parser.add_argument('--no-park', action='store_true', help='leave the light example running (default: park with board_test)') - # no cross-process flash budget with a concurrent hil_test.py run yet (would need - # a file-lock budget in hil_lock; hil_test uses in-process semaphores) — keep modest + # no cross-process flash budget against a concurrent hil_test.py run (its semaphores + # are in-process), so keep this modest parser.add_argument('-j', '--jobs', type=int, default=4) parser.add_argument('-v', '--verbose', action='store_true') args = parser.parse_args() @@ -946,12 +980,11 @@ def main() -> None: boards = [b for b in boards if b['name'] in args.board] hil_flash.build_dir = args.build_dir or 'examples' - hil_flash.verbose = args.verbose + hil_util.verbose = args.verbose if args.build_dir is None: - # default mode: search both standard layouts (cmake-build/ from tools/build.py - # + ESP-IDF, examples/ from manual builds). An EXPLICIT -B is exclusive — the - # caller named an artifact tree, so a miss must report, not silently flash an - # older build from elsewhere. hil_test's -B is likewise untouched by this. + # default mode: search both standard layouts (cmake-build/ from tools/build.py and + # ESP-IDF, examples/ from manual builds). An EXPLICIT -B stays exclusive: the caller + # named an artifact tree, so a miss must report rather than flash an older build. hil_flash.EXTRA_BUILD_DIRS = ['cmake-build', 'examples'] allow_recovery = not args.scan_only and can_recover() seen = {} @@ -971,7 +1004,7 @@ def main() -> None: rows = [check_board_safe(b, args, allow_recovery, seen) for b in boards] else: with io.StringIO() as spool, ThreadPoolExecutor(max_workers=args.jobs) as pool: - sys.stdout = spool # silence hil_flash's COMMAND FAILED dumps; say() uses __stdout__ + sys.stdout = spool # silence hil_util.run_cmd's COMMAND FAILED dumps; say() uses __stdout__ try: rows = list(pool.map(lambda b: check_board_safe(b, args, allow_recovery, seen), boards)) finally: @@ -990,9 +1023,13 @@ def main() -> None: headers = ['Board', 'Probe', 'Flash', 'Device', 'Status', 'Note'] cells = [[r['name'], r['probe'], r['flash'], r['device'], status_mark.get(r['status'], r['status']), '; '.join(r['note'])] for r in rows] - widths = [max(len(h), *(len(c[i]) for c in cells)) if cells else len(h) + # display_width, not len(): ✅ / ❌ / 🔒 / ⚠ are one character and two columns, so + # len() pads every row holding one a column short of the header rule + _w = hil_util.display_width + widths = [max(_w(h), *(_w(c[i]) for c in cells)) if cells else _w(h) for i, h in enumerate(headers)] - line = lambda vals: '| ' + ' | '.join(v.ljust(w) for v, w in zip(vals, widths)) + ' |' + line = lambda vals: ('| ' + ' | '.join(hil_util.pad(v, w) + for v, w in zip(vals, widths)) + ' |') print() print(line(headers)) print('|' + '|'.join('-' * (w + 2) for w in widths) + '|') @@ -1008,6 +1045,16 @@ def main() -> None: counts[r.get('status', 'failed')] += 1 print(f'\n{counts["ok"]} ok · {counts["flash-failed"]} flash-failed · {counts["failed"]} failed ' f'· {counts["locked"]} locked · in {time.monotonic() - t0:.0f}s') + if hil_util.sysfs_stranded(): + # Without this the table is the worst kind of wrong: a device whose `serial` never + # answered is absent from the scan, which prints as "probe MISSING"/"off bus" for + # hardware that is physically present -- during exactly the incident this tool is + # run to diagnose, and it sends the operator to power-cycle a healthy rig. + print('WARNING: at least one sysfs read did not answer within ' + f'{hil_util.SYSFS_READ_GRACE:.0f}s, so rows above that say a probe or board ' + f'is missing may be this tool losing sight of healthy hardware rather than ' + f'absent hardware. Find the wedged device (see the usb-kernel-recover ' + f'skill) and re-run before acting on the table.') sys.exit(min(counts['flash-failed'] + counts['failed'], 125)) diff --git a/test/hil/helper/hil_report.py b/test/hil/helper/hil_report.py new file mode 100644 index 000000000..c93c8e6a1 --- /dev/null +++ b/test/hil/helper/hil_report.py @@ -0,0 +1,578 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""The HIL report document: one owner for hil_report.json and hil_report.md. + +The markdown IS a rendering of the sidecar -- every writer goes through render_report(), so +a table can never contain something the JSON does not. This module owns the whole life of +that document: the cell vocabulary, the one classifier both artifacts share, rendering, the +writers, and the fold to one machine-readable verdict per board. + +Dual-mode by design: imported as `helper.hil_report` by hil_test.py, and run as a script by +the operator (see .claude/agents/hil-operator.md). A script run puts test/hil/helper on +sys.path rather than test/hil, so this module imports no sibling helper at all -- +_p and the width helpers below are defined locally for that reason. +""" +import argparse +import json +import sys +import unicodedata +from pathlib import Path + + +def _w(s: str) -> int: + """Terminal COLUMNS, not characters. Every status mark in REPORT_CELL is one Python + character and TWO columns wide, so len() pads a cell holding one a column short and + the pipes drift out of line with the header rule for the whole table. + + Local, like _p above and for the same reason: this module is also run as a script, and + under PYTHONSAFEPATH=1 a sibling import dies before argparse runs. hil_util carries the + same pair for callers that can import it. + """ + return sum(2 if unicodedata.east_asian_width(c) in 'WF' else 1 for c in s) + + +def _pad(s: str, width: int, center: bool = False) -> str: + """str.ljust/center, measured in display columns. See _w.""" + room = max(0, width - _w(s)) + if not center: + return s + ' ' * room + left = room // 2 + return ' ' * left + s + ' ' * (room - left) + + +def _p(*args, **kwargs) -> None: + """Print that cannot raise. Defined here rather than imported from hil_health: this + module is ALSO run as a script (hil-operator.md invokes it by path), and under + PYTHONSAFEPATH=1 -- which the suite's own MTP fixtures set -- sys.path[0] is not the + script dir, so any sibling import dies before argparse runs. Five lines beat that.""" + try: + print(*args, **kwargs) + except (OSError, ValueError): + # ValueError too: printing to a CLOSED stream raises "I/O operation on closed + # file", and escaping here skips the containment path's os._exit. + pass + +REPORT_MD = 'hil_report.md' +REPORT_JSON = 'hil_report.json' +# The status vocabulary, shared by the code that WRITES a cell (hil_test's test runners) and +# the code that reads one back (cell_state). One dict, so the human's table and the agent's +# verdict cannot drift apart. +REPORT_CELL = {'pass': '✅', 'fail': '❌', 'skip': '⚪'} +BOUNDARY_CELL = 'same-PID boundary' +LOCKED_CELL = 'board-locked' +# A pseudo-test column, not a real one: write_timeout_report marks the boards that were +# still dispatched when the pool guard fired. accumulate_report clears it on a retry. +POOL_TIMEOUT_CELL = 'pool-timeout' +# The other way a board can fail to report: the pool did not expire, a worker RAISED. Same +# shape, different cause, and naming the cause is the whole point of the column -- a board +# marked pool-timeout by an abort that never timed out sends the reader after the guard. +RUN_ABORTED_CELL = 'run-aborted' + + +def _load(report_dir: Path) -> tuple: + """(doc, readable) for the sidecar, coerced to the canonical shape. + + hil_ci.sh uploads a sidecar as the --accumulate merge base, so a non-conforming one is + reachable from OUTSIDE the harness -- and every writer here runs on a path where a + TypeError costs the whole report. Coerce once, at the boundary, instead of guarding + each use: `banner: null` used to kill a fully successful run with a traceback and no + artifact at all, and `cells: null` sent write_timeout_report down its fallback so a + board that ate the whole pool guard was published as a pass. + + `readable` is False only when a sidecar EXISTS but could not be parsed, or is absent -- + both mean its rows are unrecoverable, which callers use to avoid destroying a markdown + that may still hold them.""" + jpath = report_dir / REPORT_JSON + if not jpath.is_file(): + return {'rows': [], 'banner': '', 'scope': '', 'caveat': ''}, False + try: + raw = json.loads(jpath.read_text()) + if not isinstance(raw, dict): + raise ValueError('sidecar is not an object') + except (OSError, ValueError, TypeError): + return {'rows': [], 'banner': '', 'scope': '', 'caveat': ''}, False + rows = [] + # isinstance, not `or []`: a sidecar with `rows: 1` iterates an int and raises outside + # the parse handler above. + for r in (raw.get('rows') if isinstance(raw.get('rows'), list) else []): + if not isinstance(r, dict) or 'board' not in r: + continue + cells = r.get('cells') + dur = r.get('duration') + # VALUES as well as keys: render_matrix does REPORT_CELL.get(v, v), which raises + # TypeError on an unhashable value, and cell_state does v.startswith. A non-str + # cell is corrupt, and dropping it renders blank -- "not run" -- which is the + # honest reading. Coercing it to str would make it classify as a PASS. + rows.append({'board': str(r['board']), + 'cells': {str(k): v for k, v in cells.items() if isinstance(v, str)} + if isinstance(cells, dict) else {}, + 'duration': dur if isinstance(dur, str) else None}) + text = lambda k: raw[k] if isinstance(raw.get(k), str) else '' + return {'rows': rows, 'banner': text('banner'), 'scope': text('scope'), + 'caveat': text('caveat')}, True + + +def cell_state(v) -> str: + """'pass' | 'fail' | 'skip' for one report cell. + + THE classifier -- the markdown tally and the per-board verdict both call this, so they + cannot disagree. 'fail' or a fail-icon prefix is a failure, 'skip' or a skip-icon prefix + is a skip, and EVERYTHING ELSE is a pass. That last arm is load-bearing: a passing test + may return a plain metric string ('480.0 MBps') that lands in the cell unprefixed, while + failures are guaranteed marked -- TestFail's docstring pins that its metric is + icon-prefixed precisely so render and tally treat it as a failure. Classifying unknown + shapes as fail here would publish a green table as a red verdict. + + isinstance-guarded: cells are usually str but a caller may hand over None or a number, + and .startswith on those raises inside a report writer that must not raise.""" + if v == 'fail' or (isinstance(v, str) and v.startswith(REPORT_CELL['fail'])): + return 'fail' + if v == 'skip' or (isinstance(v, str) and v.startswith(REPORT_CELL['skip'])): + return 'skip' + return 'pass' + + +def render_matrix(rows_all: list) -> str: + """Render rows (list of (row_label, {example: status}, duration)) as an aligned + markdown matrix: columns = tests (bare names) centered, boards left-aligned, + per-row duration as the trailing column.""" + seen = set() + for _, cells, _ in rows_all: + seen.update(cells) + if not seen: + return 'No tests were run.' + + # metric-bearing columns pinned first, the rest alphabetical: stable regardless of the + # shuffled execution order + pinned = ['usbtest', 'cdc_msc_throughput', 'msc_file_explorer', 'msc_file_explorer_freertos'] + + def col_key(t): + name = t.rsplit('/', 1)[-1] + return (pinned.index(name) if name in pinned else len(pinned), name, t) + + columns = sorted(seen, key=col_key) + headers = [c.rsplit('/', 1)[-1] for c in columns] + ['duration'] # bare example names + + def cell(cells, col): + v = cells.get(col) + if v is None: + return '' + return REPORT_CELL.get(v, v) # status symbol, or a metric string (e.g. speed) verbatim + + rows_vals = [(lbl, [cell(cells, c) for c in columns] + [dur or '']) + for lbl, cells, dur in rows_all] + board_hdr = 'Board' + # display_width, not len(): the ✅/❌/⚪ marks are one character and two columns + board_w = max([_w(board_hdr)] + [_w(lbl) for lbl, _ in rows_vals]) + col_w = [max([_w(h)] + [_w(vals[i]) for _, vals in rows_vals]) + for i, h in enumerate(headers)] + + def line(label, values): + padded = [_pad(label, board_w)] + [_pad(v, w, center=True) + for v, w in zip(values, col_w)] + return '| ' + ' | '.join(padded) + ' |' + + header = line(board_hdr, headers) + sep = '| ' + '-' * board_w + ' | ' + ' | '.join(':' + '-' * (w - 2) + ':' for w in col_w) + ' |' + body = [line(lbl, vals) for lbl, vals in rows_vals] + + # tally run cells (not-run cells are absent from the dicts). A cell is a bare status or + # a metric string carrying its own icon ("❌ 29/30"), so classify by the leading icon -- + # through cell_state, the same call the per-board verdict makes. + kinds = [cell_state(v) for _, cells, _ in rows_all for v in cells.values()] + failed = kinds.count('fail') + skipped = kinds.count('skip') + passed = kinds.count('pass') + summary = (f'**{REPORT_CELL["pass"]} {passed} passed · {REPORT_CELL["fail"]} {failed} failed · ' + f'{REPORT_CELL["skip"]} {skipped} skipped · blank not run**') + + return summary + '\n\n' + '\n'.join([header, sep] + body) + + +def render_report(doc: dict) -> str: + """The markdown IS a rendering of the sidecar. Every writer goes through here, so a + table can never contain something the JSON does not.""" + # .get throughout, not subscripts: mark_report_abandoned renders a sidecar it did NOT + # write (hil_ci.sh reuses a persistent REMOTE_DIR, so it may be an older version's or + # a torn one) on the way to os._exit, and a KeyError there is not in its handler -- + # it would unwind into multiprocessing's unbounded join and hang the runner it is + # trying to free. Same reason summarize() below reads cells as `r.get('cells') or {}`. + md = render_matrix([(r.get('board', '?'), r.get('cells') or {}, r.get('duration')) + for r in doc.get('rows') or [] if isinstance(r, dict)]) + if doc.get('scope'): + # a scoped run's small table is otherwise indistinguishable from a full one, and + # it replaces the previous full table in the sticky PR comment + md = f'_Scoped run: {doc["scope"]}. Boards/tests not listed were not run._\n\n' + md + # banner, then caveat: a rig-health caveat outranks the table AND the scope note, and an + # abandon notice outranks even that -- the top of the report is where hil/SKILL.md tells + # the agent to look + if doc.get('banner'): + md = doc['banner'] + '\n' + md + if doc.get('caveat'): + md = doc['caveat'] + '\n' + md + return md + + +def write_report(report_dir: Path, doc: dict) -> None: + """Write both artifacts from one document. + + RAISES on failure, deliberately: every caller is on a path whose own handler exists to + report exactly this (write_timeout_report's _p warning, hil_test's fallback-of-the- + fallback). Swallowing OSError here made both of those dead code, so an unwritable or + root-owned report dir produced no artifact AND no message. + + Renders BEFORE writing anything: committing the JSON first and then raising in + render_report left a sidecar saying "abandoned" beside a markdown still reading as a + clean green table -- the one invariant this module exists to hold.""" + md = render_report(doc) + '\n' + report_dir.mkdir(parents=True, exist_ok=True) + (report_dir / REPORT_JSON).write_text(json.dumps(doc, indent=2) + '\n') + (report_dir / REPORT_MD).write_text(md, encoding='utf-8') + + +def _abandon_notice(why: str) -> str: + # Wording is a CONTRACT: .claude/skills/hil/SKILL.md pins this banner as the case where + # "the table below IS this run's ... Report the results AND the abandonment". Calling + # the table partial would send the reading agent to re-run boards that already passed. + return (f'**HIL run abandoned: {why}** The table below was collected before the ' + f'abandon; treat board results as unverified.\n') + + +def _already_abandoned(doc: dict) -> bool: + """Whether THIS attempt already recorded how it ended. + + `caveat` only. It used to check `banner` too, because hil_test.py folded its abandon + notices in there -- but banner is carried across an --accumulate retry by design, so a + stale notice from an earlier attempt silenced a genuinely new abandon and the run's own + failure went unrecorded. banner now carries rig HEALTH (which describes the conditions + the cells were collected under, and so must persist); caveat carries the run's OUTCOME + (which must not).""" + return '**HIL run ab' in doc.get('caveat', '') + + +def _stamp_markdown(report_dir: Path, notice: str) -> None: + """Last line of defence: prepend the notice to the markdown itself. + + pr_comment.yml cats only hil_report.md, so a path that gives up here publishes a clean + green table under an abandoned, non-zero job. Master did this unconditionally.""" + mpath = report_dir / REPORT_MD + if not mpath.is_file(): + return + # errors='replace' and catch ValueError: a torn report or a LANG=C locale raises + # UnicodeDecodeError -- NOT an OSError -- straight past os._exit. + body = mpath.read_text(encoding='utf-8', errors='replace') + if '**HIL run ab' not in body[:2000]: + mpath.write_text(notice + '\n' + body, encoding='utf-8') + + +def mark_report_abandoned(report_dir: Path, why: str) -> None: + """Stamp an existing report as abandoned, in BOTH artifacts. + + Best-effort and silent: this runs while the interpreter is being torn down, and an + exception here hangs the process in multiprocessing's unbounded join().""" + notice = _abandon_notice(why) + try: + doc, readable = _load(report_dir) + if readable: + if _already_abandoned(doc): + return # whoever got there first wins, WRITE included + doc['caveat'] = notice + write_report(report_dir, doc) + return + except (OSError, ValueError, TypeError, AttributeError): + pass # fall through -- a failure here must not cost the stamp entirely + # Unreadable sidecar, or the document write failed. Either way the markdown is what + # the PR comment reads, so stamp it directly rather than giving up. + try: + _stamp_markdown(report_dir, notice) + except (OSError, ValueError, TypeError, AttributeError): + pass + + +def mark_report_no_boards(report_dir: Path, msg: str, fresh: bool = True) -> None: + """Record that the board filters intersected to nothing. + + `fresh` mirrors hil_test's own flag, because this runs BEFORE the fresh wipe: without + it a fresh run whose filter emptied re-published the PREVIOUS run's green rows under + this run's red job -- the stale-table failure it exists to prevent. An --accumulate run + keeps them, since nothing this attempt did invalidates them.""" + try: + doc, _ = _load(report_dir) + if not fresh and _already_abandoned(doc): + # SKILL.md gives the two notices OPPOSITE rules, and an abandon outranks a + # filter that matched nothing -- do not overwrite the record of a failed run. + # Only while ACCUMULATING, though: this runs before the fresh wipe, so guarding + # a fresh run would leave the previous attempt's rows AND its abandon notice + # published as this run's. + return + # A fresh run carries NOTHING from the prior sidecar -- rows, banner and scope + # alike, matching accumulate_report, which builds from an empty prior when fresh. + # Resetting only rows republished a stale rig-health note and a stale scope line + # under this run's notice, from a leftover or uploaded sidecar. + prior = {'rows': [], 'banner': '', 'scope': ''} if fresh else doc + write_report(report_dir, {'rows': prior['rows'], 'banner': prior['banner'], + 'scope': prior['scope'], + 'caveat': f'**HIL run selected no boards.** {msg}\n'}) + except (OSError, ValueError, TypeError, AttributeError): + pass # loud on stdout already; the exit code is what the job reads + + +def accumulate_report(mret: list, report_dir: Path, fresh: bool, scope: str = '', + banner: str = '', caveat: str = '') -> str: + """Merge this run's results into json in report_dir, then (re)write + the markdown matrix to md. `fresh` (a first run, no --accumulate) + starts a new report; otherwise a re-run accumulates so boards/tests that + already passed are preserved while re-run cells are updated. `scope` names the + board filter, if any, so a scoped table is not mistaken for a full one. + Returns the md. + + `mret` is hil_test.py's worker-result shape (name, err, fts, rows, ...), so this one + function knows something about its caller that the rest of the module does not. Folding + mret into rows could live in hil_test and only the merge here, but that would rewrite + the subtle parts -- stale board-locked clearing, BOUNDARY_CELL dropping, duration=None + preservation -- for a tidier seam. Data-shape coupling, not an import cycle.""" + # ONE canonical load: a sidecar reaching here may have been uploaded by hil_ci.sh as + # the merge base, so it is untrusted input. `banner` carries forward -- it describes + # the conditions the earlier cells were collected under, and the .failed spec re-runs + # only FAILURES so those passes are never re-earned. `caveat` does NOT: it records how + # a RUN ENDED, and this attempt has not ended yet. Carrying it made a clean retry + # publish "HIL run abandoned" over a run where nothing was abandoned. + prior = {'rows': [], 'banner': ''} + if not fresh: + prior, _ = _load(report_dir) + acc = {r['board']: [dict(r['cells']), r['duration']] for r in prior['rows']} + prior_banner = prior['banner'] + + # current cells override prior for boards/tests that ran; a filtered run reports + # duration None, keeping the previous full-run value + for name, _, _, rows, *_ in mret: + if rows and not any(LOCKED_CELL in cells for _, cells, _ in rows): + # board ran for real: clear a stale lock-failure cell (its row is keyed by + # board name; test rows may be variant names) + stale = acc.get(name) + if stale is not None: + stale[0].pop(LOCKED_CELL, None) + # and the pool-timeout mark: write_timeout_report stamps it on a board that + # never reported, and update() below MERGES, so without this a board that + # passed clean on the retry kept a red cell for ever. + stale[0].pop(POOL_TIMEOUT_CELL, None) + stale[0].pop(RUN_ABORTED_CELL, None) + if not stale[0]: + # variant-keyed boards never repopulate the board-name row, so drop it + # or it renders as a blank ghost row + del acc[name] + for row_label, cells, dur in rows: + row = acc.setdefault(row_label, [{}, None]) + # a row that ran is no longer pool-timed-out, whatever it is keyed by + row[0].pop(POOL_TIMEOUT_CELL, None) + row[0].pop(RUN_ABORTED_CELL, None) + # the boundary cell is only ever written on failure, so a re-run of this + # variant that cleared the boundary must drop the previous attempt's ❌ + if BOUNDARY_CELL not in cells: + row[0].pop(BOUNDARY_CELL, None) + row[0].update(cells) + if dur is not None: + row[1] = dur + + report_dir.mkdir(parents=True, exist_ok=True) + # by LINE, deduped: attempts repeat the same caveat far more often than they add a new + # one, and three copies of the D-state note reads as three incidents + seen, merged = set(), [] + for line in (prior_banner + banner).splitlines(): + if line.strip() and line not in seen: + seen.add(line) + merged.append(line) + banner = '\n'.join(merged) + '\n' if merged else '' + doc = {'rows': [{'board': k, 'cells': c, 'duration': d} for k, (c, d) in acc.items()], + 'banner': banner, 'scope': scope, 'caveat': caveat} + # through write_report, not hand-rolled: writing the JSON and only then rendering is + # the ordering write_report exists to forbid -- a render failure left the sidecar ahead + # of the markdown, which is the one invariant this module holds. + write_report(report_dir, doc) + return render_report(doc) + + +def _write_stuck_over_prior_md(report_dir: Path, doc: dict) -> None: + """Sidecar unrecoverable: rebuild it from the stuck rows alone, but leave the + markdown's existing table beneath the caveat rather than throwing real results away. + + The one place the md-is-a-rendering-of-the-json invariant is deliberately suspended, + because there is no readable json left for it to be a rendering of.""" + try: + prior = (report_dir / REPORT_MD).read_text(encoding='utf-8') + except (OSError, ValueError): + prior = '' + # Say so explicitly: those rows exist only as rendered text, so no later --accumulate + # can merge them back. Claiming the sidecar represents them would be false. + note = ('_The table below is a previous attempt\'s rendered output. The sidecar could ' + 'not be read, so those rows are NOT in it and will not survive another run._\n') + head = (doc['banner'] + '\n' if doc['banner'] else '') + doc['caveat'] + '\n' + note + body = prior if prior.strip() else render_matrix( + [(r['board'], r['cells'], r['duration']) for r in doc['rows']]) + report_dir.mkdir(parents=True, exist_ok=True) + (report_dir / REPORT_JSON).write_text(json.dumps(doc, indent=2) + '\n') + (report_dir / REPORT_MD).write_text(head + '\n' + body, encoding='utf-8') + + +def write_timeout_report(report_dir: Path, boards, secs: int, + banner: str = '', prefix: str = '', + cell: str = POOL_TIMEOUT_CELL) -> None: + """Leave a report behind when the worker pool has to be abandoned. + + map_async is all-or-nothing, so a timeout loses every per-board result and the report + dir would stay empty with no reason for the failure. Any prior attempt's rows are kept + and each stuck board is marked with a POOL_TIMEOUT_CELL beside them. + + `prefix` is the preflight rig-health verdict and goes to the BANNER, where rig health + lives and where an --accumulate retry carries it forward; the abandon notice goes to + the caveat, which does not carry. Folding both into the caveat is what made a clean + retry report an abandonment that had not happened.""" + try: + # names INSIDE the try: a roster entry that is not a dict raises here, and outside + # it that escaped and stranded the runner. + names = [b.get('name', '?') if isinstance(b, dict) else '?' for b in boards] + caveat = banner or ( + f'**HIL run abandoned: worker pool timed out after {secs}s.**\n\n' + f'No per-board results could be collected for this attempt. Rows other than ' + f'the {cell} cells below are from an earlier attempt. Boards ' + f'dispatched:\n\n' + '\n'.join(f'- {n}' for n in names) + '\n') + doc, readable = _load(report_dir) + rows = doc['rows'] + by_board = {r['board']: r for r in rows} + for name in names: + row = by_board.get(name) + if row is None: + rows.append({'board': name, 'cells': {cell: 'fail'}, + 'duration': None}) + else: + # _load guarantees `cells` is a dict, so a null-cells row from an uploaded + # sidecar can no longer send this down the fallback and publish a board + # that ate the whole pool guard as a pass. + row['cells'][cell] = 'fail' + out = {'rows': rows, 'scope': doc['scope'], 'caveat': caveat, + 'banner': ((doc['banner'] + prefix) if prefix not in doc['banner'] + else doc['banner'])} + if not readable and (report_dir / REPORT_MD).is_file(): + # `readable` covers ABSENT as well as torn: an absent sidecar beside an intact + # markdown used to re-render from the stuck row alone and destroy real results. + _write_stuck_over_prior_md(report_dir, out) + return + write_report(report_dir, out) + except Exception as e: # noqa: BLE001 + # Deliberately broad: this is the first statement of the pool-abandon path, so ANY + # escape skips kill_pool_children and os._exit and strands the runner. + _p(f'warning: cannot write {REPORT_MD} to {report_dir}: {e}', flush=True) + try: + # Same wording as above and the same guarded name extraction -- the fallback + # used to re-derive b.get("name") outside any try and raise identically, so a + # malformed roster left NO artifact at all. + names = [b.get('name', '?') if isinstance(b, dict) else '?' for b in boards] + head = (prefix + '\n' if prefix else '') + (banner or ( + f'**HIL run abandoned: worker pool timed out after {secs}s.**\n\n' + f'No per-board results could be collected for this attempt, so the table ' + f'below (if any) is from an earlier one. Boards dispatched:\n\n' + + '\n'.join(f'- {n}' for n in names) + '\n')) + try: + prior = (report_dir / REPORT_MD).read_text(encoding='utf-8') + except (OSError, ValueError): + prior = '' + report_dir.mkdir(parents=True, exist_ok=True) + (report_dir / REPORT_MD).write_text( + head + (f'\n{prior}' if prior else ''), encoding='utf-8') + except Exception as e2: # noqa: BLE001 + _p(f'warning: fallback {REPORT_MD} write failed too: {e2}', flush=True) + + +def variants_of(cfg: dict, board: str) -> list: + for b in cfg.get('boards', []): + if b['name'] == board: + return [v['name'] for v in (b.get('variant') or [])] or [board] + return [board] + + +def summarize(cfg: dict, boards: list, report: dict) -> dict: + # .get, not a subscript: this is the one reader an agent's verdict depends on, and a + # row without 'board' used to kill the CLI with a traceback and no results at all -- + # hil-validate.js then reports every board as "hil-operator returned no entry". + rows = {r['board']: r.get('cells') or {} + for r in (report.get('rows') or []) + if isinstance(r, dict) and 'board' in r} + owner = {v['name']: b['name'] for b in cfg.get('boards', []) + for v in (b.get('variant') or [])} + results = [] + for board in boards: + names = variants_of(cfg, board) + mine = {n: rows[n] for n in names if n in rows} + # a variant name that is neither declared nor prefixed cannot be attributed; the + # `<board>-` fallback only helps ad-hoc builds, it is not the primary path. It must + # also never steal a row DECLARED by another board: a declared variant need not start + # with its own board's name, so it may happen to start with this board's name plus '-'. + mine.update({n: c for n, c in rows.items() + if n.startswith(f'{board}-') and n not in mine + and owner.get(n, board) == board}) + # the BOARD-name row too: hil_test writes lock contention and pool timeouts keyed + # by board name, but variants_of returns only DECLARED variant names -- and + # nanoch32v203 / ch32v307v_r1_1v0 declare none equal to their board name. Without + # this those rows are invisible, so a lock held by concurrent CI is published as a + # hardware FAIL and hil-validate.js never retries it. + if board in rows and board not in mine: + mine[board] = rows[board] + if not mine: + results.append({'board': board, 'ran': False, 'pass': False, 'locked': False, + 'detail': 'no report row for this board'}) + continue + # a wedge outranks lock contention: `locked` short-circuits `detail` below, so a + # stale board-locked cell from an earlier attempt used to mask the pool-timeout + # cell the retry added -- publishing a board that hung the rig as LOCKED, which + # hil-validate.js then RE-RUNS, paying another pool guard on it. RUN_ABORTED_CELL + # is written by the same _abort_report path for a board the guard never reached, + # and must outrank it for the same reason. + wedged = any(POOL_TIMEOUT_CELL in cells or RUN_ABORTED_CELL in cells + for cells in mine.values()) + locked = not wedged and any(LOCKED_CELL in cells for cells in mine.values()) + bad = [] + for vname, cells in sorted(mine.items()): + for test, val in sorted(cells.items()): + if test == LOCKED_CELL: + continue + if cell_state(val) == 'fail': + bad.append(f'{vname} {test}: {val}') + ok = not bad and not locked + if locked: + detail = 'held by another holder; not flashed' + elif bad: + detail = '; '.join(bad) + else: + detail = f'{len(mine)} variant(s), {sum(len(c) for c in mine.values())} cell(s) ok' + results.append({'board': board, 'ran': True, 'pass': ok, 'locked': locked, + 'detail': detail}) + # `caveat` too: an abandoned or no-boards run says so THERE, and this JSON is all + # an agent gets -- leaving it in the sidecar puts it back where only a human looks. + return {'results': results, 'banner': report.get('banner', ''), + 'caveat': report.get('caveat', '')} + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument('config_file') + ap.add_argument('-b', '--board', action='append', default=[], + help='boards to report on; default: every board in the config') + ap.add_argument('--report-dir', default='.', help=f'where {REPORT_JSON} lives (default: cwd)') + a = ap.parse_args() + + cfg = json.loads(Path(a.config_file).read_text()) + boards = a.board or [b['name'] for b in cfg.get('boards', [])] + jpath = Path(a.report_dir) / REPORT_JSON + if not jpath.is_file(): + print(f'error: {jpath} not found -- did hil_test.py run in this directory?', + file=sys.stderr) + return 1 + # through _load, like every writer: feeding raw JSON to summarize left the one reader an + # agent's verdict depends on crashing on the malformed sidecars the writers tolerate. + doc, _ = _load(Path(a.report_dir)) + json.dump(summarize(cfg, boards, doc), sys.stdout, indent=2) + print() + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/test/hil/helper/hil_util.py b/test/hil/helper/hil_util.py new file mode 100644 index 000000000..6f84c143d --- /dev/null +++ b/test/hil/helper/hil_util.py @@ -0,0 +1,571 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Bottom layer of the HIL harness: the bounded command runner plus the shared helpers and +# data every other module needs. Stays stdlib-only; its one local dependency is +# tools/rtt.py (the RTT console, loaded by path below) -- everything +# else imports this, including the unit tests on GitHub's bare runner; never import them +# from here. Callers set the module global `verbose`. + +from __future__ import annotations + +import glob +import os +import signal +import subprocess +import unicodedata +import threading +import sys +from pathlib import Path +from typing import Any + + +# ------------------------------------------------------------- +# HIL example test lists, shared by hil_test.py (runner) and ci_select.py (PR-diff +# selector). Run order is shuffled per board (see test_board); every example carries a +# unique hardcoded idProduct (see its usb_descriptors.c). +# ------------------------------------------------------------- + +# device tests +device_tests = [ + 'device/cdc_dual_ports', + 'device/cdc_msc', + 'device/dfu', + 'device/cdc_msc_throughput', + 'device/audio_test_freertos', + 'device/dfu_runtime', + 'device/cdc_msc_freertos', + 'device/hid_boot_interface', + 'device/msc_dual_lun', + 'device/hid_generic_inout', + 'device/printer_to_cdc', + 'device/midi_test', + 'device/mtp', + 'device/usbtest', # cafe:4010, unique PID; runs the Linux testusb tier-4 battery via usbtest.py + # 'device/net_lwip_webserver', # disabled for PR #3605: USB net iface enum is flaky on the CI HIL host +] + +dual_tests = [ + 'dual/host_info_to_device_cdc', +] + +host_test = [ + 'host/cdc_msc_hid', + 'host/msc_file_explorer', + 'host/msc_file_explorer_freertos', + 'host/device_info', +] + +verbose = False + +def pos_int_env(name: str, default: int) -> int: + # One parsing policy for every HIL_* knob: a bare int() crashes every run at import + # on a malformed value, and 0/negative silently removes the bound the knob enforces. + try: + v = int(os.getenv(name, str(default))) + except ValueError: + print(f'warning: {name} is not an integer; using {default}', + file=sys.stderr, flush=True) + return default + if v <= 0: + print(f'warning: {name}={v} is not usable; using {default}', + file=sys.stderr, flush=True) + return default + return v + + +def pos_float_env(name: str, default: float) -> float: + try: + v = float(os.getenv(name, str(default))) + except ValueError: + print(f'warning: {name} is not a number; using {default}', + file=sys.stderr, flush=True) + return default + # float() accepts 'inf'/'nan': an infinite serial timeout is an unbounded read, the + # very thing these knobs exist to prevent, and nan fails every comparison silently + if not (v > 0 and v < float('inf')): + print(f'warning: {name}={v} is not usable; using {default}', + file=sys.stderr, flush=True) + return default + return v + + +CMD_TIMEOUT = pos_int_env('HIL_CMD_TIMEOUT', 180) +# Post-SIGKILL reap, spent ON TOP of a run_cmd timeout whenever the child has to be killed. +# A caller budgeting several bounded steps must add one of these PER STEP, or its own outer +# bound fires mid-step -- for a flasher, orphaning it on the probe. +REAP_GRACE = 10 + +TINYUSB_ROOT = Path(__file__).resolve().parents[3] # test/hil/helper/ -> repo root + + +def display_width(s: str) -> int: + """Terminal COLUMNS, not characters. + + The status marks the reports use -- ✅ ❌ ⚪ ⚠ 🔒 -- are one Python character and TWO + columns wide. Measuring with len() pads every cell containing one a column short, so + the pipes drift out of line against the header rule for the whole table. + """ + return sum(2 if unicodedata.east_asian_width(c) in 'WF' else 1 for c in s) + + +def pad(s: str, width: int, center: bool = False) -> str: + """str.ljust/center, measured in display columns. See display_width.""" + room = max(0, width - display_width(s)) + if not center: + return s + ' ' * room + left = room // 2 + return ' ' * left + s + ' ' * (room - left) + + +def cmd_stdout_text(out: Any) -> str: + if out is None: + return '' + if isinstance(out, bytes): + return out.decode('utf-8', errors='ignore') + return str(out) + + +def _banner_body(out: Any, err: Any) -> str: + # split_stderr callers keep the diagnostic in stderr — a banner of stdout alone + # would be blank exactly when something went wrong + body = cmd_stdout_text(out) + err_text = cmd_stdout_text(err) + if err_text: + body = f'{body}\n{err_text}' if body else err_text + return body + + +# Shared with compact_output's stripper in hil_test: duplicated literals let the two +# layers drift and reintroduce literal marker noise mid-row in the GitHub log. +GROUP_MARK, ENDGROUP_MARK = '::group::', '::endgroup::' + + +def strip_workflow_markers(line: str) -> str: + # run_cmd only ever emits markers at line start; mid-line is not a real case. + return line.removeprefix(GROUP_MARK).removeprefix(ENDGROUP_MARK) + + +def _ci_log_groups() -> bool: + # GitHub folds ::group::/::endgroup:: only at line start of the JOB's real stdout; a + # pool worker's capture is compacted into one row line, where they render literally. + return bool(os.getenv('CI')) and sys.stdout is sys.__stdout__ + + +def _print_banner(title: str, out: Any, err: Any) -> None: + print() + if _ci_log_groups(): + print(f'{GROUP_MARK}{title}') + print(_banner_body(out, err)) + print(ENDGROUP_MARK) + else: + print(title) + print(_banner_body(out, err)) + + +SYSFS_READ_GRACE = 2.0 # default bound on one attribute read; see read_sysfs + +# path -> the kernfs inode the node had when its bounded read gave up. Keyed by INODE, not +# by path alone: a busport does not change when a board returns to the same physical port, +# so a path-only blacklist outlives the wedge -- hil_pool_check resets or reflashes the +# board, wait_device polls that busport for the new inode, and the scan it polls through +# would never look at the device again. A re-enumeration destroys the kernfs node and makes +# a new one, so a CHANGED inode is the all-clear. os.stat is safe on a wedged device: it +# does not call ->show(), so it cannot block on the lock the reader is stuck behind. +_stranded: dict = {} +_strand_hits: dict = {} # path -> how many times it has stranded, ever +_refused: set = set() # paths answered None WITHOUT reading, once past _STRAND_MAX +_strand_lock = threading.Lock() +_ever_stranded = False + +# Each strand costs a thread AND an fd for the life of the process -- on sysfs the open() +# SUCCEEDS and only the read blocks. Two ceilings, because they bound different things: +# +# _PATH_STRAND_MAX -- a device that FLAPS while still wedged re-enumerates, clears the +# inode memo, and strands again. Per path, so one sick board cannot leak without bound. +# After this many it stays memoised whatever its inode says. +# _STRAND_MAX -- a whole-process backstop against RLIMIT_NOFILE or the thread ceiling, +# which would raise inside a worker and lose every board's result. Counted PER PATH, not +# per reader: hil_pool_check runs four poll threads over one bus, and counting each +# reader let four threads on ONE wedged device spend four credits between them. With +# per-path counting a 27-board rig cannot approach this. +_PATH_STRAND_MAX = 4 +_STRAND_MAX = 64 + + +def sysfs_stranded() -> bool: + """True once any bounded read has given up, and it STAYS true. + + A sticky, process-wide fact, so it answers exactly one question: "could anything in + this process's output be the tool losing sight of healthy hardware?" -- which is what + hil_pool_check's footer needs. It canNOT answer "is THIS device unreadable" for a + caller deciding what a single missing device means; use path_stranded() for that. + """ + return _ever_stranded + + +def strand_note() -> str: + """Suffix for an absence claim, so "not found" never reads as proven absence. + + Lives here because every caller that can say "not found" needs the same sentence, and + the one that had to re-invent it got missed: a wedged-but-enumerated printer was + reported as an enumeration failure, sending a maintainer after firmware. + """ + return (' (a bounded sysfs read gave up, so "not found" here means "could not tell"' + ' -- see the usb-kernel-recover skill)') if sysfs_stranded() else '' + + +def path_stranded(path: str) -> bool: + """Whether THIS attribute is currently memoised as unreadable. + + The per-device question sysfs_stranded() cannot answer. usbtest uses it to tell a DUT + whose `serial` is held under device_lock from one that genuinely left the bus, because + the difference decides whether it performs driver-registry writes that take the + UNINTERRUPTIBLE device_lock. + """ + with _strand_lock: + return path in _stranded or path in _refused + + +def read_sysfs(path: str, timeout: float = SYSFS_READ_GRACE) -> str | None: + """A sysfs attribute's value, or None when it did not answer. + + BOUNDED BY DEFAULT, and it has to be. `serial` is served by usb_string_attr, which + takes usb_lock_device_interruptible (v6.12.96 sysfs.c:141-143) -- the same lock a + wedged usbfs ioctl holds. Every OTHER attribute the harness reads (idVendor, idProduct, + bcdDevice, busnum, devnum, speed) is a lock-free sysfs_emit from a cached field and + cannot block. + + "Only the wedged board's own worker pays" is FALSE, which is why the bound is not + opt-in: usb_scan reads `serial` on every device matching the VID to find the one it + wants, so resolving MY board touches every peer's locked attribute. hil_lock's + controller_of does that from controller_permit, on essentially every board -- one + wedged DUT would stall every worker, not one. hil_pool_check has no guard at all. + + A give-up reads as None, the same as unreadable: there is no third value and no + per-attribute blindness. The memo is keyed by inode so the cost stays on the device + that is actually wedged; path_stranded() tells a caller which device that was. + """ + with _strand_lock: + was = _stranded.get(path) + stuck_for_good = _strand_hits.get(path, 0) >= _PATH_STRAND_MAX + budget_spent = len(_stranded) >= _STRAND_MAX + if was is not None: + try: + if os.stat(path).st_ino == was: + return None # same kernfs node, still wedged + except OSError: + pass # gone: let the read below report it + if stuck_for_good: + return None # flapped too many times; see _PATH_STRAND_MAX + with _strand_lock: + _stranded.pop(path, None) # a different inode is the all-clear + elif budget_spent: + # see _STRAND_MAX. Recorded, not just returned: usbtest fails CLOSED on + # path_stranded() before the lock-taking cleanup, and a path we declined to read + # is exactly the case it must not be told is readable-and-absent. + with _strand_lock: + _refused.add(path) + return None + + # BEFORE the read, not after: a node that re-enumerates DURING the grace would + # otherwise have its brand-new HEALTHY inode recorded as the wedged one, and only a + # second re-enumeration could ever clear it. If it cannot be stat'd there is no key to + # memoise against, so the path is simply re-read next time -- the open fails fast. + try: + ino = os.stat(path).st_ino + except OSError: + ino = None + out: dict = {} + + def _read(): + try: + with open(path) as f: + out['v'] = f.read().strip() + except (OSError, ValueError): + pass + + t = threading.Thread(target=_read, daemon=True) + t.start() + t.join(timeout) + # `out` FIRST: a reader can deposit its value and still be alive for a moment + # afterwards, and counting that as a strand blacklists a healthy attribute forever + if 'v' in out: + # a path that answered is not refused any more: _refused feeds path_stranded(), + # and a stale entry makes usbtest read a LATER genuine disconnect as "cannot tell" + with _strand_lock: + _refused.discard(path) + if t.is_alive() and 'v' not in out: + global _ever_stranded + announce = False + if ino is None: + # the pre-read stat lost a race the open then won -- the node was replaced + # between them. Re-stat now: the reader is blocked on whatever node exists, + # so this is the key it is stuck on. Without a key nothing is memoised and + # every later poll starts another permanent thread and fd for this path. + try: + ino = os.stat(path).st_ino + except OSError: + pass + with _strand_lock: + _ever_stranded = True + if ino is not None: + first = path not in _stranded # count the PATH once, not each reader + _stranded[path] = ino + if first: + _strand_hits[path] = _strand_hits.get(path, 0) + 1 + announce = len(_stranded) == _STRAND_MAX + else: + _refused.add(path) # unkeyable: at least do not vouch for it + if announce: + print(f'warning: {_STRAND_MAX} devices have unreadable sysfs attributes; ' + f'refusing to start more bounded readers, so later reads answer None ' + f'without looking. Find the wedged device (usb-kernel-recover skill).', + file=sys.stderr, flush=True) + return None + return out.get('v') + + +def usb_scan(vid_pid=None, serial=None, vid=None, timeout=SYSFS_READ_GRACE) -> list: + """Enumerated USB devices matching the filters: [{busport, dir, vid, pid, serial}]. + + Three rules, one implementation for every caller: + + * Root hubs excluded (glob `*-*`): no DUT is one, and scans including them measured + seconds slower (observation, no mechanism -- the "autosuspend wake" explanation was + wrong; usb_string_attr reads a cached string, sysfs.c:141-143). + * idVendor/idProduct first: lock-free `sysfs_emit` from udev->descriptor + (sysfs.c:688-705), so they rule out nearly every device for free. + * `serial` LAST and BOUNDED: it is the only attribute here served under the device + lock, so it is the only one that can block. Filtering on the lock-free pair first + keeps most devices out of it, but a scan for ONE board still reads the serial of + every peer that shares its VID -- so the bound is what stops one wedged DUT from + stalling every caller (see read_sysfs). + """ + out = [] + for d in glob.glob('/sys/bus/usb/devices/*-*'): + # `in`, not endswith: an interface is '<busport>:<cfg>.<ifnum>' (2-4:1.0), which + # CONTAINS the colon rather than ending with it. Screening them out here is worth + # real time -- they were 31 of 44 matches on this rig. + if ':' in os.path.basename(d): + continue + try: + with open(os.path.join(d, 'idVendor')) as f: + dev_vid = f.read().strip() + with open(os.path.join(d, 'idProduct')) as f: + dev_pid = f.read().strip() + except OSError: + continue # vanished mid-walk, or not a device dir: a fact, not unknown + if vid_pid is not None and (dev_vid, dev_pid) != tuple(vid_pid): + continue # ruled out for free, without touching the locked attribute + if vid is not None and dev_vid != vid: + continue # same, for callers that know the VID but not the PID + sn = read_sysfs(os.path.join(d, 'serial'), timeout) + if sn is None: + continue # no serial attribute + if serial is not None and sn.lower() != serial.lower(): + continue + out.append({'busport': os.path.basename(d), 'dir': d, + 'vid': dev_vid, 'pid': dev_pid, 'serial': sn}) + return out + + +def _close_pipes(p: subprocess.Popen) -> None: + """Close OUR ends of an abandoned child's pipes. Never raises.""" + for pipe in (p.stdout, p.stderr, p.stdin): + try: + if pipe is not None: + pipe.close() + except OSError: + pass + + +def run_alongside(argv: list, work, timeout: int) -> subprocess.CompletedProcess: + """Run `argv` alongside `work()`, which runs in THIS thread, then reap it -- bounded. + + The read-while-we-write shape run_cmd cannot express: the caller needs the child + RUNNING while it does something else. Everything else about the contract is run_cmd's + -- own session, killpg, bounded reap, our pipe ends closed, rc 124 on the kill. + + A PROCESS, not a thread: an abandoned thread keeps the fd, and usblp_open returns + -EBUSY while usblp->used (v6.12.96 usblp.c), so every later open in this long-lived + worker would read as a wedged device. A killed process takes its fd with it. + + stdout is captured as BYTES and kept CLEAN -- a caller byte-compares it against the + payload it sent, so a single stderr byte (a PYTHONWARNINGS chirp, a sitecustomize + print, a .pth deprecation from a venv) would read as USB data corruption. stderr gets + its own pipe; communicate() drains both, so the split cannot deadlock. + `work` runs even if the child dies immediately -- the caller's own asserts decide. + """ + p = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + start_new_session=True) + + def _reap() -> subprocess.CompletedProcess: + try: + out, err = p.communicate(timeout=timeout) + return subprocess.CompletedProcess(argv, p.returncode, out, err) + except subprocess.TimeoutExpired: + try: + os.killpg(p.pid, signal.SIGKILL) + except OSError: + p.kill() + try: + out, err = p.communicate(timeout=REAP_GRACE) + except subprocess.TimeoutExpired: + # Outlasted SIGKILL: uninterruptible, still holding whatever it opened. + # Abandoned like any other stray -- but as a real child in its own + # session, so the containment sweep FINDS it (child_procs walks the ppid + # tree) and the report names it. That is the whole difference from a + # blocked thread, which no sweep can see and no signal can reach. + out, err = b'', b'' + _close_pipes(p) # our own fds must not leak either + return subprocess.CompletedProcess(argv, 124, out, err) + + try: + work() + except BaseException: + # Reap first so the child never outlives us, then let the caller's error through. + # A `return` inside a `finally` would SWALLOW it -- an assert in `work` would + # vanish and the caller would compare data it never finished sending. + _reap() + raise + return _reap() + + +# The RTT console implementation lives in tools/rtt.py (importable classes + CLI, +# stdlib-only, harness-critical — see its module docstring). Loaded by file path so +# no sys.path entry for tools/ can shadow other imports; re-exported here so the +# harness keeps addressing hil_util.JlinkRtt. +import importlib.util as _ilu + +_rtt_path = TINYUSB_ROOT / 'tools' / 'rtt.py' +if not _rtt_path.exists(): + # name the real cause: a bare FileNotFoundError out of an exec_module here reads + # as a harness bug, when the actual problem is an incompletely staged tree + raise ImportError(f'{_rtt_path} is missing — the RTT console lives there and the ' + f'harness depends on it; stage it alongside test/hil (hil_ci.sh does)') +_rtt_spec = _ilu.spec_from_file_location('tinyusb_tools_rtt', _rtt_path) +_rtt = _ilu.module_from_spec(_rtt_spec) +sys.modules[_rtt_spec.name] = _rtt # registered: RttError must be picklable across the fork Pool +_rtt_spec.loader.exec_module(_rtt) +JlinkRtt = _rtt.JlinkRtt +OpenocdRtt = _rtt.OpenocdRtt +RttError = _rtt.RttError +RTT_BANNER_RE = _rtt.RTT_BANNER_RE +strip_banner = _rtt.strip_banner + + +def _cmd_label(cmd) -> str: + """A one-line name for a banner. An argv whose payload is a `python3 -c` program would + otherwise dump the whole body into the CI log, where run_cmd's banners are already the + noisiest thing in a failing row.""" + if isinstance(cmd, str): + return cmd + parts = [a if len(a) <= 60 else f'<{len(a)}-char program>' for a in cmd] + return ' '.join(parts) + + +def run_cmd(cmd: str | list, cwd: str | None = None, timeout: int | None = None, + binary: bool = False, split_stderr: bool = False, + quiet: bool = False) -> subprocess.CompletedProcess: + """Bounded subprocess: own session, killpg on expiry, rc 124 when it had to be killed. + + `cmd` is a shell STRING or an argv LIST. argv exists for a program that cannot survive + a trip through the shell -- a multi-line `python3 -c` body -- which is how the harness + runs a library call that no in-process bound can contain. A daemon thread cannot bound + a C call that holds the GIL, so for those the child process IS the bound. + """ + if timeout is None: + timeout = CMD_TIMEOUT + # binary: raw bytes (text mode's errors='replace' mangles non-UTF-8 file content). + # split_stderr: keep stderr out of stdout, for callers that parse stdout. quiet: no + # COMMAND FAILED banner, for retry loops that report failures themselves (timeouts + # still print: a killed child is always noteworthy). + popen_kwargs = { + 'cwd': cwd, + # a list goes straight to execve; only a string needs a shell to parse it + 'shell': isinstance(cmd, str), + 'stdout': subprocess.PIPE, + 'stderr': subprocess.PIPE if split_stderr else subprocess.STDOUT, + } + if not binary: + popen_kwargs.update({'text': True, 'encoding': 'utf-8', 'errors': 'replace'}) + # C-level setsid, same process-group semantics as preexec_fn=os.setsid but safe when + # called from threads (pool_check runs flashes from a thread pool) + popen_kwargs['start_new_session'] = True + + p = subprocess.Popen(cmd, **popen_kwargs) + try: + out, err = p.communicate(timeout=timeout) + r = subprocess.CompletedProcess(args=cmd, returncode=p.returncode, stdout=out, stderr=err) + except subprocess.TimeoutExpired as ex: + try: + os.killpg(p.pid, signal.SIGKILL) + except OSError: + # ProcessLookupError: already gone. PermissionError: an all-root group refuses + # the group kill -- letting either escape would skip the bounded reap, the pipe + # close and the rc-124 return this handler exists for. + pass + try: + out, err = p.communicate(timeout=REAP_GRACE) + except subprocess.TimeoutExpired: + # Something in the group outlived SIGKILL: D state (truly unkillable), or + # root-owned because sudo FORKS rather than execs, so the wrapper dies and its + # root child does not. Abandon it and let the report name it; the harness never + # sudo-kills its way out. Our ends of its pipes must not leak, though: a pool + # worker lives for the whole run, so every wedged command would cost it two fds. + out, err = None, None + _close_pipes(p) + # prefer the post-kill buffers (supersets of the exception's), falling back to ex.* + # when the child was unkillable. TimeoutExpired carries BYTES even for a text-mode + # Popen, so the fallbacks must be decoded or a text-mode caller gets bytes exactly + # when the child wedged in D state. + def _typed(v): + if not binary and isinstance(v, bytes): + return v.decode('utf-8', errors='replace') + return v + + timeout_out = _typed(out or ex.stdout) or (b'' if binary else '') + # ...and never None: with split_stderr the SUCCESS path always yields a str/bytes, + # so a caller that does `r.stderr.strip()` works everywhere except the timeout -- + # the one path it was written for. Without split_stderr stderr stays None, as on + # the success path (it was merged into stdout). + timeout_err = _typed(err if err is not None else ex.stderr) + if split_stderr and timeout_err is None: + timeout_err = b'' if binary else '' + _print_banner(f'COMMAND TIMEOUT ({timeout}s): {_cmd_label(cmd)}', timeout_out, timeout_err) + return subprocess.CompletedProcess(args=cmd, returncode=124, stdout=timeout_out, stderr=timeout_err) + except BaseException: + # BaseException, not Exception (as in CPython's own subprocess.run): + # KeyboardInterrupt is the case that matters, and start_new_session put the child in + # its OWN group, so it never got the terminal's SIGINT -- without this, Ctrl-C + # leaves the flasher or testusb holding the probe and its usbfs node. Kill and + # close, never wait: this path must not add a hang of its own. + try: + os.killpg(p.pid, signal.SIGKILL) + except OSError: + pass + _close_pipes(p) + raise + + if r.returncode != 0 and not quiet: + _print_banner(f'COMMAND FAILED: {_cmd_label(cmd)}', r.stdout, r.stderr) + elif verbose: + print(cmd) + print(cmd_stdout_text(r.stdout)) + return r + + +# get usb serial by id +def get_serial_dev(id, vendor_str, product_str, ifnum): + if vendor_str and product_str: + # known vendor and product + vendor_str = vendor_str.replace(' ', '_') + product_str = product_str.replace(' ', '_') + return f'/dev/serial/by-id/usb-{vendor_str}_{product_str}_{id}-if{ifnum:02d}' + else: + # just use id: mostly for cp210x/ftdi flasher + pattern = f'/dev/serial/by-id/usb-*_{id}-if*' + port_list = glob.glob(pattern) + if len(port_list) == 0: + raise RuntimeError(f'No serial device found for {pattern}') + return port_list[0] diff --git a/test/hil/hfp.json b/test/hil/hfp.json index 735d5a402..2babcaaf3 100644 --- a/test/hil/hfp.json +++ b/test/hil/hfp.json @@ -7,9 +7,8 @@ "device": true, "host": false, "dual": false }, "flasher": { - "name": "jlink", - "uid": "774470029", - "args": "-device STM32L412KB" + "name": "stlink", + "uid": "0673FF575051717867034946" } }, { diff --git a/test/hil/hil_ci.sh b/test/hil/hil_ci.sh index ef93bcb49..43ede5795 100644 --- a/test/hil/hil_ci.sh +++ b/test/hil/hil_ci.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash # Run HIL test remotely on ci.lan -# Usage: test/hil/hil_ci.sh [-b BOARD] [-t TEST] [extra hil_test.py args...] +# Usage: test/hil/hil_ci.sh [-b BOARD]... [-t TEST] [extra hil_test.py args...] # Example: # test/hil/hil_ci.sh -b stm32f723disco +# test/hil/hil_ci.sh -b stm32f723disco -b raspberry_pi_pico # test/hil/hil_ci.sh -b stm32f723disco -t host/cdc_msc_hid -r 1 # # Env overrides: REMOTE, REMOTE_DIR, CONFIG (path to HIL config json), @@ -20,17 +21,70 @@ CONFIG=${CONFIG:-$ROOT_DIR/test/hil/tinyusb.json} exit 1 } -# Parse -b BOARD from arguments to know which build to copy -BOARD="" +# REMOTE_DIR reaches the rig as `rm -rf` input, an scp remote path and an rsync remote +# path -- the remote shell re-splits and expands all three, so no amount of LOCAL quoting +# protects them (and %q would escape the ~ that REMOTE_DIR=~/dir needs). Screen it once. +# The tilde is the whole hazard: the REMOTE shell expands it, so `~/` alone -- one typo +# away from the documented ~/dir override -- means `rm -rf` on that account's HOME. Hence +# `/` or `~/` followed by at least one named component, ending in a name character. +[[ $REMOTE_DIR =~ ^(/|~/)[A-Za-z0-9_.~/-]*[A-Za-z0-9_-]$ && $REMOTE_DIR != *..* + && $REMOTE_DIR != *//* ]] || { + echo "error: REMOTE_DIR must be /path or ~/path of [A-Za-z0-9_.~/-], no '..', no" \ + "trailing slash -- it is an rm -rf target on $REMOTE: $REMOTE_DIR" >&2 + exit 1 +} + +# --build would run tools/build.py ON THE RIG, and this script stages binaries, not the +# build tree -- it is not copied, so the run dies there with a confusing missing-file +# error. Building is the local half of this workflow by design. +for a in "$@"; do + [ "$a" = "--build" ] || continue + echo "error: --build builds on the REMOTE, but this script copies prebuilt binaries" >&2 + echo " (tools/build.py is not staged). Build locally first, then re-run:" >&2 + echo " cd examples && cmake --preset <board> && cmake --build --preset <board>" >&2 + exit 1 +done + +# Parse -b BOARD from arguments to know which builds to copy. Repeatable: hil_test.py +# takes the whole board set in ONE run (it schedules them across host controllers and +# budgets the flashes itself), so every -b needs its binaries staged, not just the last. +BOARDS=() ARGS=() while [[ $# -gt 0 ]]; do case "$1" in - -b) - [[ $# -ge 2 ]] || { echo "error: -b requires a BOARD argument" >&2; exit 1; } - BOARD="$2" + # hil_test.py declares `-b, --board` with action='append', so argparse also accepts + # --board=X and -bX. Recognising only the bare `-b X` forwarded the others to the rig + # while never staging them: the board ran with no firmware and reported a green row. + -b|--board) + [[ $# -ge 2 ]] || { echo "error: $1 requires a BOARD argument" >&2; exit 1; } + BOARDS+=("$2") + ARGS+=("$1" "$2") + shift 2 + ;; + --board=*) + BOARDS+=("${1#--board=}") + ARGS+=("$1") + shift + ;; + # -bt (--board-test) BEFORE the glued -b?* arm, mirroring argparse's longest-match: it is + # the form <config>.failed uses, and a bare -b?* would register a board named "t..." that + # the roster check below rejects -- killing every documented retry. + -bt|--board-test) + [[ $# -ge 2 ]] || { echo "error: $1 requires NAME:tests" >&2; exit 1; } ARGS+=("$1" "$2") shift 2 ;; + -bt?*|--board-test=*) + ARGS+=("$1") + shift + ;; + # glued short form: argparse resolves -bNAME to --board NAME, so staging must too -- + # unparsed it fell through to the all-boards branch and silently staged everything built + -b?*) + BOARDS+=("${1#-b}") + ARGS+=("$1") + shift + ;; *) ARGS+=("$1") shift @@ -38,29 +92,205 @@ while [[ $# -gt 0 ]]; do esac done -# Setup remote directory. Use `bash -s` + heredoc so REMOTE_DIR (user-overridable) -# is passed as a positional parameter and never reinterpreted by the remote shell. +# Resolve a board to its build dirs: its own dir, the cmake-build-<board>-* glob (ad-hoc +# local builds), and the variant dirs named in $CONFIG -- variant names are NOT required to +# be prefixed with the board name, so the glob alone is not enough. Prints one dir per line. +variant_names() { + python3 -c ' +import json, sys +cfg = json.load(open(sys.argv[1])) +for b in cfg.get("boards", []): + if b["name"] == sys.argv[2]: + for v in b.get("variant") or []: + print(v["name"]) +' "$CONFIG" "$1" +} + +resolve_build_dirs() { + local board="$1" d v + declare -A seen=() + shopt -s nullglob + for d in "$ROOT_DIR"/examples/cmake-build-"$board" "$ROOT_DIR"/examples/cmake-build-"$board"-*; do + [[ -d $d && -z ${seen[$d]:-} ]] && { seen[$d]=1; printf '%s\n' "$d"; } + done + shopt -u nullglob + # to a file, not a process substitution: `set -e`/pipefail cannot see the exit status of + # the latter, so a malformed roster silently yielded zero variant dirs + local vf; vf=$(mktemp) + variant_names "$board" > "$vf" || { rm -f "$vf"; echo "Error: could not read variants for $board from $CONFIG" >&2; exit 1; } + while IFS= read -r v; do + d="$ROOT_DIR/examples/cmake-build-$v" + [[ -d $d && -z ${seen[$d]:-} ]] && { seen[$d]=1; printf '%s\n' "$d"; } + done < "$vf" + rm -f "$vf" +} + +# Pre-flight: EVERY board must resolve to at least one build dir before anything is wiped or +# copied. This check used to live in the copy loop, so an unbuilt board late in the list +# aborted the run after the remote tree had been rm -rf'd and earlier boards fully rsynced -- +# zero coverage, a half-staged rig, and a stale local hil_report.md left in place. Report all +# missing boards at once so one build round fixes them. +MANIFEST=$(mktemp) +trap 'rm -f "$MANIFEST"' EXIT +# Roster membership first: hil_test.py rejects an unknown -b with sys.exit(1) for the WHOLE +# run (hil_test.py:2297), and it does so AFTER this script has wiped REMOTE_DIR and staged +# every board -- one typo then costs the entire batch. We already parse $CONFIG here, so +# catch it before anything is touched. Note -b matches board names only, never variant names. +if [ ${#BOARDS[@]} -gt 0 ]; then +ROSTER=$(python3 -c ' +import json, sys +print("\n".join(b["name"] for b in json.load(open(sys.argv[1])).get("boards", []))) +' "$CONFIG") || { echo "error: could not read the board roster from $CONFIG" >&2; exit 1; } +notinroster=() +for b in ${BOARDS[@]+"${BOARDS[@]}"}; do + grep -qxF -- "$b" <<< "$ROSTER" || notinroster+=("$b") +done +if [ ${#notinroster[@]} -gt 0 ]; then + echo "error: not in $(basename "$CONFIG"): ${notinroster[*]}" >&2 + echo " (-b takes board names, not variant names)" >&2 + exit 1 +fi +fi # BOARDS non-empty: nothing to validate for an all-boards run + +missing=() +for b in ${BOARDS[@]+"${BOARDS[@]}"}; do + dirs=$(resolve_build_dirs "$b") + if [ -z "$dirs" ]; then + missing+=("$b") + else + while IFS= read -r d; do printf '%s\t%s\n' "$b" "$d" >> "$MANIFEST"; done <<< "$dirs" + # A declared variant with no build dir is NOT an error -- no cmake preset is + # variant-suffixed, so this is the normal state for e.g. the -DMA variants. It is worth + # saying out loud: hil_test.py logs `Skip (no binary)` and counts zero errors for it, so + # the run exits 0 and the operator reads a green table for cells that never ran. + # plain assignment, not process substitution: set -e sees a variant_names failure here, + # the same trap the comment in resolve_build_dirs warns about + vnames=$(variant_names "$b") + while IFS= read -r v; do + [ -z "$v" ] && continue + # whole lines: a substring match lets cmake-build-<v>-DMA silence the warning for <v> + grep -qxF -- "$ROOT_DIR/examples/cmake-build-$v" <<< "$dirs" \ + || echo "warning: $b variant '$v' has no build dir -- its cells will be skipped, not tested" >&2 + done <<< "$vnames" + fi +done +if [ ${#missing[@]} -gt 0 ]; then + echo "Error: no build directory under $ROOT_DIR/examples/ for: ${missing[*]}" >&2 + for b in "${missing[@]}"; do + echo " cd examples && cmake --preset $b && cmake --build --preset $b" >&2 + done + exit 1 +fi + +# The all-boards form needs its emptiness check HERE too: below the setup ssh it fired after +# the remote tree was already rm -rf'd, destroying the previous run's report and re-run spec +# on the rig before deciding there was nothing to do. +if [ ${#BOARDS[@]} -eq 0 ]; then + shopt -s nullglob + allbuilds=("$ROOT_DIR"/examples/cmake-build-*/) + shopt -u nullglob + if [ ${#allbuilds[@]} -eq 0 ]; then + echo "error: no examples/cmake-build-* directories under $ROOT_DIR -- nothing to test" >&2 + echo " build first, e.g.: cd examples && cmake --preset <board> && cmake --build --preset <board>" >&2 + exit 1 + fi +fi + +# Setup remote directory. `bash -s` + heredoc so REMOTE_DIR arrives as a positional +# parameter, keeping the `rm -rf` target out of the command string the heredoc runs. echo "==> Setting up remote $REMOTE:$REMOTE_DIR" ssh "$REMOTE" bash -s -- "$REMOTE_DIR" <<'REMOTE' set -e +# Second gate, on the side that knows what ~ expanded to: only here is $HOME a value +# rather than a guess, and this is the line that actually runs rm -rf. +case "$1" in + ''|/|"$HOME"|"$HOME"/) echo "refusing to rm -rf '$1'" >&2; exit 1 ;; +esac rm -rf -- "$1" -# .claude path: usbtest.py's HUNG recovery resolves usb_recover.sh relative to the -# staged repo root — without it, recovery ENOENTs and the wedge is left in place -mkdir -p -- "$1/test/hil" "$1/examples" "$1/.claude/skills/usb-kernel-recover/scripts" +mkdir -p -- "$1/test/hil/helper" "$1/examples" REMOTE +# The --accumulate merge base. The wipe above just cleared REMOTE_DIR, and +# accumulate_report merges onto the sidecar in the RUN's cwd (hil_test.py:2193 sets +# `fresh = not args.accumulate`, and only a non-fresh run reads it) -- so without this a +# remote retry starts from nothing and its one-row table REPLACES the full-fleet one it was +# meant to extend. The copy-back at the end of this script has always existed; this is the +# other half of it. +# +# Gated, not unconditional: a fresh run unlinks the sidecar anyway (hil_test.py:2244), so +# uploading there is wasted work that also obscures what the wipe means. +# +# <config>.failed is deliberately NOT uploaded: hil_test.py only ever writes it, never +# reads it -- the retry spec reaches the rig as the -b/-bt arguments the caller expanded +# from it (`hil_ci.sh $(cat <config>.failed)`). +# argparse decides, not a case arm: hil_test.py declares `-a, --accumulate`, so argparse +# also accepts `-av`, `-va`, `--accum` and `--acc` -- and hil-validate.js tells the +# operator to retry "adding -v", which makes `-av` the natural spelling. A hand-rolled +# match missed all four: no upload, and the else-branch warning never fired either, so the +# one-row table replaced the full-fleet one in silence. +ACCUMULATE=$(python3 - ${ARGS[@]+"${ARGS[@]}"} <<'PY' +import argparse, sys +p = argparse.ArgumentParser(add_help=False) +p.add_argument('-a', '--accumulate', action='store_true') +p.add_argument('-v', '--verbose', action='store_true') # so -av/-va bundle as they do there +print(1 if p.parse_known_args(sys.argv[1:])[0].accumulate else 0) +PY +) || ACCUMULATE=0 +if [ "$ACCUMULATE" = 1 ]; then + if [ -f "$ROOT_DIR/hil_report.json" ]; then + # Provenance: hil_report.json is not namespaced by CONFIG or REMOTE (build.yml and + # pr_comment.yml read that exact name), so a `REMOTE=hifiphile CONFIG=.../hfp.json` + # run leaves an hfp sidecar behind that a later ci.lan retry would merge, publishing + # boards that never ran here. Require at least one row to belong to THIS roster. + if python3 - "$ROOT_DIR/hil_report.json" "$CONFIG" <<'PY' +import json, sys +try: + rows = json.load(open(sys.argv[1])).get('rows') or [] + cfg = json.load(open(sys.argv[2])).get('boards') or [] +except Exception: + sys.exit(1) +known = set() +for b in cfg: + known.add(b.get('name')) + known.update(v.get('name') for v in (b.get('variant') or [])) +sys.exit(0 if not rows or any(r.get('board') in known for r in rows if isinstance(r, dict)) + else 1) +PY + then + echo "==> Uploading hil_report.json as the --accumulate merge base" + scp -q "$ROOT_DIR/hil_report.json" "$REMOTE:$REMOTE_DIR/" + else + echo "==> warning: $ROOT_DIR/hil_report.json holds no board from $(basename "$CONFIG")" \ + "-- it is from another rig or config, so it is NOT being uploaded; this run's" \ + "table will REPLACE rather than extend" >&2 + fi + else + # Loud, because this is the failure mode: the run still succeeds, and quietly + # publishes a small table where a full one used to be. + echo "==> warning: --accumulate was requested but $ROOT_DIR/hil_report.json does not" \ + "exist, so there is nothing to merge onto -- this run's table will REPLACE the" \ + "previous one rather than extend it" >&2 + fi +fi + # Copy HIL test script and config echo "==> Copying test scripts" scp -q "$ROOT_DIR/test/hil/hil_test.py" \ "$ROOT_DIR/test/hil/hil_flash.py" \ - "$ROOT_DIR/test/hil/hil_lock.py" \ "$ROOT_DIR/test/hil/usbtest.py" \ - "$ROOT_DIR/test/hil/hil_examples.py" \ "$ROOT_DIR/test/hil/pymtp.py" \ + "$ROOT_DIR/test/hil/mtp_test.py" \ "$CONFIG" \ "$REMOTE:$REMOTE_DIR/test/hil/" -scp -q "$ROOT_DIR/.claude/skills/usb-kernel-recover/scripts/usb_recover.sh" \ - "$REMOTE:$REMOTE_DIR/.claude/skills/usb-kernel-recover/scripts/" +scp -q "$ROOT_DIR/test/hil/helper/__init__.py" \ + "$ROOT_DIR/test/hil/helper/hil_util.py" \ + "$ROOT_DIR/test/hil/helper/hil_health.py" \ + "$ROOT_DIR/test/hil/helper/hil_lock.py" \ + "$ROOT_DIR/test/hil/helper/hil_report.py" \ + "$REMOTE:$REMOTE_DIR/test/hil/helper/" +# the rtt console/capture tool (rtt skill), harness-critical: hil_util imports it +ssh "$REMOTE" mkdir -p "$REMOTE_DIR/tools" +scp -q "$ROOT_DIR/tools/rtt.py" "$REMOTE:$REMOTE_DIR/tools/" # Copy only firmware binaries (elf/bin/hex) plus esptool metadata # (config.env + flash_args needed by the esptool flasher), preserving structure @@ -73,43 +303,23 @@ copy_board_binaries() { "$src" "$REMOTE:$REMOTE_DIR/examples/" } -if [ -n "$BOARD" ]; then - # Copy the board's build dir plus its variant dirs. Variant names come from - # $CONFIG (they are not required to be prefixed with the board name); the - # cmake-build-<BOARD>-* glob is kept as a fallback for ad-hoc local builds. - # Collect only dirs that actually exist, deduplicated. - declare -A SEEN_DIRS=() - BUILD_DIRS=() - add_build_dir() { - [[ -d "$1" && -z "${SEEN_DIRS[$1]:-}" ]] || return 0 - SEEN_DIRS[$1]=1 - BUILD_DIRS+=("$1") - } - shopt -s nullglob - for d in "$ROOT_DIR"/examples/cmake-build-"$BOARD" "$ROOT_DIR"/examples/cmake-build-"$BOARD"-*; do - add_build_dir "$d" - done - shopt -u nullglob - while IFS= read -r v; do - add_build_dir "$ROOT_DIR/examples/cmake-build-$v" - done < <(python3 -c ' -import json, sys -cfg = json.load(open(sys.argv[1])) -for b in cfg.get("boards", []): - if b["name"] == sys.argv[2]: - for v in b.get("variant") or []: - print(v["name"]) -' "$CONFIG" "$BOARD") - if [ ${#BUILD_DIRS[@]} -eq 0 ]; then - echo "Error: no build directory found for $BOARD under $ROOT_DIR/examples/" - echo "Build first with: cd examples && cmake --preset $BOARD && cmake --build --preset $BOARD" - exit 1 - fi - echo "==> Copying binaries for $BOARD (${#BUILD_DIRS[@]} build dir(s))" - for d in "${BUILD_DIRS[@]}"; do - copy_board_binaries "$d" +if [ ${#BOARDS[@]} -gt 0 ]; then + # Replay the pre-flight manifest: the dirs were already resolved and proved non-empty + # for every board, so nothing here can abort mid-staging. Plain reads of the manifest -- + # a process substitution would hide a reader failure from set -e (the comment in + # resolve_build_dirs is about exactly that trap). + for b in "${BOARDS[@]}"; do + dirs=() + while IFS=$'\t' read -r bb d; do + [ "$bb" = "$b" ] && [ -n "$d" ] && dirs+=("$d") + done < "$MANIFEST" + echo "==> Copying binaries for $b (${#dirs[@]} build dir(s))" + for d in ${dirs[@]+"${dirs[@]}"}; do + copy_board_binaries "$d" + done done else + # emptiness was already refused in pre-flight, before the remote wipe echo "==> Copying all built binaries" # Use `%/` parameter expansion to strip the trailing slash from the glob — # rsync needs the bare dir name so the per-board cmake-build-<BOARD>/ subdir @@ -119,14 +329,50 @@ else done fi -# Run test. Use `bash -s` so REMOTE_DIR + ARGS reach the remote shell as positional -# parameters; quoting and metacharacters in args are preserved. -CONFIG_BASENAME="$(basename "$CONFIG")" +# Run test via `bash -s`, so REMOTE_DIR and the args arrive as positional parameters. +# %q the ARGS -- ssh joins its argv into ONE string that the remote shell re-splits, so +# `-t 'host/cdc msc'` would arrive as two arguments and hil_test.py would see a stray +# word where it expects the config path. REMOTE_DIR is deliberately NOT quoted here: it +# is screened above precisely so it can keep its ~ expansion. +ARGS_Q=() +for a in ${ARGS[@]+"${ARGS[@]}"}; do ARGS_Q+=("$(printf '%q' "$a")"); done +# same re-split, same fix: CONFIG is a user-supplied path and its basename lands in the +# command string too +CONFIG_Q="$(printf '%q' "test/hil/$(basename "$CONFIG")")" echo "==> Running HIL test on $REMOTE" rc=0 -ssh "$REMOTE" bash -s -- "$REMOTE_DIR" "${ARGS[@]}" "test/hil/$CONFIG_BASENAME" <<'REMOTE' || rc=$? +# --retry 1 FIRST, before the user's args: this targets the same shared rig CI uses, and the +# pool guard is a flat constant that does not scale with max_retry, so a few flaky boards can +# re-pay ~510s each until the 3600s guard fires, abandoning the pool and holding board flocks +# against concurrent CI. hil_test.py's own default is already 1; passing it explicitly keeps +# that true if the default ever moves. Placed first, not appended, so argparse's last-wins +# means `hil_ci.sh -r 3` still gets 3. +# Forward the HIL_* knobs (HIL_NO_BOARD_LOCK for an authorized force, the parallel widths, +# HIL_POOL_TIMEOUT). ssh passes no environment and joins its argv into one string the remote +# shell re-splits, so a bare NAME=value element would arrive as a positional argument to +# hil_test.py and argparse would exit 2. Build `export` lines instead and hand them over as a +# single %q-quoted word for the remote to eval. +# Joined with '; ', NOT newlines: %q renders a newline as bash-only $'...' quoting, which the +# remote LOGIN shell must parse from the joined command string -- under dash the force arrives +# as garbage and silently does nothing. Backslash escaping round-trips in both shells. +# HIL_REPORT_DIR stays local: where the report lands on the rig is this script's contract +# (REMOTE_DIR, where all three copy-backs below look), so forwarding it would relocate the +# report and every copy-back would come home empty. +HIL_EXPORTS="" +while IFS= read -r v; do + [ -z "$v" ] && continue + HIL_EXPORTS+="export $(printf '%s=%q' "$v" "${!v}"); " +done < <(compgen -v | grep -x 'HIL_[A-Z0-9_]*' | grep -vxE 'HIL_EXPORTS|HIL_REPORT_DIR' || true) +[ -n "$HIL_EXPORTS" ] && echo "==> Forwarding: $HIL_EXPORTS" +# One %q-quoted word, so ssh's argv join and the remote shell's re-split hand it back +# byte-for-byte, and the remote evals it. Empty stays `''` -- a real, shiftable argument -- +# rather than vanishing from the joined string and shifting the run's own flags out of place. +HIL_EXPORTS_Q=$(printf '%q' "$HIL_EXPORTS") + +ssh "$REMOTE" bash -s -- "$REMOTE_DIR" "$HIL_EXPORTS_Q" --retry 1 ${ARGS_Q[@]+"${ARGS_Q[@]}"} "$CONFIG_Q" <<'REMOTE' || rc=$? cd -- "$1" shift +eval "$1"; shift # HIL_* exports, %q-quoted locally into one word # Flasher CLIs live in the user bin dirs on ci.lan (esptool/idf in ~/.local/bin, # STM32CubeProgrammer's STM32_Programmer_CLI in ~/bin); the non-interactive shell # subprocess used for flashing doesn't source profile/rc, so add them explicitly. @@ -136,8 +382,41 @@ REMOTE # Copy the generated report back to the local checkout (best-effort; the run's # exit code is preserved regardless of whether a report was produced). -scp -q "$REMOTE:$REMOTE_DIR/hil_report.md" "$ROOT_DIR/hil_report.md" \ - && echo "==> Report copied to $ROOT_DIR/hil_report.md" \ - || echo "==> warning: no hil_report.md copied back" >&2 +# rm -f FIRST, exactly as the sidecar loop below does: the markdown and the JSON are two +# halves of ONE document now, so leaving a stale table behind when the copy fails -- beside +# a sidecar that was correctly removed -- publishes last run's green results under this +# run's red job, and the operator's hil_report.py call exits 1 against the missing sidecar. +# Fetch BOTH halves to temps and commit them as a pair. Separate fetch/rename meant a +# markdown that arrived beside a sidecar that did not left the local pair failing the +# rendering invariant, and the next --accumulate retry merging the wrong base. Deleting +# first and then scp'ing was worse still: an ssh drop at the end of a 60-minute run +# destroyed the report outright. +md_ok=0; json_ok=0 +scp -q "$REMOTE:$REMOTE_DIR/hil_report.md" "$ROOT_DIR/hil_report.md.tmp" 2>/dev/null \ + && [ -f "$ROOT_DIR/hil_report.md.tmp" ] && md_ok=1 +scp -q "$REMOTE:$REMOTE_DIR/hil_report.json" "$ROOT_DIR/hil_report.json.tmp" 2>/dev/null \ + && [ -f "$ROOT_DIR/hil_report.json.tmp" ] && json_ok=1 +if [ "$md_ok" = 1 ] && [ "$json_ok" = 1 ]; then + mv -f "$ROOT_DIR/hil_report.md.tmp" "$ROOT_DIR/hil_report.md" + mv -f "$ROOT_DIR/hil_report.json.tmp" "$ROOT_DIR/hil_report.json" + echo "==> Report copied to $ROOT_DIR/hil_report.md (+ sidecar)" +else + rm -f "$ROOT_DIR/hil_report.md.tmp" "$ROOT_DIR/hil_report.json.tmp" + # All or nothing: a half-copied pair is worse than none. The stale local markdown goes + # because that is what gets pasted into a PR as this run's results; the stale sidecar + # goes with it so the two cannot disagree. + rm -f "$ROOT_DIR/hil_report.md" "$ROOT_DIR/hil_report.json" + echo "==> warning: report copy-back incomplete (md=$md_ok json=$json_ok); removed the" \ + "stale local pair -- an --accumulate retry has no merge base until a run succeeds" >&2 +fi + +# The re-run spec lives in the run's cwd on the rig and the next invocation rm -rf's it. +# Delete the local copy first: a green run writes no .failed, so a silent no-op scp would +# leave last run's spec looking current and "retry from the spec" would re-flash boards +# that passed. +spec="$(basename "$CONFIG").failed" +rm -f "$ROOT_DIR/$spec" +scp -q "$REMOTE:$REMOTE_DIR/$spec" "$ROOT_DIR/$spec" 2>/dev/null \ + && echo "==> $spec copied to $ROOT_DIR/$spec" || true exit $rc diff --git a/test/hil/hil_ci_set_matrix.py b/test/hil/hil_ci_set_matrix.py deleted file mode 100644 index bca989bd1..000000000 --- a/test/hil/hil_ci_set_matrix.py +++ /dev/null @@ -1,90 +0,0 @@ -import argparse -import json -import os - - -def _resolve_config_path(config_file): - if os.path.exists(config_file): - return config_file - - script_relative = os.path.join(os.path.dirname(__file__), config_file) - if os.path.exists(script_relative): - return script_relative - - raise FileNotFoundError(f'Config file not found: {config_file}') - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('config_files', nargs='+', help='Configuration JSON file(s)') - parser.add_argument('--select', help='hil_select.py JSON; scopes boards when full=false') - args = parser.parse_args() - - selected = None - sel = json.loads(args.select) if args.select else None - if sel and not sel.get('full'): - selected = set(sel.get('boards', {})) - - # Toolchain buckets must match the toolchains instantiated by the hil-build - # job in .github/workflows/build.yml. Keep all keys present (even if empty) - # so `fromJSON(hil_json)[toolchain]` always resolves to a list. - matrix = { - 'arm-gcc': [], - 'riscv-gcc': [], - 'esp-idf': [] - } - - seen = {toolchain: set() for toolchain in matrix} - - def append_build_arg(toolchain, build_arg): - if build_arg not in seen[toolchain]: - seen[toolchain].add(build_arg) - matrix[toolchain].append(build_arg) - - for config_file in args.config_files: - with open(_resolve_config_path(config_file)) as f: - config = json.load(f) - - for board in config['boards']: - if selected is not None and board['name'] not in selected: - continue - name = board['name'] - flasher = board['flasher'] - # esptool boards must build under esp-idf; others default to arm-gcc - # but may opt into another bucket via an explicit "toolchain" field - # (e.g. RISC-V boards like ch32v20x need "riscv-gcc"). - if flasher['name'] == 'esptool': - toolchain = 'esp-idf' - else: - toolchain = board.get('toolchain', 'arm-gcc') - if toolchain not in matrix: - # a board in no bucket would never be built, and the bare KeyError - # below would only say so as a traceback from the set-matrix job - raise SystemExit( - f'{name}: toolchain {toolchain!r} is not a build bucket ' - f'({", ".join(matrix)}); add it here and to the hil-build / ' - f'hil-build-esp jobs in .github/workflows/build.yml') - - build_board = f'-b {name}' - if 'build' in board and 'args' in board['build']: - build_board += ' ' + ' '.join(f'-D{a}' for a in board['build']['args']) - - # Each variant builds into cmake-build-<variant.name> with its own cmake - # -D defines and raw CFLAGS. No 'variant' -> a single build named after - # the board. - variants = board.get('variant') or [{'name': name, 'flags': ''}] - for v in variants: - arg = build_board - if v['name'] != name: - arg += f' --build-name {v["name"]}' - for d in v.get('defines', []): - arg += f' -D{d}' - for tok in v.get('flags', '').split(): - arg += f' --cflag={tok}' - append_build_arg(toolchain, arg) - - print(json.dumps(matrix)) - - -if __name__ == '__main__': - main() diff --git a/test/hil/hil_examples.py b/test/hil/hil_examples.py deleted file mode 100644 index 4c8b6918b..000000000 --- a/test/hil/hil_examples.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MIT -# HIL example test lists, shared by hil_test.py (runner) and hil_select.py -# (PR-diff selector). Stdlib-only: hil_select runs on bare CI runners. - -# The per-board run order is shuffled (see test_board). -# Every example carries a unique hardcoded idProduct (see its usb_descriptors.c) - -# device tests -device_tests = [ - 'device/cdc_dual_ports', - 'device/cdc_msc', - 'device/dfu', - 'device/cdc_msc_throughput', - 'device/audio_test_freertos', - 'device/dfu_runtime', - 'device/cdc_msc_freertos', - 'device/hid_boot_interface', - 'device/msc_dual_lun', - 'device/hid_generic_inout', - 'device/printer_to_cdc', - 'device/midi_test', - 'device/mtp', - 'device/usbtest', # cafe:4010, unique PID; runs the Linux testusb tier-4 battery via usbtest.py - # 'device/net_lwip_webserver', # disabled for PR #3605: USB net iface enum is flaky on the CI HIL host -] - -dual_tests = [ - 'dual/host_info_to_device_cdc', -] - -host_test = [ - 'host/cdc_msc_hid', - 'host/msc_file_explorer', - 'host/msc_file_explorer_freertos', - 'host/device_info', -] diff --git a/test/hil/hil_flash.py b/test/hil/hil_flash.py index da81fcc97..15f476ccd 100755 --- a/test/hil/hil_flash.py +++ b/test/hil/hil_flash.py @@ -1,138 +1,48 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: MIT -# Firmware flashing for the TinyUSB HIL rig: run_cmd, one flash_*/reset_* pair per -# flasher type (dispatched by config name via getattr), find_firmware, and the -# fixture serial-port resolver get_serial_dev (here, not hil_test: flash_esptool -# needs it and helpers must not import hil_test). -# Callers set module globals `build_dir` and `verbose` (hil_test.main from argparse, -# pool_check directly) exactly as they set hil_test's globals today. -# -# from __future__ import annotations (below): some moved function signatures use -# type hints (Any, Board) not defined in this module; postponed evaluation (PEP -# 563) keeps those as unevaluated strings so the verbatim-moved defs still load. +# Firmware flashing for the TinyUSB HIL rig: one flash_*/reset_* pair per flasher type +# (dispatched by config name via getattr) plus find_firmware. The bounded runner run_cmd +# lives in hil_util (never import hil_test here). Callers set the module global +# `build_dir`. `from __future__ import annotations` keeps the Board hints below +# unevaluated: the type is not defined in this module. from __future__ import annotations -import glob import json -import os -import signal +import re import subprocess from pathlib import Path -verbose = False -build_dir = 'cmake-build' +import os +import sys -CMD_TIMEOUT = int(os.getenv('HIL_CMD_TIMEOUT', '180')) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # PYTHONSAFEPATH drops it +from helper import hil_util + +build_dir = 'cmake-build' # flasher names (dispatch key, board['flasher']['name'].lower()) whose reset_* is a no-op RESET_NOOP = {'esptool', 'lm4flash'} # extra parents find_firmware ALSO searches after build_dir. Empty by default so -# hil_test's -B stays authoritative (a board missing there must report "Skip (no -# binary)", never silently flash a stale binary from another tree); pool_check -# opts in to cover both standard layouts. +# hil_test's -B stays authoritative: a board missing there must report "Skip (no +# binary)", never silently flash a stale binary from another tree. EXTRA_BUILD_DIRS: list = [] - -def cmd_stdout_text(out: Any) -> str: - if out is None: - return '' - if isinstance(out, bytes): - return out.decode('utf-8', errors='ignore') - return str(out) - - -# ------------------------------------------------------------- -# Path -# ------------------------------------------------------------- -TINYUSB_ROOT = Path(__file__).resolve().parents[2] - -# get usb serial by id -def get_serial_dev(id, vendor_str, product_str, ifnum): - if vendor_str and product_str: - # known vendor and product - vendor_str = vendor_str.replace(' ', '_') - product_str = product_str.replace(' ', '_') - return f'/dev/serial/by-id/usb-{vendor_str}_{product_str}_{id}-if{ifnum:02d}' - else: - # just use id: mostly for cp210x/ftdi flasher - pattern = f'/dev/serial/by-id/usb-*_{id}-if*' - port_list = glob.glob(pattern) - if len(port_list) == 0: - raise RuntimeError(f'No serial device found for {pattern}') - return port_list[0] +_VID_PID_WARNED: set = set() # one warning per probe, not per command # ------------------------------------------------------------- # Flashing firmware # ------------------------------------------------------------- -def run_cmd(cmd: str, cwd: str | None = None, timeout: int = CMD_TIMEOUT) -> subprocess.CompletedProcess: - popen_kwargs = { - 'cwd': cwd, - 'shell': True, - 'stdout': subprocess.PIPE, - 'stderr': subprocess.STDOUT, - 'text': True, - 'encoding': 'utf-8', - 'errors': 'replace', - } - if os.name != 'nt': - # C-level setsid, same process-group semantics as preexec_fn=os.setsid but - # safe when called from threads (pool_check runs flashes from a thread pool) - popen_kwargs['start_new_session'] = True - - p = subprocess.Popen(cmd, **popen_kwargs) - try: - out, _ = p.communicate(timeout=timeout) - r = subprocess.CompletedProcess(args=cmd, returncode=p.returncode, stdout=out) - except subprocess.TimeoutExpired as ex: - if os.name != 'nt': - try: - os.killpg(p.pid, signal.SIGKILL) - except ProcessLookupError: - pass - else: - p.kill() - try: - out, _ = p.communicate(timeout=10) - except subprocess.TimeoutExpired: # unkillable (e.g. D-state on wedged USB) - out = None - timeout_out = ex.stdout or out or b'' - title = f'COMMAND TIMEOUT ({timeout}s): {cmd}' - print() - if os.getenv('CI'): - print(f"::group::{title}") - print(cmd_stdout_text(timeout_out)) - print(f"::endgroup::") - else: - print(title) - print(cmd_stdout_text(timeout_out)) - return subprocess.CompletedProcess(args=cmd, returncode=124, stdout=timeout_out) - - if r.returncode != 0: - title = f'COMMAND FAILED: {cmd}' - print() - if os.getenv('CI'): - print(f"::group::{title}") - print(cmd_stdout_text(r.stdout)) - print(f"::endgroup::") - else: - print(title) - print(cmd_stdout_text(r.stdout)) - elif verbose: - print(cmd) - print(cmd_stdout_text(r.stdout)) - return r - - -def flash_jlink(board: Board, firmware: str) -> subprocess.CompletedProcess: +def flash_jlink(board: Board, firmware: str, timeout=None) -> subprocess.CompletedProcess: flasher = board['flasher'] script = ['halt', 'r', f'loadfile {firmware}', 'r', 'go', 'exit'] f_jlink = Path(f'{board["name"]}_{Path(firmware).name}.jlink') with f_jlink.open('w') as f: f.writelines(f'{s}\n' for s in script) - ret = run_cmd(f'JLinkExe -USB {flasher["uid"]} {flasher["args"]} -if swd -JTAGConf -1,-1 -speed auto -NoGui 1 -ExitOnError 1 -CommandFile {f_jlink}') + ret = hil_util.run_cmd(f'JLinkExe -USB {flasher["uid"]} {flasher["args"]} -if swd -JTAGConf -1,-1 -speed auto -NoGui 1 -ExitOnError 1 -CommandFile {f_jlink}', + timeout=timeout) f_jlink.unlink(missing_ok=True) return ret @@ -144,89 +54,208 @@ def reset_jlink(board: Board) -> subprocess.CompletedProcess: if not f_jlink.exists(): with f_jlink.open('w') as f: f.writelines(f'{s}\n' for s in script) - ret = run_cmd(f'JLinkExe -USB {flasher["uid"]} {flasher["args"]} -if swd -JTAGConf -1,-1 -speed auto -NoGui 1 -ExitOnError 1 -CommandFile {f_jlink}') + ret = hil_util.run_cmd(f'JLinkExe -USB {flasher["uid"]} {flasher["args"]} -if swd -JTAGConf -1,-1 -speed auto -NoGui 1 -ExitOnError 1 -CommandFile {f_jlink}') return ret -def flash_stlink(board, firmware): +def flash_stlink(board, firmware, timeout=None): + # --verify catches the partial/corrupt write that exits 0 and sends the test phase + # off to exercise bad firmware. Opt-IN here ("verify": true), unlike flash_openocd's + # opt-out: a default-on read-back silently changes every roster entry that lacks the + # key, including boards on rigs this was never validated against. flasher = board['flasher'] - return run_cmd(f'STM32_Programmer_CLI --connect port=swd sn={flasher["uid"]} --write {firmware} --go') + verify = ' --verify' if flasher.get('verify', False) else '' + return hil_util.run_cmd(f'STM32_Programmer_CLI --connect port=swd sn={flasher["uid"]} --write {firmware}{verify} --go', + timeout=timeout) def reset_stlink(board): flasher = board['flasher'] - return run_cmd(f'STM32_Programmer_CLI --connect port=swd sn={flasher["uid"]} --rst --go') + return hil_util.run_cmd(f'STM32_Programmer_CLI --connect port=swd sn={flasher["uid"]} --rst --go') def _openocd_cmd_base(flasher): + # Optional roster field vid_pid, openocd-verbatim (e.g. "0x1a86 0x8010"), pins probe + # discovery to the probe's IDs so openocd never opens foreign usbfs nodes to read + # strings -- a wedged node makes that open hang unkillably (the 2026-08-10 convoy). + # BEFORE args, because the rescue cfgs run `init` internally and reject (or never see) + # a config command that follows it. + vid_pid = '' + if 'vid_pid' in flasher: + # Validated HERE too, not just in convoy_safe: openocd only warns ("incomplete + # vid_pid configuration directive") and exits 0 on a malformed value, so the pin + # silently does not apply and discovery goes back to opening every usbfs node -- + # the convoy this field exists to stop. The same key name carries a DIFFERENT + # syntax under tests.dev_attached ('1a86_55d4'), so the typo is one copy away. + if valid_vid_pid(flasher['vid_pid']): + vid_pid = f'-c "adapter usb vid_pid {flasher["vid_pid"]}" ' + else: + # stderr + once-per-probe, like the missing-pin branch below: stdout here is + # captured by test_example's redirect_stdout (shown only when the test FAILS) + # and by hil_pool_check's StringIO spool, so on a PASSING run the operator + # would never learn the pin was silently dropped. + uid = flasher.get('uid', '?') + if uid not in _VID_PID_WARNED: + _VID_PID_WARNED.add(uid) + print(f'warning: {uid} has a malformed vid_pid {flasher["vid_pid"]!r} ' + f'(want "0xVVVV 0xPPPP"); probe pin DROPPED, so discovery will open ' + f'foreign usbfs nodes', file=sys.stderr, flush=True) + elif flasher.get('uid') not in _VID_PID_WARNED: + # stderr, once per probe: test_example captures stdout, so a passing run would + # swallow this and the operator would never learn discovery still opens every + # usbfs node + _VID_PID_WARNED.add(flasher.get('uid')) + print(f'warning: openocd flasher {flasher.get("uid", "?")} has no vid_pid pin; ' + f'probe discovery will open every usbfs node (hangs on a wedged one)', + file=sys.stderr, flush=True) return (f'openocd -c "tcl_port disabled" -c "gdb_port disabled" -c "telnet_port disabled" ' - f'-c "adapter serial {flasher["uid"]}" {flasher["args"]}') + f'-c "adapter serial {flasher["uid"]}" {vid_pid}{flasher["args"]}') -# `verify` is on by default and opted out per board with "verify": false in the roster. -# WCH targets must opt out: flash read-back over the WCH-Link sdi transport returns a -# repeated word instead of memory contents, so verification always reports a mismatch and -# fails the flash (measured on ch32v103r and ch32v307v, 2026-07-30). Do NOT drop verify -# fleet-wide to accommodate them — every other openocd board can read back, and without it -# a partial or corrupt write exits 0 and the test phase runs bad firmware. -def flash_openocd(board, firmware): +# `verify` is on by default, opted out per board with "verify": false. WCH targets must +# opt out: read-back over the WCH-Link sdi transport returns a repeated word instead of +# memory contents, so verification always mismatches (measured on ch32v103r and ch32v307v, +# 2026-07-30). Do NOT drop verify fleet-wide for them — every other openocd board reads +# back, and without it a partial or corrupt write exits 0 and the tests run bad firmware. +def flash_openocd(board, firmware, timeout=None): flasher = board['flasher'] verify = ' verify' if flasher.get('verify', True) else '' - ret = run_cmd(f'{_openocd_cmd_base(flasher)} -c "program {firmware}{verify} reset exit"') + ret = hil_util.run_cmd(f'{_openocd_cmd_base(flasher)} -c "program {firmware}{verify} reset exit"', + timeout=timeout) return ret -def reset_openocd(board): +def reset_openocd(board, timeout=None): + # timeout: usbtest's post-hang recovery bounds this (RECOVER_RESET_TIMEOUT); an + # unbounded reset there would outlive the caller's outer kill and orphan openocd on + # the probe, which is the stray the recovery exists to avoid. flasher = board['flasher'] - ret = run_cmd(f'{_openocd_cmd_base(flasher)} -c "init; reset run; exit"') + ret = hil_util.run_cmd(f'{_openocd_cmd_base(flasher)} -c "init; reset run; exit"', + timeout=timeout) return ret # OpenOCD's messages for "the target's debug port did not answer". The probe is fine when -# these appear (the log still shows "CMSIS-DAP: Interface ready"); the chip's debug clock -# is gone, which no reset the probe can drive would fix -- the CMSIS-DAP Debug Probe has no -# nRESET line at all. Which message you get depends on the DAP topology, NOT on the board: -# rp2040.cfg creates three multidrop DAPs (cores 0/1 and the Rescue DP at instance 0xf) so -# it fails in swd_multidrop_select, while rp2350.cfg creates a single plain ADIv6 DAP that -# fails earlier in swd_connect. A dead RP2040 can also produce the second one if the very -# first DP read never gets through, so both are accepted for both chips -- it is the target -# cfg in the roster args, below, that picks how to rescue. +# these appear ("CMSIS-DAP: Interface ready" is still logged); the chip's debug clock is +# gone, which no probe-driven reset fixes -- the CMSIS-DAP probe has no nRESET line. Which +# message appears depends on DAP topology, not the board, so both are accepted for both +# chips; RESCUE_CFG below picks the rescue. DAP_WEDGED = ('Failed to connect multidrop', 'Error connecting DP: cannot read IDR') -# How each RP target reaches its Rescue DP, keyed by the target cfg named in flasher args. -# (cfg substitution, extra args): rp2040.cfg drives the Rescue DP itself behind a RESCUE -# flag and calls init/shutdown on its own; rp2350 has a separate cfg that pokes the rescue -# bit via an AP register but never shuts down, so it would sit in the server loop until -# CMD_TIMEOUT without an explicit one. +# How each RP target reaches its Rescue DP, keyed by the target cfg named in flasher args: +# (cfg substitution, pre args, post args). rp2040.cfg drives the Rescue DP behind a RESCUE +# flag and init/shutdowns itself; rp2350-rescue.cfg never shuts down, so it needs an +# explicit one or it sits in the server loop until CMD_TIMEOUT. RESCUE_CFG = { 'target/rp2040.cfg': ('target/rp2040.cfg', '-c "set RESCUE 1" ', ''), 'target/rp2350.cfg': ('target/rp2350-rescue.cfg', '', ' -c "shutdown"'), } -def rescue_openocd(board, flash_out: str = '') -> bool: +def rescue_openocd(board, flash_out: str = '', timeout=None) -> bool: """Power-on-reset a wedged RP2040/RP2350 through its Rescue DP, the one debug port not - gated by the system clock (RP2040 datasheet 2.3.4.2): setting CDBGPWRUPREQ hard-resets - the chip, and the bootrom halts it in a safe state ready to be flashed. This is the - only way back for a target whose cores have stopped answering -- otherwise the board - needs a physical replug, since the probe carries no reset line. + gated by the system clock (RP2040 datasheet 2.3.4.2): CDBGPWRUPREQ hard-resets the + chip and the bootrom halts it ready to be flashed. Without it the board needs a + physical replug -- the probe carries no reset line. - No-op (returns False) unless this is an openocd RP board AND the flash output shows the - wedge, so a flash that failed for any other reason still just retries. Returns True - when a rescue was attempted; the caller should retry the flash afterwards.""" + No-op (False) unless this is an openocd RP board AND the flash output shows the wedge, + so a flash that failed for any other reason still just retries. True when a rescue was + attempted; the caller should retry the flash afterwards.""" flasher = board['flasher'] if flasher['name'].lower() != 'openocd' or not any(m in flash_out for m in DAP_WEDGED): return False for cfg, (rescue_cfg, pre, post) in RESCUE_CFG.items(): if cfg in flasher['args']: args = flasher['args'].replace(cfg, rescue_cfg) - return run_cmd(f'{_openocd_cmd_base({**flasher, "args": pre + args})}{post}').returncode == 0 + return hil_util.run_cmd(f'{_openocd_cmd_base({**flasher, "args": pre + args})}{post}', + timeout=timeout).returncode == 0 return False -def flash_esptool(board: Board, firmware: str) -> subprocess.CompletedProcess: +# openocd's own syntax: one or more "0xVVVV 0xPPPP" pairs. Validated rather than merely +# tested for truthiness -- `vid_pid` is a hand-edited roster field whose NAME is also used, +# with a different syntax, by tests.dev_attached, and convoy_safe reads a non-empty value +# as PROOF the flasher can deliver a recovery past a poisoned node. A typo there silently +# promised a recovery that openocd would reject at startup. +_VID_PID_RE = re.compile(r'^0x[0-9a-fA-F]{4}(\s+0x[0-9a-fA-F]{4})+$') + + +def valid_vid_pid(value) -> bool: + return isinstance(value, str) and bool(_VID_PID_RE.match(value.strip())) + + +def recover_flasher(board: dict) -> dict: + """The flasher that delivers RECOVERY for this board. + + Optional roster key `flasher_recover`, else the primary. It exists because delivery and + normal flashing have different requirements: a board flashed by jlink/stlink/lm4flash + cannot reach its probe past a poisoned usbfs node, but the same probe driven by openocd + often can (see convoy_safe). Keeping it a separate key rather than a list means the + primary's shape never changes, so nothing that reads board['flasher'] has to care. + """ + return board.get('flasher_recover') or board['flasher'] + + +def convoy_safe(flasher: dict) -> bool: + """Can this flasher DELIVER a recovery while a usbfs node on the rig is poisoned? + + A post-HUNG reflash only helps if the flasher reaches its probe without opening the + wedged node. Two shapes qualify: + + * openocd pinned with the roster's `vid_pid` -- the match is made from the cached + descriptor and the loop `continue`s BEFORE libusb_open, so a foreign node is never + opened. On 2026-08-12 it was the only flasher that still reached its probe. + * esptool -- delivery is `-p <ttyACM>`, a named port; it never enumerates usbfs. + + Everything else enumerates by OPENING nodes, would block in D state on the poisoned + one, survive SIGKILL and become a second stray. JLinkExe cannot be pinned: selection + is serial-only (-USB/-SelectEmuBySN) and reading a serial requires the open (J-Link + Commander V9.66 exposes no VID/PID filter), so those boards can only become + convoy-safe by moving to openocd. + + Verified against openocd 0ce743125 (the rig's build), because the INVERSE is what + bites: cmsis_dap_usb_bulk.c:107 skips on `id_filter && !id_match`, and `id_filter` is + only `vids[0] || pids[0]` -- so without the pin nothing is skipped and every device on + the bus is opened, which the code itself expects to mostly fail. Enumeration cannot + block: libusb reads the `descriptors` sysfs attribute, and descriptors_read (v6.12.101 + drivers/usb/core/sysfs.c) is a memcpy from udev->rawdescriptors under no lock. + + The pin gates the BULK backend, which is the one that runs: `auto` tries usb_bulk -> + hid -> tcp (cmsis_dap.c:62) and stops at the first that opens, so a CMSIS-DAP v2 probe + never reaches the rest. It does NOT cover the HID fallback that a v1 probe or a failed + bulk open takes -- cmsis_dap_usb_hid.c:91 calls hid_enumerate(0x0, 0x0), pin ignored, + and filters afterwards, while hidapi's hidraw backend reads `manufacturer` and + `product` for every HID device it lists (linux/hid.c:744), both usb_string_attr and so + served under the device lock. A wedged DUT running hid_generic_inout, + hid_boot_interface or hid_composite_freertos is a HID device and would stall that walk + -- interruptibly, so it hangs rather than joining the D-state convoy and run_cmd's + timeout ends it, but "never opens a foreign node" is true of the bulk path, not of + every path openocd can take. + """ + name = (flasher.get('name') or '').lower() + if name == 'esptool': + return True + # EXACT, not startswith: rescue_openocd and usbtest's + # getattr(hil_flash, f'flash_{name}') both require the exact name, so an + # 'openocd_wch'-style entry would pass this gate, reserve the Rescue-DP legs, + # and then find no recovery path at all -- paying for a path that cannot fire, which + # is the precise cost this gate exists to avoid. + if name != 'openocd': + return False + if valid_vid_pid(flasher.get('vid_pid')): + return True + # openocd over the JLINK driver is safe WITHOUT a pin, and cannot use one: jlink.c + # never reads adapter_usb_get_vids/pids (selection is adapter serial / usb address / + # usb location), but libjaylink's discovery returns early unless idVendor == 0x1366 and + # the PID is in its table, and only THEN calls libusb_open (discovery_usb.c). So it + # never opens a foreign node -- which is exactly what JLinkExe, SEGGER's own tool, + # does do. Verified against openocd 0ce743125 and libjaylink master. + return 'interface/jlink.cfg' in (flasher.get('args') or '') + + +def flash_esptool(board: Board, firmware: str, timeout=None) -> subprocess.CompletedProcess: flasher = board['flasher'] - port = get_serial_dev(flasher["uid"], None, None, 0) + port = hil_util.get_serial_dev(flasher["uid"], None, None, 0) fw_dir = Path(firmware).parent with (fw_dir / 'config.env').open() as f: idf_target = json.load(f)['IDF_TARGET'] @@ -234,32 +263,39 @@ def flash_esptool(board: Board, firmware: str) -> subprocess.CompletedProcess: flash_args = f.read().strip().replace('\n', ' ') command = (f'esptool --chip {idf_target} -p {port} {flasher["args"]} ' f'--before=default_reset --after=hard_reset write_flash {flash_args}') - ret = run_cmd(command, cwd=str(fw_dir)) + ret = hil_util.run_cmd(command, cwd=str(fw_dir), timeout=timeout) return ret def reset_esptool(board): - flasher = board['flasher'] + # NO-OP, and marked as one: esptool's reset would be `--after hard_reset`, which is not + # wired here. Returning rc 0 without resetting is why callers must never read the exit + # code as proof -- usbtest's recovery skips a primitive carrying `no_op`. return subprocess.CompletedProcess(args=['dummy'], returncode=0) -def flash_lm4flash(board, firmware): +reset_esptool.no_op = True + + +def flash_lm4flash(board, firmware, timeout=None): # TI Tiva-C / Stellaris ICDI: lightweight lm4flash, resets and runs after write flasher = board['flasher'] - ret = run_cmd(f'lm4flash -s {flasher["uid"]} {flasher["args"]} {firmware}') + ret = hil_util.run_cmd(f'lm4flash -s {flasher["uid"]} {flasher["args"]} {firmware}', + timeout=timeout) return ret def reset_lm4flash(board): # lm4flash has no reset-only mode; it resets+runs on flash, so reset is a no-op - flasher = board['flasher'] return subprocess.CompletedProcess(args=['dummy'], returncode=0) -# The one place a flasher's firmware extension is decided: find_firmware resolves the -# path with it and the flash_* functions pass that path through untouched. A flasher -# added here without an entry falls back to .elf-or-.bin and can be handed the wrong -# file — test_hil_select's TestRosterFlashersDispatch fails if a roster names one. +reset_lm4flash.no_op = True + + +# The one place a flasher's firmware extension is decided. A flasher with no entry falls +# back to .elf-or-.bin and can be handed the wrong file — test_ci_select's +# TestRosterFlashersDispatch fails if a roster names one. FLASHER_SUFFIX = { 'esptool': '.bin', 'jlink': '.elf', @@ -271,13 +307,12 @@ FLASHER_SUFFIX = { def find_firmware(variant: str, example: str, roots: list | None = None, flasher: str | None = None): """Locate a built example's firmware under <build_dir>/cmake-build-<variant>/<example>/, - then under EXTRA_BUILD_DIRS (empty unless the caller opts in — see its comment). - `roots` overrides that search list entirely for one call (e.g. to find a build just - produced by tools/build.py in its fixed cmake-build/ layout without widening the - global policy). `flasher` is the roster flasher name: it selects which extension - counts (see FLASHER_SUFFIX), so a build that produced only the other one is reported - missing — a clean "Skip (no binary)" — instead of being handed to the flasher, which - would fail opaquely on the absent file and burn every retry plus the board lock. + then under EXTRA_BUILD_DIRS. `roots` overrides that search list entirely for one call + (e.g. a build just produced by tools/build.py in its fixed cmake-build/ layout) + without widening the global policy. `flasher` is the roster flasher name and selects + which extension counts (FLASHER_SUFFIX), so a build that produced only the other one + is reported missing — a clean "Skip (no binary)" — instead of being handed to the + flasher, which would fail opaquely and burn every retry plus the board lock. Accepts the single-config layout (firmware directly in the example dir) or Ninja Multi-Config (a per-config subdir like RelWithDebInfo/). Returns the full Path INCLUDING extension, or None if not built.""" @@ -286,7 +321,7 @@ def find_firmware(variant: str, example: str, roots: list | None = None, flasher if not suffixes or suffixes == [None]: suffixes = ['.elf', '.bin'] for bd in dict.fromkeys(roots if roots is not None else [build_dir, *EXTRA_BUILD_DIRS]): - fw_dir = TINYUSB_ROOT / bd / f'cmake-build-{variant}' / example + fw_dir = hil_util.TINYUSB_ROOT / bd / f'cmake-build-{variant}' / example if not fw_dir.is_dir(): continue for cand in [fw_dir / base, fw_dir / 'RelWithDebInfo' / base, diff --git a/test/hil/hil_select.py b/test/hil/hil_select.py deleted file mode 100755 index 3ac3f1fdb..000000000 --- a/test/hil/hil_select.py +++ /dev/null @@ -1,520 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MIT -"""PR-diff -> HIL selection: which rig boards and which tests a change can affect. - -Stdlib-only (runs on bare CI runners; never imports hil_test/hil_flash/hil_lock). -Fail-open: any file no rule classifies forces the full matrix. See -docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md. - -JSON: full, boards (name -> 'all' | [tests]), families (bsp families the diff -touches, including ones with no rig board - build-only consumers such as /pre-pr -sample from these), args (hil_test.py args per config) and args_flasher (the same -args split by each board's flasher, for CI legs that split one rig by flasher). -""" -import argparse -import functools -import glob -import json -import os -import re -import subprocess -import sys - -from hil_examples import device_tests, dual_tests, host_test - -ALL_TESTS = {'device': device_tests, 'dual': dual_tests, 'host': host_test} - -# class dir -> config macro suffix exceptions (rule 3); dfu is per-file, handled inline -NET_MACROS = ('ECM_RNDIS', 'NCM') - -_NONCODE_RE = re.compile( - r'^(docs/|\.claude/|.*\.(md|rst)$|LICENSE)') -_FULL_RE = re.compile( - r'^(src/common/|src/osal/|src/tusb\.c$|src/tusb\.h$|src/tusb_option\.h$|' - r'test/hil/|\.github/workflows/build.*\.yml$|\.github/actions/|' - r'tools/build\.py$|tools/get_deps\.py$|tools/cmake/|hw/mcu/|lib/|' - r'hw/bsp/(family_support\.cmake|board_api\.h|board\.c|ansi_escape\.h)$|' - r'examples/build_system/|examples/CMakeLists\.txt$|' - # board_test is HIL infrastructure, not a test: hil_test.py flashes it to park - # every board (variant boundary + end-of-board teardown), so every board depends on it - r'examples/device/board_test/)') - -# --no-renames: with rename detection git reports only a rename's destination, so code -# moved out of an HIL-relevant path would be classified by its new path alone -GIT_DIFF_ARGV = ['git', 'diff', '--no-renames', '--name-only'] - - -def test_role(test: str) -> str: - return test.split('/', 1)[0] # 'device' | 'dual' | 'host' - - -def board_roles(board: dict) -> set: - t = board.get('tests', {}) - roles = set() - if t.get('device'): - roles.add('device') - if t.get('host'): - roles.add('host') - if t.get('dual'): - roles.update(('device', 'host')) - for only in t.get('only', []): - r = test_role(only) - roles.update(('device', 'host') if r == 'dual' else (r,)) - return roles - - -def board_tests(board: dict) -> list: - """Every test this board would run today (mirrors hil_test.test_board's default).""" - t = board.get('tests', {}) - if 'only' in t: - run = list(t['only']) - else: - run = [] - if t.get('device'): - run += device_tests - if t.get('dual'): - run += dual_tests - if t.get('host'): - run += host_test - return [x for x in run if x not in t.get('skip', [])] - - -# cached: called per changed file x roster board, and the tree doesn't change mid-run [email protected]_cache(maxsize=None) -def board_family(board_name: str, repo_root: str): - hits = glob.glob(os.path.join(repo_root, 'hw/bsp/*/boards', board_name)) - return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None - - -# `if (OPTION STREQUAL "1")` guards in family_support.cmake, and the option tokens -# a roster entry passes to the build (NAME=VALUE / -DNAME=VALUE) -_CM_IF_RE = re.compile(r'if\s*\(') -_CM_ELSE_RE = re.compile(r'else(if)?\s*\(') -_CM_ENDIF_RE = re.compile(r'endif\s*\(') -_CM_OPT_RE = re.compile(r'if\s*\(\s*\$?\{?([A-Za-z_]\w*)\}?\s+STREQUAL\s+"?1"?\s*\)') -_CM_PORT_RE = re.compile(r'src/portable/((?:[^/\s]+/)?[^/\s]+)/') -_FALSY = ('', '0', 'off', 'false', 'no') - - [email protected]_cache(maxsize=None) -def port_option_gates(repo_root: str) -> dict: - """port dir -> build options that compile it regardless of the board's family - file, e.g. {'analog/max3421': {'MAX3421_HOST'}} from family_support.cmake.""" - gates = {} - try: - text = open(os.path.join(repo_root, 'hw/bsp/family_support.cmake')).read() - except OSError: - return gates - stack = [] # one entry per open if(): its option, or None - for line in text.splitlines(): - line = line.strip() - if _CM_IF_RE.match(line): - m = _CM_OPT_RE.match(line) - stack.append(m.group(1) if m else None) - elif _CM_ELSE_RE.match(line): - if stack: - stack[-1] = None # the guard doesn't hold in this branch - elif _CM_ENDIF_RE.match(line): - if stack: - stack.pop() - opts = {o for o in stack if o} - m = _CM_PORT_RE.search(line) - if opts and m: - gates.setdefault(m.group(1), set()).update(opts) - return gates - - -_CM_SET_RE = re.compile(r'set\s*\(\s*([A-Za-z_]\w*)\s+([^)\s]+)\s*\)') - - -# cached: called per changed portable file x roster board [email protected]_cache(maxsize=None) -def bsp_board_options(board_name: str, repo_root: str) -> frozenset: - """Build options a board turns on in its own BSP: `set(<OPT> <value>)` in - hw/bsp/<family>/boards/<board>/board.cmake, e.g. MAX3421_HOST on the espressif - and rp2040 max3421 boards. CMake only - HIL CI builds nothing with Make, so a - board.mk-only option (e.g. nrf5340dk's MAX3421_HOST) compiles no port here.""" - fam = board_family(board_name, repo_root) - if not fam: - return frozenset() - path = os.path.join(repo_root, 'hw/bsp', fam, 'boards', board_name, 'board.cmake') - try: - text = open(path).read() - except OSError: - return frozenset() - out = set() - for line in text.splitlines(): - line = line.strip() - if line.startswith('#'): - continue - m = _CM_SET_RE.match(line) - if m and m.group(2).strip('"').lower() not in _FALSY: - out.add(m.group(1)) - return frozenset(out) - - -def board_options(board: dict, repo_root: str) -> set: - """Build options a board has truthy: the roster entry's build.args plus each - variant's defines (NAME=VALUE) and raw CFLAGS (-DNAME=VALUE), plus whatever its - own board.cmake sets (a board can enable a gated port without the roster saying so).""" - toks = list(board.get('build', {}).get('args', [])) - for v in board.get('variant', []): - toks += list(v.get('defines', [])) - toks += v.get('flags', '').split() - out = set(bsp_board_options(board['name'], repo_root)) - for t in toks: - name, _, val = (t[2:] if t.startswith('-D') else t).partition('=') - if name and val.strip().strip('"').lower() not in _FALSY: - out.add(name.strip()) - return out - - [email protected]_cache(maxsize=None) -def port_families(port_dir: str, repo_root: str) -> set: - """Board families that compile this src/portable dir. CMake only: HIL CI builds - every board with CMake, so a port wired up in family.mk alone is compiled for no - HIL board and must not select one. family.cmake lists portable sources directly - for most families; espressif instead references them from a nested component - CMakeLists.txt (hw/bsp/espressif/components/tinyusb_src/CMakeLists.txt).""" - fams = set() - bsp_root = os.path.join(repo_root, 'hw/bsp') - # trailing '/' so a port dir is not a prefix of a sibling: bare 'microchip/pic' - # would otherwise match '.../microchip/pic32mz/...' and inherit its families - needle = port_dir + '/' - for f in glob.glob(os.path.join(bsp_root, '*/family.cmake')) + \ - glob.glob(os.path.join(bsp_root, '*/components/*/CMakeLists.txt')): - try: - if needle in open(f).read(): - fam = os.path.relpath(f, bsp_root).split(os.sep, 1)[0] - fams.add(fam) - except OSError: - pass - return fams - - -_CLS_INC_RE = re.compile(r'#\s*include\s*[<"]class/([^/"<>]+)/([^"<>]+)[">]') - - [email protected]_cache(maxsize=None) -def class_include_edges(repo_root: str) -> dict: - """'<class>/<header>' -> the other class dirs that include it. A class header - pulled in by a second class ships in every firmware enabling that second class: - src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, and - net_device.h includes class/cdc/cdc.h. The class rule derives macros from the - directory name alone, so without this edge a change to the included header - selects only its own class's examples - and on a board that skips those (e.g. - metro_m4_express skips audio_test_freertos), nothing at all. - - Derived from the actual #include lines rather than a hand-written table so it - cannot rot when a class picks up or drops a cross-class include.""" - edges = {} - for f in sorted(glob.glob(os.path.join(repo_root, 'src/class/*/*.[ch]'))): - cls = os.path.basename(os.path.dirname(f)) - try: - text = open(f).read() - except OSError: - continue - for inc_cls, inc_hdr in _CLS_INC_RE.findall(text): - if inc_cls != cls: - edges.setdefault(f'{inc_cls}/{inc_hdr}', set()).add(cls) - return edges - - -def class_macros(cls: str, base: str, prefix: str) -> list: - """Config macros that compile a class dir's code, for role prefix TUD/TUH. - `base` refines dfu only (it splits DFU from DFU_RUNTIME per file); pass '' for - a class reached through an include edge, where the widest set is correct.""" - if cls == 'net': - return [f'CFG_{prefix}_{m}' for m in NET_MACROS] - if cls == 'dfu': - if base.startswith('dfu_rt'): - return [f'CFG_{prefix}_DFU_RUNTIME'] - if base.startswith('dfu_device') or base.startswith('dfu_host'): - return [f'CFG_{prefix}_DFU'] - return [f'CFG_{prefix}_DFU', f'CFG_{prefix}_DFU_RUNTIME'] - return [f'CFG_{prefix}_{cls.upper()}'] - - -def _config_enables(cfg_path: str, macros) -> bool: - try: - text = open(cfg_path).read() - except OSError: - return False - return any(re.search(rf'#define\s+{m}\s+\(?\s*0*[1-9]', text) for m in macros) - - -def roster_only_tests(all_boards) -> set: - """Test paths that only appear in a roster board's tests.only list (e.g. - espressif boards), not in the shared device/dual/host_test lists.""" - out = set() - for b in all_boards: - out.update(b.get('tests', {}).get('only', [])) - return out - - -def class_examples(macros, role: str, repo_root: str, extra_tests: set) -> set: - """Tests (from role's + dual lists, plus roster-only-list tests of that role) - whose example config enables any macro.""" - pool = role_tests({role}, extra_tests) - out = set() - for test in pool: - cfg = os.path.join(repo_root, 'examples', test, 'src', 'tusb_config.h') - if _config_enables(cfg, macros): - out.add(test) - return out - - -def role_tests(roles: set, extras: set) -> set: - """Every test for the given role(s): each role's own list + dual tests, - plus roster-only-list tests (extras) matching those roles or 'dual'.""" - pool = set(dual_tests) - for r in roles: - pool |= set(ALL_TESTS[r]) - pool |= {t for t in extras if test_role(t) in roles or test_role(t) == 'dual'} - return pool - - -class _Sel: - """Accumulates contributions. board->set(tests) plus 'all-board' markers.""" - def __init__(self): - self.full = False - self.by_board = {} # name -> set of tests, or 'all' - self.roles = set() # roles touched by any contribution - self.families = set() # bsp families touched (incl. off-rig ones: build-only consumers) - self.reasons = [] - - def add(self, boards, tests, reason): - """tests: 'all' or iterable of test paths.""" - self.reasons.append(reason) - for b in boards: - cur = self.by_board.get(b) - if tests == 'all' or cur == 'all': - self.by_board[b] = 'all' - else: - self.by_board[b] = (cur or set()) | set(tests) - - def force_full(self, reason): - self.full = True - self.reasons.append(reason) - - -def _classify_one(path, repo_root, roster_boards, extras: set, s: _Sel): - base = os.path.basename(path) - if _NONCODE_RE.match(path): - s.reasons.append(f'{path}: non-code, no contribution') - return - if _FULL_RE.match(path): - s.force_full(f'{path}: core/infra -> full matrix') - return - - m = re.match(r'src/portable/((?:[^/]+/)?[^/]+)/', path) - if m: - port = m.group(1) - if re.match(r'(dcd_|.*_device)', base): - roles = {'device'} - elif re.match(r'(hcd_|.*_host)', base): - roles = {'host'} - else: - roles = {'device', 'host'} - fams = port_families(port, repo_root) - if not fams: - # no family references this port: either a new/renamed port dir or a - # family.cmake layout the scan misses - widen instead of contributing nothing - s.force_full(f'{path}: port {port} maps to no board family -> full matrix') - return - s.families.update(fams) - # a board can also pull the port in through a build option (e.g. MAX3421_HOST=1 - # from the roster on metro_m4_express, or from its own board.cmake), which its - # family file never names - gates = port_option_gates(repo_root).get(port, set()) - boards = [b['name'] for b in roster_boards - if (board_family(b['name'], repo_root) in fams or - (gates and board_options(b, repo_root) & gates)) and (board_roles(b) & roles)] - tests = role_tests(roles, extras) - s.roles.update(roles) - why = f'{path}: port {port} -> families {sorted(fams)}' - if gates: - why += f' + option {sorted(gates)}' - s.add(boards, tests, f'{why} -> boards {boards} ({"/".join(sorted(roles))})') - return - - m = re.match(r'src/class/([^/]+)/', path) - if m: - cls = m.group(1) - if re.search(r'_device\.[ch]$', base): - roles = {'device'} - elif re.search(r'_host\.[ch]$', base): - roles = {'host'} - else: - roles = {'device', 'host'} - # this file's own class, plus any class whose headers include it - via = sorted(class_include_edges(repo_root).get(f'{cls}/{base}', ())) - - def macros(prefix): - return (class_macros(cls, base, prefix) + - [m2 for c in via for m2 in class_macros(c, '', prefix)]) - tests = set() - if 'device' in roles: - tests |= class_examples(macros('TUD'), 'device', repo_root, extras) - if 'host' in roles: - tests |= class_examples(macros('TUH'), 'host', repo_root, extras) - boards = [b['name'] for b in roster_boards if board_roles(b) & roles] - s.roles.update(roles) - why = f'{path}: class {cls}' + (f' (+ included by {via})' if via else '') - s.add(boards, tests, f'{why} -> {sorted(tests)} ({"/".join(sorted(roles))})') - return - - m = re.match(r'src/(device|host)/', path) - if m: - role = m.group(1) - boards = [b['name'] for b in roster_boards if role in board_roles(b)] - s.roles.add(role) - s.add(boards, role_tests({role}, extras), f'{path}: core {role} stack -> all {role} tests') - return - - m = re.match(r'hw/bsp/([^/]+)/(?:boards/([^/]+)/)?', path) - if m: - fam, brd = m.group(1), m.group(2) - s.families.add(fam) - if brd: - boards = [b['name'] for b in roster_boards if b['name'] == brd] - why = f'{path}: bsp board {brd}' - else: - boards = [b['name'] for b in roster_boards - if board_family(b['name'], repo_root) == fam] - why = f'{path}: bsp family {fam}' - s.roles.update(('device', 'host')) - s.add(boards, 'all', f'{why} -> boards {boards}') - return - - m = re.match(r'examples/(device|host|dual)/([^/]+)/', path) - if m: - test = f'{m.group(1)}/{m.group(2)}' - known = any(test in pool for pool in ALL_TESTS.values()) or test in extras - if known: - boards = [b['name'] for b in roster_boards] - role = test_role(test) - s.roles.update(('device', 'host') if role == 'dual' else (role,)) - s.add(boards, [test], f'{path}: example -> {test} on all boards') - else: - s.reasons.append(f'{path}: example not in HIL lists, no contribution') - return - - s.force_full(f'{path}: unclassified -> full matrix') - - -def classify(changed_files, repo_root, rosters): - all_boards = [] - seen = set() - for _, boards in rosters: - for b in boards: - if b['name'] not in seen: - seen.add(b['name']) - all_boards.append(b) - - extras = roster_only_tests(all_boards) - s = _Sel() - # no early exit once full: keep classifying so `families` still reports every - # family the diff touches (build-only consumers need it). Nothing after the first - # force_full can change full/boards/args - the full branch below ignores by_board. - for path in changed_files: - _classify_one(path, repo_root, all_boards, extras, s) - - if s.full: - return {'full': True, 'boards': {b['name']: 'all' for b in all_boards}, - 'families': sorted(s.families), 'reasons': s.reasons} - - # role pruning: single-role selections drop the other role's tests and boards - by_name = {b['name']: b for b in all_boards} - out = {} - for name, tests in s.by_board.items(): - allowed = board_tests(by_name[name]) - if tests == 'all': - kept = list(allowed) - else: - kept = [t for t in allowed if t in tests] - if s.roles and s.roles != {'device', 'host'}: - role = next(iter(s.roles)) - kept = [t for t in kept if test_role(t) in (role, 'dual')] - if kept: - out[name] = 'all' if set(kept) == set(allowed) else sorted(kept) - return {'full': False, 'boards': out, 'families': sorted(s.families), - 'reasons': s.reasons} - - -def _board_args(name, chosen) -> list: - parts = [f'-b {name}'] - if chosen != 'all': - parts.append(f'-bt {name}:{",".join(chosen)}') - return parts - - -def selection_args(sel, rosters): - """hil_test.py args per config. Empty means either 'full matrix' or 'nothing - selected' - callers must read sel['full'] to tell them apart.""" - args = {} - for cfg_path, boards in rosters: - parts = [] - if not sel['full']: - for b in boards: - chosen = sel['boards'].get(b['name']) - if chosen is not None: - parts += _board_args(b['name'], chosen) - args[os.path.basename(cfg_path)] = ' '.join(parts) - return args - - -def selection_args_by_flasher(sel, rosters): - """{config: {flasher name: args}}. CI runs one rig as several jobs split by - flasher (esptool vs the rest); each must gate on its own subset, otherwise the - other leg runs a filter matching zero boards and reports a vacuous green.""" - out = {} - for cfg_path, boards in rosters: - per = {} - if not sel['full']: - for b in boards: - chosen = sel['boards'].get(b['name']) - if chosen is None: - continue - per.setdefault(b.get('flasher', {}).get('name', ''), []).extend( - _board_args(b['name'], chosen)) - out[os.path.basename(cfg_path)] = {f: ' '.join(p) for f, p in per.items()} - return out - - -def changed_files_from_git(base, repo_root): - mb = subprocess.run(['git', 'merge-base', 'HEAD', base], cwd=repo_root, - capture_output=True, text=True, check=True).stdout.strip() - diff = subprocess.run(GIT_DIFF_ARGV + [f'{mb}..HEAD'], cwd=repo_root, - capture_output=True, text=True, check=True).stdout - return [l for l in diff.splitlines() if l.strip()] - - -def main(): - ap = argparse.ArgumentParser(description=__doc__) - g = ap.add_mutually_exclusive_group(required=True) - g.add_argument('--base', help='git ref to diff against (merge-base..HEAD)') - g.add_argument('--diff-file', help='newline-separated changed-file list') - ap.add_argument('configs', nargs='+', help='rig roster JSON file(s)') - a = ap.parse_args() - - repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - rosters = [] - for c in a.configs: - with open(c) as f: - rosters.append((c, json.load(f)['boards'])) - - files = (open(a.diff_file).read().splitlines() if a.diff_file - else changed_files_from_git(a.base, repo_root)) - files = [f for f in files if f.strip()] - - s = classify(files, repo_root, rosters) - s['args'] = selection_args(s, rosters) - s['args_flasher'] = selection_args_by_flasher(s, rosters) - for r in s['reasons']: - print(f'hil_select: {r}', file=sys.stderr) - print(json.dumps(s)) - - -if __name__ == '__main__': - main() diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py index e7f82bd7f..b2b74b13c 100755 --- a/test/hil/hil_test.py +++ b/test/hil/hil_test.py @@ -23,9 +23,10 @@ # THE SOFTWARE. # Host setup (required: a missing tool fails its test rather than skipping it): -# - System packages: sudo apt install mtools libmtp9 alsa-utils iperf +# - System packages: sudo apt install mtools libmtp9 libmtp-runtime alsa-utils iperf # mtools read_disk_file (device/cdc_msc, device/msc_dual_lun) # libmtp9 pymtp ctypes load (device/mtp); Debian 13 uses libmtp9t64 +# libmtp-runtime mtp-probe and the completed-device /dev/libmtp-* marker # alsa-utils arecord (device/audio_test_freertos) # iperf throughput tests (device/net_lwip_*) # openocd unified openocd from https://github.com/hathach/openocd (branch tinyusb) for wch, rp2040/rp2350, analog max32 @@ -43,8 +44,10 @@ import itertools import os import random import re -import select +import signal +import shlex import sys +import tempfile import time from contextlib import redirect_stdout from pathlib import Path @@ -52,30 +55,30 @@ from typing import TypedDict, NotRequired, cast import serial import subprocess +import traceback import json import glob import multiprocessing from multiprocessing import TimeoutError as MpTimeoutError +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # PYTHONSAFEPATH drops it import hil_flash -import hil_lock -from hil_examples import device_tests, dual_tests, host_test +import usbtest # for the recovery bounds only; hil_test runs it as a subprocess +from helper import hil_health, hil_lock, hil_report, hil_util +from helper.hil_util import device_tests, dual_tests, host_test + +# Raw Lock/Semaphore objects in Pool initargs are inheritable only under fork +# (spawn/forkserver pickle them and fail at Pool creation), so pin it against an +# interpreter default change. -# Raw Lock/Semaphore objects passed via Pool initargs are inheritable only under the fork -# start method (spawn/forkserver pickle them and fail at Pool creation) — pin it so a -# future interpreter default change cannot break the run at startup. _mp = multiprocessing.get_context('fork') Pool, Lock, Semaphore, Manager = _mp.Pool, _mp.Lock, _mp.Semaphore, _mp.Manager -import hashlib -import ctypes -from pymtp import MTP import string -# Enumeration wait budget. The first attempt gets ENUM_TIMEOUT; retry attempts get the -# shorter ENUM_TIMEOUT_RETRY - the board was just re-flashed again, and a device that is -# going to enumerate shows up within a few seconds, so a failing test costs ~3-5x a -# passing one instead of 10-30x. Per-attempt value is set by test_example(); each pool -# worker is its own process, so a module global is safe. +# Enumeration wait budget: first attempt ENUM_TIMEOUT, retries the shorter +# ENUM_TIMEOUT_RETRY -- a device that will enumerate shows up within seconds, so a failing +# test costs ~3-5x a passing one instead of 10-30x. Set per attempt by test_example(); a +# module global is safe because each pool worker is its own process. ENUM_TIMEOUT = 8 ENUM_TIMEOUT_RETRY = 4 _enum_timeout = ENUM_TIMEOUT @@ -86,11 +89,11 @@ def enum_timeout() -> int: return _enum_timeout -def wait_until(predicate, step: float = 1.0): +def wait_until(predicate, step: float = 1.0, timeout: float | None = None): """Poll predicate under the per-attempt enum budget. Deadline-based so a slow predicate - body (subprocess, libmtp scan) counts against the budget. Returns the first truthy - predicate value, or None on timeout.""" - deadline = time.monotonic() + enum_timeout() + body (subprocess, libmtp scan) counts against the budget. An explicit timeout overrides + that budget. Returns the first truthy predicate value, or None on timeout.""" + deadline = time.monotonic() + (enum_timeout() if timeout is None else timeout) while True: r = predicate() if r: @@ -103,26 +106,33 @@ STATUS_OK = "\033[32mOK\033[0m" STATUS_FAILED = "\033[31mFailed\033[0m" STATUS_SKIPPED = "\033[33mSkipped\033[0m" -# Plain (non-ANSI) cell symbols for the markdown matrix report (hil_report.md). -# A missing binary is reported as skipped too. -REPORT_CELL = {'pass': '✅', 'fail': '❌', 'skip': '⚪'} - class TestFail(AssertionError): """Fail a test but still surface a metric string in its report cell (e.g. usbtest's '❌ 29/30' instead of a bare ❌). The cell metric is icon-prefixed so render/tally treat it as a failure.""" - def __init__(self, msg: str, metric: str | None = None): + def __init__(self, msg: str, metric: str | None = None, parsed: bool = False): super().__init__(msg) self.metric = metric + # parsed=True: a real per-case verdict, so a retry would only re-observe it + # (test_example skips the rest). A failure to RUN the tool stays retryable. + self.parsed = parsed verbose = False +# Set when a HUNG usbtest case could not be recovered: the DUT's usbfs node still has a +# D-state holder, so every later flash on that board enumerates into it, blocks, survives +# SIGKILL and becomes another stray. maxtasksperchild=1 gives each board its own worker, +# so this global is board-scoped; test_board resets it anyway. +board_wedged = '' +max_retry = 1 # mirrors argparse's -r default (see main); defined HERE too so + # test_example is callable (and testable) without going through main() PROFILE = os.environ.get('HIL_PROFILE') == '1' # timestamped logs + permit/flash timing + ctrl-map dump test_only = [] board_test = {} skip_flash = False print_lock = None shuffle_seed = None # per-run seed for the per-board test-order shuffle (HIL_SHUFFLE_SEED to replay) +_current_fw = None # firmware test_example resolved for the RUNNING test (set before each test fn) def init_worker(lock, seed, b_mutexes, f_sems, cmap, cmeta, hints_by_uid): @@ -146,13 +156,21 @@ def log_line(msg: str) -> None: def compact_output(raw: str) -> str: if not raw: return '' - lines = [ln.strip() for ln in raw.replace('\r', '\n').split('\n') if ln.strip()] + # Defense in depth (the emitter already suppresses them, see _ci_log_groups): markers + # piped into this capture land mid-row, where GitHub renders them literally. + lines = [] + for ln in raw.replace('\r', '\n').split('\n'): + ln = hil_util.strip_workflow_markers(ln.strip()).strip() + if ln: + lines.append(ln) return ' | '.join(lines) class FlasherCfg(TypedDict): name: str uid: str - args: str + args: NotRequired[str] # stlink entries carry no args + vid_pid: NotRequired[str] # openocd probe pin, verbatim (e.g. "0x2e8a 0x000c") + verify: NotRequired[bool] # openocd read-back verify opt-out (WCH) class AttachedDevCfg(TypedDict, total=False): @@ -173,10 +191,6 @@ class TestsCfg(TypedDict, total=False): dev_attached: list[AttachedDevCfg] -class BuildCfg(TypedDict, total=False): - args: list[str] - - class VariantCfg(TypedDict, total=False): name: str # build dir (cmake-build-<name>) and HIL report row flags: str # raw CFLAGS, e.g. "-DCFG_TUD_DWC2_DMA_ENABLE=1" @@ -188,17 +202,52 @@ class Board(TypedDict): uid: str tests: TestsCfg flasher: FlasherCfg - build: NotRequired[BuildCfg] + # every build knob lives here, including a board's always-on defines: a board that + # needs one carries a single variant named after itself (metro_m4_express / + # MAX3421_HOST=1), which is exactly what the `or [...]` default below synthesises variant: NotRequired[list[VariantCfg]] + logger: NotRequired[str] # "rtt": console = the debug probe's RTT channel 0, not a VCOM (rtt skill) toolchain: NotRequired[str] # CI build bucket override, e.g. "riscv-gcc" (consumed by hil_ci_set_matrix.py) class HilConfig(TypedDict): boards: list[Board] -POOL_TIMEOUT = int(os.getenv('HIL_POOL_TIMEOUT', '4200')) # usbtest batteries are serialized fleet-wide, lengthening the tail -SERIAL_READ_TIMEOUT = float(os.getenv('HIL_SERIAL_READ_TIMEOUT', '5')) -SERIAL_WRITE_TIMEOUT = float(os.getenv('HIL_SERIAL_WRITE_TIMEOUT', '10')) +# Below the CI job ceilings so THIS guard fires first and still writes a report, well +# above a healthy fleet run (~14 min measured), and deliberately generous: firing early +# abandons boards that were still in flight (30 min fired on 5 of the last 8 HIL jobs), +# while firing late costs minutes on an already-wedged run. The drain keeps whatever had +# already finished either way. +POOL_TIMEOUT = hil_util.pos_int_env('HIL_POOL_TIMEOUT', 3600) + + +# The post-hang recovery reserve is PER BOARD and lives in usbtest.recovery_reserve(), +# derived from the ladder that file itself declares. Reserved whole, which is what lets the +# child run the ladder straight through instead of asking "does the next step still fit?" +# before each step. It only ELAPSES when cases actually time out; a healthy battery returns +# in ~200s and never touches it. + +# How long usbtest.py may keep starting new cases (--budget). The outer run_cmd timeout is +# always this PLUS the overshoot PLUS the recovery reserve when one can run, never a +# separate literal, or lowering one eats the room the other needs. 0 is refused (usbtest.py +# reads it as "no limit"); the margin over a healthy battery (~200s) keeps contention from +# becoming BUDGET entries. +USBTEST_BATTERY_BUDGET = hil_util.pos_int_env('HIL_USBTEST_BATTERY_BUDGET', 260) + +# The battery checks its budget BEFORE dispatching a case, so it can overshoot by one +# already-started case. Our outer kill must sit ABOVE that or we SIGKILL the battery just +# as it goes to print its JSON, turning ~29 real per-case verdicts into "usbtest did not +# run" and re-paying the whole battery on retry. +# Worst case, from usbtest.py: --timeout 60 (the case) + 5s post-SIGKILL reap + +# dmesg_tail(), bounded by HELPER_TIMEOUT=30 and run on BOTH the FAIL and HUNG timeout +# paths = 95s. 120 leaves a margin. Re-derive it if any of those three moves -- dmesg_tail +# is the one easily missed, and without it the estimate lands 20s short. +USBTEST_OVERSHOOT = 120 +# Named, not a literal, so the unit tests can zero it: every test that drives +# test_device_usbtest against a fake rig otherwise pays a real 3s (ten of them, 30s a run). +USBTEST_SETTLE = 3 +SERIAL_READ_TIMEOUT = hil_util.pos_float_env('HIL_SERIAL_READ_TIMEOUT', 5) +SERIAL_WRITE_TIMEOUT = hil_util.pos_float_env('HIL_SERIAL_WRITE_TIMEOUT', 10) MSC_README_TXT = \ @@ -206,7 +255,6 @@ b"This is tinyusb's MassStorage Class demo.\r\n\r\n\ If you find any bugs or get any questions, feel free to file an\r\n\ issue at github.com/hathach/tinyusb" -# get usb disk by id def get_disk_dev(id, vendor_str, lun): return f'/dev/disk/by-id/usb-{vendor_str}_Mass_Storage_{id}-0:{lun}' @@ -234,8 +282,7 @@ def open_serial_dev(port: str): while timeout > 0: if os.path.exists(port): try: - # write_timeout: a wedged device otherwise blocks ser.write() forever, - # hanging the worker until the pool/job timeout kills the whole run + # write_timeout: see serial_write_all ser = serial.Serial(port, baudrate=115200, timeout=SERIAL_READ_TIMEOUT, write_timeout=SERIAL_WRITE_TIMEOUT) break @@ -250,19 +297,155 @@ def open_serial_dev(port: str): return ser +def open_board_console(board: Board): + """The board's log console: its probe's VCOM, or RTT when the probe has none. + + Both ends expose the same read/in_waiting/write/close surface, so the tests read one + the same way they read the other.""" + if board.get('logger') == 'rtt': + # JlinkRtt speaks JLinkExe only; an openocd/stlink flasher would yield + # `-device ''` and fail 15 s later with a misleading port error. The OpenOCD + # RTT route is validated manually on native probes but has no harness backend + # yet (rtt skill; followup doc) — and never point it at ea4088's LPC-Link2 + # (measured: knocks that probe off USB; other J-Link-OB probes untested) + assert board['flasher']['name'].lower() == 'jlink', \ + f'{board["name"]}: "logger": "rtt" needs a jlink flasher, not {board["flasher"]["name"]}' + return hil_util.JlinkRtt(board) + ser = open_serial_dev(hil_util.get_serial_dev(board['flasher']["uid"], None, None, 0)) + ser.timeout = 0.1 + return ser + + def serial_write_all(ser: serial.Serial, data: bytes): - # write_timeout is a total deadline for the whole call (pyserial keeps partial progress - # internally). A timeout means the device stopped draining — treat it as fatal: pyserial - # loses the partial-write count on raise, so retrying would duplicate bytes on the wire. + # write_timeout is a deadline for the whole call. A timeout means the device stopped + # draining, and it is fatal: pyserial loses the partial-write count on raise, so + # retrying would duplicate bytes on the wire. try: ser.write(data) except serial.SerialTimeoutException: raise AssertionError(f'Serial write timeout after {SERIAL_WRITE_TIMEOUT:.1f}s') + except hil_util.RttError as e: + # the RTT console's failure contract (stall/closed/peer death): same + # drain-stopped meaning as the serial timeout -- a test failure, not a harness + # crash. Deliberately NOT bare RuntimeError: NotImplementedError and CPython's + # own 'dictionary changed size during iteration' are RuntimeErrors too, and a + # harness bug must not be reported as this board misbehaving. + raise AssertionError(f'Console write failed: {e}') + + +# J-Link Commander's telnet greeting: never target output (defined with the console +# in tools/rtt.py; hil_pool_check strips it through the same object) +RTT_BANNER_RE = hil_util.RTT_BANNER_RE + +LP_OPEN_TIMEOUT = 5 # bound on opening the printer lp node; see test_device_printer_to_cdc +# Runs under hil_util.run_alongside as `python3 -c`. Inline rather than a file so hil_ci.sh's +# staging list does not need another entry to keep the rig working. +LP_READER = ( + 'import os, sys\n' + 'fd = os.open(sys.argv[1], os.O_RDONLY)\n' + # readiness marker: the parent must not send a byte before the node is open, or the + # bytes are lost. A blind sleep raced CPython start-up on a loaded rig. + 'open(sys.argv[3], "w").close()\n' + 'want = int(sys.argv[2])\n' + 'buf = b""\n' + 'while len(buf) < want:\n' + ' chunk = os.read(fd, min(64, want - len(buf)))\n' + ' if not chunk:\n' + ' break\n' + ' buf += chunk\n' + 'sys.stdout.buffer.write(buf)\n' +) +# Runs under hil_util.run_cmd as `python3 -c`, argv so the body needs no shell quoting. +# A PROCESS, not a thread, and not optional: cython-hidapi wraps hid_enumerate in +# `with nogil` but calls hid_open and hid_close BARE (hidapi 0.15.0 hid.pyx), so those hold +# the GIL for their whole blocking call. A daemon thread cannot bound that -- the waiter +# parks off-GIL but must reacquire the GIL to return, which the stuck thread never yields +# -- so an in-process bound is inert exactly where it is needed, and the whole worker +# freezes rather than just the call. killpg reaches a child regardless. +# +# What blocks: hidapi's hidraw backend reads `manufacturer` and `product` via udev for each +# device that reaches create_device_info_for_device, via copy_udev_string(usb_dev, +# "manufacturer"/"product") -- both usb_string_attr, served under the device lock a wedged +# usbfs ioctl holds (v6.12.96 sysfs.c:141-143). +# +# Passing BOTH ids is what keeps a wedged peer out of that path, and it does more than skip +# non-matches: hidapi only runs the cheap pre-check `if (vendor_id != 0 || product_id != 0)` +# (0.15.0 linux/hid.c:962), so an unfiltered walk sends EVERY device straight to the locked +# reads. The pre-check itself is free -- parse_hid_vid_pid_from_sysfs parses +# <sysfs_path>/device/uevent (:532) -- and both `continue`s precede +# create_device_info_for_device (:966-970 before :976). Six examples in this tree expose a +# HID interface under VID cafe, so a VID-only walk would stall on any of them wedged on a +# peer. hid_open passes the same ids through to hid_enumerate internally (:1030), so the +# filter narrows that walk too -- but a peer running THIS example still matches both ids, +# which is why the child process, not the filter, is what bounds this. +HID_ECHO = r""" +import hid, random, sys, time + +uid, budget, want_pid = sys.argv[1], float(sys.argv[2]), int(sys.argv[3], 16) +deadline = time.monotonic() + budget + +dev = None +while dev is None: + for d in hid.enumerate(0xCafe, want_pid): + if d["serial_number"] == uid: + dev = d + break + if dev is not None or time.monotonic() >= deadline: + break + time.sleep(1) +if dev is None: + sys.exit(f"HID device not found for {uid}") + +h = hid.device() +h.open(dev["vendor_id"], dev["product_id"], uid) +try: + for size in (8, 32, 63): + # Report ID (0) + payload, padded to 64 bytes + payload = bytes(random.randint(1, 255) for _ in range(size)) + h.write(bytes([0]) + payload + bytes(64 - size)) + echo = h.read(64, 2000) + if not echo or len(echo) < size: + sys.exit(f"HID echo timeout or short read ({size} bytes)") + if bytes(echo[:size]) != payload: + sys.exit(f"HID echo wrong data ({size} bytes): " + f"sent {payload.hex()} received {bytes(echo[:size]).hex()}") +finally: + h.close() +""" +# The write half, same shape and same reason: usblp_open() ignores O_NONBLOCK and stalls in +# usb_autopm_get_interface() on a wedged device, holding the driver-global usblp_mutex. A +# blocked THREAD cannot be abandoned without keeping the fd, and usblp allows a single opener +# (v6.12.96 usblp.c), so the next open of this node returns -EBUSY for the life of the worker. +# A killed process takes its fd with it. O_NONBLOCK is kept because usblp DOES honour it on +# write, which is what the select()/partial-write loop below relies on. +LP_WRITER = ( + 'import os, random, select, sys\n' + 'lp, payload_path, ready = sys.argv[1], sys.argv[2], sys.argv[3]\n' + 'data = open(payload_path, "rb").read()\n' + 'fd = os.open(lp, os.O_WRONLY | os.O_NONBLOCK)\n' + # readiness marker, as in LP_READER: the parent must not read CDC before the node is open + 'open(ready, "w").close()\n' + 'off = 0\n' + 'while off < len(data):\n' + ' n = min(random.randint(1, 64), len(data) - off)\n' + ' buf, w = data[off:off + n], 0\n' + ' while w < len(buf):\n' + ' _, wr, _ = select.select([], [fd], [], 5.0)\n' + ' if not wr:\n' + ' sys.exit("printer write timeout (firmware not draining OUT endpoint)")\n' + ' w += os.write(fd, buf[w:])\n' + ' off += n\n' +) +MTYPE_TIMEOUT = 30 # a README-sized read is <1 s; bounds a D-state hang on a wedged device def read_disk_file(uid: str, lun: int, fname: str) -> bytes: - # Reads a file from a FAT volume on a block device without mounting it. - # Requires mtools: `apt install mtools` (no pip dependency). + # Reads a file from an unmounted FAT volume; needs mtools. run_cmd everywhere in this + # file rather than subprocess.run/check_output: its post-timeout reap is an unbounded + # communicate() with no killpg (CPython 3.13.5 subprocess.py:558-565 -- kill(), then + # communicate() with NO timeout), which never returns on a device wedged in D state, + # where the kill is queued and never delivered. binary + # keeps the bytes exact, split_stderr keeps mtype warnings out of them. dev = get_disk_dev(uid, 'TinyUSB', lun) last_err = None @@ -270,38 +453,34 @@ def read_disk_file(uid: str, lun: int, fname: str) -> bytes: nonlocal last_err if not os.path.exists(dev): return None - try: - data = subprocess.check_output( - ['mtype', '-i', dev, f'::/{fname}'], stderr=subprocess.PIPE) - assert data, f'Cannot read file {fname} from {dev}' - return data - except subprocess.CalledProcessError as e: - last_err = e.stderr.decode(errors='replace').strip() - return None + r = hil_util.run_cmd(f"mtype -i {shlex.quote(dev)} ::/{shlex.quote(fname)}", + timeout=MTYPE_TIMEOUT, binary=True, split_stderr=True, quiet=True) + if r.returncode == 0: + if r.stdout: + return r.stdout + # rc 0 with no data is an answer (empty file, zeroed sectors), not "not + # ready" — fail now instead of spinning the budget + raise AssertionError(f'Cannot read file {fname} from {dev}: mtype returned no data') + last_err = (r.stderr or b'').decode(errors='replace').strip() or f'mtype rc {r.returncode}' + return None data = wait_until(try_read) if data is None: - raise AssertionError(f'mtype failed on {dev}: {last_err}' if last_err else f'Storage {dev} not existed') + raise AssertionError(f'Cannot read file {fname} from {dev}: {last_err}' if last_err + else f'Storage {dev} not existed') return data -def open_mtp_dev(uid): - mtp = MTP() - - def try_open(): - # unmount gio/gvfs MTP mount which blocks libmtp from accessing the device - subprocess.run(f"gio mount -u mtp://TinyUsb_TinyUsb_Device_{uid}/", - shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - for raw in mtp.detect_devices(): - mtp.device = mtp.mtp.LIBMTP_Open_Raw_Device(ctypes.byref(raw)) - if mtp.device: - sn = mtp.get_serialnumber().decode('utf-8') - if sn == uid: - return mtp - mtp.disconnect() - return None - - return wait_until(try_open) +# ~5 KB of transfers plus libmtp setup takes seconds, not minutes; a larger value makes a +# wedged MTP board cost that much on every retry, all charged to the pool guard. +MTP_SESSION_MARGIN = 30 # transfer budget after enumeration; past it the session is killed +# room past the child's OWN enumeration budget for the echo exchange (3 x write + a 2000ms +# hidapi read) and interpreter start-up, so the outer kill only fires on a real stall +HID_ECHO_MARGIN = 30 +# hid_generic_inout's own idProduct. Pinned against the example's descriptor by +# HidEchoRunsInAChild.test_the_pid_matches_the_example, because a silent drift here would +# widen the walk back to every cafe: HID device without failing anything. +HID_INOUT_PID = 0x4012 def get_printer_dev(id: str, vendor_str, product_str, ifnum: int): @@ -310,10 +489,12 @@ def get_printer_dev(id: str, vendor_str, product_str, ifnum: int): product_str = product_str.replace(' ', '_') if product_str else '' for lp in glob.glob('/sys/class/usbmisc/lp*'): try: - sn = open(f'{lp}/device/../serial').read().strip() + sn = hil_util.read_sysfs(f'{lp}/device/../serial') + if sn is None: + continue if sn == id: return f'/dev/usb/{os.path.basename(lp)}' - except (FileNotFoundError, PermissionError, ValueError): + except OSError: # read_sysfs swallows its own OSError/ValueError; glob can race pass return None @@ -325,7 +506,8 @@ def open_printer_dev(id: str, vendor_str, product_str, ifnum: int) -> str: return lp_dev if lp_dev and os.path.exists(lp_dev) else None lp_dev = wait_until(try_find) - assert lp_dev, f'Printer device not found for {id} if{ifnum:02d}' + assert lp_dev, (f'Printer device not found for {id} if{ifnum:02d}' + + hil_util.strand_note()) return lp_dev @@ -335,18 +517,16 @@ def open_printer_dev(id: str, vendor_str, product_str, ifnum: int) -> str: def test_dual_host_info_to_device_cdc(board): uid = board['uid'] declared_devs = [f'{d["vid_pid"]}_{d["serial"]}' for d in board['tests']['dev_attached']] - port = hil_flash.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0) + port = hil_util.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0) ser = open_serial_dev(port) ser.timeout = 0.1 - # read until all expected devices are enumerated data = b'' timeout = enum_timeout() while timeout > 0: new_data = ser.read(ser.in_waiting or 1) if new_data: data += new_data - # check if all devices found enum_dev_sn = [] for l in data.decode('utf-8', errors='ignore').splitlines(): vid_pid_sn = re.search(r'ID ([0-9a-fA-F]+):([0-9a-fA-F]+) SN (\w+)', l) @@ -383,36 +563,53 @@ def test_host_device_info(board): flasher = board['flasher'] declared_devs = [f'{d["vid_pid"]}_{d["serial"]}' for d in board['tests']['dev_attached']] - port = hil_flash.get_serial_dev(flasher["uid"], None, None, 0) - ser = open_serial_dev(port) - ser.timeout = 0.1 - - # reset device since we can miss the first line - ret = getattr(hil_flash, f'reset_{flasher["name"].lower()}')(board) - assert ret.returncode == 0, 'Failed to reset device' + if board.get('logger') == 'rtt': + # The RTT console owns the probe, so reset BEFORE opening it (Commander then + # delivers the buffered boot burst). Unconditional, not only under --skip-flash: + # a previous run's console drained the ring, and the enumeration lines print + # only once — without this a re-run on unchanged firmware reads an empty ring. + ret = getattr(hil_flash, f'reset_{flasher["name"].lower()}')(board) + assert ret.returncode == 0, 'Failed to reset device' + ser = open_board_console(board) + try: + if board.get('logger') != 'rtt': + # reset device since we can miss the first line; on the VCOM the console + # survives the reset, so resetting after open catches the boot banner. + ret = getattr(hil_flash, f'reset_{flasher["name"].lower()}')(board) + assert ret.returncode == 0, 'Failed to reset device' - # read until all expected devices are enumerated - data = b'' - timeout = enum_timeout() - while timeout > 0: - new_data = ser.read(ser.in_waiting or 1) - if new_data: - data += new_data - # check if all devices found - enum_dev_sn = [] - for l in data.decode('utf-8', errors='ignore').splitlines(): - vid_pid_sn = re.search(r'ID ([0-9a-fA-F]+):([0-9a-fA-F]+) SN (\w+)', l) - if vid_pid_sn: - enum_dev_sn.append(f'{vid_pid_sn.group(1)}_{vid_pid_sn.group(2)}_{vid_pid_sn.group(3)}') - if set(declared_devs).issubset(set(enum_dev_sn)): - break - time.sleep(0.1) - timeout -= 0.1 - ser.close() + data = b'' + timeout = enum_timeout() + while timeout > 0: + # infra death is not a board failure: without this a dead JLinkExe/probe + # would burn the whole timeout and report as 'No data from device' + assert not getattr(ser, 'eof', False), \ + 'RTT console died (its server exited or the probe dropped off USB)' + new_data = ser.read(ser.in_waiting or 1) + if new_data: + data += new_data + enum_dev_sn = [] + for l in data.decode('utf-8', errors='ignore').splitlines(): + vid_pid_sn = re.search(r'ID ([0-9a-fA-F]+):([0-9a-fA-F]+) SN (\w+)', l) + if vid_pid_sn: + enum_dev_sn.append(f'{vid_pid_sn.group(1)}_{vid_pid_sn.group(2)}_{vid_pid_sn.group(3)}') + if set(declared_devs).issubset(set(enum_dev_sn)): + break + time.sleep(0.1) + timeout -= 0.1 + finally: + ser.close() - if len(data) == 0: - assert False, 'No data from device' lines = data.decode('utf-8', errors='ignore').splitlines() + if board.get('logger') == 'rtt': + # JLinkExe's telnet banner is delivered at connect, whether or not it ever + # finds the control block, so len(data) alone cannot tell "board said nothing" + # from "console never attached to the ring" -- drop the banner first + target_lines = hil_util.strip_banner(data).splitlines() + assert target_lines, ('No data from device: the RTT console attached but the target ' + 'produced nothing -- firmware built without LOGGER=rtt, or SWD lost') + elif len(data) == 0: + assert False, 'No data from device' enum_dev_sn = [] for l in lines: @@ -462,7 +659,7 @@ def test_host_cdc_msc_hid(board): if not cdc_devs and not msc_devs: return 'skipped' - port = hil_flash.get_serial_dev(flasher["uid"], None, None, 0) + port = hil_util.get_serial_dev(flasher["uid"], None, None, 0) ser = open_serial_dev(port) ser.timeout = 0.1 @@ -470,7 +667,6 @@ def test_host_cdc_msc_hid(board): ret = getattr(hil_flash, f'reset_{flasher["name"].lower()}')(board) assert ret.returncode == 0, 'Failed to reset device' - # Wait for all expected mount messages data = b'' timeout = enum_timeout() wait_cdc = len(cdc_devs) > 0 @@ -486,7 +682,6 @@ def test_host_cdc_msc_hid(board): time.sleep(0.1) timeout -= 0.1 - # Lookup serial chip name from vid_pid vid_pid_name = { '0403_6001': 'FTDI', '0403_6010': 'FTDI', '0403_6011': 'FTDI', '0403_6014': 'FTDI', '10c4_ea60': 'CP210x', '10c4_ea70': 'CP210x', @@ -497,7 +692,6 @@ def test_host_cdc_msc_hid(board): lines = data.decode('utf-8', errors='ignore').splitlines() - # Verify and print CDC mount if cdc_devs: assert b'CDC Interface is mounted' in data, 'CDC device not mounted on host' dev = cdc_devs[0] @@ -506,7 +700,6 @@ def test_host_cdc_msc_hid(board): if 'CDC Interface is mounted' in l: print(f'\r\n {chip_name}: {l} ', end='') - # Verify and print MSC mount (inquiry + disk size) if msc_devs: assert b'MassStorage device is mounted' in data, 'MSC device not mounted on host' assert b'Disk Size' in data, 'MSC Disk Size not reported' @@ -526,7 +719,6 @@ def test_host_cdc_msc_hid(board): packet_size = 64 - # Echo test: write random 1-packet_size chunks, wait for echo before sending next echo_len = 1024 echo_data = rand_ascii(echo_len) ser.reset_input_buffer() @@ -534,7 +726,6 @@ def test_host_cdc_msc_hid(board): while offset < echo_len: chunk_size = min(random.randint(1, packet_size), echo_len - offset) serial_write_all(ser, echo_data[offset:offset + chunk_size]) - # wait until this chunk is echoed back echo = b'' t_end = time.monotonic() + 1.0 while time.monotonic() < t_end and len(echo) < chunk_size: @@ -555,7 +746,7 @@ def test_host_msc_file_explorer(board): if not msc_devs: return 'skipped' - port = hil_flash.get_serial_dev(flasher["uid"], None, None, 0) + port = hil_util.get_serial_dev(flasher["uid"], None, None, 0) ser = open_serial_dev(port) ser.timeout = 0.1 @@ -563,7 +754,6 @@ def test_host_msc_file_explorer(board): ret = getattr(hil_flash, f'reset_{flasher["name"].lower()}')(board) assert ret.returncode == 0, 'Failed to reset device' - # Wait for MSC mount (Disk Size message) data = b'' timeout = enum_timeout() while timeout > 0: @@ -600,14 +790,12 @@ def test_host_msc_file_explorer(board): if MSC_README_TXT.decode() in resp_text: print('README.TXT matched ', end='') - # MSC throughput test: send dd command to read sectors time.sleep(0.5) ser.reset_input_buffer() for ch in 'dd 1024\r': serial_write_all(ser, ch.encode()) time.sleep(0.002) - # Read dd output until prompt resp = b'' t = 30.0 while t > 0: @@ -642,15 +830,14 @@ def test_host_msc_file_explorer_freertos(board): # Tests: device # ------------------------------------------------------------- def test_device_board_test(board): - # Dummy test pass def test_device_cdc_dual_ports(board): uid = board['uid'] port = [ - hil_flash.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0), - hil_flash.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 2) + hil_util.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0), + hil_util.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 2) ] ser = [open_serial_dev(p) for p in port] @@ -689,7 +876,7 @@ def test_device_cdc_dual_ports(board): def test_device_cdc_msc(board): uid = board['uid'] # CDC Echo test - port = hil_flash.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0) + port = hil_util.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0) ser = open_serial_dev(port) def rand_ascii(length): @@ -718,6 +905,20 @@ def test_device_cdc_msc_freertos(board): test_device_cdc_msc(board) +def link_is_fs(speed) -> bool: + """Payload scaling from a `speed` attribute. Anything not positively read as high + speed counts as FS, None included: the FS payload merely tests an HS board less, while + the HS payload hard-fails a healthy FS board.""" + return speed not in ('480', '5000', '10000') + + +def dd_timeout(mib: float) -> int: + """Bound one dd by what was ASKED for: 2.5 s/MiB is the slowest rate this test has + measured (FS CDC, ~420 kB/s), over a 30 s floor. A flat bound fails a healthy board as + soon as the payload grows or the leaf-hub uplink is shared.""" + return int(30 + 2.5 * mib) + + def test_device_cdc_msc_throughput(board): uid = board['uid'] @@ -728,7 +929,6 @@ def test_device_cdc_msc_throughput(board): return f'{float(m.group(1)):.1f} {m.group(2)}ps' return '?' - # Wait for MSC disk enumeration dev = get_disk_dev(uid, 'TinyUSB', 0) timeout = enum_timeout() while timeout > 0: @@ -737,8 +937,7 @@ def test_device_cdc_msc_throughput(board): time.sleep(0.1); timeout -= 0.1 assert timeout > 0, f'Disk {dev} not found' - # Wait for CDC tty enumeration - tty = hil_flash.get_serial_dev(uid, 'TinyUSB', 'Throughput', 0) + tty = hil_util.get_serial_dev(uid, 'TinyUSB', 'Throughput', 0) timeout = enum_timeout() while timeout > 0: if os.path.exists(tty): @@ -746,41 +945,47 @@ def test_device_cdc_msc_throughput(board): time.sleep(0.1); timeout -= 0.1 assert timeout > 0, f'CDC tty {tty} not found' - # Detect speed (12 Mbps FS / 480 Mbps HS) for payload scaling - is_fs = False - for f in glob.glob('/sys/bus/usb/devices/*/serial'): - try: - if open(f).read().strip().lower() == uid.lower(): - is_fs = (open(os.path.join(os.path.dirname(f), 'speed')).read().strip() == '12') - break - except (OSError, ValueError): - pass + # Detect speed (12 Mbps FS / 480 Mbps HS) for payload scaling; a device we never find + # keeps the FS payload (see link_is_fs) + # usb_scan, not a private glob: it skips root hubs and filters on the lock-free + # descriptor pair before touching `serial`. + is_fs = True + speed_known = False + devs = hil_util.usb_scan(vid='cafe', serial=uid) + if devs: + speed = hil_util.read_sysfs(os.path.join(devs[0]['dir'], 'speed')) + is_fs = link_is_fs(speed) + speed_known = speed is not None # Put tty in raw mode so dd sees pure binary throughput. - rs = hil_flash.run_cmd(f'timeout 30 stty -F {tty} raw -echo') - assert rs.returncode == 0, f'stty failed: {hil_flash.cmd_stdout_text(rs.stdout)}' + rs = hil_util.run_cmd(f'timeout 30 stty -F {tty} raw -echo') + assert rs.returncode == 0, f'stty failed: {hil_util.cmd_stdout_text(rs.stdout)}' # Payload aim: ~5 s per direction at FS (~830 kB/s), much less at HS. msc_count = 2 if is_fs else 16 # bs=1M cdc_count = 16 if is_fs else 128 # bs=64K tmp_file = f'/tmp/cdc_msc_tp_{uid}.bin' + t_cdc, t_msc = dd_timeout(cdc_count / 16), dd_timeout(msc_count) - rw = hil_flash.run_cmd(f'timeout 30 dd if=/dev/zero of={tty} bs=64K count={cdc_count} 2>&1') - assert rw.returncode == 0, f'CDC dd write failed: {hil_flash.cmd_stdout_text(rw.stdout)}' - cdc_w = parse_speed(hil_flash.cmd_stdout_text(rw.stdout)) + rw = hil_util.run_cmd(f'timeout {t_cdc} dd if=/dev/zero of={tty} bs=64K count={cdc_count} 2>&1') + assert rw.returncode == 0, f'CDC dd write failed: {hil_util.cmd_stdout_text(rw.stdout)}' + cdc_w = parse_speed(hil_util.cmd_stdout_text(rw.stdout)) - rr = hil_flash.run_cmd(f'timeout 30 dd if={tty} of=/dev/null bs=64K count={cdc_count} iflag=fullblock 2>&1') - assert rr.returncode == 0, f'CDC dd read failed: {hil_flash.cmd_stdout_text(rr.stdout)}' - cdc_r = parse_speed(hil_flash.cmd_stdout_text(rr.stdout)) + rr = hil_util.run_cmd(f'timeout {t_cdc} dd if={tty} of=/dev/null bs=64K count={cdc_count} iflag=fullblock 2>&1') + assert rr.returncode == 0, f'CDC dd read failed: {hil_util.cmd_stdout_text(rr.stdout)}' + cdc_r = parse_speed(hil_util.cmd_stdout_text(rr.stdout)) - rmr = hil_flash.run_cmd(f'dd if={dev} of={tmp_file} bs=1M count={msc_count} iflag=direct 2>&1') - assert rmr.returncode == 0, f'MSC dd read failed: {hil_flash.cmd_stdout_text(rmr.stdout)}' - msc_r = parse_speed(hil_flash.cmd_stdout_text(rmr.stdout)) + # inner bound, like the CDC pair above: run_cmd's SIGKILL is merely QUEUED against a + # dd blocked in the block layer on a half-dead device, so without one the call rides + # CMD_TIMEOUT and is abandoned holding the disk and usbfs nodes. + rmr = hil_util.run_cmd(f'timeout {t_msc} dd if={dev} of={tmp_file} bs=1M count={msc_count} iflag=direct 2>&1') + assert rmr.returncode == 0, f'MSC dd read failed: {hil_util.cmd_stdout_text(rmr.stdout)}' + msc_r = parse_speed(hil_util.cmd_stdout_text(rmr.stdout)) - rmw = hil_flash.run_cmd(f'dd if={tmp_file} of={dev} bs=1M count={msc_count} oflag=direct 2>&1') - assert rmw.returncode == 0, f'MSC dd write failed: {hil_flash.cmd_stdout_text(rmw.stdout)}' - msc_w = parse_speed(hil_flash.cmd_stdout_text(rmw.stdout)) + rmw = hil_util.run_cmd(f'timeout {t_msc} dd if={tmp_file} of={dev} bs=1M count={msc_count} oflag=direct 2>&1') + assert rmw.returncode == 0, f'MSC dd write failed: {hil_util.cmd_stdout_text(rmw.stdout)}' + msc_w = parse_speed(hil_util.cmd_stdout_text(rmw.stdout)) try: os.remove(tmp_file) @@ -789,8 +994,7 @@ def test_device_cdc_msc_throughput(board): print(f' CDC read {cdc_r} write {cdc_w}, MSC read {msc_r} write {msc_w} ', end='') - # compact read/write speeds for the report cell, e.g. "✅ C 652/422k M 1.1M/783k" - # (C=CDC, M=MSC; the unit is shown once when both sides share it) + # report cell, e.g. "✅ C 652/422k M 1.1M/783k" (C=CDC, M=MSC; shared unit shown once) def short(s): return (s.split()[0].rstrip('0').rstrip('.') + s.split()[-1][0]) if ' ' in s else s @@ -800,20 +1004,29 @@ def test_device_cdc_msc_throughput(board): r = r[:-1] return f'{r}/{w}' - return f'{REPORT_CELL["pass"]} C {pair(cdc_r, cdc_w)} M {pair(msc_r, msc_w)}' + # 'FS?' when the speed could not be read: the numbers below were produced against the FS + # payload, so an HS board reads as suspiciously slow. Say so rather than publish a green + # cell whose scale is a guess. + scale = '' if speed_known else ' FS?' + return f'{hil_report.REPORT_CELL["pass"]} C {pair(cdc_r, cdc_w)} M {pair(msc_r, msc_w)}{scale}' def test_device_dfu(board): uid = board['uid'] + vid_pid = 'cafe:400b' - # Wait device enum. Deadline-based: dfu-util -l itself takes ~1 s per call, which a - # per-iteration countdown would not charge against the budget. + # Deadline-based: dfu-util takes ~1 s per call, which a countdown would not charge + # against the budget. -d pins enumeration to THIS example's ids: a bare `-l` opens every + # DFU-capable node, and one wedged node blocks that open in D state. The pair is doubled + # because dfu-util matches run-time and DFU-mode devices against SEPARATE id pairs + # (parse_vendprod: an omitted DFU-mode pair matches ANY DFU-mode device). The deadline + # is only tested BETWEEN calls, so the per-call bound is what caps a blocked open. deadline = time.monotonic() + enum_timeout() found = False while time.monotonic() < deadline: - ret = hil_flash.run_cmd(f'dfu-util -l') - stdout = hil_flash.cmd_stdout_text(ret.stdout) - if f'serial="{uid}"' in stdout and 'Found DFU: [cafe:400b]' in stdout: + ret = hil_util.run_cmd(f'dfu-util -d {vid_pid},{vid_pid} -l', timeout=15) + stdout = hil_util.cmd_stdout_text(ret.stdout) + if f'serial="{uid}"' in stdout and f'Found DFU: [{vid_pid}]' in stdout: found = True break time.sleep(1) @@ -823,17 +1036,23 @@ def test_device_dfu(board): f_dfu0 = f'dfu0_{uid}' f_dfu1 = f'dfu1_{uid}' - # Test upload try: os.remove(f_dfu0) os.remove(f_dfu1) except OSError: pass - ret = hil_flash.run_cmd(f'dfu-util -S {uid} -a 0 -U {f_dfu0}') + # -d as well as -S: dfu-util matches the SERIAL only after libusb_open() (dfu_util.c + # probes the descriptor for iSerialNumber), so -S alone still opens every DFU-capable + # node. The id filter runs BEFORE the open; -S then picks our board (see the poll). + # Each partition is one short string, so a healthy upload is ~1 s; the bound is there + # for a node that stops answering mid-transfer. + ret = hil_util.run_cmd(f'dfu-util -d {vid_pid},{vid_pid} -S {uid} -a 0 -U {f_dfu0}', + timeout=30) assert ret.returncode == 0, 'Upload failed' - ret = hil_flash.run_cmd(f'dfu-util -S {uid} -a 1 -U {f_dfu1}') + ret = hil_util.run_cmd(f'dfu-util -d {vid_pid},{vid_pid} -S {uid} -a 1 -U {f_dfu1}', + timeout=30) assert ret.returncode == 0, 'Upload failed' with open(f_dfu0) as f: @@ -848,13 +1067,14 @@ def test_device_dfu(board): def test_device_dfu_runtime(board): uid = board['uid'] - # Wait device enum (deadline-based, see test_device_dfu) + vid_pid = 'cafe:400c' + # enumeration pinned to this example's ids, same per-call bound (see test_device_dfu) deadline = time.monotonic() + enum_timeout() found = False while time.monotonic() < deadline: - ret = hil_flash.run_cmd(f'dfu-util -l') - stdout = hil_flash.cmd_stdout_text(ret.stdout) - if f'serial="{uid}"' in stdout and 'Found Runtime: [cafe:400c]' in stdout: + ret = hil_util.run_cmd(f'dfu-util -d {vid_pid},{vid_pid} -l', timeout=15) + stdout = hil_util.cmd_stdout_text(ret.stdout) + if f'serial="{uid}"' in stdout and f'Found Runtime: [{vid_pid}]' in stdout: found = True break time.sleep(1) @@ -867,7 +1087,6 @@ def test_device_hid_boot_interface(board): kbd = get_hid_dev(uid, 'TinyUSB', 'TinyUSB_Device', 'event-kbd') mouse1 = get_hid_dev(uid, 'TinyUSB', 'TinyUSB_Device', 'if01-event-mouse') mouse2 = get_hid_dev(uid, 'TinyUSB', 'TinyUSB_Device', 'if01-mouse') - # Wait device enum timeout = enum_timeout() while timeout > 0: if os.path.exists(kbd) and os.path.exists(mouse1) and os.path.exists(mouse2): @@ -884,12 +1103,9 @@ def test_device_hid_composite_freertos(id): def test_device_printer_to_cdc(board): - import threading - uid = board['uid'] - # Wait for CDC port and printer device - cdc_port = hil_flash.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0) + cdc_port = hil_util.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0) ser = open_serial_dev(cdc_port) lp_dev = open_printer_dev(uid, 'TinyUSB', 'TinyUSB_Device', 2) @@ -909,162 +1125,172 @@ def test_device_printer_to_cdc(board): sizes = [32, 64, 128, 256, 512, random.randint(2000, 5000)] - # flush any stale data ser.reset_input_buffer() # Test 1: Printer -> CDC with multiple sizes, write in random 1-64 byte chunks - LP_WRITE_TIMEOUT = 5.0 # seconds; firmware may stall draining the printer OUT endpoint + # The write runs in a PROCESS for the same reason the read below does: see LP_WRITER. for size in sizes: test_data = rand_ascii(size) ser.reset_input_buffer() - rd = b'' - offset = 0 - lp_fd = os.open(lp_dev, os.O_WRONLY | os.O_NONBLOCK) + rd = bytearray() + + payload = Path(tempfile.gettempdir()) / f'hil-lp-tx-{os.getpid()}-{size}' + ready = Path(tempfile.gettempdir()) / f'hil-lp-ready-{os.getpid()}-{size}' + payload.write_bytes(test_data) + ready.unlink(missing_ok=True) + # +5 like write_cdc's sibling wait below: the bound is on the OPEN, and the child + # must first fork, exec and boot CPython, which on a loaded rig routinely exceeds + # LP_OPEN_TIMEOUT on its own. A tighter wait here reports a slow interpreter start + # as a wedged node. + open_deadline = time.monotonic() + LP_OPEN_TIMEOUT + 5 + saw_ready = False + + def read_cdc(): + # WAIT for the writer to have the node open, as Test 2's write_cdc does: the + # child has to fork, exec and boot CPython, and reading before it starts just + # burns the serial timeout. + # ONE deadline, shared with the child's bound below. Two different ones let + # the writer open after the parent gave up: it writes the whole payload with + # nobody reading, exits 0, and the byte-compare reports FIRMWARE DATA + # CORRUPTION for a board whose only problem was a slow open. + nonlocal saw_ready + while not ready.exists(): + if time.monotonic() > open_deadline: + return # never opened; the assert below reports THAT, not data + time.sleep(0.02) + saw_ready = True + # fullspeed devices may need extra time; ser.read is bounded by + # SERIAL_READ_TIMEOUT, so an empty return means the stream went quiet + while len(rd) < size: + chunk = ser.read(size - len(rd)) + if not chunk: + break + rd.extend(chunk) # in place: `rd +=` would rebind it as a local + try: - while offset < size: - chunk_size = min(random.randint(1, 64), size - offset) - buf = test_data[offset:offset + chunk_size] - written = 0 - while written < len(buf): - _, wr, _ = select.select([], [lp_fd], [], LP_WRITE_TIMEOUT) - assert wr, f'Printer write timeout after {LP_WRITE_TIMEOUT}s (firmware not draining OUT endpoint)' - n = os.write(lp_fd, buf[written:]) - written += n - rd += ser.read(chunk_size) - offset += chunk_size + r = hil_util.run_alongside( + [sys.executable, '-c', LP_WRITER, lp_dev, str(payload), str(ready)], + read_cdc, LP_OPEN_TIMEOUT + 12) finally: - os.close(lp_fd) - # read any remaining bytes (fullspeed devices may need extra time) - while len(rd) < size: - remaining = ser.read(size - len(rd)) - if not remaining: - break - rd += remaining - assert rd == test_data, (f'Printer->CDC wrong data ({size} bytes):\n' - f' expected: {test_data[:64]}\n received: {rd[:64]}') + ready.unlink(missing_ok=True) + payload.unlink(missing_ok=True) + # rc 124 is run_alongside's kill, i.e. the open blocked -- and stderr is EMPTY + # there, so without the fallback the cell reads 'failed (32 bytes, rc 124):' and + # nothing, for the one failure this conversion exists to contain. An OSError is a + # FACT about the node (EBUSY from usblp's single-opener rule, ENOENT from a + # re-enumeration race) and must not send the operator to usb-kernel-recover. + # The bound covers the open AND the whole write, so rc 124 alone does not mean a + # wedged node. `ready` is written on the line after os.open() returns, so its + # ABSENCE is what says the open never completed -- the case that sends an operator + # to usb-kernel-recover. Anything else killed on the bound was a slow drain. + detail = hil_util.cmd_stdout_text(r.stderr).strip()[:200] + # FIRST: a child that exited on its OWN carries the concrete errno, and only one + # we KILLED (rc 124) can be diagnosed as an open that never completed. Asserting + # the marker before this reported EBUSY/ENOENT as a wedged node -- the conflation + # the comment above exists to prevent. rc is in the message because a child killed + # by a signal leaves `detail` empty. + assert r.returncode in (0, 124), ( + f'Printer->CDC writer failed ({size} bytes, rc {r.returncode}): {detail}') + # saw_ready, not ready.exists(): a marker that appeared AFTER read_cdc gave up + # means the child wrote with nobody reading, and the byte-compare below would call + # that firmware data corruption. Report the slow open instead. + assert saw_ready, (f'printer: {lp_dev} was not opened for write within ' + f'{LP_OPEN_TIMEOUT + 5}s (device wedged, or the writer never ' + f'started); rc {r.returncode}') + assert r.returncode == 0, ( + f'Printer->CDC writer killed on its bound after opening {lp_dev} ' + f'(rc {r.returncode}): the firmware stopped draining the OUT endpoint') + assert bytes(rd) == test_data, (f'Printer->CDC wrong data ({size} bytes):\n' + f' expected: {test_data[:64]}\n' + f' received: {bytes(rd)[:64]}') - # Test 2: CDC -> Printer with multiple sizes, write in random 1-64 byte chunks - # Use a thread to read from printer since /dev/usb/lp read blocks + # Test 2: CDC -> Printer with multiple sizes, write in random 1-64 byte chunks. + # The lp read runs in a PROCESS, not a thread: /dev/usb/lp* blocks on read, usblp + # allows a SINGLE opener, and a blocked thread cannot be abandoned without keeping + # that fd -- which poisoned the node for every later test this worker ran. A killed + # process takes its fd with it. ser.reset_input_buffer() time.sleep(0.5) for size in sizes: test_data = rand_ascii(size) - rd_result = [b'', None] # [data, error] - reader_ready = threading.Event() - - def lp_reader(): - try: - rd = b'' - fd = os.open(lp_dev, os.O_RDONLY) - reader_ready.set() - try: - while len(rd) < size: - chunk = os.read(fd, min(64, size - len(rd))) - if not chunk: - break - rd += chunk - finally: - os.close(fd) - rd_result[0] = rd - except Exception as e: - rd_result[1] = e - reader_ready.set() - reader = threading.Thread(target=lp_reader, daemon=True) - reader.start() - # wait for reader to open lp device before writing - reader_ready.wait(timeout=5) - time.sleep(0.1) + ready = Path(tempfile.gettempdir()) / f'hil-lp-ready-{os.getpid()}-{size}' + ready.unlink(missing_ok=True) - # Write to CDC in small chunks with flush to avoid overflowing device FIFO - offset = 0 - while offset < size: - chunk_size = min(random.randint(1, 64), size - offset) - serial_write_all(ser, test_data[offset:offset + chunk_size]) - time.sleep(0.01) - offset += chunk_size + def write_cdc(): + # WAIT for the reader to have the node open. The child has to fork, exec and + # boot a CPython interpreter; on a loaded rig that routinely exceeds the 0.3s + # this used to sleep, and every byte sent early is lost -- surfacing as a + # spurious data mismatch rather than a timeout. + deadline = time.monotonic() + LP_OPEN_TIMEOUT + 5 + while not ready.exists(): + if time.monotonic() > deadline: + return # reader never opened; the rc/compare below reports it + time.sleep(0.02) + offset = 0 + while offset < size: + chunk_size = min(random.randint(1, 64), size - offset) + serial_write_all(ser, test_data[offset:offset + chunk_size]) + time.sleep(0.01) + offset += chunk_size - reader.join(timeout=10) - assert not reader.is_alive(), f'CDC->Printer timeout ({size} bytes)' - assert rd_result[1] is None, f'CDC->Printer read error: {rd_result[1]}' - assert rd_result[0] == test_data, (f'CDC->Printer wrong data ({size} bytes):\n' - f' expected: {test_data[:64]}\n received: {rd_result[0][:64]}') + try: + r = hil_util.run_alongside( + [sys.executable, '-c', LP_READER, lp_dev, str(size), str(ready)], + write_cdc, LP_OPEN_TIMEOUT + 12) + finally: + ready.unlink(missing_ok=True) + # stderr, not stdout: run_alongside keeps the payload stream clean, so a traceback + # from the reader now arrives on its own pipe + # rc 124 is run_alongside's kill -- a blocked usblp_open leaves stderr EMPTY, so + # without the fallback this renders as 'failed (32 bytes, rc 124):' and nothing + rdetail = hil_util.cmd_stdout_text(r.stderr).strip()[:200] + assert r.returncode == 0, ( + f'CDC->Printer reader failed ({size} bytes): {rdetail}' if rdetail else + f'printer: reading {lp_dev} blocked (device wedged): the reader was killed on ' + f'its bound (rc {r.returncode})') + assert r.stdout == test_data, (f'CDC->Printer wrong data ({size} bytes):\n' + f' expected: {test_data[:64]}\n received: {r.stdout[:64]}') time.sleep(0.2) ser.close() def test_device_mtp(board): + # The whole session lives in mtp_test.py under run_cmd: libmtp calls are synchronous + # ctypes that block unkillably (D state) on a wedged device, so a disposable process is + # the only thing the harness can walk away from. uid = board['uid'] - - # --- BEFORE: mute C-level stderr for libmtp vid/pid warnings --- - fd = sys.stderr.fileno() - _saved = os.dup(fd) - _null = os.open(os.devnull, os.O_WRONLY) - os.dup2(_null, fd) - - mtp = open_mtp_dev(uid) - - # --- AFTER: restore stderr --- - os.dup2(_saved, fd) - os.close(_null) - os.close(_saved) - - if mtp is None or mtp.device is None: - assert False, 'MTP device not found' - - try: - assert b"TinyUSB" == mtp.get_manufacturer(), 'MTP wrong manufacturer' - assert b"MTP Example" == mtp.get_modelname(), 'MTP wrong model' - assert b'1.0' == mtp.get_deviceversion(), 'MTP wrong version' - assert b'TinyUSB MTP' == mtp.get_devicename(), 'MTP wrong device name' - - # read and compare readme.txt and logo.png - f1_expect = b'TinyUSB MTP Filesystem example' - f2_md5_expect = '40ef23fc2891018d41a05d4a0d5f822f' # md5sum of logo.png - f1 = uid.encode("utf-8") + b'_file1' - f2 = uid.encode("utf-8") + b'_file2' - f3 = uid.encode("utf-8") + b'_file3' - mtp.get_file_to_file(1, f1) - with open(f1, 'rb') as file: - f1_data = file.read() - os.remove(f1) - assert f1_data == f1_expect, 'MTP file1 wrong data' - mtp.get_file_to_file(2, f2) - with open(f2, 'rb') as file: - f2_data = file.read() - os.remove(f2) - assert f2_md5_expect == hashlib.md5(f2_data).hexdigest(), 'MTP file2 wrong data' - # test send file - with open(f3, "wb") as file: - f3_data = os.urandom(random.randint(1024, 3*1024)) - file.write(f3_data) - file.close() - fid = mtp.send_file_from_file(f3, b'file3') - f3_readback = f3 + b'_readback' - mtp.get_file_to_file(fid, f3_readback) - with open(f3_readback, 'rb') as f: - f3_rb_data = f.read() - os.remove(f3_readback) - assert f3_rb_data == f3_data, 'MTP file3 wrong data' - os.remove(f3) - mtp.delete_object(fid) - finally: - mtp.disconnect() + script = Path(__file__).resolve().parent / 'mtp_test.py' + # 2x, as master's in-process open_mtp_dev used: libmtp-runtime publishes + # /dev/libmtp-* only after its SYNCHRONOUS mtp-probe finishes, seconds on a freshly + # flashed FS board, and the gio unmount eats part of what is left before the first + # probe. Extracting the session into a subprocess halved this by accident (8s/4s), + # which fails healthy hardware on the retry. + t = 2 * enum_timeout() + r = hil_util.run_cmd( + f'{shlex.quote(sys.executable)} {shlex.quote(str(script))} --uid {shlex.quote(uid)} --timeout {t}', + timeout=t + MTP_SESSION_MARGIN) + if r.returncode == 124: + # "abandoned", not "killed": a session blocked in a usbfs ioctl (D state) never + # receives the SIGKILL -- it lingers until its device path clears, by design + raise AssertionError(f'MTP session wedged (abandoned after {t + MTP_SESSION_MARGIN}s; ' + f'the session process may linger unkillable in D state)') + assert r.returncode == 0, f'MTP session failed (rc {r.returncode}):\n{r.stdout}' def test_device_net_lwip_webserver(board): # MAC hard-coded in examples/device/net_lwip_webserver/src/main.c; Linux names the - # USB network interface enx<MAC_lowercase_no_colons>. Device IP is 192.168.7.1 and - # the example runs an iperf2 TCP server on port 5001 (INCLUDE_IPERF). + # iface enx<MAC_lowercase_no_colons>. Device IP 192.168.7.1, iperf2 TCP server on 5001 + # (INCLUDE_IPERF). import socket mac_no_colons = '0202846a9600' iface = 'enx' + mac_no_colons device_ip = '192.168.7.1' iperf_port = 5001 - # Wait for the host to get an IPv4 address in the device's subnet (DHCP served by the device). - # USB enum + DHCP serve can take longer on the CI HIL hardware than on local — give it 30s. + # Wait for an IPv4 address in the device's subnet (it serves DHCP); 30s because USB + # enum + DHCP serve is slower on the CI HIL hardware than locally. iface_timeout = 30 deadline = time.monotonic() + iface_timeout host_ip = None @@ -1078,8 +1304,7 @@ def test_device_net_lwip_webserver(board): time.sleep(0.5) assert host_ip, f'USB net iface {iface} did not come up with 192.168.7.x within {iface_timeout}s' - # Poll the iperf TCP port until the device is accepting. The net stack comes up a bit - # after DHCP completes; iperf server binding isn't instantaneous after reflash. + # Poll until the device accepts: the net stack and the iperf bind come up after DHCP. deadline = time.monotonic() + enum_timeout() last_err = None while time.monotonic() < deadline: @@ -1092,12 +1317,12 @@ def test_device_net_lwip_webserver(board): time.sleep(0.3) assert last_err is None, f'iperf TCP {device_ip}:{iperf_port} not accepting within {enum_timeout()}s: {last_err}' - # Throughput: 5-second iperf2 TCP test, CSV output for stable parsing. - # iperf2 CSV final summary line: timestamp,src_ip,src_port,dst_ip,dst_port,id,interval,bytes,bps - ret = subprocess.run(['iperf', '-c', device_ip, '-t', '5', '-y', 'C'], - capture_output=True, text=True, timeout=30) - stderr = ret.stderr.strip() - stdout = ret.stdout.strip() + # 5-second iperf2 TCP test; -y C for stable parsing (final summary line is + # timestamp,src_ip,src_port,dst_ip,dst_port,id,interval,bytes,bps). + ret = hil_util.run_cmd(f'iperf -c {device_ip} -t 5 -y C', + timeout=30, split_stderr=True, quiet=True) + stderr = (ret.stderr or '').strip() + stdout = (ret.stdout or '').strip() assert ret.returncode == 0, f'iperf rc={ret.returncode}: stderr={stderr!r} stdout={stdout!r}' lines = [l for l in stdout.splitlines() if l] assert lines, f'iperf produced no output (rc={ret.returncode}, stderr={stderr!r})' @@ -1108,19 +1333,16 @@ def test_device_net_lwip_webserver(board): mbps = bps / 1e6 print(f' iperf {mbps:5.1f} Mbps', end='') - # Reject implausibly low throughput - a working USB-net link should clear this easily. assert mbps >= 1.0, f'iperf throughput too low: {mbps:.2f} Mbps' def test_device_msc_dual_lun(board): uid = board['uid'] - # Read README from LUN 0 data0 = read_disk_file(uid, 0, 'README0.TXT') readme0 = b"LUN0: " + MSC_README_TXT assert data0 == readme0, f'MSC LUN0 wrong data in README0.TXT\n expected: {readme0}\n received: {data0}' - # Read README from LUN 1 data1 = read_disk_file(uid, 1, 'README1.TXT') readme1 = b"LUN1: " + MSC_README_TXT assert data1 == readme1, f'MSC LUN1 wrong data in README1.TXT\n expected: {readme1}\n received: {data1}' @@ -1129,7 +1351,6 @@ def test_device_msc_dual_lun(board): def test_device_midi_test(board): uid = board['uid'] - # Find MIDI device via /dev/snd/by-id using board UID timeout = enum_timeout() midi_port = None while timeout > 0: @@ -1147,31 +1368,40 @@ def test_device_midi_test(board): timeout -= 1 assert midi_port is not None, f'MIDI device not found for {uid}' - # Read MIDI messages and verify note on/off import select - with open(midi_port, 'rb') as f: - notes = [] + midi_fd = os.open(midi_port, os.O_RDONLY | os.O_NONBLOCK) + try: + data = bytearray() # Read for up to 3 seconds to capture a few notes (286ms interval) end_time = time.monotonic() + 3 - while time.monotonic() < end_time: - ready, _, _ = select.select([f], [], [], 0.5) - if ready: - data = f.read(64) - if data: - # Parse MIDI bytes: note_on = 0x90, note_off = 0x80 - i = 0 - while i + 2 < len(data): - status = data[i] - if (status & 0xF0) == 0x90: # Note On - notes.append(data[i + 1]) - i += 3 - elif (status & 0xF0) == 0x80: # Note Off - i += 3 - else: - i += 1 + while (remaining := end_time - time.monotonic()) > 0: + ready, _, _ = select.select([midi_fd], [], [], min(0.5, remaining)) + if not ready: + continue + try: + chunk = os.read(midi_fd, 64) + except BlockingIOError: + continue + if not chunk: + break + data.extend(chunk) + finally: + os.close(midi_fd) + + notes = [] + # Parse MIDI bytes: note_on = 0x90, note_off = 0x80 + i = 0 + while i + 2 < len(data): + status = data[i] + if (status & 0xF0) == 0x90: # Note On + notes.append(data[i + 1]) + i += 3 + elif (status & 0xF0) == 0x80: # Note Off + i += 3 + else: + i += 1 assert len(notes) >= 2, f'Expected at least 2 MIDI notes, got {len(notes)}' - # Verify notes are from the expected sequence note_sequence = [ 74, 78, 81, 86, 90, 93, 98, 102, 57, 61, 66, 69, 73, 78, 81, 85, 88, 92, 97, 100, 97, 92, 88, 85, 81, 78, 74, 69, 66, 62, 57, 62, @@ -1185,9 +1415,6 @@ def test_device_midi_test(board): def test_device_audio_test_freertos(board): uid = board['uid'] - if os.name == 'nt': - return 'skipped' - pcm = None timeout = enum_timeout() while timeout > 0: @@ -1212,8 +1439,11 @@ def test_device_audio_test_freertos(board): raw_path, ] - ret = subprocess.run(cmd, capture_output=True, text=True, timeout=20) - assert ret.returncode == 0, f'arecord failed: {ret.stderr.strip() or ret.stdout.strip()}' + # run_cmd: ALSA capture from a wedged device blocks in D state (see read_disk_file) + ret = hil_util.run_cmd(' '.join(shlex.quote(c) for c in cmd), + timeout=20, split_stderr=True, quiet=True) + assert ret.returncode == 0, \ + f'arecord failed: {(ret.stderr or "").strip() or (ret.stdout or "").strip()}' try: with open(raw_path, 'rb') as f: @@ -1231,121 +1461,213 @@ def test_device_audio_test_freertos(board): samples = [int.from_bytes(raw[i:i + 2], 'little', signed=False) for i in range(0, len(raw), 2)] assert sample_count > 1024, f'Not enough samples captured: {sample_count}' - # The firmware sends a continuous uint16 ramp. Using ALSA hw: capture bypasses - # PulseAudio processing, so most adjacent samples should differ by exactly 1. - total_diffs = sample_count - 1 - one_step = 0 - near_step = 0 - for i in range(total_diffs): - d = (samples[i + 1] - samples[i]) & 0xFFFF - if d == 1: - one_step += 1 - if d in (0, 1, 2, 47, 48, 49): - near_step += 1 + # The producer is already running while ALSA activates streaming, so the + # initial overwritable software FIFO (at most 224 samples) can transition + # between ramp generations. After that startup window, require an exact ramp. + startup_samples = 256 + for i in range(startup_samples, sample_count - 1): + expected = (samples[i] + 1) & 0xFFFF + assert samples[i + 1] == expected, ( + f'Audio mismatch at sample {i + 1}: expected {expected}, got {samples[i + 1]}') - one_ratio = one_step / total_diffs - near_ratio = near_step / total_diffs - assert one_ratio >= 0.85, f'Unexpected audio pattern (strict ratio={one_ratio:.3f})' - assert near_ratio >= 0.98, f'Unexpected audio pattern (relaxed ratio={near_ratio:.3f})' - - print(f' ALSA {pcm} strict={one_ratio:.3f} relaxed={near_ratio:.3f}', end='') + print(f' ALSA {pcm}', end='') def test_device_hid_generic_inout(board): + # The whole exchange runs in a child (see HID_ECHO): hidapi's blocking calls hold the + # GIL, so nothing in-process can bound them. run_cmd's killpg can. uid = board['uid'] - import hid # cython-hidapi (pip: hidapi, apt: python3-hid) - - # Find HID device by UID (VID=0xCafe) - timeout = enum_timeout() - dev = None - while timeout > 0: - for d in hid.enumerate(0xCafe): - if d['serial_number'] == uid: - dev = d - break - if dev: - break - time.sleep(1) - timeout -= 1 - assert dev is not None, f'HID device not found for {uid}' - - h = hid.device() - h.open(dev['vendor_id'], dev['product_id'], uid) - try: - # Echo test: send random data and verify echo - for size in [8, 32, 63]: - # Report ID (0) + payload, padded to 64 bytes - payload = bytes([random.randint(1, 255) for _ in range(size)]) - report = bytes([0]) + payload + bytes(64 - size) - h.write(report) - echo = h.read(64, 2000) - assert echo and len(echo) >= size, ( - f'HID echo timeout or short read ({size} bytes)') - assert bytes(echo[:size]) == payload, ( - f'HID echo wrong data ({size} bytes):\n' - f' expected: {payload.hex()}\n received: {bytes(echo[:size]).hex()}') - finally: - h.close() + r = hil_util.run_cmd( + [sys.executable, '-c', HID_ECHO, uid, str(enum_timeout()), f'{HID_INOUT_PID:#06x}'], + timeout=enum_timeout() + HID_ECHO_MARGIN, split_stderr=True) + # rc 124 is run_cmd's kill: the child was still inside a hidapi call, which is the + # wedge this runs in a child FOR -- and stderr is empty there, so say so rather than + # render a bare trailing colon + detail = hil_util.cmd_stdout_text(r.stderr).strip()[:300] + assert r.returncode == 0, (f'hid_generic_inout: {detail}' if detail else + f'hid_generic_inout: the child was killed on its bound ' + f'(rc {r.returncode}) -- a hidapi call did not return') def test_device_usbtest(board): - # Run the Linux testusb tier-4 battery (test/hil/usbtest.py) against the enumerated cafe:4010 - # device; surface the pass count in the report cell ("✅ 30/30", or "❌ 29/30" on a partial). + global board_wedged + # Runs test/hil/usbtest.py against the cafe:4010 device; the pass count goes in the + # report cell ("✅ 30/30", or "❌ 29/30" on a partial). uid = board['uid'] def usbtest_enumerated(): - # match VID:PID too, not just the serial: right after flashing, the previous example's - # enumeration (same serial, different PID) can linger and would fail usbtest.py's lookup - for f in glob.glob('/sys/bus/usb/devices/*/serial'): - d = os.path.dirname(f) - try: - if (open(f).read().strip().lower() == uid.lower() - and open(os.path.join(d, 'idVendor')).read().strip() == 'cafe' - and open(os.path.join(d, 'idProduct')).read().strip() == '4010'): - return True - except OSError: - pass - return False + # vid_pid FIRST: right after flashing, the previous example's enumeration (same + # serial, different PID) can linger and would fail usbtest.py's lookup -- and + # filtering on the two lock-free descriptor fields rules out every other device + # on the bus before the one read that can block. + return bool(hil_util.usb_scan(vid_pid=('cafe', '4010'), serial=uid)) end = time.monotonic() + enum_timeout() - while time.monotonic() < end and not usbtest_enumerated(): + seen = usbtest_enumerated() + while time.monotonic() < end and not seen: time.sleep(0.2) + seen = usbtest_enumerated() # fail before usbtest_permit: an absent device would otherwise queue on the battery # mutex for minutes behind real batteries just to have usbtest.py report "no device" - if not usbtest_enumerated(): + if not seen: # 0/30 rather than a bare cell: the battery never ran (30 = standard case count) - raise TestFail(f'no cafe:4010 device with serial {uid}', - metric=f'{REPORT_CELL["fail"]} 0/30') - # settle: right after flashing the enumeration can bounce once (and on dual-port parts like - # CH32V307 the other port's stale usbtest node — same serial and PID — lingers a moment); - # running testusb into that gap sees the device drop mid-case - time.sleep(3) + # maxtasksperchild=1, so this worker only ever handled THIS board: a give-up here + # is about this device. Without the caveat a wedged-but-present DUT reads as a + # positive absence claim -- the conflation this whole path exists to avoid. + raise TestFail(f'no cafe:4010 device with serial {uid}{hil_util.strand_note()}', + metric=f'{hil_report.REPORT_CELL["fail"]} 0/30') + # settle: right after flashing the enumeration can bounce once (and on dual-port parts + # the other port's stale node — same serial and PID — lingers), and testusb run into + # that gap sees the device drop mid-case + time.sleep(USBTEST_SETTLE) # --keep-binding is required for concurrent batteries: usbtest.py's cleanup unbinds - # EVERY usbtest-bound interface (releasing stale same-PID grabs), which would kill a - # peer battery mid-run under USBTEST_PARALLEL > 1; the unbind path has also wedged a - # host xHCI (usb_hcd_alloc_bandwidth) on this rig. Leaving bindings is harmless with - # unique example PIDs - the next example re-enumerates under a different PID and binds - # its normal driver. usbtest_permit budgets USBTEST_PARALLEL batteries per controller. + # EVERY usbtest-bound interface, killing a peer battery under USBTEST_PARALLEL > 1, and + # that unbind path has also wedged a host xHCI (usb_hcd_alloc_bandwidth) here. Harmless + # to leave: the next example enumerates under a different PID. script = Path(__file__).resolve().parent / 'usbtest.py' - cmd = f'python3 "{script}" --serial "{uid}" --json --keep-binding --timeout 60' + # --budget makes the battery a real bound: repeated case timeouts (a FAIL, not a HUNG, + # so the battery keeps going) can otherwise spend the whole outer timeout inside the + # case loop, leaving the recovery below nothing. + cmd = (f'{shlex.quote(sys.executable)} {shlex.quote(str(script))} ' + f'--serial {shlex.quote(uid)} --json --keep-binding ' + f'--timeout 60 --budget {USBTEST_BATTERY_BUDGET}') + # Post-hang recovery reflashes the DUT through its own probe, NEVER a root-port cycle + # (one board reached instead of every fixture under the port; see usb-kernel-recover). + # _current_fw is the artifact test_example flashed for THIS test: re-deriving it from + # board['name'] reflashes the wrong build on variant-only boards. Our run_cmd bound + # below RESERVES the whole ladder (usbtest.recovery_reserve), which is what lets the + # child run it straight through without an outer kill landing mid-flash and orphaning + # the flasher (own session) on the probe. Never under --skip-flash -- and say so: a + # HUNG case then holds the DUT's usbfs lock for the rest of the run, and a probe reset + # is no substitute (the DWC2 pullup survives a core halt). + # ...and only when this flasher can DELIVER that reflash past a poisoned node + # (hil_flash.convoy_safe). Otherwise the flags cost twice: the delivery adds a SECOND + # stray, and the board reserves recovery budget for a path that cannot fire. + # The RECOVERY flasher, which may be the roster's optional `flasher_recover` rather + # than the primary -- a jlink/stlink board can name an openocd entry that reaches the + # same probe convoy-safely without changing how the board is normally flashed. + _rec_flasher = hil_flash.recover_flasher(board) + recovery = bool(_current_fw and not skip_flash and hil_flash.convoy_safe(_rec_flasher)) + # ONE bound: run_cmd's kill below. It carries the recovery reserve only when a + # recovery can actually run, and only what THIS flasher's ladder can spend -- a board + # that cannot recover used to hold a pool worker AND its battery permit idle for a + # reserve it had no way to spend, under a usbtest width of 2. + outer = USBTEST_BATTERY_BUDGET + USBTEST_OVERSHOOT + ( + usbtest.recovery_reserve(_rec_flasher) if recovery else 0) + if _current_fw and skip_flash: + print('note: --skip-flash disables usbtest hang recovery; a HUNG case will leave ' + 'the device wedged until it is reflashed', flush=True) + elif _current_fw and not recovery: + print(f'note: {_rec_flasher["name"]} cannot deliver a reflash past a poisoned ' + f'usbfs node, so usbtest hang recovery is disabled for {board["name"]}; a ' + f'HUNG case will leave it wedged for the rest of the run', flush=True) + if recovery: + # ship the RECOVERY flasher as `flasher`: usbtest.py and convoy_safe both read + # board['flasher'], so substituting here keeps the entire child side unaware that + # a second roster entry exists + rb = json.dumps({'name': board['name'], 'flasher': _rec_flasher}) + cmd += f' --recover-board {shlex.quote(rb)} --recover-fw {shlex.quote(_current_fw)}' + # The reserve above USBTEST_BATTERY_BUDGET exists because the battery can overrun by + # one already-started case, and a hang there needs room for the recovery (whose reflash + # is bounded by usbtest.RECOVER_FLASH_TIMEOUT, not HIL_CMD_TIMEOUT). Without it run_cmd + # SIGKILLs usbtest.py mid-recovery, losing the JSON and the diagnosis. with hil_lock.usbtest_permit(uid): - r = hil_flash.run_cmd(cmd, timeout=200) - out = hil_flash.cmd_stdout_text(r.stdout) + # split_stderr: the battery's final JSON is parsed from stdout, and stderr is the + # only detail left when the outer timeout kills the battery before it prints + r = hil_util.run_cmd(cmd, timeout=outer, split_stderr=True) + out = hil_util.cmd_stdout_text(r.stdout) brace = out.find('{') try: + # brace < 0 would slice from the END ('...rc 0' -> '0' -> int 0, whose subscript + # raises TypeError outside the tuple below and loses the diagnosis) + if brace < 0: + raise ValueError('no JSON object on stdout') data = json.loads(out[brace:]) passed, failed = int(data['passed']), int(data['failed']) - except (ValueError, KeyError, json.JSONDecodeError): - raise TestFail(f'usbtest did not run: {compact_output(out) or hil_flash.cmd_stdout_text(r.stderr)}', - metric=f'{REPORT_CELL["fail"]} 0/30') + except (ValueError, KeyError, TypeError, json.JSONDecodeError): + # compact BOTH, never `or`: a battery SIGKILLed mid-print leaves a truthy JSON + # fragment on stdout, so an `or` drops the stderr that explains the failure + parts = [compact_output(hil_util.cmd_stdout_text(r.stderr)), compact_output(out)] + detail = ' | '.join(p for p in parts if p) + # Retryable even on rc 124 (run_cmd's outer kill), though the retry re-pays the + # whole budget: 124 only says the timer expired, which a healthy battery can hit + # under load, and test_example REFLASHES before each attempt. Where usbtest's + # in-band recovery is off (--skip-flash, a flasher failing convoy_safe, a terminal + # wedge) that reflash is the only thing left to unpoison the DUT for the boards + # that share its controller. + # No JSON to read the verdict from, so fall back to the text: a battery SIGKILLed + # mid-hang still says HUNG on stdout, and this raise happens BEFORE the latch below + # -- which is why the outer-timeout case, the likeliest real wedge, never latched. + if 'HUNG' in out: + board_wedged = (f'{board["name"]}: usbtest reported a hang and was killed ' + f'before it could report a verdict') + raise TestFail(f'usbtest did not run: {detail}', + metric=f'{hil_report.REPORT_CELL["fail"]} 0/30') + + return _usbtest_verdict(board, data, out, passed, failed, recovery, + _rec_flasher) - total = passed + failed - if failed == 0 and total > 0: - return f'{REPORT_CELL["pass"]} {passed}/{total}' - bad = [c.get('num') for c in data.get('cases', []) if c.get('status') != 'PASS'] - raise TestFail(f'usbtest {passed}/{total} (cases failed: {bad})', - metric=f'{REPORT_CELL["fail"]} {passed}/{total}') + +def _usbtest_verdict(board: Board, data: dict, out: str, passed: int, failed: int, + recovery: bool, rec_flasher: dict) -> str: + """The report cell for a battery that produced JSON, or a TestFail carrying one. + + Also latches board_wedged, which stops the REST of this board's examples: each would + flash THROUGH the poisoned usbfs node, block, survive SIGKILL and add another stray -- + one wedge becoming one stray per remaining example, which is the convoy this whole + containment path exists to prevent. + """ + global board_wedged + # A HUNG case that recovery could not clear leaves a D-state holder on this board's + # usbfs node. Latch it: the remaining examples would each flash THROUGH that node, + # block, survive SIGKILL and add another stray -- turning one wedge into one stray per + # remaining example, which is the convoy this branch exists to contain. + # The battery's OWN verdict first: `recovery` only says the flags were passed, not that + # the reflash worked, so a convoy-safe board whose recovery failed used to come back + # unlatched and flash every remaining example through the poisoned node. + if data.get('wedged') or (not recovery and 'HUNG' in out): + # rec_flasher, NOT board['flasher']: recovery was decided against recover_flasher() + # in the caller, and the two diverge as soon as a roster carries the + # optional `flasher_recover` key -- naming the wrong one sends the operator to the + # wrong probe. The wording stays on what usbtest actually reported ("still wedged"), + # because unrecovered_hang is also set by the ambiguous abort, where + # nothing hung and the old text was false on both clauses. + board_wedged = (f'{board["name"]}: usbtest reports the device still wedged ' + + (f'after a recovery reflash via {rec_flasher["name"]}' if recovery + else f'and {rec_flasher["name"]} cannot deliver a recovery reflash')) + + # notrun counts toward the denominator but is NOT a failure: listing cases that never + # ran as failures sends a maintainer bisecting one of them. + notrun = int(data.get('notrun', 0)) + total = passed + failed + notrun + if board_wedged and failed == 0 and notrun == 0: + # Every case passed and the device STILL wedged -- usbtest's ambiguous + # abort fires after the last case, so nothing back-fills a BUDGET entry. Reporting + # the pass would exit 0 with a D-state holder on the rig and the board absent from + # the re-run spec. parsed=True: a retry re-pays the whole battery to re-observe a + # wedge, and flashes through the poisoned node to do it. + raise TestFail(f'usbtest {passed}/{total} but the device wedged ({board_wedged})', + metric=f'{hil_report.REPORT_CELL["fail"]} {passed}/{total}', parsed=True) + if failed == 0 and notrun == 0 and total > 0: + return f'{hil_report.REPORT_CELL["pass"]} {passed}/{total}' + bad = [c.get('num') for c in data.get('cases', []) + if c.get('status') not in ('PASS', 'BUDGET')] + why = f'usbtest {passed}/{total}' + if bad: + why += f' (cases failed: {bad})' + if notrun: + # the reason is per BUDGET entry: a hang or a device drop also aborts the battery, + # and blaming the budget points the maintainer at the wrong thing + reasons = {c.get('detail', '') for c in data.get('cases', []) + if c.get('status') == 'BUDGET'} + reason = (reasons.pop().replace('not run: ', '') if len(reasons) == 1 + else 'the battery stopped early') + why += f'; {notrun} case(s) never ran ({reason}), so this says nothing about them' + # parsed ONLY when every case ran: an aborted battery (budget expiry, kernel hang, bus + # drop) leaves BUDGET entries, and those are exactly what a reflash retry can fix. + raise TestFail(why, metric=f'{hil_report.REPORT_CELL["fail"]} {passed}/{total}', + parsed=(notrun == 0)) # ------------------------------------------------------------- @@ -1370,42 +1692,68 @@ def test_example(board: Board, variant: str, example: str) -> tuple[int, str, st test_name = f'{variant:40} {example:30} ...' - # --skip-flash runs whatever is already on the board, so any build counts as present: - # only the flashing path needs the artifact this board's flasher actually consumes. - # Filtering there too would skip the test as "no binary" over an extension it never uses. + # --skip-flash runs whatever is already on the board, so any build counts as present; + # filtering by flasher there would skip the test over an extension it never uses. fw_name = hil_flash.find_firmware(variant, example, flasher=None if skip_flash else board['flasher']['name']) if fw_name is None: log_line(f'{test_name} Skip (no binary)') return 0, 'skip', None + # usbtest's hang recovery reflashes the exact artifact under test; re-deriving it from + # board['name'] breaks on variant-only boards + global _current_fw + _current_fw = str(fw_name) if verbose: log_line(f'Firmware {fw_name}') - # flash firmware (unless --skip-flash), then run the test. Both may fail randomly, - # retry a few times. global _enum_timeout start_s = time.time() flash_ok = True last_err = '' last_detail = '' + wedge_break = False for i in range(max_retry): + if board_wedged and i: + # The latch is set MID-attempt (a HUNG usbtest whose flasher cannot recover), + # so test_board's check between tests is too late for THIS test's own retries: + # every further attempt re-flashes into the D-state-held node, blocks, survives + # SIGKILL and leaves another stray. The wedge is not something a retry can fix. + log_line(f'{test_name} not retrying: {board_wedged}') + # COUNT it. Breaking out here skips the i == max_retry - 1 branch that would + # have incremented err_count, so the board rendered a red cell, contributed 0 + # to the exit status and was omitted from the re-run spec -- a rig left with a + # D-state holder published under sys.exit(0). Latent at CI's --retry 1, live + # for every local run and for the workflows that pass no -r. + wedge_break = True + break _enum_timeout = ENUM_TIMEOUT if i == 0 else ENUM_TIMEOUT_RETRY attempt_out = io.StringIO() with redirect_stdout(attempt_out): if not skip_flash: with hil_lock.flash_permit(board['uid']): t_flash = time.monotonic() - ret = getattr(hil_flash, f'flash_{board["flasher"]["name"].lower()}')(board, str(fw_name)) + try: + ret = getattr(hil_flash, + f'flash_{board["flasher"]["name"].lower()}')(board, str(fw_name)) + except Exception as e: + # A flasher that RAISES (esptool's get_serial_dev when the adapter + # drops off the bus, a missing config.env, an unwritable CWD) would + # propagate out of the worker and abort the whole drain, costing + # every board still in flight. + print(f'flash raised: {type(e).__name__}: {e}', flush=True) + ret = subprocess.CompletedProcess(args='flash', returncode=1, + stdout=f'{type(e).__name__}: {e}') if PROFILE: log_line(f'[prof] {variant} {example} flash attempt {i + 1}: ' f'{time.monotonic() - t_flash:.1f}s rc={ret.returncode}') flash_ok = (ret.returncode == 0) - # A wedged RP2040/RP2350 DAP answers nothing and the probe has no reset - # line, so the retry would fail identically; POR it via the Rescue DP - # first. No-op for every other board and every other flash failure. - if not flash_ok and i + 1 < max_retry and \ - hil_flash.rescue_openocd(board, hil_flash.cmd_stdout_text(ret.stdout)): + # A wedged RP2040/RP2350 DAP answers nothing and the probe has no + # reset line, so the retry fails identically; POR it via the Rescue DP + # first (no-op otherwise). NOT gated on a remaining attempt: CI HIL jobs + # run --retry 1, and this leaves the DAP POR'd for the jobs that follow. + if not flash_ok and \ + hil_flash.rescue_openocd(board, hil_util.cmd_stdout_text(ret.stdout)): log_line(f'{variant} {example}: DAP wedged, rescued via Rescue DP') if flash_ok: try: @@ -1417,7 +1765,6 @@ def test_example(board: Board, variant: str, example: str) -> tuple[int, str, st else: status = STATUS_OK result_status = 'pass' - # a test may return a string to show in its report cell (e.g. speed) metric = tret if isinstance(tret, str) else None msg = f'{test_name} {status}' if last_detail: @@ -1428,9 +1775,20 @@ def test_example(board: Board, variant: str, example: str) -> tuple[int, str, st except Exception as e: last_err = str(e) last_detail = compact_output(attempt_out.getvalue()) + if getattr(e, 'parsed', False): + # a PARSED per-case result (usbtest's "29/30"): retrying re-pays + # the whole battery, inside the fleet's usbtest permit, to + # re-observe a number the JSON already reported. Only that case. + err_count += 1 + metric = getattr(e, 'metric', None) + msg = f'{test_name} {STATUS_FAILED}: {e}' + if last_detail: + msg += f' {last_detail}' + msg += f' in {time.time() - start_s:.1f}s' + log_line(msg) + break if i == max_retry - 1: err_count += 1 - # a failing test may still carry a metric to show in its cell (e.g. "❌ 29/30") metric = getattr(e, 'metric', None) msg = f'{test_name} {STATUS_FAILED}: {e}' if last_detail: @@ -1463,23 +1821,27 @@ def test_example(board: Board, variant: str, example: str) -> tuple[int, str, st msg += f' in {time.time() - start_s:.1f}s' log_line(msg) + if wedge_break and not err_count: + # ONE error for the test, never two: a board that also failed to flash has already + # been counted just above. Without this the test returns 0 -- red cell, clean exit + # status, absent from the re-run spec. + err_count += 1 return err_count, result_status, metric def build_board(board: Board) -> tuple[str, int]: """Build firmware for this board via tools/build.py. - Honors board config's variant list and build.args defines. - Output goes to cmake-build/cmake-build-<variant>/ (tools/build.py layout).""" + Honors board config's variant list. + Output goes to cmake-build/cmake-build-<variant>/ (tools/build.py layout). + + Unbounded on purpose: --build is a local convenience (no CI workflow passes it), so + the developer watching the build is the timeout.""" name = board['name'] - bcfg = cast(BuildCfg, board.get('build', {})) - extra_defs = bcfg.get('args', []) variants = board.get('variant') or [{'name': name, 'flags': ''}] failed = 0 for v in variants: - cmd = [sys.executable, str(hil_flash.TINYUSB_ROOT / 'tools' / 'build.py'), '-b', name] - for d in extra_defs: - cmd += ['-D', d] + cmd = [sys.executable, str(hil_util.TINYUSB_ROOT / 'tools' / 'build.py'), '-b', name] if v['name'] != name: cmd += ['--build-name', v['name']] for d in v.get('defines', []): @@ -1489,69 +1851,87 @@ def build_board(board: Board) -> tuple[str, int]: if verbose: cmd.append('-v') print(f' + {" ".join(cmd)}') - r = subprocess.run(cmd, cwd=hil_flash.TINYUSB_ROOT) - if r.returncode != 0: + # stdio is inherited so the build STREAMS: a silent buffer is + # indistinguishable from a stall. + proc = subprocess.Popen(cmd, cwd=hil_util.TINYUSB_ROOT, start_new_session=True) + try: + rc = proc.wait() + except KeyboardInterrupt: + # start_new_session means the build never saw the terminal's SIGINT + try: + os.killpg(proc.pid, signal.SIGKILL) + except OSError: + proc.kill() + raise + if rc != 0: failed += 1 return name, failed -# pseudo-test column for a variant boundary the park-flash could not clear (see below) -BOUNDARY_CELL = 'same-PID boundary' +def _tests_for(board: Board) -> list: + """Which examples this board runs, in roster order. + + Three sources, most specific first: an explicit -bt list for this board, a global -t + list filtered against what the board can actually do, or the roster's own capability + flags. The -t filter is not cosmetic -- without it a device-only board runs host/dual + tests whose `dev_attached` roster entry does not exist. + """ + name = board['name'] + if name in board_test: + return list(board_test[name]) + + board_tests = board.get('tests', {}) + if test_only: + if 'only' in board_tests: + allowed = set(board_tests['only']) + return [t for t in test_only if t in allowed] + return [t for t in test_only + if board_tests.get(t.split('/', 1)[0]) is True] + + if 'tests' not in board: + return [] + test_list: list = [] + if board_tests.get('device') is True: + test_list += list(device_tests) + if board_tests.get('dual') is True: + test_list += dual_tests + if board_tests.get('host') is True: + test_list += host_test + if 'only' in board_tests: + test_list = list(board_tests['only']) + for skip in board_tests.get('skip', []): + if skip in test_list: + test_list.remove(skip) + log_line(f'{name:25} {skip:30} ... Skip') + return test_list -def test_board(board: Board) -> tuple[str, int, list[str], list, float]: +def test_board(board: Board) -> tuple: + # (name, err_count, failed_tests, rows, duration[, strays]) -- the board-LOCKED early + # return is 5 wide, the normal one 6. _stray_note reads index 5 behind a len() guard, + # so a field inserted anywhere before it silently reports a duration as a stray count. + swept = False name = board['name'] flasher = board['flasher'] + global board_wedged + board_wedged = '' try: _lock_fh = hil_lock.acquire_board_lock(name) except RuntimeError as e: log_line(f'{name:25} {STATUS_FAILED}: {e}') - # visible report row so the ❌ matches the exit code; failed-tests stays - # empty so a re-run repeats the whole board (no bogus -bt test filter) - return name, 1, [], [(name, {'board-locked': 'fail'}, None)], 0.0 + # visible report row so the ❌ matches the exit code; failed-tests stays empty so a + # re-run repeats the whole board (no bogus -bt filter) + return name, 1, [], [(name, {hil_report.LOCKED_CELL: 'fail'}, None)], 0.0 # after the lock: flock wait behind a concurrent run is not board cost t_board = time.monotonic() try: - # default to all tests - test_list = [] - - if name in board_test: - test_list = board_test[name] - elif len(test_only) > 0: - # Explicit -t: filter against the board's capabilities so a device-only - # board doesn't try to run host/dual tests (the test functions need a - # `dev_attached` entry in the board config that won't exist). - board_tests = board.get('tests', {}) - if 'only' in board_tests: - allowed = set(board_tests['only']) - test_list = [t for t in test_only if t in allowed] - else: - for t in test_only: - category = t.split('/', 1)[0] - if board_tests.get(category) is True: - test_list.append(t) - else: - if 'tests' in board: - board_tests = board['tests'] - if board_tests.get('device') is True: - test_list += list(device_tests) - if board_tests.get('dual') is True: - test_list += dual_tests - if board_tests.get('host') is True: - test_list += host_test - if 'only' in board_tests: - test_list = board_tests['only'] - if 'skip' in board_tests: - for skip in board_tests['skip']: - if skip in test_list: - test_list.remove(skip) - log_line(f'{name:25} {skip:30} ... Skip') + test_list = _tests_for(board) err_count = 0 failed_tests = [] board_wide_fail = False # re-run the whole board, not a subset of its tests - rows = [] # list of (row_label, {example: status}, duration) — one row per build variant + rows = [] # list of (row_label, {example: status}, duration) — one per build variant # a -t/-bt filtered run times only a subset; report no duration so an accumulate # re-run keeps the previous full-run value partial = bool(test_only) or name in board_test @@ -1560,11 +1940,10 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: prev_last = None # last test of the previous variant: the variant boundary is an adjacency too for v in variants: vname = v['name'] - # Shuffle each (board, variant)'s run order — de-synchronizes the worker pool so - # usbtest batteries and flash churn spread across the timeline instead of convoying, - # and surfaces order-dependent bugs. Seeded for replay (HIL_SHUFFLE_SEED, logged by - # main). Unique per-example PIDs make any two different examples re-enumerate; only - # the variant boundary can repeat the same example (same PID) — swap it away. + # Shuffle each (board, variant)'s run order: spreads batteries and flash churn + # across the timeline instead of convoying, and surfaces order-dependent bugs. + # Seeded for replay (HIL_SHUFFLE_SEED). Unique per-example PIDs re-enumerate + # between examples; only the variant boundary can repeat one. run_list = list(test_list) if shuffle_seed is not None and len(run_list) > 1: random.Random(f'{shuffle_seed}:{name}:{vname}').shuffle(run_list) @@ -1572,24 +1951,34 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: run_list[0], run_list[-1] = run_list[-1], run_list[0] cells = {} if run_list and run_list[0] == prev_last and not skip_flash: - # Same example (same PID) still repeats across the boundary: a one-test - # list (the common case for a -bt scoped run) leaves nothing to swap - # with. Park on board_test first - it disables the board's USB, so the - # PID goes away and the next flash must re-enumerate to be seen. + # Same example (same PID) still repeats across the boundary (a one-test + # -bt run has nothing to swap with). Park on board_test first: it disables + # the board's USB, so the next flash must re-enumerate to be seen. t_park = time.monotonic() - park_ec, park_status, _ = test_example(board, vname, 'device/board_test') + # _should_park, same as the teardown park: this is attempt 0, so + # test_example's retry guard does not stop it flashing into a poisoned node + park_ec, park_status, _ = ( + test_example(board, vname, 'device/board_test') if _should_park(skip_flash) + else (0, 'skip', None)) if park_ec or park_status == 'skip': - # Boundary not cleared: the previous variant's device may still be - # enumerated under the same PID, so this variant's tests could pass - # against its firmware. Skip them - a false green proves nothing and - # is worse than a gap - and record the boundary itself as the failure - # (a visible ❌ cell, mirroring the board-lock row above) so the report - # matches the exit code instead of rendering all-green. - why = 'no board_test binary' if park_status == 'skip' else 'park flash failed' + # Boundary not cleared: the previous variant may still be enumerated + # under the same PID, so this variant's tests could pass against ITS + # firmware. Skip them and record the boundary as the failure, so the + # report matches the exit code instead of rendering all-green. + # A 'skip' here has two very different causes: no board_test build, or + # _should_park refusing to flash a WEDGED board. Reporting the latter as + # a missing binary sends the operator hunting a build that exists. + wedge_skip = park_status == 'skip' and bool(board_wedged) + why = ('the board is wedged' if wedge_skip else + 'no board_test binary' if park_status == 'skip' else + 'park flash failed') log_line(f'{vname:40} {"same-PID boundary":30} {STATUS_FAILED}: not cleared ({why}); ' f'skipping {len(run_list)} test(s) on this variant') - err_count += 1 - cells[BOUNDARY_CELL] = 'fail' + # the wedge already charged its own error through test_device_usbtest; + # charging again would double-count one incident in the exit code + if not wedge_skip: + err_count += 1 + cells[hil_report.BOUNDARY_CELL] = 'fail' # blaming run_list[0] would re-run an innocent test that then passes, # leaving the boundary unretested; re-run the whole board instead board_wide_fail = True @@ -1601,43 +1990,79 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: prev_last = run_list[-1] t_variant = time.monotonic() for test in run_list: + if board_wedged: + # Do NOT flash through a poisoned node: each attempt enumerates into + # it, blocks uninterruptibly and leaves another stray behind. Report + # the skip so the cell is not mistaken for a pass. + cells[test] = f'{hil_report.REPORT_CELL["skip"]} board wedged' + # ...and re-run the WHOLE board, like the boundary-failure path above: + # these tests never executed, so naming them individually in the .failed + # spec is not enough -- an --accumulate re-run that fixes only the wedged + # test would merge a green cell over it and leave these skips standing + # from the earlier attempt, forever, under a green job. + board_wide_fail = True + continue ec, status, metric = test_example(board, vname, test) err_count += ec cells[test] = metric if metric else status if ec > 0: failed_tests.append(test) + if board_wedged: + log_line(f'{vname:40} SKIPPING the rest of this board: {board_wedged}; ' + f'flashing through the poisoned node would add a stray per test') dur = f'{time.monotonic() - t_variant:.0f}s' if run_list and not partial else None rows.append((vname, cells, dur)) - # board duration excludes the teardown park-flash below; a partial (filtered) - # run reports 0.0 so it never overwrites a cached full-run duration + # excludes the teardown park-flash below; a partial (filtered) run reports 0.0 so + # it never overwrites a cached full-run duration t_total = 0.0 if partial else time.monotonic() - t_board - # flash board_test last to disable board's usb (skipped when --skip-flash is set); - # this is teardown/park, not a test — not recorded in the report - if not skip_flash: + # park: flash board_test last to disable the board's usb; teardown, not a test, + # so it is not recorded in the report. + # + # NOT on a wedged board: the latch has just skipped every remaining test precisely + # because flashing through a D-state-held node blocks, survives SIGKILL and leaves + # a stray -- and this park is a flash like any other. test_example's own guard does + # not stop it (that one only suppresses RETRIES, and this is attempt 0), so the + # containment path would add the very stray it exists to prevent. + if _should_park(skip_flash): test_example(board, variants[0]['name'], 'device/board_test') - return name, err_count, [] if board_wide_fail else sorted(set(failed_tests)), rows, t_total + # Sweep HERE, not in main()'s finally: maxtasksperchild=1 retires this process as + # soon as it returns, reparenting anything it spawned to init and off the pool's + # ppid tree, so the main-side sweep walks fresh idle workers and finds nothing. + # Measured: 4 tasks, zero overlap, sweep 0, all 4 strays alive. + stray = hil_health.kill_own_children() + swept = True + + # LAST field: what this worker could not kill. Only the worker can answer it, and + # the result tuple already crosses back, so no Manager round-trip. + return (name, err_count, [] if board_wide_fail else sorted(set(failed_tests)), + rows, t_total, stray) finally: + # A raise skips the sweep above, and maxtasksperchild=1 retires this process + # immediately afterwards -- reparenting its flasher to init and erasing the ppid + # link, so main's sweep cannot see it either. The count cannot reach the report on + # this path (there is no result tuple), but the KILL still frees the probe. + if not swept: + try: + hil_health.kill_own_children() + except Exception as se: # noqa: BLE001 - never mask the original failure + print(f'warning: stray sweep failed: {type(se).__name__}: {se}', flush=True) if _lock_fh: try: - # clear our pid record before dropping the flock: this worker - # process lives on (pool reuse), so a stale record would make - # hil_lock.py's pid-liveness checks report a freed board as - # still locked for the rest of the run + # clear our pid record before dropping the flock: this worker process + # lives on (pool reuse), so a stale record would make hil_lock's + # pid-liveness checks report a freed board as locked for the rest of the run _lock_fh.truncate(0) except OSError: pass _lock_fh.close() -REPORT_MD = 'hil_report.md' -REPORT_JSON = 'hil_report.json' -# controller hints learned from previous runs: uid -> {'name', 'pci', 'duration'}. Only -# 'pci' is consumed (dispatch order and first-flash budgeting, never battery -# serialization); name/duration are informational. PCI addresses are boot-stable (bus -# numbers are not), so the cache survives reboots and only goes stale on re-cabling. +# controller hints from previous runs: uid -> {'name', 'pci', 'duration'}. Only 'pci' is +# consumed (dispatch order and first-flash budgeting, never battery serialization). PCI +# addresses are boot-stable, so the cache survives reboots and goes stale on re-cabling. CONTROLLER_CACHE = Path.home() / '.cache' / 'tinyusb-hil' / 'controller_cache.json' @@ -1652,120 +2077,278 @@ def schedule_boards(boards: list, pci_of_uid: dict) -> list: return [b for grp in itertools.zip_longest(*buckets.values()) for b in grp if b is not None] -def render_matrix(rows_all: list) -> str: - """Render rows (list of (row_label, {example: status}, duration)) as an aligned - markdown matrix: columns = tests (bare names) centered, boards left-aligned, - per-row duration as the trailing column.""" - seen = set() - for _, cells, _ in rows_all: - seen.update(cells) - if not seen: - return 'No tests were run.' +def _write_failed_spec(failed_fname: Path, report_dir: Path, mret: list) -> None: + """Re-run spec: only the failed boards (-b), each restricted to its own failed tests + (-bt); a board with failures but no test list re-runs entirely. - # metric-bearing columns pinned first (usbtest score, throughput, explorer read speed), - # the rest alphabetical by bare test name: stable regardless of the (shuffled) execution order - pinned = ['usbtest', 'cdc_msc_throughput', 'msc_file_explorer', 'msc_file_explorer_freertos'] + Shared with the pool-guard path, which feeds it the boards that never reported. That + path used to leave this unwritten -- and a fresh run has already unlinked it -- so + build.yml's "Get re-run spec" step found nothing and the GitHub re-run repeated the + whole fleet to find the one board that wedged.""" + parts = ['--accumulate'] + for name, err, fts, *_ in mret: + if err > 0: + parts.append(f'-b {name}') + if fts: + parts.append(f'-bt {name}:{",".join(fts)}') + if len(parts) > 1: # build-only failures have no boards to re-run + report_dir.mkdir(parents=True, exist_ok=True) + with failed_fname.open('w') as f: + f.write(' '.join(parts)) + else: + failed_fname.unlink(missing_ok=True) - def col_key(t): - name = t.rsplit('/', 1)[-1] - return (pinned.index(name) if name in pinned else len(pinned), name, t) - columns = sorted(seen, key=col_key) - headers = [c.rsplit('/', 1)[-1] for c in columns] + ['duration'] # bare example names +class PoolDrainTimeout(MpTimeoutError): + """Guard expiry, carrying the rows that DID finish. - def cell(cells, col): - v = cells.get(col) - if v is None: - return '' - return REPORT_CELL.get(v, v) # status symbol, or a metric string (e.g. speed) verbatim + They ride on the exception because the raise is the containment path: losing them here + is what map_async did, and what the drain exists to stop. + """ - rows_vals = [(lbl, [cell(cells, c) for c in columns] + [dur or '']) - for lbl, cells, dur in rows_all] - board_hdr = 'Board' - board_w = max([len(board_hdr)] + [len(lbl) for lbl, _ in rows_vals]) - col_w = [max([len(h)] + [len(vals[i]) for _, vals in rows_vals]) - for i, h in enumerate(headers)] + def __init__(self, finished: list): + super().__init__() + self.finished = finished - def line(label, values): - padded = [label.ljust(board_w)] + [v.center(w) for v, w in zip(values, col_w)] - return '| ' + ' | '.join(padded) + ' |' - header = line(board_hdr, headers) - sep = '| ' + '-' * board_w + ' | ' + ' | '.join(':' + '-' * (w - 2) + ':' for w in col_w) + ' |' - body = [line(lbl, vals) for lbl, vals in rows_vals] +def drain_pool(it, boards: list, deadline: float, out: list | None = None) -> list: + """Collect imap_unordered results against ONE deadline. Returns the finished rows. - # tally run cells (blank/not-run cells are absent from the dicts). A cell is a bare status - # ('pass'/'fail'/'skip') or a metric string that carries its own icon (e.g. "❌ 29/30" is a - # fail, "✅ 30/30" / "✅ CDC …" a pass), so classify by the leading icon. - def cell_kind(v): - if v == 'fail' or (isinstance(v, str) and v.startswith(REPORT_CELL['fail'])): - return 'fail' - if v == 'skip' or (isinstance(v, str) and v.startswith(REPORT_CELL['skip'])): - return 'skip' - return 'pass' - kinds = [cell_kind(v) for _, cells, _ in rows_all for v in cells.values()] - failed = kinds.count('fail') - skipped = kinds.count('skip') - passed = kinds.count('pass') - summary = (f'**{REPORT_CELL["pass"]} {passed} passed · {REPORT_CELL["fail"]} {failed} failed · ' - f'{REPORT_CELL["skip"]} {skipped} skipped · blank not run**') + Raises PoolDrainTimeout (carrying those same rows) when the deadline passes with boards + still in flight -- the caller keeps them, names only what is missing, and writes a + re-run spec covering just those. - return summary + '\n\n' + '\n'.join([header, sep] + body) + A function, not an inline loop, so the tests can call THIS instead of a copy of it: the + loop's previous test built its own ThreadPool and its own drain and asserted on those, + so deleting the real one outright kept the suite green. + """ + # `out` is the CALLER's list: a worker that raises something other than a timeout + # (get_serial_dev on a dropped adapter, a Manager EOFError) propagates bare, and a + # local accumulator would take every finished board with it -- the exact loss the + # drain replaced map_async to prevent. + mret: list = out if out is not None else [] + for _ in boards: + left = deadline - time.monotonic() + if left <= 0: + raise PoolDrainTimeout(mret) + try: + mret.append(it.next(timeout=left)) + except MpTimeoutError: + raise PoolDrainTimeout(mret) from None + return mret -def accumulate_report(mret: list, report_dir: Path, fresh: bool, scope: str = '') -> str: - """Merge this run's results into hil_report.json in report_dir, then (re)write - the markdown matrix to hil_report.md. `fresh` (a first run, no --accumulate) - starts a new report; otherwise a re-run accumulates so boards/tests that - already passed are preserved while re-run cells are updated. `scope` names the - board filter, if any, so a scoped table is not mistaken for a full one. - Returns the md.""" - acc = {} # ordered {row_label: [cells dict, duration str|None]} - jpath = report_dir / REPORT_JSON - if not fresh and jpath.is_file(): - try: - saved = json.loads(jpath.read_text()) - # CI keys the report dir by run id, so the sidecar can only have been - # written by an earlier attempt of the same run - for entry in saved.get('rows', []): - acc[entry['board']] = [dict(entry['cells']), entry.get('duration')] - except (ValueError, KeyError, TypeError): - pass # corrupt/old sidecar: start fresh +def _should_park(skip_flash: bool) -> bool: + """Flash the teardown park (device/board_test, to switch the DUT's USB off)? + + Not on a wedged board. The latch has just skipped every remaining test precisely + because flashing through a D-state-held node blocks, survives SIGKILL and leaves a + stray -- and the park is a flash like any other. test_example's own guard does not stop + it either: that one only suppresses RETRIES, and the park is always attempt 0. So the + containment path would end by adding the very stray it exists to prevent. + """ + return not skip_flash and not board_wedged + + +def _stray_note(mret: list) -> str: + """Name the strays the workers could not kill, for the report banner. + + Summed from the result tuples rather than computed in main()'s finally: that finally + runs AFTER accumulate_report on both abort paths, so a banner appended there was + written to a variable nobody read again. + """ + dirty = [(r[0], r[5]) for r in mret if len(r) > 5 and r[5]] + if not dirty: + return '' + total = sum(n for _, n in dirty) + return (f'> **Rig dirty.** {total} process(es) survived SIGKILL and still hold a probe ' + f'or usbfs node into the next job: ' + f'{", ".join(f"{b} ({n})" for b, n in dirty)}.\n') + + +# containment paths print through hil_health._p: stdout may already be a dead pipe (a +# dropped ssh session), and a BrokenPipeError there would skip os._exit +_p = hil_health._p + + +def _abandon_exit(pool, mgr, abandoned: bool, err_count: int, + report_dir: Path | None = None) -> None: + """Free the runner when the pool could not be shut down. Returns only if not abandoned. + + Must run even while an exception is propagating: multiprocessing's atexit handler + SIGTERMs its daemon workers (ignored in uninterruptible sleep) and then join()s them + with NO timeout, so an abandoned pool plus any raise between the pool's finally and + here hangs the interpreter until the job ceiling kills it. Reproduced: rc=124 at 25s + with SIGTERM-ignoring workers standing in for D state.""" + if not abandoned: + return + try: + if sys.exc_info()[0] is not None: + # os._exit below discards the traceback, and this is often the only place the + # real failure would ever be printed + traceback.print_exc() + except OSError: + pass + # Word this on evidence: shutdown_pool also returns False when terminate() RAISES, and + # a live worker after terminate() is what distinguishes a wedge from a harness bug. + # Count WORKERS only -- _pool_procs appends the Manager, our own healthy child, so + # including it made n >= 1 always and the harness-error branch unreachable. It is killed + # separately: os._exit skips its finalizer, and orphaned it holds the runner's stdout. + n = hil_health.kill_pool_children(pool) + hil_health.kill_pool_children(None, mgr) + if n: + _p(f'HIL worker pool would not terminate ({n} worker(s) still live, ' + f'uninterruptible); SIGKILLed them and abandoned the rest to free the ' + f'runner. Boards held by any leaked worker stay locked until the host is ' + f'power-cycled.', flush=True) + else: + _p('HIL worker pool shutdown failed but left no live worker behind, so this is ' + 'a harness error rather than a wedged rig -- see the Pool.terminate() ' + 'warning above. Exiting early anyway to free the runner; no board should ' + 'stay locked.', flush=True) + # A report already written by accumulate_report says nothing about the abandon, and a + # green table under a red job is how an agent ends up pasting it as this run's result. + # Set the caveat in the DOCUMENT -- prepending to the markdown alone left the sidecar, + # which is all hil_report.summarize() and therefore an agent ever sees, saying nothing. + # Best-effort, never at the cost of exiting. + if report_dir is not None: + hil_report.mark_report_abandoned(report_dir, 'the worker pool would not shut down.') + try: + sys.stdout.flush() + except OSError: + pass + # Clamped: os._exit takes a status byte, so err_count == 256 would truncate to 0 and + # report a failing, abandoned run as green. + os._exit(min(err_count, 125) if err_count else 1) + + +def _load_controller_hints() -> tuple[dict, dict]: + """The uid -> {name, pci, duration} cache, plus the uid -> pci view scheduling wants. + + Best effort throughout: a missing, hand-edited or torn cache costs dispatch ORDER, + never the run. + """ + hints: dict = {} + try: + with CONTROLLER_CACHE.open() as f: + loaded = json.load(f) + if isinstance(loaded, dict): # keep only the expected uid -> dict shape + hints = {k: v for k, v in loaded.items() if isinstance(v, dict)} + except (OSError, ValueError): + pass + return hints, {uid: h['pci'] for uid, h in hints.items() if h.get('pci')} + + +def _save_controller_hints(hints: dict, mret: list, uid_of: dict, cmap) -> None: + """Fold this run's PCI resolutions and durations back into the cache, atomically. + + Merge-on-write: another HIL job (the esp split) may have finished since our startup + read, so overlay only this run's boards rather than publishing our whole view. + """ + for name, _, _, _, dur, *_ in mret: + uid = uid_of.get(name) + if uid is None: + continue + h = dict(hints.get(uid) or {}) + h['name'] = name # informational: the cache is keyed by uid + h['pci'] = cmap.get(f'uid:{uid}') or h.get('pci') + if dur > 0: # test_board reports 0.0 for filtered (partial) runs + h['duration'] = round(dur, 1) + hints[uid] = h + merged: dict = {} + try: + with CONTROLLER_CACHE.open() as f: + cur = json.load(f) + if isinstance(cur, dict): + merged = {k: v for k, v in cur.items() if isinstance(v, dict)} + except (OSError, ValueError): + pass + # overlay onto what the CACHE now holds, not onto our startup snapshot: another HIL + # job may have written a newer duration/pci for these boards since we read it + for name, *_ in mret: + uid = uid_of.get(name) + if uid is not None and uid in hints: + merged[uid] = {**merged.get(uid, {}), **hints[uid]} + CONTROLLER_CACHE.parent.mkdir(parents=True, exist_ok=True) + tmp = CONTROLLER_CACHE.with_suffix('.json.tmp') + with tmp.open('w') as f: + json.dump(merged, f, indent=1, sort_keys=True) + tmp.replace(CONTROLLER_CACHE) + + +def _abort_report(reason: str, mret: list, config_boards: list, failed_fname: Path, + report_dir: Path, fresh: bool, health_banner: str, + timeout_secs: int | None = None) -> None: + """Keep what finished, name what did not, and get a report on disk. Never raises. + + Both abort paths -- the pool guard expiring and a worker raising -- need exactly this, + and in this order. The re-run spec goes FIRST: a fresh run already unlinked it, and + leaving it unwritten is what made a GitHub re-run repeat the whole fleet. Only the + boards that never reported go in it. + + The report follows, before anything that can block, and the caller raises afterwards + into the one containment path. `timeout_secs` adds the pool-guard fallback: when + accumulate_report itself fails -- an unwritable report dir, a torn JSON -- + _abandon_exit can only stamp a report that EXISTS, so without it the artifact upload + finds nothing and the sticky PR comment keeps the previous push's green table under a + red job. + """ + stuck = [b['name'] for b in config_boards if b['name'] not in {r[0] for r in mret}] + try: + _write_failed_spec(failed_fname, report_dir, + [(n, 1, [], None, 0) for n in stuck] + + [r for r in mret if r[1] > 0]) + except Exception as werr: # noqa: BLE001 - it mkdir()s and open()s the very report dir + # the fallback below is FOR an unwritable/root-owned report dir; letting the spec + # raise here replaces the caller's RuntimeError, so the operator never sees the + # 'pool timed out' line and no report is written at all + print(f'warning: re-run spec failed: {type(werr).__name__}: {werr}', flush=True) + banner = (f"**HIL run {reason}.** {len(mret)} board(s) below finished and are this " + f"run's; {len(stuck)} never reported and are NOT in the table: " + f"{', '.join(stuck)}. The re-run spec covers those.\n") + try: + hil_report.accumulate_report(mret, report_dir, fresh, '', + health_banner + _stray_note(mret), caveat=banner) + return + except Exception as rerr: # noqa: BLE001 - the caller's raise must still happen + print(f'warning: partial report failed: {type(rerr).__name__}: {rerr}' + + '; falling back to the board list', flush=True) + try: + # banner=, or write_timeout_report's default caveat publishes 'No per-board + # results could be collected' onto a report where mret DID hold finished rows + # the CELL names the cause: a board the pool guard never reached did not + # "pool-timeout", and marking it so sends the reader after a guard that did not fire + hil_report.write_timeout_report( + report_dir, [b for b in config_boards if b['name'] in stuck], + timeout_secs or 0, banner=banner, prefix=health_banner, + cell=(hil_report.POOL_TIMEOUT_CELL if timeout_secs + else hil_report.RUN_ABORTED_CELL)) + except Exception as re2: # noqa: BLE001 + print(f'warning: fallback report failed too: {type(re2).__name__}: {re2}', + flush=True) + - # merge this run: current cells override prior for boards/tests that ran; a filtered - # run reports duration None, keeping the previous full-run value - for name, _, _, rows, _ in mret: - if rows and not any('board-locked' in cells for _, cells, _ in rows): - # board ran for real this time: clear a stale lock-failure cell - # (its row is keyed by board name; test rows may be variant names) - stale = acc.get(name) - if stale is not None: - stale[0].pop('board-locked', None) - if not stale[0]: - # variant-keyed boards never repopulate the board-name row — - # drop it or it renders as a blank ghost row - del acc[name] - for row_label, cells, dur in rows: - row = acc.setdefault(row_label, [{}, None]) - # the boundary cell is only ever written on failure, so a re-run of this - # variant that cleared the boundary must drop the previous attempt's ❌ - if BOUNDARY_CELL not in cells: - row[0].pop(BOUNDARY_CELL, None) - row[0].update(cells) - if dur is not None: - row[1] = dur +def _start_pool(mgr, seed: str, hints_by_uid: dict): + """(cmap, pool). Split out so main()'s try/finally reads as one shape. - report_dir.mkdir(parents=True, exist_ok=True) - jpath.write_text(json.dumps({'rows': [{'board': k, 'cells': c, 'duration': d} - for k, (c, d) in acc.items()]}, indent=2) + '\n') + The Manager is created by the CALLER and passed in: Pool() forks, and after a convoy + that fork is what hits EAGAIN/ENOMEM. Creating the Manager here too would leave main() + with `mgr` still None while a live SyncManager child exists -- os._exit skips its + finalizer and the orphan holds the runner's stdout, so the job step never completes. - md = render_matrix([(k, c, d) for k, (c, d) in acc.items()]) - if scope: - # a scoped run's small table is otherwise indistinguishable from a full one, - # and it replaces the previous full table in the sticky PR comment - md = f'_Scoped run: {scope}. Boards/tests not listed were not run._\n\n' + md - (report_dir / REPORT_MD).write_text(md + '\n', encoding='utf-8') - return md + maxtasksperchild=1: a fresh worker per board makes cross-board contamination + structural rather than dependent on every module global being reset by hand + (board_wedged, _current_fw, hil_flash's warn-once sets). The extra fork is noise + against a flash+test cycle. + """ + cmap = mgr.dict() + initargs = (Lock(), seed, + hil_lock.make_permit_sems(Semaphore, hil_lock.USBTEST_PARALLEL), + hil_lock.make_permit_sems(Semaphore, hil_lock.FLASH_PARALLEL), + cmap, Lock(), hints_by_uid) + pool = Pool(processes=os.cpu_count() or 1, initializer=init_worker, + initargs=initargs, maxtasksperchild=1) + return cmap, pool def main() -> None: @@ -1797,14 +2380,21 @@ def main() -> None: help='Per-board test list as BOARD:test1,test2 (overrides -t for that board); repeat for multiple boards') parser.add_argument('-B', '--build-dir', default='cmake-build', help='Build folder name (default: cmake-build)') parser.add_argument('--build', action='store_true', help='Build firmware for selected boards with cmake before running tests') - parser.add_argument('-r', '--retry', type=int, default=3, help='Retry count for failed tests (default: 3)') + # default 1, not 3: the pool guard is a FLAT 3600s that does not scale with max_retry, + # and one usbtest test at default 3 can burn 1530s of it (510s outer x3) for a single + # board. Every CI caller already pins --retry 1; the bare invocations in the hil skill + # and hil-validate.js run against the same one-slot rig and used to inherit 3. + parser.add_argument('-r', '--retry', type=int, default=1, help='Retry count for failed tests (default: 1)') parser.add_argument('-v', '--verbose', action='store_true', help='Verbose output') args = parser.parse_args() + if args.retry < 1: + # 0 would make every test loop body never run: all-red cells, exit 0 + parser.error('--retry must be >= 1') config_file = Path(args.config_file) boards = args.board verbose = args.verbose - hil_flash.verbose = args.verbose + hil_util.verbose = args.verbose test_only = args.test_only for entry in args.board_test: bname, _, tnames = entry.partition(':') @@ -1833,6 +2423,80 @@ def main() -> None: config_boards = [e for e in config_boards if e['flasher']['name'] not in args.exclude_flasher and (not args.flasher or e['flasher']['name'] in args.flasher)] + # fail rtt misconfigurations before the first flash cycle -- but only for boards + # this run actually touches: one bad roster entry must not abort other runs' subsets + def _rtt_config_abort(msg: str): + # loud AND leaving evidence, like the no-boards branch below: exiting with no + # report at all lets the PR comment keep the previous push's stale table + print(f'ERROR: {msg}', flush=True) + rd = Path(os.environ.get('HIL_REPORT_DIR', '.')) + hil_report.mark_report_no_boards(rd, f'config error: {msg}', fresh=not args.accumulate) + sys.exit(1) + + bad_logger = [e['name'] for e in config_boards if e.get('logger') not in (None, 'rtt')] + if bad_logger: + # only the exact string activates RTT handling; anything else would silently + # mean VCOM and reproduce the misleading 'No serial device found' failure + _rtt_config_abort(f'unknown "logger" value (only "rtt" is supported): {", ".join(bad_logger)}') + bad_rtt = [e['name'] for e in config_boards + if e.get('logger') == 'rtt' and e['flasher']['name'].lower() != 'jlink'] + if bad_rtt: + # JlinkRtt speaks JLinkExe only (the OpenOCD RTT route is manual — rtt skill) + _rtt_config_abort(f'"logger": "rtt" needs a jlink flasher: {", ".join(bad_rtt)}') + rtt_no_logger_def = [e['name'] for e in config_boards + if e.get('logger') == 'rtt' + and any('LOGGER=rtt' not in (v.get('defines') or []) + for v in (e.get('variant') or [{}]))] + if rtt_no_logger_def: + # a prebuilt cmake-build-<board> configured with -DLOGGER=rtt is a legitimate + # build path the roster need not describe, so warn there -- but when this run is + # responsible for the firmware (--build, or CI where the hil-build job compiled + # the artifact from these same defines) the flashed image is UART-logger and every + # test times out as 'the target produced nothing'. An always-on define is + # expressed as a single self-named variant (see the Board comment). + msg = (f'"logger": "rtt" board has a variant without LOGGER=rtt in its defines ' + f'({", ".join(rtt_no_logger_def)})') + if args.build or os.environ.get('GITHUB_ACTIONS'): + _rtt_config_abort(f'{msg} -- the firmware built for this run cannot serve the ' + f'configured RTT console') + print(f'warning: {msg} -- fine for prebuilt example sets, wrong for --build/CI ' + f'builds', flush=True) + rtt_fixture = [e['name'] for e in config_boards + if e.get('logger') == 'rtt' + and any(d.get('is_cdc') or d.get('is_msc') + for d in e.get('tests', {}).get('dev_attached', []))] + if rtt_fixture: + # interim guard, removed when the followup lands: cdc_msc_hid/msc_file_explorer + # still open the flasher VCOM directly and would die mid-run on an rtt board + _rtt_config_abort(f'"logger": "rtt" boards cannot carry is_cdc/is_msc fixtures yet ' + f'(host cdc/msc tests bypass the RTT console — see ' + f'the rtt harness-adoption doc in docs/superpowers/followup/): {", ".join(rtt_fixture)}') + + if not config_boards: + # same reason the unknown -b board exits 1: 'No tests were run.' with rc 0 reads as + # a green HIL leg, so a roster edit emptying a leg's filter stops testing silently + msg = (f'No boards left after the flasher filter (--flasher ' + f'{args.flasher or "-"}, --exclude-flasher {args.exclude_flasher or "-"})') + print(msg, flush=True) + # loud AND leaving evidence: exiting with no report at all lets the PR comment + # keep the previous push's stale table under a red job + rd = Path(os.environ.get('HIL_REPORT_DIR', '.')) + # fresh must be threaded through: this runs BEFORE the `if fresh:` wipe below, so + # defaulting it here wiped an --accumulate run's accumulated rows -- the exact + # regression the parameter exists to prevent. + hil_report.mark_report_no_boards(rd, msg, fresh=not args.accumulate) + sys.exit(1) + + + # Before the build: the probe needs nothing from it, and the annotation is more useful + # early than after a multi-board cmake build has been paid for. + # One line, not a probe: a D-state pid at start-up is a hint for whoever reads a red + # cell, never a reason to refuse the run. hil_pool_check does diagnosis. + note = hil_health.d_state_note() + if note: + log_line(f'rig note: {note}') + health_banner = f'> **Rig note.** {note}. Not a fault on its own -- a healthy testusb sits in D state for most of every case.\n' if note else '' + build_err = 0 if args.build: if hil_flash.build_dir != 'cmake-build': @@ -1848,128 +2512,169 @@ def main() -> None: print(f'Build phase done: {build_err} failed') print('-' * 30) - # HIL report sidecar (hil_report.json/.md) and the .failed re-run spec live in - # report_dir (CI keys it by run id, so it persists across run attempts but is - # private to one run). A full run starts fresh; a re-run (--accumulate, which - # the generated .failed spec always starts with) merges so already-passed - # boards/tests are preserved. Clear prior state up front on a fresh run so a - # crash mid-run can't leave a stale report or re-run spec for a retry. - # -bt alone is not a re-run marker: PR-scoped first attempts pass -bt too. + # The report sidecar and the .failed re-run spec live in report_dir (CI keys it by run + # id: persistent across attempts, private to one run). A full run starts fresh; a re-run + # (--accumulate, which .failed always starts with) merges so already-passed boards + # survive. -bt alone is not a re-run marker. report_dir = Path(os.environ.get('HIL_REPORT_DIR', '.')) failed_fname = report_dir / (config_file.name + '.failed') fresh = not args.accumulate - if fresh: - report_dir.mkdir(parents=True, exist_ok=True) - for f in (REPORT_JSON, REPORT_MD): - (report_dir / f).unlink(missing_ok=True) - failed_fname.unlink(missing_ok=True) seed = os.getenv('HIL_SHUFFLE_SEED') or str(int(time.time())) log_line(f'test-order shuffle seed: {seed} (HIL_SHUFFLE_SEED={seed} to replay); ' f'flash/usbtest parallel per controller: {hil_lock.FLASH_PARALLEL}/{hil_lock.USBTEST_PARALLEL}; ' - f'enum timeout first/retry: {ENUM_TIMEOUT}/{ENUM_TIMEOUT_RETRY}s') + f'enum timeout first/retry: {ENUM_TIMEOUT}/{ENUM_TIMEOUT_RETRY}s; ' + # all three are env-tunable, so a run that dies on the guard is otherwise + # unattributable from the log alone + f'pool guard: {POOL_TIMEOUT}s') - hints = {} - try: - with CONTROLLER_CACHE.open() as f: - loaded = json.load(f) - # tolerate a hand-edited/torn cache: keep only the expected uid -> dict shape - if isinstance(loaded, dict): - hints = {k: v for k, v in loaded.items() if isinstance(v, dict)} - except (OSError, ValueError): - pass - hints_by_uid = {uid: h['pci'] for uid, h in hints.items() if h.get('pci')} + hints, hints_by_uid = _load_controller_hints() config_boards = schedule_boards(config_boards, hints_by_uid) log_line('dispatch order: ' + ', '.join(b['name'] for b in config_boards)) - mgr = Manager() - cmap = mgr.dict() - initargs = (Lock(), seed, - [Semaphore(hil_lock.USBTEST_PARALLEL) for _ in range(hil_lock.CONTROLLER_SLOTS)], - [Semaphore(hil_lock.FLASH_PARALLEL) for _ in range(hil_lock.CONTROLLER_SLOTS)], - cmap, Lock(), hints_by_uid) - with Pool(processes=os.cpu_count() or 1, initializer=init_worker, initargs=initargs) as pool: - async_ret = pool.map_async(test_board, config_boards) + # Bound BEFORE the try so the finally can name them whatever failed: Pool() forks, and + # the EAGAIN/ENOMEM the wipe comment below worries about is most likely to come from + # that fork -- after a convoy, where every stranded read holds a thread and an fd. Left + # outside, an OSError there escaped with mgr LIVE and `pool` unbound, so no report was + # written and the interpreter unwound into multiprocessing's unbounded atexit join. + pool = mgr = cmap = None + # Defined before the pool so _abandon_exit always has a value: a raise before + # `err_count = build_err + ...` would turn the containment path into a NameError. + err_count = build_err + # Fail CLOSED: only a shutdown_pool() that actually returned True clears this, and the + # assignment sits at the END of the inner finally, so anything raising before it + # (kill_worker_children, a BrokenPipeError from its print) leaves _abandon_exit armed. + pool_abandoned = True + # BEFORE Manager()/Pool(), not inside the try: hil_ci.sh reuses a persistent REMOTE_DIR + # and scp's the report back unconditionally, so if a fork failure (OSError/EAGAIN right + # after a convoy -- the case this whole block guards) skipped the wipe, the finally's + # _abandon_exit would stamp "HIL run abandoned" onto the PREVIOUS run's report and + # publish last night's board results as this run's. Nothing is live yet here, so an + # OSError from the wipe itself just exits with its traceback -- it cannot strand the + # interpreter in multiprocessing's unbounded atexit join, which is what deferring it + # was protecting against. + if fresh: + report_dir.mkdir(parents=True, exist_ok=True) + for f in (hil_report.REPORT_JSON, hil_report.REPORT_MD): + (report_dir / f).unlink(missing_ok=True) + failed_fname.unlink(missing_ok=True) + try: + # BOUND FIRST, in main's own scope: a Pool fork failure inside _start_pool must + # still leave a live Manager reachable by the finally below, or its child is + # orphaned holding the runner's stdout. + mgr = Manager() + cmap, pool = _start_pool(mgr, seed, hints_by_uid) + # OUTER: encloses the pool block too, not just the reporting below. An exception + # escaping async_ret.get() (a worker exception, a Ctrl-C) runs the pool finally and + # then propagates straight out of main(); with _abandon_exit in a sibling try it + # was never reached. try: - mret = async_ret.get(timeout=POOL_TIMEOUT) - except MpTimeoutError: - pool.terminate() - pool.join() - raise RuntimeError(f'HIL worker pool timed out after {POOL_TIMEOUT}s') + # imap_unordered, NOT map_async: map_async is all-or-nothing, so a guard expiry + # threw away every board that had already finished -- up to a worker-width of + # completed rig time -- and left the re-run spec unwritten, so CI re-tested all + # ~26 boards to find the one that wedged. Draining as results arrive keeps what + # finished and names only what was still in flight. + it = pool.imap_unordered(test_board, config_boards) + mret = [] + deadline = time.monotonic() + POOL_TIMEOUT + try: + mret = drain_pool(it, config_boards, deadline, out=mret) + except MpTimeoutError as te: + # RAISE afterwards into the ONE containment path: the inner finally runs + # the ordered sweep (kill_worker_children BEFORE terminate, or a reaped + # worker's flasher reparents out of reach), the outer one os._exit's. + mret = te.finished + _abort_report(f'abandoned: worker pool timed out after {POOL_TIMEOUT}s', + mret, config_boards, failed_fname, report_dir, fresh, + health_banner, timeout_secs=POOL_TIMEOUT) + _p(f'HIL worker pool timed out after {POOL_TIMEOUT}s; sweeping and ' + f'shutting it down (abandoning it if a worker is unkillable)', + flush=True) + raise RuntimeError(f'HIL worker pool timed out after {POOL_TIMEOUT}s') + except Exception as e: + # A worker RAISED -- e.g. a flasher adapter dropping off the bus makes + # get_serial_dev raise in the worker's flash section, which no per-test + # handler guards. The drain means `mret` already holds every board that + # finished, so keep those rows and name only the ones still in flight. + _abort_report(f'aborted: a worker raised {type(e).__name__}: {e}', + mret, config_boards, failed_fname, report_dir, fresh, + health_banner) + raise - err_count = build_err + sum(e[1] for e in mret) - # generate the re-run spec if anything failed: run ONLY the failed boards (-b), - # each restricted to its own failed tests (-bt); a board with failures but no - # test list (e.g. board-locked) re-runs entirely. --accumulate preserves the - # already-passed cells in the report. - parts = ['--accumulate'] - for name, err, fts, _, _ in mret: - if err > 0: - parts.append(f'-b {name}') - if fts: - parts.append(f'-bt {name}:{",".join(fts)}') - if len(parts) > 1: # build-only failures have no boards to re-run - report_dir.mkdir(parents=True, exist_ok=True) - with failed_fname.open('w') as f: - f.write(' '.join(parts)) - else: - failed_fname.unlink(missing_ok=True) + err_count = build_err + sum(e[1] for e in mret) + _write_failed_spec(failed_fname, report_dir, mret) + finally: + # Not `with Pool(...)`: its __exit__ joins the workers unbounded and hangs on + # any worker in uninterruptible sleep. shutdown_pool bounds the same terminate() + # and returns False when the pool is NOT cleanly closed. + # + # Sweep BEFORE shutdown: what the workers spawned must be snapshotted and + # killed while its parent is alive, or terminate() reparents it out of reach. + # + # Both calls stay guarded and neither exits: a raise here would skip + # accumulate_report and publish an empty report dir for a run whose boards all + # passed. pool_abandoned is fail-CLOSED, so _abandon_exit still arms. + try: + # Still worth running for the TIMEOUT path, where the workers are + # genuinely stuck mid-task and their children are still reachable through + # the pool's ppid tree. On the normal path every worker has already swept + # its own (kill_own_children) and retired, so this finds nothing. + # + # No banner from here: this finally runs AFTER accumulate_report on both + # abort paths, so anything appended to health_banner now is written to a + # variable nobody reads again. The report gets its count from the result + # tuples instead, via _stray_note. + hil_health.kill_worker_children(pool, mgr) + except Exception as e: + print(f'warning: worker-child sweep failed: {type(e).__name__}: {e}', + flush=True) + try: + pool_abandoned = not hil_health.shutdown_pool(pool) + except Exception as e: + print(f'warning: pool shutdown failed: {type(e).__name__}: {e}', flush=True) - # refresh controller hints: pci resolved this run, plus board durations when the - # full test list ran (a -t/-bt filtered run would understate the board's real cost) - try: - if PROFILE: - # debug snapshot of the run's live uid->PCI / PCI->slot resolutions - report_dir.mkdir(parents=True, exist_ok=True) - with (report_dir / 'hil_profile_ctrl.json').open('w') as f: - json.dump(dict(cmap), f, indent=1, sort_keys=True) - uid_of = {b['name']: b['uid'] for b in config['boards']} - for name, _, _, _, dur in mret: - uid = uid_of.get(name) - if uid is None: - continue - h = dict(hints.get(uid) or {}) - h['name'] = name # informational: cache is keyed by uid - h['pci'] = cmap.get(f'uid:{uid}') or h.get('pci') - if dur > 0: # test_board reports 0.0 for filtered (partial) runs - h['duration'] = round(dur, 1) - hints[uid] = h - # merge-on-write: another HIL job (e.g. the esp split) may have finished since - # our startup read - re-read and overlay only this run's boards so its entries - # survive, then replace atomically so a concurrent reader never sees a torn file - merged = {} + # refresh controller hints: pci resolved this run, plus durations from full runs + # only (a filtered run would understate the board's real cost) try: - with CONTROLLER_CACHE.open() as f: - cur = json.load(f) - if isinstance(cur, dict): - merged = {k: v for k, v in cur.items() if isinstance(v, dict)} - except (OSError, ValueError): - pass - merged.update({uid_of[n]: hints[uid_of[n]] for n, *_ in mret if n in uid_of}) - CONTROLLER_CACHE.parent.mkdir(parents=True, exist_ok=True) - tmp = CONTROLLER_CACHE.with_suffix('.json.tmp') - with tmp.open('w') as f: - json.dump(merged, f, indent=1, sort_keys=True) - tmp.replace(CONTROLLER_CACHE) - except OSError as e: - print(f'warning: cannot persist controller hints to {CONTROLLER_CACHE}: {e}') + if PROFILE: + # debug snapshot of the run's live uid->PCI / PCI->slot resolutions + report_dir.mkdir(parents=True, exist_ok=True) + with (report_dir / 'hil_profile_ctrl.json').open('w') as f: + json.dump(dict(cmap), f, indent=1, sort_keys=True) + _save_controller_hints( + hints, mret, {b['name']: b['uid'] for b in config['boards']}, cmap) + except Exception as e: + # Deliberately broad, and it must stay that way: this best-effort refresh makes + # Manager proxy RPCs that raise EOFError / BrokenPipeError / RemoteError when + # the Manager child has died, none of them OSErrors -- an OSError-only guard let + # those skip accumulate_report(). Nothing here is worth the report. + print(f'warning: cannot persist controller hints to {CONTROLLER_CACHE}: ' + f'{type(e).__name__}: {e}') + - # board x test result matrix -> hil_report.md (accumulates across re-runs) + stdout - # -b/-bt in play means a filtered run (PR selection or a re-run spec): say so in the - # report, which otherwise looks exactly like a full run that happened to be small - scoped = sorted(set(args.board) | set(board_test)) - scope = f'{len(scoped)} board(s) — {", ".join(scoped)}' if scoped else '' - report = accumulate_report(mret, report_dir, fresh, scope) - print() - print(report) - print(f'\nReport written to {(report_dir / REPORT_MD).resolve()}') + # board x test result matrix -> hil_report.md (accumulates across re-runs) + stdout. + # -b/-bt means a filtered run (PR selection or a re-run spec): say so, or the report + # looks exactly like a full run that happened to be small + scoped = sorted(set(args.board) | set(board_test)) + scope = f'{len(scoped)} board(s) — {", ".join(scoped)}' if scoped else '' + report = hil_report.accumulate_report(mret, report_dir, fresh, scope, + health_banner + _stray_note(mret)) + print() + print(report) + print(f'\nReport written to {(report_dir / hil_report.REPORT_MD).resolve()}') - duration = time.time() - duration - print() - print("-" * 30) - print(f'Total failed: {err_count} in {duration:.1f}s') - print("-" * 30) - sys.exit(err_count) + duration = time.time() - duration + print() + print("-" * 30) + print(f'Total failed: {err_count} in {duration:.1f}s') + print("-" * 30) + finally: + # In the finally, not after: any raise above (accumulate_report sits outside the + # OSError handler) would skip the abandon path and unwind into multiprocessing's + # unbounded atexit join, hanging the runner. + _abandon_exit(pool, mgr, pool_abandoned, err_count, report_dir) + # Same clamp: exit status is a byte either way, so 256 failures would report green. + sys.exit(min(err_count, 125)) if __name__ == '__main__': diff --git a/test/hil/mtp_test.py b/test/hil/mtp_test.py new file mode 100644 index 000000000..92d54bdbe --- /dev/null +++ b/test/hil/mtp_test.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# One MTP test session for one board, in a disposable process. Every libmtp call is +# synchronous ctypes in our own address space and blocks in a usbfs ioctl in D state on +# a wedged device, where not even SIGKILL is delivered — so the session must be +# something the harness can abandon: hil_test.test_device_mtp runs it under +# hil_util.run_cmd (killpg + bounded reap, rc 124 on timeout). Imports stay stdlib + +# pymtp: nothing here may pull in the harness. +# +# Exit 0 on a fully passing session; 1 with the failure on stdout/stderr otherwise. +import argparse +import ctypes +import glob +import hashlib +import os +import signal +import subprocess +import sys +import threading +import time + +sys.path.append(os.path.dirname(os.path.abspath(__file__))) # PYTHONSAFEPATH drops it +# -- APPEND so PYTHONPATH still wins (the tests steer a fake pymtp that way) + +from pathlib import Path +from pymtp import LIBMTP_DeviceEntry, LIBMTP_RawDevice, MTP + +FILE1_EXPECT = b'TinyUSB MTP Filesystem example' +FILE2_MD5_EXPECT = '40ef23fc2891018d41a05d4a0d5f822f' # md5sum of logo.png + + +# Real paths by default; the offline tests point these at a fixture tree, the same way +# they steer the pymtp fake through FAKE_PYMTP_*. +# The one test seam: '' in production, a tmpdir in the offline tests, which mirror the +# real layout beneath it. This runs as a SUBPROCESS (a libmtp call blocked in a usbfs +# ioctl hangs its thread forever, so the session must be somewhere killable), and neither +# monkeypatching nor import shadowing crosses that boundary -- unlike the fake pymtp, +# which the tests inject through PYTHONPATH alone. +_ROOT = os.environ.get('HIL_MTP_FAKE_ROOT', '') +_MARKER_GLOB = f'{_ROOT}/dev/libmtp-*' +_SYS_USB = Path(f'{_ROOT}/sys/bus/usb/devices') +_USB_DEV = Path(f'{_ROOT}/dev/bus/usb') + + +def _bounded_read(path, grace: float = 2.0): + """Read a sysfs attribute with a wall-clock bound, or return None. + + `serial` is served under the device lock a wedged usbfs ioctl holds, and EVERY MTP DUT + is cafe:4017 -- so the vid/pid filter below cannot rule out a wedged NEIGHBOUR, and an + unbounded read of its serial would burn this session's whole budget and report a + healthy board as wedged. Stdlib only by design (this file never imports the harness), + so this is a small local twin of hil_util.read_sysfs. + """ + out = {} + + def _read(): + try: + out['v'] = path.read_text().strip() + except OSError: + pass + + t = threading.Thread(target=_read, daemon=True) + t.start() + t.join(grace) + return out.get('v') + + +def _ready_marker(uid: str): + """(busnum, devnum) of the udev-ready MTP device with this serial, or None. + + /dev/libmtp-<sysname> is published by libmtp-runtime AFTER its synchronous mtp-probe + accepts the device, so this set is both small and ready -- unlike a sysfs-wide scan, + which races re-enumerations from other boards' jobs. Requires the libmtp-runtime + package. + """ + for marker_name in glob.glob(_MARKER_GLOB): + marker = Path(marker_name) + try: + dev = _SYS_USB / marker.name[len('libmtp-'):] + # vid/pid first: lock-free descriptor fields, so they rule out every other + # device before the `serial` read, which the kernel serves under the device + # lock a wedged usbfs ioctl would hold + if ((dev / 'idVendor').read_text().strip() != 'cafe' + or (dev / 'idProduct').read_text().strip() != '4017'): + continue + # bounded: this one CAN block, and a wedged neighbour shares the vid/pid above + serial = _bounded_read(dev / 'serial') + if serial is None or serial.lower() != uid.lower(): + continue + busnum = int((dev / 'busnum').read_text()) + devnum = int((dev / 'devnum').read_text()) + node = _USB_DEV / f'{busnum:03d}' / f'{devnum:03d}' + if marker.resolve(strict=True) != node or not os.access(node, os.R_OK | os.W_OK): + continue + return busnum, devnum + except (OSError, ValueError): + # a marker can vanish while another board flashes: not our device's problem + continue + return None + + +def _gvfs_unmount(uid: str, deadline: float) -> None: + """Drop any gvfs claim on this device, immediately before opening it. + + Called only once the udev marker exists. gvfs claims an MTP device AFTER udev + probing, so before the marker there is nothing to unmount: an earlier call is a + guaranteed no-op that still forks a process, and it leaves the gap between the + unmount and the open unprotected -- the hang this exists to prevent. Per-iteration + calls also forked one gio per second of the enumeration budget. + """ + # Popen, not run(timeout=): run's post-timeout reap is an unbounded wait(), and a gio + # blocked in D state on a wedged usbfs node does not die on SIGKILL, so run(timeout=2) + # can hang for good. Bounded by at most HALF of what is LEFT of our own budget, never + # a fixed sub-bound: the parent gives us --timeout 8 (4 on a retry), so anything larger + # collapsed the poll loop to one attempt and made a slow gio look like a wedged session. + gio_bound = max(0.5, min(3.0, (deadline - time.monotonic()) / 2)) + try: + # argv, not shell=True: uid comes from a hand-edited roster and is board firmware + # output, so a space or $(...) would unmount the wrong URI (leaving the gvfs mount + # held) or run as us. + gio = subprocess.Popen(['gio', 'mount', '-u', + f'mtp://TinyUsb_TinyUsb_Device_{uid}/'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, start_new_session=True) + except OSError: + # glib2.0-bin absent (ci.lan has no gio at all): nothing holds a gvfs mount + # either, so go straight on to the open. + return + try: + gio.wait(timeout=gio_bound) + except subprocess.TimeoutExpired: + try: + os.killpg(gio.pid, signal.SIGKILL) + except OSError: + gio.kill() + try: + gio.wait(timeout=2) # reap it: an abandoned gio leaves a zombie + except subprocess.TimeoutExpired: + pass + print('gio unmount timed out; continuing', file=sys.stderr) + + +def open_mtp_dev(uid: str, timeout: float): + mtp = MTP() + deadline = time.monotonic() + timeout + while True: + try: + # pymtp raises USB_LAYER/PTP_LAYER/GENERAL/AlreadyConnected on a board still + # settling right after a flash; an unguarded raise would skip the rest of the + # enumeration budget (and the disconnect) instead of retrying. + # + # Never detect_devices(): that PROBES every MTP device on the rig, so a board + # still initialising in a parallel job answers our scan (the race #3790 fixed). + # libmtp-runtime publishes /dev/libmtp-<sysname> only after its own mtp-probe + # has accepted a device, so start from that small, ready-only set and open OUR + # device directly by bus/dev address. + target = _ready_marker(uid) + if target: + # ready first, THEN unmount, then open -- see _gvfs_unmount + _gvfs_unmount(uid, deadline) + busnum, devnum = target + # TinyUSB needs no libmtp quirks, so the raw entry can be built here + entry = LIBMTP_DeviceEntry(None, 0xcafe, None, 0x4017, 0) + raw = LIBMTP_RawDevice(entry, busnum, devnum) + mtp.device = mtp.mtp.LIBMTP_Open_Raw_Device(ctypes.byref(raw)) + if mtp.device: + serial = mtp.get_serialnumber() + if (serial.decode('utf-8') if serial else '').lower() == uid.lower(): + return mtp + mtp.disconnect() + except Exception as e: + print(f'mtp poll: {type(e).__name__}: {e}', file=sys.stderr) + # only when a device was actually opened: pymtp's `self.device == None` + # guard does NOT catch a ctypes NULL pointer (falsy, but != None), so + # disconnecting blindly calls LIBMTP_Release_Device(NULL) + if getattr(mtp, 'device', None): + try: + mtp.disconnect() + except Exception: + pass + mtp.device = None + if time.monotonic() >= deadline: + return None + time.sleep(1) + + +def run_session(uid: str, timeout: float) -> int: + mtp = open_mtp_dev(uid, timeout) + if mtp is None or mtp.device is None: + print('MTP device not found') + return 1 + + try: + assert b"TinyUSB" == mtp.get_manufacturer(), 'MTP wrong manufacturer' + assert b"MTP Example" == mtp.get_modelname(), 'MTP wrong model' + assert b'1.0' == mtp.get_deviceversion(), 'MTP wrong version' + assert b'TinyUSB MTP' == mtp.get_devicename(), 'MTP wrong device name' + + f1 = uid.encode("utf-8") + b'_file1' + f2 = uid.encode("utf-8") + b'_file2' + f3 = uid.encode("utf-8") + b'_file3' + mtp.get_file_to_file(1, f1) + with open(f1, 'rb') as file: + f1_data = file.read() + os.remove(f1) + assert f1_data == FILE1_EXPECT, 'MTP file1 wrong data' + mtp.get_file_to_file(2, f2) + with open(f2, 'rb') as file: + f2_data = file.read() + os.remove(f2) + assert FILE2_MD5_EXPECT == hashlib.md5(f2_data).hexdigest(), 'MTP file2 wrong data' + with open(f3, "wb") as file: + # 1524-byte payload + 12-byte MTP header = 3 full 512-byte buffers, so this + # exercises delivery of the final OUT payload before its ZLP. Deliberate and + # FIXED: a random size hits that boundary in ~0.2% of runs, which is not a test + # of it. Deterministic content so a mismatch is reproducible. + f3_data = bytes((i % 251) + 1 for i in range(1524)) + file.write(f3_data) + file.close() + fid = mtp.send_file_from_file(f3, b'file3') + f3_readback = f3 + b'_readback' + mtp.get_file_to_file(fid, f3_readback) + with open(f3_readback, 'rb') as f: + f3_rb_data = f.read() + os.remove(f3_readback) + assert f3_rb_data == f3_data, 'MTP file3 wrong data' + os.remove(f3) + mtp.delete_object(fid) + except AssertionError as e: + print(e) + return 1 + finally: + mtp.disconnect() + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument('--uid', required=True, help='board_get_unique_id serial to match') + parser.add_argument('--timeout', type=float, default=30, help='enumeration wait budget (s)') + args = parser.parse_args() + return run_session(args.uid, args.timeout) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/test/hil/pymtp.py b/test/hil/pymtp.py index 8b694df94..fc0c66104 100644 --- a/test/hil/pymtp.py +++ b/test/hil/pymtp.py @@ -420,6 +420,8 @@ _libmtp.LIBMTP_Get_Playlist.restype = ctypes.POINTER(LIBMTP_Playlist) _libmtp.LIBMTP_Get_Folder_List.restype = ctypes.POINTER(LIBMTP_Folder) _libmtp.LIBMTP_Find_Folder.restype = ctypes.POINTER(LIBMTP_Folder) _libmtp.LIBMTP_Get_Errorstack.restype = ctypes.POINTER(LIBMTP_Error) +_libmtp.LIBMTP_Dump_Errorstack.argtypes = [ctypes.POINTER(LIBMTP_MTPDevice)] +_libmtp.LIBMTP_Dump_Errorstack.restype = None _libmtp.LIBMTP_Open_Raw_Device.restype = ctypes.POINTER(LIBMTP_MTPDevice) _libmtp.LIBMTP_Open_Raw_Device.argtypes = [ctypes.POINTER(LIBMTP_RawDevice)] @@ -451,16 +453,14 @@ class MTP: def debug_stack(self): """ - Checks if __DEBUG__ is set, if so, prints and clears the - errorstack. + Checks if __DEBUG__ is set, and if so prints the error stack. @rtype: None @return: None """ - if __DEBUG__: - self.mtp.LIBMTP_Dump_Errorstack() - #self.mtp.LIBMTP_Clear_Errorstack() + if __DEBUG__ and self.device: + self.mtp.LIBMTP_Dump_Errorstack(self.device) def detect_devices(self): """ diff --git a/test/hil/requirements.txt b/test/hil/requirements.txt index ef1cf575b..abfb93783 100644 --- a/test/hil/requirements.txt +++ b/test/hil/requirements.txt @@ -1,7 +1,8 @@ # System packages (install separately): -# sudo apt install mtools libmtp9 alsa-utils iperf +# sudo apt install mtools libmtp9 libmtp-runtime alsa-utils iperf # mtools - read_disk_file (device/cdc_msc, device/msc_dual_lun) # libmtp9 - pymtp ctypes load (device/mtp); Debian 13 uses libmtp9t64 +# libmtp-runtime - mtp-probe and the completed-device /dev/libmtp-* marker # alsa-utils - arecord (device/audio_test_freertos) # iperf - throughput tests (device/net_lwip_*) hidapi diff --git a/test/hil/test/stubs/hid.py b/test/hil/test/stubs/hid.py new file mode 100644 index 000000000..20a6cccef --- /dev/null +++ b/test/hil/test/stubs/hid.py @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: MIT +"""Scripted stand-in for cython-hidapi, for the HID_ECHO child tests. + +A real wedge cannot be manufactured on demand, so the failure modes are scripted here and +selected with FAKE_HID_MODE. Mirrors test/stubs/pymtp.py, which does the same for libmtp. +""" +import ctypes +import ctypes.util +import os +import time + +_MODE = os.environ.get('FAKE_HID_MODE', 'ok') +_UID = os.environ.get('FAKE_HID_UID', 'CAFE01') + + +def _gil_stall(): + """Block forever WITHOUT releasing the GIL -- the shape cython-hidapi's bare + hid_open()/hid_close() calls have, and the one an in-process bound cannot touch. + + PyDLL, not CDLL: CDLL releases the GIL around the call, which would make this the + easy case instead of the hard one. Resolved through find_library so a non-glibc libc + still works; PyDLL(None) is not usable here (its `sleep` returns immediately). + """ + ctypes.PyDLL(ctypes.util.find_library('c') or 'libc.so.6').sleep(3600) +_PID = int(os.environ.get('FAKE_HID_PID', '0x4012'), 16) + + +def enumerate(vid=0, pid=0): + """Real hid.enumerate(vid, pid) filters on both ids -- 0 means "any" -- and returns a + 'path' key too. The filters are applied BEFORE the locked manufacturer/product reads, + which is why passing both narrows what a wedged peer can stall.""" + if _MODE == 'wedged_enumerate': + # hidapi's hidraw backend reads `manufacturer`/`product` for every device it + # lists, both served under the device lock -- this is that stall. + while True: + time.sleep(3600) + if _MODE == 'absent': + return [] + if vid not in (0, 0xCafe) or pid not in (0, _PID): + return [] + return [{'serial_number': _UID, 'vendor_id': 0xCafe, 'product_id': _PID, + 'path': b'/dev/hidraw0'}] + + +class device: + def __init__(self): + self._last = b'' + + def open(self, vid, pid, serial): + # HID_ECHO really does call this, and usb_autopm/hidraw can block in it, so the + # child must be bounded here too -- exercised by test_a_wedged_open_is_killed. + if _MODE == 'wedged_open': + while True: + time.sleep(3600) + if _MODE == 'wedged_open_gil': + # a thread-based bound is inert against this; only killing the process works + _gil_stall() + + def write(self, report): + self._last = bytes(report) + + def read(self, size, timeout_ms): + if _MODE == 'wedged_read': + while True: + time.sleep(3600) + if _MODE == 'short_read': + return list(self._last[1:4]) + if _MODE == 'wrong_data': + return list(bytes(b ^ 0xFF for b in self._last[1:])) + return list(self._last[1:]) # the device echoes the payload, minus report ID + + def close(self): + if _MODE == 'wedged_close': + # also GIL-holding in cython-hidapi, and it runs in HID_ECHO's finally on + # every failure path + _gil_stall() diff --git a/test/hil/test/stubs/pymtp.py b/test/hil/test/stubs/pymtp.py new file mode 100644 index 000000000..2720321f6 --- /dev/null +++ b/test/hil/test/stubs/pymtp.py @@ -0,0 +1,125 @@ +# SPDX-License-Identifier: MIT +# Fake pymtp for the hil unit tests — stands in both for the import (GitHub's bare +# pre-commit runner has no libmtp/pymtp) and for a scripted MTP device. Behavior is +# driven by env vars so subprocesses (mtp_test.py under run_cmd) can be steered: +# FAKE_PYMTP_MODE absent (default) | ok | hang +# FAKE_PYMTP_UID serial number the fake device reports +# FAKE_PYMTP_FILE1 text served as file id 1 (README.TXT) +# FAKE_PYMTP_LOGO path to the logo bytes served as file id 2 +# File contents come from env, not constants: the test extracts them from the example's +# own sources, so this stub cannot drift out of sync with the firmware. +# 'hang' blocks forever inside detect_devices — the in-process libmtp equivalent of a +# D-state usbfs ioctl on a wedged device. +import ctypes +import os +import time + + +class NotConnected(Exception): + pass + + +class LIBMTP_DeviceEntry(ctypes.Structure): + """Real pymtp exposes this; mtp_test builds one to open a KNOWN device instead of + probing every MTP device on the bus.""" + _fields_ = [('vendor', ctypes.c_char_p), ('vendor_id', ctypes.c_uint16), + ('product', ctypes.c_char_p), ('product_id', ctypes.c_uint16), + ('device_flags', ctypes.c_uint32)] + + +class LIBMTP_RawDevice(ctypes.Structure): + _fields_ = [('device_entry', LIBMTP_DeviceEntry), ('bus_location', ctypes.c_uint32), + ('devnum', ctypes.c_uint8)] + + +class _LibShim: + @staticmethod + def LIBMTP_Open_Raw_Device(_ref): + # mtp_test no longer calls detect_devices() (it probed every MTP device on the + # rig), so the scripted modes have to act here -- this is the only libmtp entry + # point the marker-based open goes through. + mode = os.environ.get('FAKE_PYMTP_MODE', 'absent') + if mode == 'hang': + time.sleep(10000) + if mode == 'error': + raise RuntimeError('CommandFailed: LIBMTP_ERROR_USB_LAYER') + if mode == 'error_then_ok': + flag = os.environ.get('FAKE_PYMTP_ERRED_MARKER', '/tmp/.fake_pymtp_erred') + if not os.path.exists(flag): + open(flag, 'w').close() + raise RuntimeError('CommandFailed: LIBMTP_ERROR_PTP_LAYER') + if mode == 'absent': + return ctypes.POINTER(ctypes.c_int)() # NULL: nothing to open + # the real one has restype POINTER(LIBMTP_MTPDevice): a failed open returns a + # NULL pointer, which is FALSY but compares unequal to None -- the distinction + # mtp_test's `if mtp.device:` guards depend on + if os.environ.get('FAKE_PYMTP_OPEN') == 'null': + return ctypes.POINTER(ctypes.c_int)() + return 1 + + +class MTP: + def __init__(self): + self.mtp = _LibShim() + self.device = None + self._sent = {} + self._next_id = 3 + + def detect_devices(self): + mode = os.environ.get('FAKE_PYMTP_MODE', 'absent') + if mode == 'hang': + time.sleep(10000) + if mode == 'error': + # real pymtp raises for USB_LAYER/PTP_LAYER/GENERAL/AlreadyConnected; + # the first poll after a flash routinely hits one + raise RuntimeError('CommandFailed: LIBMTP_ERROR_USB_LAYER') + if mode == 'error_then_ok': + if not getattr(self, '_erred', False): + self._erred = True + raise RuntimeError('CommandFailed: LIBMTP_ERROR_USB_LAYER') + return [ctypes.c_int(1)] + if mode != 'ok': + return [] + return [ctypes.c_int(1)] + + def get_serialnumber(self): + return os.environ.get('FAKE_PYMTP_UID', '').encode() + + def get_manufacturer(self): + return b'TinyUSB' + + def get_modelname(self): + return b'MTP Example' + + def get_deviceversion(self): + return b'1.0' + + def get_devicename(self): + return b'TinyUSB MTP' + + def get_file_to_file(self, fid, path): + if fid == 1: + data = os.environ['FAKE_PYMTP_FILE1'].encode() + elif fid == 2: + with open(os.environ['FAKE_PYMTP_LOGO'], 'rb') as f: + data = f.read() + else: + data = self._sent[fid] + with open(path, 'wb') as f: + f.write(data) + + def send_file_from_file(self, path, _name): + with open(path, 'rb') as f: + self._sent[self._next_id] = f.read() + self._next_id += 1 + return self._next_id - 1 + + def delete_object(self, fid): + del self._sent[fid] + + def disconnect(self): + # vendored pymtp raises when nothing is connected; a stub that silently accepts + # it hides a LIBMTP_Release_Device(NULL) call on real hardware + if self.device is None: + raise NotConnected('no device connected') + self.device = None diff --git a/test/hil/test/test_ci_metrics.py b/test/hil/test/test_ci_metrics.py new file mode 100644 index 000000000..6f1511913 --- /dev/null +++ b/test/hil/test/test_ci_metrics.py @@ -0,0 +1,581 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests for the by-example half of tools/metrics.py and the (family, example) +# pair-compare script. Stdlib only; synthetic map.json fixtures, no builds. +# python3 test/hil/test/test_ci_metrics.py +import json +import os +import subprocess +import sys +import tempfile +import unittest + +REPO = os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.abspath(__file__))))) +METRICS = os.path.join(REPO, 'tools', 'metrics.py') + + +def fake_map(path, files): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'w') as f: + json.dump({'files': files}, f) + + +def entry(name, size, path_prefix='tinyusb/src'): + return {'file': name, 'path': f'{path_prefix}/{name}', 'size': size, + 'symbols': [{'name': f'{name}_fn', 'size': size}], 'sections': {'.text': size}} + + +class TestByExample(unittest.TestCase): + def build_tree(self, td): + fake_map(os.path.join(td, 'device', 'cdc_msc', 'cdc_msc.map.json'), + [entry('usbd.c', 100), entry('cdc_device.c', 50)]) + fake_map(os.path.join(td, 'host', 'bare_api', 'bare_api.map.json'), + [entry('usbh.c', 200)]) + + def test_by_example_output(self): + with tempfile.TemporaryDirectory() as td: + self.build_tree(td) + out = os.path.join(td, 'metrics') + r = subprocess.run([sys.executable, METRICS, 'combine', '-q', '-j', + '--by-example', '-o', out, + os.path.join(td, '*', '*', '*.map.json')], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + by_ex = json.load(open(out + '_by_example.json')) + self.assertEqual(set(by_ex), {'device/cdc_msc', 'host/bare_api'}) + self.assertEqual({f['file'] for f in by_ex['device/cdc_msc']['files']}, + {'usbd.c', 'cdc_device.c'}) + # the plain averaged output is unchanged by the extra flag + avg = json.load(open(out + '.json')) + self.assertIn('files', avg) + + def test_by_example_json_roundtrips_as_combine_input(self): + with tempfile.TemporaryDirectory() as td: + self.build_tree(td) + out = os.path.join(td, 'metrics') + subprocess.run([sys.executable, METRICS, 'combine', '-q', '-j', '--by-example', + '-o', out, os.path.join(td, '*', '*', '*.map.json')], check=True) + out2 = os.path.join(td, 'sub') + r = subprocess.run([sys.executable, METRICS, 'combine', '-q', '-j', + '-o', out2, out + '_by_example.json'], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + sub = json.load(open(out2 + '.json')) + names = {f['file'] for f in sub['files']} + # one data entry per example, not one blob: reading it as an ordinary + # metrics.json would double-count every file + self.assertIn('usbd.c', names) + self.assertIn('cdc_device.c', names) + self.assertNotIn('TOTAL', {n.upper() for n in names}) + + def test_by_example_expansion_is_keyed_on_the_filename(self): + # the '_by_example.json' suffix IS the contract (write_by_example, the CMake + # rule and metrics_pair_compare all spell it). A shape-sniff would reroute + # any coincidentally-shaped JSON into the per-example branch instead. + with tempfile.TemporaryDirectory() as td: + look_alike = os.path.join(td, 'metrics.json') + with open(look_alike, 'w') as f: + json.dump({'device/cdc_msc': {'files': [entry('usbd.c', 100)]}}, f) + out = os.path.join(td, 'combined') + r = subprocess.run([sys.executable, METRICS, 'combine', '-q', '-j', + '-o', out, look_alike], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + combined = json.load(open(out + '.json')) + self.assertNotIn('usbd.c', {f['file'] for f in combined.get('files', [])}) + + +PAIR_COMPARE = os.path.join(REPO, '.github/scripts/metrics_pair_compare.py') + + +def fake_by_example(root, board, data): + d = os.path.join(root, f'cmake-build-{board}') + os.makedirs(d, exist_ok=True) + with open(os.path.join(d, 'metrics_by_example.json'), 'w') as f: + json.dump(data, f) + + +class TestPairCompare(unittest.TestCase): + def test_intersection_compare(self): + with tempfile.TemporaryDirectory() as td: + base, new = os.path.join(td, 'base'), os.path.join(td, 'new') + # real board names so board->family resolution works against hw/bsp + fake_by_example(base, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('usbd.c', 100)]}, + 'device/dfu': {'files': [entry('dfu_device.c', 10)]}}) + fake_by_example(new, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('usbd.c', 120)]}}) + out = os.path.join(td, 'cmp') + r = subprocess.run([sys.executable, PAIR_COMPARE, '--base-dir', base, + '--new-dir', new, '--out', out], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + md = open(out + '.md').read() + self.assertIn('usbd.c', md) + self.assertNotIn('dfu_device.c', md) # not on both sides + self.assertIn('raspberry_pi_pico', md) # scope footer names the board + self.assertIn('device/dfu', md) # named as dropped + + def test_a_different_board_of_the_same_family_is_not_compared(self): + """--one-first returns all_boards[0], so adding a board can shift which one a + family builds. Keyed on the family, the base run's sizes and the PR run's sizes + would land under one key and the difference between two unrelated MCUs would be + published as this PR's code-size impact.""" + with tempfile.TemporaryDirectory() as td: + base, new = os.path.join(td, 'base'), os.path.join(td, 'new') + # both rp2040, both device/cdc_msc - only the board differs + fake_by_example(base, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('usbd.c', 100)]}}) + fake_by_example(new, 'adafruit_fruit_jam', + {'device/cdc_msc': {'files': [entry('usbd.c', 900)]}}) + out = os.path.join(td, 'cmp') + r = subprocess.run([sys.executable, PAIR_COMPARE, '--base-dir', base, + '--new-dir', new, '--out', out], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + md = open(out + '.md').read() + self.assertIn('skipped', md) + self.assertNotIn('+800', md) + + def test_empty_intersection_writes_note(self): + with tempfile.TemporaryDirectory() as td: + base, new = os.path.join(td, 'base'), os.path.join(td, 'new') + fake_by_example(base, 'raspberry_pi_pico', {'device/dfu': {'files': [entry('a.c', 1)]}}) + fake_by_example(new, 'stm32f407disco', {'device/cdc_msc': {'files': [entry('b.c', 1)]}}) + out = os.path.join(td, 'cmp') + r = subprocess.run([sys.executable, PAIR_COMPARE, '--base-dir', base, + '--new-dir', new, '--out', out], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn('skipped', open(out + '.md').read()) + + def test_malformed_files_are_skipped_with_stderr_note(self): + with tempfile.TemporaryDirectory() as td: + base, new = os.path.join(td, 'base'), os.path.join(td, 'new') + # good pair on both sides -- must survive the malformed siblings below + fake_by_example(base, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('usbd.c', 100)]}}) + fake_by_example(new, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('usbd.c', 120)]}}) + # well-formed JSON, wrong shape (a list, not a {example: {files: [...]}} dict) + wrong_shape = os.path.join(base, 'cmake-build-stm32f407disco', 'metrics_by_example.json') + os.makedirs(os.path.dirname(wrong_shape), exist_ok=True) + with open(wrong_shape, 'w') as f: + json.dump(['not', 'a', 'dict'], f) + # metrics_by_example.json not under a cmake-build-<board> dir + misplaced = os.path.join(base, 'not_a_board_dir', 'metrics_by_example.json') + os.makedirs(os.path.dirname(misplaced), exist_ok=True) + with open(misplaced, 'w') as f: + json.dump({'device/dfu': {'files': [entry('dfu_device.c', 10)]}}, f) + + out = os.path.join(td, 'cmp') + r = subprocess.run([sys.executable, PAIR_COMPARE, '--base-dir', base, + '--new-dir', new, '--out', out], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) # fail-open: never crash the job + md = open(out + '.md').read() + self.assertIn('usbd.c', md) # good pair still compared + self.assertIn(wrong_shape, r.stderr) + self.assertIn(misplaced, r.stderr) + self.assertIn('skipping', r.stderr) + + + def test_missing_base_baseline_gets_its_own_note(self): + # interim state right after this feature merges: master has not uploaded a + # per-example baseline yet, so the BASE side collects nothing. The generic + # "no pair on both sides" note misattributes that to the PR's own scoping. + with tempfile.TemporaryDirectory() as td: + base, new = os.path.join(td, 'base'), os.path.join(td, 'new') + os.makedirs(base) + fake_by_example(new, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('usbd.c', 100)]}}) + out = os.path.join(td, 'cmp') + r = subprocess.run([sys.executable, PAIR_COMPARE, '--base-dir', base, + '--new-dir', new, '--out', out], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + md = open(out + '.md').read() + self.assertIn('No per-example baseline from the base branch yet', md) + self.assertIn('next push', md) + self.assertNotIn('comparison skipped', md) + + def test_a_partially_malformed_file_contributes_nothing(self): + """A file that blows up half way through must drop WHOLE. Entries parsed + before the malformation used to stay in the comparison while stderr claimed + the file had been skipped - a silently truncated table published as the + code-size verdict. A non-list 'files' (TypeError) also has to be caught.""" + with tempfile.TemporaryDirectory() as td: + base, new = os.path.join(td, 'base'), os.path.join(td, 'new') + # good entry FIRST, malformed second: the leak is order-dependent + fake_by_example(base, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('leaked.c', 100)]}, + 'device/dfu': {'files': 42}}) + fake_by_example(new, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('leaked.c', 120)]}}) + # a sibling file that is fine on both sides must still be compared + fake_by_example(base, 'stm32f407disco', + {'device/cdc_msc': {'files': [entry('good.c', 10)]}}) + fake_by_example(new, 'stm32f407disco', + {'device/cdc_msc': {'files': [entry('good.c', 12)]}}) + out = os.path.join(td, 'cmp') + r = subprocess.run([sys.executable, PAIR_COMPARE, '--base-dir', base, + '--new-dir', new, '--out', out], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + md = open(out + '.md').read() + self.assertIn('good.c', md) + self.assertNotIn('leaked.c', md) + self.assertIn('skipping', r.stderr) + self.assertIn(os.path.join(base, 'cmake-build-raspberry_pi_pico'), r.stderr) + + def test_dropped_footer_is_summarised_not_dumped(self): + """The sticky PR comment is capped at 65,536 chars by GitHub; a broad scoped + PR drops hundreds of (family, example) pairs and the full list alone ran to + tens of KB, pushing the comment past the cap and reddening code-metrics.""" + with tempfile.TemporaryDirectory() as td: + base, new = os.path.join(td, 'base'), os.path.join(td, 'new') + common = {'device/cdc_msc': {'files': [entry('usbd.c', 100)]}} + extra = {f'device/example_{i:03d}': {'files': [entry(f'f{i}.c', i + 1)]} + for i in range(30)} + fake_by_example(base, 'raspberry_pi_pico', dict(common, **extra)) + fake_by_example(new, 'raspberry_pi_pico', common) + out = os.path.join(td, 'cmp') + r = subprocess.run([sys.executable, PAIR_COMPARE, '--base-dir', base, + '--new-dir', new, '--out', out], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + md = open(out + '.md').read() + footer = md[md.index('_Scoped compare:'):] + self.assertLess(len(footer), 2048, footer) + self.assertIn('30', footer) # the count is still reported + self.assertIn('more', footer) # truncation marker + self.assertIn('device/example_029', r.stderr) # full list on stderr + + +CIRCLECI = os.path.join(REPO, '.circleci') +SENTINELS = ('example-map-default', 'build-filtered-default') + + +class TestCircleCiSentinelContract(unittest.TestCase): + """config.yml's set-matrix rewrites config2.yml's parameter defaults by matching + a sentinel comment line — the only way past /pipeline/continue's 512-char + parameter cap. Renaming or reformatting either side is a silent full-build + fallback that no CI job reports, so pin the contract here.""" + + def setUp(self): + self.config = open(os.path.join(CIRCLECI, 'config.yml')).read() + self.config2 = open(os.path.join(CIRCLECI, 'config2.yml')).read() + + def test_each_sentinel_appears_once_on_a_default_line(self): + for tag in SENTINELS: + marker = f'# {tag}: rewritten in-place by config.yml set-matrix' + hits = [l for l in self.config2.splitlines() if l.strip().endswith(marker)] + self.assertEqual(len(hits), 1, f'{tag}: {len(hits)} sentinel lines in config2.yml') + self.assertIn('default:', hits[0], f'{tag}: sentinel is not on a default: line') + + def test_the_selection_travels_as_a_file(self): + # a mass-sweep selection runs to hundreds of KB: handed to ci_set_matrix as one + # argv it E2BIGs the step before the `||` fallback can fire, and EXAMPLE_MAP / + # BUILD_FILTERED (derived with jq, no argv limit) would then label a FULL build + # scoped -- the build and its label disagreeing is worse than either alone + self.assertIn('--select-file', self.config) + self.assertNotIn('--select "', self.config) + + def test_the_rewriter_names_the_same_sentinels(self): + for tag in SENTINELS: + self.assertIn(f"'{tag}'", self.config, + f'{tag}: config.yml rewrite block does not name this sentinel') + self.assertIn("# {tag}: rewritten in-place by config.yml set-matrix", self.config, + 'config.yml no longer builds the sentinel comment it matches on') + + def test_the_rewrite_precedes_the_scoped_entries(self): + # the scoping is all-or-nothing: config2's checked-in defaults are {} / false = + # unfiltered, so a rewrite that fails AFTER the family entries were generated + # leaves a subset of families built and code-metrics told it was a full build. + # Rewrite first, and on failure drop the scoping (back to the full matrix). + rewrite = self.config.index("p = '.circleci/config2.yml'") + entries = self.config.index('gen_build_entry() {') + self.assertLess(rewrite, entries, + 'the sentinel rewrite must run before any build entry is generated') + tail = self.config[rewrite:entries] + self.assertIn('MATRIX_JSON="$FULL_MATRIX_JSON"', tail, + 'a failed rewrite must fall back to the FULL matrix, not keep the ' + 'scoped one') + # and that fallback must be a plain assignment: a second `python ...` here is an + # unguarded command under CircleCI's `set -e`, inside the one branch whose whole + # job is to keep the pipeline green + self.assertNotIn('ci_set_matrix.py)', tail) + + def test_the_selector_gate_runs_both_suites(self): + # test_ci_select.py owns the rules; this file owns the sentinel contract the + # very same job rewrites. Gating on one of the two leaves the other unguarded. + for suite in ('test_ci_select.py', 'test_ci_metrics.py'): + self.assertIn(suite, self.config, f'{suite} does not gate the CircleCI selector') + + +class TestWorkflowSelectionHandOff(unittest.TestCase): + """build.yml's counterpart of the CircleCI contract above: same E2BIG limit, same + consequence (the scoping silently turns itself off on exactly the PRs where it + saves most), plus the GITHUB_ENV lines that carry PR-derived values.""" + + def setUp(self): + wf = os.path.join(os.path.dirname(CIRCLECI), '.github', 'workflows') + self.build = open(os.path.join(wf, 'build.yml')).read() + self.util = open(os.path.join(wf, 'build_util.yml')).read() + + def test_no_step_execs_with_the_selection_in_its_environment(self): + # SELECT_JSON="$SELECT_JSON" python3 -c ... E2BIGs at ~128KiB: measured 261KB + # for a `git ls-files hw/bsp/**` sweep. Every reader takes the file instead. + self.assertNotIn('SELECT_JSON="$SELECT_JSON"', self.build) + self.assertIn('json.load(open("ci_select_out.json"))', self.build) + + def test_the_file_is_written_before_its_first_reader(self): + self.assertLess(self.build.index("printf '%s' \"$SELECT_JSON\" > ci_select_out.json"), + self.build.index('json.load(open("ci_select_out.json"))'), + 'the selection file must exist before the step that reads it') + + def test_pr_derived_env_values_are_character_guarded(self): + # values reach GITHUB_ENV/GITHUB_OUTPUT as bare NAME=VALUE lines; a newline in + # one (git allows it in a path, and both the example map and the roster are + # PR-editable) writes extra variables into every later step of a job that runs + # with secrets - and for run_*, flips which rig jobs execute + for name in ('EX_ARGS', 'ARTIFACT_TAG'): + self.assertIn(f'echo "{name}=', self.util) + # per guard, not a sum: `count(a) + count(b) == 2` stays green when one guard is + # deleted and the other duplicated + for guard in ('case "$EX_ARGS" in', 'case "$TAG" in'): + self.assertEqual(self.util.count(guard), 1, + f'{guard}: each GITHUB_ENV write screens its value exactly once') + # CircleCI builds from the same PR-derived map and uses $EX_ARGS unquoted + cci = open(os.path.join(CIRCLECI, 'config2.yml')).read() + self.assertIn('case "$EX_ARGS" in', cci, + 'the CircleCI copy of the example filter needs the same screen') + self.assertIn('case "$BUILD_ARGS" in', self.build) + self.assertIn('unexpected characters in the " + key', self.build, + 'the args_*/run_* emitter must screen each board filter') + + def test_the_guards_accept_what_the_selector_actually_emits(self): + """A guard that rejects a NORMAL value is worse than no guard: build.yml throws + the whole selection away, warns, and both axes fall back to full - silently + turning the feature off. So run the real character classes over real selections + rather than only asserting that the guard text is present. + + The one that got away: `[-A-Za-z0-9_/ .=+]` has no ':' or ',', and every partial + board filter is `-bt <board>:<test>,<test>`.""" + import re, subprocess, sys, tempfile, json + repo = os.path.dirname(CIRCLECI) + # the character classes, lifted from the three places they are written + classes = {} + m = re.search(r're\.fullmatch\(r"\[([^"]+)\]\*"', self.build) + self.assertTrue(m, 'args_*/run_* guard not found in build.yml') + classes['args'] = m.group(1) + for name, text in (('BUILD_ARGS', self.build), ('EX_ARGS', self.util), + ('TAG', self.util)): + m = re.search(r'case "\$%s" in\s*\n\s*\*\[!([^\]]+)\]\*\)' % name, text) + self.assertTrue(m, f'{name} guard not found') + classes[name] = m.group(1).replace('\\', '') + + def ok(cls, value): + return re.fullmatch('[%s]*' % cls.replace('!', ''), value) is not None + + with tempfile.TemporaryDirectory() as d: + for path in ('src/class/cdc/cdc_device.c', 'src/device/usbd.c', + 'src/portable/synopsys/dwc2/dcd_dwc2.c', + 'examples/device/cdc_msc/src/main.c', + 'hw/bsp/stm32f4/family.cmake'): + f = os.path.join(d, 'diff.txt') + with open(f, 'w') as fh: + fh.write(path + '\n') + r = subprocess.run([sys.executable, os.path.join(repo, 'tools/ci_select.py'), + '--diff-file', f, + os.path.join(repo, 'test/hil/tinyusb.json'), + os.path.join(repo, 'test/hil/hfp.json')], + capture_output=True, text=True, cwd=repo) + self.assertEqual(r.returncode, 0, r.stderr) + s = json.loads(r.stdout) + for flasher, a in s.get('args_flasher', {}).get('tinyusb.json', {}).items(): + self.assertTrue(ok(classes['args'], a), + f'{path}/{flasher}: the args guard rejects {a!r}') + hfp = s.get('args', {}).get('hfp.json', '') + self.assertTrue(ok(classes['args'], hfp), f'{path}: hfp {hfp!r}') + # BUILD_ARGS is the hfp job's `-b <board> [-e ...]` list, not the -bt + # test filter above - screen the value that step actually builds + with open(os.path.join(d, 'sel.json'), 'w') as fh: + fh.write(r.stdout) + hm = subprocess.run( + [sys.executable, os.path.join(repo, '.github/scripts/hil_ci_set_matrix.py'), + '--select-file', os.path.join(d, 'sel.json'), + os.path.join(repo, 'test/hil/hfp.json')], + capture_output=True, text=True, cwd=repo) + self.assertEqual(hm.returncode, 0, hm.stderr) + build_args = ' '.join(json.loads(hm.stdout)['arm-gcc']) + self.assertTrue(ok(classes['BUILD_ARGS'], build_args), + f'{path}: the BUILD_ARGS guard rejects {build_args!r}') + for entry in json.loads(hm.stdout)['arm-gcc']: + tag = re.sub(r' -e [^ ]+', '', entry) + self.assertTrue(ok(classes['TAG'], tag), + f'{path}: the artifact-name guard rejects {tag!r}') + for fam, exs in (s.get('build', {}).get('family_examples') or {}).items(): + ex_args = ' '.join('-e ' + e for e in exs) + self.assertTrue(ok(classes['EX_ARGS'], ex_args), + f'{path}/{fam}: the EX_ARGS guard rejects {ex_args!r}') + + def test_an_unusable_selection_is_unusable_for_both_matrices(self): + # hil_ci_set_matrix reads "full false with no boards map" as unusable and falls + # open to the whole roster; if this emitter instead computed run_*=false, the + # rig jobs would skip while all 37 build legs ran - a full build and still zero + # hardware coverage, which is the outcome the guard exists to prevent + self.assertIn('isinstance(s.get("boards"), dict)', self.build) + + def test_the_build_extras_drop_when_the_matrix_falls_open(self): + # ci_set_matrix falls open with rc 0, so the example map and family regex must + # follow it or a nominally full build is filtered and labelled as a scoped one + self.assertIn("grep -q 'ci_set_matrix: UNSCOPED'", self.build) + self.assertIn('BUILD_SELECT_FILE', self.build) + scripts = os.path.join(os.path.dirname(CIRCLECI), '.github', 'scripts') + matrix = open(os.path.join(scripts, 'ci_set_matrix.py')).read() + # count-independent: pin the INVARIANT, not the number of fall-open paths - + # every message that emits the full matrix must carry the marker, and a purely + # informational note (a partial family miss) must not claim to have done so. + # Adjacent string literals are joined first, since these messages wrap. + import re as _re + flat = _re.sub(r"['\"]\s*\n\s*f?['\"]", '', matrix) + hits = [m.start() for m in _re.finditer('emitting the full ', flat)] + self.assertGreaterEqual(len(hits), 2, 'fall-open messages not found') + for i in hits: + self.assertIn('UNSCOPED', flat[max(0, i - 200):i], + 'a fall-open path without the marker build.yml greps for') + + def _run_extras_block(self, sel): + """Extract the build-extras shell block from build.yml and run it for real. + Nothing else exercises it, which is why the empty/rejected conflation shipped.""" + import re as _re, shlex, subprocess, tempfile, json as _json + repo = os.path.dirname(CIRCLECI) + i = self.build.index("EXAMPLE_MAP='{}'\n BUILD_FILTERED='false'") + i = self.build.rindex('\n', 0, i) + 1 + j = self.build.index(' echo "matrix=$MATRIX_JSON"', i) + block = _re.sub(r'^ {10}', '', self.build[i:j], flags=_re.M) + with tempfile.TemporaryDirectory() as d: + selp = os.path.join(d, 'sel.json') + with open(selp, 'w') as fh: + _json.dump(sel, fh) + matrix = subprocess.run( + [sys.executable, os.path.join(repo, '.github/scripts/ci_set_matrix.py'), + '--select-file', selp], capture_output=True, text=True, cwd=repo).stdout.strip() + self.assertTrue(matrix, 'ci_set_matrix produced nothing') + sh = os.path.join(d, 'probe.sh') + with open(sh, 'w') as fh: + # shlex.quote, not hand-rolled quoting: a TMPDIR with a space in it + # made this fail for a reason that had nothing to do with the block + fh.write('BUILD_SELECT_FILE=' + shlex.quote(selp) + '\n') + fh.write('MATRIX_JSON=' + shlex.quote(matrix) + '\n') + fh.write(block) + # sentinel + newline separated: the block itself writes ::warning:: to + # stdout, and '|' would collide with the regex's own separator + fh.write('\nprintf "@@R@@\\n%s\\n%s\\n%s" "$MATRIX_JSON" "$BUILD_FILTERED" "$FAMILY_REGEX"\n') + r = subprocess.run(['bash', sh], capture_output=True, text=True, cwd=repo) + self.assertEqual(r.returncode, 0, r.stderr) + mj, filtered, regex = r.stdout.split('@@R@@\n', 1)[1].split('\n', 2) + return sum(len(v) for v in _json.loads(mj).values()), filtered, regex + + def test_an_empty_family_list_is_not_treated_as_unusable(self): + """.build.families is read twice - as a count and as a `|`-joined regex. An EMPTY + list and one REJECTED by the charset guard both leave the regex empty and mean + opposite things, so the block has to branch on which happened. + + Testing `-z "$FAMILY_REGEX"` alone sent every nothing-selected PR down the + fall-open path and discarded the correct all-empty matrix: #3842 (docs + + .gitignore) and #3840 (test/hil only) each rebuilt all 74 cmake legs after the + selector had correctly chosen none.""" + legs, filtered, regex = self._run_extras_block( + {'build': {'full': False, 'families': [], 'family_examples': {}}}) + self.assertEqual(legs, 0, 'an empty families list must keep the all-empty matrix') + self.assertEqual(filtered, 'false', 'nothing was built, so nothing to compare') + self.assertEqual(regex, '') + + def test_a_real_family_list_stays_scoped(self): + legs, filtered, regex = self._run_extras_block( + {'build': {'full': False, 'families': ['stm32f4', 'rp2040'], + 'family_examples': {}}}) + self.assertGreater(legs, 0) + self.assertEqual(filtered, 'true') + self.assertEqual(regex, 'stm32f4|rp2040') + + def test_a_regex_metacharacter_in_a_family_name_falls_open(self): + # the name is interpolated raw into a name_is_regexp artifact pattern, so a + # metacharacter would match another family's baseline - reject and widen + legs, filtered, regex = self._run_extras_block( + {'build': {'full': False, 'families': ['stm32f4.*'], 'family_examples': {}}}) + self.assertGreater(legs, 100, 'a rejected family list must fall open to full') + self.assertEqual(filtered, 'false') + self.assertEqual(regex, '') + + def test_membrowse_upload_is_not_scoped_by_the_pr_filter(self): + # by decision, the upload runs unfiltered so the size history stays keyed on the + # family's preferred board whatever the PR touched. $EX_ARGS would not have + # scoped the targets either way - `examples-membrowse-upload` is not `all`, so + # resolve_example_target_groups passes it through as the aggregate - but it DID + # move the board, because --one-first picks one that can build the -e set. + # + # The accepted cost: on a family whose preferred board cannot build that set, + # the upload lands on a board the Build step never compiled and every example + # goes up --identical. test_the_upload_board_can_diverge_from_the_built_board + # keeps that consequence measured rather than assumed. + line = [l for l in self.util.splitlines() + if '--target examples-membrowse-upload' in l][0] + self.assertNotIn('$EX_ARGS', line) + self.assertNotIn('-e ', line) + + def test_the_upload_board_can_diverge_from_the_built_board(self): + """Pins the SIZE of what the removal gave up, so it cannot grow unnoticed. + + --one-first with no -e returns preferred_list[0]; with one it returns the first + preferred board that can build it. Where those differ, the Membrowse Upload step + configures a build dir the Build step never wrote. + + ci=True unconditionally, as _prune_buildable does and for the same reason: the + answer must be the runner's, not the developer's. The CI skip lists are off by + default locally, which moves the pick on three families - this test asserted the + local set and went red on its first CI run.""" + sys.path.insert(0, os.path.join(REPO, 'tools')) + import build as build_py + roles = ('device', 'host', 'dual') + exs = sorted(f'{r}/{n}' for r in roles + for n in os.listdir(os.path.join(REPO, 'examples', r)) + if os.path.isdir(os.path.join(REPO, 'examples', r, n))) + fams = sorted(d for d in os.listdir(os.path.join(REPO, 'hw/bsp')) + if os.path.isdir(os.path.join(REPO, 'hw/bsp', d, 'boards'))) + cwd = os.getcwd() + os.chdir(REPO) + try: + diverging = set() + for fam in fams: + try: + base = build_py.get_family_boards(fam, False, True, None, 'cmake', + (), ci=True) + except Exception: + continue + if not base: + continue + for e in exs: + try: + one = build_py.get_family_boards(fam, False, True, [e], 'cmake', + (), ci=True) + except Exception: + continue + if one and one[0] != base[0]: + diverging.add(fam) + break + finally: + os.chdir(cwd) + self.assertEqual(diverging, {'imxrt', 'lpc11', 'lpc18', 'lpc54', 'mcx', 'rx', + 'samd11', 'samd2x_l2x', 'samd5x_e5x', 'stm32l0', + 'stm32l4', 'tm4c'}, + 'the set of families whose membrowse upload can land on an ' + 'uncompiled board changed; re-check whether dropping $EX_ARGS ' + 'from the upload step is still the right trade') + + +if __name__ == '__main__': + unittest.main() diff --git a/test/hil/test/test_ci_select.py b/test/hil/test/test_ci_select.py new file mode 100644 index 000000000..22fbde17b --- /dev/null +++ b/test/hil/test/test_ci_select.py @@ -0,0 +1,2602 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests for ci_select.py — pure logic, no hardware, no git. Run directly: +# python3 test/hil/test/test_ci_select.py +# +# Imports stay stdlib + ci_select/hil_util/hil_flash ONLY: the pre-commit hil-test +# hook runs this suite, on GitHub's bare runner in the pre-commit workflow as well as +# locally, and that runner has no pyserial/pymtp. hil_flash is admissible because it +# is stdlib + hil_util only (test_hil_util.BottomLayer enforces the stdlib closure of +# both) and the roster-dispatch tests need its flash_* table; never import hil_test, +# which pulls pyserial. +import contextlib +import glob +import io +import json +import os +import pathlib +import re +import subprocess +import sys +import unittest + +REPO = os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.abspath(__file__))))) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # test/hil, for hil_flash/helper +sys.path.insert(0, os.path.join(REPO, 'tools')) +import hil_flash +import ci_select +from helper.hil_util import device_tests, dual_tests + + +def real_rosters(): + """The actual rig rosters, for regression tests that need real-world data + (a specific board/family/only-list) rather than the synthetic ROSTER above.""" + rosters = [] + for name in ('tinyusb.json', 'hfp.json'): + path = os.path.join(REPO, 'test/hil', name) + with open(path) as f: + rosters.append((f'test/hil/{name}', json.load(f)['boards'])) + return rosters + + +def roster_flashers(): + """(roster path, board) for every board in the live rosters, `boards-skip` + included: a parked board's flasher name must still dispatch, so that unparking it + is not what discovers the name went stale.""" + for name in ('tinyusb.json', 'hfp.json'): + path = os.path.join(REPO, 'test/hil', name) + with open(path) as f: + cfg = json.load(f) + for key in ('boards', 'boards-skip'): + for b in cfg.get(key, []): + yield f'test/hil/{name}', b + + +def on_roster(tc, *names): + """The subset of `names` currently in the live rig rosters, skipping the test + when none are, because parking/unparking a board is routine rig maintenance. + + That skip now matters MORE than it used to, not less: this suite is a blocking + pre-commit hook AND build.yml's selector steps gate on it (a failing suite falls + open to the full matrix), so an assertion that depends on a specific board being + present goes red on every PR -- including src/-only ones that never touched the + rig -- until someone fixes the roster. Keep roster-dependent assertions behind + on_roster.""" + have = {b['name'] for _, boards in real_rosters() for b in boards} + got = [n for n in names if n in have] + if not got: + tc.skipTest(f'not in the rig roster: {", ".join(names)}') + return got + + +ROSTER = [ + # device-only, rp2040 family + {'name': 'raspberry_pi_pico', 'uid': 'u1', 'flasher': {'name': 'openocd'}, + 'tests': {'device': True, 'host': True, 'dual': True}}, + # device-only, stm32f4 family + {'name': 'stm32f407disco', 'uid': 'u2', 'flasher': {'name': 'jlink'}, + 'tests': {'device': True, 'host': False, 'dual': False}}, + # host-only board + {'name': 'raspberry_pi_pico2', 'uid': 'u3', 'flasher': {'name': 'openocd'}, + 'tests': {'device': False, 'host': True, 'dual': False}}, + # only-list board (espressif-style), flashed by the CI leg that splits on esptool + {'name': 'espressif_s3_devkitm', 'uid': 'u4', 'flasher': {'name': 'esptool'}, + 'tests': {'only': ['device/cdc_msc_freertos', 'host/device_info']}}, +] +ROSTERS = [('test/hil/tinyusb.json', ROSTER)] + + +def sel(files): + return ci_select.classify(files, REPO, ROSTERS) + + +class TestPortRule(unittest.TestCase): + def test_dcd_rp2040_selects_pico_family_only(self): + s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico', s['boards']) + self.assertNotIn('stm32f407disco', s['boards']) + self.assertNotIn('espressif_s3_devkitm', s['boards']) + # device role: no host tests in pico's list + self.assertTrue(all(not t.startswith('host/') for t in s['boards']['raspberry_pi_pico'])) + # host-only boards drop out entirely on a device-role change + self.assertNotIn('raspberry_pi_pico2', s['boards']) + + def test_shared_port_file_is_both_roles(self): + s = sel(['src/portable/synopsys/dwc2/dwc2_common.c']) + self.assertFalse(s['full']) + self.assertNotIn('raspberry_pi_pico', s['boards']) # rp2040 is not a dwc2 family + self.assertIn('stm32f407disco', s['boards']) # stm32f4 is + + +class TestCoreRoleRule(unittest.TestCase): + def test_usbd_selects_all_device_tests_everywhere(self): + s = sel(['src/device/usbd.c']) + self.assertFalse(s['full']) + self.assertNotIn('raspberry_pi_pico2', s['boards']) # host-only board dropped + pico = s['boards']['raspberry_pi_pico'] + self.assertTrue(set(device_tests).issubset(set(pico))) + self.assertTrue(set(dual_tests).issubset(set(pico))) # dual survives device role + self.assertTrue(all(not t.startswith('host/') for t in pico)) + # only-list board: selection intersects its only-list + esp = s['boards']['espressif_s3_devkitm'] + self.assertEqual(esp, ['device/cdc_msc_freertos']) + + def test_host_change_drops_device(self): + s = sel(['src/host/usbh.c']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico2', s['boards']) + self.assertNotIn('stm32f407disco', s['boards']) # device-only board dropped + + +class TestClassRule(unittest.TestCase): + def test_cdc_device_selects_cdc_examples_only(self): + s = sel(['src/class/cdc/cdc_device.c']) + self.assertFalse(s['full']) + pico = s['boards']['raspberry_pi_pico'] + self.assertIn('device/cdc_msc', pico) + self.assertIn('device/cdc_dual_ports', pico) + self.assertNotIn('device/msc_dual_lun', pico) # CFG_TUD_CDC 0 there + self.assertNotIn('device/usbtest', pico) # CFG_TUD_CDC 0 there + self.assertTrue(all(not t.startswith('host/') for t in pico)) + + def test_msc_host_selects_host_side(self): + s = sel(['src/class/msc/msc_host.c']) + self.assertFalse(s['full']) + self.assertNotIn('stm32f407disco', s['boards']) # device-only board + pico2 = s['boards']['raspberry_pi_pico2'] + self.assertIn('host/msc_file_explorer', pico2) + self.assertTrue(all(not t.startswith('device/') for t in pico2)) + + +class TestClassIncludeEdges(unittest.TestCase): + """A class header another class includes reaches that class's examples too. + src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, so + midi_test's firmware contains audio.h - but the class rule derives macros from + the directory name alone, so an audio.h change used to select only + device/audio_test_freertos. On boards that skip that example the per-board + intersection emptied and an audio.h-only PR ran ZERO HIL on them.""" + def test_edges_derived_from_includes(self): + edges = ci_select.class_include_edges(REPO) + self.assertEqual(edges.get('audio/audio.h'), {'midi'}) + self.assertEqual(edges.get('cdc/cdc.h'), {'net'}) + + def test_audio_header_selects_midi_example(self): + s = ci_select.classify(['src/class/audio/audio.h'], REPO, real_rosters()) + self.assertFalse(s['full']) + # every board that runs device/midi_test at all must run it here (boards with + # a tests.only list, e.g. espressif, run the freertos examples instead) + by_name = {b['name']: b for _, bs in real_rosters() for b in bs} + checked = 0 + for name, tests in s['boards'].items(): + if 'device/midi_test' in ci_select.board_tests(by_name[name]): + self.assertIn('device/midi_test', tests, name) + checked += 1 + self.assertTrue(checked) + + def test_audio_header_reaches_boards_that_skip_audio(self): + # both skip device/audio_test_freertos: without the midi edge their + # intersection is empty and they drop out of the selection entirely + boards = on_roster(self, 'metro_m4_express', 'nrf54lm20dk') + s = ci_select.classify(['src/class/audio/audio.h'], REPO, real_rosters()) + for board in boards: + self.assertEqual(s['boards'].get(board), ['device/midi_test'], board) + + def test_edge_is_per_header_not_per_class(self): + # midi includes audio.h, not audio_device.h: an audio_device change must + # not drag midi's examples in + s = ci_select.classify(['src/class/audio/audio_device.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for tests in s['boards'].values(): + if tests != 'all': + self.assertNotIn('device/midi_test', tests) + + +class TestFallbackRules(unittest.TestCase): + def test_unknown_tool_is_full(self): + s = sel(['tools/random_new_script.py']) + self.assertTrue(s['full']) + + def test_docs_only_is_empty_not_full(self): + s = sel(['docs/info/contributing.rst', 'README.rst']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + def test_bsp_family_selects_family_boards(self): + s = sel(['hw/bsp/rp2040/family.cmake']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico', s['boards']) + self.assertEqual(s['boards']['raspberry_pi_pico'], 'all') + self.assertNotIn('stm32f407disco', s['boards']) + + def test_bsp_board_narrows_to_board(self): + s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) + self.assertFalse(s['full']) + self.assertEqual(list(s['boards'].keys()), ['raspberry_pi_pico']) + + def test_example_change_selects_that_example(self): + s = sel(['examples/device/cdc_msc/src/main.c']) + self.assertFalse(s['full']) + self.assertEqual(s['boards']['raspberry_pi_pico'], ['device/cdc_msc']) + + def test_core_common_is_full(self): + for f in ['src/tusb.c', 'src/common/tusb_fifo.c', 'src/osal/osal_freertos.h']: + self.assertTrue(sel([f])['full'], f) + + def test_board_test_example_is_full(self): + # board_test is the park/teardown firmware hil_test.py flashes on every board, + # not an unlisted example: a regression there must not skip the whole rig + for f in ['examples/device/board_test/src/main.c', + 'examples/device/board_test/CMakeLists.txt']: + self.assertTrue(sel([f])['full'], f) + + def test_harness_is_full(self): + # hw/mcu/ is no longer here: it resolves to families/boards via mcu_families() + # instead of forcing full - see TestMcuHilRule + for f in ['test/hil/hil_test.py', '.github/workflows/build.yml']: + self.assertTrue(sel([f])['full'], f) + + def test_mixed_roles_no_pruning(self): + s = sel(['src/device/usbd.c', 'src/host/usbh.c']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico2', s['boards']) + self.assertIn('stm32f407disco', s['boards']) + + def test_cmakelists_and_requirements_are_full(self): + for f in ['src/CMakeLists.txt', 'examples/CMakeLists.txt', + 'examples/device/CMakeLists.txt', 'test/hil/requirements.txt']: + self.assertTrue(sel([f])['full'], f) + + def test_docs_txt_is_noncode(self): + s = sel(['docs/info/changelog.txt']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + +class TestArgsEmission(unittest.TestCase): + def test_args_for_scoped_selection(self): + s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + args = ci_select.selection_args(s, ROSTERS) + a = args['tinyusb.json'] + self.assertIn('-b raspberry_pi_pico', a) + self.assertNotIn('stm32f407disco', a) + self.assertIn('-bt raspberry_pi_pico:', a) # device-only subset of a device+host board + + def test_args_full_is_empty(self): + s = sel(['tools/random_new_script.py']) + self.assertEqual(ci_select.selection_args(s, ROSTERS), {'tinyusb.json': ''}) + + def test_args_all_board_gets_bare_b(self): + s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) + a = ci_select.selection_args(s, ROSTERS)['tinyusb.json'] + self.assertIn('-b raspberry_pi_pico', a) + self.assertNotIn('-bt', a) + + def test_args_by_flasher_splits_esp_from_the_rest(self): + s = sel(['src/device/usbd.c']) + per = ci_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json'] + self.assertIn('espressif_s3_devkitm', per['esptool']) + self.assertIn('raspberry_pi_pico', per['openocd']) + self.assertNotIn('espressif_s3_devkitm', per.get('openocd', '') + per.get('jlink', '')) + + def test_args_by_flasher_omits_a_flasher_with_no_selected_board(self): + # the esp CI leg must see no args at all here, not a filter matching zero boards + s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) + per = ci_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json'] + self.assertEqual(per, {'openocd': '-b raspberry_pi_pico'}) + + def test_args_by_flasher_full_is_empty(self): + s = sel(['tools/random_new_script.py']) + self.assertEqual(ci_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}}) + + def test_cli_diff_file(self): + import subprocess, tempfile, json as j + with tempfile.NamedTemporaryFile('w', suffix='.txt', delete=False) as f: + f.write('src/class/cdc/cdc_device.c\n') + path = f.name + r = subprocess.run([sys.executable, os.path.join(REPO, 'tools/ci_select.py'), + '--diff-file', path, os.path.join(REPO, 'test/hil/tinyusb.json')], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + out = j.loads(r.stdout) + self.assertFalse(out['full']) + self.assertIn('tinyusb.json', out['args']) + # reasons are a stderr diagnostic, deliberately NOT in the payload: they were + # 97% of a 9.8 MB JSON on a dep bump, and every consumer re-parses that file + self.assertNotIn('reasons', out, 'reasons must not ride in the machine-read JSON') + self.assertNotIn('reasons', out['build']) + self.assertIn('cdc_device', r.stderr) + # A core-class diff must select boards THROUGH THE CLI: the in-process tests + # inject their own repo root, so only this subprocess path catches a broken + # repo_root derivation -- which once made every repo-relative glob match + # nothing and turned this exact diff into a silent full-HIL skip. + self.assertTrue(out['boards'], + 'CLI selected zero boards for a src/class change: repo_root broken?') + os.unlink(path) + + +class TestRealRosterPortFamilies(unittest.TestCase): + """Regression for port_families() missing espressif's dwc2 reference, which + lives in a component CMakeLists.txt rather than family.cmake/family.mk.""" + def test_dwc2_change_selects_espressif_boards(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = ci_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertIn(board, s['boards']) + + +class TestOptionGatedPort(unittest.TestCase): + """Regression: family_support.cmake compiles some ports from a build option + (MAX3421_HOST=1 -> hcd_max3421.c), so a board's family file never names them.""" + # host-side option board (max3421 as host controller), off any max3421 family + OPT_ROSTER = [('test/hil/opt.json', [ + {'name': 'fake_dual_board', 'uid': 'o1', 'flasher': {'name': 'jlink'}, + 'variant': [{'name': 'fake_dual_board', 'defines': ['MAX3421_HOST=1']}], + 'tests': {'device': True, 'host': False, 'dual': True}}, + {'name': 'fake_host_board', 'uid': 'o2', 'flasher': {'name': 'jlink'}, + 'variant': [{'name': 'fake_host_board', 'flags': '-DMAX3421_HOST=1'}], + 'tests': {'device': False, 'host': True, 'dual': False}}, + {'name': 'fake_off_board', 'uid': 'o3', 'flasher': {'name': 'jlink'}, + 'variant': [{'name': 'fake_off_board', 'defines': ['MAX3421_HOST=0']}], + 'tests': {'device': True, 'host': True, 'dual': True}}, + ])] + + def test_real_roster_max3421_selects_option_board(self): + boards = on_roster(self, 'metro_m4_express') + s = ci_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertIn(board, s['boards']) + + def test_option_selects_via_defines_and_flags(self): + s = ci_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, self.OPT_ROSTER) + self.assertFalse(s['full']) + self.assertIn('fake_dual_board', s['boards']) # variant defines + self.assertIn('fake_host_board', s['boards']) # variant flags + self.assertNotIn('fake_off_board', s['boards']) # variant defines, but =0 + + def test_device_role_port_does_not_pull_host_only_option_board(self): + s = ci_select.classify(['src/portable/analog/max3421/dcd_max3421.c'], REPO, self.OPT_ROSTER) + self.assertFalse(s['full']) + self.assertNotIn('fake_host_board', s['boards']) # host-only board, device change + self.assertIn('fake_dual_board', s['boards']) # device-capable option board + + def test_gates_parsed_from_family_support(self): + self.assertEqual(ci_select.port_option_gates(REPO).get('analog/max3421'), + {'MAX3421_HOST'}) + + def test_board_cmake_option_counts(self): + """A board can enable a gated port in its own BSP rather than via the roster + (hw/bsp/espressif/boards/*/board.cmake -> set(MAX3421_HOST 1)); board_options() + must see those too, or such a board joining the roster is silently dropped.""" + self.assertIn('MAX3421_HOST', + ci_select.bsp_board_options('adafruit_feather_esp32s3', REPO)) + self.assertIn('CFG_TUH_RPI_PIO_USB', + ci_select.bsp_board_options('adafruit_fruit_jam', REPO)) + # commented-out `# set(MAX3421_HOST 1)` must not count + self.assertNotIn('MAX3421_HOST', + ci_select.bsp_board_options('feather_nrf52840_express', REPO)) + + def test_board_cmake_option_selects_off_family_board(self): + # adafruit_feather_esp32s3 is not on any rig roster; stand it in as one to + # prove the BSP-sourced option alone pulls a max3421 change onto the board + roster = [('test/hil/opt.json', [ + {'name': 'adafruit_feather_esp32s3', 'uid': 'o1', 'flasher': {'name': 'esptool'}, + 'tests': {'device': False, 'host': True, 'dual': False}}])] + s = ci_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster) + self.assertFalse(s['full']) + self.assertIn('adafruit_feather_esp32s3', s['boards']) + + def test_board_mk_option_is_ignored(self): + """Make-only options must not select: HIL CI builds with CMake exclusively, so + hw/bsp/nrf/boards/nrf5340dk/board.mk's MAX3421_HOST compiles nothing here.""" + roster = [('test/hil/opt.json', [ + {'name': 'nrf5340dk', 'uid': 'o1', 'flasher': {'name': 'jlink'}, + 'tests': {'device': False, 'host': True, 'dual': False}}])] + s = ci_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + +class TestPortFamiliesCmakeOnly(unittest.TestCase): + """port_families() is CMake-only (HIL CI never builds with Make) and matches on + 'port_dir/' so a port dir is not a prefix of a sibling.""" + def test_make_only_family_is_not_a_family(self): + # hw/bsp/pic32mz has family.mk but no family.cmake + self.assertEqual(ci_select.port_families('microchip/pic32mz', REPO), set()) + + def test_prefix_port_does_not_inherit_sibling_families(self): + # bare-substring matching let 'microchip/pic' match '.../microchip/pic32mz/...' + self.assertEqual(ci_select.port_families('microchip/pic', REPO), set()) + + def test_make_only_port_contributes_nothing(self): + s = sel(['src/portable/microchip/pic32mz/dcd_pic32mz.c']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons']) + + def test_cmake_families_still_found(self): + self.assertEqual(ci_select.port_families('raspberrypi/rp2040', REPO), {'rp2040'}) + self.assertIn('stm32f4', ci_select.port_families('synopsys/dwc2', REPO)) + + +class TestPortFamiliesCoverage(unittest.TestCase): + """Systematic guard: every real dcd_*/hcd_* port directory should map to at + least one board family, so a future family.cmake/CMakeLists.txt layout that + port_families() doesn't scan fails loudly instead of silently dropping boards + (as espressif's dwc2 reference did - see TestRealRosterPortFamilies).""" + # Ports with no board family: not a bug, just not wired into any rig board. + # Add here (with a reason) only if port_families() legitimately can't find one. + # A port listed here contributes NOTHING on either axis (empty means empty), so + # this list is the tripwire: a port that stops resolving must show up as a test + # failure, not as a PR that quietly builds and tests nothing. + NO_FAMILY = { + 'template', # reference/example port, not built by any board + # hw/bsp/pic32mz has family.mk only (no family.cmake), and port_families() + # is CMake-only because HIL CI builds every board with CMake - so this port + # is compiled for no HIL board. + 'microchip/pic32mz', + 'microchip/pic', # same: only ever referenced from pic32mz's family.mk + } + + @staticmethod + def _dcd_hcd_ports(): + portable_root = os.path.join(REPO, 'src/portable') + ports = [] + for entry in sorted(os.listdir(portable_root)): + d = os.path.join(portable_root, entry) + if not os.path.isdir(d): + continue + if glob.glob(os.path.join(d, 'dcd_*.c')) or glob.glob(os.path.join(d, 'hcd_*.c')): + ports.append(entry) + continue + for sub in sorted(os.listdir(d)): + sd = os.path.join(d, sub) + if os.path.isdir(sd) and (glob.glob(os.path.join(sd, 'dcd_*.c')) or + glob.glob(os.path.join(sd, 'hcd_*.c'))): + ports.append(f'{entry}/{sub}') + return ports + + def test_every_port_maps_to_a_family(self): + ports = self._dcd_hcd_ports() + self.assertTrue(ports) # sanity: the scan itself found something + for port in ports: + if port in self.NO_FAMILY: + continue + fams = ci_select.port_families(port, REPO) + self.assertTrue(fams, f'{port}: no family references this port ' + f'(port_families() scan gap, or add to NO_FAMILY)') + + +class TestRealRosterOnlyListTests(unittest.TestCase): + """Regression for roster-only-list tests (e.g. espressif's hid_composite_freertos) + being invisible to the selector because it only knew the shared hil_util lists.""" + def test_only_list_example_change_selects_it(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = ci_select.classify(['examples/device/hid_composite_freertos/src/main.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertEqual(s['boards'][board], ['device/hid_composite_freertos']) + + def test_class_change_includes_only_list_boards(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = ci_select.classify(['src/class/hid/hid_device.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertIn(board, s['boards']) + + +class TestPortAndCoreRoleUseExtras(unittest.TestCase): + """Regression: the port rule and core-role rule must thread the roster-only + test universe (extras) the same way the class rule already does, so a DCD + or device-stack change doesn't silently drop espressif's only-list tests + (e.g. hid_composite_freertos) that aren't in the shared device_tests list.""" + def test_dcd_change_includes_only_list_test(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = ci_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + tests = s['boards'][board] + if tests == 'all': + continue # a board whose whole allowed set is selected collapses to 'all' + self.assertIn('device/hid_composite_freertos', tests) + self.assertIn('device/cdc_msc_freertos', tests) + self.assertIn('device/audio_test_freertos', tests) + self.assertIn('device/usbtest', tests) + + def test_core_device_change_includes_only_list_test(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = ci_select.classify(['src/device/usbd.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + tests = s['boards'][board] + if tests == 'all': + continue # a board whose whole allowed set is selected collapses to 'all' + self.assertIn('device/hid_composite_freertos', tests) + self.assertIn('device/cdc_msc_freertos', tests) + self.assertIn('device/audio_test_freertos', tests) + self.assertIn('device/usbtest', tests) + + def test_host_change_does_not_leak_device_only_list_test(self): + s = ci_select.classify(['src/host/usbh.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board, tests in s['boards'].items(): + if tests == 'all': + continue + self.assertNotIn('device/hid_composite_freertos', tests, board) + + +class TestFamilies(unittest.TestCase): + """`families` exists for consumers that build (not just test) the diff: most + families have no rig board, so `boards` alone would compile nothing for them.""" + def test_off_rig_port_still_reports_family(self): + s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) # no same7x board on the rig + self.assertEqual(s['families'], ['same7x']) + + def test_port_families_are_reported(self): + s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + self.assertIn('rp2040', s['families']) + + def test_bsp_family_and_board_report_family(self): + self.assertEqual(sel(['hw/bsp/rp2040/family.cmake'])['families'], ['rp2040']) + self.assertEqual(sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h'])['families'], + ['rp2040']) + + def test_docs_only_has_no_families(self): + self.assertEqual(sel(['docs/info/contributing.rst'])['families'], []) + + def test_full_selection_still_reports_families(self): + """A full-matrix file must not hide the families of the other changed files: + consumers that build from `families` (e.g. /pre-pr) ignore `boards` when full.""" + s = sel(['src/common/tusb_fifo.c', 'src/portable/microchip/samx7x/dcd_samx7x.c']) + self.assertTrue(s['full']) + self.assertIn('same7x', s['families']) + # full stays full: every roster board, and no args to narrow the run + self.assertEqual(set(s['boards']), {b['name'] for b in ROSTER}) + self.assertTrue(all(v == 'all' for v in s['boards'].values())) + self.assertEqual(ci_select.selection_args(s, ROSTERS), {'tinyusb.json': ''}) + self.assertEqual(ci_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}}) + + def test_family_order_does_not_matter(self): + # same as above with the full-matrix file last (was the only order that worked) + s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c', 'src/common/tusb_fifo.c']) + self.assertTrue(s['full']) + self.assertIn('same7x', s['families']) + + +class TestGitDiffArgv(unittest.TestCase): + def test_diff_disables_rename_detection(self): + """Without --no-renames git reports only a rename's destination, so moving an + HIL-relevant file to a non-code path would be classified as non-code only.""" + self.assertIn('--no-renames', ci_select.GIT_DIFF_ARGV) + + +class TestPortWithoutFamilyContributesNothing(unittest.TestCase): + """A port dir no family file references contributes nothing on BOTH axes (the + maintainer's empty-means-empty ruling): nothing compiles the file, so there is + nothing to run. Forcing the full 30-board rig here bought no coverage - the build + walk answered the identical condition with zero families for the same path.""" + def test_unreferenced_port_contributes_nothing(self): + orig = ci_select.port_families + ci_select.port_families = lambda port_dir, repo_root: set() + try: + s = sel(['src/portable/vendor/newip/dcd_newip.c']) + b = ci_select.classify_build(['src/portable/vendor/newip/dcd_newip.c'], REPO) + finally: + ci_select.port_families = orig + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons']) + self.assertFalse(b['full']) + self.assertEqual(b['families'], []) + + +class TestOpenocdVidPid(unittest.TestCase): + """The roster's optional flasher `vid_pid` field (openocd-verbatim, e.g. + "0x1a86 0x8010", more pairs appended) pins openocd's probe discovery so it + never opens foreign usbfs nodes. It must be emitted BEFORE the args: the + rescue cfgs run `init` internally (rp2350-rescue.cfg errors on any + config-stage command after its init; rp2040.cfg under RESCUE scans before a + trailing flag is even parsed), and no rig cfg sets a competing list + (the 2026-08-10 convoy mechanism).""" + + def test_vid_pid_flag_precedes_args(self): + cmd = hil_flash._openocd_cmd_base( + {'uid': 'S1', 'args': '-f target/wch-riscv.cfg', 'vid_pid': '0x1a86 0x8010'}) + self.assertIn('-c "adapter usb vid_pid 0x1a86 0x8010" -f target/wch-riscv.cfg', cmd) + self.assertTrue(cmd.endswith('-f target/wch-riscv.cfg'), cmd) + + def test_rescue_cfg_command_keeps_vid_pid_before_init(self): + """rescue_openocd swaps the target cfg for one that runs `init` internally; + a vid_pid flag after the args would error there (rp2350) or be skipped + (rp2040) -- in exactly the wedged-rig scenario the pin exists for.""" + flasher = {'name': 'openocd', 'uid': 'S1', 'vid_pid': '0x2e8a 0x000c', + 'args': '-c "set RESCUE 1" -f target/rp2040.cfg'} + cmd = hil_flash._openocd_cmd_base(flasher) + self.assertLess(cmd.index('adapter usb vid_pid'), cmd.index('-f target/'), cmd) + + def test_vid_pid_multiple_pairs(self): + cmd = hil_flash._openocd_cmd_base( + {'uid': 'S1', 'args': '-f i.cfg', 'vid_pid': '0x2e8a 0x000c 0x2e8a 0x000d'}) + self.assertIn('-c "adapter usb vid_pid 0x2e8a 0x000c 0x2e8a 0x000d"', cmd) + + def test_no_field_no_flag_but_warns(self): + # the roster lint only covers the committed rosters; a dev PC's local.json entry + # without the field must at least say what it is giving up -- on STDERR, since + # hil_test captures stdout per test and would swallow it on a passing run + import io + from contextlib import redirect_stderr + hil_flash._VID_PID_WARNED.discard('S-warn') + cap = io.StringIO() + with redirect_stderr(cap): + cmd = hil_flash._openocd_cmd_base({'uid': 'S-warn', 'args': '-f i.cfg'}) + self.assertNotIn('vid_pid', cmd) + self.assertIn('vid_pid', cap.getvalue()) + + def test_roster_openocd_entries_all_pin_vid_pid(self): + # every openocd probe on the rig has a known VID/PID; a new entry without the + # pin silently reintroduces open-everything discovery + for path, board in roster_flashers(): + f = board['flasher'] + # tinyusb.json only: hfp.json is the hifiphile rig owner's file, and a + # blocking repo-wide lint over someone else's roster would red every PR the + # moment they add an openocd board (hil_flash treats the field as optional) + if f['name'] == 'openocd' and path.endswith('tinyusb.json'): + self.assertIn('vid_pid', f, + f"{path}: {board['name']} openocd flasher lacks vid_pid") + self.assertNotIn('vid_pid', f.get('args', ''), + f"{path}: {board['name']} packs vid_pid into args; use the field") + + +class TestRosterFlashersDispatch(unittest.TestCase): + """hil_test and hil_pool_check resolve a board's flasher with a bare + getattr(hil_flash, f'flash_{name}'), and hil_test does it inside a redirect_stdout — + so a renamed or typo'd roster name raises an AttributeError whose output is swallowed, + with nothing pointing at the roster as the thing to edit. Renaming a flash_*/reset_* + pair without updating every roster must fail here instead.""" + + def test_flash_and_reset_exist_for_every_roster_flasher(self): + for path, board in roster_flashers(): + name = board['flasher']['name'].lower() + for fn in (f'flash_{name}', f'reset_{name}'): + self.assertTrue(callable(getattr(hil_flash, fn, None)), + f'{path}: {board["name"]} uses flasher "{name}" ' + f'but hil_flash.{fn} does not exist') + + def test_firmware_suffix_known_for_every_roster_flasher(self): + """find_firmware falls back to accepting .elf-or-.bin when a flasher is missing + from FLASHER_SUFFIX, silently restoring the mismatch that map exists to catch.""" + for path, board in roster_flashers(): + name = board['flasher']['name'].lower() + self.assertIn(name, hil_flash.FLASHER_SUFFIX, + f'{path}: {board["name"]} uses flasher "{name}" ' + f'with no hil_flash.FLASHER_SUFFIX entry') + + +class FlasherRecoverEntry(unittest.TestCase): + """Optional roster key: a SECOND flasher used only to deliver recovery while a usbfs + node is poisoned. Boards whose primary flasher cannot get past a convoy (jlink, + stlink, lm4flash) name an openocd entry here instead of changing how they are + normally flashed.""" + + def test_recover_flasher_prefers_the_optional_entry(self): + prim = {'name': 'jlink', 'uid': 'X', 'args': '-device MIMXRT1064xxx6A'} + rec = {'name': 'openocd', 'uid': 'X', 'args': '-f interface/jlink.cfg -f target/foo.cfg'} + self.assertEqual(hil_flash.recover_flasher({'flasher': prim, 'flasher_recover': rec}), rec) + self.assertEqual(hil_flash.recover_flasher({'flasher': prim}), prim) + + def test_openocd_over_jlink_is_convoy_safe_without_a_pin(self): + """libjaylink discovery returns early unless idVendor == 0x1366 (SEGGER) and the PID + is in its table, and only THEN calls libusb_open (discovery_usb.c) -- it never opens + a foreign node. `adapter usb vid_pid` is a no-op for this driver: jlink.c reads + adapter_serial / usb address / usb location, never the vid/pid.""" + self.assertTrue(hil_flash.convoy_safe( + {'name': 'openocd', 'args': '-f interface/jlink.cfg -f target/stm32f4x.cfg'})) + + def test_openocd_with_neither_a_pin_nor_jlink_is_not_safe(self): + self.assertFalse(hil_flash.convoy_safe( + {'name': 'openocd', 'args': '-f interface/stlink.cfg -f target/stm32h7x.cfg'})) + + def test_the_existing_rules_are_unchanged(self): + self.assertTrue(hil_flash.convoy_safe( + {'name': 'openocd', 'vid_pid': '0x2e8a 0x000c', 'args': '-f interface/cmsis-dap.cfg'})) + self.assertFalse(hil_flash.convoy_safe({'name': 'jlink', 'uid': 'X'})) + self.assertTrue(hil_flash.convoy_safe({'name': 'esptool'})) + + +class TestModuleMove(unittest.TestCase): + def test_repo_root_guard(self): + # __file__-derived root: moving the module without re-deriving the parent + # count re-points every scan at the wrong tree (it happened once already) + self.assertTrue(os.path.isdir(os.path.join(ci_select._REPO_ROOT, 'src'))) + self.assertTrue(os.path.isdir(os.path.join(ci_select._REPO_ROOT, 'hw', 'bsp'))) + self.assertEqual(os.path.realpath(ci_select._REPO_ROOT), os.path.realpath(REPO)) + + +class TestPathFamilies(unittest.TestCase): + def test_port_wrapper_unchanged(self): + self.assertEqual(ci_select.port_families('raspberrypi/rp2040', REPO), {'rp2040'}) + self.assertIn('stm32f4', ci_select.port_families('synopsys/dwc2', REPO)) + + def test_boundary_without_trailing_slash(self): + # hw/bsp/nrf/family.cmake writes `${TOP}/hw/mcu/nordic/nrfx` — no trailing + # slash; the match must accept a directory-boundary end-of-token + self.assertEqual(ci_select.path_families('hw/mcu/nordic/nrfx', REPO), {'nrf'}) + + def test_boundary_rejects_prefix_sibling(self): + # 'microchip/pic' must not inherit pic32mz's references (and pic32mz itself + # is family.mk-only, which the CMake-only scan never reads) + self.assertEqual(ci_select.port_families('microchip/pic', REPO), set()) + self.assertEqual(ci_select.port_families('microchip/pic32mz', REPO), set()) + + def test_mcu_families_prefix_walk(self): + self.assertEqual(ci_select.mcu_families('hw/mcu/nordic/nrf5x/nrf_clock.h', REPO), {'nrf'}) + self.assertEqual(ci_select.mcu_families('hw/mcu/dialog/da1469x/x.h', REPO), {'da1469x'}) + self.assertEqual(ci_select.mcu_families('hw/mcu/no_such_vendor/x.c', REPO), set()) + + +class TestMcuHilRule(unittest.TestCase): + def test_mcu_no_longer_forces_full(self): + s = ci_select.classify(['hw/mcu/nordic/nrf5x/nrf_clock.h'], REPO, ROSTERS) + self.assertFalse(s['full']) + self.assertIn('nrf', s['families']) # recorded even with no nrf rig board + + def test_mcu_selects_family_boards(self): + got = on_roster(self, 'feather_nrf52840_express', 'pca10056', 'pca10095') + s = ci_select.classify(['hw/mcu/nordic/nrf5x/nrf_clock.h'], REPO, real_rosters()) + self.assertFalse(s['full']) + for b in got: + self.assertIn(b, s['boards']) + + def test_unresolved_mcu_path_selects_nothing(self): + # empty means empty (maintainer ruling): if no family's build references the + # path, no build consumes the change - there is nothing to compile or run. + # test_tracked_mcu_vendors_resolve is the drift guard for a real vendor dir + s = ci_select.classify(['hw/mcu/no_such_vendor/x.c'], REPO, ROSTERS) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + self.assertEqual(s['families'], []) + + +class TestOrphanInvariant(unittest.TestCase): + ALLOW = {'microchip/pic', 'microchip/pic32mz'} # spec: known orphans, CMake builds neither + + def test_every_port_resolves_to_a_family(self): + for d in sorted(glob.glob(os.path.join(REPO, 'src/portable/*/*'))): + if not os.path.isdir(d): + continue + port = os.path.relpath(d, os.path.join(REPO, 'src/portable')).replace(os.sep, '/') + fams = ci_select.port_families(port, REPO) + if port in self.ALLOW: + self.assertEqual(fams, set(), f'{port}: no longer an orphan - drop it from ALLOW') + else: + self.assertTrue(fams, f'{port}: no family.cmake references it - wire it up or allowlist it') + + def test_tracked_mcu_vendors_resolve(self): + import subprocess as sp + r = sp.run(['git', 'ls-files', 'hw/mcu'], cwd=REPO, capture_output=True, text=True) + if r.returncode != 0: + self.skipTest('not a git checkout') + vendors = sorted({'/'.join(p.split('/')[:3]) for p in r.stdout.split()}) + for v in vendors: + self.assertTrue(ci_select.mcu_families(v + '/x.c', REPO), f'{v}: resolves to no family') + + # hw/bsp families ci_set_matrix's family_list does not map to any toolchain. Master + # gave a PR touching one of these no compile coverage either - none of the other 64 + # families compiles same7x's board.h - so this is not new. What IS new is that the + # gap used to be masked by a full matrix and is now the whole answer, which is why + # ci_set_matrix treats a selection that intersects family_list to NOTHING as + # unusable (UNSCOPED -> full matrix) rather than emitting an all-empty one. + # espressif is here because hil-build-esp builds its boards by name rather than by + # family - though only on hathach/tinyusb: that job is gated on repository_owner, + # so on a fork an espressif-only PR builds nowhere. + UNBUILT_FAMILIES = {'cxd56', 'efm32', 'espressif', 'f1c100s', 'pic32mz', 'py32f0', + 'same7x'} + + def test_every_bsp_family_is_in_the_ci_matrix(self): + sys.path.insert(0, os.path.join(REPO, '.github/scripts')) + import ci_set_matrix + fams = set(ci_select.all_bsp_families(REPO)) + self.assertEqual(fams - set(ci_set_matrix.family_list), self.UNBUILT_FAMILIES, + 'a hw/bsp family that no toolchain in ci_set_matrix.family_list ' + 'builds: a PR touching only it now selects zero build legs. Wire ' + 'it into family_list, or add it here with a reason.') + + def test_every_get_deps_family_token_resolves_or_is_a_known_alias(self): + """Same drift guard, dep side. A token naming no hw/bsp dir makes the entry + unreachable for its family in get_deps.py itself (`f in entry[2].split()`), and + makes a bump of it select nothing here. The four known ones are pinned; a fifth + appearing is a real bug in get_deps.py, not something to swallow.""" + sys.path.insert(0, os.path.join(REPO, 'tools')) + import get_deps + fams = set(ci_select.all_bsp_families(REPO)) + stale = {} + for name, d in (('deps_mandatory', get_deps.deps_mandatory), + ('deps_optional', get_deps.deps_optional)): + for path, entry in d.items(): + for tok in str(entry[2]).split(): + if tok != 'all' and tok not in fams: + stale.setdefault(tok, []).append(f'{name}[{path}]') + # subset, not equality: correcting a token in get_deps.py (fc100s -> f1c100s) + # should be a one-file change, while a NEW unmappable token - which force-fulls + # every get_deps edit that touches its entry - has to be a deliberate act + self.assertFalse(set(stale) - set(ci_select._DEPS_ALIAS_TOKENS), + f'get_deps family tokens naming no hw/bsp dir: ' + f'{ {k: v for k, v in stale.items() if k not in ci_select._DEPS_ALIAS_TOKENS} }') + + +class TestRostersDoNotOverlap(unittest.TestCase): + """sel['boards'] is one map across every roster, so a board listed in TWO rosters + with different test lists would get the union - and hil_test.py on the rig that + only runs half of them would be handed a -t it has no fixture for. No overlap + exists today; this is the tripwire for the day one is added.""" + + def test_no_board_name_is_in_two_rosters(self): + seen = {} + for name in ('tinyusb.json', 'hfp.json'): + cfg = json.load(open(os.path.join(REPO, 'test/hil', name))) + for b in cfg['boards']: + if b['name'] in seen: + self.assertEqual( + seen[b['name']], b.get('tests'), + f"{b['name']}: on two rosters with different test lists - " + f"selection_args must then filter per roster, not from the union") + seen[b['name']] = b.get('tests') + + +class TestTypecRule(unittest.TestCase): + """Rule 12b. src/typec/usbc.c is listed unconditionally by src/CMakeLists.txt and + src/tinyusb.mk, but its whole body is `#if CFG_TUC_ENABLED`, which only + examples/typec/power_delivery sets - so it is parsed by every build and compiled by + one. Same shape as the class rule, same answer. Before this rule it matched nothing + and force-fulled 82 families and all 30 rig boards.""" + + def test_build_axis_selects_only_the_typec_examples(self): + s = ci_select.classify_build(['src/typec/usbc.c'], REPO) + self.assertFalse(s['full']) + self.assertTrue(s['families'], 'typec must be compiled somewhere') + self.assertTrue(s['family_examples'], 'and the examples must be named') + for fam, exs in s['family_examples'].items(): + self.assertTrue(exs, fam) + for e in exs: + self.assertTrue(e.startswith('typec/'), f'{fam}: {e} is not a typec example') + + def test_every_typec_file_answers_the_same(self): + for f in ('src/typec/usbc.c', 'src/typec/usbc.h', 'src/typec/tcd.h', + 'src/typec/pd_types.h'): + s = ci_select.classify_build([f], REPO) + self.assertFalse(s['full'], f) + self.assertTrue(s['families'], f) + + def test_no_rig_board_runs_typec(self): + # typec is not a HIL role, so the rig cannot exercise it whatever it selects + s = sel(['src/typec/usbc.c']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + def test_it_tracks_the_enabling_config_rather_than_a_hardcoded_list(self): + # the answer must come from CFG_TUC_ENABLED in the example configs, so it + # follows a new typec example (or an old one switched off) on its own + want = ci_select.examples_enabling( + ci_select.role_examples(REPO, ('typec',)), ('CFG_TUC_ENABLED',), REPO) + self.assertTrue(want, 'no example enables CFG_TUC_ENABLED - rule 12b is dead') + got = set() + for exs in ci_select.classify_build(['src/typec/usbc.c'], REPO)['family_examples'].values(): + got |= set(exs) + self.assertEqual(got, want) + + +class TestCachesAreKeyedOnTheTree(unittest.TestCase): + """build_utils caches on repo-RELATIVE paths while ci_select._in_repo() chdirs + between trees, so the cwd has to be part of every cache key. Without it a second + tree gets the first tree's skip.txt/only.txt and FAMILY_MCUS - which is exactly the + base-vs-branch comparison the code-size skill does in one process.""" + + def test_a_second_tree_is_not_answered_from_the_first(self): + import build_utils, tempfile + old = os.getcwd() + try: + os.chdir(REPO) + self.assertFalse(build_utils.skip_example('host/bare_api', 'metro_m0_express')) + with tempfile.TemporaryDirectory() as d: + os.makedirs(os.path.join(d, 'hw/bsp'), exist_ok=True) + os.chdir(d) + # the board does not exist in this tree at all -> unknown board -> skip + self.assertTrue(build_utils.skip_example('host/bare_api', 'metro_m0_express'), + 'the empty tree was answered from the repo tree cache') + os.chdir(REPO) + self.assertFalse(build_utils.skip_example('host/bare_api', 'metro_m0_express'), + 'and the repo answer must survive the excursion') + finally: + os.chdir(old) + + +class TestClassesWithNoEnablingExample(unittest.TestCase): + """The class rule is the one rule with no drift guard: ports, hw/mcu, get_deps + tokens and bsp families all have one. A class dir that no example config enables + selects NOTHING on both axes (the maintainer's empty-means-empty ruling), which is + right - but it must be a listed state, not a surprise, or a class added before its + first example silently stops being built.""" + + # class dirs no example's tusb_config.h turns on, for either role. Must only shrink: + # a new entry means a class nothing compiles, so a break in it reaches master. + NO_EXAMPLE = {'bth'} + + def test_only_the_known_classes_select_nothing(self): + import glob as _glob + dead = set() + for d in sorted(_glob.glob(os.path.join(REPO, 'src/class/*'))): + if not os.path.isdir(d): + continue + cls = os.path.basename(d) + hit = False + for base in sorted(os.path.basename(f) for f in _glob.glob(os.path.join(d, '*.[ch]'))): + roles = ci_select._class_roles(base) + if ci_select._build_class_examples(cls, base, roles, REPO): + hit = True + break + if not hit: + dead.add(cls) + self.assertEqual(dead, self.NO_EXAMPLE, + 'a class dir enabled by no example config: it selects nothing on ' + 'both axes, so nothing compiles it until the next master push') + + +class TestTheHarnessTestsAreNotTheHarness(unittest.TestCase): + """test/hil/test/ selects nothing; test/hil/ itself still selects everything. + + Rule 2 is a bare `test/hil/` prefix, so the harness's own unit tests were booking + the full 27-board rig - ~11 minutes of exclusive hardware for a diff that cannot + reach it. Nothing on the rig runs them: pre-commit does, and build.yml runs + test_ci_select.py as the gate before trusting a selection at all. + + The carve-out is only safe while that directory holds nothing rig-affecting, which + is what the second test pins.""" + + def test_the_harness_own_tests_select_nothing_on_either_axis(self): + for p in ('test/hil/test/test_ci_select.py', 'test/hil/test/test_ci_metrics.py', + 'test/hil/test/test_hil_bounded.py', 'test/hil/test/stubs/pymtp.py', + 'test/hil/test/stubs/hid.py'): + s = ci_select.classify([p], REPO, ROSTERS) + self.assertFalse(s['full'], p) + self.assertFalse(s['boards'], p) + b = ci_select.classify_build([p], REPO) + self.assertFalse(b['full'], p) + self.assertFalse(b['families'], p) + + def test_the_harness_itself_still_takes_the_whole_rig(self): + # the thing rule 2 exists for: these decide what the rig does, so they cannot be + # trusted to narrow their own blast radius + for p in ('test/hil/hil_test.py', 'test/hil/tinyusb.json', + 'test/hil/helper/hil_ci_set_matrix.py'): + s = ci_select.classify([p], REPO, ROSTERS) + self.assertTrue(s['full'], f'{p} must still force the full rig') + + def test_nothing_rig_affecting_has_moved_into_the_carve_out(self): + """The carve-out is a claim about that directory's contents; pin them. + + A new file there that the rig DOES read would silently stop selecting the rig. + Listing them costs one line per file and makes that a failing test instead.""" + out = subprocess.run(['git', 'ls-files', 'test/hil/test'], cwd=REPO, + capture_output=True, text=True, check=True) + self.assertEqual(sorted(out.stdout.split()), [ + 'test/hil/test/stubs/hid.py', + 'test/hil/test/stubs/pymtp.py', + 'test/hil/test/test_ci_metrics.py', + 'test/hil/test/test_ci_select.py', + 'test/hil/test/test_hil_bounded.py', + 'test/hil/test/test_hil_health.py', + 'test/hil/test/test_hil_report.py', + 'test/hil/test/test_hil_rtt.py', + 'test/hil/test/test_hil_util.py', + ], 'test/hil/test/ gained or lost a file; it is carved out of rule 2, so confirm ' + 'the rig still does not read anything in there before updating this list') + + +class TestExampleMapOmitsFullFamilies(unittest.TestCase): + """A family whose selection is ALREADY everything it can build carries no -e list. + + Sixth of the same shape as the class below, found the same way: a perf rewrite of + _prune_buildable dropped the `set(kept) != set(buildable)` test and all 216 tests + stayed green. The build outcome is identical either way -- build.py applies the same + skip_example the pruner just did -- so nothing compiled differently and only the + payload grew (22 families x 33 examples on one dcd_dwc2.c diff). That is exactly the + kind of drift no build failure ever reports.""" + + def test_a_device_only_port_diff_still_omits_families_it_cannot_narrow(self): + # dcd_dwc2.c selects device+dual examples only, but a family whose host examples + # are all unbuildable anyway ends up wanting its entire buildable set + b = ci_select.classify_build(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO) + self.assertFalse(b['full']) + self.assertTrue(b['families']) + omitted = [f for f in b['families'] if f not in b['family_examples']] + self.assertTrue(omitted, 'no family omitted its -e list; the "already everything ' + 'this family builds" case stopped being detected') + for fam in omitted: + self.assertNotIn(fam, b['family_examples']) + + def test_a_family_that_can_build_more_than_the_diff_wants_keeps_its_list(self): + # the other direction: one example selects itself and nothing else, so every + # family it lands on must carry an explicit -e or CI builds all 46 + b = ci_select.classify_build(['examples/device/cdc_msc/src/main.c'], REPO) + self.assertFalse(b['full']) + for fam in b['families']: + self.assertEqual(b['family_examples'].get(fam), ['device/cdc_msc'], fam) + + +class TestSelectionBehavioursThatHadNoTest(unittest.TestCase): + """Five behaviours a reviewer's mutation pass proved were unpinned: break each one + and the whole suite stayed green. Each test here fails against its mutant. + + They are grouped because they share a shape - every one is a small expression whose + removal silently NARROWS the selection, which is the failure direction that merges a + regression rather than wasting a runner.""" + + def test_build_defines_reach_the_prefilter(self): + # mutant: `defines = ()` in build.py's build_boards_list. metro_m4_express gets + # MAX3421_HOST=1 from its roster variant, never from its BSP, so without the + # defines the -e prefilter drops the rig's only MAX3421 firmware and hil-tinyusb + # has nothing to flash. + import build as build_py, build_utils, inspect + src = inspect.getsource(build_py.build_boards_list) + self.assertIn('defines = tuple(sorted(build_defines))', src, + 'the -D tokens must reach cmake_board/skip_example') + old = os.getcwd() + os.chdir(REPO) + try: + ex, board = 'dual/host_info_to_device_cdc', 'metro_m4_express' + self.assertTrue(build_utils.skip_example(ex, board), + 'without the define this example is correctly skipped') + self.assertFalse(build_utils.skip_example(ex, board, ('MAX3421_HOST=1',)), + 'with it, it must build - that is what the roster passes') + finally: + os.chdir(old) + + def test_one_first_prefers_a_board_that_can_build_the_filter(self): + # mutant: buildable() -> True, i.e. back to all_boards[0]. lpc54's first board + # skips every msc_file_explorer example, so the leg would compile nothing. + import build as build_py + old_env, old = os.environ.get('GITHUB_ACTIONS'), os.getcwd() + os.environ['GITHUB_ACTIONS'] = 'true' + os.chdir(REPO) + try: + unfiltered = build_py.get_family_boards('lpc54', False, True) + filtered = build_py.get_family_boards('lpc54', False, True, + ['host/msc_file_explorer']) + self.assertEqual(unfiltered, ['lpcxpresso54114'], 'unfiltered pick must not move') + self.assertNotEqual(filtered, unfiltered, + 'the -e pick must avoid a board that skips the whole filter') + import build_utils + self.assertFalse(build_utils.skip_example('host/msc_file_explorer', filtered[0]), + f'{filtered[0]} must actually build the filtered example') + finally: + os.chdir(old) + if old_env is None: + os.environ.pop('GITHUB_ACTIONS', None) + else: + os.environ['GITHUB_ACTIONS'] = old_env + + def test_a_class_file_selects_its_own_macro_not_just_the_directory(self): + # mutant: delete the _CLS_STEM_RE block. src/class/midi holds MIDI 1.0 AND 2.0; + # examples/device/midi2_device is the only example enabling CFG_TUD_MIDI2 and the + # only one that compiles midi2_device.c, but the directory macro alone misses it. + got = ci_select._build_class_examples('midi', 'midi2_device.c', {'device'}, REPO) + self.assertIn('device/midi2_device', got, + 'a midi2 change must select the example that compiles it') + host = ci_select._build_class_examples('midi', 'midi2_host.c', {'host'}, REPO) + self.assertIn('host/midi2_host', host) + # and the plain midi files must NOT drag midi2 in + plain = ci_select._build_class_examples('midi', 'midi_device.c', {'device'}, REPO) + self.assertNotIn('device/midi2_device', plain) + + def test_a_port_change_selects_the_dual_examples(self): + # mutant: drop `+ ('dual',)`. A dcd/hcd change must build the dual examples - + # they exercise both stacks on one board, so a dwc2 break lands there first. + s = ci_select.classify_build(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO) + duals = {e for exs in s['family_examples'].values() for e in exs + if e.startswith('dual/')} + self.assertTrue(duals, 'a dcd change selected no dual example') + + def test_the_selector_answers_the_same_with_and_without_ci_env(self): + # mutant: drop ci=True from _prune_buildable. ci_skip_boards/ci_preferred_boards + # only apply when GITHUB_ACTIONS/CIRCLECI is set, so without the pin a laptop and + # a runner disagree - and /pre-pr would report a family list CI will not build. + files = ['examples/host/cdc_msc_hid_freertos/src/main.c'] + old = os.environ.get('GITHUB_ACTIONS') + os.environ.pop('GITHUB_ACTIONS', None) + try: + local = ci_select.classify_build(files, REPO)['families'] + os.environ['GITHUB_ACTIONS'] = 'true' + import importlib + importlib.reload(ci_select) + runner = ci_select.classify_build(files, REPO)['families'] + finally: + if old is None: + os.environ.pop('GITHUB_ACTIONS', None) + else: + os.environ['GITHUB_ACTIONS'] = old + import importlib + importlib.reload(ci_select) + self.assertEqual(local, runner, 'the selector must not depend on the CI env vars') + + +class TestRuleTableIsCarbonOfTheSpec(unittest.TestCase): + """ci_select's module docstring carries the rule table so a reader landing in the + code does not have to open the spec to learn what rule 6 is. Both are maintained by + hand, so this pins them cell-for-cell: edit one without the other and this fails. + + It also pins the table against the CODE - every rule id the docstring claims must + appear as a `# rule N` marker on a branch of _classify_build_one, so a row cannot be + documented without a branch, or a branch renumbered without the table.""" + + @staticmethod + def _rows(text): + import re as _re + out = [] + for l in text.splitlines(): + if not l.startswith('| '): + continue + c = [x.strip() for x in l.strip().strip('|').split('|')] + if len(c) == 5 and _re.fullmatch(r'\d+[a-z]?', c[0]): + out.append(c) + return out + + def test_docstring_table_matches_the_spec(self): + spec = open(os.path.join( + REPO, 'docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md')).read() + doc, spec_rows = self._rows(ci_select.__doc__), self._rows(spec) + self.assertTrue(spec_rows, 'no rule table found in the spec') + self.assertEqual([r[0] for r in doc], [r[0] for r in spec_rows], + 'rule ids differ between ci_select.__doc__ and the spec') + for d, s in zip(doc, spec_rows): + self.assertEqual(d, s, f'rule {d[0]} differs between the docstring and the spec') + + def test_every_documented_rule_has_a_branch(self): + import re as _re + src = open(os.path.join(REPO, 'tools/ci_select.py')).read() + marked = set() + # handles `# rule 6`, `# rules 1, 1b` and `# rules 8-10` + for m in _re.finditer(r'#\s*rules?\s+([0-9a-z, -]+)', src): + for tok in _re.split(r',\s*', m.group(1).strip()): + rng = _re.fullmatch(r'(\d+)\s*-\s*(\d+)', tok.strip()) + if rng: + marked.update(str(n) for n in range(int(rng.group(1)), int(rng.group(2)) + 1)) + elif _re.fullmatch(r'\d+[a-z]?', tok.strip()): + marked.add(tok.strip()) + documented = {r[0] for r in self._rows(ci_select.__doc__)} + missing = sorted(documented - marked, key=lambda s: (int(_re.match(r'\d+', s).group()), s)) + self.assertEqual(missing, [], f'documented rules with no `# rule N` branch marker: {missing}') + + +class TestNoTrackedFileIsUnclassified(unittest.TestCase): + """Rule 17 (unclassified -> full on both axes) is the fail-open net for paths nobody + anticipated. It must stay that way - a wrong `full` costs runner minutes and is + visible in the run, a wrong `empty` costs a merged regression and is invisible - but + nothing in the tree should REACH it. Every tracked file is classified by a rule, so + 17 fires only for genuinely new shapes, and this test is what tells the author to + write the row instead of letting the fall-through pick an answer for them. + + Before this guard, 254 tracked files reached 17: .gitignore took a docs-only PR to + 74 cmake legs and the whole rig, while examples/<role>/CMakeLists.txt got the RIGHT + answer from the wrong rule - row 15 names it, the regex never matched it.""" + + def _unclassified(self, axis): + import subprocess as sp + r = sp.run(['git', 'ls-files'], cwd=REPO, capture_output=True, text=True) + if r.returncode != 0: + self.skipTest('not a git checkout') + files = r.stdout.split() + self.assertGreater(len(files), 1000, 'suspiciously few tracked files') + out = [] + for f in files: + s = (ci_select.classify_build([f], REPO) if axis == 'build' + else ci_select.classify([f], REPO, real_rosters())) + if any('unclassified' in why for why in s['reasons']): + out.append(f) + return out + + def test_build_axis(self): + left = self._unclassified('build') + self.assertEqual(left, [], f'{len(left)} tracked files fall through to rule 17 on ' + f'the build axis, e.g. {left[:5]} - classify them, or ' + f'add the pattern to _META_RE if no build reads them') + + def test_hil_axis(self): + left = self._unclassified('hil') + self.assertEqual(left, [], f'{len(left)} tracked files fall through to rule 17 on ' + f'the HIL axis, e.g. {left[:5]}') + + +class TestLibRule(unittest.TestCase): + """lib/** is not a full-matrix path: only the examples that build the lib need it.""" + + def b(self, files): + return ci_select.classify_build(files, REPO) + + def test_lib_examples_ground_truth(self): + self.assertEqual(ci_select.lib_examples('embedded-cli', REPO), + {'host/msc_file_explorer', 'host/msc_file_explorer_freertos'}) + self.assertEqual(ci_select.lib_examples('networking', REPO), + {'device/net_lwip_webserver'}) + # only family_support.cmake's LOGGER=rtt plumbing names it, and no CI example + # build turns that on - the scan is per-example on purpose + self.assertEqual(ci_select.lib_examples('SEGGER_RTT', REPO), set()) + self.assertEqual(ci_select.lib_examples('rt-thread', REPO), set()) + + def test_lib_examples_matches_at_a_directory_boundary(self): + # 'lib/net' must not inherit lib/networking's example + self.assertEqual(ci_select.lib_examples('net', REPO), set()) + + def test_build_lib_selects_only_the_using_examples(self): + s = self.b(['lib/embedded-cli/embedded_cli.h']) + self.assertFalse(s['full']) + self.assertTrue(s['families']) + want = {'host/msc_file_explorer', 'host/msc_file_explorer_freertos'} + mapped = set() + for fam, exs in s['family_examples'].items(): + self.assertTrue(set(exs) <= want, f'{fam}: {exs}') + mapped |= set(exs) + self.assertEqual(mapped, want) + + def test_build_lib_nobody_builds_selects_nothing(self): + s = self.b(['lib/SEGGER_RTT/RTT/SEGGER_RTT.c']) + self.assertFalse(s['full']) + self.assertEqual(s['families'], []) + self.assertEqual(s['family_examples'], {}) + + def test_hil_lib_selects_the_using_tests(self): + s = sel(['lib/embedded-cli/embedded_cli.h']) + self.assertFalse(s['full']) + want = {'host/msc_file_explorer', 'host/msc_file_explorer_freertos'} + self.assertEqual(set(s['boards']['raspberry_pi_pico']), want) + self.assertEqual(set(s['boards']['raspberry_pi_pico2']), want) + # device-only board and the only-list board run neither test + self.assertNotIn('stm32f407disco', s['boards']) + self.assertNotIn('espressif_s3_devkitm', s['boards']) + + def test_hil_lib_used_only_by_a_disabled_test_selects_nothing(self): + # device/net_lwip_webserver is commented out of hil_util.device_tests, so the + # intersection with the HIL universe is empty + s = sel(['lib/networking/dhserver.c']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + def test_hil_lib_nobody_builds_selects_nothing(self): + s = sel(['lib/SEGGER_RTT/RTT/SEGGER_RTT.c']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + +# A miniature get_deps.py: the module shape the parser must cope with (imports, +# both dep dicts, the derived deps_all, a function) without the real 300-entry file. +_GD_BASE = """#!/usr/bin/env python3 +import argparse + +deps_mandatory = { + 'lib/fatfs': ['https://github.com/abbrev/fatfs.git', 'aaa', 'all'], +} + +deps_optional = { + 'hw/mcu/st/cmsis_device_f4': ['https://github.com/x/f4.git', 'bbb', 'stm32f4 stm32f7'], + 'hw/mcu/nordic/nrfx': ['https://github.com/x/nrfx.git', 'ccc', 'nrf'], +} + +deps_all = {**deps_mandatory, **deps_optional} + + +def main(): + return 1 +""" + + +class TestGetDepsChangedFamilies(unittest.TestCase): + """Pure text-in, families-out: no git, no exec of the parsed module.""" + + def f(self, head, base=_GD_BASE): + return ci_select.get_deps_changed_families(base, head, REPO) + + def test_no_change_selects_nothing(self): + self.assertEqual(self.f(_GD_BASE), set()) + + def test_comment_only_change_selects_nothing(self): + self.assertEqual(self.f(_GD_BASE.replace('import argparse', + 'import argparse # noqa')), set()) + + def test_optional_commit_bump_selects_its_families(self): + self.assertEqual(self.f(_GD_BASE.replace("'bbb'", "'bbb2'")), + {'stm32f4', 'stm32f7'}) + + def test_mandatory_all_entry_is_full(self): + self.assertIsNone(self.f(_GD_BASE.replace("'aaa'", "'aaa2'"))) + + def test_logic_change_is_full(self): + self.assertIsNone(self.f(_GD_BASE.replace('return 1', 'return 2'))) + + def test_unparseable_text_is_full(self): + self.assertIsNone(self.f('def broken(:\n')) + + def test_unresolvable_token_is_full(self): + # a changed entry we cannot map to a family is NOT "nothing changed": reading it + # that way empties the whole build matrix for a dep bump. Fall open instead - + # even when a sibling token does resolve, because the unmapped one may be the + # family that actually needed the new revision + base = _GD_BASE.replace("'ccc', 'nrf'", "'ccc', 'zz_gone samd5x_e5x'") + self.assertIsNone(self.f(base.replace("'ccc'", "'ccc2'"), base)) + base = _GD_BASE.replace("'ccc', 'nrf'", "'ccc', 'zz_gone'") + self.assertIsNone(self.f(base.replace("'ccc'", "'ccc2'"), base)) + + def test_family_token_change_unions_both_sides(self): + # the family list itself edited: both sides contribute + head = _GD_BASE.replace("'ccc', 'nrf'", "'ccc', 'rp2040 samd5x_e5x'") + self.assertEqual(self.f(head), {'nrf', 'rp2040', 'samd5x_e5x'}) + + def test_known_alias_tokens_select_nothing(self): + # the tokens in _DEPS_ALIAS_TOKENS name no hw/bsp dir: either a pre-rename + # spelling sitting beside the current name in the same entry, or a family with + # no boards in the tree. Changing one selects nothing rather than force-fulling + # every get_deps edit that touches its entry. + base = _GD_BASE.replace("'ccc', 'nrf'", "'ccc', 'stm32l5'") + self.assertEqual(self.f(base.replace("'ccc'", "'ccc2'"), base), set()) + + def test_moving_an_entry_between_the_two_dicts_is_seen(self): + # value untouched, dict changed: mandatory deps are fetched for every family, so + # demoting one stops families fetching it. Merging the dicts before diffing (or + # comparing the ast dump of deps_all) hides this completely. + head = _GD_BASE.replace( + " 'hw/mcu/nordic/nrfx': ['https://github.com/x/nrfx.git', 'ccc', 'nrf'],\n", '') + head = head.replace( + "deps_mandatory = {\n", + "deps_mandatory = {\n 'hw/mcu/nordic/nrfx': ['https://github.com/x/nrfx.git', 'ccc', 'nrf'],\n") + self.assertEqual(self.f(head), {'nrf'}) + + def test_added_entry_selects_its_families(self): + head = _GD_BASE.replace( + "deps_optional = {\n", + "deps_optional = {\n 'hw/mcu/x': ['https://github.com/x/x.git', 'ddd', 'rp2040'],\n") + self.assertEqual(self.f(head), {'rp2040'}) + + def test_removed_entry_selects_its_base_side_families(self): + head = _GD_BASE.replace( + " 'hw/mcu/nordic/nrfx': ['https://github.com/x/nrfx.git', 'ccc', 'nrf'],\n", '') + self.assertEqual(self.f(head), {'nrf'}) + + def test_family_list_change_unions_both_sides(self): + head = _GD_BASE.replace("'stm32f4 stm32f7'", "'stm32f4 stm32h7'") + self.assertEqual(self.f(head), {'stm32f4', 'stm32f7', 'stm32h7'}) + + def test_real_get_deps_parses(self): + with open(os.path.join(REPO, 'tools/get_deps.py')) as f: + real = f.read() + self.assertEqual(ci_select.get_deps_changed_families(real, real, REPO), set()) + # a real optional entry bumped resolves to that entry's real family. The commit + # is read out of get_deps.py rather than pinned here - a routine dep bump must + # not fail this suite, and pinning a hash tests the tree, not the code + sys.path.insert(0, os.path.join(REPO, 'tools')) + import get_deps + commit, tokens = get_deps.deps_optional['hw/mcu/nordic/nrfx'][1:3] + bumped = real.replace(commit, '0' * len(commit)) + self.assertNotEqual(bumped, real) + self.assertEqual(ci_select.get_deps_changed_families(real, bumped, REPO), + set(tokens.split())) + + +class TestGetDepsRule(unittest.TestCase): + """tools/get_deps.py: the changed dep entries' families, or full when unknowable.""" + + def test_build_selects_the_changed_families(self): + s = ci_select.classify_build(['tools/get_deps.py'], REPO, + get_deps_families={'stm32f4'}) + self.assertFalse(s['full']) + self.assertEqual(s['families'], ['stm32f4']) + self.assertNotIn('stm32f4', s['family_examples']) # every example it builds + + def test_build_without_a_base_is_full(self): + # --diff-file mode has no git and so no base content: fail open + self.assertTrue(ci_select.classify_build(['tools/get_deps.py'], REPO)['full']) + + def test_build_no_dep_entry_changed_selects_nothing(self): + s = ci_select.classify_build(['tools/get_deps.py'], REPO, get_deps_families=set()) + self.assertFalse(s['full']) + self.assertEqual(s['families'], []) + + def test_hil_selects_the_changed_families_boards(self): + s = ci_select.classify(['tools/get_deps.py'], REPO, ROSTERS, + get_deps_families={'stm32f4'}) + self.assertFalse(s['full']) + self.assertEqual(list(s['boards']), ['stm32f407disco']) + self.assertEqual(s['families'], ['stm32f4']) + + def test_hil_without_a_base_is_full(self): + self.assertTrue(ci_select.classify(['tools/get_deps.py'], REPO, ROSTERS)['full']) + + def test_hil_no_dep_entry_changed_selects_nothing(self): + s = ci_select.classify(['tools/get_deps.py'], REPO, ROSTERS, get_deps_families=set()) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + def test_cli_diff_file_mode_is_full(self): + import tempfile, json as j + with tempfile.NamedTemporaryFile('w', suffix='.txt', delete=False) as f: + f.write('tools/get_deps.py\n') + path = f.name + r = subprocess.run([sys.executable, os.path.join(REPO, 'tools/ci_select.py'), + '--diff-file', path, os.path.join(REPO, 'test/hil/tinyusb.json')], + capture_output=True, text=True) + os.unlink(path) + self.assertEqual(r.returncode, 0, r.stderr) + out = j.loads(r.stdout) + self.assertTrue(out['full']) + self.assertTrue(out['build']['full']) + + +class TestGetDepsGitPlumbing(unittest.TestCase): + """--base mode: merge-base, the diff, and both blobs come from git, and only + tools/get_deps.py in the diff triggers the blob reads.""" + + HEAD = _GD_BASE.replace("'bbb'", "'bbb2'") + + def run_main(self, diff): + from unittest import mock + calls = [] + + def fake_run(argv, **kw): + calls.append(argv) + if argv[:2] == ['git', 'merge-base']: + out = 'MB123\n' + elif argv[:3] == ci_select.GIT_DIFF_ARGV[:3]: + out = diff + elif argv[:2] == ['git', 'show']: + out = _GD_BASE if argv[2].startswith('MB123:') else self.HEAD + else: + raise AssertionError(f'unexpected git call: {argv}') + return subprocess.CompletedProcess(argv, 0, stdout=out, stderr='') + + buf = io.StringIO() + argv = [sys.executable, '--base', 'origin/master'] + with mock.patch.object(ci_select.subprocess, 'run', fake_run), \ + mock.patch.object(sys, 'argv', argv), \ + contextlib.redirect_stdout(buf), contextlib.redirect_stderr(io.StringIO()): + ci_select.main() + return json.loads(buf.getvalue()), calls + + def test_base_mode_reads_the_merge_base_blob(self): + out, calls = self.run_main('tools/get_deps.py\n') + self.assertIn(['git', 'show', 'MB123:tools/get_deps.py'], calls) + self.assertIn(['git', 'show', 'HEAD:tools/get_deps.py'], calls) + self.assertFalse(out['build']['full']) + self.assertEqual(out['build']['families'], ['stm32f4', 'stm32f7']) + + def test_no_get_deps_in_the_diff_reads_no_blob(self): + out, calls = self.run_main('src/class/cdc/cdc_device.c\n') + self.assertFalse(any(c[:2] == ['git', 'show'] for c in calls)) + self.assertFalse(out['build']['full']) + + def test_git_failure_falls_open(self): + from unittest import mock + + def fake_run(argv, **kw): + if argv[:2] == ['git', 'show']: + raise subprocess.CalledProcessError(128, argv) + out = 'MB123\n' if argv[:2] == ['git', 'merge-base'] else 'tools/get_deps.py\n' + return subprocess.CompletedProcess(argv, 0, stdout=out, stderr='') + + buf = io.StringIO() + with mock.patch.object(ci_select.subprocess, 'run', fake_run), \ + mock.patch.object(sys, 'argv', [sys.executable, '--base', 'origin/master']), \ + contextlib.redirect_stdout(buf), contextlib.redirect_stderr(io.StringIO()): + ci_select.main() + self.assertTrue(json.loads(buf.getvalue())['build']['full']) + + +class TestBuildClassifier(unittest.TestCase): + def b(self, files): + return ci_select.classify_build(files, REPO) + + def test_noncode_and_test_hil_contribute_nothing(self): # rules 1, 2 + s = self.b(['docs/info/index.rst', 'README.rst', 'test/hil/hil_test.py', '.claude/skills/hil/SKILL.md']) + self.assertFalse(s['full']) + self.assertEqual(s['families'], []) + self.assertEqual(s['family_examples'], {}) + + def test_port_device_rule(self): # rule 3 + s = self.b(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + self.assertFalse(s['full']) + self.assertEqual(s['families'], ['rp2040']) + exs = s['family_examples']['rp2040'] + self.assertIn('device/cdc_msc', exs) + self.assertFalse(any(e.startswith(('host/', 'typec/')) for e in exs)) + # dual inclusion asserted on the pure role helper: whether a dual example + # survives Task 4's buildability pruning depends on the environment-gated + # CI board pick, so the classifier-output assertion must not rely on it + self.assertIn('dual/host_info_to_device_cdc', + ci_select.role_examples(REPO, ('device', 'dual'))) + self.assertNotIn('host/bare_api', ci_select.role_examples(REPO, ('device', 'dual'))) + + def test_port_host_rule(self): # rule 4 + s = self.b(['src/portable/analog/max3421/hcd_max3421.c']) + self.assertFalse(s['full']) + # rp2040's family.cmake unconditionally lists hcd_max3421.c as a source of its + # tinyusb_host_max3421 INTERFACE lib (linked only when MAX3421_HOST=1, e.g. the + # real feather_rp2040_max3421 board) and espressif's component CMakeLists also + # references it — so the raw (unpruned) scan legitimately finds both; Task 4's + # buildability post-filter is what may later prune either away + # non-empty FIRST: a subset assertion is satisfied by set(), and since ports are + # now empty-means-empty (fail-closed) an unnoticed regression to zero families + # would select no build leg at all and merge an uncompiled HCD + self.assertTrue(s['families'], 'a host-port change must select some family') + self.assertLessEqual(set(s['families']), {'espressif', 'rp2040'}) + self.assertTrue(s['family_examples'], 'and must name the examples for them') + for exs in s['family_examples'].values(): + self.assertTrue(exs) + self.assertFalse(any(e.startswith(('device/', 'typec/')) for e in exs)) + + def test_port_shared_file_selects_all_examples(self): # rule 5 + s = self.b(['src/portable/synopsys/dwc2/dwc2_common.c']) + self.assertFalse(s['full']) + self.assertIn('stm32f4', s['families']) + self.assertNotIn('rp2040', s['families']) + self.assertNotIn('stm32f4', s['family_examples']) # 'all' => no map key + + def test_bsp_family_rule(self): # rule 6 + s = self.b(['hw/bsp/stm32f4/boards/stm32f407disco/board.h']) + self.assertEqual(s['families'], ['stm32f4']) + self.assertNotIn('stm32f4', s['family_examples']) + + def test_bsp_top_level_file_is_full(self): # rule 16 + self.assertTrue(self.b(['hw/bsp/board.c'])['full']) + self.assertTrue(self.b(['hw/bsp/family_support.cmake'])['full']) + + def test_mcu_rule(self): # rule 7 + s = self.b(['hw/mcu/nordic/nrf5x/nrf_clock.h']) + self.assertEqual(s['families'], ['nrf']) + # empty means empty: no family's build references the path, so no build + # compiles it - nothing to select + s = self.b(['hw/mcu/no_such_vendor/x.c']) + self.assertFalse(s['full']) + self.assertEqual(s['families'], []) + self.assertEqual(s['family_examples'], {}) + + def test_class_device_rule(self): # rule 8 + s = self.b(['src/class/cdc/cdc_device.c']) + self.assertFalse(s['full']) + # near-all families (Task 4's pruning may drop a few); never equality + # against all_bsp_families — that's a tuple, and pruning shrinks the list + self.assertIn('stm32f4', s['families']) + self.assertGreater(len(s['families']), 50) + exs = s['family_examples']['stm32f4'] + self.assertIn('device/cdc_msc', exs) + self.assertNotIn('device/hid_composite', exs) + self.assertNotIn('host/cdc_msc_hid', exs) # TUH_CDC examples are rule 9's + + def test_class_host_rule(self): # rule 9 + s = self.b(['src/class/msc/msc_host.c']) + exs = s['family_examples']['stm32f4'] + self.assertIn('host/msc_file_explorer', exs) + self.assertNotIn('device/cdc_msc', exs) + + def test_class_shared_header_and_include_edge(self): # rule 10 + s = self.b(['src/class/audio/audio.h']) + exs = s['family_examples']['stm32f4'] + self.assertIn('device/audio_test', exs) + self.assertIn('device/midi_test', exs) # midi headers include audio.h + + def test_core_device_rule(self): # rule 11 + s = self.b(['src/device/usbd.c']) + exs = s['family_examples']['stm32f4'] + self.assertIn('device/cdc_msc', exs) + # no dual In-assertion: dual examples are only.txt-gated to max3421/pio-usb + # boards, so pruning legitimately drops them on a plain stm32f4 board + self.assertFalse(any(e.startswith(('host/', 'typec/')) for e in exs)) + + def test_core_host_rule(self): # rule 12 + s = self.b(['src/host/usbh.c']) + exs = s['family_examples']['stm32f4'] + self.assertFalse(any(e.startswith(('device/', 'typec/')) for e in exs)) + + def test_example_rule(self): # rules 13, 14 + s = self.b(['examples/device/cdc_msc/src/main.c']) + self.assertEqual(s['family_examples']['stm32f4'], ['device/cdc_msc']) + s = self.b(['examples/device/board_test/src/main.c']) + self.assertEqual(s['family_examples']['stm32f4'], ['device/board_test']) + s = self.b(['examples/device/no_such_example/src/main.c']) # deleted example: nothing + self.assertFalse(s['full']) + self.assertEqual(s['families'], []) + + def test_full_paths(self): # rules 15-17 + for p in ('src/common/tusb_fifo.c', 'src/osal/osal.h', 'src/tusb.c', + 'src/tusb_option.h', + 'tools/build.py', 'tools/cmake/cpu/cortex-m4.cmake', + 'examples/CMakeLists.txt', 'examples/device/CMakeLists.txt', + 'examples/build_system/cmake/cpu.cmake', '.github/workflows/build.yml', + '.circleci/config.yml', 'src/CMakeLists.txt', 'src/tinyusb.mk', + 'hw/bsp/family_support.mk', 'tools/build_utils.py', + 'some/unknown/path.c'): + self.assertTrue(self.b([p])['full'], p) + + def test_repo_metadata_is_not_a_build_input(self): + # these used to reach `full` through rule 17: a PR touching only .gitignore and a + # README created 74 cmake legs and booked the whole rig. No Build step reads them. + for p in ('sonar-project.properties', '.gitignore', '.gitattributes', + '.clang-format', '.idea/misc.xml', 'version.yml', 'library.json', + 'examples/CMakePresets.json', 'test/fuzz/fuzz.cc', + 'test/unit-test/project.yml', '.github/workflows/pr_comment.yml', + 'tools/gen_doc.py'): + s = self.b([p]) + self.assertFalse(s['full'], p) + self.assertEqual(s['families'], [], p) + + def test_the_build_machinery_is_still_full(self): + # the other side of the same line: these DECIDE what gets built + for p in ('.circleci/config.yml', '.github/workflows/build.yml', + '.github/scripts/ci_set_matrix.py', 'tools/ci_select.py', + 'tools/build_utils.py', 'tools/metrics.py'): + self.assertTrue(self.b([p])['full'], p) + + def test_mixed_diff_unions_per_family(self): + s = self.b(['src/portable/raspberrypi/rp2040/dcd_rp2040.c', 'src/class/cdc/cdc_device.c']) + self.assertFalse(s['full']) + self.assertIn('stm32f4', s['families']) + self.assertGreater(len(s['families']), 50) + self.assertIn('device/hid_composite', s['family_examples']['rp2040']) # from the dcd rule + self.assertNotIn('device/hid_composite', s['family_examples']['stm32f4']) # cdc-only there + + def test_example_names_are_real_dirs(self): + for ex in ci_select.all_examples(REPO): + role, name = ex.split('/') + self.assertTrue(os.path.isdir(os.path.join(REPO, 'examples', role, name)), ex) + self.assertRegex(ex, r'^(device|dual|host|typec)/[A-Za-z0-9_]+$') + + +class TestBuildPostFilter(unittest.TestCase): + def test_kept_examples_are_buildable(self): + import build_utils, build as build_py + s = ci_select.classify_build(['src/class/msc/msc_host.c'], REPO) + self.assertFalse(s['full']) + # families that cannot build a single TUH_MSC example drop out entirely + self.assertNotIn('msp430', s['families']) + old = os.getcwd() + os.chdir(REPO) + try: + # buildable on SOME board of the family - CircleCI builds them all + for fam, exs in s['family_examples'].items(): + boards = build_py.get_family_boards(fam, False, False) + for e in exs: + self.assertTrue(any(not build_utils.skip_example(e, b) for b in boards), + f'{fam}: {e}') + finally: + os.chdir(old) + + def test_unfiltered_family_has_no_map_key(self): + s = ci_select.classify_build(['hw/bsp/stm32f4/family.c'], REPO) + self.assertEqual(s['families'], ['stm32f4']) + self.assertEqual(s['family_examples'], {}) + + def test_espressif_prunes_to_what_its_build_path_can_build(self): + # build.py's espressif branch builds get_examples('espressif') only (the + # *_freertos examples plus a short extra list), so keeping espressif for a + # device/mtp diff spins CircleCI's most expensive leg up to skip everything + s = ci_select.classify_build(['examples/device/mtp/src/main.c'], REPO) + self.assertFalse(s['full']) + self.assertNotIn('espressif', s['families']) + + def test_espressif_survives_an_example_it_does_build(self): + s = ci_select.classify_build(['examples/device/cdc_msc_freertos/src/main.c'], REPO) + self.assertFalse(s['full']) + self.assertIn('espressif', s['families']) + + def test_ra_survives_the_dual_example_prune(self): + # ra's only buildable dual example is gated on only.txt's mcu:ra6m5, which + # exists only if the ${MCU_VARIANT} token in FAMILY_MCUS resolves + s = ci_select.classify_build( + ['examples/dual/host_info_to_device_cdc/src/main.c'], REPO) + self.assertFalse(s['full']) + self.assertIn('ra', s['families'], s['families']) + + def test_deleted_family_dir_does_not_crash(self): + # rule 6 extracts a family from the path; a PR that deletes or renames + # hw/bsp/<fam> used to traceback in get_family_boards' scandir + s = ci_select.classify_build(['hw/bsp/no_such_family_xyz/family.cmake'], REPO) + self.assertFalse(s['full']) + self.assertEqual(s['families'], []) + self.assertTrue(any('gone from tree' in r for r in s['reasons']), s['reasons']) + + def test_class_source_selecting_nothing_selects_nothing(self): + # a class-with-no-enabling-config case: no config enables CFG_TUH_VENDOR, so + # nothing exercises it and nothing builds - empty means empty (maintainer + # decision; the file is still parsed by every full master-push build, which is + # the accepted net for a break outside its #if guard). src/class/bth is the + # live instance of this state today; TestClassesWithNoEnablingExample pins the + # whole set, so a new one cannot appear unnoticed. + # src/class/bth/bth_device.c, a file that EXISTS: the old assertion named + # src/class/vendor/vendor_host.c, deleted by the same branch, so any made-up + # path reached the same branch and the test passed vacuously. + real = os.path.join(REPO, 'src/class/bth/bth_device.c') + self.assertTrue(os.path.isfile(real), 'the case needs a file that exists') + s = ci_select.classify_build(['src/class/bth/bth_device.c'], REPO) + self.assertFalse(s['full']) + self.assertEqual(s['families'], []) + self.assertTrue(any('no contribution' in r for r in s['reasons']), s['reasons']) + # and the reason must name the class, not just any empty answer + self.assertTrue(any('bth' in r for r in s['reasons']), s['reasons']) + + def test_class_source_with_examples_still_scopes(self): + s = ci_select.classify_build(['src/class/cdc/cdc_device.c'], REPO) + self.assertFalse(s['full']) + + def test_no_stdout_pollution(self): + # get_family_boards prints on odd families; the selector's stdout is JSON + import io, contextlib + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + ci_select.classify_build(['src/class/msc/msc_host.c'], REPO) + self.assertEqual(buf.getvalue(), '') + + +class TestNoContributionPaths(unittest.TestCase): + """Paths that are inside build.yml's code filter but cannot change a compiled byte. + Unclassified means FULL on both axes, so a metrics-only PR would otherwise cost the + whole build matrix plus an exclusive full-rig sweep - where master ran nothing.""" + + def test_metrics_scripts_run_on_no_board_but_still_build(self): + # HIL axis only. tools/metrics.py IS executed by a build - examples/CMakeLists.txt + # makes it the `tinyusb_metrics` target and build_util.yml adds + # `--target tinyusb_metrics` - so the build axis must keep exercising it, or a + # break merges green and reds the next master push. Nothing on the rig runs it. + for p in ('tools/metrics.py', '.github/scripts/metrics_pair_compare.py'): + h = sel([p]) + self.assertFalse(h['full'], p) + self.assertEqual(h['boards'], {}, p) + self.assertTrue(ci_select.classify_build([p], REPO)['full'], p) + + def test_typec_example_builds_but_runs_nothing(self): + # examples/typec is compiled by the build matrix and run by no rig board; the + # HIL walk used to not recognise the role at all -> unclassified -> full rig + p = 'examples/typec/power_delivery/src/main.c' + h = sel([p]) + self.assertFalse(h['full']) + self.assertEqual(h['boards'], {}) + b = ci_select.classify_build([p], REPO) + self.assertFalse(b['full']) + self.assertTrue(b['families'], 'typec still has to be compiled somewhere') + + +class TestHilExamples(unittest.TestCase): + def test_board_test_always_present_and_full_emits(self): + s = ci_select.classify(['src/common/tusb_fifo.c'], REPO, ROSTERS) # full + he = ci_select.hil_examples(s, ROSTERS) + self.assertEqual(set(he), {b['name'] for b in ROSTER}) + for name, exs in he.items(): + self.assertIn('device/board_test', exs) + + def test_narrowed_board_gets_chosen_tests_only(self): + s = ci_select.classify(['examples/device/cdc_msc/src/main.c'], REPO, ROSTERS) + he = ci_select.hil_examples(s, ROSTERS) + self.assertEqual(he['stm32f407disco'], ['device/board_test', 'device/cdc_msc']) + + def test_full_board_gets_its_whole_test_list(self): + s = ci_select.classify(['hw/bsp/stm32f4/boards/stm32f407disco/board.h'], REPO, ROSTERS) + he = ci_select.hil_examples(s, ROSTERS) + want = set(ci_select.board_tests(ROSTER[1])) | {'device/board_test'} + self.assertEqual(set(he['stm32f407disco']), want) + self.assertNotIn('raspberry_pi_pico', he) # deselected board: no firmware needed + + +class TestHilExamplesDuplicateRosters(unittest.TestCase): + """Rosters are disjoint today, but a board moved between rigs (or listed on both + during a migration) must get the UNION of its test lists: superset firmware is + harmless, a missing image fails the run on whichever rig lost the coin toss.""" + + ROSTERS = [ + ('test/hil/a.json', [{'name': 'dup_board', 'uid': 'd1', 'flasher': {'name': 'jlink'}, + 'tests': {'only': ['device/cdc_msc']}}]), + ('test/hil/b.json', [{'name': 'dup_board', 'uid': 'd1', 'flasher': {'name': 'jlink'}, + 'tests': {'only': ['device/hid_boot_interface']}}]), + ] + + def test_duplicate_board_unions_the_test_lists(self): + he = ci_select.hil_examples({'full': True, 'boards': {}}, self.ROSTERS) + self.assertEqual(he['dup_board'], + ['device/board_test', 'device/cdc_msc', + 'device/hid_boot_interface']) + + +class TestCliJson(unittest.TestCase): + def test_build_key_without_rosters(self): + r = subprocess.run([sys.executable, os.path.join(REPO, 'tools/ci_select.py'), + '--diff-file', '/dev/null'], capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + j = json.loads(r.stdout) + self.assertIn('build', j) + self.assertNotIn('hil_examples', j) # rosters not given + + def test_build_and_hil_keys_with_rosters(self): + import tempfile + with tempfile.NamedTemporaryFile('w', suffix='.txt', delete=False) as f: + f.write('src/portable/raspberrypi/rp2040/dcd_rp2040.c\n') + df = f.name + r = subprocess.run([sys.executable, os.path.join(REPO, 'tools/ci_select.py'), + '--diff-file', df, os.path.join(REPO, 'test/hil/tinyusb.json')], + capture_output=True, text=True) + os.unlink(df) + j = json.loads(r.stdout) + self.assertEqual(j['build']['families'], ['rp2040']) + self.assertIn('hil_examples', j) + for exs in j['hil_examples'].values(): + self.assertIn('device/board_test', exs) + + +SET_MATRIX = os.path.join(REPO, '.github/scripts/ci_set_matrix.py') + + +class TestCiSetMatrix(unittest.TestCase): + def run_matrix(self, *args): + return subprocess.run([sys.executable, SET_MATRIX, *args], + capture_output=True, text=True) + + def test_no_flags_is_todays_output(self): + r = self.run_matrix() + self.assertEqual(r.returncode, 0, r.stderr) + self.baseline = json.loads(r.stdout) + self.assertIn('stm32f4', self.baseline['arm-gcc']) + + def test_select_full_is_identical(self): + base = json.loads(self.run_matrix().stdout) + sel = json.dumps({'build': {'full': True, 'families': [], 'family_examples': {}}}) + self.assertEqual(json.loads(self.run_matrix('--select', sel).stdout), base) + + def test_select_narrow_is_a_subset(self): + sel = json.dumps({'build': {'full': False, 'families': ['rp2040', 'stm32f4'], + 'family_examples': {}}}) + m = json.loads(self.run_matrix('--select', sel).stdout) + self.assertEqual(m['arm-gcc'], ['rp2040', 'stm32f4']) + self.assertEqual(m['riscv-gcc'], []) + self.assertEqual(set(m), set(json.loads(self.run_matrix().stdout))) # all keys kept + + def test_malformed_select_falls_open(self): + base = json.loads(self.run_matrix().stdout) + r = self.run_matrix('--select', 'not json {') + self.assertEqual(r.returncode, 0) + self.assertEqual(json.loads(r.stdout), base) + self.assertIn('ci_set_matrix: UNSCOPED', r.stderr) # build.yml greps this + + def test_wrong_shaped_select_falls_open_too(self): + # valid JSON, wrong types: the matrix is built AFTER main()'s try/except, so an + # AttributeError here reds the step - the very outcome that handler exists to + # prevent (GHA and CircleCI only survive it through their own shell `||`) + base = json.loads(self.run_matrix().stdout) + for bad in ('{"build": ["stm32f4"]}', '{"build": {"full": false}}', + '{"build": {"full": false, "families": "stm32f4"}}', '["stm32f4"]'): + r = self.run_matrix('--select', bad) + self.assertEqual(r.returncode, 0, f'{bad}: {r.stderr}') + self.assertEqual(json.loads(r.stdout), base, bad) + + def test_base_flag_with_empty_diff_selects_nothing(self): + # --base HEAD => empty diff => build.families [] => every toolchain scopes to [] + base = json.loads(self.run_matrix().stdout) + r = self.run_matrix('--base', 'HEAD') + self.assertEqual(r.returncode, 0, r.stderr) + m = json.loads(r.stdout) + self.assertEqual(set(m), set(base)) + self.assertTrue(all(v == [] for v in m.values()), m) + + def test_select_file_matches_select(self): + # build.yml hands the selection over as a FILE: a ~128KiB step env var makes + # the step's own exec fail with E2BIG before any fallback can run + import tempfile + sel = json.dumps({'build': {'full': False, 'families': ['rp2040'], + 'family_examples': {}}}) + with tempfile.NamedTemporaryFile('w', suffix='.json', delete=False) as f: + f.write(sel) + path = f.name + try: + self.assertEqual(self.run_matrix('--select-file', path).stdout, + self.run_matrix('--select', sel).stdout) + finally: + os.unlink(path) + + def test_absent_families_key_falls_open(self): + # `{"build": {"full": false}}` with no families key is an unusable selection, + # not "nothing selected": scoping every toolchain to [] would report a + # vacuous green with zero families built + base = json.loads(self.run_matrix().stdout) + r = self.run_matrix('--select', json.dumps({'build': {'full': False}})) + self.assertEqual(r.returncode, 0) + self.assertEqual(json.loads(r.stdout), base) + self.assertIn('ci_set_matrix: UNSCOPED', r.stderr) # build.yml greps this + + def test_families_no_toolchain_builds_falls_open(self): + # hw/bsp/same7x is real but in no toolchain's list, so scoping to it emits an + # all-empty matrix: every leg skips and the PR goes green from a build job that + # ran no compiler. Unusable, not "nothing selected" - and the marker matters, + # because that is what build.yml and CircleCI grep to drop the build extras too. + base = json.loads(self.run_matrix().stdout) + r = self.run_matrix('--select', + json.dumps({'build': {'full': False, 'families': ['same7x']}})) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertEqual(json.loads(r.stdout), base) + self.assertIn('ci_set_matrix: UNSCOPED', r.stderr) + + def test_a_partial_toolchain_miss_still_scopes(self): + # one buildable family is real coverage: scope to it and just note the other + r = self.run_matrix('--select', json.dumps( + {'build': {'full': False, 'families': ['stm32f4', 'same7x']}})) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertEqual(json.loads(r.stdout)['arm-gcc'], ['stm32f4']) + self.assertNotIn('UNSCOPED', r.stderr) + self.assertIn('same7x', r.stderr) + + def test_explicit_empty_families_selects_nothing(self): + # an explicit [] IS a legitimate answer (a diff that builds nothing) + r = self.run_matrix('--select', + json.dumps({'build': {'full': False, 'families': []}})) + self.assertEqual(r.returncode, 0) + self.assertEqual(set().union(*json.loads(r.stdout).values()), set()) + + def test_missing_select_file_falls_open(self): + base = json.loads(self.run_matrix().stdout) + r = self.run_matrix('--select-file', '/no/such/selection.json') + self.assertEqual(r.returncode, 0) + self.assertEqual(json.loads(r.stdout), base) + self.assertIn('ci_set_matrix: UNSCOPED', r.stderr) # build.yml greps this + + def test_base_flag_bad_ref_falls_open(self): + base = json.loads(self.run_matrix().stdout) + r = self.run_matrix('--base', 'no-such-ref-xyz') + self.assertEqual(r.returncode, 0) + self.assertEqual(json.loads(r.stdout), base) + self.assertIn('ci_set_matrix: UNSCOPED', r.stderr) # build.yml greps this + + +HIL_SET_MATRIX = os.path.join(REPO, '.github/scripts/hil_ci_set_matrix.py') + + +class TestHilCiSetMatrixExamples(unittest.TestCase): + def run_matrix(self, *args): + r = subprocess.run([sys.executable, HIL_SET_MATRIX, *args, + os.path.join(REPO, 'test/hil/tinyusb.json')], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + return r.stdout + + def test_no_hil_examples_is_byte_identical(self): + plain = self.run_matrix() + sel = json.dumps({'full': True, 'boards': {}}) + self.assertEqual(self.run_matrix('--select', sel), plain) + + def test_absent_boards_key_falls_open_to_the_full_roster(self): + # the mirror of ci_set_matrix's families guard: reading an ABSENT boards key as + # "nothing selected" filters every board out, so every hil-build leg skips and + # both rig jobs skip through needs: - an all-green PR with zero hardware + # coverage. An explicit boards: {} stays a legitimate nothing-selected. + plain = self.run_matrix() + for bad in ('{"full": false, "hil_examples": {}}', '{"full": false, "boards": []}', + 'not json {', '["a board"]', + # the whole selection is unusable, hil_examples included: keeping the + # -e lists builds a few examples per board while the rig, unfiltered, + # runs that board's whole test list + '{"full": false, "hil_examples": {"frdm_k64f": ["device/cdc_msc"]}}'): + self.assertEqual(self.run_matrix('--select', bad), plain, bad) + self.assertNotEqual(self.run_matrix('--select', '{"full": false, "boards": {}}'), + plain, 'an explicit empty boards map still means nothing') + + def test_select_file_matches_select(self): + # hil-hfp-iar passes the whole selection; as one argv it can exceed + # MAX_ARG_STRLEN on a big diff, so the file form must be equivalent + import tempfile + board = on_roster(self, 'stm32f407disco')[0] + sel = json.dumps({'full': False, 'boards': {board: 'all'}, + 'hil_examples': {board: ['device/board_test']}}) + with tempfile.NamedTemporaryFile('w', suffix='.json', delete=False) as f: + f.write(sel) + path = f.name + try: + self.assertEqual(self.run_matrix('--select-file', path), + self.run_matrix('--select', sel)) + finally: + os.unlink(path) + + def test_examples_appended_per_board(self): + board = on_roster(self, 'stm32f407disco')[0] + sel = json.dumps({'full': False, 'boards': {board: 'all'}, + 'hil_examples': {board: ['device/board_test', 'device/cdc_msc']}}) + m = json.loads(self.run_matrix('--select', sel)) + entries = [e for entries in m.values() for e in entries] + self.assertTrue(entries) + for e in entries: + self.assertIn(f'-b {board}', e) + self.assertIn('-e device/board_test', e) + self.assertIn('-e device/cdc_msc', e) + + +class TestBuildPyExampleFilter(unittest.TestCase): + def setUp(self): + import build as build_py + self.build = build_py + self.old = os.getcwd() + os.chdir(REPO) # skip_example uses repo-relative paths + + def tearDown(self): + os.chdir(self.old) + + def test_all_maps_to_example_names(self): + # ONE group: the examples of a '--target all' build go into a single + # `cmake --build --target a b c`, so they build in parallel + t = self.build.resolve_example_target_groups(['all'], ['device/cdc_msc', 'device/dfu'], + 'stm32f407disco') + self.assertEqual(t, [['cdc_msc', 'dfu']]) + + def test_other_targets_pass_through_in_their_own_group(self): + # a target that is not 'all' keeps its own invocation, so ordering against the + # examples is preserved (tinyusb_metrics runs after them, as it did unfiltered) + t = self.build.resolve_example_target_groups(['all', 'tinyusb_metrics'], + ['device/cdc_msc'], 'stm32f407disco') + self.assertEqual(t, [['cdc_msc'], ['tinyusb_metrics']]) + + def test_unbuildable_examples_drop_and_empty_is_none(self): + # typec/power_delivery only builds on stm32g4-class parts, never on f4 + t = self.build.resolve_example_target_groups(['all'], + ['typec/power_delivery', 'device/cdc_msc'], + 'stm32f407disco') + self.assertEqual(t, [['cdc_msc']]) + self.assertIsNone(self.build.resolve_example_target_groups(['all'], + ['typec/power_delivery'], + 'stm32f407disco')) + + def test_espressif_empty_intersection_skips_without_building(self): + # cmake_board's espressif branch must short-circuit on an empty -e + # intersection the same way the generic cmake/make branches do, and + # must do so before touching idf.py (no real esp-idf build here). + calls = [] + real_run_cmd = self.build.run_cmd # `del` here would drop the real one + self.build.run_cmd = lambda cmd: calls.append(cmd) # would only run for a real build + try: + r = self.build.cmake_board('espressif_s3_devkitc', [], None, [], ['all'], + examples=['nonexistent/example']) + finally: + self.build.run_cmd = real_run_cmd + self.assertEqual(r, [0, 0, 1]) + self.assertEqual(calls, []) + + def test_make_one_example_uses_make_semantics(self): + # F1 end to end: the make path must ask skip_example with build_system='make', + # or lpc54's cmake-only FAMILY_MCUS un-skips a host example whose make build + # compiles no HCD source and fails to link + calls = [] + real_run_cmd = self.build.run_cmd + self.build.run_cmd = lambda cmd: calls.append(cmd) + try: + r = self.build.make_one_example('host/msc_file_explorer_freertos', + 'lpcxpresso54628', '', ['all']) + finally: + self.build.run_cmd = real_run_cmd + self.assertEqual(r, [0, 0, 1]) # skipped, nothing handed to make + self.assertEqual(calls, []) + + def test_example_flag_rejects_a_bare_name(self): + # `-e cdc_msc` (no role) used to IndexError inside the target resolver; + # argparse rejects the shape now, with a message that names it + r = subprocess.run([sys.executable, os.path.join(REPO, 'tools', 'build.py'), + '-b', 'stm32f407disco', '-e', 'cdc_msc'], + capture_output=True, text=True, cwd=REPO) + self.assertEqual(r.returncode, 2, r.stdout + r.stderr) + self.assertIn('role/name', r.stderr) + + def test_no_example_basename_is_reused_across_roles(self): + # -e maps role/name onto the BARE cmake target name, so device/foo and host/foo + # would collapse into one `--target foo`: one of them would never build while + # the post-configure check still reports both as covered. No collision today, + # and the -e lists are machine-generated, so nothing else would notice one. + seen = {} + for ex in ci_select.all_examples(REPO): + role, name = ex.split('/', 1) + self.assertNotIn(name, seen, + f'{ex} and {seen.get(name)}/{name} share a cmake target name; ' + f'build.py -e cannot tell them apart') + seen[name] = role + + def test_example_flag_rejects_a_name_no_example_dir_answers_to(self): + # right shape, no such dir: every board would report Skipped and the run would + # still exit 0 (main returns the FAILED count), so an entirely stale -e list - + # from the example map or from a roster test name - reads as a green build + r = subprocess.run([sys.executable, os.path.join(REPO, 'tools', 'build.py'), + '-b', 'stm32f407disco', '-e', 'device/no_such_example'], + capture_output=True, text=True, cwd=REPO) + self.assertEqual(r.returncode, 2, r.stdout + r.stderr) + self.assertIn('no such example directory', r.stderr) + + def test_pr_filter_answers_before_configuring(self): + # nothing the -e list names is buildable here: the skip.txt mirror needs no + # configure output, so the whole cmake run must be skipped, not just its build + calls = [] + real_run_cmd = self.build.run_cmd + self.build.run_cmd = lambda cmd: calls.append(cmd) + try: + r = self.build.cmake_board('stm32f407disco', [], None, [], ['all'], + examples=['typec/power_delivery']) + finally: + self.build.run_cmd = real_run_cmd + self.assertEqual(r, [0, 0, 1]) + self.assertEqual(calls, []) + + def _cmake_board_with_targets(self, registered, examples): + """cmake_board with the configure/build stubbed and CMake's registered-target + list forced. Returns (result, target names handed to `cmake --build`).""" + class Ok: + returncode = 0 + calls = [] + + def fake_run(cmd): + calls.append(cmd) + return Ok() + real_run_cmd = self.build.run_cmd + real_targets = self.build.cmake_registered_targets + self.build.run_cmd = fake_run + self.build.cmake_registered_targets = lambda d: registered + try: + r = self.build.cmake_board('stm32f407disco', [], None, [], ['all'], + examples=examples) + finally: + self.build.run_cmd = real_run_cmd + self.build.cmake_registered_targets = real_targets + # everything after --target: one invocation carries the whole group + built = [c[c.index('--target') + 1:] for c in calls if '--target' in c] + return r, built + + def test_example_without_a_cmake_target_is_dropped(self): + # an example dir CMake never registered (absent from the role CMakeLists, or + # a stale roster name) must not reach `cmake --build --target <it>`: that is a + # hard red, and skip.txt cannot see it + r, built = self._cmake_board_with_targets({'cdc_msc'}, + ['device/cdc_msc', 'device/dfu']) + self.assertEqual(built, [['cdc_msc']]) + self.assertEqual(r, [1, 0, 0]) + + def test_the_selected_examples_build_in_one_invocation(self): + # one `cmake --build --target a b c`, not one invocation per example: the + # per-example loop serialised every scoped leg, and hil-build gets an -e list + # on EVERY PR (~14 examples per board), so it is on the critical path to the rig + r, built = self._cmake_board_with_targets({'cdc_msc', 'dfu', 'hid_generic_inout'}, + ['device/cdc_msc', 'device/dfu', + 'device/hid_generic_inout']) + self.assertEqual(built, [['cdc_msc', 'dfu', 'hid_generic_inout']]) + + def test_no_registered_target_at_all_skips_the_build(self): + r, built = self._cmake_board_with_targets({'cdc_msc'}, ['device/dfu']) + self.assertEqual(built, []) + self.assertEqual(r, [0, 0, 1]) + + def test_unparseable_target_help_keeps_the_skip_txt_answer(self): + # ground truth unavailable (a non-Ninja generator, an old cmake): fall back + # to the mirror rather than dropping every example + r, built = self._cmake_board_with_targets(None, ['device/cdc_msc']) + self.assertEqual(built, [['cdc_msc']]) + + def test_target_help_parse(self): + text = ('[1/1] All primary targets available:\n' + 'tinyusb_metrics: phony\n' + 'cdc_msc: phony\n' + 'cdc_msc-membrowse-upload: phony\n' + 'device/edit_cache: phony\n' + '/abs/build/device/cdc_msc/CMakeFiles/cdc_msc-jlink: CUSTOM_COMMAND\n') + self.assertEqual(self.build.parse_target_help(text), + {'tinyusb_metrics', 'cdc_msc', 'cdc_msc-membrowse-upload'}) + + def test_build_defines_reach_the_example_filter(self): + # metro_m4_express gets MAX3421_HOST=1 from its roster variant, never + # from its BSP: without threading them through, -e drops the rig's only + # MAX3421 dual firmware that --target all used to build + self.assertIsNone(self.build.resolve_example_target_groups( + ['all'], ['dual/host_info_to_device_cdc'], 'metro_m4_express')) + self.assertEqual(self.build.resolve_example_target_groups( + ['all'], ['dual/host_info_to_device_cdc'], 'metro_m4_express', + extra_defines=('MAX3421_HOST=1',)), [['host_info_to_device_cdc']]) + + + +class TestFamilyMcusFallback(unittest.TestCase): + """A family whose family.cmake sets FAMILY_MCUS only inside if() blocks gets its + whole MCU answer from _board_mcu's CFG_TUSB_MCU scrape (build_utils._family_mcus + does not evaluate cmake conditionals). For mcx that answer is load-bearing - six + examples' skip.txt name mcu:MCXA15 - and it comes out right only because every + mcx board still carries the token in a make-only board.mk the scrape falls + through to. A board.cmake-only board (MCU_VARIANT, no CFG_TUSB_MCU) would scrape + 'NONE' and silently skip EVERY example on it, in CI as well as in -e.""" + + @staticmethod + def conditional_only_families(): + """hw/bsp/<family> dirs whose family.cmake has no unconditional + set(FAMILY_MCUS ...) - computed, not listed, so a family that grows or loses + one moves in and out of this guard on its own.""" + import build_utils + out = [] + for fc in sorted(glob.glob(os.path.join(REPO, 'hw/bsp/*/family.cmake'))): + depth, uncond = 0, False + for line in open(fc).read().splitlines(): + line = line.strip() + if build_utils._FAMILY_MCUS_RE.match(line) and depth == 0: + uncond = True + if re.match(r'if\s*\(', line): + depth += 1 + elif re.match(r'endif\s*\(', line): + depth = max(0, depth - 1) + if not uncond: + out.append(os.path.dirname(fc)) + return out + + def test_every_board_of_such_a_family_scrapes_an_mcu(self): + import build_utils + fams = self.conditional_only_families() + self.assertTrue(fams, 'no family sets FAMILY_MCUS conditionally any more') + for fam_dir in fams: + fam = os.path.basename(fam_dir) + for bd in sorted(glob.glob(os.path.join(fam_dir, 'boards', '*'))): + if not os.path.isdir(bd): + continue + mcu, _ = build_utils._board_mcu(bd, fam_dir, fam) + self.assertNotEqual( + mcu, 'NONE', + f'{fam}/{os.path.basename(bd)}: nothing to scrape a CFG_TUSB_MCU ' + f'token from, and {fam}/family.cmake sets FAMILY_MCUS only inside ' + f'if() - skip_example would skip every example on this board. Fix ' + f'by evaluating the if(MCU_VARIANT STREQUAL ...) branches.') + + +class TestMcuTokensResolve(unittest.TestCase): + """The cmake-side MCU mirror must never answer with an unexpanded ${VAR} or with + nothing at all: both make every `mcu:` token miss, which reads as 'skip' for any + example carrying an only.txt and silently drops compile coverage.""" + + @staticmethod + def _every_board(): + import build as build_py + old = os.getcwd() + os.chdir(REPO) + try: + for fam in sorted(os.path.basename(os.path.dirname(f)) + for f in glob.glob(os.path.join(REPO, 'hw/bsp/*/boards'))): + for b in build_py.get_family_boards(fam, False, False): + yield fam, b + finally: + os.chdir(old) + + def test_no_board_answers_with_an_unexpanded_variable(self): + import build_utils + for fam, board in self._every_board(): + fam_dir, board_dir = f'{REPO}/hw/bsp/{fam}', f'{REPO}/hw/bsp/{fam}/boards/{board}' + mcus = set(build_utils._family_mcus(fam_dir, board_dir)) + mcus.add(build_utils._board_mcu(board_dir, fam_dir, fam)[0]) + self.assertFalse([m for m in mcus if '${' in m], + f'{fam}/{board}: unexpanded cmake variable in {sorted(mcus)} - ' + f'teach build_utils._cmake_expand the construct that produces it') + self.assertTrue(mcus - {'NONE'}, + f'{fam}/{board}: no MCU name resolved at all') + + # skip.txt/only.txt tokens no board in the tree answers to: stale spellings left + # behind by a family rename. Each one silently changes what CI builds, so this list + # must only ever SHRINK - a new entry means either a live token the mirror cannot + # produce, or a rename nobody followed through. `family:samd21` was one of these + # until the nine examples/host/*/only.txt files were corrected to samd2x_l2x. + # + # The remaining `mcu:` entries sit beside a live token in the same file, so they gate + # nothing either way. MKL25ZXX (7 files) and SAME5X (1) were dead too, but unlike + # these they were the ONLY token for their board - the examples were already being + # built on the very boards those lines meant to exclude. Dropping them is a no-op for + # the build (verified per example) and was chosen over re-pointing, which would have + # removed working coverage. + UNREACHABLE_TOKENS = { + 'mcu': {'LPC177X_8X', 'MIMXRT10XX', 'MIMXRT11XX', 'STM32U3'}, + 'family': set(), + 'board': set(), + } + + def test_every_skip_only_token_is_reachable(self): + import build_utils + wanted = {ns: set() for ns in self.UNREACHABLE_TOKENS} + for f in glob.glob(os.path.join(REPO, 'examples/*/*/*.txt')): + if os.path.basename(f) in ('skip.txt', 'only.txt'): + for tok in open(f).read().split(): + ns, _, name = tok.partition(':') + if ns in wanted and name: + wanted[ns].add(name) + have = {ns: set() for ns in wanted} + have['mcu'].add('MAX3421') # synthetic, from family_support.cmake:940 + for fam, board in self._every_board(): + fam_dir, board_dir = f'{REPO}/hw/bsp/{fam}', f'{REPO}/hw/bsp/{fam}/boards/{board}' + have['family'].add(fam) + have['board'].add(board) + have['mcu'] |= set(build_utils._family_mcus(fam_dir, board_dir)) + have['mcu'].add(build_utils._board_mcu(board_dir, fam_dir, fam)[0]) + have['mcu'].add(build_utils._scrape_mcu(pathlib.Path(fam_dir), + pathlib.Path(board_dir), fam)[0]) # make + for ns in wanted: + self.assertEqual( + wanted[ns] - have[ns], self.UNREACHABLE_TOKENS[ns] & wanted[ns], + f'a skip.txt/only.txt {ns}: token nothing in hw/bsp answers to. Either ' + f'the token is stale (a rename just changed what CI builds), or the ' + f'mirror cannot produce it - both silently skip that example everywhere.') + + def test_the_mcx_skip_tokens_are_still_live(self): + # the reason the mcx scrape is load-bearing rather than academic + named = [os.path.dirname(f) for f in glob.glob(os.path.join(REPO, 'examples/*/*/skip.txt')) + if 'mcu:MCXA15' in open(f).read().split()] + self.assertTrue(named, 'no skip.txt names mcu:MCXA15 any more') + + +class TestSkipExampleMirrorsFamilyFilter(unittest.TestCase): + """build_utils.skip_example is the python mirror of CMake's family_filter + (hw/bsp/family_support.cmake:171-207). family_filter loops over the whole + FAMILY_MCUS list; a per-board CFG_TUSB_MCU scrape alone lets -e ask for a + target CMake never created, and `cmake --build --target <it>` hard-fails.""" + + def setUp(self): + import build_utils + self.build_utils = build_utils + self.old = os.getcwd() + os.chdir(REPO) # skip_example uses repo-relative paths + + def tearDown(self): + os.chdir(self.old) + + def test_any_family_mcu_can_skip(self): + # broadcom_64bit: set(FAMILY_MCUS BCM2711 BCM2835); raspberrypi_cm4 is + # BCM2711, and examples/device/dfu/skip.txt lists mcu:BCM2835 + self.assertTrue(self.build_utils.skip_example('device/dfu', 'raspberrypi_cm4')) + + def test_any_family_mcu_can_satisfy_only(self): + # lpc55: family.mk says LPC55XX, family.cmake sets FAMILY_MCUS LPC55, and + # host/cdc_msc_hid/only.txt lists mcu:LPC55 - CMake builds it + self.assertFalse(self.build_utils.skip_example('host/cdc_msc_hid', 'lpcxpresso55s69')) + + def test_existing_decisions_are_unchanged(self): + self.assertFalse(self.build_utils.skip_example('device/cdc_msc', 'stm32f407disco')) + self.assertTrue(self.build_utils.skip_example('typec/power_delivery', 'stm32f407disco')) + + def test_build_define_enables_max3421_only_list(self): + # family_support.cmake:940 appends MAX3421 to FAMILY_MCUS when + # MAX3421_HOST=1; on metro_m4_express that define comes from the roster + # variant defines, so skip_example has to be told about it + ex = 'dual/host_info_to_device_cdc' + self.assertTrue(self.build_utils.skip_example(ex, 'metro_m4_express')) + self.assertFalse(self.build_utils.skip_example(ex, 'metro_m4_express', + extra_defines=('MAX3421_HOST=1',))) + + def test_family_mcus_variable_token_resolves(self): + """hw/bsp/ra/family.cmake: `set(FAMILY_MCUS RAXXX ${MCU_VARIANT})`, and + ra6m5_ek/board.cmake sets MCU_VARIANT ra6m5 — which is exactly the token + dual/host_info_to_device_cdc/only.txt spells (mcu:ra6m5). Dropping the + ${...} token silently removed ra from every scoped dual-example build.""" + self.assertFalse(self.build_utils.skip_example( + 'dual/host_info_to_device_cdc', 'ra6m5_ek')) + + def test_board_cmake_max3421_counts(self): + """feather_rp2040_max3421/board.cmake sets MAX3421_HOST 1 while the MCU + token comes from rp2040's family.cmake; scanning only the file the token + came from misses it, and only.txt's mcu:MAX3421 never matches.""" + self.assertFalse(self.build_utils.skip_example( + 'host/cdc_msc_hid_freertos', 'feather_rp2040_max3421')) + + +class TestSkipExampleMakeSemantics(unittest.TestCase): + """FAMILY_MCUS is a CMAKE fact. hw/bsp/lpc54/family.cmake sets it to LPC54 and + wires the ohci host sources; family.mk builds OPT_MCU_LPC54XXX and compiles no + HCD source at all — so applying the cmake MCU union to a Make build un-skips + the 9 host examples only.txt gates on mcu:LPC54 and they fail to link + (undefined reference to hcd_init). Make keeps master's exact algorithm.""" + + def setUp(self): + import build_utils + self.build_utils = build_utils + self.old = os.getcwd() + os.chdir(REPO) # skip_example uses repo-relative paths + + def tearDown(self): + os.chdir(self.old) + + def test_make_keeps_cmake_only_family_mcus_out(self): + self.assertTrue(self.build_utils.skip_example( + 'host/msc_file_explorer_freertos', 'lpcxpresso54628', build_system='make')) + + def test_make_does_not_skip_on_a_sibling_family_mcu(self): + # broadcom_64bit sets FAMILY_MCUS "BCM2711 BCM2835"; raspberrypi_cm4 is the + # BCM2711 one and device/dfu/skip.txt names mcu:BCM2835. The aarch64 make leg + # built device/dfu before the union and must keep building it. + for ex in ('device/dfu', 'device/usbtmc'): + self.assertFalse(self.build_utils.skip_example( + ex, 'raspberrypi_cm4', build_system='make'), ex) + + def test_cmake_is_the_default_and_still_unions(self): + self.assertTrue(self.build_utils.skip_example('device/dfu', 'raspberrypi_cm4')) + self.assertEqual( + self.build_utils.skip_example('device/dfu', 'raspberrypi_cm4'), + self.build_utils.skip_example('device/dfu', 'raspberrypi_cm4', + build_system='cmake')) + + def test_build_system_is_part_of_the_cache_key(self): + # one lru_cache shared by both semantics would answer the second caller + # with the first caller's verdict + ex, board = 'host/msc_file_explorer_freertos', 'lpcxpresso54628' + self.assertFalse(self.build_utils.skip_example(ex, board, build_system='cmake')) + self.assertTrue(self.build_utils.skip_example(ex, board, build_system='make')) + self.assertFalse(self.build_utils.skip_example(ex, board, build_system='cmake')) + + +class TestConfigEnables(unittest.TestCase): + """_config_enables decides which examples a class change selects, on BOTH the + build and the HIL axis. A define it cannot evaluate must read as ON: reading + it as OFF is fail-closed, and lets a compile break merge green.""" + + def test_identifier_value_is_enabled(self): + # examples/host/midi_rx: `#define CFG_TUH_MIDI CFG_TUH_DEVICE_MAX` + cfg = os.path.join(REPO, 'examples/host/midi_rx/src/tusb_config.h') + self.assertTrue(ci_select._config_enables(cfg, ['CFG_TUH_MIDI'])) + + def test_literal_zero_is_disabled(self): + import tempfile + with tempfile.TemporaryDirectory() as td: + cfg = os.path.join(td, 'tusb_config.h') + with open(cfg, 'w') as f: + f.write('#define CFG_TUD_CDC 0\n' + '#define CFG_TUD_MSC (0)\n' + '#define CFG_TUD_HID 00\n' + '#define CFG_TUH_HID 0 // typical keyboard + mouse\n' + '#define CFG_TUD_MIDI 01\n' + '#define CFG_TUD_DFU (1)\n') + for m in ('CFG_TUD_CDC', 'CFG_TUD_MSC', 'CFG_TUD_HID', 'CFG_TUH_HID'): + self.assertFalse(ci_select._config_enables(cfg, [m]), m) + for m in ('CFG_TUD_MIDI', 'CFG_TUD_DFU'): + self.assertTrue(ci_select._config_enables(cfg, [m]), m) + self.assertFalse(ci_select._config_enables(cfg, ['CFG_TUD_VIDEO'])) + + def test_two_branch_define_reads_on(self): + # examples/device/uac2_speaker_fb defines CFG_TUD_HID 1 under + # `#if CFG_AUDIO_DEBUG` and 0 in the #else. The default build (CFG_AUDIO_DEBUG + # defaults to 1) compiles the HID class in, so a CFG_TUD_HID change must keep + # this example on both axes - the #else's zero must not decide it. + cfg = os.path.join(REPO, 'examples/device/uac2_speaker_fb/src/tusb_config.h') + self.assertTrue(ci_select._config_enables(cfg, ['CFG_TUD_HID'])) + + def test_any_nonzero_define_wins_over_a_zero_one(self): + import tempfile + with tempfile.TemporaryDirectory() as td: + cfg = os.path.join(td, 'tusb_config.h') + with open(cfg, 'w') as f: + f.write('#if FOO\n#define CFG_TUD_MSC 1\n#else\n' + '#define CFG_TUD_MSC 0\n#endif\n' + '#if BAR\n#define CFG_TUD_CDC 0\n#else\n' + '#define CFG_TUD_CDC (0)\n#endif\n') + self.assertTrue(ci_select._config_enables(cfg, ['CFG_TUD_MSC'])) + self.assertFalse(ci_select._config_enables(cfg, ['CFG_TUD_CDC'])) + + def test_midi_host_change_selects_midi_rx(self): + s = ci_select.classify_build(['src/class/midi/midi_host.c'], REPO) + self.assertFalse(s['full']) + self.assertTrue(s['families'], 'a TUH_MIDI change must select some family') + self.assertTrue(any('host/midi_rx' in exs + for exs in s['family_examples'].values()), + s['family_examples']) + + +class TestPruneUsesEveryFamilyBoard(unittest.TestCase): + """CircleCI's cmake legs build EVERY board of a family, so an example gated to + one board (only.txt board:mimxrt1060_evk) must keep its family even though the + family's one-first board cannot build it.""" + + def test_board_gated_example_keeps_its_family(self): + s = ci_select.classify_build( + ['examples/dual/host_hid_to_device_cdc/src/main.c'], REPO) + self.assertFalse(s['full']) + self.assertIn('imxrt', s['families'], s['families']) + self.assertEqual(s['family_examples'].get('imxrt'), + ['dual/host_hid_to_device_cdc']) + + def test_either_build_system_keeps_the_family(self): + """This one family list gates CircleCI's MAKE legs too, and the two build + systems answer skip.txt differently. device/dfu carries mcu:BCM2835, which the + cmake FAMILY_MCUS union (BCM2711 BCM2835) applies to every broadcom_64bit board + and the make scrape applies to none - asking cmake alone drops the only + aarch64-gcc family in the matrix, so build-make-aarch64-gcc silently stops + compiling dfu at all.""" + import build_utils + old = os.getcwd() + os.chdir(REPO) + try: + self.assertTrue(build_utils.skip_example('device/dfu', 'raspberrypi_cm4')) + self.assertFalse(build_utils.skip_example('device/dfu', 'raspberrypi_cm4', + (), 'make')) + finally: + os.chdir(old) + s = ci_select.classify_build(['examples/device/dfu/src/main.c'], REPO) + self.assertIn('broadcom_64bit', s['families'], s['families']) + + +class TestPrunePoolIsBuildPys(unittest.TestCase): + """_prune_buildable asks build.py what each family's build path can see, the same + way for every family - the espressif carve-out lives in build.py.get_examples and + needs no second copy here. Measured identical on all 82 families.""" + + def setUp(self): + import build as build_py + self.build_py = build_py + self.old = os.getcwd() + os.chdir(REPO) # get_examples scans relative paths + + def tearDown(self): + os.chdir(self.old) + + def test_only_espressif_narrows_the_pool(self): + allex = list(ci_select.all_examples(REPO)) + for fam in ci_select.all_bsp_families(REPO): + pool = [e for e in allex if e in set(self.build_py.get_examples(fam))] + if fam == 'espressif': + self.assertNotEqual(pool, allex) # the carve-out is real + else: + self.assertEqual(pool, allex, f'{fam}: build.py narrows this family') + + def test_selections_are_what_the_espressif_only_rule_gave(self): + # espressif's own list is the one value that ever differed from the unfiltered + # example set. Recomputed from build.py rather than pinned as literals: a new + # board, family or example moves the counts, and a suite that fails for that + # teaches people to edit the numbers instead of reading the diff. What is pinned + # is the RELATION - espressif gets exactly the rule's answer narrowed to its own + # pool, every other family gets the answer unnarrowed. + pool = set(self.build_py.get_examples('espressif')) + # the third diff names an example espressif DOES build, so there is nothing for + # the carve-out to remove - it pins that the narrowing does not over-reach + for files, carve in ((['src/portable/synopsys/dwc2/dcd_dwc2.c'], True), + (['src/class/msc/msc_host.c'], True), + (['examples/device/cdc_msc_freertos/src/main.c'], False)): + s = ci_select.classify_build(files, REPO) + self.assertFalse(s['full'], files) + self.assertIn('espressif', s['families'], files) + esp = set(s['family_examples'].get('espressif') or []) + self.assertTrue(esp, f'{files}: espressif selected nothing') + # the pool narrowing is what _prune_buildable adds here, so it must hold... + self.assertTrue(esp <= pool, f'{files}: {sorted(esp - pool)} is outside the pool') + # ...and it must actually bite: some other family was given an example that + # espressif's build path cannot see, and espressif did not get it + other = set().union(*(set(v) for f, v in s['family_examples'].items() + if f != 'espressif'), set()) + self.assertEqual(bool(other - pool), carve, + f'{files}: carve-out expected={carve}, other-side extras ' + f'{sorted(other - pool)}') + self.assertFalse(esp & (other - pool), files) + + +class TestGetDepsExampleShim(unittest.TestCase): + """hil_ci_set_matrix emits `-b <board> -e role/name` entries that .github/actions/ + get_deps and build.yml's hfp job hand verbatim to get_deps.py. argparse must not + reject -e there (exit 2 = every PR's Get Dependencies step red).""" + + # get_deps.main() with its process pool stubbed out: argparse runs for real, + # nothing is cloned (this suite also runs on GitHub's bare pre-commit runner) + CODE = ('import sys\n' + 'import get_deps\n' + 'class P:\n' + ' def __enter__(self): return self\n' + ' def __exit__(self, *a): return False\n' + ' def map(self, fn, items): return [0] * len(items)\n' + 'get_deps.Pool = P\n' + "sys.argv = ['get_deps.py'] + sys.argv[1:]\n" + 'sys.exit(get_deps.main())\n') + + def run_get_deps(self, *args): + env = dict(os.environ, PYTHONPATH=os.path.join(REPO, 'tools')) + return subprocess.run([sys.executable, '-c', self.CODE, *args], + capture_output=True, text=True, cwd=REPO, env=env) + + def test_example_flag_is_accepted(self): + r = self.run_get_deps('-b', 'stm32f407disco', '-e', 'device/cdc_msc') + self.assertNotIn('unrecognized arguments', r.stderr) + self.assertEqual(r.returncode, 0, r.stderr) + + def test_plain_board_still_works(self): + r = self.run_get_deps('-b', 'stm32f407disco') + self.assertEqual(r.returncode, 0, r.stderr) + + +if __name__ == '__main__': + unittest.main(verbosity=1) diff --git a/test/hil/test/test_hil_bounded.py b/test/hil/test/test_hil_bounded.py new file mode 100644 index 000000000..c30c58cbd --- /dev/null +++ b/test/hil/test/test_hil_bounded.py @@ -0,0 +1,1821 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests proving hil_test's storage and MTP helpers cannot hang the worker: a +# wedged device blocks the call in D state forever (child process or in-process ioctl), +# so these paths go through a bounded runner. Fakes stand in for the wedge (a real one +# cannot be manufactured on demand): a PATH-injected `mtype` script and a +# PYTHONPATH-injected `pymtp` module, each with a mode that blocks forever. +# Scope: mtype, the gio unmount, the libmtp session, the arecord/iperf reaps, and the +# printer read (a process now, via run_alongside, so a killed reader takes its fd with +# it -- usblp allows ONE opener, and a blocked thread kept the node for the worker's life). +# Known residue (unbounded, backstopped only by the pool guard): hid open/write and +# midi's read(64). +# +# hil_test imports pyserial, which GitHub's bare pre-commit runner does not have — so +# an inert serial module is stubbed into sys.modules BEFORE the import (nothing here +# exercises serial paths). MTP traffic never touches hil_test: it all goes through the +# mtp_test.py subprocess, which gets the fake pymtp via PYTHONPATH. +# Run directly: +# python3 test/hil/test/test_hil_bounded.py +import os +import stat +import sys +import threading +from multiprocessing import TimeoutError as MpTimeoutError +import time +import types +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory + +TEST_DIR = os.path.dirname(os.path.abspath(__file__)) +# the modules under test live in the parent dir (test/hil), not here +sys.path.insert(0, os.path.dirname(TEST_DIR)) + +serial_stub = types.ModuleType('serial') +serial_stub.Serial = type('Serial', (), {}) +serial_stub.SerialException = type('SerialException', (Exception,), {}) +serial_stub.SerialTimeoutException = type('SerialTimeoutException', (Exception,), {}) +sys.modules.setdefault('serial', serial_stub) +import hil_flash +import hil_test + + +def write_script(path: Path, body: str) -> None: + path.write_text('#!/bin/sh\n' + body + '\n') + path.chmod(path.stat().st_mode | stat.S_IEXEC) + + +def no_settle(case): + """Zero test_device_usbtest's post-flash settle for one test. + + Real hardware needs it -- the enumeration can bounce once after a flash, and on + dual-port parts the stale same-serial node lingers. A fake rig has neither, and ten + tests drive that path, so leaving it real cost 30s of every suite run. + """ + case.addCleanup(setattr, hil_test, 'USBTEST_SETTLE', hil_test.USBTEST_SETTLE) + hil_test.USBTEST_SETTLE = 0 + + +def run_bounded(fn, timeout: float): + """Run fn in a daemon thread; return (finished, exception). A still-running thread is + the hang under test — leave it to die with the interpreter.""" + exc = [] + + def wrapper(): + try: + fn() + except BaseException as e: # noqa: BLE001 - tests inspect the exception + exc.append(e) + + t = threading.Thread(target=wrapper, daemon=True) + t.start() + t.join(timeout) + return not t.is_alive(), exc[0] if exc else None + + +class ReadDiskFile(unittest.TestCase): + def setUp(self): + self.tmp = TemporaryDirectory() + tmp = Path(self.tmp.name) + self.addCleanup(self.tmp.cleanup) + # fake block device node: get_disk_dev is patched to this existing path + self.dev = tmp / 'fakedev' + self.dev.write_bytes(b'') + # addCleanup, not tearDown: tearDown does NOT run when setUp raises, and a leaked + # PATH entry points at a temp bin dir this class already deleted. + for name in ('get_disk_dev', '_enum_timeout', 'MTYPE_TIMEOUT'): + self.addCleanup(setattr, hil_test, name, getattr(hil_test, name)) + hil_test.get_disk_dev = lambda uid, vendor, lun: str(self.dev) + hil_test._enum_timeout = 1 # the wait these tests must outlast; keep it small + self.bin = tmp / 'bin' + self.bin.mkdir() + self.addCleanup(os.environ.__setitem__, 'PATH', os.environ['PATH']) + os.environ['PATH'] = f'{self.bin}:{os.environ["PATH"]}' + self.pidfile = tmp / 'mtype.pid' + self.addCleanup(self._reap_mtype) + + def _reap_mtype(self): + if self.pidfile.exists(): # reap a leaked hang-mode mtype + try: + os.kill(int(self.pidfile.read_text()), 9) + except (OSError, ValueError): + pass + + def test_returns_exact_bytes_despite_stderr_noise(self): + # \377 is invalid UTF-8 and stderr noise must not leak into the data + write_script(self.bin / 'mtype', r"printf 'R\377EADME-DATA'; printf 'vfat warning' >&2") + data = hil_test.read_disk_file('uid0', 0, 'README.TXT') + self.assertEqual(data, b'R\xffEADME-DATA') + + def test_failure_message_carries_mtype_stderr_and_fname(self): + write_script(self.bin / 'mtype', "printf 'mtype: cannot read' >&2; exit 1") + with self.assertRaises(AssertionError) as cm: + hil_test.read_disk_file('uid0', 0, 'README.TXT') + self.assertIn('cannot read', str(cm.exception)) + self.assertIn('README.TXT', str(cm.exception)) + + def test_empty_read_fails_immediately_with_fname(self): + # rc 0 with no data is a real answer (bad sectors, empty file), not "not ready": + # fail at once like the old assert did, naming the file — don't spin the budget + write_script(self.bin / 'mtype', 'exit 0') + t0 = time.monotonic() + with self.assertRaises(AssertionError) as cm: + hil_test.read_disk_file('uid0', 0, 'README.TXT') + # BELOW one full _enum_timeout wait, not above it: "fails immediately" is the + # claim, and a bound of 1.5 against a 1s budget passes for code that spun the + # whole budget -- which is the regression this test exists to catch. + self.assertLess(time.monotonic() - t0, hil_test._enum_timeout, + 'read_disk_file spun the enumeration budget on a real answer') + self.assertIn('README.TXT', str(cm.exception)) + + def test_hung_mtype_cannot_hang_the_worker(self): + # a D-state child never exits; the bounded runner must give up without it + write_script(self.bin / 'mtype', f'echo $$ > {self.pidfile}; exec sleep 1000') + hil_test.MTYPE_TIMEOUT = 2 + finished, exc = run_bounded(lambda: hil_test.read_disk_file('uid0', 0, 'README.TXT'), 20) + self.assertTrue(finished, 'read_disk_file hung on a stuck mtype') + self.assertIsInstance(exc, AssertionError) + + +class CompactOutput(unittest.TestCase): + def test_strips_workflow_command_markers(self): + """Defense-in-depth: the historical marker source was worker-side run_cmd + (now suppressed at the emitter); anything future that pipes markers into a + captured stdout would land them mid-row where GitHub renders them literally.""" + raw = '::group::COMMAND TIMEOUT (1s): x\nboom\n::endgroup::\ntail' + self.assertEqual(hil_test.compact_output(raw), 'COMMAND TIMEOUT (1s): x | boom | tail') + + +class UsbtestRecovery(unittest.TestCase): + def test_recovery_flags_and_flash_bound_fit_the_reserve(self): + """The post-hang reflash plumbing: the CLI flags exist, and the bounded reflash + and the reserve that pays for them is derived per flasher (see the two tests + below), not pinned.""" + import subprocess + hil_dir = Path(TEST_DIR).parents[0] + r = subprocess.run([sys.executable, str(hil_dir / 'usbtest.py'), '--help'], + capture_output=True, text=True, timeout=30) + self.assertEqual(r.returncode, 0, r.stderr) + for flag in ('--recover-board', '--recover-fw'): + self.assertIn(flag, r.stdout) + + def test_the_reserve_covers_every_step_of_its_own_ladder(self): + """Enumerated from the SIDE EFFECTS usbtest performs, so dropping a step from + recovery_reserve() fails here. Overrun means run_cmd's outer kill lands MID-FLASH + and orphans the flasher (start_new_session, so killpg misses it) on the probe. + """ + import usbtest + from helper import hil_util as _hu + # each bounded step costs its timeout PLUS run_cmd's post-SIGKILL reap + flash = usbtest.RECOVER_FLASH_TIMEOUT + _hu.REAP_GRACE + reset = usbtest.RECOVER_RESET_TIMEOUT + _hu.REAP_GRACE + fixed = 2 * usbtest.RECOVER_SETTLE + usbtest.RECOVER_OVERHEAD + rp = {'name': 'openocd', 'args': '-f target/rp2040.cfg'} + for flasher, steps in ( + # an RP openocd board: reset, reflash, then Rescue-DP POR + one retry + (rp, reset + flash + 2 * flash + fixed), + # openocd on a NON-RP target: rescue_openocd has no RESCUE_CFG entry for + # it, so its two legs are time the board can never spend + ({'name': 'openocd', 'args': '-f target/wch-riscv.cfg'}, + reset + flash + fixed), + # esptool: reset_esptool is a stub (no_op) and rescue refuses a + # non-openocd flasher, so ONE reflash is all it can ever spend + ({'name': 'esptool', 'args': ''}, flash + fixed)): + self.assertEqual(usbtest.recovery_reserve(flasher), steps, + f'{flasher} reserves time it cannot spend, or too little') + + def test_the_reserve_leaves_room_for_the_work_no_step_bounds(self): + """The ladder's step timeouts do not cover the two /proc walks, the roster + json.loads, the child's first import, or the JSON print. With zero margin any + env-overridable bound moving up puts the outer killpg inside the reflash.""" + import usbtest + self.assertGreater(usbtest.RECOVER_OVERHEAD, 0) + rp = {'name': 'openocd', 'args': '-f target/rp2350.cfg'} + bounded = (usbtest.RECOVER_RESET_TIMEOUT + 3 * usbtest.RECOVER_FLASH_TIMEOUT) + self.assertGreaterEqual(usbtest.recovery_reserve(rp) - bounded, + usbtest.RECOVER_OVERHEAD, + 'the reserve equals its own worst case with no margin') + + def test_a_flasher_reserves_nothing_for_a_rescue_it_cannot_run(self): + """rescue_openocd returns False for anything but openocd, so reserving its two + legs elsewhere holds a pool worker AND a usbtest permit for 200s of dead time.""" + import usbtest + self.assertLess(usbtest.recovery_reserve({'name': 'esptool', 'args': ''}), + usbtest.recovery_reserve({'name': 'openocd', + 'args': '-f target/rp2040.cfg'})) + + +class UsbtestRunHelper(unittest.TestCase): + """usbtest.run() is the bounded replacement for subprocess.run: sysfs_write feeds it + input=, and every battery calls that before case 1.""" + + def setUp(self): + import usbtest + self.usbtest = usbtest + + def test_input_kwarg_is_honoured(self): + r = self.usbtest.run(['cat'], input='payload', timeout=10) + self.assertEqual(r.returncode, 0) + self.assertEqual(r.stdout, 'payload') + + def test_capture_output_kwarg_is_accepted(self): + r = self.usbtest.run(['printf', 'x'], capture_output=True, timeout=10) + self.assertEqual(r.stdout, 'x') + + def test_timeout_is_bounded_and_raises(self): + import subprocess + t0 = time.monotonic() + with self.assertRaises(subprocess.TimeoutExpired): + self.usbtest.run(['sleep', '30'], timeout=1) + self.assertLess(time.monotonic() - t0, 15) + + +class BuildBoardContract(unittest.TestCase): + def test_every_return_path_is_a_pair(self): + """main() unpacks `_, nfail = build_board(board)`; a bare int on any path + (the timeout path did) raises TypeError before the pool exists.""" + import ast + src = (Path(TEST_DIR).parents[0] / 'hil_test.py').read_text() + fn = next(n for n in ast.walk(ast.parse(src)) + if isinstance(n, ast.FunctionDef) and n.name == 'build_board') + for node in ast.walk(fn): + if isinstance(node, ast.Return) and node.value is not None: + self.assertIsInstance(node.value, ast.Tuple, + f'build_board returns a non-tuple at line {node.lineno}') + + +class RemoteStaging(unittest.TestCase): + def test_import_closure_is_staged_to_the_rig(self): + # hil_ci.sh stages an explicit scp whitelist; a module that is not on it exists + # locally and in CI checkouts but silently never reaches the remote rig (how + # mtp_test.py was first missed). Walk the local-import closure of everything + # the rig executes and require each file's exact scp entry — a bare-substring + # match would be satisfied by a mention in a comment or the run line. + import ast + hil_dir = Path(TEST_DIR).parents[0] + staged = (hil_dir / 'hil_ci.sh').read_text() + + def imported_paths(pyfile): + # ast, not regex: an earlier regex walker went silently vacuous on a + # multi-line import. ast also sees function-local deferred imports + # (usbtest.py's `import hil_flash` inside the recovery branch). + for node in ast.walk(ast.parse(pyfile.read_text())): + if isinstance(node, ast.Import): + for a in node.names: + yield a.name.replace('.', '/') + '.py' + elif isinstance(node, ast.ImportFrom) and node.module: + if node.module == 'helper': + for a in node.names: + yield f'helper/{a.name}.py' + else: + yield node.module.replace('.', '/') + '.py' + + seeds = ['hil_test.py', 'usbtest.py', 'mtp_test.py'] # CLI + spawned helpers + for f in seeds: # a renamed seed must fail loudly, not fall out of the walk + self.assertTrue((hil_dir / f).exists(), f'stale RemoteStaging seed: {f}') + todo, seen = list(seeds), set() + while todo: + f = todo.pop() + if f in seen or not (hil_dir / f).exists(): + continue # stdlib/site-packages imports have no test/hil file + seen.add(f) + todo += list(imported_paths(hil_dir / f)) + for f in sorted(seen): + self.assertIn(f'"$ROOT_DIR/test/hil/{f}"', staged, + f'{f} runs on the rig but hil_ci.sh does not scp it') + + +class _MtpFakeRig: + """The fake rig shared by the MTP cases: a udev-marker tree under one tmp root and + the scripted pymtp on PYTHONPATH. A plain mixin, NOT a TestCase -- subclassing a + TestCase to reuse a fixture re-runs every inherited test in each subclass.""" + + @classmethod + def setUpClass(cls): + # both file fixtures come from the example's sources, so drift there fails here: + # file id 1 is README.TXT (C define), file id 2 is logo.png (C byte array) + import hashlib + import re + src = Path(TEST_DIR).parents[2] / 'examples/device/mtp/src' + m = re.search(r'#define README_TXT_CONTENT "([^"]+)"', (src / 'mtp_fs_example.c').read_text()) + assert m, 'README_TXT_CONTENT define not found in mtp_fs_example.c' + cls.readme = m.group(1) + data = bytes(int(x, 16) for x in + re.findall(r'0x([0-9a-fA-F]{2})', (src / 'tinyusb_logo_png.h').read_text())) + assert hashlib.md5(data).hexdigest() == '40ef23fc2891018d41a05d4a0d5f822f' + cls.logo = data + + def setUp(self): + self.tmp = TemporaryDirectory() + tmp = Path(self.tmp.name) + self.addCleanup(self.tmp.cleanup) + logo = tmp / 'logo.bin' + logo.write_bytes(self.logo) + self.board = {'uid': 'CAFE01', 'name': 'fakeboard'} + # addCleanup, not tearDown: tearDown does NOT run when setUp raises, and a leaked + # chdir into a deleted temp dir breaks every test after it. + self.saved_env = {k: os.environ.get(k) for k in + ('FAKE_PYMTP_MODE', 'FAKE_PYMTP_UID', 'FAKE_PYMTP_LOGO', + 'FAKE_PYMTP_FILE1', 'PYTHONPATH', 'PYTHONSAFEPATH', + 'HIL_MTP_FAKE_ROOT', 'FAKE_PYMTP_ERRED_MARKER')} + self.addCleanup(self._restore_env) + # A udev-ready marker tree: libmtp-runtime publishes /dev/libmtp-<sysname> only + # after mtp-probe accepts a device, and mtp_test opens THAT device directly rather + # than probing every MTP device on the rig (the parallel-probe race #3790 fixed). + # mirrors the real layout under one root, so <tmp>/sys/bus/usb/devices/1-1 reads + # as the stand-in for /sys/bus/usb/devices/1-1 that it is + dev = tmp / 'sys/bus/usb/devices/1-1' + usbdev = tmp / 'dev/bus/usb/001' + markers = tmp / 'dev' # created by usbdev's parents=True + dev.mkdir(parents=True); usbdev.mkdir(parents=True) + (dev / 'idVendor').write_text('cafe\n') + (dev / 'idProduct').write_text('4017\n') + (dev / 'serial').write_text(self.board['uid'] + '\n') + (dev / 'busnum').write_text('1\n') + (dev / 'devnum').write_text('2\n') + node = usbdev / '002' + node.write_bytes(b'') + (markers / 'libmtp-1-1').symlink_to(node) + os.environ['HIL_MTP_FAKE_ROOT'] = str(tmp) + os.environ['FAKE_PYMTP_ERRED_MARKER'] = str(tmp / 'erred') + os.environ['FAKE_PYMTP_UID'] = self.board['uid'] + os.environ['FAKE_PYMTP_LOGO'] = str(logo) + os.environ['FAKE_PYMTP_FILE1'] = self.readme + stubs = os.path.join(TEST_DIR, 'stubs') + pp = self.saved_env['PYTHONPATH'] + os.environ['PYTHONPATH'] = stubs if not pp else f'{stubs}:{pp}' + # pymtp is vendored next to mtp_test.py, and a script's own dir (sys.path[0]) + # outranks PYTHONPATH — safe-path mode (3.11+) drops it so the fake wins there + os.environ['PYTHONSAFEPATH'] = '1' + for name in ('_enum_timeout', 'MTP_SESSION_MARGIN'): + self.addCleanup(setattr, hil_test, name, getattr(hil_test, name)) + hil_test._enum_timeout = 1 # the wait these tests must outlast; keep it small + # the session scratch files land in cwd + self.addCleanup(os.chdir, os.getcwd()) + os.chdir(tmp) + + def _restore_env(self): + for k, v in self.saved_env.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + + [email protected](sys.version_info < (3, 11), 'fake-pymtp steering needs PYTHONSAFEPATH') +class DeviceMtp(_MtpFakeRig, unittest.TestCase): + """test_device_mtp end to end: the real mtp_test.py subprocess under run_cmd, + with the scripted pymtp fake steered in via PYTHONPATH.""" + + def test_mtp_session_passes_against_scripted_device(self): + os.environ['FAKE_PYMTP_MODE'] = 'ok' + hil_test.test_device_mtp(self.board) # no exception + + def test_absent_device_fails_cleanly(self): + os.environ['FAKE_PYMTP_MODE'] = 'absent' + finished, exc = run_bounded(lambda: hil_test.test_device_mtp(self.board), 30) + self.assertTrue(finished) + self.assertIsInstance(exc, AssertionError) + self.assertIn('MTP device not found', str(exc)) + + def test_libmtp_error_on_one_poll_retries_instead_of_dying(self): + """pymtp raises for USB_LAYER/PTP_LAYER errors -- routine on the first poll + after a flash. An unguarded raise skipped the whole enumeration budget.""" + os.environ['FAKE_PYMTP_MODE'] = 'error_then_ok' + hil_test.test_device_mtp(self.board) # retries past the error, then passes + + def test_libmtp_error_every_poll_fails_cleanly(self): + os.environ['FAKE_PYMTP_MODE'] = 'error' + finished, exc = run_bounded(lambda: hil_test.test_device_mtp(self.board), 30) + self.assertTrue(finished) + self.assertIsInstance(exc, AssertionError) + + def test_hung_mtp_stack_cannot_hang_the_worker(self): + # in-process libmtp blocking in a usbfs ioctl (D state) hangs whatever thread + # made the call, forever — the session must be somewhere disposable + os.environ['FAKE_PYMTP_MODE'] = 'hang' + hil_test.MTP_SESSION_MARGIN = 3 + finished, exc = run_bounded(lambda: hil_test.test_device_mtp(self.board), 25) + self.assertTrue(finished, 'test_device_mtp hung on a wedged MTP stack') + self.assertIsInstance(exc, AssertionError) + + +class ConvoySafeFlasher(unittest.TestCase): + """hil_flash.convoy_safe decides whether a board gets post-HUNG recovery at all. + + It must be true ONLY for flashers that can reach their probe without opening the + poisoned usbfs node: openocd pinned with a roster vid_pid (filters on kernel-cached + sysfs descriptors) and esptool (delivers to a named tty, never enumerates usbfs). + Anything else enumerates by opening nodes, would block in D state on the wedged one + and become a second stray -- JLinkExe included, whose selection is serial-only and + so cannot be pinned at all.""" + + def setUp(self): + import hil_flash + self.f = hil_flash.convoy_safe + + def test_pinned_openocd_is_safe(self): + self.assertTrue(self.f({'name': 'openocd', 'vid_pid': '0x2e8a 0x000c'})) + + def test_unpinned_openocd_is_not(self): + self.assertFalse(self.f({'name': 'openocd'})) + self.assertFalse(self.f({'name': 'openocd', 'vid_pid': ''})) + + def test_esptool_is_safe_without_a_pin(self): + """Delivery is `-p <ttyACM>`; there is no usbfs walk to poison.""" + self.assertTrue(self.f({'name': 'esptool'})) + + def test_enumerating_flashers_are_not(self): + for name in ('jlink', 'stlink', 'lm4flash', 'dfu-util'): + self.assertFalse(self.f({'name': name, 'vid_pid': '0x1366 0x1024'}), + f'{name} must not be treated as convoy-safe') + + def test_missing_or_odd_name_is_not_safe(self): + for flasher in ({}, {'name': None}, {'name': ''}): + self.assertFalse(self.f(flasher)) + + +class UnresolvedControllerBucket(unittest.TestCase): + """An unresolved controller must budget in ONE bucket. Taking a permit on every slot + serialized the whole fleet the moment a single board could not be resolved.""" + + def setUp(self): + import threading + from helper import hil_lock + self.hil_lock = hil_lock + self.saved = (hil_lock.controller_map, hil_lock.controller_meta, + hil_lock.controller_hints, hil_lock.log) + hil_lock.controller_map, hil_lock.controller_meta = {}, threading.Lock() + hil_lock.controller_hints, hil_lock.log = {}, lambda *a, **k: None + + def tearDown(self): + (self.hil_lock.controller_map, self.hil_lock.controller_meta, + self.hil_lock.controller_hints, self.hil_lock.log) = self.saved + + def _slots(self, uid, warn): + import threading + sems = self.hil_lock.make_permit_sems(threading.Semaphore, 2) + return self.hil_lock.controller_permit(sems, uid, warn_unknown=warn).slots + + def test_unresolved_boards_share_one_slot(self): + for warn in (False, True): + slots = self._slots('NOSUCHUID', warn) + self.assertEqual(len(slots), 1, 'unresolved uid took more than one slot') + self.assertEqual(slots, self._slots('OTHERUID', warn), + 'unresolved boards must share the bucket, not spread over it') + + def test_the_semaphore_array_is_long_enough_for_the_unknown_slot(self): + """UNKNOWN_SLOT indexes one PAST the real slots. An array sized to + CONTROLLER_SLOTS IndexErrors on the first unresolved board, inside a pool worker, + which map_async turns into a total loss of every board's results.""" + import threading + sems = self.hil_lock.make_permit_sems(threading.Semaphore, 2) + self.assertGreater(len(sems), self.hil_lock.UNKNOWN_SLOT) + + def test_the_unknown_bucket_never_lends_a_controller_a_second_budget(self): + """A private FULL budget let 2 unknown batteries join 2 resolved ones on the same + physical controller -- 4 where the width is 2. One at a time caps that at +1.""" + import threading + sems = self.hil_lock.make_permit_sems(threading.Semaphore, 2) + first = self.hil_lock.controller_permit(sems, 'NOSUCHUID') + first.__enter__() + self.addCleanup(first.__exit__) + second = self.hil_lock.controller_permit(sems, 'OTHERUID') + self.assertFalse(sems[second.slots[0]].acquire(blocking=False), + 'a second unresolved board got in alongside the first') + + def test_every_real_slot_keeps_the_full_width(self): + import threading + sems = self.hil_lock.make_permit_sems(threading.Semaphore, 2) + for s in sems[:self.hil_lock.CONTROLLER_SLOTS]: + self.assertTrue(s.acquire(blocking=False) and s.acquire(blocking=False)) + self.assertFalse(s.acquire(blocking=False)) + + +class ThroughputPayloadBound(unittest.TestCase): + """An unknown link speed must pick the FS payload, and each dd must be bounded by the + payload actually requested.""" + + def test_only_a_read_high_speed_gets_the_big_payload(self): + for speed in (None, '12', '1.5'): + self.assertTrue(hil_test.link_is_fs(speed), f'{speed!r} must scale as FS') + for speed in ('480', '5000', '10000'): + self.assertFalse(hil_test.link_is_fs(speed)) + + def test_dd_bound_scales_with_the_payload_and_stays_bounded(self): + self.assertGreater(hil_test.dd_timeout(16), hil_test.dd_timeout(1)) + self.assertGreaterEqual(hil_test.dd_timeout(1), 30) # setup + flush floor + # still an INNER bound: run_cmd's own timeout must stay the outer one + self.assertLess(hil_test.dd_timeout(16), hil_test.hil_util.CMD_TIMEOUT) + + +class FindDeviceCache(unittest.TestCase): + """usbtest.find_device's cache is keyed by sysname, a bus-topology path: after a + renumber it can name a different cafe:4010 board, and idVendor/idProduct are identical + on every one of them. Only `serial` tells them apart.""" + + def setUp(self): + import usbtest + self.usbtest = usbtest + self.tmp = TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.saved_sys_usb = usbtest.SYS_USB + usbtest.SYS_USB = Path(self.tmp.name) + usbtest._DEV_CACHE.clear() + self._dev('1-2', 'AAAA', devnum=2) + self._dev('1-3', 'BBBB', devnum=3) + + def tearDown(self): + self.usbtest.SYS_USB = self.saved_sys_usb + self.usbtest._DEV_CACHE.clear() + + def _dev(self, sysname, serial, devnum): + d = Path(self.tmp.name) / sysname + d.mkdir() + for name, val in (('idVendor', self.usbtest.VID), ('idProduct', self.usbtest.PID), + ('serial', serial), ('busnum', '1'), ('devnum', str(devnum)), + ('speed', '480'), ('bcdDevice', '0104')): + (d / name).write_text(val + '\n') + + def test_cached_sysname_with_another_boards_serial_is_rejected(self): + self.usbtest._DEV_CACHE['bbbb'] = '1-2' # renumbered: 1-2 is board AAAA now + dev = self.usbtest.find_device('BBBB') + self.assertEqual(dev['sysname'], '1-3') + self.assertEqual(dev['serial'], 'BBBB') + self.assertEqual(self.usbtest._DEV_CACHE['bbbb'], '1-3') + + def test_cached_sysname_with_the_right_serial_is_kept(self): + self.usbtest._DEV_CACHE['bbbb'] = '1-3' + dev = self.usbtest.find_device('BBBB') + self.assertEqual((dev['sysname'], dev['serial']), ('1-3', 'BBBB')) + + def test_a_cached_device_that_vanished_falls_back_to_the_scan(self): + self.usbtest._DEV_CACHE['bbbb'] = '1-9' # gone from sysfs + self.assertEqual(self.usbtest.find_device('BBBB')['sysname'], '1-3') + + +class ReRunSpecNamesOnlyWhatFailed(unittest.TestCase): + """The pool-guard path used to leave this unwritten -- and a fresh run has already + unlinked it -- so build.yml's re-run step found nothing and GitHub re-tested all ~26 + boards to find the one that wedged.""" + + def test_only_failed_boards_and_their_failed_tests(self): + with TemporaryDirectory() as td: + d = Path(td) + spec = d / 'cfg.failed' + hil_test._write_failed_spec(spec, d, [ + ('good', 0, [], None, 1.0), + ('bad', 2, ['device/cdc_msc'], None, 1.0), + ('wedged', 1, [], None, 0.0), # never reported: no test list + ]) + got = spec.read_text() + self.assertIn('-b bad', got) + self.assertIn('-bt bad:device/cdc_msc', got) + self.assertIn('-b wedged', got) + self.assertNotIn('good', got) + + def test_an_all_green_run_removes_a_stale_spec(self): + with TemporaryDirectory() as td: + d = Path(td) + spec = d / 'cfg.failed' + spec.write_text('--accumulate -b stale') + hil_test._write_failed_spec(spec, d, [('good', 0, [], None, 1.0)]) + self.assertFalse(spec.exists(), 'a stale spec would re-run last time\'s boards') + + +class WedgedPidsFailsClosed(unittest.TestCase): + """A scan that could not SEE the holder must not report "no holder". The holder is + root-owned (run_case uses sudo -n when the node is not writable) and that is exactly + what a hidepid/ProtectProc mount hides — so an unreadable /proc reading as clear + clears unrecovered_hang and lets cleanup unbind a device whose usbfs lock is still + held, which deadlocks the bus rather than one board.""" + + def test_returns_a_completeness_flag_not_just_pids(self): + import usbtest + got = usbtest.wedged_pids('/dev/bus/usb/999/999') + self.assertIsInstance(got, tuple) + self.assertEqual(len(got), 2, 'the caller needs (pids, complete)') + + def test_a_restricted_proc_is_reported_incomplete(self): + import usbtest + self.addCleanup(setattr, usbtest.os, 'geteuid', usbtest.os.geteuid) + self.addCleanup(setattr, usbtest.os, 'access', usbtest.os.access) + usbtest.os.geteuid = lambda: 1000 # not root + usbtest.os.access = lambda p, m: False # /proc/1/cmdline unreadable + _, complete = usbtest.wedged_pids('/dev/bus/usb/999/999') + self.assertFalse(complete, 'a hidden holder was reported as absent') + + +class MtpGioOrdering(_MtpFakeRig, unittest.TestCase): + """gio must not run until the device is READY. + + gvfs claims an MTP device only AFTER udev probing, so the mount this unmounts cannot + exist before /dev/libmtp-<sysname> is published -- an unmount issued earlier is a + guaranteed no-op that still forks a process, and it leaves the window between the + unmount and the open unprotected, which is the hang it exists to prevent. Running it + per poll iteration also forks one gio per second of the enumeration budget.""" + + def setUp(self): + super().setUp() + tmp = Path(self.tmp.name) + self.gio_log = tmp / 'gio.log' + binn = tmp / 'bin'; binn.mkdir() + (binn / 'gio').write_text('#!/bin/sh\necho "$@" >> "$GIO_LOG"\n') + (binn / 'gio').chmod(0o755) + for k in ('PATH', 'GIO_LOG'): + old = os.environ.get(k) + self.addCleanup(lambda k=k, v=old: os.environ.__setitem__(k, v) + if v is not None else os.environ.pop(k, None)) + os.environ['GIO_LOG'] = str(self.gio_log) + os.environ['PATH'] = f'{binn}:{os.environ["PATH"]}' + + def _gio_calls(self): + return self.gio_log.read_text().splitlines() if self.gio_log.exists() else [] + + def test_gio_does_not_run_before_the_device_is_ready(self): + (Path(self.tmp.name) / 'dev' / 'libmtp-1-1').unlink() # never becomes ready + os.environ['FAKE_PYMTP_MODE'] = 'absent' + run_bounded(lambda: hil_test.test_device_mtp(self.board), 30) + calls = self._gio_calls() + self.assertEqual(calls, [], f'gio ran {len(calls)}x with no device ready: {calls}') + + def test_gio_still_runs_once_the_device_is_ready(self): + """The guard must delay the unmount, not delete it.""" + os.environ['FAKE_PYMTP_MODE'] = 'ok' + hil_test.test_device_mtp(self.board) + self.assertTrue(self._gio_calls(), 'gio never ran for a ready device') + + +class MtpGioFallthrough(unittest.TestCase): + """The missing-gio path must fall THROUGH to detection. `continue` there skips the + deadline check and the sleep as well, spinning at 100% CPU until the caller's outer + kill — reported as a wedged DUT for a missing apt package.""" + + def test_a_missing_gio_still_bounds_the_session(self): + import subprocess + with TemporaryDirectory() as td: + env = {**os.environ, 'PATH': td, # no gio, no anything + 'PYTHONPATH': os.path.join(TEST_DIR, 'stubs'), + 'FAKE_PYMTP_MODE': 'none', 'PYTHONSAFEPATH': '1'} + t0 = time.monotonic() + r = subprocess.run([sys.executable, + str(Path(TEST_DIR).parents[0] / 'mtp_test.py'), + '--uid', 'CAFE01', '--timeout', '1'], + capture_output=True, text=True, timeout=60, env=env) + elapsed = time.monotonic() - t0 + self.assertLess(elapsed, 30, f'did not honour --timeout 1 ({elapsed:.1f}s)') + self.assertNotEqual(r.returncode, 0) + # The assertions above are satisfied by an immediate CRASH, which is exactly what + # shipped through this test once: `pass` left gio unbound and the next line + # dereferenced it. Assert the behaviour the docstring names -- it POLLED for the + # device (so it spent its budget) and did not die on a traceback. + self.assertGreater(elapsed, 0.8, + f'exited without polling ({elapsed:.1f}s) -- it crashed') + self.assertNotIn('Traceback', r.stderr) + self.assertIn('MTP device not found', r.stdout + r.stderr) + + +class RunWhileContract(unittest.TestCase): + """The read-while-we-write runner. Its child can still outlast SIGKILL -- but unlike + the thread it replaced, an abandoned child is a real process in its own session, so + the containment sweep finds it and the report names it.""" + + def setUp(self): + from helper import hil_util + self.hil_util = hil_util + + def test_an_error_in_work_is_not_swallowed(self): + """A `return` inside the reap's `finally` discarded it: an assert in the CDC + write half vanished and the caller went on to compare data it never sent.""" + def boom(): + raise AssertionError('the write failed') + with self.assertRaises(AssertionError): + self.hil_util.run_alongside(['sh', '-c', 'printf X'], boom, 5) + + def test_the_child_is_reaped_even_when_work_raises(self): + seen = {} + + def boom(): + raise AssertionError('x') + # a duration no other process would plausibly pick: `pgrep -f` searches the WHOLE + # machine, so a bare `sleep 20` matched an unrelated background job -- another + # agent session's retry loop, in the case that exposed this -- and failed a test + # about our own child. Observed failing 3/3 in isolation while that loop ran. + sentinel = '20.0451' + with self.assertRaises(AssertionError): + self.hil_util.run_alongside(['sleep', sentinel], boom, 1) + # nothing of ours is left running: the reap ran on the error path too + import subprocess + out = subprocess.run(['pgrep', '-f', f'^sleep {sentinel}'], + capture_output=True, text=True) + seen['strays'] = [p for p in out.stdout.split() if p] + self.assertEqual(seen['strays'], [], 'work() raising leaked the child') + + def test_an_abandoned_child_is_in_its_own_session(self): + """killpg on it reaps whatever it spawned, and it cannot take our group with it.""" + import subprocess + pgids = {} + + def check(): + time.sleep(0.2) + pgids['child'] = os.getpgid(self._proc_pid) + + real_popen = subprocess.Popen + + def spy(argv, **kw): + p = real_popen(argv, **kw) + self._proc_pid = p.pid + return p + self.addCleanup(setattr, subprocess, 'Popen', real_popen) + subprocess.Popen = spy + self.hil_util.run_alongside(['sleep', '0.5'], check, 5) + subprocess.Popen = real_popen + self.assertNotEqual(pgids['child'], os.getpgid(0)) + + +class UsbScanIsTheOneWalk(unittest.TestCase): + """Three call sites each had a different subset of the three things this must get + right; none had all three. The expensive read is `serial` -- served under the device + lock a wedged usbfs ioctl holds -- so it must come LAST, only for devices the free + descriptor fields could not rule out, and never twice for a path that stranded.""" + + def setUp(self): + from helper import hil_util + self.hil_util = hil_util + self.td = TemporaryDirectory() + self.addCleanup(self.td.cleanup) + self.root = Path(self.td.name) + self.reads = [] + real = hil_util.read_sysfs + + def counting(path, *a, **k): + self.reads.append(path) + return real(path, *a, **k) + self.addCleanup(setattr, hil_util, 'read_sysfs', real) + hil_util.read_sysfs = counting + + def _dev(self, name, vid, pid, serial='S1'): + d = self.root / name + d.mkdir() + (d / 'idVendor').write_text(vid + '\n') + (d / 'idProduct').write_text(pid + '\n') + (d / 'serial').write_text(serial + '\n') + return d + + def _scan(self, **kw): + import glob as _g + real_glob = _g.glob + self.addCleanup(setattr, self.hil_util.glob, 'glob', real_glob) + self.hil_util.glob.glob = lambda pat: [str(p) for p in self.root.iterdir()] + return self.hil_util.usb_scan(**kw) + + def test_a_mismatched_vid_pid_costs_no_serial_read(self): + """`serial` is the ONE attribute here served under the device lock, so it is the + one that can block on a wedged device. Filtering on the lock-free descriptor pair + first is what keeps a scan for our board off every other board's locked read.""" + self._dev('1-1', '1234', '5678') + self._dev('1-2', 'cafe', '4010', serial='UID1') + devs = self._scan(vid_pid=('cafe', '4010')) + self.assertEqual([d['serial'] for d in devs], ['UID1']) + # the ruled-out device's locked attribute was never touched + self.assertNotIn(str(self.root / '1-1' / 'serial'), self.reads) + + +class AbandonExitSurvivesAFailedFork(unittest.TestCase): + """Pool() forks, and after a convoy -- every stranded read holding a thread and an fd -- + that fork is what hits EAGAIN/ENOMEM. It now runs inside the try, so the finally can + reach _abandon_exit with pool and mgr still None.""" + + def test_none_pool_and_manager_still_write_the_banner(self): + # a subprocess, because _abandon_exit ends in os._exit: in-process it would take + # the test runner with it, before any assertion could run + import json + import subprocess + with TemporaryDirectory() as td: + rd = Path(td) + # it takes the report DIRECTORY now and re-renders both artifacts from the + # sidecar, so seed the sidecar -- the markdown is output, not input + (rd / 'hil_report.json').write_text(json.dumps( + {'rows': [{'board': 'boardA', 'cells': {'cdc_msc': 'pass'}, + 'duration': '1s'}], + 'banner': '', 'scope': '', 'caveat': ''})) + src = ( + 'import sys, types\n' + f'sys.path.insert(0, {str(Path(TEST_DIR).parents[0])!r})\n' + 'st = types.ModuleType("serial")\n' + 'st.Serial = type("Serial", (), {})\n' + 'st.SerialException = type("SerialException", (Exception,), {})\n' + 'st.SerialTimeoutException = type("E2", (Exception,), {})\n' + 'sys.modules.setdefault("serial", st)\n' + 'import hil_test\n' + f'hil_test._abandon_exit(None, None, True, 1, __import__("pathlib")' + f'.Path({str(rd)!r}))\n') + r = subprocess.run([sys.executable, '-c', src], capture_output=True, + text=True, timeout=120) + self.assertEqual(r.returncode, 1, r.stderr) + self.assertTrue((rd / 'hil_report.md').read_text().startswith( + '**HIL run abandoned'), 'the abandon banner never reached the report') + self.assertIn('abandoned', + json.loads((rd / 'hil_report.json').read_text())['caveat']) + + def test_kill_pool_children_tolerates_a_pool_that_never_existed(self): + from helper import hil_health + self.assertEqual(hil_health.kill_pool_children(None), 0) + self.assertEqual(hil_health.kill_pool_children(None, None), 0) + + +class UsbtestOuterBoundIsOneValue(unittest.TestCase): + """run_cmd's kill is the ONE bound, and it must carry a recovery reserve only when a + recovery can actually run. Otherwise a board on a path that cannot recover holds a pool + worker and its battery permit idle for the difference, under a usbtest width of 2.""" + + def _invoke(self, flasher, skip_flash=False): + from contextlib import contextmanager + from helper import hil_lock, hil_util + + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + dev = Path(td.name) / 'dev1' + dev.mkdir() + for attr, val in (('serial', 'UID1'), ('idVendor', 'cafe'), ('idProduct', '4010')): + (dev / attr).write_text(val + '\n') + + def patch(obj, name, value): + self.addCleanup(setattr, obj, name, getattr(obj, name)) + setattr(obj, name, value) + + def _permit(uid): + yield + + seen = {} + + def fake_run(cmd, **kw): + import subprocess + seen['cmd'], seen['timeout'] = cmd, kw.get('timeout') + return subprocess.CompletedProcess(cmd, 1, stdout=b'', stderr=b'stub') + + from helper import hil_util as _hu + patch(_hu, 'glob', types.SimpleNamespace(glob=lambda p: [str(dev)])) + patch(hil_test, 'USBTEST_SETTLE', 0) # see no_settle + patch(hil_lock, 'usbtest_permit', contextmanager(_permit)) + patch(hil_test, 'skip_flash', skip_flash) + patch(hil_test, '_current_fw', '/tmp/fw.elf') + patch(hil_util, 'run_cmd', fake_run) + with self.assertRaises(hil_test.TestFail): + hil_test.test_device_usbtest({'name': 'b', 'uid': 'UID1', 'flasher': flasher}) + return seen + + def test_a_recoverable_board_reserves_the_recovery_budget(self): + import usbtest + flasher = {'name': 'openocd', 'vid_pid': '0x1366 0x1024', + 'args': '-f target/rp2040.cfg'} + seen = self._invoke(flasher) + want = (hil_test.USBTEST_BATTERY_BUDGET + hil_test.USBTEST_OVERSHOOT + + usbtest.recovery_reserve(flasher)) + self.assertEqual(seen['timeout'], want) + + def test_the_reserve_follows_the_board_not_a_fleet_constant(self): + """Two convoy-safe openocd boards, one RP and one not: the non-RP board cannot + run rescue_openocd, so reserving its two legs holds a pool worker and a usbtest + permit for 200s of dead time.""" + rp = self._invoke({'name': 'openocd', 'vid_pid': '0x1366 0x1024', + 'args': '-f target/rp2040.cfg'}) + wch = self._invoke({'name': 'openocd', 'vid_pid': '0x1366 0x1024', + 'args': '-f target/wch-riscv.cfg'}) + self.assertLess(wch['timeout'], rp['timeout']) + + def test_a_board_with_no_recovery_does_not_pay_for_one(self): + seen = self._invoke({'name': 'stlink', 'uid': 'X'}) # never convoy_safe + # It does not carry the RECOVERY reserve it cannot spend + self.assertEqual(seen['timeout'], + hil_test.USBTEST_BATTERY_BUDGET + hil_test.USBTEST_OVERSHOOT) + # ...but it MUST still exceed the child's own --budget. The battery checks the + # budget before dispatching, so it can overshoot by one already-started case; an + # equal bound SIGKILLs it just as it goes to print, turning ~29 real per-case + # verdicts into "usbtest did not run" and re-paying the whole battery on retry. + toks = seen['cmd'].split() + budget = int(toks[toks.index('--budget') + 1]) + case_timeout = int(toks[toks.index('--timeout') + 1]) + self.assertGreaterEqual(seen['timeout'] - budget, case_timeout, + 'the outer kill can land mid-case, before the JSON') + + def test_skip_flash_still_bounds_the_child(self): + """--skip-flash disables recovery, so the child must not be given a reserve it + cannot spend -- but it MUST still be bounded.""" + seen = self._invoke({'name': 'openocd', 'vid_pid': '0x1366 0x1024'}, skip_flash=True) + self.assertEqual(seen['timeout'], + hil_test.USBTEST_BATTERY_BUDGET + hil_test.USBTEST_OVERSHOOT) + + +class UsbtestRetryPolicy(unittest.TestCase): + """The pool guard bounds ONE battery; the retry loop multiplies it by max_retry. + So the loop must retry only what a retry can fix.""" + + def _patch(self, obj, name, value): + # addCleanup, not a finally: a failing assert must not leave the real module + # patched for whatever test runs next (max_retry only exists once main() ran, + # so restoring it means DELETING it again) + if hasattr(obj, name): + self.addCleanup(setattr, obj, name, getattr(obj, name)) + else: + self.addCleanup(delattr, obj, name) + setattr(obj, name, value) + + def _attempts(self, exc): + """How many times test_example runs the test fn before giving up.""" + import hil_flash + calls = [] + + def fake_test(board): + calls.append(1) + raise exc + + self._patch(hil_flash, 'find_firmware', lambda *a, **k: Path('/nonexistent/fw.elf')) + self._patch(hil_test, 'skip_flash', True) # no probe, no hardware + self._patch(hil_test, 'max_retry', 3) + self._patch(hil_test, 'log_line', lambda *a, **k: None) + hil_test.test_fake_example = fake_test + self.addCleanup(delattr, hil_test, 'test_fake_example') + hil_test.test_example({'name': 'b', 'uid': 'u', 'flasher': {'name': 'openocd'}}, + 'v', 'fake/example') + return len(calls) + + def test_a_per_case_verdict_is_not_retried(self): + # re-running the battery only re-observes a number the JSON already reported + self.assertEqual(self._attempts(hil_test.TestFail('29/30', parsed=True)), 1) + + def test_a_transient_failure_is_retried(self): + self.assertEqual(self._attempts(hil_test.TestFail('usbtest did not run')), 3) + + +class UsbtestOuterKillStaysRetryable(unittest.TestCase): + """rc 124 is run_cmd's timer expiring, NOT proof the DUT is wedged -- a healthy + battery can hit it under load. Suppressing the retry to save the budget also + suppresses the reflash test_example does before each attempt, which is the only + thing left to unpoison the DUT where usbtest's in-band recovery is off.""" + + def setUp(self): + from contextlib import contextmanager + from helper import hil_lock + self.td = TemporaryDirectory() + self.addCleanup(self.td.cleanup) + dev = Path(self.td.name) / 'dev1' + dev.mkdir() + # a real (readable) fake sysfs node, so the bounded reads run unmodified + for attr, val in (('serial', 'UID1'), ('idVendor', 'cafe'), ('idProduct', '4010')): + (dev / attr).write_text(val + '\n') + self.dev = dev + + def patch(obj, name, value): + saved = getattr(obj, name) + self.addCleanup(setattr, obj, name, saved) + setattr(obj, name, value) + + from helper import hil_util as _hu + patch(_hu, 'glob', types.SimpleNamespace(glob=lambda p: [str(dev)])) + patch(hil_test, 'USBTEST_SETTLE', 0) # see no_settle + def _permit(uid): # a real generator: a lambda returning an iterator has + yield # no .throw(), so any raise inside the `with` would + # surface as an AttributeError from contextlib instead + patch(hil_lock, 'usbtest_permit', contextmanager(_permit)) + patch(hil_test, 'skip_flash', True) + + def test_rc_124_stays_retryable(self): + import subprocess + from helper import hil_util + saved = hil_util.run_cmd + self.addCleanup(setattr, hil_util, 'run_cmd', saved) + hil_util.run_cmd = lambda *a, **k: subprocess.CompletedProcess( + 'usbtest', 124, stdout=b'', stderr=b'killed on the outer bound') + with self.assertRaises(hil_test.TestFail) as cm: + hil_test.test_device_usbtest({'name': 'b', 'uid': 'UID1', + 'flasher': {'name': 'openocd'}}) + self.assertFalse(cm.exception.parsed, + 'the retry is the last reflash a poisoned DUT gets') + + def test_a_crashed_tool_stays_retryable(self): + import subprocess + from helper import hil_util + saved = hil_util.run_cmd + self.addCleanup(setattr, hil_util, 'run_cmd', saved) + hil_util.run_cmd = lambda *a, **k: subprocess.CompletedProcess( + 'usbtest', 1, stdout=b'', stderr=b'ImportError: no module named usbtest') + with self.assertRaises(hil_test.TestFail) as cm: + hil_test.test_device_usbtest({'name': 'b', 'uid': 'UID1', + 'flasher': {'name': 'openocd'}}) + self.assertFalse(cm.exception.parsed) + + +class RemoteDirIsScreened(unittest.TestCase): + """REMOTE_DIR reaches the rig through `rm -rf`, an scp remote path and an rsync + remote path -- all re-split and expanded by the REMOTE shell, none of them + protectable by quoting the local variable. So the script screens the value once + instead: it must survive that re-split unchanged, and `~` must keep working.""" + + def _run(self, remote_dir, *args, keep_going=False): + import subprocess + with TemporaryDirectory() as td: + # real ssh/scp/rsync would reach the rig; these just record the argv. Exit 77 + # unless the caller needs the script to run on to the second ssh. + rc = 0 if keep_going else 77 + for tool in ('ssh', 'scp', 'rsync'): + write_script(Path(td) / tool, f'echo "stub-{tool} $*" >&2; exit {rc}') + # hil_ci.sh now refuses an all-boards run with nothing built, so this arg-quoting + # test needs a checkout stub with one build dir to reach the run invocation + root = Path(td) / 'root' + (root / 'test' / 'hil').mkdir(parents=True) + (root / 'test' / 'hil' / 'hil_test.py').touch() + (root / 'examples' / 'cmake-build-alpha').mkdir(parents=True) + env = {**os.environ, 'REMOTE_DIR': remote_dir, 'REMOTE': 'stub', + 'ROOT_DIR': str(root), + 'PATH': td + os.pathsep + os.environ['PATH']} + return subprocess.run( + ['bash', str(Path(TEST_DIR).parents[0] / 'hil_ci.sh'), *args], + capture_output=True, text=True, timeout=60, env=env) + + def test_whitespace_is_refused(self): + # unscreened, the remote `rm -rf -- "$1"` gets a TRUNCATED path and deletes + # the wrong tree + r = self._run('/tmp/hil dir') + self.assertNotEqual(r.returncode, 0) + self.assertIn('REMOTE_DIR', r.stderr) + + def test_command_substitution_is_refused(self): + r = self._run('/tmp/$(touch pwned)') + self.assertNotEqual(r.returncode, 0) + self.assertIn('REMOTE_DIR', r.stderr) + + def test_bare_root_is_refused(self): + r = self._run('/') + self.assertNotEqual(r.returncode, 0) + self.assertIn('REMOTE_DIR', r.stderr) + + def test_a_tilde_path_is_accepted(self): + """The one override %q broke: `~` must reach the remote shell UNESCAPED or it + creates a literal '~' directory in the login dir.""" + r = self._run('~/tinyusb-hil') + self.assertIn('~/tinyusb-hil', r.stderr) # got as far as the first ssh + self.assertNotIn('\\~', r.stderr) # %q escapes it; the remote shell won't + + def test_paths_that_would_rm_rf_something_huge_are_refused(self): + """Passing the tilde through UNESCAPED is what makes this dangerous: the remote + shell expands `~/` to the login dir, so `rm -rf -- "$1"` takes out $HOME -- one + typo away from the documented REMOTE_DIR=~/dir override. A bare root, a + no-component path and a foreign ~user are the same class.""" + for bad in ('~/', '~root/x', '~-', '//', '/.', '/tmp/hil/'): + with self.subTest(remote_dir=bad): + r = self._run(bad) + self.assertNotEqual(r.returncode, 0, f'{bad!r} was accepted') + self.assertIn('REMOTE_DIR', r.stderr) + + def test_an_arg_containing_a_space_survives_the_remote_resplit(self): + """ssh joins its argv into ONE string the remote shell re-splits, so an unquoted + `-t 'host/cdc msc'` arrives as two arguments and hil_test.py sees a stray word + where it expects the config path.""" + r = self._run('/tmp/tinyusb-hil', '-t', 'host/cdc msc', keep_going=True) + run_line = [l for l in r.stderr.splitlines() if 'bash -s --' in l][-1] + self.assertIn(r'host/cdc\ msc', run_line) + + +class EveryBoardIsStaged(unittest.TestCase): + """One hil_test.py run takes several `-b` flags, and hil-operator hands it the whole board + set that way. The `-b` parse loop kept a single BOARD, so only the LAST board's binaries + were rsynced and every other board died on the rig with a missing firmware path -- after + its flash slot and lock were already spent. + + Three of these five fail against the pre-fix script (the discriminating unbuilt case + puts the board FIRST, because the old single-BOARD parse happened to handle a trailing + one correctly); the run-line and variant-dir tests are characterization -- the old script + already forwarded ARGS whole and read variants from the config for its one board.""" + + def _run(self, boards, cfg_boards=None, variants=None): + import json + import subprocess + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + root = Path(td.name) / 'root' + (root / 'test' / 'hil').mkdir(parents=True) + (root / 'test' / 'hil' / 'hil_test.py').touch() + built = cfg_boards if cfg_boards is not None else boards + (root / 'examples').mkdir(parents=True, exist_ok=True) + for b in built: + (root / 'examples' / f'cmake-build-{b}').mkdir(parents=True) + roster = [{'name': b} for b in boards] + for entry in roster: + for v in (variants or {}).get(entry['name'], []): + entry.setdefault('variant', []).append({'name': v}) + cfg = root / 'test' / 'hil' / 'cfg.json' + cfg.write_text(json.dumps({'boards': roster})) + stubs = Path(td.name) / 'bin' + stubs.mkdir() + # real ssh/scp/rsync would reach the rig; these just record the argv + for tool in ('ssh', 'scp', 'rsync'): + write_script(stubs / tool, f'echo "stub-{tool} $*" >&2; exit 0') + env = {**os.environ, 'REMOTE': 'stub', 'ROOT_DIR': str(root), 'CONFIG': str(cfg), + 'PATH': str(stubs) + os.pathsep + os.environ['PATH']} + args = [a for b in boards for a in ('-b', b)] + r = subprocess.run(['bash', str(Path(TEST_DIR).parents[0] / 'hil_ci.sh'), *args], + capture_output=True, text=True, timeout=60, env=env) + r.rsyncs = [l for l in r.stderr.splitlines() if l.startswith('stub-rsync')] + # the RUN ssh is the one carrying hil_test.py's args; the setup ssh is not + r.run_lines = [l for l in r.stderr.splitlines() if '--retry 1' in l] + return r + + def test_binaries_for_every_requested_board_are_copied(self): + r = self._run(['alpha', 'beta', 'gamma']) + self.assertEqual(r.returncode, 0, r.stderr) + for b in ('alpha', 'beta', 'gamma'): + self.assertTrue(any(f'cmake-build-{b} ' in l for l in r.rsyncs), + f'{b} binaries never staged: {r.rsyncs}') + + def test_every_board_reaches_hil_test(self): + r = self._run(['alpha', 'beta']) + self.assertEqual(len(r.run_lines), 1, r.stderr) + self.assertIn('-b alpha', r.run_lines[0]) + self.assertIn('-b beta', r.run_lines[0]) + + def test_an_unbuilt_board_aborts_before_anything_is_staged(self): + """The discriminating case: the unbuilt board is FIRST. The pre-fix script kept only + the last -b, found it built, and ran happily while silently testing one board. It also + has to fail BEFORE staging -- the old in-loop check fired after the remote tree was + wiped and earlier boards were rsynced, costing a run and leaving a half-staged rig.""" + r = self._run(['alpha', 'beta'], cfg_boards=['beta']) + self.assertNotEqual(r.returncode, 0, 'unbuilt first board was accepted') + self.assertIn('alpha', r.stdout + r.stderr) + self.assertEqual(r.rsyncs, [], f'staged despite an unbuilt board: {r.rsyncs}') + self.assertEqual(r.run_lines, [], 'reached the run despite an unbuilt board') + + def test_a_board_whose_firmware_is_only_a_variant_dir_is_accepted(self): + """Variant names are not required to be prefixed with the board name, so a board can + own no `cmake-build-<board>` dir at all. A pre-flight that only globs the board name + rejects it and tells the user to build firmware that is already there.""" + r = self._run(['alpha', 'beta'], cfg_boards=['alpha', 'odd-name-v'], + variants={'beta': ['odd-name-v']}) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertTrue(any('cmake-build-odd-name-v ' in l for l in r.rsyncs), + f"beta's variant dir never staged: {r.rsyncs}") + + def test_all_unbuilt_boards_are_named_at_once(self): + """One build round should fix every complaint, so the guard reports the whole set.""" + r = self._run(['alpha', 'beta', 'gamma'], cfg_boards=['beta']) + self.assertNotEqual(r.returncode, 0) + out = r.stdout + r.stderr + self.assertIn('alpha', out) + self.assertIn('gamma', out) + + +class StagingCoversEveryBoardForm(unittest.TestCase): + """hil_test.py declares `-b, --board` with action='append', so argparse accepts --board X, + --board=X and -bX too. Staging only the bare form sent boards to the rig with no firmware, + where every test logs `Skip (no binary)` and counts zero errors -- a green row for a board + that was never flashed. Also covers the roster check, which has to fire BEFORE the remote + tree is wiped, since hil_test.py rejects an unknown -b for the whole run.""" + + def _run(self, argv, built, roster=None, variants=None, env_extra=None, stale=None): + import json + import subprocess + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + root = Path(td.name) / 'root' + (root / 'test' / 'hil').mkdir(parents=True) + (root / 'test' / 'hil' / 'hil_test.py').touch() + (root / 'examples').mkdir(parents=True, exist_ok=True) + for b in built: + (root / 'examples' / f'cmake-build-{b}').mkdir(parents=True) + entries = [{'name': b} for b in (roster if roster is not None else built)] + for e in entries: + for v in (variants or {}).get(e['name'], []): + e.setdefault('variant', []).append({'name': v}) + cfg = root / 'test' / 'hil' / 'cfg.json' + cfg.write_text(json.dumps({'boards': entries})) + stubs = Path(td.name) / 'bin' + stubs.mkdir() + # ssh joins its argv into ONE string that the REMOTE shell re-splits, and feeds the + # heredoc on stdin. A stub that echoes "$*" hides exactly that, which is how a + # completely broken env-forwarding change once passed its own test -- so this stub + # re-splits like the real thing and reports the script body separately. + write_script(stubs / 'ssh', 'shift; printf "REMOTE-ARGV: %s\\n" "$*" >&2; ' + 'body=$(cat); printf "REMOTE-BODY: %s\\n" "$body" >&2; exit 0') + for tool in ('scp', 'rsync'): + write_script(stubs / tool, f'echo "stub-{tool} $*" >&2; exit 0') + for name, content in (stale or {}).items(): + (root / name).write_text(content) + env = {**os.environ, 'REMOTE': 'stub', 'ROOT_DIR': str(root), 'CONFIG': str(cfg), + 'PATH': str(stubs) + os.pathsep + os.environ['PATH'], **(env_extra or {})} + r = subprocess.run(['bash', str(Path(TEST_DIR).parents[0] / 'hil_ci.sh'), *argv], + capture_output=True, text=True, timeout=60, env=env) + r.rsyncs = [l for l in r.stderr.splitlines() if l.startswith('stub-rsync')] + r.run_lines = [l for l in r.stderr.splitlines() if '--retry 1' in l] + r.body = '\n'.join(l for l in r.stderr.splitlines() if l.startswith('REMOTE-BODY')) + r.stale_left = {name: (root / name).exists() for name in (stale or {})} + return r + + def test_long_board_forms_are_staged_and_only_that_board(self): + """Two boards are built so the pre-fix 'copy all built binaries' else-branch cannot + stage the right one by accident -- that is what made the first version of this test + pass against master while the feature was broken. -balpha is the glued short form + argparse resolves to --board alpha; unparsed it fell through to the all-boards branch + and silently staged everything built with no roster check.""" + for argv in (['--board', 'alpha'], ['--board=alpha'], ['-balpha']): + with self.subTest(argv=argv): + r = self._run(argv, built=['alpha', 'beta'], roster=['alpha', 'beta']) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertTrue(any('cmake-build-alpha ' in l for l in r.rsyncs), + f'{argv} never staged: {r.rsyncs}') + self.assertFalse(any('cmake-build-beta ' in l for l in r.rsyncs), + f'{argv} staged an unrequested board: {r.rsyncs}') + + def test_board_test_flag_is_not_mistaken_for_a_board(self): + """-bt is hil_test.py's --board-test and is exactly what <config>.failed contains, so + a glued -b?* pattern turns the documented retry into 'not in the roster: t'.""" + for argv in (['-b', 'alpha', '-bt', 'alpha:device/cdc_msc'], + ['-b', 'alpha', '-btalpha:device/cdc_msc']): + with self.subTest(argv=argv): + r = self._run(argv, built=['alpha']) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertNotIn('not in', r.stderr) + self.assertTrue(any('alpha:device/cdc_msc' in l for l in r.run_lines), + f'-bt never reached the rig: {r.run_lines}') + + def test_a_board_outside_the_roster_is_refused_before_staging(self): + r = self._run(['-b', 'alpha', '-b', 'ghost'], built=['alpha', 'ghost'], roster=['alpha']) + self.assertNotEqual(r.returncode, 0) + self.assertIn('ghost', r.stdout + r.stderr) + self.assertEqual(r.rsyncs, [], 'staged despite an unknown board') + self.assertEqual(r.run_lines, [], 'reached the run despite an unknown board') + + def test_a_variant_with_no_build_dir_warns_instead_of_passing_silently(self): + r = self._run(['-b', 'alpha'], built=['alpha'], variants={'alpha': ['alpha-DMA']}) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn('alpha-DMA', r.stderr) + self.assertIn('skipped, not tested', r.stderr) + + def test_no_build_dirs_at_all_aborts_the_all_boards_form(self): + """`hil_ci.sh` with no -b stages everything built. With nothing built it used to wipe + the rig, stage nothing, and return a green all-skip table.""" + r = self._run([], built=[], roster=['alpha']) + self.assertNotEqual(r.returncode, 0) + self.assertIn('nothing to test', r.stdout + r.stderr) + self.assertEqual(r.run_lines, [], 'reached the run with nothing staged') + self.assertNotIn('Setting up remote', r.stdout + r.stderr, + 'the guard fired only after the remote tree was already wiped') + + def test_hil_env_reaches_the_rig_as_environment_not_argv(self): + """An authorized force is HIL_NO_BOARD_LOCK=1. Passed through ssh's argv it arrives as a + positional argument and argparse exits 2, so it has to travel in the script body.""" + r = self._run(['-b', 'alpha'], built=['alpha'], env_extra={'HIL_NO_BOARD_LOCK': '1'}) + self.assertEqual(r.returncode, 0, r.stderr) + # one %q-quoted word of `export NAME=value; ` fragments, evaluated by the remote — + # NOT a bare NAME=value element, which hil_test.py's argparse takes as a positional. + # %q backslash-escapes the spaces, so match the pieces rather than the plain phrase. + run = '\n'.join(r.run_lines) + self.assertIn('HIL_NO_BOARD_LOCK=1', run) + self.assertIn('export', run) + self.assertFalse(any(' HIL_NO_BOARD_LOCK=1 ' in l for l in r.run_lines), + 'env reached argv unquoted, where hil_test.py sees a positional') + + def test_a_value_with_spaces_survives_forwarding(self): + r = self._run(['-b', 'alpha'], built=['alpha'], + env_extra={'HIL_SCRATCH': '/tmp/my scratch'}) + self.assertEqual(r.returncode, 0, r.stderr) + run = '\n'.join(r.run_lines) + self.assertIn('HIL_SCRATCH', run) + self.assertIn('scratch', run) + + def test_hil_report_dir_is_never_forwarded(self): + """Where the report lands on the rig is this script's contract (REMOTE_DIR, where all + three copy-backs look); forwarding a local HIL_REPORT_DIR relocates it there and every + copy-back comes home empty -- two of the three silently.""" + r = self._run(['-b', 'alpha'], built=['alpha'], + env_extra={'HIL_REPORT_DIR': '/tmp/elsewhere'}) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertFalse(any('HIL_REPORT_DIR' in l for l in r.run_lines), + f'HIL_REPORT_DIR reached the rig: {r.run_lines}') + + def test_a_stale_local_failed_spec_does_not_survive_a_green_run(self): + """A green run writes no .failed on the rig, so the copy-back scp no-ops; the local + spec from a previous FAILED run must not survive it looking current -- a later + "retry from the spec" would re-flash boards that already passed.""" + r = self._run(['-b', 'alpha'], built=['alpha'], + stale={'cfg.json.failed': '--accumulate -b alpha'}) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertFalse(r.stale_left['cfg.json.failed'], + "last run's re-run spec survived a green run") + + +class PoolGuardKeepsWhatFinished(unittest.TestCase): + """The guard's 30-minute predecessor fired on 5 of the last 8 HIL jobs, so this is the + common failure, not an edge case: map_async discarded every board that had finished and + left the re-run spec unwritten, so CI re-tested all ~26 to find the one that wedged. + + Calls hil_test.drain_pool -- the loop main() actually runs. The predecessor of this test + built its own ThreadPool and its own drain loop and asserted on those, so deleting the + production drain outright left it green.""" + + class _It: + """Stands in for imap_unordered: yields, then blocks past any deadline.""" + + def __init__(self, ready): + self.ready, self.i = ready, 0 + + def next(self, timeout=None): + if self.i < len(self.ready): + self.i += 1 + return self.ready[self.i - 1] + raise MpTimeoutError + + def test_finished_rows_survive_a_guard_expiry(self): + boards = [{'name': 'fast1'}, {'name': 'fast2'}, {'name': 'wedged'}] + rows = [('fast1', 0, [], [], 1.0, False), ('fast2', 0, [], [], 1.0, False)] + with self.assertRaises(hil_test.PoolDrainTimeout) as cm: + hil_test.drain_pool(self._It(rows), boards, time.monotonic() + 5) + self.assertEqual([r[0] for r in cm.exception.finished], ['fast1', 'fast2']) + + def test_an_expired_deadline_stops_before_asking_for_more(self): + """Left <= 0 must not be handed to it.next() as a zero/negative timeout.""" + boards = [{'name': 'a'}, {'name': 'b'}] + it = self._It([('a', 0, [], [], 1.0, False)]) + with self.assertRaises(hil_test.PoolDrainTimeout) as cm: + hil_test.drain_pool(it, boards, time.monotonic() - 1) # already past + self.assertEqual(cm.exception.finished, []) + self.assertEqual(it.i, 0, 'asked the pool for a result after the deadline') + + def test_rows_collected_before_the_deadline_expires_are_kept_too(self): + """The OTHER raise site: boards finish, then the clock runs out between results. + Both sites must carry the rows -- a bare raise here loses a worker-width of rig + time just as map_async did, and the it.next() path alone does not prove it.""" + class Slow(self._It): + def next(self, timeout=None): + time.sleep(0.2) # each result eats into the deadline + return super().next(timeout) + + boards = [{'name': n} for n in ('a', 'b', 'c', 'd')] + rows = [(n, 0, [], [], 1.0, False) for n in ('a', 'b', 'c', 'd')] + with self.assertRaises(hil_test.PoolDrainTimeout) as cm: + hil_test.drain_pool(Slow(rows), boards, time.monotonic() + 0.3) + self.assertTrue(cm.exception.finished, 'rows collected before the expiry were lost') + + def test_every_board_finishing_returns_them_all(self): + boards = [{'name': 'a'}, {'name': 'b'}] + rows = [('a', 0, [], [], 1.0, False), ('b', 1, [], [], 2.0, False)] + got = hil_test.drain_pool(self._It(rows), boards, time.monotonic() + 5) + self.assertEqual(got, rows) + + +class WedgedBoardCosts(unittest.TestCase): + """Two decisions the containment latch makes, tested as decisions rather than through + test_board's loop -- the loop-level predecessor of these tests reimplemented that loop + and asserted on its own copy, which is how both defects survived it.""" + + def setUp(self): + self.addCleanup(setattr, hil_test, 'board_wedged', hil_test.board_wedged) + + def test_a_board_that_wedged_still_counts_as_an_error(self): + """It rendered a red cell but returned err_count 0, so main()'s sys.exit(err_count) + reported success and _write_failed_spec (`if err > 0`) left the board out of the + re-run entirely: a rig holding a D-state process published as a clean pass.""" + hil_test.board_wedged = 'usbtest HUNG' + # no real flasher: skip_flash isolates the accounting from hil_flash + self.addCleanup(setattr, hil_test, 'skip_flash', hil_test.skip_flash) + hil_test.skip_flash = True + # a firmware path must resolve or test_example returns 'skip (no binary)' before + # ever reaching the retry loop this is about + self.addCleanup(setattr, hil_flash, 'find_firmware', hil_flash.find_firmware) + hil_flash.find_firmware = lambda *a, **k: Path('fw.elf') + + def boom(*a, **k): + raise hil_test.TestFail('usbtest did not run') # unparsed: retryable + + self.addCleanup(setattr, hil_test, 'test_device_usbtest', hil_test.test_device_usbtest) + hil_test.test_device_usbtest = boom + board = {'name': 'b', 'uid': 'U', 'flasher': {'name': 'openocd'}, 'tests': []} + err, _status, _metric = hil_test.test_example(board, 'b', 'device/usbtest') + self.assertEqual(err, 1, 'a wedged board contributed nothing to the exit status') + + def test_the_teardown_park_does_not_flash_a_wedged_board(self): + """The park is a flash like any other: on a D-state-held node it blocks, survives + SIGKILL and leaves a stray -- added by the path that just declared the board wedged + and skipped every test for exactly that reason.""" + hil_test.board_wedged = '' + self.assertTrue(hil_test._should_park(False), 'a healthy board must still park') + hil_test.board_wedged = 'usbtest HUNG' + self.assertFalse(hil_test._should_park(False), + 'the teardown park would flash through the poisoned node') + self.assertFalse(hil_test._should_park(True), '--skip-flash must still suppress it') + + +class WedgeVerdictReachesTheLatch(unittest.TestCase): + """usbtest computes `unrecovered_hang` but never reported it, so hil_test inferred the + latch from `not recovery and 'HUNG' in out` and missed three cases: recovery ran and + FAILED (convoy-safe boards -- max32666fthr HUNG in the 08-14 run), the `ambiguous` + abort (which sets the flag but leaves no case at status HUNG), and an unparsable JSON, + which is the outer-timeout kill and the case where a wedge is most likely.""" + + def setUp(self): + self.addCleanup(setattr, hil_test, 'board_wedged', hil_test.board_wedged) + hil_test.board_wedged = '' + no_settle(self) + + def _run(self, stdout, rc=0): + from helper import hil_lock, hil_util + class R: + returncode = rc + stderr = b'' + R.stdout = stdout.encode() + self.addCleanup(setattr, hil_util, 'run_cmd', hil_util.run_cmd) + hil_util.run_cmd = lambda *a, **k: R() + # usbtest_enumerated is nested in test_device_usbtest, so stub what it calls + self.addCleanup(setattr, hil_util, 'usb_scan', hil_util.usb_scan) + hil_util.usb_scan = lambda **k: ([{'busport': '1-1', 'dir': '/x', 'vid': 'cafe', + 'pid': '4010', 'serial': 'U'}], False) + self.addCleanup(setattr, hil_lock, 'usbtest_permit', hil_lock.usbtest_permit) + from contextlib import contextmanager + hil_lock.usbtest_permit = contextmanager(lambda uid: iter([None])) + board = {'name': 'b', 'uid': 'U', 'flasher': {'name': 'openocd', 'vid_pid': '0x1 0x2'}} + try: + hil_test.test_device_usbtest(board) + except Exception: + pass + return hil_test.board_wedged + + def test_a_reported_wedge_latches_even_when_recovery_ran(self): + """`recovery` True means the flags were PASSED, not that they worked.""" + js = '{"serial":"U","speed":"480","tier":1,"passed":1,"failed":1,"notrun":0,' '"wedged":true,"cases":[{"num":1,"status":"FAIL"}]}' + self.assertTrue(self._run(js), 'a reported wedge did not latch') + + def test_no_wedge_reported_does_not_latch(self): + js = '{"serial":"U","speed":"480","tier":1,"passed":2,"failed":0,"notrun":0,' '"wedged":false,"cases":[]}' + self.assertFalse(self._run(js)) + + def test_an_unparseable_battery_that_mentions_HUNG_still_latches(self): + """rc 124 mid-print: no JSON to read, and this is the likeliest real wedge.""" + self.assertTrue(self._run('TEST 10 HUNG: device wedged mid-transfer', rc=124)) + + +class WedgedBoardCannotReportAPass(unittest.TestCase): + """The latch alone is not enough: it is set BEFORE the pass return, so an all-green + battery that still wedged returned `PASS 30/30`. That board then contributes 0 to + err_count, is omitted from the .failed re-run spec (which keys on err > 0), and the job + exits 0 with a D-state holder on the rig -- the exact silence this branch exists to end. + usbtest's `ambiguous` abort fires AFTER the last case, so nothing + back-fills a BUDGET entry to make failed/notrun non-zero.""" + + def setUp(self): + self.addCleanup(setattr, hil_test, 'board_wedged', hil_test.board_wedged) + hil_test.board_wedged = '' + no_settle(self) + + def _cell(self, js): + """Returns ('pass', cell) or ('fail', message).""" + from helper import hil_lock, hil_util + class R: + returncode = 0 + stderr = b'' + R.stdout = js.encode() + self.addCleanup(setattr, hil_util, 'run_cmd', hil_util.run_cmd) + hil_util.run_cmd = lambda *a, **k: R() + self.addCleanup(setattr, hil_util, 'usb_scan', hil_util.usb_scan) + hil_util.usb_scan = lambda **k: ([{'busport': '1-1', 'dir': '/x', 'vid': 'cafe', + 'pid': '4010', 'serial': 'U'}], False) + self.addCleanup(setattr, hil_lock, 'usbtest_permit', hil_lock.usbtest_permit) + from contextlib import contextmanager + hil_lock.usbtest_permit = contextmanager(lambda uid: iter([None])) + board = {'name': 'b', 'uid': 'U', 'flasher': {'name': 'openocd', 'vid_pid': '0x1 0x2'}} + try: + return ('pass', hil_test.test_device_usbtest(board)) + except hil_test.TestFail as e: + return ('fail', str(e)) + + def test_an_all_pass_battery_that_wedged_is_not_a_pass(self): + kind, detail = self._cell('{"serial":"U","speed":"480","tier":1,"passed":30,' + '"failed":0,"notrun":0,"wedged":true,"cases":[]}') + self.assertEqual(kind, 'fail', f'a wedged board reported a green cell: {detail}') + self.assertIn('wedged', detail) + + def test_an_all_pass_battery_that_did_not_wedge_is_still_a_pass(self): + """The guard must key on the latch, not merely on having parsed a battery.""" + kind, cell = self._cell('{"serial":"U","speed":"480","tier":1,"passed":30,' + '"failed":0,"notrun":0,"wedged":false,"cases":[]}') + self.assertEqual(kind, 'pass', f'a healthy board was failed: {cell}') + self.assertIn('30/30', cell) + + +def _gil_stall_available() -> bool: + """Whether the hid stub can simulate a GIL-HOLDING stall on this host. + + It needs a libc with sleep(3) loaded through ctypes.PyDLL. Everywhere the HIL harness + actually runs that is present; where it is not, the two tests that depend on it skip + rather than fail, because their subject is the bound, not ctypes. + """ + import ctypes + import ctypes.util + try: + ctypes.PyDLL(ctypes.util.find_library('c') or 'libc.so.6') + return True + except OSError: + return False + + +class HidEchoRunsInAChild(unittest.TestCase): + """hidapi's blocking calls hold the GIL -- cython-hidapi wraps hid_enumerate in + `with nogil` but calls hid_open and hid_close bare -- so a daemon thread cannot bound + them: the waiter parks off-GIL but must reacquire the GIL to return, which the stuck + thread never yields. Only a child process can be killed regardless, which is what + run_cmd's killpg does.""" + + def _run(self, mode, uid='CAFE01', budget='0', timeout=20, pid=None): + saved = {k: os.environ.get(k) for k in ('FAKE_HID_MODE', 'FAKE_HID_UID', + 'FAKE_HID_PID', 'PYTHONPATH', + 'PYTHONSAFEPATH')} + + def restore(): + for k, v in saved.items(): + os.environ.pop(k, None) if v is None else os.environ.__setitem__(k, v) + self.addCleanup(restore) + os.environ['FAKE_HID_MODE'] = mode + os.environ['FAKE_HID_UID'] = uid + stubs = os.path.join(TEST_DIR, 'stubs') + pp = saved['PYTHONPATH'] + os.environ['PYTHONPATH'] = stubs if not pp else f'{stubs}:{pp}' + # `python3 -c` puts the cwd at sys.path[0], AHEAD of PYTHONPATH, so any hid.py + # reachable from the suite's cwd would displace the stub and every mode-driven + # test below would pass or fail for the wrong reason. Safe-path mode drops it -- + # the same practice _MtpFakeRig documents. + os.environ['PYTHONSAFEPATH'] = '1' + from helper import hil_util + want = pid or f'{hil_test.HID_INOUT_PID:#06x}' + return hil_util.run_cmd( + [sys.executable, '-c', hil_test.HID_ECHO, uid, budget, want], + timeout=timeout, split_stderr=True, quiet=True) + + def _stderr(self, r): + from helper import hil_util + return hil_util.cmd_stdout_text(r.stderr) + + def test_a_healthy_device_passes(self): + r = self._run('ok') + self.assertEqual(r.returncode, 0, self._stderr(r)) + + def test_the_pid_matches_the_example(self): + """The walk filters on BOTH ids, and hidapi applies them before the locked + manufacturer/product reads. Six examples in this tree expose a HID interface under + VID cafe, so a stale PID here silently widens the walk back to all of them -- and + nothing else would fail. Pinned against the descriptor rather than restated.""" + import re + src = (Path(TEST_DIR).parents[2] + / 'examples/device/hid_generic_inout/src/usb_descriptors.c').read_text() + m = re.search(r'#define\s+USB_PID\s+(0x[0-9a-fA-F]+)', src) + self.assertIsNotNone(m, 'hid_generic_inout no longer defines USB_PID') + self.assertEqual(hil_test.HID_INOUT_PID, int(m.group(1), 16), + 'HID_INOUT_PID drifted from the example descriptor') + + def test_a_peer_running_another_example_is_filtered_out(self): + """The point of the PID filter: a wedged sibling on a different example never + reaches the locked reads at all.""" + r = self._run('ok', pid='0x400f') # hid_composite, not ours + self.assertNotEqual(r.returncode, 0) + self.assertIn('HID device not found', self._stderr(r)) + + @unittest.skipUnless(_gil_stall_available(), 'no libc for a GIL-holding stall') + def test_a_gil_holding_stall_is_still_killed(self): + """THE case an in-process bound cannot cover. hid_open is not `with nogil`, so a + thread-based guard is inert there; the child is killed anyway.""" + t0 = time.monotonic() + r = self._run('wedged_open_gil', timeout=2) + self.assertEqual(r.returncode, 124, + 'a GIL-holding hidapi stall must still be killed on the bound') + self.assertLess(time.monotonic() - t0, 20, 'run_cmd did not bound the child') + + def test_a_wedged_enumerate_is_killed_on_the_bound(self): + r = self._run('wedged_enumerate', timeout=2) + self.assertEqual(r.returncode, 124) + + @unittest.skipUnless(_gil_stall_available(), 'no libc for a GIL-holding stall') + def test_a_wedged_close_is_killed_on_the_bound(self): + """close() runs in the child's finally on EVERY failure path and is also + GIL-holding; hidraw_release takes the same rwsem hidraw_open needs.""" + r = self._run('wedged_close', timeout=3) + self.assertEqual(r.returncode, 124) + + def test_an_absent_device_reports_why(self): + r = self._run('absent') + self.assertNotEqual(r.returncode, 0) + self.assertIn('HID device not found', self._stderr(r)) + + def test_a_bad_echo_reports_both_payloads(self): + r = self._run('wrong_data') + self.assertNotEqual(r.returncode, 0) + msg = self._stderr(r) + self.assertIn('wrong data', msg) + self.assertIn('sent', msg) + self.assertIn('received', msg) + + def test_a_short_echo_is_not_read_as_a_pass(self): + r = self._run('short_read') + self.assertNotEqual(r.returncode, 0) + self.assertIn('short read', self._stderr(r)) + + +class StrayNoteSurvivesTheTupleWidth(unittest.TestCase): + """_stray_note reads r[5] -- and three producers build this tuple at three widths, so + `len(r) > 5 and r[5]` reads a WRONG SLOT rather than raising if a field is ever + inserted. The live handoff pr3840-mret-board-result.md proposes exactly that, and the + report would then say "no strays" while probes and usbfs nodes stay held into the next + job. The index changed once already in this branch (r[6] -> r[5]).""" + + def test_it_names_the_board_and_the_count(self): + wide = ('dirty', 1, [], [], 9.0, 2) + clean = ('fine', 0, [], [], 8.0, 0) + note = hil_test._stray_note([wide, clean]) + self.assertIn('dirty (2)', note) + self.assertIn('2 process(es)', note) + self.assertNotIn('fine', note, 'a clean board must not appear in the note') + + def test_a_narrow_row_from_the_timeout_path_is_not_misread(self): + """The abort paths synthesise 5-field rows for boards that never reported.""" + self.assertEqual(hil_test._stray_note([('stuck', 1, [], None, 0)]), '') + self.assertEqual(hil_test._stray_note([('fine', 0, [], [], 8.0, 0)]), '') + + def test_the_slot_it_reads_is_the_slot_test_board_writes(self): + """Pins the index against the producer, so inserting a field fails HERE rather + than silently reporting a duration as a stray count.""" + import ast + src = (Path(TEST_DIR).parents[0] / 'hil_test.py').read_text() + fn = next(n for n in ast.walk(ast.parse(src)) + if isinstance(n, ast.FunctionDef) and n.name == 'test_board') + widths = sorted({len(n.value.elts) for n in ast.walk(fn) + if isinstance(n, ast.Return) and isinstance(n.value, ast.Tuple)}) + # the board-LOCKED early return is 5 wide and carries no stray count; the normal + # one is 6, with strays last + self.assertEqual(widths, [5, 6], + 'the result tuple changed width; _stray_note reads index 5') + + +class MixedWidthRowsSurviveTheReportWriters(unittest.TestCase): + """_abort_report hands `[(n, 1, [], None, 0) for n in stuck] + [r for r in mret ...]` + to both writers -- 5-field synthetic rows mixed with 6-field worker rows. Every other + test uses uniform widths, so replacing either `*_` unpack with a fixed-width one keeps + the suite green and raises only INSIDE the containment path, where a raise costs every + board's results.""" + + def _mixed(self): + return [('stuck', 1, [], None, 0), # synthetic, 5 wide + ('ran', 1, ['device/dfu'], + [('ran', {'device/dfu': '❌ boom'}, '8s')], 8.0, 2)] # worker, 6 wide + + def test_the_rerun_spec_accepts_both_widths(self): + with TemporaryDirectory() as td: + rd = Path(td) + hil_test._write_failed_spec(rd / 'c.json.failed', rd, self._mixed()) + spec = (rd / 'c.json.failed').read_text() + self.assertIn('stuck', spec) + self.assertIn('ran', spec) + + def test_the_cell_names_the_cause_of_the_abort(self): + """A board the pool guard never reached did not "pool-timeout". Marking it so + sends whoever reads the table after a guard that never fired.""" + from helper import hil_report + real = hil_report.accumulate_report + + def render(reason, secs): + hil_report.accumulate_report = lambda *a, **k: (_ for _ in ()).throw( + OSError('report dir unwritable')) + try: + with TemporaryDirectory() as td: + rd = Path(td) + hil_test._abort_report(reason, [], [{'name': 'boardA'}], + rd / 'c.failed', rd, True, '', + timeout_secs=secs) + return (rd / hil_report.REPORT_MD).read_text() + finally: + hil_report.accumulate_report = real + + guard = render('abandoned: worker pool timed out after 3600s', 3600) + self.assertIn(hil_report.POOL_TIMEOUT_CELL, guard) + raised = render('aborted: a worker raised ValueError: x', None) + self.assertIn(hil_report.RUN_ABORTED_CELL, raised) + self.assertNotIn(hil_report.POOL_TIMEOUT_CELL, raised, + 'a run that aborted on a raise is not a pool timeout') + # and the fallback must still fire on BOTH paths -- that is what it is for + for md in (guard, raised): + self.assertIn('boardA', md) + + def test_only_the_rerun_spec_sees_the_synthetic_rows(self): + """accumulate_report gets `mret` alone -- worker rows, always 4th field a real + list. Widening _abort_report to hand it the synthetic list too would crash the + containment path: those rows carry rows=None and render_matrix iterates it.""" + import ast + src = (Path(TEST_DIR).parents[0] / 'hil_test.py').read_text() + fn = next(n for n in ast.walk(ast.parse(src)) + if isinstance(n, ast.FunctionDef) and n.name == '_abort_report') + calls = {ast.unparse(n.func): ast.unparse(n) + for n in ast.walk(fn) if isinstance(n, ast.Call) + and ast.unparse(n.func).endswith(('_write_failed_spec', + 'accumulate_report'))} + self.assertEqual( + ast.unparse(ast.parse(calls['hil_report.accumulate_report']).body[0] + ).split('(', 1)[1].split(',')[0], 'mret', + 'accumulate_report must receive worker rows only -- the synthetic rows carry ' + 'rows=None and render_matrix iterates that field') + self.assertIn('stuck', calls['_write_failed_spec'], + 'the re-run spec must still name the boards that never reported') + + +class UsbtestAbsentDeviceVerdict(unittest.TestCase): + """The arm that fails BEFORE usbtest_permit: an absent device must not queue on the + battery mutex for minutes just to have usbtest.py report "no device", and the cell + needs the 0/30 denominator or the row reads as a bare failure.""" + + def setUp(self): + self.addCleanup(setattr, hil_test, 'board_wedged', hil_test.board_wedged) + hil_test.board_wedged = '' + no_settle(self) + from helper import hil_lock, hil_util + self.addCleanup(setattr, hil_util, 'usb_scan', hil_util.usb_scan) + hil_util.usb_scan = lambda **k: [] # a readable bus, no such device + self.addCleanup(setattr, hil_test, '_enum_timeout', hil_test._enum_timeout) + hil_test._enum_timeout = 0 + self.addCleanup(setattr, hil_lock, 'usbtest_permit', hil_lock.usbtest_permit) + from contextlib import contextmanager + + def boom(uid): + raise AssertionError('took the battery permit for an absent device') + yield + hil_lock.usbtest_permit = contextmanager(boom) + + def test_a_readable_bus_without_the_device_says_absent_with_a_denominator(self): + with self.assertRaises(hil_test.TestFail) as cm: + hil_test.test_device_usbtest({'name': 'b', 'uid': 'NOPE', + 'flasher': {'name': 'stlink', 'uid': 'X'}}) + self.assertIn('no cafe:4010 device', str(cm.exception)) + self.assertIn('0/30', cm.exception.metric) + + def test_a_scan_that_gave_up_says_could_not_tell_instead(self): + """The conflation this whole path exists to avoid: an unreadable DUT is not an + absent one, and the bare string sends a maintainer after a firmware regression on + hardware that is merely wedged.""" + from helper import hil_util + self.addCleanup(setattr, hil_util, '_ever_stranded', hil_util._ever_stranded) + hil_util._ever_stranded = True + with self.assertRaises(hil_test.TestFail) as cm: + hil_test.test_device_usbtest({'name': 'b', 'uid': 'NOPE', + 'flasher': {'name': 'stlink', 'uid': 'X'}}) + self.assertIn('could not tell', str(cm.exception)) + + +class UsbtestStartupDoesNotClaimAbsenceBlind(unittest.TestCase): + """usbtest.py's own startup lookup, the sibling of the arm above. hil_test relays its + stderr verbatim into the report cell, so a positive 'no cafe:4010 device' from a scan + that gave up is the same conflation one process further out. Structural because the + exit sits mid-main(), behind argparse and the testusb probe.""" + + def test_the_sysfs_backed_absence_claims_carry_the_note(self): + """Both claims that a bounded read can turn into a false absence. The printer one + was missed: read_sysfs folds a timed-out `serial` into None, so a wedged-but- + enumerated printer read as 'Printer device not found' -- an enumeration verdict for + hardware that is merely unreadable. The MIDI lookup is deliberately NOT here: it + globs /dev/snd/by-id and readlinks it, so no bounded read can blind it.""" + import ast + tree = ast.parse(Path(hil_test.__file__).read_text()) + claims = [ast.unparse(n) for n in ast.walk(tree) + if isinstance(n, (ast.Assert, ast.Raise)) + and ('Printer device not found' in ast.unparse(n) + or 'no cafe:4010 device' in ast.unparse(n))] + self.assertEqual(len(claims), 2, 'a sysfs-backed absence claim moved or was added') + for c in claims: + self.assertIn('strand_note', c, f'absence claimed without the note: {c[:70]}') + + def test_the_absence_exit_carries_the_stranded_caveat(self): + import ast + import usbtest + tree = ast.parse(Path(usbtest.__file__).read_text()) + exits = [n for n in ast.walk(tree) + if isinstance(n, ast.Call) and ast.unparse(n.func) == 'sys.exit' + and 'no {VID}:{PID} device' in ast.unparse(n)] + self.assertEqual(len(exits), 1, 'the absence exit moved; retarget this test') + self.assertIn('strand_note', ast.unparse(exits[0]), + 'usbtest claims absence without consulting sysfs_stranded()') + + +class UsbtestGlobalCleanupStaysProcessWide(unittest.TestCase): + """The strand flag has TWO consumers at different scopes. The per-case verdict is + per-DUT -- a peer that stranded must not make OUR board report wedged. But the finally + block's cleanup is GLOBAL: remove_id plus an unbind of every interface under the + usbtest driver, including that peer's. Those writes take the uninterruptible + device_lock, so the global path has to stay gated on the process-wide question.""" + + def test_the_global_unbind_consults_the_process_wide_flag(self): + import ast + import usbtest + tree = ast.parse(Path(usbtest.__file__).read_text()) + fins = [n for n in ast.walk(tree) if isinstance(n, ast.Try) and n.finalbody + and 'remove_id' in ast.unparse(ast.Module(body=n.finalbody, type_ignores=[]))] + self.assertEqual(len(fins), 1, 'the cleanup finally moved; retarget this test') + body = ast.unparse(ast.Module(body=fins[0].finalbody, type_ignores=[])) + self.assertIn('sysfs_stranded', body, + 'global remove_id/unbind runs without the process-wide strand gate') + + +if __name__ == '__main__': + unittest.main() diff --git a/test/hil/test/test_hil_health.py b/test/hil/test/test_hil_health.py new file mode 100644 index 000000000..ceecc8d37 --- /dev/null +++ b/test/hil/test/test_hil_health.py @@ -0,0 +1,596 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests for hil_health.py — pure logic against a synthetic /proc, no hardware. A real +# wedge cannot be manufactured on demand, so the detectors are exercised against fabricated +# inputs. hil_health is stdlib-only on purpose, so all of this runs on a bare CI runner +# with nothing skipped. Run directly: +# python3 test/hil/test/test_hil_health.py +import os +import signal +import sys +import threading +import time +import subprocess +import unittest +from multiprocessing import Pool +from pathlib import Path +from tempfile import TemporaryDirectory + +# the module under test lives in the parent dir (test/hil), not here +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from helper import hil_health + +REAL_PROC = hil_health.PROC + + +def make_proc(root: Path, procs: dict, with_pid1: bool = True) -> None: + """Build a synthetic /proc. `procs` maps pid -> (comm, state, cmdline); a None comm or + cmdline omits that file. `with_pid1=False` simulates a restricted /proc (hidepid=2), + where an empty scan must not be read as an all-clear.""" + for pid, (comm, state, cmdline) in procs.items(): + d = root / str(pid) + d.mkdir() + if comm is not None: + (d / 'comm').write_text(comm + '\n') + if cmdline is not None: + (d / 'cmdline').write_bytes(cmdline) + # field 2 is comm in parens; the state letter follows it. Deliberately use a comm + # containing ')' so a naive split() would pick the wrong field. + (d / 'stat').write_text(f'{pid} (we)ird) {state} 1 1 0 0 -1 0 0\n') + if with_pid1 and 1 not in procs: + d = root / '1' + d.mkdir() + (d / 'comm').write_text('systemd\n') + (d / 'cmdline').write_bytes(b'/sbin/init\0') + (d / 'stat').write_text('1 (systemd) S 0 1 0 0 -1 0 0\n') + (root / 'not-a-pid').mkdir() + + +class PatchCase(unittest.TestCase): + """For classes that patch PROCESS-GLOBAL state (os.kill, time.sleep, subprocess.Popen). + + addCleanup, never tearDown: tearDown does NOT run when setUp raises, so a no-op + os.kill or time.sleep would survive into every later test in this blocking pre-commit + suite -- turning one setUp failure into a cascade of nonsense results.""" + + def patch(self, obj, name, value): + self.addCleanup(setattr, obj, name, getattr(obj, name)) + setattr(obj, name, value) + + def restore(self, obj, name): + """Same guarantee for state a TEST BODY assigns directly: register the restore + from setUp so it holds even when the assert between fails.""" + self.addCleanup(setattr, obj, name, getattr(obj, name)) + + +class ProcCase(unittest.TestCase): + """Every subclass repoints hil_health.PROC at a temp tree; restore it so a later test + cannot silently keep scanning a deleted directory.""" + + def tearDown(self): + hil_health.PROC = REAL_PROC + + +class ShutdownPool(unittest.TestCase): + def test_returns_true_when_the_pool_terminates(self): + pool = Pool(processes=1) + try: + self.assertTrue(hil_health.shutdown_pool(pool, grace=30)) + finally: + pool.terminate() + + def test_returns_false_instead_of_blocking_forever(self): + """The real failure is a worker in uninterruptible sleep, which cannot be created + from userspace. What matters is that shutdown_pool gives up on the deadline rather + than hanging, because the caller must then abandon the pool to free the job slot.""" + # Cancellable, not time.sleep(3600): shutdown_pool returns while its daemon thread + # is still inside terminate(), and an uninterruptible sleep there outlives the test. + # The next test alphabetically forks a real Pool, so the leaked thread made it + # fork-from-multithreaded ('DeprecationWarning: ... may lead to deadlocks in the + # child') and its result order-dependent. addCleanup releases it either way. + release = threading.Event() + self.addCleanup(release.set) + + class NeverDies: + def terminate(self): + release.wait(3600) + + start = time.monotonic() + self.assertFalse(hil_health.shutdown_pool(NeverDies(), grace=0.5)) + self.assertLess(time.monotonic() - start, 10) + + def test_a_raising_terminate_counts_as_failure(self): + """The thread dies on the exception, so is_alive() goes False -- which would report + success for a pool that is just as alive as if terminate() had hung.""" + class Explodes: + def terminate(self): + raise RuntimeError('boom') + + self.assertFalse(hil_health.shutdown_pool(Explodes(), grace=5)) + + +class ChildProcs(ProcCase): + """A pool worker's own group is OUR group (multiprocessing never setpgid's), so its + children can only be found by walking ppid -> pgrp in /proc.""" + + def test_grandchildren_are_swept_too(self): + """usbtest.py (child, own session) spawns its recovery reflash via run_cmd (own + session again): the flasher is a GRANDCHILD no direct-child walk covers, and a + pool-guard kill mid-recovery would orphan it on the probe.""" + got = self.scan([100], { + 100: ('worker', 1, 4242), + 200: ('usbtest.py', 100, 200), # child, own session + 300: ('openocd', 200, 300), # grandchild flasher, own session + 999: ('unrelated', 1, 999), + }) + self.assertEqual(sorted(got.get(100, [])), [(200, 200), (300, 300)]) + + def scan(self, pids, procs): + """`procs` maps pid -> (comm, ppid, pgrp); a None comm omits the stat file.""" + with TemporaryDirectory() as td: + root = Path(td) + for p, (comm, ppid, pgrp) in procs.items(): + d = root / str(p) + d.mkdir() + if comm is not None: + (d / 'stat').write_text(f'{p} ({comm}) S {ppid} {pgrp} 0 0 -1 0 0\n') + (root / 'not-a-pid').mkdir() + hil_health.PROC = root + return hil_health.child_procs(pids) + + def test_finds_direct_children_only(self): + got = self.scan([100], { + 100: ('python3', 1, 4242), # the worker itself + 201: ('openocd', 100, 201), # its detached flasher + 202: ('usbtest.py', 100, 202), # a second detached session + 303: ('unrelated', 7, 303), # someone else's child + }) + self.assertEqual({100: [(201, 201), (202, 202)]}, + {k: sorted(v) for k, v in got.items()}) + + def test_covers_every_parent_in_one_walk(self): + """One pass for all workers, not one pass each: this runs on the free-the-runner + path, and per-parent walks would also see different snapshots.""" + got = self.scan([100, 101], { + 201: ('openocd', 100, 201), + 202: ('JLinkExe', 101, 202), + }) + self.assertEqual(got, {100: [(201, 201)], 101: [(202, 202)]}) + + def test_parses_a_comm_containing_spaces_and_parens(self): + """A naive split() on the whole line would read the wrong fields.""" + got = self.scan([100], {500: ('we ) ird', 100, 500)}) + self.assertEqual(got, {100: [(500, 500)]}) + + def test_reports_a_child_that_shares_our_group(self): + """subprocess.run children (arecord, iperf) get no new session, so they land in + our group. They must still be REPORTED -- kill_pool_children signals them by pid, + since killpg on that group would take down the run itself.""" + got = self.scan([100], {201: ('arecord', 100, 4242)}) + self.assertEqual(got, {100: [(201, 4242)]}) + + def test_tolerates_unreadable_and_truncated_entries(self): + got = self.scan([100], { + 201: (None, 0, 0), # stat missing (exited mid-scan) + 202: ('openocd', 100, 202), # still found + }) + self.assertEqual(got, {100: [(202, 202)]}) + + def test_returns_empty_when_proc_is_unreadable(self): + hil_health.PROC = Path('/nonexistent-proc-for-test') + self.assertEqual(hil_health.child_procs([100]), {}) + + +class FakeProc: + """Stands in for a multiprocessing worker: kill_pool_children goes through + is_alive() and Process.kill(), whose internal returncode guard is what protects + against signalling a recycled pid.""" + + def __init__(self, pid, alive=True, wedged=False): + self.pid = pid + self._alive = alive + self._wedged = wedged # D state: ignores SIGKILL, so is_alive() stays True + self.killed = False + + def is_alive(self): + return self._alive + + def kill(self): + self.killed = True + # A signalled worker DIES unless it is wedged. Modelling every worker as an + # unkillable survivor sent all of them down the confirm/sudo ladder, which is + # what let literal pids reach the real os.kill. + if not self._wedged: + self._alive = False + + +class KillWorkerChildren(PatchCase): + # os.getpgid/killpg are stubbed for the whole class: FakeProc pids are literals like + # 101, which are live pids on a real machine, so an unstubbed killpg SIGKILLs a real + # process GROUP. That happened while writing this and killed the test run itself. + """What the workers spawned, killed while their parents are still alive. + + Verified premise: Pool.terminate() reaps a worker that is merely waiting in + communicate() on a wedged flasher, reparenting that flasher to init -- so this must + run BEFORE shutdown_pool(), or the ppid link is gone and a successful terminate() + skips the cleanup entirely.""" + + OWN_PGID = 4242 + + def setUp(self): + # _kill_and_confirm's grace poll must not touch the real /proc: fake pid + # 900 can be a live process on the host, which stalls the poll for the full grace + # and prints a false survivor warning into the blocking pre-commit hook. + self.proc_tmp = TemporaryDirectory() + self.addCleanup(self.proc_tmp.cleanup) + self.patch(hil_health, 'PROC', Path(self.proc_tmp.name)) # empty: unreadable -> gone + self.patch(hil_health, 'CONFIRM_KILL_GRACE', 0.05) + # the sweep runs two passes with a real gap; the fakes never respawn, so + # stub the wait rather than pay it in every test + self.patch(hil_health.time, 'sleep', lambda _s: None) + self.groups, self.pids = [], [] + self.children = {} # worker pid -> [(pid, pgid), ...] + self.patch(hil_health, 'child_procs', lambda pids: self.children) + self.patch(os, 'killpg', lambda pgid, sig: self.groups.append((pgid, sig))) + self.patch(os, 'kill', lambda pid, sig: self.pids.append((pid, sig))) + self.patch(os, 'getpgid', lambda pid: self.OWN_PGID) + # overwritten directly by some test bodies below (eperm/boom fakes) + self.restore(hil_health, '_kill_and_confirm') + + def test_kills_a_detached_child_by_group(self): + """Flashers are spawned with start_new_session=True, so one killpg also reaps + whatever they spawned; a plain kill would leave them holding the probe with no + timeout enforcer left alive.""" + w = FakeProc(101) + self.children = {101: [(900, 900)]} + + class FakePool: + _pool = [w] + self.assertEqual(hil_health.kill_worker_children(FakePool()), 0) # none survived + # by GROUP, so whatever the flasher spawned dies with it + self.assertEqual(self.groups, [(900, signal.SIGKILL)]) + # and then confirmed by pid: killpg reports success when it reached ANY member, + # so the group kill alone is not evidence this one died + self.assertIn((900, 0), self.pids) + self.assertFalse(w.killed) # the WORKER is not this one's job + + def test_a_root_owned_group_is_still_confirmed_and_reported(self): + """killpg on an all-root session raises EPERM: the sudo wrapper died and only its + root members remain. That is the one case this handler exists for, so it must + still reach the confirm step -- otherwise the holder that strands the NEXT job is + the one holder the report never names.""" + w = FakeProc(101) + self.children = {101: [(900, 900)]} + + def eperm(pgid, sig): + raise PermissionError + os.killpg = eperm + + class FakePool: + _pool = [w] + hil_health.kill_worker_children(FakePool()) + # confirmed by pid: a liveness probe on the member killpg could not touch + self.assertIn((900, 0), self.pids) + + def test_kills_a_same_group_child_by_pid(self): + """arecord/iperf/gio go through plain subprocess.run and stay in OUR group, where + killpg would take down the run itself -- but they must still die, or a blocked + arecord keeps holding the wedged device.""" + w = FakeProc(101) + self.children = {101: [(900, self.OWN_PGID)]} + + class FakePool: + _pool = [w] + hil_health.kill_worker_children(FakePool()) + self.assertEqual(self.groups, []) # never our own group + # SIGKILL, then a (pid, 0) probe: signalling is not dying, so the kill is always + # confirmed -- see _kill_and_confirm. + self.assertIn((900, signal.SIGKILL), self.pids) + self.assertIn((900, 0), self.pids) + + + def test_signals_pids_only_when_our_group_is_unknown(self): + """If getpgid(0) fails we cannot tell our group from a detached one, so killpg is + never safe -- fall back to per-pid signals rather than guessing.""" + def boom(pid): + raise OSError('no pgid') + os.getpgid = boom + w = FakeProc(101) + self.children = {101: [(900, 900)]} + + class FakePool: + _pool = [w] + hil_health.kill_worker_children(FakePool()) + self.assertEqual(self.groups, []) + self.assertIn((900, signal.SIGKILL), self.pids) + + def test_covers_a_dead_workers_orphans(self): + """A worker reaped between the snapshot and now leaves its flasher running. The + children are keyed off the snapshot, not off is_alive(), so they still die.""" + w = FakeProc(101, alive=False) + self.children = {101: [(900, 900)]} + + class FakePool: + _pool = [w] + self.assertEqual(hil_health.kill_worker_children(FakePool()), 0) # none survived + self.assertEqual(self.groups, [(900, signal.SIGKILL)]) + + def test_a_survivor_is_returned_so_the_report_can_say_the_rig_is_dirty(self): + """A stray that ignores SIGKILL is in D state on a usbfs node or holds a probe, and + it persists into the NEXT job. The count used to be discarded by the caller (the + return was the signalled-child count, which nothing read), so the only trace was a + line in the log -- and the run still published a table that looks clean.""" + w = FakeProc(101) + # TWO strays, only ONE unkillable: signalled=2, survivors=1, so this cannot pass + # by accident on the old return value + self.children = {101: [(900, 900), (901, 901)]} + self.patch(hil_health, '_kill_and_confirm', lambda pids: [p for p in pids if p == 901]) + + class FakePool: + _pool = [w] + self.assertEqual(hil_health.kill_worker_children(FakePool()), 1) + + def test_no_signal_when_the_workers_spawned_nothing(self): + w = FakeProc(101) # no self.children entry + + class FakePool: + _pool = [w] + self.assertEqual(hil_health.kill_worker_children(FakePool()), 0) + self.assertEqual((self.groups, self.pids), ([], [])) + + def test_includes_the_managers_children(self): + mgr_proc = FakeProc(402) + self.children = {402: [(900, 900)]} + + class FakePool: + _pool = [] + + class FakeManager: + _process = mgr_proc + self.assertEqual(hil_health.kill_worker_children(FakePool(), FakeManager()), 0) + + +class ConfirmTailIsOneGrace(PatchCase): + """The grace is ONE window for the whole set, not one per pid. Paid serially it + scaled with stray count: 30 strays x 16 workers spent ~154s inside the path whose + only job is to free the runner's single job slot -- which is exactly the + 'multi-stray convoy tail is minutes' the CI ceilings budget +30 min for.""" + + def setUp(self): + self.proc_tmp = TemporaryDirectory() + self.addCleanup(self.proc_tmp.cleanup) + root = Path(self.proc_tmp.name) + # every fake pid is alive and NOT a zombie, so all of them outlast the grace + make_proc(root, {900 + i: ('flasher', 'D', b'openocd\x00') for i in range(20)}) + self.patch(hil_health, 'PROC', root) + self.patch(hil_health, 'CONFIRM_KILL_GRACE', 0.3) + self.patch(os, 'kill', lambda pid, sig: None) # never signal a real pid + + def test_twenty_survivors_cost_one_grace_not_twenty(self): + pids = [900 + i for i in range(20)] + t0 = time.monotonic() + still = hil_health._kill_and_confirm(pids) + elapsed = time.monotonic() - t0 + self.assertEqual(sorted(still), pids) # all reported, none lost + self.assertLess(elapsed, 0.3 * 4, + f'the grace is paid per pid ({elapsed:.2f}s for 20)') + + +class KillPoolChildren(PatchCase): + """The worker processes themselves. + + Verified premise: an orphaned pool worker keeps the CI runner's stdout pipe open, so a + reader never sees EOF even after the parent exits. + + Fakes throughout: FakeProc.kill() only sets a flag, so nothing here can signal a real + process. That matters historically -- an earlier revision drove this through + os.pidfd_open with literal pids (101, 102), which exist on a real machine, so the suite + was asking the kernel to signal unrelated system processes and was saved only by EPERM. + Keep the fake in charge of kill(); never let a test reach os.kill/os.killpg with a + live pid. FakeProc.kill() alone is NOT enough for that: it leaves is_alive() True, so + the pid reaches the confirm/sudo ladder, which signals for real. Stub that too.""" + + def setUp(self): + # Pids 101/102/201 are ordinary user processes on a container or a fresh runner -- + # and pre-commit.yml runs this suite on GitHub's. Unstubbed, the ladder ran + # os.kill(101, SIGKILL) and forked `sudo -n kill -9 101` on an account with + # passwordless sudo, and the assertions passed only because those pids happen to + # be unkillable kernel threads here. + self.signals = [] + self.patch(os, 'kill', lambda pid, sig: self.signals.append((pid, sig))) + self.patch(os, 'killpg', lambda pgid, sig: self.signals.append((pgid, sig))) + self.proc_tmp = TemporaryDirectory() + self.addCleanup(self.proc_tmp.cleanup) + self.patch(hil_health, 'PROC', Path(self.proc_tmp.name)) # empty: unreadable -> gone + self.patch(hil_health, 'CONFIRM_KILL_GRACE', 0.05) + + def test_a_healthy_worker_never_reaches_the_signalling_ladder(self): + """The premise every assertion below rests on. Process.kill() is the fake's job; + only a worker that SURVIVES it goes on to raw os.kill/sudo, and these pids are + literals that belong to somebody else.""" + a, b = FakeProc(101), FakeProc(102) + + class FakePool: + _pool = [a, b] + hil_health.kill_pool_children(FakePool()) + self.assertEqual(self.signals, [], 'a literal pid reached the raw-signal ladder') + + def test_signals_every_live_worker(self): + a, b = FakeProc(101), FakeProc(102) + + class FakePool: + _pool = [a, b] + # 0, not 2: the RETURN is confirmed survivors, and workers that die to SIGKILL are + # not survivors. The operator verdict ("power-cycle the host") hangs off this. + self.assertEqual(hil_health.kill_pool_children(FakePool()), 0) + self.assertTrue(a.killed and b.killed) + + def test_a_wedged_worker_is_reported_as_a_survivor(self): + """The number the power-cycle verdict is worded on.""" + make_proc(Path(self.proc_tmp.name), {301: ('python3', 'D', b'python3 hil_test.py\x00')}) + wedged = FakeProc(301, wedged=True) + + class FakePool: + _pool = [wedged] + self.assertEqual(hil_health.kill_pool_children(FakePool()), 1) + + def test_skips_a_reaped_worker(self): + """Process.kill() re-checks returncode internally, but skipping a dead child keeps + the harness from signalling a pid the OS may have recycled.""" + live, dead = FakeProc(201), FakeProc(202, alive=False) + + class FakePool: + _pool = [live, dead] + self.assertEqual(hil_health.kill_pool_children(FakePool()), 0) + self.assertTrue(live.killed) + self.assertFalse(dead.killed) + + def test_also_kills_the_manager(self): + """Manager() is a separate child holding the same descriptors, and os._exit skips + its finalizer, so leaving it behind defeats the whole purpose. The RETURN is the + confirmed-survivor count (the caller words a power-cycle verdict on it), so a + clean kill of both reports 0.""" + worker, mgr_proc = FakeProc(401), FakeProc(402) + + class FakePool: + _pool = [worker] + + class FakeManager: + _process = mgr_proc + self.assertEqual(hil_health.kill_pool_children(FakePool(), FakeManager()), 0) + self.assertTrue(worker.killed and mgr_proc.killed) + self.assertTrue(mgr_proc.killed) + + def test_tolerates_a_pool_without_workers(self): + class NoPool: + _pool = None + self.assertEqual(hil_health.kill_pool_children(NoPool()), 0) + + +class WorkerSweepsItsOwnChildren(unittest.TestCase): + """maxtasksperchild=1 makes a worker exit the moment its task returns, so by the time + main()'s finally sweeps, the strays have been reparented to init and are off the pool's + ppid tree entirely. Measured over 4 tasks: pool._pool held two FRESH workers with zero + overlap with the four that ran, child_procs() returned {}, the sweep reported 0, and all + four strays were alive. Inside the worker the ppid link is still there.""" + + def test_a_detached_child_is_killed_and_confirmed(self): + kid = subprocess.Popen(['sleep', '120'], start_new_session=True) + self.addCleanup(lambda: kid.poll() is None and kid.kill()) + time.sleep(0.3) # let it appear in /proc + + stray = hil_health.kill_own_children() + + self.assertEqual(stray, 0, 'a killable stray was reported as a survivor') + kid.wait(timeout=5) # TimeoutExpired here means it outlived us + self.assertIsNotNone(kid.poll()) + + def test_no_children_is_not_an_error(self): + self.assertEqual(hil_health.kill_own_children(), 0) + + +class PermitReleasesOnlyWhatItTook(unittest.TestCase): + """The bounded acquire skips a slot it could not get ('proceeding over-subscribed') and + deliberately leaves it out of `taken`, but __exit__ released every slot in self.slots. + multiprocessing.Semaphore is unbounded, so each timeout permanently widened that + controller's permit -- the throttle this branch NARROWED (FLASH_PARALLEL 8->4, + USBTEST_PARALLEL 4->2) for xHCI bandwidth margin.""" + + def test_a_timed_out_slot_is_not_released_on_exit(self): + from helper import hil_lock + import multiprocessing + + sems = [multiprocessing.Semaphore(1)] + sems[0].acquire() # width 1, already held: the next wait times out + self.addCleanup(setattr, hil_lock, 'PERMIT_TIMEOUT', hil_lock.PERMIT_TIMEOUT) + hil_lock.PERMIT_TIMEOUT = 0.1 + + permit = hil_lock.controller_permit(sems, 'UID') + permit.slots = [0] + with permit: + pass + + # one holder still holds it, so a correct exit leaves it unavailable + self.assertFalse(sems[0].acquire(timeout=0.1), + 'the permit released a slot it never acquired: width grew') + + +class RecoveryUsesAResetOnlyWhenThereIsARealOne(unittest.TestCase): + """usbtest's recovery runs the reset unconditionally before the reflash -- it is + non-destructive (the wedged firmware survives for autopsy), writes no flash, cannot + brick SWD the way a bad park image has (mimxrt1064_evk, max32666fthr), and is measured + at 128-129 ms against a full erase+program. + + Two things still gate it, and both are what this pins: a flasher may have no reset + primitive at all, and reset_esptool/reset_lm4flash return rc 0 WITHOUT resetting + anything. Running those makes the log say "resetting <board> via <flasher>" for a step + that did nothing. wedged_pids() arbitrates either way, so behaviour was always right -- + the record was not, and a false record is what keeps having to be unpicked.""" + + def setUp(self): + import usbtest # test/hil is already on sys.path (see top of file) + # PRODUCTION, not a copy: re-implementing the screen here let the real gate be + # deleted with the suite still green, which is the failure mode this pins. + self._reset_fn = usbtest.reset_primitive + + def test_a_stub_that_resets_nothing_is_not_claimed(self): + for name in ('esptool', 'lm4flash'): + self.assertIsNone(self._reset_fn(name), + f'reset_{name} returns rc 0 without resetting; claiming it ' + f'puts a step that did nothing in the record') + + def test_a_real_reset_primitive_is_used(self): + for name in ('openocd', 'jlink', 'stlink'): + self.assertIsNotNone(self._reset_fn(name)) + + def test_a_flasher_with_no_reset_primitive_goes_straight_to_the_reflash(self): + self.assertIsNone(self._reset_fn('nosuchflasher')) + + def test_the_reset_is_attempted_before_the_reflash(self): + """Order matters and now lives only in main()'s inline ladder, where no test + reaches it -- swapping the two blocks kept the suite green. Reset first is + non-destructive: the firmware under test survives for autopsy, no flash is + written, and it cannot brick SWD the way a bad park image has on mimxrt1064_evk + and max32666fthr.""" + import ast + import usbtest + src = Path(usbtest.__file__).read_text() + fn = next(n for n in ast.walk(ast.parse(src)) + if isinstance(n, ast.FunctionDef) and n.name == 'main') + seg = ast.get_source_segment(src, fn) + reset_at = seg.index('reset_fn = reset_primitive(') + flash_at = seg.index("flash_fn(board, args.recover_fw") + self.assertLess(reset_at, flash_at, + 'the reflash is attempted before the non-destructive reset') + + + +class SudoSoftNeverRaises(unittest.TestCase): + """Two of its four call sites are inside run_case's timeout handler, where ANY raise + costs the HUNG verdict, the recovery and the JSON report -- and sudo() sys.exit()s on + 'a password is required', which is a raise like any other.""" + + def setUp(self): + import usbtest + self.u = usbtest + self.addCleanup(setattr, usbtest, 'sudo', usbtest.sudo) + + def _check(self, exc): + def boom(*a, **k): + raise exc + self.u.sudo = boom + r = self.u._sudo_soft(['dmesg']) # must not propagate + self.assertEqual(r.returncode, 1) + + def test_systemexit_from_a_password_prompt_is_contained(self): + self._check(SystemExit('sudo needs a password')) + + def test_oserror_is_contained(self): + self._check(OSError('no such binary')) + + def test_subprocess_error_is_contained(self): + self._check(subprocess.SubprocessError('timed out')) + + +if __name__ == '__main__': + unittest.main(verbosity=1) diff --git a/test/hil/test/test_hil_report.py b/test/hil/test/test_hil_report.py new file mode 100644 index 000000000..7c7a097ef --- /dev/null +++ b/test/hil/test/test_hil_report.py @@ -0,0 +1,1173 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests for the report document: the vocabulary, the one cell classifier, rendering, +# the four writers, and the fold to per-board verdicts. Split out of test_hil_bounded.py +# and test_hil_health.py when the report code moved into helper/hil_report.py. +# Run directly: +# python3 test/hil/test/test_hil_report.py +import json +import os +import subprocess +import sys +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory + +TEST_DIR = os.path.dirname(os.path.abspath(__file__)) +HIL_DIR = os.path.dirname(TEST_DIR) +sys.path.insert(0, HIL_DIR) + +from helper import hil_report + + +class OneClassifierForBothArtifacts(unittest.TestCase): + """The markdown tally and the agent's verdict used to classify cells with two separate + copies of one rule -- hil_test's cell_kind against REPORT_CELL, and hil_summary's + cell_state against its own re-typed '❌'/'⚪' literals. Change the icons and the table + and the verdict silently disagree.""" + + def test_bare_states(self): + self.assertEqual(hil_report.cell_state('fail'), 'fail') + self.assertEqual(hil_report.cell_state('skip'), 'skip') + self.assertEqual(hil_report.cell_state('pass'), 'pass') + + def test_icon_prefixed_metrics_carry_their_verdict(self): + self.assertEqual(hil_report.cell_state(f'{hil_report.REPORT_CELL["fail"]} 29/30'), 'fail') + self.assertEqual(hil_report.cell_state(f'{hil_report.REPORT_CELL["skip"]} board wedged'), + 'skip') + + def test_an_unprefixed_metric_is_a_pass(self): + """Load-bearing: a passing test may return a plain metric string. Classifying + unknown shapes as fail would publish a green table as a red verdict.""" + self.assertEqual(hil_report.cell_state('480.0 MBps'), 'pass') + self.assertEqual(hil_report.cell_state('1103 KB/s'), 'pass') + + def test_a_non_string_cell_does_not_raise(self): + """render_matrix's copy guarded with isinstance; hil_summary's did not, because its + caller str()'d first. The merged one keeps the guard -- it is the safer superset.""" + self.assertEqual(hil_report.cell_state(None), 'pass') + + def test_the_icons_come_from_REPORT_CELL(self): + """No second copy of the emoji anywhere in the module.""" + src = (Path(HIL_DIR) / 'helper' / 'hil_report.py').read_text(encoding='utf-8') + # CODE only: prose may quote an icon to explain a rule. The old assertion counted + # the single-quoted spelling `'❌'`, which a second copy written as "❌" would have + # sailed past. + code = '\n'.join(line.split('#', 1)[0] for line in src.splitlines()) + for icon in ('❌', '⚪', '✅'): + self.assertEqual(code.count(icon), 1, + f'{icon} is spelled in code more than once; REPORT_CELL is' + f' meant to be the one source') + + +class ModuleWorksImportedAndAsAScript(unittest.TestCase): + """It is imported as helper.hil_report by hil_test, and run as a script by the operator + (.claude/agents/hil-operator.md). A script run puts helper/ on sys.path, NOT test/hil, + so a plain `from helper import hil_health` breaks the CLI and only the CLI.""" + + def test_importable_as_a_package_module(self): + r = subprocess.run( + [sys.executable, '-c', + f'import sys; sys.path.insert(0, {HIL_DIR!r}); ' + f'from helper import hil_report; print(hil_report.REPORT_JSON)'], + capture_output=True, text=True, timeout=60) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn('hil_report.json', r.stdout) + + def test_runnable_as_a_script(self): + r = subprocess.run( + [sys.executable, str(Path(HIL_DIR) / 'helper' / 'hil_report.py'), '--help'], + capture_output=True, text=True, timeout=60) + self.assertEqual(r.returncode, 0, r.stderr) + + +class RenderReportIsPureFunctionOfTheDocument(unittest.TestCase): + """Four writers used to compose the markdown independently, so a table could carry + something the sidecar did not. One renderer, and the ordering it guarantees, is what + stops that -- pinned here rather than left to the order of three concatenations.""" + + def _doc(self, **kw): + d = {'rows': [{'board': 'boardA', 'cells': {'cdc_msc': 'pass'}, 'duration': '1s'}], + 'banner': '', 'scope': '', 'caveat': ''} + d.update(kw) + return d + + def test_table_comes_from_rows(self): + md = hil_report.render_report(self._doc()) + self.assertIn('boardA', md) + self.assertIn('cdc_msc', md) + + def test_scope_note_appears_above_the_table(self): + md = hil_report.render_report(self._doc(scope='-b boardA')) + self.assertLess(md.index('Scoped run'), md.index('boardA')) + + def test_banner_outranks_the_scope_note(self): + md = hil_report.render_report(self._doc(scope='-b boardA', + banner='> **Rig dirty.** x\n')) + self.assertLess(md.index('Rig dirty'), md.index('Scoped run')) + + def test_caveat_is_outermost(self): + md = hil_report.render_report(self._doc(banner='> **Rig dirty.** x\n', + caveat='**HIL run abandoned.**\n')) + self.assertLess(md.index('abandoned'), md.index('Rig dirty')) + + def test_a_document_with_no_rows_still_renders(self): + md = hil_report.render_report(self._doc(rows=[])) + self.assertIn('No tests were run.', md) + + def test_a_malformed_row_does_not_raise(self): + """mark_report_abandoned renders a sidecar it did not write -- hil_ci.sh reuses a + persistent REMOTE_DIR, so it can be an older version's or a torn one -- and it runs + on the way to os._exit, where a KeyError hangs the runner in multiprocessing's + unbounded join() instead of freeing it.""" + md = hil_report.render_report(self._doc( + rows=[{'board': 'boardA', 'cells': {'cdc_msc': 'pass'}}, {'board': 'half'}, + {}])) + self.assertIn('boardA', md) # the intact row still renders ... + self.assertIn('half', md) # ... and a cell-less one becomes a blank row + + +class ScopeSurvivesInTheJson(unittest.TestCase): + """A scoped run's small table is indistinguishable from a full run that lost boards. + The markdown says so; the JSON did not, so any JSON consumer could not tell.""" + + def _rows(self, board, cell): + return [(board, 0, 0, [(board, {cell: 'OK'}, '1s')], 0)] + + def test_scope_is_recorded_in_the_sidecar(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report(self._rows('boardA', 'cdc_msc'), rd, True, + '-b boardA', '') + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertEqual(doc['scope'], '-b boardA') + + def test_an_unscoped_run_records_an_empty_scope(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report(self._rows('boardA', 'cdc_msc'), rd, True, '', '') + self.assertEqual(json.loads((rd / hil_report.REPORT_JSON).read_text())['scope'], '') + + +class EveryExitPathLeavesBothArtifacts(unittest.TestCase): + """summarize() builds an agent's verdicts from the JSON. A path that writes only + markdown reports the whole fleet as 'no report row' while a human sees the real story.""" + + def test_the_no_boards_exit_writes_json_too(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.write_report(rd, {'rows': [], 'banner': '', 'scope': '', + 'caveat': '**HIL run selected no boards.** why\n'}) + self.assertIn('selected no boards', (rd / hil_report.REPORT_MD).read_text()) + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertEqual(doc['rows'], []) + self.assertIn('selected no boards', doc['caveat']) + + def test_write_report_raises_so_its_callers_can_report_it(self): + """write_report is NOT best-effort. Swallowing the OSError made + write_timeout_report's _p warning and hil_test's fallback-of-the-fallback dead + code -- an unwritable report dir produced no artifact and no message.""" + with self.assertRaises(OSError): + hil_report.write_report(Path('/proc/nonexistent/nope'), + {'rows': [], 'banner': '', 'scope': '', 'caveat': 'x\n'}) + + def test_the_guarded_callers_still_do_not_raise(self): + """They are the ones on the way to os._exit, where a raise hangs the interpreter + in multiprocessing's unbounded join().""" + bad = Path('/proc/nonexistent/nope') + hil_report.mark_report_abandoned(bad, 'the worker pool would not shut down.') + hil_report.mark_report_no_boards(bad, 'filters intersected to nothing') + import io + from contextlib import redirect_stdout + with redirect_stdout(io.StringIO()): + hil_report.write_timeout_report(bad, [{'name': 'b1'}], 3600) + + +class AbandonNoticeLandsInBothArtifacts(unittest.TestCase): + """_abandon_exit did a text prepend on a file it had not written, so the caveat never + reached the JSON and an agent reading the sidecar saw a clean partial report under a + red job.""" + + def test_abandon_sets_the_caveat_not_just_the_markdown(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('boardA', 0, 0, [('boardA', {'cdc_msc': 'OK'}, '1s')], 0)], rd, True, '', '') + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertIn('abandoned', doc['caveat']) + self.assertEqual(len(doc['rows']), 1, 'the finished board must survive') + md = (rd / hil_report.REPORT_MD).read_text() + self.assertLess(md.index('abandoned'), md.index('boardA')) + + def test_marking_a_missing_report_is_a_no_op(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + hil_report.mark_report_abandoned(Path(td.name), 'x') # must not raise + + def test_a_sidecar_with_a_malformed_row_still_gets_stamped(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_JSON).write_text(json.dumps( + {'rows': [{'board': 'boardA'}], 'banner': '', 'scope': '', 'caveat': ''})) + hil_report.mark_report_abandoned(rd, 'x') + self.assertIn('abandoned', + json.loads((rd / hil_report.REPORT_JSON).read_text())['caveat']) + self.assertIn('abandoned', (rd / hil_report.REPORT_MD).read_text()) + + def test_a_torn_sidecar_is_a_no_op(self): + """This runs while the interpreter is being torn down: a raise here hangs the + process in multiprocessing's unbounded join().""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_JSON).write_text('{ truncated mid-') + hil_report.mark_report_abandoned(rd, 'x') # must not raise + + def test_an_existing_abandon_caveat_is_not_overwritten(self): + """The pool-timeout path names the stuck boards and the rig-health verdict; this + one only knows the pool would not shut down. Whoever got there first wins -- + the guard _abandon_exit used to spell as "'**HIL run ab' not in body[:2000]".""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.write_timeout_report(rd, [{'name': 'stuck'}], 3600, + prefix='> **wedged usb_hub_wq worker.**\n') + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertIn('timed out after 3600s', doc['caveat']) + self.assertIn('wedged usb_hub_wq worker', doc['banner']) # rig health, not outcome + self.assertNotIn('would not shut down', doc['caveat']) + + +class CaveatSurvivesAccumulate(unittest.TestCase): + """CI reruns with --accumulate: the sidecar keeps every earlier attempt's cells, but the + banner was recomputed per attempt. A first attempt on a degraded rig and a clean rerun + therefore published the degraded attempt's PASSES with no caveat on them -- and the + generated .failed spec reruns only failures, so those cells are never re-earned.""" + + def _rows(self, board, cell): + return [(board, 0, 0, [(board, {cell: 'OK'}, '1s')], 0)] + + def test_an_earlier_attempts_caveat_is_still_on_the_report(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + banner = '> **Rig note.** 2 process(es) in D state at start.\n' + + hil_report.accumulate_report(self._rows('boardA', 'cdc_msc'), rd, True, '', banner) + self.assertIn('Rig note', (rd / hil_report.REPORT_MD).read_text()) + + # the rerun: clean rig, so this attempt contributes no banner of its own + md = hil_report.accumulate_report(self._rows('boardB', 'cdc_msc'), rd, False, '', '') + self.assertIn('boardA', md) # the earlier cells are kept ... + self.assertIn('Rig note', md, + 'the caveat the earlier cells were collected under was dropped') + + def test_the_same_caveat_twice_is_not_stacked(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + banner = '> **Rig note.** 2 process(es) in D state at start.\n' + hil_report.accumulate_report(self._rows('boardA', 'cdc_msc'), rd, True, '', banner) + md = hil_report.accumulate_report(self._rows('boardB', 'cdc_msc'), rd, False, '', banner) + self.assertEqual(md.count('Rig note'), 1) + + +class MarkdownIsAlwaysARenderingOfTheJson(unittest.TestCase): + """The property this whole change buys: whatever wrote the report, re-rendering the + sidecar reproduces the markdown byte for byte. Four writers, one renderer -- asserted + directly rather than inferred from the writers.""" + + def _check(self, rd): + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertEqual((rd / hil_report.REPORT_MD).read_text(), + hil_report.render_report(doc) + '\n') + + def test_normal_path(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('boardA', 0, 0, [('boardA', {'cdc_msc': 'OK'}, '1s')], 0)], rd, True, + '-b boardA', '> **Rig note.** x\n') + self._check(rd) + + def test_after_an_accumulate_rerun(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('boardA', 0, 0, [('boardA', {'cdc_msc': 'OK'}, '1s')], 0)], rd, True, '', '') + hil_report.accumulate_report( + [('boardB', 0, 0, [('boardB', {'cdc_msc': 'OK'}, '1s')], 0)], rd, False, '', '') + self._check(rd) + + def test_after_abandonment(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('boardA', 0, 0, [('boardA', {'cdc_msc': 'OK'}, '1s')], 0)], rd, True, '', '') + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + self._check(rd) + + def test_no_boards_exit(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.write_report(rd, {'rows': [], 'banner': '', 'scope': '', + 'caveat': '**HIL run selected no boards.** why\n'}) + self._check(rd) + + def test_the_pool_guard_fallback(self): + """The last writer to join the invariant: it composed its own markdown only because + hil_health could not import the renderer.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('done', 0, 0, [('done', {'cdc_msc': 'OK'}, '1s')], 0)], rd, True, '', '') + hil_report.write_timeout_report(rd, [{'name': 'stuck'}], 3600, + prefix='> **wedged usb_hub_wq worker.**\n') + self._check(rd) + +class WriteTimeoutReport(unittest.TestCase): + def test_prefix_carries_the_preflight_diagnosis(self): + """The timeout aborts before accumulate_report, so without the prefix the artifact + and the PR comment lose the one line saying WHY the pool never finished.""" + with TemporaryDirectory() as td: + d = Path(td) + hil_report.write_timeout_report(d, [{'name': 'b1'}], 4200, + prefix='> **wedged usb_hub_wq worker.**\n') + out = (d / hil_report.REPORT_MD).read_text() + # the abandon notice leads (run outcome), the rig-health prefix follows in the + # banner -- prefix used to be folded INTO the caveat, which is what made a clean + # --accumulate retry inherit an abandonment that had not happened + self.assertTrue(out.startswith('**HIL run abandoned: worker pool timed out'), out[:80]) + self.assertIn('> **wedged usb_hub_wq worker.**', out) + self.assertIn('timed out after 4200s', out) + self.assertIn('- b1', out) + + def test_writes_a_report_where_there_would_be_none(self): + with TemporaryDirectory() as td: + hil_report.write_timeout_report(Path(td), [{'name': 'ra6m5_ek'}], 4200) + md = (Path(td) / hil_report.REPORT_MD).read_text() + self.assertIn('4200s', md) + self.assertIn('ra6m5_ek', md) + + def test_the_prior_attempts_rows_survive(self): + """Was: the prior MARKDOWN TEXT survives below the banner. It now re-renders from + the merged sidecar, so the guarantee is stated against rows -- one table with the + stuck boards in it, rather than a banner stapled above a duplicate table.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('done', 0, 0, [('done', {'cdc_msc': 'OK'}, '1s')], 0)], rd, True, '', '') + hil_report.write_timeout_report(rd, [{'name': 'stuck'}], 3600) + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertEqual([r['board'] for r in doc['rows']], ['done', 'stuck']) + md = (rd / hil_report.REPORT_MD).read_text() + self.assertIn('done', md) + self.assertIn('stuck', md) + self.assertIn('abandoned', md) + self.assertLess(md.index('abandoned'), md.index('done')) + self.assertEqual(md.count('| Board'), 1, 'the prior table was duplicated, not merged') + + def test_custom_banner_is_used(self): + with TemporaryDirectory() as td: + hil_report.write_timeout_report(Path(td), [], 0, + banner='**refused to start.**\n') + self.assertIn('refused to start', (Path(td) / hil_report.REPORT_MD).read_text()) + + def test_timeout_report_writes_the_sidecar(self): + """This path used to write markdown only, so summarize() -- which is all an + agent gets -- reported the whole fleet as 'no report row' on exactly the runs + that failed.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.write_timeout_report(rd, [{'name': 'boardA'}], 3600) + self.assertTrue((rd / hil_report.REPORT_JSON).is_file()) + self.assertIn('boardA', (rd / hil_report.REPORT_JSON).read_text()) + + def test_the_sidecar_keeps_a_previous_attempts_rows(self): + """An earlier attempt's finished boards are real results and this attempt has none + of its own, so the rows merge rather than replace.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_JSON).write_text(json.dumps( + {'rows': [{'board': 'done', 'cells': {'cdc_msc': 'pass'}, 'duration': '1s'}], + 'banner': '', 'scope': '', 'caveat': ''})) + hil_report.write_timeout_report(rd, [{'name': 'stuck'}], 3600) + rows = json.loads((rd / hil_report.REPORT_JSON).read_text())['rows'] + self.assertEqual([r['board'] for r in rows], ['done', 'stuck']) + + def test_a_torn_sidecar_does_not_lose_the_stuck_boards(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_JSON).write_text('{ truncated mid-') + hil_report.write_timeout_report(rd, [{'name': 'stuck'}], 3600) + rows = json.loads((rd / hil_report.REPORT_JSON).read_text())['rows'] + self.assertEqual([r['board'] for r in rows], ['stuck']) + + def test_a_roster_entry_without_a_name_does_not_escape(self): + """The broad handler exists to stop a KeyError here stranding the runner, but a + report that silently loses its only board is worse than one saying '?'.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.write_timeout_report(rd, [{}], 3600) + rows = json.loads((rd / hil_report.REPORT_JSON).read_text())['rows'] + self.assertEqual([r['board'] for r in rows], ['?']) + + def test_unwritable_dir_does_not_raise(self): + """The caller may be about to os._exit; losing the report must not also lose the + exit path.""" + hil_report.write_timeout_report(Path('/proc/nonexistent/nope'), [], 0) + + +class SummaryFoldsReportToBoards(unittest.TestCase): + """summarize() replaces the agent retyping the markdown table. Report rows are named per + VARIANT and a variant need not start with the board name, so the config is what maps them + back -- the previous string-matching design produced a defect in each of four review rounds.""" + + def _sum(self, boards, rows, cfg_boards=None, banner=''): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + d = Path(td.name) + (d / 'hil_report.json').write_text(json.dumps( + {'rows': [{'board': b, 'cells': c, 'duration': '1s'} for b, c in rows], + 'banner': banner})) + cfg = d / 'cfg.json' + cfg.write_text(json.dumps({'boards': cfg_boards or [{'name': b} for b in boards]})) + args = [a for b in boards for a in ('-b', b)] + r = subprocess.run(['python3', str(Path(TEST_DIR).parents[0] / 'helper' / 'hil_report.py'), + str(cfg), *args, '--report-dir', str(d)], + capture_output=True, text=True, timeout=60) + self.assertEqual(r.returncode, 0, r.stderr) + return json.loads(r.stdout)['results'] + + def test_variant_rows_fold_onto_their_board(self): + """nanoch32v203 never produces a row named after the board.""" + got = self._sum(['nanoch32v203'], + [('nanoch32v203-fsdev', {'usbtest': 'pass'}), + ('nanoch32v203-usbfs', {'usbtest': 'pass'})], + cfg_boards=[{'name': 'nanoch32v203', + 'variant': [{'name': 'nanoch32v203-fsdev'}, + {'name': 'nanoch32v203-usbfs'}]}]) + self.assertEqual([r['board'] for r in got], ['nanoch32v203']) + self.assertTrue(got[0]['pass']) + self.assertTrue(got[0]['ran']) + + def test_one_failing_variant_fails_the_board(self): + got = self._sum(['nano'], + [('nano-a', {'usbtest': 'pass'}), ('nano-b', {'usbtest': '❌ 29/30'})], + cfg_boards=[{'name': 'nano', 'variant': [{'name': 'nano-a'}, + {'name': 'nano-b'}]}]) + self.assertFalse(got[0]['pass']) + self.assertIn('29/30', got[0]['detail']) + + def test_lock_contention_is_a_field_not_a_prefix(self): + got = self._sum(['alpha'], [('alpha', {'board-locked': 'fail'})]) + self.assertTrue(got[0]['locked']) + self.assertFalse(got[0]['pass']) + + def test_a_board_with_no_row_is_marked_not_run(self): + got = self._sum(['alpha', 'beta'], [('alpha', {'usbtest': 'pass'})]) + self.assertTrue(got[0]['ran']) + self.assertFalse(got[1]['ran']) + self.assertFalse(got[1]['pass']) + + def test_a_metric_cell_counts_by_its_icon(self): + got = self._sum(['a', 'b'], [('a', {'cdc_msc_throughput': '✅ C 1.2 M 3.4'}), + ('b', {'cdc_msc_throughput': '❌ C 0.0 M 0.0'})]) + self.assertTrue(got[0]['pass']) + self.assertFalse(got[1]['pass']) + + def test_skipped_cells_do_not_fail_a_board(self): + got = self._sum(['a'], [('a', {'usbtest': 'skip', 'cdc_msc': 'pass'})]) + self.assertTrue(got[0]['pass']) + + def test_a_plain_metric_cell_is_a_pass(self): + """Mirrors hil_test.py's own tally (cell_kind): failures are ALWAYS marked -- 'fail' + or a ❌ prefix, per TestFail's docstring -- while a passing test may return a plain + metric string that lands in the cell unprefixed. Treating unknown shapes as fail + would publish a green table as a red verdict.""" + got = self._sum(['a'], [('a', {'device_speed': '480.0 MBps'})]) + self.assertTrue(got[0]['pass']) + + def test_a_declared_variant_of_another_board_is_not_stolen(self): + """A declared variant need not start with its own board's name, so it may start with + a DIFFERENT board's name plus '-'. The prefix fallback must not attribute it twice.""" + got = self._sum(['alpha', 'beta'], + [('beta-x', {'usbtest': 'fail'})], + cfg_boards=[{'name': 'alpha', 'variant': [{'name': 'beta-x'}]}, + {'name': 'beta'}]) + self.assertTrue(got[0]['ran']) + self.assertFalse(got[0]['pass']) + self.assertFalse(got[1]['ran'], "beta must not inherit alpha's row") + + + def test_the_caveat_reaches_the_agents_verdict(self): + """The abandon/no-boards notice lives in the document now, and this JSON is all an + agent gets -- dropping it here puts the caveat back where only a human sees it.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + d = Path(td.name) + (d / 'hil_report.json').write_text(json.dumps( + {'rows': [{'board': 'boardA', 'cells': {'cdc_msc': 'pass'}, 'duration': '1s'}], + 'banner': '', 'scope': '', + 'caveat': '**HIL run abandoned: the worker pool would not shut down.**\n'})) + cfg = d / 'cfg.json' + cfg.write_text(json.dumps({'boards': [{'name': 'boardA'}]})) + r = subprocess.run( + ['python3', str(Path(TEST_DIR).parents[0] / 'helper' / 'hil_report.py'), + str(cfg), '-b', 'boardA', '--report-dir', str(d)], + capture_output=True, text=True, timeout=60) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn('abandoned', json.loads(r.stdout)['caveat']) + + def test_an_older_sidecar_without_a_caveat_still_summarises(self): + got = self._sum(['boardA'], [('boardA', {'cdc_msc': 'pass'})]) + self.assertTrue(got[0]['pass']) + + def test_the_old_entry_point_is_gone(self): + """hil_summary.py's CLI moved here. A leftover file would keep working while + drifting from the module that now owns the fold.""" + self.assertFalse((Path(HIL_DIR) / 'helper' / 'hil_summary.py').exists()) + + +class AbandonStampIsNotDestructive(unittest.TestCase): + """mark_report_abandoned runs on the way to os._exit, on a report it did not write. + Every case here was a live regression found by review.""" + + def _doc(self, **kw): + d = {'rows': [{'board': 'OLD', 'cells': {'t': 'pass'}, 'duration': '9s'}], + 'banner': '', 'scope': '', 'caveat': ''} + d.update(kw) + return d + + def test_declining_to_stamp_does_not_republish_the_markdown(self): + """The guard skipped the caveat assignment but write_report ran anyway, so a + no-op call still overwrote THIS run's table with a re-render of an older sidecar.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_JSON).write_text(json.dumps(self._doc( + caveat='**HIL run abandoned: worker pool timed out after 3600s.**\n'))) + (rd / hil_report.REPORT_MD).write_text('THIS RUN table with boardX\n') + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + self.assertEqual((rd / hil_report.REPORT_MD).read_text(), + 'THIS RUN table with boardX\n') + + def test_a_banner_borne_abandon_notice_also_wins(self): + """The pool-timeout path puts its notice in `banner` (hil_test.py:2300), not + `caveat`. SKILL.md gives the two notices OPPOSITE rules, so stamping the vaguer + one on top tells the agent to publish rows it is meant to discard.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('a', 0, 0, [('a', {'t': 'OK'}, '1s')], 0)], rd, True, '', '', + caveat='**HIL run abandoned: worker pool timed out after 3600s.** 2 never' + ' reported.\n') + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + md = (rd / hil_report.REPORT_MD).read_text() + self.assertTrue(md.startswith('**HIL run abandoned: worker pool timed out'), md[:80]) + self.assertNotIn('would not shut down', md) + + def test_a_missing_sidecar_still_stamps_the_markdown(self): + """Master read the MARKDOWN and prepended unconditionally, so it always stamped. + pr_comment.yml cats only hil_report.md -- giving up here publishes a clean green + table under an abandoned, non-zero job.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_MD).write_text('**✅ 27 passed · ❌ 0 failed**\n') + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + md = (rd / hil_report.REPORT_MD).read_text(encoding='utf-8') + self.assertIn('abandoned', md) + self.assertIn('27 passed', md) + + def test_a_torn_sidecar_still_stamps_the_markdown(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_JSON).write_text('{ truncated mid-') + (rd / hil_report.REPORT_MD).write_text('**✅ 27 passed**\n') + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + self.assertIn('abandoned', (rd / hil_report.REPORT_MD).read_text(encoding='utf-8')) + + def test_the_wording_matches_the_skill_contract(self): + """SKILL.md pins this banner as 'the table below IS this run's ... Report the + results AND the abandonment'. Calling it 'partial' sends the agent to re-run + boards that already passed.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_JSON).write_text(json.dumps(self._doc())) + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + caveat = json.loads((rd / hil_report.REPORT_JSON).read_text())['caveat'] + self.assertNotIn('partial', caveat) + self.assertIn('unverified', caveat) + + +class WriteReportFailsLoudly(unittest.TestCase): + def test_a_render_failure_does_not_leave_a_committed_json(self): + """It wrote the JSON, then rendered. A render raise left the sidecar saying + 'abandoned' beside a markdown that still read as a clean green table -- breaking + the one invariant this module exists to hold.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_MD).write_text('STALE GREEN TABLE\n') + (rd / hil_report.REPORT_JSON).write_text(json.dumps( + {'rows': ['boardA'], 'banner': '', 'scope': '', 'caveat': ''})) + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + md = (rd / hil_report.REPORT_MD).read_text() + # either both moved or neither did -- never a sidecar the markdown contradicts + self.assertEqual('abandoned' in doc.get('caveat', ''), 'abandoned' in md, + 'the sidecar was committed without its markdown') + + def test_a_non_dict_row_does_not_cost_the_abandon_stamp(self): + """A row that is a bare string raised out of render_report, so the stamp was lost + entirely -- the failure mode this whole function exists to prevent.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_JSON).write_text(json.dumps( + {'rows': ['boardA', {'board': 'good', 'cells': {'t': 'pass'}}], + 'banner': '', 'scope': '', 'caveat': ''})) + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + md = (rd / hil_report.REPORT_MD).read_text() + self.assertIn('abandoned', md) + self.assertIn('good', md) + + def test_an_unwritable_dir_reaches_the_callers_warning(self): + """write_report swallowing OSError made write_timeout_report's broad handler -- + and hil_test's fallback-of-the-fallback -- dead code: no artifact, no message.""" + import io + from contextlib import redirect_stdout + buf = io.StringIO() + with redirect_stdout(buf): + hil_report.write_timeout_report(Path('/proc/nonexistent/nope'), + [{'name': 'b1'}], 3600, prefix='x\n') + self.assertIn('warning', buf.getvalue().lower(), 'the failure was silent') + + +class PoolTimeoutCellIsHonest(unittest.TestCase): + def test_a_stuck_board_with_a_prior_row_still_gets_the_cell(self): + """`not in done` skipped the cell for any board carrying an earlier attempt's row, + so a board that just ate the 60-minute guard summarized as pass:true.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('stm32f4', 0, 0, [('stm32f4', {'cdc_msc': 'OK'}, '1s')], 0)], rd, True, '', '') + hil_report.write_timeout_report(rd, [{'name': 'stm32f4'}], 3600) + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + verdict = hil_report.summarize({'boards': [{'name': 'stm32f4'}]}, ['stm32f4'], doc) + self.assertFalse(verdict['results'][0]['pass'], + 'a board that hung the pool was published as a pass') + + def test_a_clean_retry_clears_the_cell(self): + """accumulate_report clears stale board-locked and BOUNDARY_CELL cells but not + this one, so a board that passed clean on the retry stayed red forever.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.write_timeout_report(rd, [{'name': 'stuck'}], 3600) + hil_report.accumulate_report( + [('stuck', 0, 0, [('stuck', {'cdc_msc': 'OK'}, '2s')], 0)], rd, False, '', '') + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertNotIn('pool-timeout', doc['rows'][0]['cells']) + verdict = hil_report.summarize({'boards': [{'name': 'stuck'}]}, ['stuck'], doc) + self.assertTrue(verdict['results'][0]['pass']) + + def test_a_torn_sidecar_does_not_destroy_an_intact_markdown(self): + """Re-rendering from an unusable sidecar threw away real results the human copy + still had. Master concatenated below its banner and kept them.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_MD).write_text('| Board | t |\n| a | OK |\n| b | OK |\n') + (rd / hil_report.REPORT_JSON).write_text('{ truncated') + hil_report.write_timeout_report(rd, [{'name': 'stuck'}], 3600) + md = (rd / hil_report.REPORT_MD).read_text() + self.assertIn('| a | OK |', md, "an earlier attempt's real results were destroyed") + self.assertIn('stuck', md) + + +class SummarizeSeesEveryRow(unittest.TestCase): + def test_board_name_rows_reach_a_variant_boards_verdict(self): + """hil_test writes lock-contention and pool-timeout rows keyed by BOARD name, but + variants_of returns only declared variant names -- so for nanoch32v203 and + ch32v307v_r1_1v0 those rows were invisible and a held lock published as a + hardware FAIL that hil-validate.js never retried.""" + cfg = {'boards': [{'name': 'nano', + 'variant': [{'name': 'nano-fsdev'}, {'name': 'nano-usbfs'}]}]} + doc = {'rows': [{'board': 'nano', 'cells': {'board-locked': 'fail'}, + 'duration': None}], 'banner': '', 'scope': '', 'caveat': ''} + r = hil_report.summarize(cfg, ['nano'], doc)['results'][0] + self.assertTrue(r['ran']) + self.assertTrue(r['locked'], 'a held lock was published as a hardware failure') + + def test_a_malformed_row_does_not_kill_the_cli(self): + """summarize is the one reader with no defense, and it is the only one an agent's + verdict depends on.""" + out = hil_report.summarize({'boards': [{'name': 'a'}]}, ['a'], + {'rows': [{'cells': {}}, {'board': 'a', + 'cells': {'t': 'pass'}}]}) + self.assertTrue(out['results'][0]['pass']) + + +class NoBoardsExitKeepsWhatRan(unittest.TestCase): + def test_it_does_not_wipe_an_accumulated_sidecar(self): + """Master wrote only markdown here, so the sidecar survived. Writing rows:[] + unconditionally makes an --accumulate rerun whose filters empty erase every + board that had already passed.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('a', 0, 0, [('a', {'t': 'OK'}, '1s')], 0)], rd, True, '', '') + hil_report.mark_report_no_boards(rd, 'No boards left after the flasher filter', + fresh=False) + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertEqual([r['board'] for r in doc['rows']], ['a']) + self.assertIn('selected no boards', doc['caveat']) + self.assertIn('selected no boards', (rd / hil_report.REPORT_MD).read_text()) + + +class TheMergeBehavioursAreActuallyPinned(unittest.TestCase): + """accumulate_report's docstring cites these three as the reason not to split it, yet + deleting any of them left the whole suite green. Mutation-verified.""" + + def test_a_cleared_boundary_drops_the_previous_attempts_mark(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('b', 0, 0, [('b-v', {hil_report.BOUNDARY_CELL: 'fail'}, '1s')], 0)], + rd, True, '', '') + hil_report.accumulate_report( + [('b', 0, 0, [('b-v', {'cdc_msc': 'OK'}, '2s')], 0)], rd, False, '', '') + cells = json.loads((rd / hil_report.REPORT_JSON).read_text())['rows'][0]['cells'] + self.assertNotIn(hil_report.BOUNDARY_CELL, cells) + + def test_a_board_that_really_ran_drops_its_stale_lock_cell(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('b', 0, 0, [('b', {hil_report.LOCKED_CELL: 'fail'}, None)], 0)], rd, True, '', '') + hil_report.accumulate_report( + [('b', 0, 0, [('b', {'cdc_msc': 'OK'}, '2s')], 0)], rd, False, '', '') + rows = json.loads((rd / hil_report.REPORT_JSON).read_text())['rows'] + self.assertEqual([r['board'] for r in rows], ['b']) + self.assertNotIn(hil_report.LOCKED_CELL, rows[0]['cells']) + + def test_a_filtered_rerun_keeps_the_previous_duration(self): + """A -t-filtered re-run reports duration None; blanking the column loses the only + record of how long the full run took.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('b', 0, 0, [('b', {'cdc_msc': 'OK'}, '119s')], 0)], rd, True, '', '') + hil_report.accumulate_report( + [('b', 0, 0, [('b', {'cdc_msc': 'OK'}, None)], 0)], rd, False, '', '') + self.assertEqual(json.loads( + (rd / hil_report.REPORT_JSON).read_text())['rows'][0]['duration'], '119s') + + +class TheFooterCountsAreNotSwapped(unittest.TestCase): + """SKILL.md tells the operator to paste the footer counts verbatim, and swapping the + failed/skipped tallies left the suite green.""" + + def test_each_kind_is_counted_under_its_own_label(self): + md = hil_report.render_matrix([ + ('b', {'p1': 'pass', 'p2': 'pass', 'f1': 'fail', + 's1': f'{hil_report.REPORT_CELL["skip"]} board wedged'}, '1s')]) + self.assertIn(f'{hil_report.REPORT_CELL["pass"]} 2 passed', md) + self.assertIn(f'{hil_report.REPORT_CELL["fail"]} 1 failed', md) + self.assertIn(f'{hil_report.REPORT_CELL["skip"]} 1 skipped', md) + + +class HilCiUploadsTheAccumulateMergeBase(unittest.TestCase): + """hil_ci.sh rm -rf's REMOTE_DIR at the start of every run, and accumulate_report + merges onto the sidecar in the run's cwd -- so without an upload a remote + `--accumulate` retry silently starts from nothing and its one-row table REPLACES the + full-fleet one. The copy-back at the end has always existed; the upload did not.""" + + def _gate(self, *args): + """Run the real gate block out of hil_ci.sh and return its ACCUMULATE verdict. + + Executed, not grepped: the previous pair of tests searched the source text and + stayed green when `if [ "$ACCUMULATE" = 1 ]` was mutated to `if true`, because the + comment block above it mentions --accumulate five times.""" + sh = (Path(HIL_DIR) / 'hil_ci.sh').read_text(encoding='utf-8') + a = sh.index('ACCUMULATE=$(python3 -') + b = sh.index(') || ACCUMULATE=0', a) + len(') || ACCUMULATE=0') + script = 'ARGS=("$@")\n' + sh[a:b] + '\necho "$ACCUMULATE"' + r = subprocess.run(['bash', '-c', script, '_', *args], + capture_output=True, text=True, timeout=60) + self.assertEqual(r.returncode, 0, r.stderr) + return r.stdout.strip() + + def test_every_spelling_argparse_accepts_is_detected(self): + """hil_test.py declares `-a, --accumulate`, so argparse also takes -av, -va, + --accum and --acc; hil-validate.js tells the operator to retry 'adding -v'.""" + for spelling in ('--accumulate', '-a', '-av', '-va', '--accum', '--acc'): + self.assertEqual(self._gate(spelling), '1', f'{spelling} was not detected') + + def test_a_run_without_it_is_not_treated_as_accumulate(self): + for spelling in ('-b', '-v', '--retry'): + self.assertEqual(self._gate(spelling), '0', f'{spelling} falsely detected') + + def test_the_sidecar_is_uploaded_and_gated(self): + sh = (Path(HIL_DIR) / 'hil_ci.sh').read_text(encoding='utf-8') + up = [ln for ln in sh.splitlines() + if 'scp' in ln and 'hil_report.json' in ln and '$REMOTE:' in ln] + self.assertTrue(up, 'nothing uploads hil_report.json; --accumulate has no merge base') + self.assertIn('if [ "$ACCUMULATE" = 1 ]', sh, 'the upload is not gated') + + def test_a_missing_merge_base_is_loud(self): + """The damage: --accumulate with nothing to merge onto succeeds and quietly + publishes a small table where a full one used to be.""" + warn = [ln for ln in (Path(HIL_DIR) / 'hil_ci.sh').read_text().splitlines() + if 'warning' in ln.lower() and 'accumulate' in ln.lower()] + self.assertTrue(warn, 'no warning when --accumulate has no local sidecar') + + +class RunOutcomeAndRigHealthAreSeparate(unittest.TestCase): + """`banner` describes the CONDITIONS cells were collected under, so it carries across a + retry. `caveat` describes how a RUN ENDED, so it must not: a clean retry that reports + an earlier attempt's abandonment tells the agent a green run failed.""" + + def test_a_clean_retry_drops_the_previous_abandon_notice(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report([('a', 0, 0, [('a', {'t': 'OK'}, '1s')], 0)], + rd, True, '', '') + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + hil_report.accumulate_report([('a', 0, 0, [('a', {'t': 'OK'}, '2s')], 0)], + rd, False, '', '') + self.assertEqual(json.loads((rd / hil_report.REPORT_JSON).read_text())['caveat'], '') + + def test_rig_health_still_carries_across_the_retry(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.write_timeout_report(rd, [{'name': 's'}], 3600, + prefix='> **Rig note.** wedged\n') + hil_report.accumulate_report([('a', 0, 0, [('a', {'t': 'OK'}, '1s')], 0)], + rd, False, '', '') + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertIn('Rig note', doc['banner']) + + def test_a_second_attempts_abandon_is_recorded(self): + """_already_abandoned matched a notice carried forward from an EARLIER attempt, so + a genuinely new abandon wrote nothing and the run's own failure vanished.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('a', 0, 0, [('a', {'t': 'OK'}, '1s')], 0)], rd, True, '', + '> **Rig note.** x\n', + caveat='**HIL run abandoned: worker pool timed out after 3600s.**\n') + hil_report.accumulate_report([('a', 0, 0, [('a', {'t': 'OK'}, '2s')], 0)], + rd, False, '', '') + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + self.assertIn('would not shut down', + json.loads((rd / hil_report.REPORT_JSON).read_text())['caveat']) + + +class AMalformedSidecarNeverCostsTheReport(unittest.TestCase): + """hil_ci.sh now uploads a sidecar as the merge base, so a non-conforming one is + reachable from outside the harness.""" + + def _write(self, rd, doc): + (rd / hil_report.REPORT_JSON).write_text(json.dumps(doc)) + + def test_a_null_banner_does_not_kill_a_successful_run(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + self._write(rd, {'rows': [{'board': 'a', 'cells': {'t': 'pass'}, 'duration': '1s'}], + 'banner': None, 'caveat': None, 'scope': ''}) + hil_report.accumulate_report([('b', 0, 0, [('b', {'t': 'OK'}, '1s')], 0)], + rd, False, '', '') + self.assertTrue((rd / hil_report.REPORT_MD).is_file()) + + def test_a_null_cells_row_still_gets_its_pool_timeout_cell(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + self._write(rd, {'rows': [{'board': 'boardA', 'cells': None, 'duration': '61s'}], + 'banner': '', 'caveat': '', 'scope': ''}) + hil_report.write_timeout_report(rd, [{'name': 'boardA'}], 3600) + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + v = hil_report.summarize({'boards': [{'name': 'boardA'}]}, ['boardA'], doc) + self.assertFalse(v['results'][0]['pass'], + 'a board that ate the whole pool guard was published as a pass') + + def test_an_awkward_sidecar_still_gets_the_abandon_stamp(self): + """Any raise inside the dict branch was swallowed and the markdown fallback was + unreachable, so the stamp was lost from BOTH artifacts.""" + for bad in ({'rows': [{'board': 'a', 'cells': {'t': 'p'}, 'duration': 120}], + 'banner': '', 'caveat': '', 'scope': ''}, + {'rows': [{'board': 'a', 'cells': {'t': ['x']}, 'duration': '1s'}], + 'banner': None, 'caveat': '', 'scope': ''}): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + self._write(rd, bad) + (rd / hil_report.REPORT_MD).write_text('**✅ 27 passed**\n') + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + self.assertIn('abandoned', (rd / hil_report.REPORT_MD).read_text(encoding='utf-8'), + f'no stamp for {bad}') + + def test_a_malformed_roster_entry_still_leaves_an_artifact(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + import io + from contextlib import redirect_stdout + with redirect_stdout(io.StringIO()): + hil_report.write_timeout_report(rd, ['plainstring'], 3600) + self.assertTrue((rd / hil_report.REPORT_MD).is_file(), 'no artifact at all') + + +class PoolTimeoutOutranksAStaleLock(unittest.TestCase): + def test_a_wedge_is_not_published_as_lock_contention(self): + """locked was computed across every cell and short-circuited detail, so a board + that wedged the rig on the retry was reported as LOCKED -- and hil-validate.js + re-runs those, paying another pool guard on a board that just hung it.""" + doc = {'rows': [{'board': 'boardX', + 'cells': {'board-locked': 'fail', 'pool-timeout': 'fail'}, + 'duration': None}], 'banner': '', 'caveat': '', 'scope': ''} + r = hil_report.summarize({'boards': [{'name': 'boardX'}]}, ['boardX'], doc)['results'][0] + self.assertFalse(r['locked'], 'a wedge was published as lock contention') + self.assertFalse(r['pass']) + + def test_a_run_aborted_board_is_not_published_as_lock_contention(self): + """run-aborted is written by the same _abort_report path as pool-timeout, for a + board the guard never reached. It has to outrank a stale lock cell for the same + reason -- otherwise hil-validate.js re-runs a board whose worker RAISED.""" + doc = {'rows': [{'board': 'boardX', + 'cells': {'board-locked': 'fail', 'run-aborted': 'fail'}, + 'duration': None}], 'banner': '', 'caveat': '', 'scope': ''} + r = hil_report.summarize({'boards': [{'name': 'boardX'}]}, ['boardX'], doc)['results'][0] + self.assertFalse(r['locked'], 'an aborted run was published as lock contention') + self.assertFalse(r['pass']) + + +class NoBoardsExitRespectsFreshness(unittest.TestCase): + def test_a_fresh_run_does_not_republish_the_previous_rows(self): + """It is called BEFORE the fresh wipe, so it re-published last run's green table + under this run's red job -- the stale-table failure it exists to prevent.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report([('a', 0, 0, [('a', {'t': 'OK'}, '1s')], 0)], + rd, True, '', '') + hil_report.mark_report_no_boards(rd, 'filters emptied', fresh=True) + self.assertEqual(json.loads((rd / hil_report.REPORT_JSON).read_text())['rows'], []) + + def test_an_accumulate_run_keeps_them(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report([('a', 0, 0, [('a', {'t': 'OK'}, '1s')], 0)], + rd, True, '', '') + hil_report.mark_report_no_boards(rd, 'filters emptied', fresh=False) + self.assertEqual([r['board'] for r in json.loads( + (rd / hil_report.REPORT_JSON).read_text())['rows']], ['a']) + + def test_it_does_not_overwrite_an_abandon_notice(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report([('a', 0, 0, [('a', {'t': 'OK'}, '1s')], 0)], + rd, True, '', '') + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + hil_report.mark_report_no_boards(rd, 'filters emptied', fresh=False) + self.assertIn('abandoned', + json.loads((rd / hil_report.REPORT_JSON).read_text())['caveat']) + + +class TheNoBoardsCallSiteIsWired(unittest.TestCase): + """The fresh/accumulate branches of mark_report_no_boards were tested by calling it + DIRECTLY, so both passed while hil_test.py's one real call site never passed the flag + at all -- an --accumulate run whose filter emptied still wiped the accumulated rows. + This drives hil_test.py itself; the no-boards exit needs only a config and a filter + that matches nothing, so it costs no hardware.""" + + def _run(self, *extra): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / 'cfg.json').write_text(json.dumps( + {'boards': [{'name': 'alpha', 'uid': '1', 'flasher': {'name': 'jlink', 'uid': '2'}}]})) + (rd / hil_report.REPORT_JSON).write_text(json.dumps( + {'rows': [{'board': 'earlier', 'cells': {'t': 'pass'}, 'duration': '1s'}], + 'banner': '', 'scope': '', 'caveat': ''})) + r = subprocess.run( + [sys.executable, str(Path(HIL_DIR) / 'hil_test.py'), + '--flasher', 'nonexistent', *extra, str(rd / 'cfg.json')], + capture_output=True, text=True, timeout=120, + env={**os.environ, 'HIL_REPORT_DIR': str(rd)}) + self.assertEqual(r.returncode, 1, r.stdout + r.stderr) + return json.loads((rd / hil_report.REPORT_JSON).read_text()) + + def test_an_accumulate_run_keeps_the_accumulated_rows(self): + doc = self._run('--accumulate') + self.assertEqual([r['board'] for r in doc['rows']], ['earlier'], + "the call site did not pass fresh=not args.accumulate") + self.assertIn('selected no boards', doc['caveat']) + + def test_a_fresh_run_does_not_republish_them(self): + doc = self._run() + self.assertEqual(doc['rows'], []) + self.assertIn('selected no boards', doc['caveat']) + + +class EveryWriterRendersBeforeItCommits(unittest.TestCase): + def test_accumulate_report_does_not_commit_json_then_fail_to_render(self): + """accumulate_report hand-rolled the write instead of calling write_report, so a + render failure left the sidecar ahead of the markdown -- the exact ordering + write_report's docstring forbids.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_JSON).write_text(json.dumps( + {'rows': [{'board': 'boardA', 'cells': {'t': 'pass'}, 'duration': 119.0}], + 'banner': '', 'caveat': '', 'scope': ''})) + hil_report.accumulate_report([('boardB', 0, 0, [('boardB', {'t': 'OK'}, '1s')], 0)], + rd, False, '', '') + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertEqual((rd / hil_report.REPORT_MD).read_text(), + hil_report.render_report(doc) + '\n') + + +class MissingSidecarDoesNotDestroyTheMarkdown(unittest.TestCase): + def test_an_absent_sidecar_keeps_the_prior_table(self): + """`recovered` was only cleared when the sidecar was TORN, not when it was absent + -- reachable from hil_ci.sh's asymmetric copy-back and build.yml's skip marker.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_MD).write_text('| Board | t |\n| a | OK |\n| b | OK |\n') + hil_report.write_timeout_report(rd, [{'name': 'stuck'}], 3600) + self.assertIn('| a | OK |', (rd / hil_report.REPORT_MD).read_text()) + + +class LoadIsTheOnlyTrustBoundary(unittest.TestCase): + """hil_ci.sh uploads a sidecar as the merge base, so these shapes arrive from OUTSIDE + the harness. Every one of these raised past a handler before.""" + + def _seed(self, rd, raw): + (rd / hil_report.REPORT_JSON).write_text(raw if isinstance(raw, str) + else json.dumps(raw)) + + def test_a_non_list_rows_does_not_raise(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + self._seed(rd, {'rows': 1, 'banner': '', 'caveat': '', 'scope': ''}) + hil_report.accumulate_report([('a', 0, 0, [('a', {'t': 'OK'}, '1s')], 0)], + rd, False, '', '') + self.assertTrue((rd / hil_report.REPORT_MD).is_file()) + + def test_an_unhashable_cell_value_does_not_raise(self): + """render_matrix does REPORT_CELL.get(v, v); an unhashable value raised TypeError + on the NORMAL accumulate path.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + self._seed(rd, {'rows': [{'board': 'a', 'cells': {'t': ['x'], 'u': 'pass'}, + 'duration': '1s'}], + 'banner': '', 'caveat': '', 'scope': ''}) + hil_report.accumulate_report([('b', 0, 0, [('b', {'t': 'OK'}, '1s')], 0)], + rd, False, '', '') + cells = {r['board']: r['cells'] + for r in json.loads((rd / hil_report.REPORT_JSON).read_text())['rows']} + self.assertNotIn('t', cells['a'], 'a corrupt cell must drop, not become a pass') + self.assertIn('u', cells['a']) + + def test_summarize_survives_a_malformed_sidecar_via_load(self): + """The CLI is the one reader an agent's verdict depends on.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / 'cfg.json').write_text(json.dumps({'boards': [{'name': 'a'}]})) + self._seed(rd, {'rows': [{'board': 1, 'cells': 'notadict'}, + {'board': 'a', 'cells': {'t': 'pass'}}], + 'banner': '', 'caveat': '', 'scope': ''}) + r = subprocess.run( + [sys.executable, str(Path(HIL_DIR) / 'helper' / 'hil_report.py'), + str(rd / 'cfg.json'), '-b', 'a', '--report-dir', str(rd)], + capture_output=True, text=True, timeout=60) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertTrue(json.loads(r.stdout)['results'][0]['pass']) + + +class NoBoardsGuardOnlyAppliesWhenAccumulating(unittest.TestCase): + def test_a_fresh_run_carries_nothing_from_the_prior_sidecar(self): + """rows were reset on fresh but banner and scope were not, so a leftover or + uploaded sidecar republished a stale rig-health note and a stale scope line under + this run's notice.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report([('old', 0, 0, [('old', {'t': 'OK'}, '1s')], 0)], + rd, True, '3 board(s) — a, b, c', + '> **Rig note.** stale D-state holder\n') + hil_report.mark_report_no_boards(rd, 'filters emptied', fresh=True) + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertEqual(doc['rows'], []) + self.assertEqual(doc['banner'], '', 'a stale rig-health banner was republished') + self.assertEqual(doc['scope'], '', 'a stale scope note was republished') + self.assertNotIn('Rig note', (rd / hil_report.REPORT_MD).read_text()) + + def test_an_accumulate_run_keeps_banner_and_scope(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report([('old', 0, 0, [('old', {'t': 'OK'}, '1s')], 0)], + rd, True, '3 board(s) — a, b, c', + '> **Rig note.** real\n') + hil_report.mark_report_no_boards(rd, 'filters emptied', fresh=False) + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertIn('Rig note', doc['banner']) + self.assertEqual([r['board'] for r in doc['rows']], ['old']) + + def test_a_fresh_run_is_not_blocked_by_a_prior_abandon(self): + """The guard runs BEFORE the fresh wipe, so guarding a fresh run left the previous + attempt's rows AND its abandon notice published as this run's.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report([('old', 0, 0, [('old', {'t': 'OK'}, '1s')], 0)], + rd, True, '', '') + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + hil_report.mark_report_no_boards(rd, 'filters emptied', fresh=True) + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertEqual(doc['rows'], []) + self.assertIn('selected no boards', doc['caveat']) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/hil/test/test_hil_rtt.py b/test/hil/test/test_hil_rtt.py new file mode 100644 index 000000000..3a07f13ec --- /dev/null +++ b/test/hil/test/test_hil_rtt.py @@ -0,0 +1,506 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests for hil_util.JlinkRtt and the rtt.py CLI against a fake JLinkExe +# on PATH — real subprocesses and sockets, no hardware, stdlib only, so the pre-commit +# hil-test hook can run this on GitHub's bare runner. Run directly: +# python3 test/hil/test/test_hil_rtt.py +import os +import subprocess +import sys +import tempfile +import time +import unittest +from contextlib import suppress as contextlib_suppress +from pathlib import Path + +# the module under test lives in the parent dir's helper/ package +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from helper import hil_util + +CLI = Path(__file__).resolve().parents[3] / 'tools' / 'rtt.py' + +# Serves -RTTTelnetPort like J-Link Commander: greets, echoes input uppercased, exits on +# stdin 'exit' (JlinkRtt.close()'s contract). FAKE_JLINK_MODE=die_after_greet sends the +# greeting then drops the connection and exits — the probe-unplug/crash case; +# FAKE_JLINK_MODE=tick also streams a line every 50 ms — the continuous-capture case. +FAKE_JLINK = '''#!/usr/bin/env python3 +import os, socket, sys, threading, time +port = int(sys.argv[sys.argv.index('-RTTTelnetPort') + 1]) +srv = socket.socket(); srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +srv.bind(('127.0.0.1', port)); srv.listen(1) +mode = os.environ.get('FAKE_JLINK_MODE', '') +def serve(): + conn, _ = srv.accept() + # the real server sends its banner AT CONNECT, before the control block is + # found — target data only flows later; the CLI's -i gate must not release + # on the banner + conn.sendall(b'SEGGER J-Link fake - Real time terminal output\\r\\n' + b'J-Link FakeProbe V1.0, SN=000\\r\\nProcess: JLinkExe\\r\\n') + if mode == 'banner_only': + while True: + if not conn.recv(4096): os._exit(0) + if mode == 'rst': + import struct + conn.recv(4096) # wait for the client to speak, then reset the connection + conn.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) + conn.close(); os._exit(0) + if mode == 'late_cb': + # models JLinkExe before it finds the control block: client bytes sent in + # this window are silently dropped, output starts only after the "attach" + end = time.time() + 1.0 + conn.setblocking(False) + while time.time() < end: + try: + conn.recv(4096) # discard early input like the real server + except OSError: + pass + time.sleep(0.05) + conn.setblocking(True) + conn.sendall(b'hello from target\\r\\n') + if mode == 'die_after_greet': + conn.close(); os._exit(0) + if mode == 'tick': + def tick(): + try: + while True: + time.sleep(0.05); conn.sendall(b'tick\\r\\n') + except OSError: + pass + threading.Thread(target=tick, daemon=True).start() + while True: + d = conn.recv(4096) + if not d: return + conn.sendall(d.upper()) +threading.Thread(target=serve, daemon=True).start() +for line in sys.stdin: + if line.strip() == 'exit': break +''' + +BOARD = {'flasher': {'uid': '000', 'args': '-device FAKE'}} + + [email protected](os.name == 'nt', 'POSIX PATH/exec semantics') +class JlinkRttFakeProbe(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls._dir = tempfile.TemporaryDirectory() + fake = Path(cls._dir.name) / 'JLinkExe' + fake.write_text(FAKE_JLINK) + fake.chmod(0o755) + cls._path = f'{cls._dir.name}{os.pathsep}{os.environ["PATH"]}' + + @classmethod + def tearDownClass(cls): + cls._dir.cleanup() + + def _fake_path(self): + # register the restore BEFORE mutating, then prepend the fake tool dir + self.addCleanup(os.environ.__setitem__, 'PATH', os.environ['PATH']) + os.environ['PATH'] = self._path + + def _console(self, mode=''): + self._fake_path() + if mode: + os.environ['FAKE_JLINK_MODE'] = mode + self.addCleanup(os.environ.pop, 'FAKE_JLINK_MODE', None) + con = hil_util.JlinkRtt(BOARD, timeout=0.1) + self.addCleanup(con.close) + return con + + def _read_until(self, con, want, timeout=3): + out = b'' + end = time.monotonic() + timeout + while want not in out and time.monotonic() < end: + out += con.read(con.in_waiting or 1) + return out + + def test_read_and_echo_write(self): + con = self._console() + self.assertIn(b'hello from target', self._read_until(con, b'hello from target')) + self.assertEqual(con.write(b'ping'), 4) + self.assertIn(b'PING', self._read_until(con, b'PING')) + + def test_eof_latched_when_server_dies(self): + con = self._console(mode='die_after_greet') + self._read_until(con, b'hello from target') + end = time.monotonic() + 3 + while not con.eof and time.monotonic() < end: + time.sleep(0.05) + self.assertTrue(con.eof) # dead server is detected, not spun on + t0 = time.monotonic() + self.assertEqual(con.read(64), b'') # empty, paced like a serial timeout + elapsed = time.monotonic() - t0 + self.assertLess(elapsed, 0.5) # bounded by the 0.1 s timeout, not hung + self.assertGreater(elapsed, 0.02) # ...but not a busy-spin fast return + con.timeout = None # pyserial's block-forever mode must + t0 = time.monotonic() # ALSO pace (0.1 s default), not spin + self.assertEqual(con.read(64), b'') + elapsed = time.monotonic() - t0 + self.assertLess(elapsed, 0.5) + self.assertGreater(elapsed, 0.02) + con.timeout = 0.1 + + def test_reset_input_buffer(self): + con = self._console() + self._read_until(con, b'hello from target') + con.write(b'x') + time.sleep(0.3) + con.reset_input_buffer() + self.assertEqual(con.in_waiting, 0) + + def test_write_after_close_raises_runtimeerror(self): + con = self._console() + con.close() + with self.assertRaises(RuntimeError): + con.write(b'x') + + def test_write_after_server_death_raises(self): + # TCP accepts one send after peer death — write() must refuse instead of + # "succeeding" into the void + con = self._console(mode='die_after_greet') + self._read_until(con, b'hello from target') + end = time.monotonic() + 3 + while not con.eof and time.monotonic() < end: + time.sleep(0.05) + with self.assertRaises(RuntimeError): + con.write(b'ping') + + def test_read_after_close_raises_runtimeerror(self): + con = self._console() + self._read_until(con, b'hello from target') + con.close() + with self.assertRaises(RuntimeError): + con.read(1) + + def test_missing_jlinkexe_raises_runtimeerror(self): + self._fake_path() + os.environ['PATH'] = self._dir.name # no python3 either, but JLinkExe fails first + os.rename(f'{self._dir.name}/JLinkExe', f'{self._dir.name}/JLinkExe.off') + self.addCleanup(os.rename, f'{self._dir.name}/JLinkExe.off', f'{self._dir.name}/JLinkExe') + with self.assertRaises(RuntimeError): + hil_util.JlinkRtt(BOARD, timeout=0.1) + + def test_close_reaps_the_server(self): + con = self._console() + proc = con._proc + con.close() + self.assertIsNotNone(proc.poll()) # no zombie, no probe held + + def test_cli_exits_when_server_dies(self): + # --seconds 0 must end on server EOF (rc 1), not hang forever + env = dict(os.environ, PATH=self._path, FAKE_JLINK_MODE='die_after_greet') + r = subprocess.run([sys.executable, str(CLI), + '--backend', 'jlink', '--probe', '000', '--device', 'FAKE', '--seconds', '0'], + env=env, capture_output=True, timeout=20) + self.assertEqual(r.returncode, 1) + self.assertIn(b'hello from target', r.stdout) + self.assertIn(b'server closed', r.stderr) + + def test_peer_reset_latches_eof(self): + # a killed server closes with RST when bytes are unread; the read side must + # LATCH eof (so the harness's `assert not ser.eof` triage fires) and never + # leak ConnectionResetError/ValueError to in_waiting/eof callers + con = self._console(mode='rst') + # rst mode sends only the banner (it RSTs on first input) -- wait for the + # banner tail, not target output that never comes + self._read_until(con, b'Process: JLinkExe') + con.write(b'x') # fake resets the connection on input + end = time.monotonic() + 3 + try: + while not con.eof and time.monotonic() < end: + con.in_waiting # must not raise across the RST + time.sleep(0.05) + except Exception as e: # noqa: BLE001 - the regression this guards + self.fail(f'{type(e).__name__} escaped the latch-only contract: {e}') + self.assertTrue(con.eof) + with self.assertRaises(hil_util.RttError): + con.write(b'y') # dead server refuses writes + + def test_write_timeout_env_rejects_inf(self): + # hil_util's twin rejects inf for the same reason: an unbounded write is what + # this knob exists to bound + import importlib.util as ilu + from pathlib import Path as _P + spec = ilu.spec_from_file_location('rtt_env_probe', _P(CLI)) + mod = ilu.module_from_spec(spec) + old = os.environ.get('HIL_SERIAL_WRITE_TIMEOUT') + os.environ['HIL_SERIAL_WRITE_TIMEOUT'] = 'inf' + self.addCleanup(lambda: os.environ.__setitem__('HIL_SERIAL_WRITE_TIMEOUT', old) + if old is not None else os.environ.pop('HIL_SERIAL_WRITE_TIMEOUT', None)) + spec.loader.exec_module(mod) + self.assertEqual(mod.RTT_WRITE_TIMEOUT, 10) + + def test_cli_rejects_bad_seconds_and_jlink_channel(self): + def run(*a): + return subprocess.run([sys.executable, str(CLI), *a], capture_output=True, timeout=15) + for bad in ('-5', 'nan'): + r = run('--backend', 'jlink', '--probe', '000', '--device', 'FAKE', '--seconds', bad) + self.assertEqual(r.returncode, 2, f'--seconds {bad} was accepted') + # the jlink telnet route serves channel 0 only; asking for another is an error, + # not silence (--dump can read any ring, so it stays allowed there) + r = run('--backend', 'jlink', '--probe', '000', '--device', 'FAKE', '--channel', '1') + self.assertEqual(r.returncode, 2) + self.assertIn(b'channel 0 only', r.stderr) + # a negative index would walk backwards off aUp[] (dump route included) + r = run('--backend', 'jlink', '--probe', '000', '--device', 'FAKE', '--channel', '-1') + self.assertEqual(r.returncode, 2) + self.assertIn(b'>= 0', r.stderr) + + def test_pyserial_surface_contracts(self): + con = self._console() + self._read_until(con, b'hello from target') + con.write(b'abcdef') + self._read_until(con, b'ABC') # echo queued + before = con.in_waiting + self.assertEqual(con.read(0), b'') # pyserial: consumes nothing + self.assertEqual(con.read(-1), b'') # never hand over/destroy bytes + self.assertEqual(con.in_waiting, before) + con.timeout = None # pyserial: block until satisfied + con.write(b'xy') # fresh echo guarantees the read returns + self.assertEqual(len(con.read(2)), 2) + con.timeout = 0.1 + con.close() + with self.assertRaises(hil_util.RttError): + con.in_waiting # closed console reports closed, not healthy + self.assertTrue(con.eof) + + def test_context_manager_closes(self): + self._fake_path() + with hil_util.JlinkRtt(BOARD, timeout=0.1) as con: + proc = con._proc + self.assertIsNotNone(proc.poll()) # __exit__ released the probe + + def test_staging_and_banner_coupling(self): + # tripwires for couplings no import-walk can see: + # (a) hil_ci.sh must stage tools/rtt.py -- hil_util exec_module's it, so an + # unstaged rig tree kills every harness import + hil_ci = (Path(__file__).resolve().parents[1] / 'hil_ci.sh').read_text() + self.assertIn('tools/rtt.py', hil_ci) + # (b) the shared RTT banner filter must drop ALL THREE J-Link banner lines, + # including the middle one, which is the PROBE MODEL string and in + # libjlinkarm carries no 'SEGGER ' prefix (J-Link OH3, J-Trace H9...) + banner_re = hil_util.RTT_BANNER_RE + for line in ('SEGGER J-Link V9.66 - Real time terminal output', + 'SEGGER J-Link LPC-Link 2 V1.0, SN=611000000', + 'J-Link OH3 V1.0, SN=123456789', + 'J-Trace H9 V2.0, SN=123456789002', + 'Process: JLinkExe'): + self.assertTrue(banner_re.match(line), f'banner line not filtered: {line!r}') + for line in ('Hello from TinyUSB', 'USBD init on controller 0', + 'ID 1a86:8010 SN 7FD88F0604B5', 'echo:p'): + self.assertFalse(banner_re.match(line), f'target line wrongly filtered: {line!r}') + + def test_pool_check_dead_rtt_board_is_not_alive(self): + # JLinkExe's banner alone must not score a dead board 'alive': pool_check's + # rtt aliveness judges only target bytes (the bug: unfiltered, the banner + # made `not boardtest_output(data)` true on the first poll) + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + from helper import hil_pool_check + # a dead board burns the whole poll window; the verdict is the same at 0.5 s + self.addCleanup(setattr, hil_pool_check, 'SERIAL_WAIT', hil_pool_check.SERIAL_WAIT) + hil_pool_check.SERIAL_WAIT = 0.5 + self._fake_path() + os.environ['FAKE_JLINK_MODE'] = 'banner_only' + self.addCleanup(os.environ.pop, 'FAKE_JLINK_MODE', None) + board = dict(BOARD, name='deadboard', logger='rtt') + got = hil_pool_check.check_host_serial(board, do_reset=False, want_hello=True) + self.assertEqual(got, b'') # dead, not "alive on banner" + + def test_cli_arg_contract(self): + # --backend is explicit (no default); vid-pid is openocd-only; the openocd + # backend accepts --addr instead of --elf and --vid-pid instead of --probe + def run(*a, inp=b''): + return subprocess.run([sys.executable, str(CLI), *a], + input=inp, capture_output=True, timeout=15) + r = run('--probe', '000', '--device', 'FAKE') # no --backend + self.assertEqual(r.returncode, 2) + self.assertIn(b'--backend', r.stderr) + r = run('--backend', 'jlink', '--probe', '000', '--device', 'FAKE', '--vid-pid', '0x1 0x2') + self.assertEqual(r.returncode, 2) # vid-pid is openocd-only + r = run('--backend', 'openocd', '--cfg', '-f x.cfg', '--addr', '0x20000000') + self.assertEqual(r.returncode, 2) # needs --probe or --vid-pid + self.assertIn(b'vid-pid', r.stderr) + r = run('--backend', 'openocd', '--probe', '000', '--cfg', '-f x.cfg', '--addr', 'nothex') + self.assertEqual(r.returncode, 2) + self.assertIn(b'hex', r.stderr) + + def test_cli_interactive_echo(self): + env = dict(os.environ, PATH=self._path) + r = subprocess.run([sys.executable, str(CLI), + '--backend', 'jlink', '--probe', '000', '--device', 'FAKE', '--seconds', '2', '-i'], + env=env, input=b'hi', capture_output=True, timeout=20) + self.assertEqual(r.returncode, 0) + self.assertIn(b'HI', r.stdout) # bytes forwarded without needing a newline + self.assertNotIn(b'never forwarded', r.stderr) # forwarding happened: no false alarm + + def test_cli_interactive_input_held_until_output(self): + # input piped at process start must survive the server's control-block hunt + # (the real JLinkExe drops client bytes until the block is found — measured + # on the rig: instant 'ping' lost, delayed 'ping' echoed) + env = dict(os.environ, PATH=self._path, FAKE_JLINK_MODE='late_cb') + r = subprocess.run([sys.executable, str(CLI), + '--backend', 'jlink', '--probe', '000', '--device', 'FAKE', '--seconds', '3', '-i'], + env=env, input=b'hi', capture_output=True, timeout=25) + self.assertEqual(r.returncode, 0) + self.assertIn(b'HI', r.stdout) + + def test_cli_interactive_no_input_diagnostic(self): + # -i with stdin closed immediately: the diagnostic must say stdin was never + # forwarded (true), keyed on actual forwarding -- not on the attach gate, + # which releases after 5 s and forwards anyway on longer runs + env = dict(os.environ, PATH=self._path, FAKE_JLINK_MODE='banner_only') + r = subprocess.run([sys.executable, str(CLI), + '--backend', 'jlink', '--probe', '000', '--device', 'FAKE', '--seconds', '1', '-i'], + env=env, input=b'', capture_output=True, timeout=20) + self.assertEqual(r.returncode, 0) + self.assertIn(b'never forwarded', r.stderr) + self.assertIn(b'no target output', r.stderr) + + def test_cli_downstream_pipe_close(self): + # a real `rtt.py | head`-style consumer: close the read end mid-stream + # and the CLI must exit 0 via its BrokenPipe path, not traceback (this test + # fails if the handler is removed — subprocess.run capture can't cover it) + env = dict(os.environ, PATH=self._path, FAKE_JLINK_MODE='tick') + p = subprocess.Popen([sys.executable, str(CLI), + '--backend', 'jlink', '--probe', '000', '--device', 'FAKE', '--seconds', '8'], + env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p.stdout.read(10) # let it stream a little + p.stdout.close() # downstream hangs up + rc = p.wait(timeout=20) + err = p.stderr.read() + p.stderr.close() + self.assertEqual(rc, 0, err) + self.assertNotIn(b'Traceback', err) + + def test_cli_feeder_races_shutdown(self): + # a feeder still writing when --seconds expires must not crash the CLI + # (pump thread vs close() race: historically tracebacks and SIGABRT rc 134) + env = dict(os.environ, PATH=self._path) + for _ in range(3): + p = subprocess.Popen([sys.executable, str(CLI), + '--backend', 'jlink', '--probe', '000', '--device', 'FAKE', '--seconds', '1', '-i'], + env=env, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE) + try: + while True: + p.stdin.write(b'hi\n') + p.stdin.flush() + time.sleep(0.01) + except (BrokenPipeError, OSError): + pass + rc = p.wait(timeout=20) + err = p.stderr.read() + p.stderr.close() + with contextlib_suppress(OSError, ValueError): + p.stdin.close() + self.assertEqual(rc, 0, err) + self.assertNotIn(b'Exception in thread', err) + + + +class StripBanner(unittest.TestCase): + # both harness consumers (device_info verdict, pool_check aliveness) judge + # target-aliveness through this ONE filter -- pin its shape here + def test_drops_banner_keeps_target(self): + raw = (b'SEGGER J-Link V9.66 - Real time terminal output\r\n' + b'J-Link OH3 V1.0, SN=123456789\r\nProcess: JLinkExe\r\n' + b'Hello from TinyUSB\r\n') + self.assertEqual(hil_util.strip_banner(raw), b'Hello from TinyUSB') + + def test_complete_only_drops_split_banner_fragment(self): + # a poll loop can catch the banner mid-line at a read boundary; the + # fragment must not defeat the prefix regex and score as target output + frag = b'SEGGER J-Link V9.66 - Real time terminal output\r\nProce' + self.assertEqual(hil_util.strip_banner(frag, complete_only=True), b'') + # the final verdict keeps a genuine unterminated target tail + self.assertEqual(hil_util.strip_banner(b'tud_task\r\nrunn'), b'tud_task\nrunn') + self.assertEqual(hil_util.strip_banner(b'', complete_only=True), b'') + + +# Serves like `openocd ... -c "rtt server start PORT CH"`: parses the port from its +# single shell-quoted command line, greets, echoes uppercased. No banner (matches the +# real openocd rtt server, which sends target data only). +FAKE_OPENOCD = '''#!/usr/bin/env python3 +import os, re, socket, sys, threading, time +if os.environ.get('FAKE_OPENOCD_ARGV'): + open(os.environ['FAKE_OPENOCD_ARGV'], 'w').write(' '.join(sys.argv)) +port = int(re.search(r'rtt server start (\\d+)', ' '.join(sys.argv)).group(1)) +srv = socket.socket(); srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +srv.bind(('127.0.0.1', port)); srv.listen(1) +conn, _ = srv.accept() +conn.sendall(b'hello from target\\r\\n') +while True: + d = conn.recv(4096) + if not d: break + conn.sendall(d.upper()) +''' + + [email protected](os.name == 'nt', 'POSIX PATH/exec semantics') +class OpenocdRttFakeProbe(unittest.TestCase): + """The openocd-backend class shares its whole read/write/eof contract with + JlinkRtt via the base class (covered above); this exercises the parts it owns: + spawn/connect, echo round-trip, teardown.""" + + @classmethod + def setUpClass(cls): + cls._dir = tempfile.TemporaryDirectory() + fake = Path(cls._dir.name) / 'openocd' + fake.write_text(FAKE_OPENOCD) + fake.chmod(0o755) + cls._path = f'{cls._dir.name}{os.pathsep}{os.environ["PATH"]}' + + @classmethod + def tearDownClass(cls): + cls._dir.cleanup() + + def _fake_path(self): + self.addCleanup(os.environ.__setitem__, 'PATH', os.environ['PATH']) + os.environ['PATH'] = self._path + + def test_reset_before_attach_shapes_the_command(self): + # SystemView-style consumers need the server draining WHEN the target boots + # (its Init record is emitted once); the opt-in flag must put `reset run` + # between init and rtt setup, and must not appear otherwise + self._fake_path() + argv_file = os.path.join(self._dir.name, 'argv.txt') + os.environ['FAKE_OPENOCD_ARGV'] = argv_file + self.addCleanup(os.environ.pop, 'FAKE_OPENOCD_ARGV', None) + for flag, want in ((True, True), (False, False)): + con = hil_util.OpenocdRtt('-f fake.cfg', 0x20000000, 1, serial_no='000', + reset_before_attach=flag) + try: + argv = Path(argv_file).read_text() + finally: + con.close() + self.assertEqual('reset run' in argv, want, argv) + if want: # ordering is the whole point: reset, settle, THEN attach + self.assertLess(argv.index('reset run'), argv.index('rtt setup'), argv) + self.assertIn('sleep 2000', argv) + self.assertIn('rtt server start', argv) + self.assertTrue(argv.rstrip().endswith('1'), argv) # channel threaded through + + def test_openocd_route_echo_and_teardown(self): + self._fake_path() + con = hil_util.OpenocdRtt('-f fake.cfg', 0x20000000, 0, + serial_no='000', vid_pid='0x1234 0x5678') + self.addCleanup(con.close) + out = b'' + end = time.monotonic() + 3 + while b'hello from target' not in out and time.monotonic() < end: + out += con.read(con.in_waiting or 1) + self.assertIn(b'hello from target', out) + con.write(b'ping') + end = time.monotonic() + 3 + while b'PING' not in out and time.monotonic() < end: + out += con.read(con.in_waiting or 1) + self.assertIn(b'PING', out) + proc = con._proc + con.close() + self.assertIsNotNone(proc.poll()) # no zombie, no probe held + with self.assertRaises(RuntimeError): + con.write(b'x') # same post-close contract as JlinkRtt + + +if __name__ == '__main__': + unittest.main() diff --git a/test/hil/test/test_hil_util.py b/test/hil/test/test_hil_util.py new file mode 100644 index 000000000..17abe52aa --- /dev/null +++ b/test/hil/test/test_hil_util.py @@ -0,0 +1,448 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests for hil_util.run_cmd's binary/split_stderr/quiet modes — real subprocesses, no +# hardware. Stdlib + hil_util only (hil_util is stdlib-only), so the pre-commit hil-test +# hook can run this on GitHub's bare runner. Run directly: +# python3 test/hil/test/test_hil_util.py +import io +import os +import sys +import time +import threading +import unittest +from tempfile import TemporaryDirectory +from contextlib import redirect_stdout +from pathlib import Path + +# the module under test lives in the parent dir's helper/ package +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from helper import hil_util + + +class RunCmdModes(unittest.TestCase): + def test_default_mode_unchanged(self): + r = hil_util.run_cmd('printf out; printf err >&2') + self.assertEqual(r.returncode, 0) + self.assertIsInstance(r.stdout, str) + # stderr merged into stdout, as every existing caller expects + self.assertIn('out', r.stdout) + self.assertIn('err', r.stdout) + + def test_binary_stdout_is_exact_bytes(self): + # \xff is not valid UTF-8: text mode would mangle it via errors='replace' + r = hil_util.run_cmd(r"printf 'a\377\000b'", binary=True) + self.assertEqual(r.returncode, 0) + self.assertEqual(r.stdout, b'a\xff\x00b') + + def test_split_stderr_keeps_stdout_clean(self): + r = hil_util.run_cmd('printf out; printf err >&2', split_stderr=True) + self.assertEqual(r.returncode, 0) + self.assertEqual(r.stdout, 'out') + self.assertEqual(r.stderr, 'err') + + def test_binary_split_stderr_timeout_returns_124(self): + t0 = time.monotonic() + r = hil_util.run_cmd(r"printf 'p\377re'; printf warn >&2; sleep 30", + binary=True, split_stderr=True, timeout=1) + self.assertEqual(r.returncode, 124) + # killpg + bounded communicate: well under sleep 30 + self.assertLess(time.monotonic() - t0, 15) + self.assertIn(b'p\xffre', r.stdout or b'') + # stderr collected before the timeout must survive the kill + self.assertIn(b'warn', r.stderr or b'') + + def test_text_mode_timeout_stdout_stays_str(self): + r = hil_util.run_cmd('sleep 30', timeout=1) + self.assertEqual(r.returncode, 124) + # a text-mode caller must never get bytes back, even empty + self.assertIsInstance(r.stdout, str) + + def test_failed_banner_includes_split_stderr(self): + # with split_stderr the diagnostic is in .stderr; the banner must not go blank. + # The text travels via env, not the command string — the banner title echoes the + # command, which would make a literal assertion pass vacuously. + os.environ['RUN_CMD_TEST_ERR'] = 'diagnostic-xyzzy' + self.addCleanup(os.environ.pop, 'RUN_CMD_TEST_ERR', None) + cap = io.StringIO() + with redirect_stdout(cap): + r = hil_util.run_cmd('printf "$RUN_CMD_TEST_ERR" >&2; exit 3', split_stderr=True) + self.assertEqual(r.returncode, 3) + self.assertIn('COMMAND FAILED', cap.getvalue()) + self.assertIn('diagnostic-xyzzy', cap.getvalue()) + + def test_no_group_markers_when_stdout_is_captured(self): + # GitHub folds ::group:: only at line start of the JOB's real stdout. Pool + # workers run tests under redirect_stdout and compact the capture into one + # row line, where the markers land mid-line and render as literal noise. + saved_ci = os.environ.get('CI') # pre-exists on GitHub runners: restore, not pop + os.environ['CI'] = '1' + self.addCleanup(lambda: os.environ.update({'CI': saved_ci}) if saved_ci is not None + else os.environ.pop('CI', None)) + cap = io.StringIO() + with redirect_stdout(cap): + r = hil_util.run_cmd('printf boom; exit 3') + self.assertEqual(r.returncode, 3) + self.assertIn('COMMAND FAILED', cap.getvalue()) + self.assertNotIn('::group::', cap.getvalue()) + self.assertNotIn('::endgroup::', cap.getvalue()) + + def test_quiet_suppresses_failed_banner(self): + # retry-loop callers report failures themselves; per-poll banners are noise + cap = io.StringIO() + with redirect_stdout(cap): + r = hil_util.run_cmd('printf boom >&2; exit 3', quiet=True) + self.assertEqual(r.returncode, 3) + self.assertNotIn('COMMAND FAILED', cap.getvalue()) + + +class BottomLayer(unittest.TestCase): + def test_bad_timeout_env_falls_back(self): + # ci_select (the PR-diff selector) imports hil_util for the example rosters; + # a malformed HIL_CMD_TIMEOUT must not crash the selector at import and knock + # CI back to the full-matrix fallback + import subprocess + r = subprocess.run( + [sys.executable, '-c', 'from helper import hil_util; print(hil_util.CMD_TIMEOUT)'], + cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + env={**os.environ, 'HIL_CMD_TIMEOUT': 'bogus'}, + capture_output=True, text=True, timeout=30) + self.assertEqual(r.returncode, 0, r.stderr) + # the warning must NOT be on stdout: ci_select's stdout is machine-read JSON + self.assertEqual(r.stdout.strip(), '180') + self.assertIn('warning', r.stderr) # but a silent fallback hides the misconfiguration + + def test_tinyusb_root_is_the_repo_root(self): + # the constant is derived from __file__ parents[N]; moving hil_util.py without + # adjusting N silently re-points every firmware/build path (it happened) + self.assertTrue((hil_util.TINYUSB_ROOT / 'examples').is_dir(), hil_util.TINYUSB_ROOT) + self.assertTrue((hil_util.TINYUSB_ROOT / 'test' / 'hil').is_dir(), hil_util.TINYUSB_ROOT) + + def test_hil_util_is_a_single_module_instance(self): + # helper modules must be imported via the helper package everywhere: a plain + # `import hil_util` from inside helper/ creates a SECOND module object, and + # state like `verbose` set on one copy never reaches the other + import hil_flash + from helper import hil_pool_check + self.assertIs(hil_flash.hil_util, hil_util) + self.assertIs(hil_pool_check.hil_util, hil_util) + self.assertIs(hil_pool_check.hil_flash, hil_flash) + + def test_bare_runner_modules_stay_stdlib_only(self): + # hil_examples.py used to make this structural (a list of strings cannot grow a + # dependency); with the rosters folded into hil_util the invariant needs teeth: + # everything the bare GitHub runner imports (selector + this suite) must stay + # stdlib + local. Adding pyserial/pymtp here breaks ci_select on CI. + import ast + hil_dir = Path(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + # ONLY the modules the bare runner can import -- not every stem in the tree. + # Globbing the directory allowed `import pymtp` (and hil_test, usbtest, + # mtp_test) through, so the pymtp case this test names could never fail: that + # module runs ctypes.CDLL(find_library('mtp')) at import and raises where there + # is no libmtp, taking ci_select down with it. + local = {'helper', 'hil_util', 'ci_select', 'hil_flash', + 'hil_health', 'hil_lock', 'hil_pool_check', 'build', 'build_utils'} + allowed = set(sys.stdlib_module_names) | local + # hil_pool_check included: test_hil_util_is_a_single_module_instance imports it + # on the bare runner, and its `import serial` is function-local for exactly + # this reason -- hoisting it must fail HERE, not on every PR's pre-commit CI + # ../../tools/rtt: hil_util exec_module's it at import (helper/hil_util.py's + # loader block), so a non-stdlib import THERE kills ci_select on the bare + # runner just as surely -- and the spec_from_file_location call is invisible to + # the ast.Import walk below, which is why it must be listed explicitly + for mod in ('helper/hil_util', 'hil_flash', '../../tools/ci_select', + 'helper/hil_health', 'helper/hil_lock', 'helper/hil_pool_check', + '../../tools/build', '../../tools/build_utils', '../../tools/rtt'): + tree = ast.parse((hil_dir / f'{mod}.py').read_text()) + # module level only: a deferred import inside a function cannot break + # importability (hil_pool_check keeps `import serial` function-local + # for exactly that reason) + for node in tree.body: + roots = [] + if isinstance(node, ast.Import): + roots = [a.name.split('.')[0] for a in node.names] + elif isinstance(node, ast.ImportFrom) and node.module: + roots = [node.module.split('.')[0]] + for root in roots: + self.assertIn(root, allowed, + f'{mod}.py imports {root}, not stdlib/local - breaks the bare CI runner') + + +class RunAlongsideKeepsStderrOffThePayload(unittest.TestCase): + """test_device_printer_to_cdc byte-compares run_alongside's stdout against the payload + it wrote. Merging stderr into that stream turns any stray child stderr byte -- a + PYTHONWARNINGS chirp, a sitecustomize print, a venv .pth deprecation -- into + 'CDC->Printer wrong data', sending a maintainer after the printer class driver for an + interpreter warning. hil_ci.sh runs python3 with no isolating flags.""" + + def test_child_stderr_does_not_contaminate_stdout(self): + from helper import hil_util + argv = [sys.executable, '-c', + 'import sys; sys.stderr.write("noise\\n"); sys.stdout.write("PAYLOAD")'] + r = hil_util.run_alongside(argv, lambda: time.sleep(0.2), timeout=20) + self.assertEqual(r.returncode, 0) + self.assertEqual(r.stdout, b'PAYLOAD', + 'child stderr leaked into the payload stream') + + +class RunCmdCleanupShape(unittest.TestCase): + """run_cmd's two cleanup paths, asserted structurally. + + Both must kill the process GROUP: start_new_session puts the child in its own group, so + a flasher run through a shell keeps children a p.kill() cannot reach, and on the + BaseException path the child never receives the terminal's SIGINT either. + + Structural rather than behavioural on purpose. Driving a real SIGINT into a blocked + communicate() from a unit test is timing-dependent, and a flaky guard on this block is + worse than none -- while what actually breaks it is an edit that rebinds a branch. Both + times this block has been mis-edited, an `else:` ended up attached to the `try` instead + of the `if` it belonged to, so `p.kill()` ran when killpg had SUCCEEDED and its + ProcessLookupError masked the caller's exception. That is a shape, and shapes are + exactly what an AST can pin. + """ + + def _run_cmd_ast(self): + import ast + src = Path(hil_util.__file__).read_text() + return next(n for n in ast.walk(ast.parse(src)) + if isinstance(n, ast.FunctionDef) and n.name == 'run_cmd') + + def test_no_cleanup_try_has_an_else(self): + import ast + for n in ast.walk(self._run_cmd_ast()): + if isinstance(n, ast.Try) and n.orelse: + self.fail(f'try/else at line {n.lineno}: an else here runs when the kill ' + f'SUCCEEDED, and its ProcessLookupError masks the caller\'s ' + f'exception -- this block has been mis-edited that way twice') + + def test_both_cleanup_paths_kill_the_group(self): + import ast + fn = self._run_cmd_ast() + killers = [getattr(c.func, 'attr', '') for c in ast.walk(fn) + if isinstance(c, ast.Call) and getattr(c.func, 'attr', '') in + ('killpg', 'kill')] + self.assertEqual(killers.count('killpg'), 2, + 'both the timeout and the BaseException path must killpg') + self.assertEqual(killers.count('kill'), 0, + 'p.kill() reaches only the direct child; a flasher run through a ' + 'shell keeps grandchildren it cannot touch') + + def test_the_interrupt_path_reraises(self): + import ast + fn = self._run_cmd_ast() + base = [h for n in ast.walk(fn) if isinstance(n, ast.Try) for h in n.handlers + if isinstance(h.type, ast.Name) and h.type.id == 'BaseException'] + self.assertTrue(base, 'the BaseException cleanup path is gone') + for h in base: + self.assertTrue(any(isinstance(x, ast.Raise) for x in ast.walk(h)), + 'the interrupt path must re-raise, or Ctrl-C is swallowed') + + +class BoundedReadForGuardlessCallers(unittest.TestCase): + """`serial` is served under the device lock a wedged usbfs ioctl holds, so the read is + bounded BY DEFAULT -- not opt-in. usb_scan reads it on every device matching the VID to + find the one it wants, and hil_lock.controller_of does that from controller_permit on + essentially every board, so one wedged DUT would stall every worker rather than one. + hil_pool_check has no guard behind it at all.""" + + def setUp(self): + from helper import hil_util + self.hil_util = hil_util + # the pre-commit hook runs all four suites in ONE interpreter, so capture and + # restore rather than assuming these start (or end) empty + for name in ('_stranded', '_strand_hits'): + self.addCleanup(setattr, hil_util, name, dict(getattr(hil_util, name))) + getattr(hil_util, name).clear() + self.addCleanup(setattr, hil_util, '_ever_stranded', hil_util._ever_stranded) + hil_util._ever_stranded = False + self.td = TemporaryDirectory() + self.addCleanup(self.td.cleanup) + self.fifo = os.path.join(self.td.name, 'serial') + os.mkfifo(self.fifo) # a read that never answers + + def test_a_wedged_attribute_gives_up_instead_of_hanging(self): + t0 = time.monotonic() + self.assertIsNone(self.hil_util.read_sysfs(self.fifo, timeout=0.3)) + self.assertLess(time.monotonic() - t0, 5, 'the bounded read did not give up') + + def test_the_bound_is_the_default_not_an_opt_in(self): + """usb_scan reads `serial` on every device matching the VID to find the one it + wants, and hil_lock's controller_of does that from controller_permit on + essentially every board -- so an opt-in bound that ONE call site forgets lets a + single wedged DUT stall every worker, not one. Three call sites forgot it once.""" + import inspect + for fn in (self.hil_util.read_sysfs, self.hil_util.usb_scan): + default = inspect.signature(fn).parameters['timeout'].default + self.assertEqual(default, self.hil_util.SYSFS_READ_GRACE, + f'{fn.__name__} must be bounded without being asked') + t0 = time.monotonic() + self.assertIsNone(self.hil_util.read_sysfs(self.fifo)) # no timeout= passed + self.assertLess(time.monotonic() - t0, 5, 'the default path did not bound') + + def test_a_node_that_returns_during_the_grace_is_not_memoised_as_wedged(self): + """The inode must be captured BEFORE the reader starts. Stat it afterwards and a + board that came back mid-read has its brand-new HEALTHY inode recorded as the + wedged one -- only a SECOND re-enumeration could ever clear it, and hil_pool_check + would report a successful recovery as still off the bus.""" + def swap(): + time.sleep(0.15) + os.unlink(self.fifo) + Path(self.fifo).write_text('CAFE01\n') + + threading.Thread(target=swap, daemon=True).start() + self.hil_util.read_sysfs(self.fifo, timeout=0.6) + self.assertEqual(self.hil_util.read_sysfs(self.fifo, timeout=1), 'CAFE01', + 'the healthy new inode was recorded as the wedged one') + + def test_concurrent_readers_of_one_path_spend_one_credit(self): + """hil_pool_check polls one bus from four threads. Counting each READER let four + threads on ONE wedged device spend four of the process budget between them -- + latching on the single wedge the tool was run to find.""" + ts = [threading.Thread(target=lambda: self.hil_util.read_sysfs(self.fifo, timeout=0.3)) + for _ in range(4)] + [t.start() for t in ts] + [t.join() for t in ts] + self.assertEqual(len(self.hil_util._stranded), 1) + self.assertEqual(self.hil_util._strand_hits[self.fifo], 1, + 'four readers of one path spent four credits') + + def test_a_flapping_wedged_device_cannot_leak_without_bound(self): + """The inode all-clear re-arms on every re-enumeration, so a device that flaps + while STILL wedged strands again each pass -- a thread and an fd per cycle.""" + for _ in range(self.hil_util._PATH_STRAND_MAX + 4): + self.hil_util.read_sysfs(self.fifo, timeout=0.2) + os.unlink(self.fifo) + os.mkfifo(self.fifo) # back on the same path, still wedged + self.assertEqual(self.hil_util._strand_hits[self.fifo], + self.hil_util._PATH_STRAND_MAX, + 'a flapping device kept stranding past its per-path cap') + + def test_a_value_that_arrived_at_the_deadline_is_not_a_strand(self): + """`out` is checked BEFORE is_alive(): a reader can deposit its value and still be + alive for a moment after join() returns. Counting that as a strand blacklists a + healthy attribute by inode forever AND latches sysfs_stranded for the process.""" + good = Path(self.td.name) / 'idVendor' + good.write_text('cafe\n') + real_thread = threading.Thread + + class Lingering(real_thread): # deposits, then outlives the join + def run(self): + super().run() + time.sleep(2) + + self.hil_util.threading.Thread = Lingering + self.addCleanup(setattr, self.hil_util.threading, 'Thread', real_thread) + self.assertEqual(self.hil_util.read_sysfs(str(good), timeout=0.3), 'cafe') + self.assertNotIn(str(good), self.hil_util._stranded) + self.assertFalse(self.hil_util.sysfs_stranded()) + + def test_path_stranded_answers_per_device_not_per_process(self): + """usbtest decides whether to run lock-taking cleanup on this result; the sticky + process-wide flag would let any peer's wedge answer for our board.""" + other = Path(self.td.name) / 'peer' + other.write_text('PEER\n') + self.hil_util.read_sysfs(self.fifo, timeout=0.3) + self.assertTrue(self.hil_util.path_stranded(self.fifo)) + self.assertFalse(self.hil_util.path_stranded(str(other))) + self.assertTrue(self.hil_util.sysfs_stranded(), 'the process-wide flag is sticky') + + def test_a_refused_read_is_stranded_not_vouched_for(self): + """usbtest fails CLOSED on path_stranded() before running remove_id/unbind, which + take the uninterruptible device_lock. Past _STRAND_MAX read_sysfs answers None + WITHOUT looking -- so answering False there hands that guard a fabricated + all-clear for a device nobody read, and the lock-taking cleanup runs on a wedge.""" + self.hil_util._stranded.update( + {f'/sys/fake/{i}': i for i in range(self.hil_util._STRAND_MAX)}) + self.assertIsNone(self.hil_util.read_sysfs(self.fifo, timeout=0.3)) + self.assertTrue(self.hil_util.path_stranded(self.fifo), + 'a path the reader refused to open was reported readable-and-absent') + + def test_a_stat_that_races_the_reader_still_memoises(self): + """The pre-read stat is the memo KEY, and it can fail while the open that follows + succeeds and blocks -- a node replaced between the two. Without a key the give-up + records nothing, so hil_pool_check's next poll starts another permanent thread and + fd for the same path, and repeats it every pass.""" + real_stat = self.hil_util.os.stat + calls = [] + + def flaky(path, *a, **kw): + calls.append(path) + if len(calls) == 1: # only the pre-read stat loses the race + raise OSError('vanished between stat and open') + return real_stat(path, *a, **kw) + + self.addCleanup(setattr, self.hil_util.os, 'stat', real_stat) + self.hil_util.os.stat = flaky + self.assertIsNone(self.hil_util.read_sysfs(self.fifo, timeout=0.3)) + self.assertIn(self.fifo, self.hil_util._stranded, + 'a lost stat race leaks a fresh reader on every later poll') + + def test_a_successful_read_clears_an_earlier_refusal(self): + """_refused feeds path_stranded(), which usbtest reads to tell "cannot tell" from + a real disconnect. Left sticky, a board that recovered and then genuinely left the + bus is classified as an unrecovered wedge for the rest of the process.""" + good = Path(self.td.name) / 'serial2' + good.write_text('ABC123\n') + self.hil_util._refused.add(str(good)) + self.addCleanup(self.hil_util._refused.discard, str(good)) + self.assertEqual(self.hil_util.read_sysfs(str(good), timeout=0.3), 'ABC123') + self.assertFalse(self.hil_util.path_stranded(str(good)), + 'a path that answered is still reported unreadable') + + def test_a_recovered_device_is_seen_again_on_the_same_busport(self): + """THE recovery flow: hil_pool_check resets or reflashes a wedged board, then + wait_device polls find_device -> scan_usb for the NEW inode. A busport does not + change when the board comes back on the same physical port, so a path-only + blacklist would make that poll look at everything except the device it is waiting + for -- the board recovers physically and the tool reports it gone for the rest of + the run. A re-enumeration destroys the kernfs node, so a changed inode is the + all-clear.""" + self.assertIsNone(self.hil_util.read_sysfs(self.fifo, timeout=0.3)) + # re-enumeration: same path, new node + os.unlink(self.fifo) + Path(self.fifo).write_text('CAFE01\n') + self.assertEqual(self.hil_util.read_sysfs(self.fifo, timeout=1), 'CAFE01', + 'a board that came back on the same busport stayed blacklisted') + + def test_the_caveat_stays_true_after_a_recovery(self): + """Rows collected while the device was unreadable keep whatever they said, so the + footer must still warn even once the memo has cleared.""" + self.hil_util.read_sysfs(self.fifo, timeout=0.3) + os.unlink(self.fifo) + Path(self.fifo).write_text('CAFE01\n') + self.hil_util.read_sysfs(self.fifo, timeout=1) + self.assertTrue(self.hil_util.sysfs_stranded()) + + def test_a_stranded_path_is_never_read_twice(self): + """Each expiry strands a thread and an fd for the life of the process, and + hil_pool_check POLLS -- wait_device re-scans every 0.5s until its budget runs + out. Re-reading would leak one pair per poll.""" + self.hil_util.read_sysfs(self.fifo, timeout=0.3) + t0 = time.monotonic() + for _ in range(5): + self.assertIsNone(self.hil_util.read_sysfs(self.fifo, timeout=0.3)) + self.assertLess(time.monotonic() - t0, 0.3, + 'repeat reads of a known-stranded path paid the grace again') + + def test_the_caller_can_say_the_table_may_be_wrong(self): + self.assertFalse(self.hil_util.sysfs_stranded()) + self.hil_util.read_sysfs(self.fifo, timeout=0.3) + self.assertTrue(self.hil_util.sysfs_stranded(), + 'nothing would tell the operator a missing row may be this tool ' + 'losing sight of healthy hardware') + + def test_a_healthy_attribute_is_not_blacklisted(self): + good = os.path.join(self.td.name, 'idVendor') + Path(good).write_text('cafe\n') + for _ in range(3): + self.assertEqual(self.hil_util.read_sysfs(good, timeout=1), 'cafe') + self.assertFalse(self.hil_util.sysfs_stranded()) + + def test_without_a_timeout_the_read_stays_plain(self): + good = os.path.join(self.td.name, 'busnum') + Path(good).write_text('3\n') + self.assertEqual(self.hil_util.read_sysfs(good), '3') + self.assertIsNone(self.hil_util.read_sysfs(os.path.join(self.td.name, 'nope'))) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/hil/test_hil_select.py b/test/hil/test_hil_select.py deleted file mode 100644 index 5e6b16759..000000000 --- a/test/hil/test_hil_select.py +++ /dev/null @@ -1,581 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MIT -# Unit tests for hil_select.py — pure logic, no hardware, no git. Run directly: -# python3 test/hil/test_hil_select.py -import glob -import json -import os -import sys -import unittest - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import hil_flash -import hil_select -from hil_examples import device_tests, dual_tests, host_test - -REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - - -def real_rosters(): - """The actual rig rosters, for regression tests that need real-world data - (a specific board/family/only-list) rather than the synthetic ROSTER above.""" - rosters = [] - for name in ('tinyusb.json', 'hfp.json'): - path = os.path.join(REPO, 'test/hil', name) - with open(path) as f: - rosters.append((f'test/hil/{name}', json.load(f)['boards'])) - return rosters - - -def roster_flashers(): - """(roster path, board) for every board in the live rosters, `boards-skip` - included: a parked board's flasher name must still dispatch, so that unparking it - is not what discovers the name went stale.""" - for name in ('tinyusb.json', 'hfp.json'): - path = os.path.join(REPO, 'test/hil', name) - with open(path) as f: - cfg = json.load(f) - for key in ('boards', 'boards-skip'): - for b in cfg.get(key, []): - yield f'test/hil/{name}', b - - -def on_roster(tc, *names): - """The subset of `names` currently in the live rig rosters, skipping the test - when none are. Parking/unparking a board is routine rig maintenance and must not - fail this suite: CI runs it right before the selector and treats a failure as - 'selector unusable', dropping PR scoping and annotating the run.""" - have = {b['name'] for _, boards in real_rosters() for b in boards} - got = [n for n in names if n in have] - if not got: - tc.skipTest(f'not in the rig roster: {", ".join(names)}') - return got - - -ROSTER = [ - # device-only, rp2040 family - {'name': 'raspberry_pi_pico', 'uid': 'u1', 'flasher': {'name': 'openocd'}, - 'tests': {'device': True, 'host': True, 'dual': True}}, - # device-only, stm32f4 family - {'name': 'stm32f407disco', 'uid': 'u2', 'flasher': {'name': 'jlink'}, - 'tests': {'device': True, 'host': False, 'dual': False}}, - # host-only board - {'name': 'raspberry_pi_pico2', 'uid': 'u3', 'flasher': {'name': 'openocd'}, - 'tests': {'device': False, 'host': True, 'dual': False}}, - # only-list board (espressif-style), flashed by the CI leg that splits on esptool - {'name': 'espressif_s3_devkitm', 'uid': 'u4', 'flasher': {'name': 'esptool'}, - 'tests': {'only': ['device/cdc_msc_freertos', 'host/device_info']}}, -] -ROSTERS = [('test/hil/tinyusb.json', ROSTER)] - - -def sel(files): - return hil_select.classify(files, REPO, ROSTERS) - - -class TestPortRule(unittest.TestCase): - def test_dcd_rp2040_selects_pico_family_only(self): - s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) - self.assertFalse(s['full']) - self.assertIn('raspberry_pi_pico', s['boards']) - self.assertNotIn('stm32f407disco', s['boards']) - self.assertNotIn('espressif_s3_devkitm', s['boards']) - # device role: no host tests in pico's list - self.assertTrue(all(not t.startswith('host/') for t in s['boards']['raspberry_pi_pico'])) - # host-only boards drop out entirely on a device-role change - self.assertNotIn('raspberry_pi_pico2', s['boards']) - - def test_shared_port_file_is_both_roles(self): - s = sel(['src/portable/synopsys/dwc2/dwc2_common.c']) - self.assertFalse(s['full']) - self.assertNotIn('raspberry_pi_pico', s['boards']) # rp2040 is not a dwc2 family - self.assertIn('stm32f407disco', s['boards']) # stm32f4 is - - -class TestCoreRoleRule(unittest.TestCase): - def test_usbd_selects_all_device_tests_everywhere(self): - s = sel(['src/device/usbd.c']) - self.assertFalse(s['full']) - self.assertNotIn('raspberry_pi_pico2', s['boards']) # host-only board dropped - pico = s['boards']['raspberry_pi_pico'] - self.assertTrue(set(device_tests).issubset(set(pico))) - self.assertTrue(set(dual_tests).issubset(set(pico))) # dual survives device role - self.assertTrue(all(not t.startswith('host/') for t in pico)) - # only-list board: selection intersects its only-list - esp = s['boards']['espressif_s3_devkitm'] - self.assertEqual(esp, ['device/cdc_msc_freertos']) - - def test_host_change_drops_device(self): - s = sel(['src/host/usbh.c']) - self.assertFalse(s['full']) - self.assertIn('raspberry_pi_pico2', s['boards']) - self.assertNotIn('stm32f407disco', s['boards']) # device-only board dropped - - -class TestClassRule(unittest.TestCase): - def test_cdc_device_selects_cdc_examples_only(self): - s = sel(['src/class/cdc/cdc_device.c']) - self.assertFalse(s['full']) - pico = s['boards']['raspberry_pi_pico'] - self.assertIn('device/cdc_msc', pico) - self.assertIn('device/cdc_dual_ports', pico) - self.assertNotIn('device/msc_dual_lun', pico) # CFG_TUD_CDC 0 there - self.assertNotIn('device/usbtest', pico) # CFG_TUD_CDC 0 there - self.assertTrue(all(not t.startswith('host/') for t in pico)) - - def test_msc_host_selects_host_side(self): - s = sel(['src/class/msc/msc_host.c']) - self.assertFalse(s['full']) - self.assertNotIn('stm32f407disco', s['boards']) # device-only board - pico2 = s['boards']['raspberry_pi_pico2'] - self.assertIn('host/msc_file_explorer', pico2) - self.assertTrue(all(not t.startswith('device/') for t in pico2)) - - -class TestClassIncludeEdges(unittest.TestCase): - """A class header another class includes reaches that class's examples too. - src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, so - midi_test's firmware contains audio.h - but the class rule derives macros from - the directory name alone, so an audio.h change used to select only - device/audio_test_freertos. On boards that skip that example the per-board - intersection emptied and an audio.h-only PR ran ZERO HIL on them.""" - def test_edges_derived_from_includes(self): - edges = hil_select.class_include_edges(REPO) - self.assertEqual(edges.get('audio/audio.h'), {'midi'}) - self.assertEqual(edges.get('cdc/cdc.h'), {'net'}) - - def test_audio_header_selects_midi_example(self): - s = hil_select.classify(['src/class/audio/audio.h'], REPO, real_rosters()) - self.assertFalse(s['full']) - # every board that runs device/midi_test at all must run it here (boards with - # a tests.only list, e.g. espressif, run the freertos examples instead) - by_name = {b['name']: b for _, bs in real_rosters() for b in bs} - checked = 0 - for name, tests in s['boards'].items(): - if 'device/midi_test' in hil_select.board_tests(by_name[name]): - self.assertIn('device/midi_test', tests, name) - checked += 1 - self.assertTrue(checked) - - def test_audio_header_reaches_boards_that_skip_audio(self): - # both skip device/audio_test_freertos: without the midi edge their - # intersection is empty and they drop out of the selection entirely - boards = on_roster(self, 'metro_m4_express', 'nrf54lm20dk') - s = hil_select.classify(['src/class/audio/audio.h'], REPO, real_rosters()) - for board in boards: - self.assertEqual(s['boards'].get(board), ['device/midi_test'], board) - - def test_edge_is_per_header_not_per_class(self): - # midi includes audio.h, not audio_device.h: an audio_device change must - # not drag midi's examples in - s = hil_select.classify(['src/class/audio/audio_device.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for tests in s['boards'].values(): - if tests != 'all': - self.assertNotIn('device/midi_test', tests) - - -class TestFallbackRules(unittest.TestCase): - def test_unknown_tool_is_full(self): - s = sel(['tools/random_new_script.py']) - self.assertTrue(s['full']) - - def test_docs_only_is_empty_not_full(self): - s = sel(['docs/info/contributing.rst', 'README.rst']) - self.assertFalse(s['full']) - self.assertEqual(s['boards'], {}) - - def test_bsp_family_selects_family_boards(self): - s = sel(['hw/bsp/rp2040/family.cmake']) - self.assertFalse(s['full']) - self.assertIn('raspberry_pi_pico', s['boards']) - self.assertEqual(s['boards']['raspberry_pi_pico'], 'all') - self.assertNotIn('stm32f407disco', s['boards']) - - def test_bsp_board_narrows_to_board(self): - s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) - self.assertFalse(s['full']) - self.assertEqual(list(s['boards'].keys()), ['raspberry_pi_pico']) - - def test_example_change_selects_that_example(self): - s = sel(['examples/device/cdc_msc/src/main.c']) - self.assertFalse(s['full']) - self.assertEqual(s['boards']['raspberry_pi_pico'], ['device/cdc_msc']) - - def test_core_common_is_full(self): - for f in ['src/tusb.c', 'src/common/tusb_fifo.c', 'src/osal/osal_freertos.h']: - self.assertTrue(sel([f])['full'], f) - - def test_board_test_example_is_full(self): - # board_test is the park/teardown firmware hil_test.py flashes on every board, - # not an unlisted example: a regression there must not skip the whole rig - for f in ['examples/device/board_test/src/main.c', - 'examples/device/board_test/CMakeLists.txt']: - self.assertTrue(sel([f])['full'], f) - - def test_harness_is_full(self): - for f in ['test/hil/hil_test.py', '.github/workflows/build.yml', 'hw/mcu/nxp/x.c', 'lib/foo/x.c']: - self.assertTrue(sel([f])['full'], f) - - def test_mixed_roles_no_pruning(self): - s = sel(['src/device/usbd.c', 'src/host/usbh.c']) - self.assertFalse(s['full']) - self.assertIn('raspberry_pi_pico2', s['boards']) - self.assertIn('stm32f407disco', s['boards']) - - def test_cmakelists_and_requirements_are_full(self): - for f in ['src/CMakeLists.txt', 'examples/CMakeLists.txt', - 'examples/device/CMakeLists.txt', 'test/hil/requirements.txt']: - self.assertTrue(sel([f])['full'], f) - - def test_docs_txt_is_noncode(self): - s = sel(['docs/info/changelog.txt']) - self.assertFalse(s['full']) - self.assertEqual(s['boards'], {}) - - -class TestArgsEmission(unittest.TestCase): - def test_args_for_scoped_selection(self): - s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) - args = hil_select.selection_args(s, ROSTERS) - a = args['tinyusb.json'] - self.assertIn('-b raspberry_pi_pico', a) - self.assertNotIn('stm32f407disco', a) - self.assertIn('-bt raspberry_pi_pico:', a) # device-only subset of a device+host board - - def test_args_full_is_empty(self): - s = sel(['tools/random_new_script.py']) - self.assertEqual(hil_select.selection_args(s, ROSTERS), {'tinyusb.json': ''}) - - def test_args_all_board_gets_bare_b(self): - s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) - a = hil_select.selection_args(s, ROSTERS)['tinyusb.json'] - self.assertIn('-b raspberry_pi_pico', a) - self.assertNotIn('-bt', a) - - def test_args_by_flasher_splits_esp_from_the_rest(self): - s = sel(['src/device/usbd.c']) - per = hil_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json'] - self.assertIn('espressif_s3_devkitm', per['esptool']) - self.assertIn('raspberry_pi_pico', per['openocd']) - self.assertNotIn('espressif_s3_devkitm', per.get('openocd', '') + per.get('jlink', '')) - - def test_args_by_flasher_omits_a_flasher_with_no_selected_board(self): - # the esp CI leg must see no args at all here, not a filter matching zero boards - s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) - per = hil_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json'] - self.assertEqual(per, {'openocd': '-b raspberry_pi_pico'}) - - def test_args_by_flasher_full_is_empty(self): - s = sel(['tools/random_new_script.py']) - self.assertEqual(hil_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}}) - - def test_cli_diff_file(self): - import subprocess, tempfile, json as j - with tempfile.NamedTemporaryFile('w', suffix='.txt', delete=False) as f: - f.write('src/class/cdc/cdc_device.c\n') - path = f.name - r = subprocess.run([sys.executable, os.path.join(REPO, 'test/hil/hil_select.py'), - '--diff-file', path, os.path.join(REPO, 'test/hil/tinyusb.json')], - capture_output=True, text=True) - self.assertEqual(r.returncode, 0, r.stderr) - out = j.loads(r.stdout) - self.assertFalse(out['full']) - self.assertIn('tinyusb.json', out['args']) - self.assertTrue(any('cdc_device' in line for line in out['reasons'])) - os.unlink(path) - - -class TestRealRosterPortFamilies(unittest.TestCase): - """Regression for port_families() missing espressif's dwc2 reference, which - lives in a component CMakeLists.txt rather than family.cmake/family.mk.""" - def test_dwc2_change_selects_espressif_boards(self): - boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') - s = hil_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - self.assertIn(board, s['boards']) - - -class TestOptionGatedPort(unittest.TestCase): - """Regression: family_support.cmake compiles some ports from a build option - (MAX3421_HOST=1 -> hcd_max3421.c), so a board's family file never names them.""" - # host-side option board (max3421 as host controller), off any max3421 family - OPT_ROSTER = [('test/hil/opt.json', [ - {'name': 'fake_dual_board', 'uid': 'o1', 'flasher': {'name': 'jlink'}, - 'build': {'args': ['MAX3421_HOST=1']}, - 'tests': {'device': True, 'host': False, 'dual': True}}, - {'name': 'fake_host_board', 'uid': 'o2', 'flasher': {'name': 'jlink'}, - 'variant': [{'name': 'fake_host_board', 'flags': '-DMAX3421_HOST=1'}], - 'tests': {'device': False, 'host': True, 'dual': False}}, - {'name': 'fake_off_board', 'uid': 'o3', 'flasher': {'name': 'jlink'}, - 'variant': [{'name': 'fake_off_board', 'defines': ['MAX3421_HOST=0']}], - 'tests': {'device': True, 'host': True, 'dual': True}}, - ])] - - def test_real_roster_max3421_selects_option_board(self): - boards = on_roster(self, 'metro_m4_express') - s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - self.assertIn(board, s['boards']) - - def test_option_selects_via_args_defines_and_flags(self): - s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, self.OPT_ROSTER) - self.assertFalse(s['full']) - self.assertIn('fake_dual_board', s['boards']) # build.args - self.assertIn('fake_host_board', s['boards']) # variant flags - self.assertNotIn('fake_off_board', s['boards']) # variant defines, but =0 - - def test_device_role_port_does_not_pull_host_only_option_board(self): - s = hil_select.classify(['src/portable/analog/max3421/dcd_max3421.c'], REPO, self.OPT_ROSTER) - self.assertFalse(s['full']) - self.assertNotIn('fake_host_board', s['boards']) # host-only board, device change - self.assertIn('fake_dual_board', s['boards']) # device-capable option board - - def test_gates_parsed_from_family_support(self): - self.assertEqual(hil_select.port_option_gates(REPO).get('analog/max3421'), - {'MAX3421_HOST'}) - - def test_board_cmake_option_counts(self): - """A board can enable a gated port in its own BSP rather than via the roster - (hw/bsp/espressif/boards/*/board.cmake -> set(MAX3421_HOST 1)); board_options() - must see those too, or such a board joining the roster is silently dropped.""" - self.assertIn('MAX3421_HOST', - hil_select.bsp_board_options('adafruit_feather_esp32s3', REPO)) - self.assertIn('CFG_TUH_RPI_PIO_USB', - hil_select.bsp_board_options('adafruit_fruit_jam', REPO)) - # commented-out `# set(MAX3421_HOST 1)` must not count - self.assertNotIn('MAX3421_HOST', - hil_select.bsp_board_options('feather_nrf52840_express', REPO)) - - def test_board_cmake_option_selects_off_family_board(self): - # adafruit_feather_esp32s3 is not on any rig roster; stand it in as one to - # prove the BSP-sourced option alone pulls a max3421 change onto the board - roster = [('test/hil/opt.json', [ - {'name': 'adafruit_feather_esp32s3', 'uid': 'o1', 'flasher': {'name': 'esptool'}, - 'tests': {'device': False, 'host': True, 'dual': False}}])] - s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster) - self.assertFalse(s['full']) - self.assertIn('adafruit_feather_esp32s3', s['boards']) - - def test_board_mk_option_is_ignored(self): - """Make-only options must not select: HIL CI builds with CMake exclusively, so - hw/bsp/nrf/boards/nrf5340dk/board.mk's MAX3421_HOST compiles nothing here.""" - roster = [('test/hil/opt.json', [ - {'name': 'nrf5340dk', 'uid': 'o1', 'flasher': {'name': 'jlink'}, - 'tests': {'device': False, 'host': True, 'dual': False}}])] - s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster) - self.assertFalse(s['full']) - self.assertEqual(s['boards'], {}) - - -class TestPortFamiliesCmakeOnly(unittest.TestCase): - """port_families() is CMake-only (HIL CI never builds with Make) and matches on - 'port_dir/' so a port dir is not a prefix of a sibling.""" - def test_make_only_family_is_not_a_family(self): - # hw/bsp/pic32mz has family.mk but no family.cmake - self.assertEqual(hil_select.port_families('microchip/pic32mz', REPO), set()) - - def test_prefix_port_does_not_inherit_sibling_families(self): - # bare-substring matching let 'microchip/pic' match '.../microchip/pic32mz/...' - self.assertEqual(hil_select.port_families('microchip/pic', REPO), set()) - - def test_make_only_port_forces_full(self): - s = sel(['src/portable/microchip/pic32mz/dcd_pic32mz.c']) - self.assertTrue(s['full']) - self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons']) - - def test_cmake_families_still_found(self): - self.assertEqual(hil_select.port_families('raspberrypi/rp2040', REPO), {'rp2040'}) - self.assertIn('stm32f4', hil_select.port_families('synopsys/dwc2', REPO)) - - -class TestPortFamiliesCoverage(unittest.TestCase): - """Systematic guard: every real dcd_*/hcd_* port directory should map to at - least one board family, so a future family.cmake/CMakeLists.txt layout that - port_families() doesn't scan fails loudly instead of silently dropping boards - (as espressif's dwc2 reference did - see TestRealRosterPortFamilies).""" - # Ports with no board family: not a bug, just not wired into any rig board. - # Add here (with a reason) only if port_families() legitimately can't find one. - # A port listed here force-fulls (fail-open), so it is never under-selected. - NO_FAMILY = { - 'template', # reference/example port, not built by any board - # hw/bsp/pic32mz has family.mk only (no family.cmake), and port_families() - # is CMake-only because HIL CI builds every board with CMake - so this port - # is compiled for no HIL board. - 'microchip/pic32mz', - 'microchip/pic', # same: only ever referenced from pic32mz's family.mk - } - - @staticmethod - def _dcd_hcd_ports(): - portable_root = os.path.join(REPO, 'src/portable') - ports = [] - for entry in sorted(os.listdir(portable_root)): - d = os.path.join(portable_root, entry) - if not os.path.isdir(d): - continue - if glob.glob(os.path.join(d, 'dcd_*.c')) or glob.glob(os.path.join(d, 'hcd_*.c')): - ports.append(entry) - continue - for sub in sorted(os.listdir(d)): - sd = os.path.join(d, sub) - if os.path.isdir(sd) and (glob.glob(os.path.join(sd, 'dcd_*.c')) or - glob.glob(os.path.join(sd, 'hcd_*.c'))): - ports.append(f'{entry}/{sub}') - return ports - - def test_every_port_maps_to_a_family(self): - ports = self._dcd_hcd_ports() - self.assertTrue(ports) # sanity: the scan itself found something - for port in ports: - if port in self.NO_FAMILY: - continue - fams = hil_select.port_families(port, REPO) - self.assertTrue(fams, f'{port}: no family references this port ' - f'(port_families() scan gap, or add to NO_FAMILY)') - - -class TestRealRosterOnlyListTests(unittest.TestCase): - """Regression for roster-only-list tests (e.g. espressif's hid_composite_freertos) - being invisible to the selector because it only knew the shared hil_examples lists.""" - def test_only_list_example_change_selects_it(self): - boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') - s = hil_select.classify(['examples/device/hid_composite_freertos/src/main.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - self.assertEqual(s['boards'][board], ['device/hid_composite_freertos']) - - def test_class_change_includes_only_list_boards(self): - boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') - s = hil_select.classify(['src/class/hid/hid_device.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - self.assertIn(board, s['boards']) - - -class TestPortAndCoreRoleUseExtras(unittest.TestCase): - """Regression: the port rule and core-role rule must thread the roster-only - test universe (extras) the same way the class rule already does, so a DCD - or device-stack change doesn't silently drop espressif's only-list tests - (e.g. hid_composite_freertos) that aren't in the shared device_tests list.""" - def test_dcd_change_includes_only_list_test(self): - boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') - s = hil_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - tests = s['boards'][board] - self.assertIn('device/hid_composite_freertos', tests) - self.assertIn('device/cdc_msc_freertos', tests) - self.assertIn('device/audio_test_freertos', tests) - self.assertIn('device/usbtest', tests) - - def test_core_device_change_includes_only_list_test(self): - boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') - s = hil_select.classify(['src/device/usbd.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - tests = s['boards'][board] - self.assertIn('device/hid_composite_freertos', tests) - self.assertIn('device/cdc_msc_freertos', tests) - self.assertIn('device/audio_test_freertos', tests) - self.assertIn('device/usbtest', tests) - - def test_host_change_does_not_leak_device_only_list_test(self): - s = hil_select.classify(['src/host/usbh.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board, tests in s['boards'].items(): - if tests == 'all': - continue - self.assertNotIn('device/hid_composite_freertos', tests, board) - - -class TestFamilies(unittest.TestCase): - """`families` exists for consumers that build (not just test) the diff: most - families have no rig board, so `boards` alone would compile nothing for them.""" - def test_off_rig_port_still_reports_family(self): - s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c']) - self.assertFalse(s['full']) - self.assertEqual(s['boards'], {}) # no same7x board on the rig - self.assertEqual(s['families'], ['same7x']) - - def test_port_families_are_reported(self): - s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) - self.assertIn('rp2040', s['families']) - - def test_bsp_family_and_board_report_family(self): - self.assertEqual(sel(['hw/bsp/rp2040/family.cmake'])['families'], ['rp2040']) - self.assertEqual(sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h'])['families'], - ['rp2040']) - - def test_docs_only_has_no_families(self): - self.assertEqual(sel(['docs/info/contributing.rst'])['families'], []) - - def test_full_selection_still_reports_families(self): - """A full-matrix file must not hide the families of the other changed files: - consumers that build from `families` (e.g. /pre-pr) ignore `boards` when full.""" - s = sel(['src/common/tusb_fifo.c', 'src/portable/microchip/samx7x/dcd_samx7x.c']) - self.assertTrue(s['full']) - self.assertIn('same7x', s['families']) - # full stays full: every roster board, and no args to narrow the run - self.assertEqual(set(s['boards']), {b['name'] for b in ROSTER}) - self.assertTrue(all(v == 'all' for v in s['boards'].values())) - self.assertEqual(hil_select.selection_args(s, ROSTERS), {'tinyusb.json': ''}) - self.assertEqual(hil_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}}) - - def test_family_order_does_not_matter(self): - # same as above with the full-matrix file last (was the only order that worked) - s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c', 'src/common/tusb_fifo.c']) - self.assertTrue(s['full']) - self.assertIn('same7x', s['families']) - - -class TestGitDiffArgv(unittest.TestCase): - def test_diff_disables_rename_detection(self): - """Without --no-renames git reports only a rename's destination, so moving an - HIL-relevant file to a non-code path would be classified as non-code only.""" - self.assertIn('--no-renames', hil_select.GIT_DIFF_ARGV) - - -class TestPortWithoutFamilyIsFull(unittest.TestCase): - """A port dir no family file references must widen (full matrix), not silently - contribute zero boards — the fail-open contract.""" - def test_unreferenced_port_forces_full(self): - orig = hil_select.port_families - hil_select.port_families = lambda port_dir, repo_root: set() - try: - s = sel(['src/portable/vendor/newip/dcd_newip.c']) - finally: - hil_select.port_families = orig - self.assertTrue(s['full']) - self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons']) - - -class TestRosterFlashersDispatch(unittest.TestCase): - """hil_test and hil_pool_check resolve a board's flasher with a bare - getattr(hil_flash, f'flash_{name}'), and hil_test does it inside a redirect_stdout — - so a renamed or typo'd roster name raises an AttributeError whose output is swallowed, - with nothing pointing at the roster as the thing to edit. Renaming a flash_*/reset_* - pair without updating every roster must fail here instead.""" - - def test_flash_and_reset_exist_for_every_roster_flasher(self): - for path, board in roster_flashers(): - name = board['flasher']['name'].lower() - for fn in (f'flash_{name}', f'reset_{name}'): - self.assertTrue(callable(getattr(hil_flash, fn, None)), - f'{path}: {board["name"]} uses flasher "{name}" ' - f'but hil_flash.{fn} does not exist') - - def test_firmware_suffix_known_for_every_roster_flasher(self): - """find_firmware falls back to accepting .elf-or-.bin when a flasher is missing - from FLASHER_SUFFIX, silently restoring the mismatch that map exists to catch.""" - for path, board in roster_flashers(): - name = board['flasher']['name'].lower() - self.assertIn(name, hil_flash.FLASHER_SUFFIX, - f'{path}: {board["name"]} uses flasher "{name}" ' - f'with no hil_flash.FLASHER_SUFFIX entry') - - -if __name__ == '__main__': - unittest.main(verbosity=1) diff --git a/test/hil/tinyusb.json b/test/hil/tinyusb.json index c9b38992c..8fd4683a4 100644 --- a/test/hil/tinyusb.json +++ b/test/hil/tinyusb.json @@ -149,6 +149,7 @@ "flasher": { "name": "openocd", "uid": "E6614C311B597D32", + "vid_pid": "0x2e8a 0x000c", "args": "-f interface/cmsis-dap.cfg -f target/max32665.cfg", "verify": true } @@ -156,11 +157,14 @@ { "name": "metro_m4_express", "uid": "9995AD485337433231202020FF100A34", - "build": { - "args": [ - "MAX3421_HOST=1" - ] - }, + "variant": [ + { + "name": "metro_m4_express", + "defines": [ + "MAX3421_HOST=1" + ] + } + ], "tests": { "device": true, "host": false, @@ -196,6 +200,20 @@ } }, { + "name": "lpcxpresso55s28", + "uid": "2BF1839A7D51F553A15AB03FD08F70AB", + "tests": { + "device": true, + "host": false, + "dual": false + }, + "flasher": { + "name": "jlink", + "uid": "000727031389", + "args": "-device LPC55S28" + } + }, + { "name": "ra4m1_ek", "uid": "152E163038303131393346E46F26574B", "tests": { @@ -240,6 +258,7 @@ "flasher": { "name": "openocd", "uid": "E6614103E72C1D2F", + "vid_pid": "0x2e8a 0x000c", "args": "-f interface/cmsis-dap.cfg -f target/rp2040.cfg -c \"adapter speed 5000\"", "verify": true } @@ -270,6 +289,7 @@ "flasher": { "name": "openocd", "uid": "E6633861A3819D38", + "vid_pid": "0x2e8a 0x000c", "args": "-f interface/cmsis-dap.cfg -f target/rp2040.cfg -c \"adapter speed 5000\"", "verify": true }, @@ -296,6 +316,7 @@ "flasher": { "name": "openocd", "uid": "E6633861A3978538", + "vid_pid": "0x2e8a 0x000c", "args": "-f interface/cmsis-dap.cfg -f target/rp2350.cfg -c \"adapter speed 5000\"", "verify": true } @@ -326,6 +347,7 @@ "flasher": { "name": "openocd", "uid": "E663AC91D3359B38", + "vid_pid": "0x2e8a 0x000c", "args": "-f interface/cmsis-dap.cfg -f target/rp2350.cfg -c \"adapter speed 5000\"", "verify": true } @@ -407,9 +429,8 @@ "dual": false }, "flasher": { - "name": "openocd", + "name": "stlink", "uid": "004C00343137510F39383538", - "args": "-f interface/stlink.cfg -f target/stm32h7x.cfg", "verify": true } }, @@ -422,9 +443,8 @@ "dual": false }, "flasher": { - "name": "openocd", + "name": "stlink", "uid": "066FFF495087534867063844", - "args": "-f interface/stlink.cfg -f target/stm32g0x.cfg", "verify": true }, "comment": "32-bit scheme, 2KB USB SRAM" @@ -472,6 +492,7 @@ "flasher": { "name": "openocd", "uid": "A76D8F062C2A", + "vid_pid": "0x1a86 0x8010", "args": "-f target/wch-riscv.cfg", "verify": false } @@ -488,6 +509,7 @@ "flasher": { "name": "openocd", "uid": "BC4954081051", + "vid_pid": "0x1a86 0x8010", "args": "-f target/wch-riscv.cfg", "verify": false } @@ -508,6 +530,7 @@ "flasher": { "name": "openocd", "uid": "BC5DA47360D0", + "vid_pid": "0x1a86 0x8010", "args": "-f target/wch-riscv.cfg", "verify": false } @@ -524,6 +547,7 @@ "flasher": { "name": "openocd", "uid": "57468F06DC03", + "vid_pid": "0x1a86 0x8010", "args": "-f target/wch-riscv.cfg", "verify": false } diff --git a/test/hil/usbtest.py b/test/hil/usbtest.py index 83ea3e24c..d23217417 100755 --- a/test/hil/usbtest.py +++ b/test/hil/usbtest.py @@ -25,7 +25,9 @@ capability flags only unlock cases, they don't require the endpoints to exist. import argparse import json +from contextlib import redirect_stdout import os +import pathlib import re import shutil import subprocess @@ -33,16 +35,81 @@ import sys import time from pathlib import Path +sys.path.append(os.path.dirname(os.path.abspath(__file__))) # PYTHONSAFEPATH drops it + VID = 'cafe' PID = '4010' GZ_REF = '0525 a4a0' # copy Gadget Zero's capability profile (ctrl_out+iso+intr) SYS_USB = Path('/sys/bus/usb/devices') DRIVER = Path('/sys/bus/usb/drivers/usbtest') -USB_RECOVER = Path(__file__).resolve().parents[2] / '.claude/skills/usb-kernel-recover/scripts/usb_recover.sh' PATTERN_PARAM = Path('/sys/module/usbtest/parameters/pattern') +RECOVER_FLASH_TIMEOUT = 90 # bound on the post-hang reflash; typical flash is 10-20s +RECOVER_RESET_TIMEOUT = 30 # bound on the post-hang probe reset; ResetTarget measures ~130ms + + +RECOVER_SETTLE = 5 # after each step, to let a freed ioctl unwind +# The ladder's UNBOUNDED work, which no step timeout covers: two wedged_pids() /proc walks, +# json.loads of the roster entry, the child's first `import hil_flash`, convoy_safe, the +# BUDGET back-fill and the JSON print. The deleted _time_left() carried this as a bare +# '- 35'. Without it the reserve equals its own worst case exactly, and HIL_CMD_TIMEOUT and +# HIL_USBTEST_BATTERY_BUDGET are both env-overridable -- any of them moving up puts +# run_cmd's killpg back inside the reflash, orphaning the flasher on the probe. +RECOVER_OVERHEAD = 40 + + +def recovery_reserve(flasher: dict | str) -> int: + """Seconds this flasher's post-hang ladder can actually spend. + + Every bounded step can cost its own timeout PLUS run_cmd's post-SIGKILL reap, so the + caller must count REAP_GRACE per step or its outer killpg lands mid-reflash and + ORPHANS the flasher on the probe. Derived rather than pinned: the predecessor was an + independent 250s that could not contain its own ladder, which is why the child used to + re-decide before every step and skipped most of them on a real hang. + + Per FLASHER, not one number for the fleet: the Rescue-DP legs are openocd-only + (hil_flash.rescue_openocd returns False for anything else), and a stub reset is + screened out by reset_primitive -- so an esptool board reserving them would hold a + pool worker and a usbtest permit for 200s it can never spend. + """ + import hil_flash + from helper import hil_util + if isinstance(flasher, str): + flasher = {'name': flasher, 'args': ''} + name = (flasher.get('name') or '').lower() + + def step(bound): + return bound + hil_util.REAP_GRACE + + total = step(RECOVER_FLASH_TIMEOUT) + 2 * RECOVER_SETTLE + RECOVER_OVERHEAD + if reset_primitive(name): + total += step(RECOVER_RESET_TIMEOUT) + # The ARGS, not just the name: rescue_openocd also needs the target cfg to be an RP + # one (RESCUE_CFG), so the five WCH/max32666 openocd boards on this rig can never run + # it. Reserving its two legs for them holds a pool worker and a usbtest permit for + # 200s of dead time -- the same waste the esptool case exists to remove. + if name == 'openocd' and any(cfg in (flasher.get('args') or '') + for cfg in hil_flash.RESCUE_CFG): + total += 2 * step(RECOVER_FLASH_TIMEOUT) # Rescue-DP POR + one retry + return total -# Battery per tier, in run order: control sanity first, then simple bulk, -# queued, unaligned, unlink, halt/toggle, throughput last. + +def reset_primitive(flasher_name: str): + """The flasher's probe-reset callable, or None when there is nothing real to run. + + Two things gate it. A flasher may have no reset_* at all, and reset_esptool / + reset_lm4flash return rc 0 WITHOUT resetting anything -- running those makes the log + say "resetting <board> via <flasher>" for a step that did nothing. wedged_pids() + arbitrates the outcome either way, so behaviour was always right; the RECORD was not. + """ + import hil_flash # deferred: stdlib-only unless recovery actually runs + fn = getattr(hil_flash, f'reset_{flasher_name.lower()}', None) + return None if getattr(fn, 'no_op', False) else fn + + +HELPER_TIMEOUT = 30 # default bound for sudo helpers (dmesg/modprobe/setpci/tee) + +# Battery per tier, in run order: control sanity, simple bulk, queued, unaligned, unlink, +# halt/toggle, throughput last. TIER_CASES = { 1: [0, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 17, 18, 19, 20, 11, 12, 24, 13, 29, 27, 28], 2: [14, 21], @@ -50,10 +117,10 @@ TIER_CASES = { 4: [15, 16, 22, 23], } -# Per-case testusb parameters (full speed / high speed). All -s/-v values are -# multiples of 512 so transfers stay packet-aligned at both speeds: the device -# streams whole max-size packets and a non-aligned IN length would babble. -# 14/21 must never run with defaults (vary >= length is -EINVAL in the kernel). +# Per-case testusb parameters (full speed / high speed). All -s/-v values are multiples +# of 512 so transfers stay packet-aligned at both speeds: the device streams whole max-size +# packets and a non-aligned IN length would babble. 14/21 must never run with defaults +# (vary >= length is -EINVAL in the kernel). PARAMS = { 0: ('-c 1', '-c 1'), 9: ('-c 256', '-c 1000'), @@ -88,9 +155,40 @@ RE_FAIL = re.compile(r'test (\d+) --> (\d+) \((.*)\)') def run(cmd, **kw): - kw.setdefault('capture_output', True) + # NOT subprocess.run(timeout=): CPython's post-timeout path is an UNBOUNDED wait() that + # never returns on a D-state child -- the hang sysfs_write's timeout exists to catch. + timeout = kw.pop('timeout', None) + data = kw.pop('input', None) # subprocess.run-only kwarg; Popen takes stdin + kw.pop('capture_output', None) # ditto: expressed by the PIPEs below kw.setdefault('text', True) - return subprocess.run(cmd, **kw) + kw.setdefault('encoding', 'utf-8') + kw.setdefault('errors', 'replace') # strict decode would raise out of _sudo_soft + # NO start_new_session: these helpers (dmesg, modprobe, setpci, tee) must stay in our + # process group so hil_test's outer killpg reaps them with us. + timeout = timeout if timeout is not None else HELPER_TIMEOUT + proc = subprocess.Popen(cmd, stdin=subprocess.PIPE if data is not None else None, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kw) + try: + out, err = proc.communicate(input=data, timeout=timeout) + return subprocess.CompletedProcess(cmd, proc.returncode, out, err) + except subprocess.TimeoutExpired: + # Under sudo our child is only the wrapper; the root grandchild survives this and + # is left for the report and hil_pool_check to name. Close our pipe ends so an + # abandoned child costs no fds. + try: + proc.kill() # same group as us: never killpg, that would kill us too + except OSError: + pass + try: + proc.communicate(timeout=5) + except subprocess.TimeoutExpired: + for pipe in (proc.stdout, proc.stderr, proc.stdin): + try: + if pipe is not None: + pipe.close() + except OSError: + pass # unkillable: abandon it, the caller reports the timeout + raise def sudo(cmd, **kw): @@ -104,29 +202,106 @@ def sudo(cmd, **kw): def sysfs_write(path, data, check=True): - # A driver-registry write (new_id/remove_id/bind) blocks in D state when a wedged device - # holds its lock (driver_attach walks the bus): fail fast and loud instead of piling up - # unkillable writers and hanging the whole run -- the rig needs USB recovery first. + # A driver-registry write (new_id/remove_id/bind) blocks in D state when a wedged + # device holds its lock: fail fast instead of piling up unkillable writers -- the rig + # needs USB recovery first. + # + # Verified in v6.12.96: unbind_store -> device_driver_detach -> + # device_release_driver_internal -> __device_driver_lock (drivers/base/dd.c), which + # takes device_lock() -- the UNINTERRUPTIBLE variant, unlike the sysfs read path -- and + # ALSO device_lock(parent), because usb_bus_type sets need_parent_lock = true + # (drivers/usb/core/driver.c:2048). So one such write against a wedged device blocks + # unkillably while holding the HUB's lock: that is the mechanism by which a single + # wedged port takes its whole bus down, and why this fails fast instead. try: r = sudo(['tee', str(path)], input=data, timeout=15) except subprocess.TimeoutExpired: sys.exit(f'write "{data}" > {path} blocked >15s: USB subsystem is wedged ' - '(a D-state device lock exists). Recover the rig (usb_recover.sh) ' + '(a D-state device lock exists). Recover the rig (usb-kernel-recover skill) ' 'before running batteries.') if check and r.returncode != 0: sys.exit(f'write "{data}" > {path} failed: {r.stderr.strip()}') return r.returncode == 0 +def _hu(): + """The helper module, imported lazily like every other helper use in this file.""" + from helper import hil_util + return hil_util + + +SERIAL_GRACE = 1.0 # tighter than hil_util's shared default on purpose: find_device + # re-scans every cafe:4010 peer after each of ~30 cases and inside + # the 8s startup poll, so N unreadable peers cost N x this per scan + + +def _read_sysfs(path): + """The attribute's value, or None. See hil_util.read_sysfs for why `serial` can block.""" + from helper import hil_util + return hil_util.read_sysfs(str(path), SERIAL_GRACE) + + +_DEV_CACHE: dict = {} # serial -> sysname, see find_device + + +def _reread(sysname, serial): + """Re-describe an already-resolved device, CONFIRMING its serial. + + idVendor/idProduct/busnum/devnum/speed are lock-free (sysfs.c:688-705), so they cannot + block on a wedged peer -- but every identical board answers them the same, so they + prove nothing about identity. `serial` does, at one bounded read: a sysname is a + topology path, and after a renumber (controller reset, reboot) it can name a DIFFERENT + cafe:4010 board whose verdicts would be filed under this one. Returns None when the + serial is gone, mismatched or unconfirmed -- caller falls back to a full scan. + """ + d = SYS_USB / sysname + try: + if ((d / 'idVendor').read_text().strip() != VID + or (d / 'idProduct').read_text().strip() != PID): + return None + dev_serial = _read_sysfs(d / 'serial') + if not isinstance(dev_serial, str) or dev_serial.lower() != serial.lower(): + return None # gone, mismatched, or unconfirmable -> full scan decides + return { + 'sysname': sysname, + 'serial': dev_serial, + 'node': '/dev/bus/usb/%03d/%03d' % (int((d / 'busnum').read_text()), + int((d / 'devnum').read_text())), + 'speed': (d / 'speed').read_text().strip(), + 'tier': int((d / 'bcdDevice').read_text().strip()[-2:], 16), + } + except (OSError, ValueError): + return None + + def find_device(serial, first=False): - """Locate the usbtest device in sysfs, return info dict or None.""" + """Locate the usbtest device in sysfs, return info dict or None. + + Cached by serial: this is called after EVERY case, and a full scan pays a bounded + but real `serial` read for every cafe:4010 peer on the rig. With another board + wedged that cost lands on a HEALTHY battery ~30 times over, truncating it into + BUDGET entries. The fast path pays ONE bounded read -- our own device's serial, the + only attribute that tells identical boards apart (see _reread). + """ + if serial: + sysname = _DEV_CACHE.get(serial.lower()) + if sysname: + hit = _reread(sysname, serial) + if hit: + return hit + _DEV_CACHE.pop(serial.lower(), None) matches = [] for dev in SYS_USB.iterdir(): try: if (dev / 'idVendor').read_text().strip() != VID or \ (dev / 'idProduct').read_text().strip() != PID: continue - dev_serial = (dev / 'serial').read_text().strip() + # idVendor/idProduct are cached descriptors; `serial` is served under + # device_lock(), so on a wedged DUT this read blocks until the wedge clears. + # Contained by the caller's bound, not prevented here -- see hil_util.read_sysfs. + dev_serial = _read_sysfs(dev / 'serial') + if dev_serial is None: + continue if serial and dev_serial.lower() != serial.lower(): continue matches.append({ @@ -141,11 +316,13 @@ def find_device(serial, first=False): continue if not matches: return None + if serial and len(matches) == 1: + _DEV_CACHE[serial.lower()] = matches[0]['sysname'] if len(matches) > 1 and not first: if serial: - # Dual-port parts (nanoch32v203 fsdev/usbfs, ch32v307 usbhs/usbfs) briefly enumerate - # BOTH ports with the same serial around a variant reflash; picking one arbitrarily - # could bind the stale port. Report ambiguity so the caller retries until it drops. + # Dual-port parts (nanoch32v203, ch32v307) briefly enumerate BOTH ports with + # one serial around a variant reflash, and picking one could bind the stale + # port -- report ambiguity so the caller retries until it drops. return {'ambiguous': sorted(m['sysname'] for m in matches)} sys.exit(f'multiple {VID}:{PID} devices found, use --serial: ' + ', '.join(m["serial"] for m in matches)) @@ -165,8 +342,8 @@ def check_host_compat(dev): vid_did = ((pci / 'vendor').read_text().strip(), (pci / 'device').read_text().strip()) break except (OSError, ValueError): - # transient sysfs error (e.g. racing a re-enumeration): retry so a blip doesn't - # silently pass an incompatible host; if the probe truly fails, fail open but say so + # transient sysfs error (racing a re-enumeration): retry so a blip does not + # silently pass an incompatible host, then fail open but say so if attempt == 2: print('warning: cannot probe the upstream host controller; ' 'skipping the host compatibility check', file=sys.stderr) @@ -178,19 +355,16 @@ def check_host_compat(dev): 'placed in the EHCI periodic schedule and unlinked reads complete as short ' 'transfers (EREMOTEIO). Move the DUT to an xHCI port.') if drv.startswith('xhci') and vid_did in (('0x1912', '0x0014'), ('0x1912', '0x0015')): - # The Renesas uPD720201/uPD720202 must run its latest firmware (>= 2.0.2.6, - # K2026090.mem; RAM-uploaded, so it reverts to ROM on every power cycle unless - # re-loaded). On the ROM firmware its command ring intermittently dies under unlink - # stress: a Configure Endpoint command stops completing, the hub worker deadlocks - # holding the device lock (needs a host power cycle). Three separate boards killed - # it this way (ch32v307 2026-07-10; ra6m5 test 24, mimxrt1015 2026-07-11). Both - # parts expose the FW version register at PCI config offset 0x6c. NOTE this check - # is necessary, not sufficient: board-specific batteries have killed the controller - # on current firmware too (mimxrt1015, stop-endpoint timeout) - those are handled - # by per-board skips in the rig config. + # The Renesas uPD720201/uPD720202 must run firmware >= 2.0.2.6 (K2026090.mem; + # RAM-uploaded, so it reverts to ROM on every power cycle): on ROM firmware its + # command ring dies under unlink stress and the hub worker deadlocks holding the + # device lock, needing a host power cycle (ch32v307 2026-07-10; ra6m5 test 24, + # mimxrt1015 2026-07-11). Both parts expose the FW version at PCI config 0x6c. + # Necessary, not sufficient -- batteries have killed the controller on current + # firmware too, which per-board skips in the rig config handle. fw = None try: - r = sudo(['setpci', '-s', pci.name, '0x6c.l'], capture_output=True, text=True) + r = _sudo_soft(['setpci', '-s', pci.name, '0x6c.l'], capture_output=True, text=True) if r.returncode == 0: fw = int(r.stdout.strip(), 16) except (OSError, ValueError): @@ -211,7 +385,7 @@ def check_host_compat(dev): def bind_usbtest(dev): """Bind the device's interface 0 to the usbtest driver.""" if not DRIVER.exists(): - r = sudo(['modprobe', 'usbtest']) + r = _sudo_soft(['modprobe', 'usbtest']) if r.returncode != 0 or not DRIVER.exists(): sys.exit(f'cannot load usbtest module: {r.stderr.strip()}') @@ -222,8 +396,8 @@ def bind_usbtest(dev): sysfs_write(DRIVER / 'remove_id', f'{VID} {PID}', check=False) sysfs_write(DRIVER / 'new_id', f'{VID} {PID} 0 {GZ_REF}') if stale_binding: - # bound before the re-registration: that probe captured the OLD dynamic id's capability - # profile; unbind once (device is idle here) so the loop below reprobes the fresh one + # it probed against the OLD dynamic id's capability profile; unbind once (the + # device is idle here) so the loop below reprobes the fresh one sysfs_write(drv / 'unbind', intf, check=False) deadline = time.monotonic() + 3 @@ -248,51 +422,64 @@ def set_pattern(value): 'the "pattern" param, or it is not readable') +def _sudo_soft(cmd, **kw): + """sudo() for calls whose failure must never abort the battery: run() re-raises + TimeoutExpired, and two of these are evaluated inside run_case's own timeout handler + -- a raise there loses the HUNG verdict, the recovery and the JSON report.""" + try: + return sudo(cmd, **kw) + except (OSError, ValueError, subprocess.SubprocessError, SystemExit) as e: + # SystemExit too: sudo() sys.exit()s on 'a password is required', unwinding out of + # run_case's timeout handler before the HUNG verdict is recorded -- which leaves + # unrecovered_hang False and lets the finally run the remove_id/unbind that must + # never happen while a D-state device lock is held + print(f'{cmd[0]}: {type(e).__name__}: {e}', file=sys.stderr) + return subprocess.CompletedProcess(cmd, 1, '', '') + + def dmesg_tail(): - r = sudo(['dmesg']) + r = _sudo_soft(['dmesg']) lines = [l for l in r.stdout.splitlines() if 'usbtest' in l] return '\n'.join(lines[-8:]) + + def wedged_pids(devnode): - """Return (pids, complete): PIDs in uninterruptible sleep whose cmdline names devnode, i.e. - still holding its usbfs device lock, and whether every /proc entry could actually be read. + """(pids, complete): pids still in D state on `devnode` after a recovery reflash. + + Matched by device node rather than by our child's pid because run_case() may wrap + testusb in sudo: the Popen pid is then the wrapper and the blocked process is its + child. A clean flash only proves the probe wrote the MCU, not that the D-state holder + let go -- this is what tells the two apart. - Matched by device node rather than by our child's pid because run_case() may wrap testusb in - sudo, in which case the Popen pid is the wrapper and the blocked process is its child -- - killing the wrapper would make a pid-based check look clean while the real holder is stuck. + FAIL CLOSED. `complete` is False when an entry could be HIDDEN from us, and the caller + must then keep treating the hang as unrecovered: the holder is root-owned (run_case + uses `sudo -n` whenever the node is not writable) and a hidepid/ProtectProc mount + hides exactly that entry. Reporting "no holder" from a scan that could not see it + clears unrecovered_hang and lets cleanup run remove_id/unbind against a device whose + usbfs lock is still held -- which deadlocks the bus, not just this board. - complete is False when a PermissionError hid an entry (a hidepid/ProtectProc mount, or the - root-owned child of that same sudo). An entry we could not read might be the holder, so the - caller must treat that as unrecovered rather than as an all-clear.""" + Self-contained: /proc is plain text and this is one pass over it, so importing a + helper to do it would only add a failure mode on the recovery path. + """ stuck, complete = [], True - # hidepid=2 and systemd's ProtectProc=invisible omit other users' processes from iterdir() - # entirely -- no entry at all, so no PermissionError to catch -- and testusb runs under sudo - # whenever the device node is not writable. The scan would then look clean while hiding the - # very holder it exists to find. pid 1 is always root-owned, so being unable to read it means - # enumeration is restricted and no result from this scan can be trusted as complete. + # A restricted /proc hides other users' entries ENTIRELY -- no entry, so no + # PermissionError to catch -- and testusb runs under sudo, so the holder is exactly + # what is hidden. Detect the restriction itself rather than its symptom. if os.geteuid() != 0 and not os.access('/proc/1/cmdline', os.R_OK): complete = False - for entry in Path('/proc').iterdir(): - if not entry.name.isdigit(): - continue - try: - cmdline = (entry / 'cmdline').read_bytes() - except PermissionError: - complete = False # cannot rule this pid out - continue - except OSError: - continue # raced with process exit: genuinely gone, not hidden - if devnode.encode() not in cmdline: - continue + for d in pathlib.Path('/proc').glob('[0-9]*'): try: - stat = (entry / 'stat').read_text() - if stat[stat.rindex(')') + 2] == 'D': # comm may contain ')', so scan from the right - stuck.append(int(entry.name)) + st = (d / 'stat').read_bytes() + if st[st.rindex(b')') + 2:st.rindex(b')') + 3] != b'D': + continue + if devnode.encode() in (d / 'cmdline').read_bytes(): + stuck.append(int(d.name)) except PermissionError: - complete = False - except (OSError, ValueError, IndexError): - continue + complete = False # cannot rule this pid out + except (OSError, ValueError): + continue # raced with exit return stuck, complete @@ -306,17 +493,29 @@ def run_case(num, dev, testusb, quick, timeout): cmd = ['sudo', '-n'] + cmd result = {'num': num, 'name': CASE_NAMES[num], 'params': fs_hs} - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + # NO start_new_session: testusb must stay in OUR process group so the caller's outer + # killpg still reaps it; a sudo-wrapped child is escalated through sudo below instead. + # errors='replace': testusb output is not guaranteed UTF-8, and a strict decode would + # raise out of here and out of main(), printing no JSON at all (battery '0/30'). + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, encoding='utf-8', errors='replace') try: out, _ = p.communicate(timeout=timeout) except subprocess.TimeoutExpired: - p.kill() + # Under sudo we only kill the wrapper; its root-owned testusb keeps the inherited + # stdout pipe, so the reap below times out and the overrun is reported as HUNG. + # Accepted rather than escalated: the rig's udev rules make the device node + # writable, so sudo is the exception, and the harness must never sudo-kill a pid + # it cannot prove is its own. + try: + p.kill() + except OSError: + pass try: out, _ = p.communicate(timeout=5) except subprocess.TimeoutExpired: - # SIGKILL had no effect: the child is in uninterruptible sleep on an - # in-kernel usbfs ioctl (device stopped responding mid-transfer). - # Abandon it — waiting or re-signalling can never succeed. + # SIGKILL had no effect: the child is in uninterruptible sleep on an in-kernel + # usbfs ioctl. Abandon it — waiting or re-signalling can never succeed. result.update(status='HUNG', detail=f'testusb stuck in D state after {timeout}s', dmesg=dmesg_tail()) return result @@ -362,7 +561,16 @@ def main(): p.add_argument('--keep-binding', action='store_true', help='leave usbtest dynamic id registered') p.add_argument('--testusb', default=None, help='path to testusb binary') p.add_argument('--timeout', type=int, default=120, help='per-case timeout in seconds') + p.add_argument('--recover-board', help='board JSON (name + flasher) for the post-hang ' + 'reflash recovery; without it a HUNG case leaves the device wedged') + p.add_argument('--recover-fw', help='firmware path reflashed by the post-hang recovery') + p.add_argument('--budget', type=int, default=0, + help='stop starting new cases after this many seconds (0 = no limit). ' + 'Callers that impose their own outer bound set this BELOW it, ' + 'reserving the remainder for the post-hang recovery -- see ' + 'recovery_reserve() for what that ladder costs') args = p.parse_args() + t_start = time.monotonic() sys.stdout.reconfigure(line_buffering=True) # per-case results visible when piped/logged testusb = args.testusb or shutil.which('testusb') or os.path.expanduser('~/testusb') @@ -370,22 +578,32 @@ def main(): sys.exit('testusb binary not found: build kernel tools/usb/testusb.c ' 'and install it, or pass --testusb') - # retry briefly: right after a flash the enumeration may still be settling, and on dual-port - # parts the other port's stale same-serial node takes a moment to drop off (see find_device) + # retry briefly: after a flash the enumeration may still be settling, and a dual-port + # part's stale same-serial node takes a moment to drop off (see find_device) deadline = time.monotonic() + 8 while True: dev = find_device(args.serial) + # find_device returns a device or {'ambiguous': [...]}. Screening for the marker + # matters: without it the next statement subscripts dev['tier'] -> KeyError, no + # JSON on stdout, and hil_test reports "usbtest did not run / 0-30". if dev and 'ambiguous' not in dev: break if time.monotonic() > deadline: - if dev: + if dev and 'ambiguous' in dev: sys.exit(f"multiple devices with serial {args.serial}: {', '.join(dev['ambiguous'])} " '— stale enumeration from another port? replug or retry') - sys.exit(f'no {VID}:{PID} device' + (f' with serial {args.serial}' if args.serial else '')) + # a bounded `serial` read that gave up looks exactly like a disconnect from + # here, and hil_test relays this line verbatim into the report cell. The + # sticky process-wide flag is the RIGHT question at startup -- nothing but + # this scan has read anything yet -- unlike mid-battery, where a peer that + # stranded at case 2 would answer for our board at case 29. + sys.exit(f'no {VID}:{PID} device' + + (f' with serial {args.serial}' if args.serial else '') + + _hu().strand_note()) time.sleep(0.5) - # tier drives which cases run; a stale/foreign device advertising an out-of-range tier - # must not silently run an empty battery ('0/0 passed' would read as green in CI) + # a stale/foreign device advertising an out-of-range tier must not silently run an + # empty battery ('0/0 passed' would read as green in CI) tier = args.tier or dev['tier'] if not 1 <= tier <= max(TIER_CASES): sys.exit(f"device advertises tier {tier} (bcdDevice ...{tier:02x}); reflash a usbtest build " @@ -404,8 +622,7 @@ def main(): if not args.json: print(info) - # probe the upstream controller before touching the device: an incompatible host - # (MosChip MCS9990, or uPD720201 on pre-2.0.2.6 firmware) exits here, before any bind + # before touching the device: an incompatible host exits here, before any bind check_host_compat(dev) results = [] @@ -414,7 +631,15 @@ def main(): bind_usbtest(dev) set_pattern(0) # tier 1 firmware sources zeros; also required by perf cases 27/28 - for num in cases: + abort_reason = None # set on any early exit; drives the BUDGET back-fill below + for idx, num in enumerate(cases): + # Only a HUNG case aborts the battery; an ordinary case timeout is a FAIL and + # the loop continues, each burning --timeout+5s, so without this the run can + # still be in the case loop when the outer timeout SIGKILLs it before it emits + # JSON. Checked before dispatch: worst overshoot is one case. + if args.budget and time.monotonic() - t_start > args.budget: + abort_reason = f'battery budget {args.budget}s exhausted' + break results.append(run_case(num, dev, testusb, args.quick, args.timeout)) r = results[-1] if not args.json: @@ -422,112 +647,266 @@ def main(): extra += f" {r['mbps']} MB/s" if 'mbps' in r else '' print(f"test {num:2d} {r['name']:22s} {r['status']:6s}{extra}") if r['status'] == 'HUNG': - print(f'aborting battery: kernel-side hang, device wedged mid-transfer.\n' - f'auto-recovering: {USB_RECOVER.name} root-cycle {dev["sysname"]} ' - f'(see .claude/skills/usb-kernel-recover)', file=sys.stderr) - # Cutting VBUS at the root port fails the in-flight URB so the usbfs ioctl returns. - # Must run BEFORE any unbind/remove_id, which would take the device lock the stuck - # ioctl holds and deadlock the bus. + abort_reason = 'battery aborted on a kernel-side hang' + # Reflash, NEVER a root-port cycle: resetting the MCU through the DUT's own + # debug probe fails the in-flight URB at the source, so the ioctl returns, + # the queued kill lands and the cleanup below is lock-safe -- and it reaches + # exactly one board, where a root-port cycle bounces every fixture under the + # port (and could never remove power anyway; see usb-kernel-recover). + # Deliberately not gated on a hub-worker check: our own stuck testusb is + # what drives a hub worker into usb_lock_device(), so a pre-check reads + # wedged by construction. # - # Assume unrecovered until proven otherwise, so that any early exit from this block - # -- an OSError spawning the helper, a KeyboardInterrupt, a sudo prompt killing the - # run -- still reaches the finally cleanup with the flag set, instead of running - # the remove_id/unbind the comments there forbid while a device lock is held. + # Assume unrecovered until proven otherwise, so any early exit from this + # block reaches the finally with the flag set instead of running the + # remove_id/unbind that must not happen while a device lock is held. unrecovered_hang = True - # Pass the serial so the helper refuses a stale busport rather than cutting power - # to whatever else now occupies that path. Popen rather than sudo()/subprocess.run: - # run() would kill() then wait() unbounded on timeout, which never returns if - # uhubctl is itself in D state -- the case the timeout exists for. Merge stderr - # into stdout so the helper's target-identity and action lines are not lost. - # Only pass the serial when we actually have one: an empty third argument reads as - # "no expectation" and would silently disable the helper's stale-busport guard. - cmd = [str(USB_RECOVER), 'root-cycle', dev['sysname']] - if dev['serial']: - cmd.append(dev['serial']) - if os.geteuid() != 0: - cmd = ['sudo', '-n'] + cmd - try: - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - text=True) - except OSError as e: - # helper missing or not executable, or sudo unavailable. unrecovered_hang is - # already True so the finally block still skips the unsafe cleanup -- this only - # replaces a traceback with a message that says what to fix. - print(f'cannot run {USB_RECOVER}: {e}', file=sys.stderr) + print('aborting battery: kernel-side hang, device wedged mid-transfer', + file=sys.stderr) + if not (args.recover_board and args.recover_fw): + print('no --recover-board/--recover-fw: the device stays wedged and ' + 'cleanup is skipped', file=sys.stderr) break - rc = None + # Both steps are bounded (RECOVER_RESET_TIMEOUT / RECOVER_FLASH_TIMEOUT) + # and the caller RESERVES room for both -- hil_test derives its + # bound from recovery_reserve(). No re-derivation here: + # the old per-step "does it still fit?" arithmetic carried an unexplained + # 35s fudge for costs paid downstream, and nobody could re-derive it. try: - out, _ = p.communicate(timeout=60) # normal run is ~8s - rc = p.returncode - except subprocess.TimeoutExpired: - p.kill() + board = json.loads(args.recover_board) + bname, fname = board['name'], board['flasher']['name'] + import hil_flash # deferred: stdlib-only unless recovery actually runs + flash_fn = getattr(hil_flash, f'flash_{fname.lower()}') + except Exception as e: # malformed/short json, import failure, unknown flasher + print(f'reflash recovery unavailable ({e})', file=sys.stderr) + break + # DELIVERY must be convoy-safe or the recovery makes things worse: our own + # testusb is D-state on this DUT's node, so a flasher that enumerates by + # OPENING usbfs nodes blocks on it, survives SIGKILL and is abandoned -- + # a SECOND stray, the budget spent, the device still wedged. On 2026-08-12 + # a vid_pid-pinned openocd was the only flasher that still reached its + # probe; JLinkExe's ShowEmuList returned zero. See hil_flash.convoy_safe. + if not hil_flash.convoy_safe(board['flasher']): + print(f'{fname} is not convoy-safe for delivery (it enumerates by ' + f'opening usbfs nodes, and this DUT has a D-state holder on ' + f'its own node): skipping the reflash rather than adding a ' + f'second stray. Pin the roster entry with vid_pid on an ' + f'openocd flasher to enable recovery for this board.', + file=sys.stderr) + break + # RESET FIRST: a probe reset fails the in-flight URB at the source just + # as a reflash does, but it is non-destructive -- the firmware under test + # survives for autopsy -- writes no flash, and cannot brick SWD the way a + # bad park image has (mimxrt1064_evk, max32666fthr). Measured ~130 ms. + # wedged_pids is the arbiter either way: reset_esptool is a stub that + # returns rc 0 without resetting anything, so an exit code proves nothing. + reset_fn = reset_primitive(fname) + if reset_fn: + print(f'auto-recovering: resetting {bname} via {fname} probe ' + f'(non-destructive; reflash only if this does not clear it)', + file=sys.stderr) + # Inspect the signature rather than catching TypeError around the + # call: a TypeError raised INSIDE the primitive would re-run it with + # no bound (run_cmd's 180s CMD_TIMEOUT, against a 40s reserve), and a + # raise from that retry does not reach the sibling except Exception -- + # it unwinds past the recovery block, so the battery exits on a + # traceback with no JSON and ~29 real verdicts are discarded. + import inspect + kw = ({'timeout': RECOVER_RESET_TIMEOUT} + if 'timeout' in inspect.signature(reset_fn).parameters else {}) try: - out, _ = p.communicate(timeout=5) - rc = p.returncode - except subprocess.TimeoutExpired: - out = ('root-cycle abandoned after 60s: uhubctl did not die to SIGKILL, so ' - 'it is wedged too and the convoy has spread beyond this device') - if out: - print(out.strip(), file=sys.stderr) - if rc is not None: - time.sleep(5) # let the bus settle and the freed ioctl unwind - # Authoritative either way. A non-zero exit only means the device did not come - # back within the poll (a slow bootloader will do that) -- if nothing still - # holds the lock, the bus is usable and cleanup is safe. Conversely a zero exit - # only proves re-enumeration, not that the D-state holder let go. + with redirect_stdout(sys.stderr): + reset_fn(board, **kw) + except Exception as e: + print(f'probe reset raised: {e}; falling through to the reflash', + file=sys.stderr) + time.sleep(RECOVER_SETTLE) # let the freed ioctl unwind stuck, complete = wedged_pids(dev['node']) - if stuck: - print(f'{dev["sysname"]}: pid(s) {stuck} still in D state on ' - f'{dev["node"]} — the device lock was never released', file=sys.stderr) - elif not complete: - print('cannot confirm recovery: /proc is only partly readable, so a ' - 'hidden D-state holder cannot be ruled out', file=sys.stderr) - else: + if complete and not stuck: + print('probe reset cleared the wedge; skipping the reflash ' + '(firmware under test left intact for autopsy)', + file=sys.stderr) unrecovered_hang = False + break + print(f'auto-recovering: reflashing {bname} via ' + f'{fname} (see .claude/skills/usb-kernel-recover). ' + f'Unbudgeted by flash_permit, like the root-cycle it replaced: the ' + f'per-controller semaphores live in hil_test\'s process.', + file=sys.stderr) + # run_cmd bounds the flash; its banners go to stdout, which in --json mode + # carries the result object -- keep them off it. A raising flasher (missing + # serial node, unwritable CWD) must not cost the battery its JSON report. + try: + with redirect_stdout(sys.stderr): + ret = flash_fn(board, args.recover_fw, timeout=RECOVER_FLASH_TIMEOUT) + except Exception as e: + print(f'reflash raised: {e}; the device may still be wedged', file=sys.stderr) + break + if ret.returncode != 0: + # a wedged RP DAP answers nothing and the probe has no reset line; + # POR it via the Rescue DP and retry once, exactly as the normal + # flash path does (no-op for every other board/failure) + out_txt = ret.stdout if isinstance(ret.stdout, str) else '' + # inside the redirect like its siblings (hil_test slices the result + # object from the first '{' on stdout), and only if a POR + retry + # still fits before the outer kill + rescued = False + try: + with redirect_stdout(sys.stderr): + rescued = hil_flash.rescue_openocd( + board, out_txt, timeout=RECOVER_FLASH_TIMEOUT) + if rescued: + print('DAP wedged; rescued via Rescue DP, retrying reflash', + file=sys.stderr) + with redirect_stdout(sys.stderr): + ret = flash_fn(board, args.recover_fw, + timeout=RECOVER_FLASH_TIMEOUT) + except Exception as e: + # guarded like the first flash: a raise here would unwind past the + # BUDGET back-fill and the JSON print + print(f'rescue/retry raised: {e}', file=sys.stderr) + if ret.returncode != 0: + print(f'reflash failed (rc {ret.returncode}); the device may still ' + f'be wedged', file=sys.stderr) + # settle even on a non-zero exit: the reset may have landed before the + # flasher failed, and the freed ioctl needs a moment to unwind before + # wedged_pids samples + time.sleep(RECOVER_SETTLE) + # Authoritative either way: a clean flash only proves the probe wrote the + # MCU, not that the D-state holder let go. + stuck, complete = wedged_pids(dev['node']) + if stuck: + print(f'{dev["sysname"]}: pid(s) {stuck} still in D state on ' + f'{dev["node"]} — the device lock was never released', file=sys.stderr) + # No hub-worker verdict here: our own testusb still holds the DUT's + # device lock, which is what drives a hub worker into usb_lock_device() + # -- any verdict from here is confounded by construction. + elif not complete: + print('cannot confirm recovery: /proc is only partly readable, so a ' + 'hidden D-state holder cannot be ruled out', file=sys.stderr) + else: + unrecovered_hang = False + break + # re-resolve: a mid-battery re-enumeration changes the devnum and so the node + # path. Match on the concrete serial (not args.serial, which may be None) so + # this can never retarget to another device sharing the VID:PID. + # first=False: the ambiguity guard exists because ONE serial can match two + # sysfs nodes on the dual-port WCH parts, and `dev = live` below makes any + # mistake stick for the rest of the battery -- including wedged_pids() then + # scanning the wrong node and clearing unrecovered_hang on a device it never + # checked. Ambiguous comes back as {'ambiguous': [...]}, handled below. + live = find_device(dev['serial']) + if live and live.get('ambiguous'): + # two nodes now answer to one serial (the dual-port WCH parts do this + # around a re-enumeration). Picking either would file the rest of the + # battery's verdicts under a device we cannot identify, so stop here and + # keep the recovery in play rather than guess. + abort_reason = (f'serial {dev["serial"]} matches more than one device ' + f'({", ".join(live["ambiguous"])}) after case {num}') + unrecovered_hang = True break - # re-resolve: after a mid-battery re-enumeration the devnum (and thus the node - # path) changes; keep testing the live node instead of the stale one. Match on the - # concrete serial (not args.serial, which may be None) so this can never retarget to - # a different device that happens to share the VID:PID. - live = find_device(dev['serial'], first=True) if not live: - results.append({'num': num, 'status': 'FAIL', - 'detail': f'device dropped off the bus after case {num}'}) + # ABSENT vs UNREADABLE: a bounded `serial` read that gave up looks exactly + # like a disconnect from here, and the difference decides whether the + # cleanup below runs. remove_id/unbind take the UNINTERRUPTIBLE + # device_lock (see the driver-registry note above), so performing them + # against a device that is merely unreadable -- i.e. probably wedged -- + # deadlocks the bus rather than tidying up. Fail CLOSED: if anything gave + # up during this scan, treat it as the wedge it probably is, which also + # keeps the recovery and the board_wedged latch in play. + # OUR device's own attribute, not the process-wide sysfs_stranded(): + # that flag is sticky and every DUT here is cafe:4010, so a peer that + # stranded at case 2 would make a genuine disconnect at case 29 report as + # an unrecovered wedge for the rest of the run. + if _hu().path_stranded(str(SYS_USB / dev['sysname'] / 'serial')): + abort_reason = (f'cannot tell whether the device is still present ' + f'after case {num}: its serial read gave up') + unrecovered_hang = True + break + # no second entry for `num`: run_case already recorded it, and a duplicate + # inflates the denominator (31/30) and reports a PASSing case as failed + abort_reason = f'device dropped off the bus after case {num}' break dev = live + if abort_reason and all(c in {r['num'] for r in results} for c in cases) \ + and 'dropped off the bus' in abort_reason and results: + # nothing left to back-fill (the drop happened during/after the LAST case), + # so the run would report a clean pass; the case it died on is not a pass + if results[-1].get('status') == 'PASS': + # only a PASS: a real FAIL/NOTRUN verdict names the actual regression + # (errno, dmesg) and must not be overwritten by the drop message + results[-1] = dict(results[-1], status='FAIL', detail=abort_reason) + if abort_reason: + # One BUDGET entry per case never dispatched, on EVERY abort path: a shrunken + # denominator (4/5 instead of 4/30) hides that most of the battery never + # executed and makes a regression in the skipped range read as "not the + # problem". + ran = {r['num'] for r in results} + results += [{'num': n, 'status': 'BUDGET', 'detail': f'not run: {abort_reason}'} + for n in cases if n not in ran] finally: # best-effort cleanup: a sudo/sysfs failure here (sudo() may sys.exit) must not replace # an exception propagating out of the try body with a less useful one try: if unrecovered_hang: - # testusb is still stuck in a usbfs ioctl holding the device lock; remove_id/unbind - # would join the convoy and deadlock the bus (see usb-kernel-recover skill) — leave it be + # testusb still holds the device lock in a usbfs ioctl: remove_id/unbind + # would join the convoy and deadlock the bus (see usb-kernel-recover) print('skipping cleanup after unrecovered hang: ask the operator for a full PVE host ' 'power cycle (a VM reboot is not reliable — hubs latch up across the PCIe reset)', file=sys.stderr) elif not args.keep_binding: - sysfs_write(DRIVER / 'remove_id', f'{VID} {PID}', check=False) - # release every claimed interface: other devices sharing the VID:PID (stale example - # firmware on a test rig) may have been grabbed on probe and would otherwise stay - # bound to usbtest until re-plugged, hijacking the next test's device - for intf in DRIVER.glob('*:*'): - sysfs_write(DRIVER / 'unbind', intf.name, check=False) + # PROCESS-WIDE, unlike the per-case verdict above. That one is per-DUT on + # purpose -- a peer that stranded must not make OUR board report wedged. + # This cleanup is GLOBAL: it unbinds every interface under the driver, + # including the peer we could not read, and unbind takes the + # uninterruptible device_lock. Narrowing this gate to path_stranded() + # would add a driver-registry writer to an existing wedge. + # INSIDE keep_binding rather than before it: hil_test always passes that + # flag, so a check further out announced a skip of cleanup that was never + # going to run -- one line of noise ahead of the real cause in every + # stranded row. ONE line for the same reason: the finally runs before + # SystemExit's message reaches stderr. + if _hu().sysfs_stranded(): + print('cleanup skipped: a sysfs read gave up, so unbind could take a ' + 'wedged device lock', file=sys.stderr) + else: + sysfs_write(DRIVER / 'remove_id', f'{VID} {PID}', check=False) + # release every claimed interface: another device sharing the VID:PID + # (stale example firmware) may have been grabbed on probe and would + # stay bound to usbtest until re-plugged, hijacking the next test + for intf in DRIVER.glob('*:*'): + sysfs_write(DRIVER / 'unbind', intf.name, check=False) except SystemExit: pass - failed = [r for r in results if r['status'] != 'PASS'] + # BUDGET, not NOTRUN: NOTRUN is taken, for a case the KERNEL gated off (-EOPNOTSUPP, + # see run_case) -- a real result that must stay in `failed` and keep its case number. + # BUDGET keeps the denominator honest without lying about the numerator: naming cases + # that never executed as failures sends a maintainer bisecting one of them. + notrun = [r for r in results if r['status'] == 'BUDGET'] + failed = [r for r in results if r['status'] not in ('PASS', 'BUDGET')] ran = len(results) if args.json: + # `wedged` is the verdict this process ALREADY computed; without it the caller had + # to infer one from 'HUNG' in our stdout, which misses a recovery that ran and + # failed, the ambiguous abort (no case reaches status HUNG), and any battery + # killed before it printed. print(json.dumps({'serial': dev['serial'], 'speed': dev['speed'], 'tier': tier, - 'passed': ran - len(failed), 'failed': len(failed), + 'passed': ran - len(failed) - len(notrun), + 'failed': len(failed), 'notrun': len(notrun), + 'wedged': bool(unrecovered_hang), 'cases': results}, indent=2)) else: - print(f"{ran - len(failed)}/{ran} passed") + print(f"{ran - len(failed) - len(notrun)}/{ran} passed" + + (f", {len(notrun)} not run" if notrun else "")) for r in failed: print(f" FAILED test {r['num']}: {r.get('detail', '')}") if r.get('dmesg'): print(' ' + r['dmesg'].replace('\n', '\n ')) - return len(failed) + # NOTRUN counts toward the exit status even though it is reported separately: a + # standalone run whose cases were all skipped has NOT passed, and returning 0 hands a + # false success to any script driving this directly. + return len(failed) + len(notrun) if __name__ == '__main__': |