summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorHa Thach <[email protected]>2026-08-20 22:49:07 +0700
committerGitHub <[email protected]>2026-08-20 22:49:07 +0700
commit9466f3cda69b052679e7bb078b5cf7906ddbea27 (patch)
treed1cc160b3a0c34c96869a407c86e0e080c373bf1
parent7800876bf151a239046521232dc4603b156061be (diff)
parent0fa0ece024fecae0847459b5949c66f40fcc6e11 (diff)
Merge pull request #3836 from hathach/claude/hil-doc-audit
hil: one-run scheduling with JSON result handoff; audit and correct the .claude instruction surface
-rw-r--r--.claude/agents/builder.md2
-rw-r--r--.claude/agents/hil-operator.md49
-rw-r--r--.claude/skills/esp-target-debug/SKILL.md2
-rw-r--r--.claude/skills/hil-pool-check/SKILL.md81
-rw-r--r--.claude/skills/hil/SKILL.md18
-rw-r--r--.claude/skills/usb-kernel-recover/SKILL.md41
-rw-r--r--.claude/skills/usbtest/SKILL.md28
-rw-r--r--.claude/workflows/hil-validate.js134
-rw-r--r--.claude/workflows/test-hil-validate.mjs69
-rw-r--r--.gitignore1
-rw-r--r--.pre-commit-config.yaml9
-rw-r--r--CLAUDE.md4
-rw-r--r--docs/superpowers/followup/pr3836-report-single-source.md470
-rw-r--r--docs/superpowers/plans/2026-08-18-claude-doc-audit.md518
-rw-r--r--docs/superpowers/specs/2026-08-18-claude-doc-audit-design.md135
-rw-r--r--test/hil/helper/hil_pool_check.py6
-rw-r--r--test/hil/helper/hil_summary.py115
-rw-r--r--test/hil/hil_ci.sh253
-rw-r--r--test/hil/test/test_hil_bounded.py317
19 files changed, 2127 insertions, 125 deletions
diff --git a/.claude/agents/builder.md b/.claude/agents/builder.md
index 70648e80d..4edb7e0d4 100644
--- a/.claude/agents/builder.md
+++ b/.claude/agents/builder.md
@@ -25,7 +25,7 @@ cmake -S examples/<group>/<example> -B "$BUILD" -DBOARD=<BOARD> -G Ninja -DCMAKE
cmake --build "$BUILD"
```
-Espressif boards (listed under `hw/bsp/espressif/boards/`): run `. $HOME/code/esp-idf/export.sh` first; only ESP-IDF examples build for them (e.g. `cdc_msc_freertos`): `idf.py -DBOARD=<BOARD> build` from the example dir.
+Espressif boards (listed under `hw/bsp/espressif/boards/`): run `. "$IDF_PATH/export.sh"` first (`IDF_PATH` is the official ESP-IDF variable, exported per host); only ESP-IDF examples build for them (e.g. `cdc_msc_freertos`): `idf.py -DBOARD=<BOARD> build` from the example dir.
## Recovery rules
diff --git a/.claude/agents/hil-operator.md b/.claude/agents/hil-operator.md
index ebc9251cc..a37501211 100644
--- a/.claude/agents/hil-operator.md
+++ b/.claude/agents/hil-operator.md
@@ -15,14 +15,22 @@ You operate physical USB test hardware. These repo skills are your source of tru
The GitHub Actions runner keeps running during your work. Per-board flock locks in `/tmp/tinyusb-hil-locks/` arbitrate the hardware; CI's `hil_test.py` fails fast on locked boards (re-runnable later).
-- `python3 test/hil/hil_test.py ...` runs: do NOT pre-hold those boards — `hil_test.py` self-locks each board for its flash+test and would fail fast with `board locked` against your own hold.
+- `python3 test/hil/hil_test.py ...` runs: do NOT pre-hold those boards — `hil_test.py` self-locks each board for its flash+test and would fail fast with `board locked` against your own hold. Several boards go into ONE run as repeated `-b`, never into several runs.
- ANY other hardware action (JLinkExe/openocd/GDB, manual flash, usbtest.py, serial poking): hold first, release when done — release is mandatory cleanup (a crashed holder auto-releases via kernel flock, but do not rely on it):
```bash
python3 test/hil/helper/hil_lock.py hold <board...> --reason "<task>"
# ... hardware work ...
python3 test/hil/helper/hil_lock.py release <board...>
```
-- Rig-wide operations (uhubctl power cycling, controller resets — they renumber buses): `python3 test/hil/helper/hil_lock.py hold --all --reason "<why>"` first.
+- Rig-wide operations — uhubctl power cycling, `usb_recover.sh root-cycle`, pci-rebind,
+ controller resets — need `python3 test/hil/helper/hil_lock.py hold --all --config <this host's config> --reason "<why>"`
+ first, even a single root-port bounce. `--all` is coarse for a bounce, but it is the only
+ correct reservation available: the affected siblings are sysfs busports, nothing maps a
+ busport to a board name (the pool check's topology report counts devices per subtree, it does
+ not name them), and `hil_lock.py hold` validates nothing against the roster — so passing it
+ `13-1.6` creates a lock file for a board that does not exist and reserves nothing while
+ reporting success. If `--all` cannot be taken, wait: a partial hold is worse than none,
+ because it reads as protection.
- If a lock is already held by someone else: report holder/reason (`hil_lock.py status`) — never force, never kill the holder. If the holder's reason is `hil_test.py`, that is a concurrent CI job mid-test on the board: waiting a few minutes and retrying once is appropriate when your task allows; otherwise return the holder info so the orchestrator can ask the user.
- You cannot ask the user anything. Bypassing a lock (`HIL_NO_BOARD_LOCK=1`, or proceeding with manual hardware work despite a held lock) is allowed ONLY when your prompt explicitly states the user authorized forcing.
@@ -33,11 +41,40 @@ The GitHub Actions runner keeps running during your work. Per-board flock locks
Bash tool's 10 min foreground cap: run it in the background and wait
for the completion notification. A foreground timeout kills the run before hil_test.py
can write its report. NEVER cancel early.
-- One hardware action at a time. You are never run concurrently with another hil-operator.
-- On test failure: retry once with `-v -r 1` appended (one verbose attempt for diagnosis; a usbtest battery that produced per-case verdicts is NOT auto-retried, so its result already stands). If a board/fixture stops enumerating, or a tool of YOURS hangs in D state, consult usb-kernel-recover and capture `dmesg | tail -50` into `detail`; set `wedged` true. A `> **Rig note.**` banner reporting someone else's D-state process is not that — see the hil skill's banner list.
+- One hardware action at a time. You are never run concurrently with another hil-operator, and a
+ multi-board `hil_test.py` run is ONE action: hand it every board as repeated `-b` and let it
+ schedule them — it round-robins boards across host controllers and budgets simultaneous flashes
+ and usbtest batteries per controller. Those budgets live in one process, so a second
+ `hil_test.py` alongside the first does not share them and the rig sees double the configured
+ width. (Do not read that as the cause of a dead card: hil_lock.py:128-131 records that every
+ observed uPD720201 death traced to a marginal DUT port bouncing under concurrent batteries,
+ and that lowering the widths does not fix a bad port — fix the port or pull the board.)
+- On test failure, retry ONCE, with `-v` for diagnosis. Retry from the spec the run just wrote —
+ `<config>.failed`, which already begins with `--accumulate` and restricts each board to its
+ failed tests via `-bt`. If you compose the retry by hand you MUST pass `--accumulate` yourself:
+ a fresh run unlinks the report, so a hand-scoped `-b <board>` retry replaces the whole-fleet
+ table with a one-row table. A usbtest battery that produced per-case verdicts is NOT auto-retried,
+ so its result already stands. If a board/fixture stops enumerating, or a tool of YOURS hangs in D
+ state, consult usb-kernel-recover and capture `dmesg | tail -50` into `detail`; set `wedged` true.
+ A `> **Rig note.**` banner reporting someone else's D-state process is not that — see the hil
+ skill's banner list.
## Output contract
-Your final message is parsed by a program. Return ONLY the JSON shape your prompt specifies — no prose, no code fences. Typical board-run shape:
+Your final message is parsed by a program. Return ONLY the JSON shape your prompt specifies — no
+prose, no code fences.
-{"board": "raspberry_pi_pico", "pass": true, "detail": "<per-test summary or first failure>", "wedged": false}
+For a board run, do NOT transcribe the report table. Run the tests, then hand back the machine
+output verbatim:
+
+```bash
+python3 test/hil/helper/hil_summary.py <config> -b BOARD [-b BOARD...] # from the report dir
+```
+
+`{"results": <its results array, verbatim>, "banner": <its banner, verbatim>, "wedged": ["board", ...]}`
+
+`results` and `banner` are copied, never retyped, reworded or re-ordered: report rows are named
+per variant, a variant name need not start with the board name, and lock contention is a cell
+rather than a phrase, so re-deriving any of it by hand is how this contract broke before.
+`wedged` is yours — the boards your run left unresponsive, usually none — and the only field you
+author.
diff --git a/.claude/skills/esp-target-debug/SKILL.md b/.claude/skills/esp-target-debug/SKILL.md
index 1af9fcf7f..7b21788f1 100644
--- a/.claude/skills/esp-target-debug/SKILL.md
+++ b/.claude/skills/esp-target-debug/SKILL.md
@@ -48,7 +48,7 @@ UART side is also the remote reset: `esptool.py --after hard_reset read_mac`.
## Attach
```bash
-. $HOME/code/esp-idf/export.sh # openocd-esp32, riscv32-/xtensa-esp32s3-elf-gdb, esptool
+. "$IDF_PATH/export.sh" # openocd-esp32, riscv32-/xtensa-esp32s3-elf-gdb, esptool
openocd -c 'set ESP_RTOS FreeRTOS' -f board/esp32p4-builtin.cfg \
-c 'adapter serial <MAC-with-colons>' & # S3: board/esp32s3-builtin.cfg
riscv32-esp-elf-gdb -batch -ex 'target extended-remote :3333' \
diff --git a/.claude/skills/hil-pool-check/SKILL.md b/.claude/skills/hil-pool-check/SKILL.md
index 6a8f66087..65e28b65f 100644
--- a/.claude/skills/hil-pool-check/SKILL.md
+++ b/.claude/skills/hil-pool-check/SKILL.md
@@ -39,9 +39,14 @@ ssh ci.lan 'bash -lc "cd ~/code/tinyusb && python3 test/hil/helper/hil_pool_chec
## Notes
Missing firmware is **built on the spot** — never skipped (`--no-build` opts out; those boards
-then report `flash-failed`). Builds need the family env, exported on the rig in
-`~/.profile`/`~/.bashrc`: `PICO_SDK_PATH` for rp2040/rp2350 (`~/code/pico/pico-sdk`), the
-ESP-IDF env (`get-idf`) for espressif — which also needs `esptool` on PATH (pip's
+then report `flash-failed`). Builds need the family env, referenced by its OFFICIAL variable so the docs hold on any
+rig: `PICO_SDK_PATH` for rp2040/rp2350, `IDF_PATH` for espressif — activated explicitly as
+`. "$IDF_PATH/export.sh"`, never as `get-idf` (an interactive alias; aliases are not expanded
+in non-interactive shells, so scripts get `get-idf: command not found` even under `bash -lc`).
+Each host exports both vars in `~/.bashrc` ABOVE the interactive early-return, which is what
+makes a plain non-interactive `ssh <rig> 'cmd'` see them (verified on ci; where the checkouts
+live is that host's business, not this file's). It also
+needs `esptool` on PATH (pip's
`~/.local/bin/esptool`; a non-login shell may lack it — run via `bash -lc`). An explicit `-B` is
searched exclusively for *existing* firmware; builds still land in `cmake-build/` and are noted
`built <example>`. Espressif boards park too when the IDF env is present. A first run on an
@@ -57,8 +62,78 @@ are *unverified*, not healthy — read the footer, not just `$?`. A `⚠ pid …
means stale firmware or a silent flash no-op (J-Link lore); a device off the bus entirely needs
the usb-kernel-recover skill or a physical replug.
+## When the tool's probe recovery fails
+
+`flash-failed` with the probe ✅ present and a `probe toggle unconfirmed` note means the probe's
+own firmware is wedged, not the board. The tool's recovery is an `authorized` toggle, which is a
+USB re-enumeration and never removes power, so probes that keep their sysfs kobject across it
+(ST-Link, WCH-Link, CP210x, picoprobe) survive the toggle still wedged. Confirm with the flasher's
+own list — `STM32_Programmer_CLI -l st-link`, or `JLinkExe -CommandFile <script>` with
+`ShowEmuList` in it: a probe that enumerates but reports a blank serial/firmware is answering the
+kernel and not the tool, which is a host-to-probe fault. A dead target reports the opposite: the
+probe identifies itself normally and then fails to connect.
+
+The next rung is a root-port bounce, and what it buys depends on which card the probe hangs off
+(`readlink -f /sys/bus/usb/devices/usb<bus>` gives the PCI address):
+
+- **Renesas** (five cards here): `uhubctl` lists their root hubs as `ppps`-capable, but the cards
+ do not implement it — VBUS never drops, only D+/D− (see usb-kernel-recover). A cycle is therefore
+ a harder forced re-enumeration, **not** a power cycle: worth one attempt, but a probe that rode
+ out the `authorized` toggle can ride this out too. Do not read `ppps` here as power control.
+- **AMD `0000:02:00.0`** (where the WCH-Links live): no port-power switching at all — `uhubctl`
+ does not list it. There is nothing to cycle; go straight to a physical replug.
+
+The leaf hubs are ganged, so a bounce hits every device under that root port. Escalate by hand, in
+this order:
+
+1. **Let the full run finish first.** Never cycle mid-run: the bounce re-enumerates siblings and
+ would corrupt the checks still in flight for other boards.
+2. Identify the subtree and its blast radius, so the report can name what was disturbed:
+ ```bash
+ ls -d /sys/bus/usb/devices/<bus>-<rootport>.* # siblings that will be bounced
+ ```
+3. **Hold `--all` for the cycle, and release before the re-check.** `hil_lock.py status` only
+ observes; CI can take a board a second later and flash straight into the bounce. The bounce
+ hits every board under the root port and nothing maps a sysfs busport to a board name, so
+ `--all` is the only reservation that actually covers them:
+ ```bash
+ python3 test/hil/helper/hil_lock.py hold --all --config test/hil/tinyusb.json --reason "probe power cycle"
+ ```
+ It is all-or-nothing: a refusal naming `hil_test.py` means a CI job is mid-test — wait, do
+ not force, and do not substitute a partial hold. Release before step 5: `hil_pool_check.py`
+ self-locks every board it checks and reports 🔒 locked for any it cannot take, so a hold
+ still in place makes the whole verification pass report locked against you and verify nothing.
+4. Cycle the ROOT port through the recovery script — rung 2 of usb-kernel-recover, which owns this
+ invocation:
+ ```bash
+ sudo .claude/skills/usb-kernel-recover/scripts/usb_recover.sh \
+ root-cycle <the wedged probe's own busport> [expected-serial]
+ # e.g. 13-1.6, NOT the 13-1 hub path from step 2 — the script derives the root port
+ # itself. Give the full path: the script ships inside the checkout, is on no PATH, and
+ # sudo's secure_path excludes the repo, so a bare `usb_recover.sh` is command-not-found
+ ```
+ Give it the device, not the hub: the expected-serial guard and the success check both read the
+ path you pass, so handing it the hub compares the hub's serial and watches the hub's inode,
+ which always changes when its own root port is cycled — it prints success while the probe is
+ still dead. **Never a bare `uhubctl -a cycle` here.** Without `-S` it writes sysfs `disable`,
+ whose `disable_store` takes the root hub's lock uninterruptibly and then calls
+ `usb_disconnect()` on the child — against the wedged probe you are trying to clear, that blocks
+ while holding the root hub's lock and poisons the whole bus. The script passes `-S`.
+5. Release the locks, then re-check the affected boards:
+ `python3 test/hil/helper/hil_pool_check.py -b BOARD [-b …]`. Include the bounced siblings — a
+ cycle that fixes one probe can leave another unenumerated.
+
+If the second pass still fails, the probe needs a physical replug: no software rung on this rig
+removes VBUS, so there is nothing further to try.
+
## Reporting
The user-facing answer to a pool check IS the tool's summary table: paste the complete per-board
table (and footer counts) verbatim — never truncate rows or reduce it to a prose digest like
"27/27 healthy"; at most one line of commentary below it.
+
+When an escalation above was needed, add a short note under the table naming: which boards needed
+it, which root port was cycled (or that a replug was needed instead), which siblings bounced, and
+the second-pass result for each.
+Report BOTH passes — a final table showing every board ok hides the fact that a probe had to be
+power-cycled to get there, which is exactly the signal that predicts it recurring.
diff --git a/.claude/skills/hil/SKILL.md b/.claude/skills/hil/SKILL.md
index 093f345b2..f0c449d33 100644
--- a/.claude/skills/hil/SKILL.md
+++ b/.claude/skills/hil/SKILL.md
@@ -32,7 +32,7 @@ python3 test/hil/helper/hil_lock.py release BOARD [BOARD...]
```
- Never pre-hold boards you are about to run `hil_test.py` on — it self-locks and would treat your own hold as a conflict.
-- Rig-wide operations (uhubctl power cycling, controller resets — bus renumbering) affect every board: `hil_lock.py hold --all --reason "..."` first.
+- Rig-wide operations (uhubctl power cycling, `usb_recover.sh root-cycle`, pci-rebind, controller resets — bus renumbering) affect every board: `hil_lock.py hold --all --config <this host's config> --reason "..."` first — `--all` defaults to `tinyusb.json`, so on `tusb` it would reserve 27 boards that do not exist there and none of the three that do. Even a single root-port bounce needs `--all`: nothing maps a sysfs busport to a board name, and `hil_lock.py hold` accepts any string, so a "just the siblings" hold reserves nothing while reporting success.
- `hil_lock.py status` lists holders. Locks auto-release when the holder process dies (kernel flock); `/tmp` clears on reboot.
- Forcing past a lock: `HIL_NO_BOARD_LOCK=1 python3 test/hil/hil_test.py ...` bypasses the guard without killing the holder. Only with the user's explicit go-ahead — they accept the risk of colliding with whatever holds the board.
@@ -81,7 +81,7 @@ Examples must be built for the target board(s) — see CLAUDE.md "Build" → "Al
## Arguments
-- **Board:** `-b BOARD_NAME` for one board; omit to run all boards in the config.
+- **Board:** `-b BOARD_NAME`, repeatable for a subset (`-b a -b b`); omit to run all boards in the config. Give a whole set to ONE run rather than one run per board: it schedules the boards across host controllers and budgets concurrent flashes and usbtest batteries per controller (`hil_lock.py` `FLASH_PARALLEL`/`USBTEST_PARALLEL`). Those permits are in-process semaphores — a second `hil_test.py` running alongside does not share them, it multiplies the load on the same xHCI cards.
- **Pass-through:** `-v`, `-r N`, etc. forwarded unchanged.
If `local.json` is missing on a dev PC, ask the user to supply one (only fall back to `tinyusb.json` if told to).
@@ -108,10 +108,13 @@ python3 test/hil/hil_test.py -b stm32f723disco -B examples "$CONFIG"
# All boards:
bash test/hil/hil_ci.sh
-# A single board, with pass-through flags:
-bash test/hil/hil_ci.sh -b raspberry_pi_pico2 -t host/cdc_msc_hid -r 1
+# A subset — repeat -b, ONE invocation for the whole set:
+bash test/hil/hil_ci.sh -b raspberry_pi_pico2 -b stm32f723disco -t host/cdc_msc_hid -r 1
```
+One invocation per board is wrong here, not merely slow: each run `rm -rf`s `REMOTE_DIR`
+and rewrites the report, so only the last board's rows survive.
+
Env overrides: `REMOTE`, `REMOTE_DIR`, `CONFIG`. Fails fast if the build dir/repo layout is missing.
## Timing
@@ -155,5 +158,8 @@ PREFIX, since each carries trailing detail and one is a blockquote:
"device not found" from the named boards means "could not tell". Do NOT report their red
cells as broken boards.
-On failure, retry with `-v`; if that's not enough, add temporary debug prints to
-`hil_test.py`.
+On failure, retry once with `-v` — from the `<config>.failed` spec the run just wrote, which
+already begins with `--accumulate` and restricts each board to its failed tests. A hand-scoped
+`-b <board>` retry MUST pass `--accumulate` too: a fresh run unlinks the report, replacing the
+whole-fleet table with a one-row table. If that is still not enough, add temporary debug prints
+to `hil_test.py`.
diff --git a/.claude/skills/usb-kernel-recover/SKILL.md b/.claude/skills/usb-kernel-recover/SKILL.md
index 009090769..7bb5fc1ba 100644
--- a/.claude/skills/usb-kernel-recover/SKILL.md
+++ b/.claude/skills/usb-kernel-recover/SKILL.md
@@ -105,16 +105,32 @@ convoy-safe: **openocd pinned with `vid_pid`**, or esptool (`-p <ttyACM>`).
JLinkExe selects by serial, which needs `libusb_open`, so it needs the shield.
**Rung 2 — wedged PROBE: `root-cycle`.** A probe has no probe to reset it, so the
-port-side drop is the only lock-free lever left. It commands the ROOT hub and
-never touches the wedged device's lock.
+port-side drop is the only lever left that avoids the KERNEL device lock. It
+commands the ROOT hub and never touches the wedged device's `device_lock` — which
+is exactly why rungs 1 and 3 are dangerous and this one is not.
+
+That is a different lock from the rig's **board flocks**, and this rung still needs
+those: it bounces every fixture under the root port, including boards another job is
+mid-flash on. Take them first, and release after:
```bash
-sudo usb_recover.sh root-cycle <busport> [expected-serial]
+python3 test/hil/helper/hil_lock.py hold --all --config <this host's config> --reason "root-cycle <busport>"
+sudo .claude/skills/usb-kernel-recover/scripts/usb_recover.sh root-cycle <busport> [expected-serial]
+python3 test/hil/helper/hil_lock.py release --all # no --config: it walks the lock dir
```
-Bounces **every fixture under that root port** (up to 25 here). Renesas `ppps`
-disables D+/D− only — VBUS stays up, so it is a forced re-enumeration, not a
-power cycle. Success is the sysfs inode changing, not uhubctl's exit code.
+`--all` is coarse for one root port, but nothing maps a sysfs busport to a board name,
+so it is the only reservation that actually covers the blast radius; `hold` accepts any
+string, so a hand-listed "just the siblings" hold reserves nothing while reporting
+success. A refusal naming `hil_test.py` means CI is mid-test — wait, do not force. Give
+the script the wedged probe's own busport (e.g. `13-1.6`), not the `13-1` hub path: it
+derives the root port itself, and the expected-serial guard and the success check both
+read the path you pass.
+
+Bounces **every fixture under that root port** (up to 25 here). The Renesas cards
+advertise `ppps` but do not implement it: VBUS stays up and only D+/D− drop, so
+this is a forced re-enumeration, never a power cycle — a device whose firmware is
+wedged can ride it out. Success is the sysfs inode changing, not uhubctl's exit code.
**Rung 3 — terminal case: a driver ioctl that OWNS the lock.** No software cure:
the task is uninterruptible and SIGKILL is queued, not delivered. Reboot with
@@ -201,6 +217,13 @@ Observed: 5 boards missing with a completely clean D-state list, because
## Rig layout (ci.lan, bus numbers renumber every boot)
-`readlink -f /sys/bus/usb/devices/usb<N>` → its PCI address. AMD `0000:02:00.0`
-has no port-power switching; Renesas `0000:01:00.0` (probe tree) and
-`0000:03:00.0`/`0000:05:00.0` (DUT hubs) have real per-port `ppps`.
+`readlink -f /sys/bus/usb/devices/usb<N>` → its PCI address; `sudo uhubctl` lists
+the root hubs it can drive, against their PCI address. Five Renesas uPD720201 cards
+— `0000:01:00.0`, `03`, `04`, `05`, `06:00.0` — advertise per-port `ppps` on both
+their USB2 and USB3 root hubs, **but do not implement it**: the silicon never drops
+VBUS, so a cycle re-enumerates the port and nothing more (above). Do not read
+`uhubctl`'s `ppps` as power control on this rig. AMD `0000:02:00.0` does not appear
+in `uhubctl` at all — no switching of any kind. Which tree holds which probes moves
+with re-cabling, so derive it (`lsusb -s <bus>:`) rather than trusting a stored map.
+Verified 2026-08-18; `sudo` is passwordless for `hathach` here, so every rung above
+runs without a prompt.
diff --git a/.claude/skills/usbtest/SKILL.md b/.claude/skills/usbtest/SKILL.md
index 32c2be913..4d7c6e5ab 100644
--- a/.claude/skills/usbtest/SKILL.md
+++ b/.claude/skills/usbtest/SKILL.md
@@ -21,7 +21,8 @@ the failing case passing *and* the full battery still at 30/30 across reflash cy
## Run
```bash
-# build (cmake); descriptor sizes auto-adapt per MCU via src/usb_descriptors.h + src/tusb_config.h
+# build (cmake); descriptor sizes auto-adapt per MCU via the example's own
+# src/usb_descriptors.h + src/tusb_config.h (paths below are relative to it)
cd examples/device/usbtest && cmake -B build -DBOARD=<board> -G Ninja -DCMAKE_BUILD_TYPE=MinSizeRel && cmake --build build
# flash, wait ~3-5 s for enumeration to settle, then:
python3 test/hil/usbtest.py --serial <uid> --keep-binding # full battery for the advertised tier
@@ -36,9 +37,15 @@ python3 test/hil/usbtest.py --serial <uid> --keep-binding --tests 29 # one case
device wedged and skip cleanup — expected; reset or reflash it yourself.
- Always settle a few seconds after flashing — enumeration can bounce once; testusb into the gap sees
the device drop mid-case.
-- On a CI rig: stop the actions runner before touching hardware; restart after. Never run two
- batteries concurrently (hil_test.py serializes them; concurrent batteries have hard-frozen a rig
- via a fatal PCIe error on a VFIO-passed xHCI).
+- On a CI rig: hold the board lock before touching hardware and release it after — never stop the
+ actions runner. It keeps running; the per-board flock is what arbitrates (see the `hil` skill).
+ Never start a battery by hand next to a running one: `hil_test.py` budgets 2 concurrent batteries
+ per host controller (`HIL_USBTEST_PARALLEL`). The width itself is a profiled throughput/bandwidth
+ trade, not a safety ceiling (hil_lock.py:122-127) — but a battery outside the budget is a real
+ hazard, and the hazard is recorded: unbudgeted concurrent batteries have hard-frozen the rig with
+ a fatal PCIe error on a VFIO-passed xHCI, and a marginal DUT port bouncing under concurrent
+ batteries has killed a uPD720201 outright, which lowering the widths does not fix
+ (hil_lock.py:130-132).
## Porting ladder — new MCU/DCD to 30/30
@@ -47,9 +54,9 @@ python3 test/hil/usbtest.py --serial <uid> --keep-binding --tests 29 # one case
2. **Tier 2 (ctrl_out 14/21)**, **tier 3 (interrupt 25/26)**, **tier 4 (iso 15/16/22/23)** — raise
the tier only when the layer below is clean; run the *full* battery after each layer.
3. **Fit the endpoints**: tier 4 needs 6 endpoints + EP0. Small parts need per-MCU mps/epbuf
- overrides in `src/usb_descriptors.h` (`USBTEST_INT/ISO_EP_MPS_FS`) and `src/tusb_config.h`
- (`CFG_TUD_VENDOR_TX_EPSIZE`) — follow the existing CH32/LPC11 patterns. Parts that can't fit go
- in `skip.txt`.
+ overrides in the example's own `src/usb_descriptors.h` (`USBTEST_INT/ISO_EP_MPS_FS`) and
+ `src/tusb_config.h` (`CFG_TUD_VENDOR_TX_EPSIZE`) — follow the existing CH32/LPC11 patterns.
+ Parts that can't fit go in `skip.txt`.
4. **Sign-off = reliability, not one pass**: 3–10 full flash→battery cycles. One 30/30 proves
nothing on a flaky bring-up; deterministic partial counts (e.g. exactly 1-in-8 lost) are a
signature, not noise — chase them.
@@ -98,7 +105,8 @@ whether a hung case is recoverable. Fetch the rig's exact version (`uname -r`):
```bash
curl -sO "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/plain/drivers/usb/misc/usbtest.c?h=v6.12.96"
-# case N lives under `case N:` in usbtest_do_ioctl(); tools/usb/testusb.c maps the flags:
+# case N lives under `case N:` in the kernel's usbtest_do_ioctl()
+# (drivers/usb/misc/usbtest.c); kernel tools/usb/testusb.c maps the flags:
# -c = param.iterations, -s = param.length, -g = param.sglen (NOT what they read like)
```
@@ -147,7 +155,9 @@ curl -sO "https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/plain
- "usbmon shows no toggle problem" → usbmon can't see toggles.
- "It works on gcc" → clang/IAR/LTO/make still pending.
- "Fixed iso IN" → apply the same exemption to iso OUT (toggle logic is symmetric).
-- A clean single-board run does not validate concurrent/fleet behavior — batteries serialize.
+- A clean single-board run does not validate concurrent/fleet behavior — a fleet run puts up to 2
+ batteries per host controller (`HIL_USBTEST_PARALLEL`) plus concurrent flashes on the same hub
+ uplinks, which one board never exercises.
- Reasoning about a case from its name or table row → open `usbtest.c` (step 0). The
flags don't mean what they look like, and recoverability is a property of that
case's wait, not of the rig.
diff --git a/.claude/workflows/hil-validate.js b/.claude/workflows/hil-validate.js
index 136f9075e..bc0bda8b9 100644
--- a/.claude/workflows/hil-validate.js
+++ b/.claude/workflows/hil-validate.js
@@ -1,60 +1,134 @@
export const meta = {
name: 'hil-validate',
- description: 'Serialized hardware-in-the-loop run: flash+test each board with hil-operator; per-board flock locks arbitrate with concurrent CI (the actions-runner keeps running)',
- whenToUse: 'After validate passes, to exercise built firmware on the physical rig. Requires examples/cmake-build-<board> for each board. If the result has non-empty `locked`, ask the user: force (re-invoke with force: true), continue waiting (re-invoke later), or accept the partial result. Pass force: true ONLY with explicit user authorization.',
- phases: [{ title: 'HIL', detail: 'strictly serial per-board hil-operator runs' }],
+ description: 'Hardware-in-the-loop run: one hil-operator flashes and tests every board in a single hil_test.py run; per-board flock locks arbitrate with concurrent CI (the actions-runner keeps running)',
+ whenToUse: 'After validate passes, to exercise built firmware on the physical rig. Requires the boards to be built (examples/cmake-build-<board>, plus a dir per declared variant). If the result has non-empty `locked`, ask the user: force (re-invoke with force: true), continue waiting (re-invoke later), or accept the partial result. Pass force: true ONLY with explicit user authorization.',
+ phases: [{ title: 'HIL', detail: 'one hil-operator, every board in one hil_test.py run' }],
}
// args: { boards: string[], force?: boolean }
if (typeof args === 'string') { try { args = JSON.parse(args) } catch { /* not JSON: shape check below reports it */ } }
if (!args || !Array.isArray(args.boards) || args.boards.length === 0) {
- throw new Error('args must be { boards: string[], force? } with examples/cmake-build-<board> already built')
+ throw new Error('args must be { boards: string[], force? } with the boards already built')
}
+// The operator returns hil_summary.py's JSON verbatim plus its own observations. It does NOT
+// retype the report table: rows are named per variant, a variant need not start with the board
+// name, and lock contention is a cell rather than a phrase — rebuilding board identity from
+// prose produced a defect in each of four review rounds. hil_summary.py does that join against
+// the roster, so `locked` and `ran` arrive as fields and nothing here parses a detail string.
+const BOARD = {
+ type: 'object', additionalProperties: false,
+ required: ['board', 'ran', 'pass', 'locked', 'detail'],
+ properties: {
+ board: { type: 'string' }, ran: { type: 'boolean' }, pass: { type: 'boolean' },
+ locked: { type: 'boolean' }, detail: { type: 'string' },
+ },
+}
const HIL = {
type: 'object', additionalProperties: false,
- required: ['board', 'pass', 'detail', 'wedged'],
+ required: ['results', 'wedged'],
properties: {
- board: { type: 'string' }, pass: { type: 'boolean' },
- detail: { type: 'string' }, wedged: { type: 'boolean' },
+ results: { type: 'array', items: BOARD },
+ // the operator's own observation — not derivable from the report
+ wedged: { type: 'array', items: { type: 'string' } },
+ banner: { type: 'string' },
},
}
-const runBoard = (b) => agent(
- `Run the HIL test for board ${b} per .claude/skills/hil/SKILL.md. Do NOT touch the actions-runner service and do NOT pre-hold the board lock — hil_test.py self-locks the board while testing. ` +
+// ONE operator for the whole set, and one hil_test.py inside it. hil_test.py already runs the
+// boards concurrently: it round-robins them across host controllers and holds per-controller
+// permits (hil_lock.py FLASH_PARALLEL/USBTEST_PARALLEL) that bound simultaneous flashes and
+// usbtest batteries. Those permits live in one process, so a second hil_test.py does not share
+// them - N parallel single-board runs multiply the budget by N onto the same xHCI cards, for no
+// wall-clock gain over one run that already parallelizes them.
+const runBoards = (boards, isRetry = false) => agent(
+ `Run the HIL tests for these boards per .claude/skills/hil/SKILL.md: ${boards.join(', ')}. ` +
+ `Pass them ALL to ONE hil_test.py invocation as repeated -b flags (${boards.map((b) => `-b ${b}`).join(' ')}) — it schedules them across host controllers and budgets concurrent flashes and usbtest batteries itself. Never start a second hil_test.py alongside it. ` +
+ (isRetry
+ ? 'This is a RE-RUN of boards an earlier run could not take: pass --accumulate as well, or hil_test.py unlinks the report and the whole-fleet table collapses to just these boards. '
+ : '') +
+ 'Do NOT touch the actions-runner service and do NOT pre-hold the board locks — hil_test.py self-locks each board for its flash+test. ' +
(args.force
? 'THE USER HAS EXPLICITLY AUTHORIZED FORCING: run hil_test.py with HIL_NO_BOARD_LOCK=1 in the environment (bypasses the board lock check; do NOT release or kill the existing holder). '
- : 'If the run fails because the board lock is held (a dev session or concurrent CI job), report pass=false and set detail to start EXACTLY with "board locked:" followed by the holder JSON verbatim — never force the lock. ') +
- 'Reserve the phrase "board locked" strictly for lock contention; describe a frozen or non-enumerating board as "unresponsive" instead. ' +
- `Firmware is in examples/cmake-build-${b}. Use the config for this host (hostname first), single-board flag -b ${b}. Run hil_test.py as a BACKGROUND Bash task and wait for it (a stuck fleet runs to its pool guard, 60 min by default — beyond any foreground timeout); never cancel it early. ` +
- 'On non-lock failures retry once with -v -r 1 (one verbose attempt for diagnosis; note a usbtest battery that produced per-case verdicts is NOT auto-retried, so its result already stands). wedged=true if the board/fixture is unresponsive after the run (capture dmesg | tail -50 into detail).',
- { label: `hil:${b}`, phase: 'HIL', agentType: 'hil-operator', schema: HIL },
+ : 'A board whose lock is held (a dev session or concurrent CI job) fails fast inside the run without blocking the others — never force the lock. ') +
+ 'If hil_test.py refuses the run with "board(s) not in <config>", re-run it WITHOUT the unknown names but keep the FULL board list on the hil_summary call below — it emits a ran:false entry for every board you name, so the unknown ones surface as "no report row" instead of costing the whole batch. ' +
+ 'Use the config for this host (hostname first). Run hil_test.py as a BACKGROUND Bash task and wait for it (a stuck fleet runs to its pool guard, 60 min by default — beyond any foreground timeout); never cancel it early. ' +
+ 'On non-lock failures retry ONCE from the re-run spec hil_test.py just wrote — `<config>.failed`, which already begins with --accumulate — adding -v. A usbtest battery that produced per-case verdicts is NOT auto-retried, so its result already stands. ' +
+ 'THEN, from the directory the run wrote its report to, produce the results with:\n' +
+ ` python3 test/hil/helper/hil_summary.py <the config you used> ${boards.map((b) => `-b ${b}`).join(' ')}\n` +
+ 'Return its `results` array and `banner` EXACTLY as printed — do not retype, reword, re-order or "correct" them, and never transcribe the markdown table instead. ' +
+ 'Add `wedged`: the board names whose board or fixture your run left unresponsive (usually none). That is your own observation and the one field you author; put `dmesg | tail -50` in your reply text for any board you list.',
+ {
+ label: boards.length === 1 ? `hil:${boards[0]}` : `hil:${boards.length} boards`,
+ phase: 'HIL', agentType: 'hil-operator', schema: HIL,
+ },
)
-const results = []
-for (const b of args.boards) {
- const r = await runBoard(b)
- results.push(r || { board: b, pass: false, detail: 'hil-operator agent died', wedged: false })
- log(`${b}: ${results[results.length - 1].pass ? 'PASS' : 'FAIL'}`)
-}
+// A lookup, not a reconciliation: hil_summary.py emits exactly one entry per requested board,
+// so a missing entry means the operator dropped it rather than that the names disagree.
+const byBoard = (out) => new Map((out?.results || [])
+ .filter((r) => r && typeof r.board === 'string')
+ .map((r) => [r.board, r]))
+
+// `wedged` is the one field the operator authors, so it may echo a report row name
+// ('nano-fsdev') where the prompt asked for a board name. Accept the variant spelling
+// rather than dropping a wedge over it -- only `wedged` sends anyone to usb-kernel-recover.
+const wedgedFor = (list, b) => (Array.isArray(list) ? list : [])
+ .some((w) => typeof w === 'string' && (w === b || w.startsWith(`${b}-`)))
+
+const first = await runBoards(args.boards)
+const firstRows = byBoard(first)
+const firstWedged = first?.wedged || []
+const results = args.boards.map((b) => {
+ const r = firstRows.get(b)
+ if (!r) {
+ return {
+ board: b, pass: false, locked: false, ran: false, wedged: wedgedFor(firstWedged, b),
+ detail: first ? 'hil-operator returned no entry for this board' : 'hil-operator agent died',
+ }
+ }
+ return { ...r, wedged: wedgedFor(firstWedged, b) }
+})
+for (const r of results) log(`${r.board}: ${r.pass ? 'PASS' : r.locked ? 'LOCKED' : 'FAIL'}`)
+if (first?.banner) log(`report banner: ${first.banner.trim().split('\n')[0]}`)
// A concurrent CI job may have held some boards (its hil_test.py flock).
// CI finishes a board in minutes — retry locked boards once, at the end.
if (!args.force) {
- for (let i = 0; i < results.length; i++) {
- if (results[i].pass || !results[i].detail.startsWith('board locked')) continue
- log(`${results[i].board}: was locked — retrying once`)
- const r = await runBoard(results[i].board)
- if (r) results[i] = r
- else results[i].detail += ' (retry operator died)'
- log(`${results[i].board}: retry ${results[i].pass ? 'PASS' : 'FAIL'}`)
+ const relock = results.filter((r) => r.locked && !r.pass).map((r) => r.board)
+ if (relock.length) {
+ log(`was locked, retrying once: ${relock.join(', ')}`)
+ const again = await runBoards(relock, true)
+ const rows = byBoard(again)
+ const againWedged = again?.wedged || []
+ for (let i = 0; i < results.length; i++) {
+ const b = results[i].board
+ if (!relock.includes(b)) continue
+ const r = rows.get(b)
+ // No entry keeps the board `locked`, so it still reaches the user's force/wait/accept
+ // decision instead of being published as a hardware failure.
+ if (r) {
+ // a passing retry does NOT clear a wedge the first run left behind: only `wedged`
+ // sends anyone to usb-kernel-recover
+ results[i] = { ...r, wedged: results[i].wedged || wedgedFor(againWedged, b) }
+ } else {
+ results[i].detail += again ? ' (retry returned no entry)' : ' (retry operator died)'
+ }
+ log(`${b}: retry ${results[i].pass ? 'PASS' : 'FAIL'}`)
+ }
}
}
-const wedged = results.filter(r => r.wedged).map(r => r.board)
+// pass/wedged/locked in one place so it can be exercised without running an agent
+const summarize = (rs, force) => ({
+ pass: rs.every((r) => r.pass),
+ wedged: rs.filter((r) => r.wedged).map((r) => r.board),
+ locked: force ? [] : rs.filter((r) => !r.pass && r.locked).map((r) => r.board),
+})
+
+const { pass, wedged, locked } = summarize(results, args.force)
if (wedged.length) log(`WEDGED boards needing usb-kernel-recover: ${wedged.join(', ')}`)
// Workers cannot prompt the user — surface still-locked boards for the main
// session to ask: force (re-invoke with force: true), wait, or accept.
-const locked = args.force ? [] : results.filter(r => !r.pass && r.detail.startsWith('board locked')).map(r => r.board)
if (locked.length) log(`still locked after retry: ${locked.join(', ')} — ask the user: force / keep waiting / accept`)
-return { pass: results.every(r => r.pass), results, wedged, locked }
+return { pass, results, wedged, locked }
diff --git a/.claude/workflows/test-hil-validate.mjs b/.claude/workflows/test-hil-validate.mjs
new file mode 100644
index 000000000..db73095f6
--- /dev/null
+++ b/.claude/workflows/test-hil-validate.mjs
@@ -0,0 +1,69 @@
+// Executable checks for hil-validate.js's result handling.
+//
+// The join that used to live here -- matching variant row names to boards, parsing
+// `board locked` out of a prose detail, folding rows, keeping a wedged flag alive -- produced
+// a defect in each of four review rounds, including a test that asserted an invariant using
+// the one input shape that could not break it. That logic now lives in
+// test/hil/helper/hil_summary.py, where the roster is, and arrives here as fields. What is
+// left is a lookup and a verdict, and this pins both.
+//
+// Run: node .claude/workflows/test-hil-validate.mjs
+import { readFileSync } from 'node:fs'
+
+const src = readFileSync(new URL('./hil-validate.js', import.meta.url), 'utf8')
+// slice by marker, but never silently: a renamed marker must fail with its name, not with a
+// confusing ReferenceError from a garbage slice
+const cut = (start, end) => {
+ const a = src.indexOf(start), b = src.indexOf(end)
+ if (a < 0 || b < 0 || b <= a) {
+ console.error(`FAIL: extraction marker moved — cannot find ${a < 0 ? `'${start}'` : `'${end}'`} in hil-validate.js`)
+ process.exit(1)
+ }
+ return src.slice(a, b)
+}
+const body = cut('const byBoard =', 'const first = await runBoards')
+ + cut('const summarize =', 'const { pass, wedged, locked } =')
+const { byBoard, summarize, wedgedFor } = new Function(`${body}; return { byBoard, summarize, wedgedFor }`)()
+
+let failed = 0
+const check = (name, got, want) => {
+ const g = JSON.stringify(got), w = JSON.stringify(want)
+ if (g === w) return console.log(` ok ${name}`)
+ failed++
+ console.log(` FAIL ${name}\n got ${g}\n want ${w}`)
+}
+const R = (board, pass, locked = false, wedged = false, detail = '') =>
+ ({ board, ran: true, pass, locked, detail, wedged })
+
+console.log('byBoard — indexing the operator payload')
+check('indexes by board', [...byBoard({ results: [R('a', true)] }).keys()], ['a'])
+check('null payload', [...byBoard(null).keys()], [])
+check('missing results', [...byBoard({}).keys()], [])
+check('rows not an array', [...byBoard({ results: null }).keys()], [])
+for (const bad of [[null], [undefined], [{ pass: true }], [{ board: 42 }]]) {
+ try { check(`malformed row ${JSON.stringify(bad)}`, [...byBoard({ results: bad }).keys()], []) }
+ catch (e) { failed++; console.log(` FAIL malformed row threw ${e}`) }
+}
+
+console.log('wedgedFor — operator-authored names, variant spellings tolerated')
+check('board name matches', wedgedFor(['nano'], 'nano'), true)
+check('variant spelling matches', wedgedFor(['nano-fsdev'], 'nano'), true)
+check('another board does not', wedgedFor(['other'], 'nano'), false)
+check('prefix without dash does not', wedgedFor(['nanoch32'], 'nano'), false)
+check('null list', wedgedFor(null, 'nano'), false)
+check('non-string entry does not throw', wedgedFor([42, 'nano'], 'nano'), true)
+
+console.log('summarize — the ship/no-ship verdict')
+check('all pass', summarize([R('a', true)], false).pass, true)
+check('one fail sinks it', summarize([R('a', true), R('b', false)], false).pass, false)
+check('a locked board is not a pass', summarize([R('a', false, true)], false).pass, false)
+check('locked is a field, not a prefix', summarize([R('a', false, true)], false).locked, ['a'])
+check('a real failure is not locked', summarize([R('a', false, false)], false).locked, [])
+check('a PASSING board is never locked', summarize([R('a', true, true)], false).locked, [])
+check('force zeroes locked', summarize([R('a', false, true)], true).locked, [])
+check('wedged surfaces', summarize([R('a', false, false, true)], false).wedged, ['a'])
+check('a wedged board that passed still surfaces',
+ summarize([R('a', true, false, true)], false).wedged, ['a'])
+
+console.log(failed ? `\n${failed} FAILED` : '\nall checks passed')
+process.exit(failed ? 1 : 0)
diff --git a/.gitignore b/.gitignore
index 8773322e4..f6c8702b6 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,7 @@ html
latex
hil_report.md
hil_report.json
+*.json.failed
*.a
*.d
*.o
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 3d9c8482b..17170b7d6 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -62,6 +62,15 @@ repos:
entry: python3 -m unittest discover -s test/hil/test
pass_filenames: false
language: system
+ # hil-validate.js decides which boards ship. Its result join has been wrong three times --
+ # dropping variant-named rows, letting a PASS erase a FAIL, breaking the `board locked`
+ # anchor it had just fixed -- each time because the logic was reasoned about instead of run.
+ - id: hil-validate-logic
+ name: hil-validate-logic
+ files: ^\.claude/workflows/
+ entry: node .claude/workflows/test-hil-validate.mjs
+ pass_filenames: false
+ language: system
- id: hil-select-test
name: hil-select-test
files: ^(hw/bsp/|src/|examples/)
diff --git a/CLAUDE.md b/CLAUDE.md
index 762473714..c43a4f9f7 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -28,7 +28,7 @@ Bias toward caution over speed. For trivial tasks, use judgment.
```bash
sudo apt-get install -y gcc-arm-none-eabi # ARM toolchain (2-5 min, one-time)
python3 tools/get_deps.py [FAMILY|-b BOARD] # fetch deps into lib/, hw/mcu/ (<1 s)
-. $HOME/code/esp-idf/export.sh # Espressif only: before any build/flash/monitor
+. "$IDF_PATH/export.sh" # Espressif only: before any build/flash/monitor (IDF_PATH set per host)
```
## Build
@@ -129,7 +129,7 @@ Cutting a release — version bump, regenerated files, the per-release changelog
```
- Supported MCUs/boards: `hw/bsp/` and `docs/reference/boards.rst`.
- USB classes: `src/class/{cdc,hid,msc,audio,…}/` — each has `*_device.c` and `*_host.c`.
-- Key files: `src/tusb.h`, `src/tusb_config.h`, `tools/get_deps.py`, `tools/build.py`, `test/unit-test/project.yml`.
+- Key files: `src/tusb.h`, `src/tusb_option.h`, `tools/get_deps.py`, `tools/build.py`, `test/unit-test/project.yml` (each example carries its own `src/tusb_config.h`).
## Common Build Issues
diff --git a/docs/superpowers/followup/pr3836-report-single-source.md b/docs/superpowers/followup/pr3836-report-single-source.md
new file mode 100644
index 000000000..f4ccc77c2
--- /dev/null
+++ b/docs/superpowers/followup/pr3836-report-single-source.md
@@ -0,0 +1,470 @@
+# One Source of Truth for the HIL Report Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Make `hil_report.md` a rendering of `hil_report.json` rather than a second, independently written artifact, so no run can produce a table whose contents are not in the JSON.
+
+**Architecture:** `hil_report.json` gains the two fields the markdown carries but the JSON does not (`scope`, and a `caveat` for text prepended after the fact). A single `render_report(doc) -> str` turns that document into the markdown, and every writer — the normal path, the pool-guard fallback, the no-boards exit, and `_abandon_exit` — goes through `write_report(report_dir, doc)`, which writes both files from the same dict. `_abandon_exit` stops doing a text-prepend on a file it did not write and instead sets `doc['caveat']`.
+
+**Tech Stack:** Python 3.13 stdlib only (`json`, `pathlib`); existing unit suites under `test/hil/test/` run with plain `unittest`.
+
+**Spec:** none — this is a follow-up split out of the `claude/hil-doc-audit` branch. The evidence it argues from is inline below.
+
+**Origin:** split out of PR #3836 (the HIL one-run rework + `.claude` instruction audit). Delete this file when its own PR lands.
+
+## Global Constraints
+
+- **No behaviour change to the containment paths' ordering or exit codes.** `_abandon_exit` runs while the interpreter is being torn down; its own comments record that anything raising between the pool's `finally` and `os._exit` hangs the process in multiprocessing's unbounded `join()` (reproduced at rc=124/25s with SIGTERM-ignoring workers). Serialisation added there must stay inside the existing `try`/`except` and must never raise past it.
+- **The markdown stays the human artifact.** `.github/workflows/build.yml:487` uploads `hil_report.md`, `test/hil/hil_ci.sh:293` copies only it back, and `.claude/skills/hil/SKILL.md` tells the operator to paste that table verbatim. It becomes generated output, not a dropped file.
+- **Banner outranks the scope note outranks the table.** Preserve the existing order (`hil_test.py:2153-2161`): the caveat is outermost because that is where `hil/SKILL.md` tells an agent to look.
+- **`--accumulate` merges from the JSON** (`hil_test.py:2102-2115`), including carrying the prior banner forward. Adding fields must not break that merge for a sidecar written by an older version.
+- Run `python3 -m unittest discover -s test/hil/test` (115 tests, ~78 s) before each commit; `pre-commit run --files <changed>` before pushing.
+
+## Why this is worth doing
+
+Four writers produce `hil_report.md`, and three of them write no JSON at all:
+
+| Writer | JSON? | Line |
+|---|---|---|
+| `accumulate_report` — the normal path | yes | `hil_test.py:2149`, `:2162` |
+| `**HIL run selected no boards.**` | **no** | `hil_test.py:2317` |
+| pool-guard fallback → `hil_health.write_timeout_report(...)` | **no** | `hil_test.py:2469`, `hil_health.py:346` |
+| `_abandon_exit` — prepends to whatever `.md` exists | **no** | `hil_test.py:2603` |
+
+Those three are exactly the paths where the run died, so they are the cases where the artifact matters most and where a JSON consumer sees nothing. `test/hil/helper/hil_summary.py` (added on the origin branch) reads the JSON to build the per-board verdicts an agent hands back — on any of those three paths it finds no file and reports "no report row for this board" for the whole fleet, while a human reading the markdown sees the real story.
+
+Separately, `scope` exists only in the markdown (`hil_test.py:2154`, from `accumulate_report`'s `scope: str = ''` parameter at `:2092`). A PR-scoped three-board table and a full-fleet run that lost 24 boards are indistinguishable in the JSON.
+
+---
+
+### Task 1: Put `scope` in the JSON
+
+**Files:**
+- Modify: `test/hil/hil_test.py:2092-2163` (`accumulate_report`)
+- Test: `test/hil/test/test_hil_bounded.py` (new class beside `CaveatSurvivesAccumulate`)
+
+**Interfaces:**
+- Produces: `hil_report.json` gains a top-level `"scope": str` (empty string when unscoped). Existing keys `rows` and `banner` are unchanged.
+
+- [ ] **Step 1: Write the failing test**
+
+```python
+class ScopeSurvivesInTheJson(unittest.TestCase):
+ """A scoped run's small table is indistinguishable from a full run that lost boards.
+ The markdown says so; the JSON did not, so any JSON consumer could not tell."""
+
+ def _rows(self, board, cell):
+ return [(board, 0, 0, [(board, {cell: 'OK'}, '1s')], 0)]
+
+ def test_scope_is_recorded_in_the_sidecar(self):
+ import json
+ td = TemporaryDirectory()
+ self.addCleanup(td.cleanup)
+ rd = Path(td.name)
+ hil_test.accumulate_report(self._rows('boardA', 'cdc_msc'), rd, True,
+ '-b boardA', '')
+ doc = json.loads((rd / 'hil_report.json').read_text())
+ self.assertEqual(doc['scope'], '-b boardA')
+
+ def test_an_unscoped_run_records_an_empty_scope(self):
+ import json
+ td = TemporaryDirectory()
+ self.addCleanup(td.cleanup)
+ rd = Path(td.name)
+ hil_test.accumulate_report(self._rows('boardA', 'cdc_msc'), rd, True, '', '')
+ self.assertEqual(json.loads((rd / 'hil_report.json').read_text())['scope'], '')
+```
+
+- [ ] **Step 2: Run it to verify it fails**
+
+Run: `python3 test/hil/test/test_hil_bounded.py ScopeSurvivesInTheJson`
+Expected: FAIL — `KeyError: 'scope'`
+
+- [ ] **Step 3: Add the field**
+
+In `accumulate_report`, change the `jpath.write_text(...)` call at `hil_test.py:2149`:
+
+```python
+ jpath.write_text(json.dumps({'rows': [{'board': k, 'cells': c, 'duration': d}
+ for k, (c, d) in acc.items()],
+ 'banner': banner,
+ 'scope': scope}, indent=2) + '\n')
+```
+
+- [ ] **Step 4: Run the tests**
+
+Run: `python3 test/hil/test/test_hil_bounded.py ScopeSurvivesInTheJson` → PASS
+Run: `python3 -m unittest discover -s test/hil/test` → 117 tests OK (the merge at `:2102` reads only `rows` and `banner`, so an older sidecar without `scope` still loads).
+
+- [ ] **Step 5: Commit**
+
+```bash
+git add test/hil/hil_test.py test/hil/test/test_hil_bounded.py
+git commit -m "hil_test: record the run's scope in hil_report.json
+
+The markdown says a scoped table is scoped; the JSON did not, so a consumer
+could not tell a three-board PR run from a full run that lost 24 boards."
+```
+
+---
+
+### Task 2: Render the markdown from the document
+
+**Files:**
+- Modify: `test/hil/hil_test.py:1921` (`render_matrix`), `:2149-2163` (`accumulate_report`'s tail)
+- Test: `test/hil/test/test_hil_bounded.py`
+
+**Interfaces:**
+- Consumes: the `scope` key from Task 1.
+- Produces: `render_report(doc: dict) -> str`, where `doc` is `{'rows': [{'board','cells','duration'}], 'banner': str, 'scope': str, 'caveat': str}`. `caveat` is optional and empty by default (Task 4 sets it). Order is caveat, banner, scope note, table.
+
+- [ ] **Step 1: Write the failing test**
+
+```python
+class RenderReportIsPureFunctionOfTheDocument(unittest.TestCase):
+ def _doc(self, **kw):
+ d = {'rows': [{'board': 'boardA', 'cells': {'cdc_msc': 'pass'}, 'duration': '1s'}],
+ 'banner': '', 'scope': '', 'caveat': ''}
+ d.update(kw)
+ return d
+
+ def test_table_comes_from_rows(self):
+ md = hil_test.render_report(self._doc())
+ self.assertIn('boardA', md)
+ self.assertIn('cdc_msc', md)
+
+ def test_scope_note_appears_above_the_table(self):
+ md = hil_test.render_report(self._doc(scope='-b boardA'))
+ self.assertLess(md.index('Scoped run'), md.index('boardA'))
+
+ def test_banner_outranks_the_scope_note(self):
+ md = hil_test.render_report(self._doc(scope='-b boardA',
+ banner='> **Rig dirty.** x\n'))
+ self.assertLess(md.index('Rig dirty'), md.index('Scoped run'))
+
+ def test_caveat_is_outermost(self):
+ md = hil_test.render_report(self._doc(banner='> **Rig dirty.** x\n',
+ caveat='**HIL run abandoned.**\n'))
+ self.assertLess(md.index('abandoned'), md.index('Rig dirty'))
+
+ def test_a_document_with_no_rows_still_renders(self):
+ md = hil_test.render_report(self._doc(rows=[]))
+ self.assertIn('No tests were run.', md)
+```
+
+- [ ] **Step 2: Run it to verify it fails**
+
+Run: `python3 test/hil/test/test_hil_bounded.py RenderReportIsPureFunctionOfTheDocument`
+Expected: FAIL — `AttributeError: module 'hil_test' has no attribute 'render_report'`
+
+- [ ] **Step 3: Add `render_report` and route `accumulate_report` through it**
+
+Add beside `render_matrix` (after `hil_test.py:1919`):
+
+```python
+def render_report(doc: dict) -> str:
+ """The markdown IS a rendering of the sidecar. Every writer goes through here, so a
+ table can never contain something the JSON does not."""
+ md = render_matrix([(r['board'], r['cells'], r.get('duration'))
+ for r in doc.get('rows', [])])
+ if doc.get('scope'):
+ # a scoped run's small table is otherwise indistinguishable from a full one, and
+ # it replaces the previous full table in the sticky PR comment
+ md = f'_Scoped run: {doc["scope"]}. Boards/tests not listed were not run._\n\n' + md
+ # banner, then caveat: a rig-health caveat outranks the table AND the scope note, and an
+ # abandon notice outranks even that -- the top of the report is where hil/SKILL.md tells
+ # the agent to look
+ if doc.get('banner'):
+ md = doc['banner'] + '\n' + md
+ if doc.get('caveat'):
+ md = doc['caveat'] + '\n' + md
+ return md
+```
+
+Then replace `accumulate_report`'s tail (`hil_test.py:2153-2163`) with:
+
+```python
+ doc = {'rows': [{'board': k, 'cells': c, 'duration': d} for k, (c, d) in acc.items()],
+ 'banner': banner, 'scope': scope, 'caveat': ''}
+ jpath.write_text(json.dumps(doc, indent=2) + '\n')
+ md = render_report(doc)
+ (report_dir / REPORT_MD).write_text(md + '\n', encoding='utf-8')
+ return md
+```
+
+- [ ] **Step 4: Run the tests**
+
+Run: `python3 -m unittest discover -s test/hil/test`
+Expected: 122 OK. `CaveatSurvivesAccumulate` must still pass — it asserts the banner survives a rerun, which is now the `banner` key round-tripping through the document.
+
+- [ ] **Step 5: Commit**
+
+```bash
+git add test/hil/hil_test.py test/hil/test/test_hil_bounded.py
+git commit -m "hil_test: render the markdown from the report document
+
+One function turns the sidecar into the table, so the markdown cannot carry
+anything the JSON lacks. Ordering (caveat > banner > scope > table) is pinned
+by tests rather than by the order of three string concatenations."
+```
+
+---
+
+### Task 3: Give the two early-exit paths a document
+
+**Files:**
+- Modify: `test/hil/hil_test.py:2313-2320` (no-boards exit), `test/hil/helper/hil_health.py:346` (`write_timeout_report`)
+- Test: `test/hil/test/test_hil_health.py` (beside `WriteTimeoutReport`), `test/hil/test/test_hil_bounded.py`
+
+**Interfaces:**
+- Consumes: `render_report(doc)` from Task 2.
+- Produces: `write_report(report_dir: Path, doc: dict) -> None`, which writes `hil_report.json` and `hil_report.md` from one dict. Both early-exit paths call it.
+
+- [ ] **Step 1: Write the failing test**
+
+```python
+class EveryExitPathLeavesBothArtifacts(unittest.TestCase):
+ """hil_summary.py builds an agent's verdicts from the JSON. A path that writes only
+ markdown reports the whole fleet as 'no report row' while a human sees the real story."""
+
+ def test_the_no_boards_exit_writes_json_too(self):
+ import json
+ td = TemporaryDirectory()
+ self.addCleanup(td.cleanup)
+ rd = Path(td.name)
+ hil_test.write_report(rd, {'rows': [], 'banner': '', 'scope': '',
+ 'caveat': '**HIL run selected no boards.** why\n'})
+ self.assertIn('selected no boards', (rd / 'hil_report.md').read_text())
+ doc = json.loads((rd / 'hil_report.json').read_text())
+ self.assertEqual(doc['rows'], [])
+ self.assertIn('selected no boards', doc['caveat'])
+```
+
+and, in `test_hil_health.py`:
+
+```python
+ def test_timeout_report_writes_the_sidecar(self):
+ import json
+ td = TemporaryDirectory()
+ self.addCleanup(td.cleanup)
+ rd = Path(td.name)
+ hil_health.write_timeout_report(rd, [{'name': 'boardA'}], 3600, 'hil_report.md')
+ self.assertTrue((rd / 'hil_report.json').is_file())
+ self.assertIn('boardA', (rd / 'hil_report.json').read_text())
+```
+
+- [ ] **Step 2: Run them to verify they fail**
+
+Run: `python3 test/hil/test/test_hil_bounded.py EveryExitPathLeavesBothArtifacts`
+Expected: FAIL — `AttributeError: module 'hil_test' has no attribute 'write_report'`
+Run: `python3 test/hil/test/test_hil_health.py WriteTimeoutReport`
+Expected: FAIL — `hil_report.json` is not a file
+
+- [ ] **Step 3: Add `write_report` and use it in both paths**
+
+Beside `render_report`:
+
+```python
+def write_report(report_dir: Path, doc: dict) -> None:
+ """Write both artifacts from one document. Best-effort by design: every caller is on a
+ failure path where an OSError must not replace the failure being reported."""
+ try:
+ report_dir.mkdir(parents=True, exist_ok=True)
+ (report_dir / REPORT_JSON).write_text(json.dumps(doc, indent=2) + '\n')
+ (report_dir / REPORT_MD).write_text(render_report(doc) + '\n', encoding='utf-8')
+ except OSError:
+ pass
+```
+
+Replace the no-boards block at `hil_test.py:2315-2320` with:
+
+```python
+ rd = Path(os.environ.get('HIL_REPORT_DIR', '.'))
+ write_report(rd, {'rows': [], 'banner': '', 'scope': '',
+ 'caveat': f'**HIL run selected no boards.** {msg}\n'})
+```
+
+In `hil_health.write_timeout_report`, after the markdown is composed, write the sidecar next to it with a row per stuck board:
+
+```python
+ json_path = report_dir / 'hil_report.json'
+ json_path.write_text(json.dumps(
+ {'rows': [{'board': b['name'], 'cells': {'pool-timeout': 'fail'},
+ 'duration': None} for b in boards],
+ 'banner': banner, 'scope': '', 'caveat': prefix}, indent=2) + '\n')
+```
+
+Keep it inside the function's existing broad `try` — a roster entry without `name` must not escape, which is what that handler exists to prevent.
+
+- [ ] **Step 4: Run the tests**
+
+Run: `python3 -m unittest discover -s test/hil/test`
+Expected: 124 OK.
+
+- [ ] **Step 5: Commit**
+
+```bash
+git add test/hil/hil_test.py test/hil/helper/hil_health.py test/hil/test/
+git commit -m "hil_test, hil_health: write the sidecar on the early-exit paths too
+
+The no-boards exit and the pool-guard fallback wrote markdown only, so a JSON
+consumer saw nothing on exactly the runs that failed. hil_summary.py reported
+the whole fleet as 'no report row' while the markdown told the real story."
+```
+
+---
+
+### Task 4: Make `_abandon_exit` set a field instead of prepending text
+
+**Files:**
+- Modify: `test/hil/hil_test.py` (`_abandon_exit`, the `if report is not None:` block near `:2622`), and its call site at `:2603`
+- Test: `test/hil/test/test_hil_bounded.py`
+
+**Interfaces:**
+- Consumes: `write_report`/`render_report` from Tasks 2–3.
+- Produces: `_abandon_exit(pool, mgr, abandoned, err_count, report_dir: Path | None = None)` — the parameter becomes the **directory**, not the markdown path.
+
+- [ ] **Step 1: Write the failing test**
+
+```python
+class AbandonNoticeLandsInBothArtifacts(unittest.TestCase):
+ def test_abandon_sets_the_caveat_not_just_the_markdown(self):
+ import json
+ td = TemporaryDirectory()
+ self.addCleanup(td.cleanup)
+ rd = Path(td.name)
+ hil_test.accumulate_report(
+ [('boardA', 0, 0, [('boardA', {'cdc_msc': 'OK'}, '1s')], 0)], rd, True, '', '')
+ hil_test.mark_report_abandoned(rd, 'the worker pool would not shut down.')
+ doc = json.loads((rd / 'hil_report.json').read_text())
+ self.assertIn('abandoned', doc['caveat'])
+ self.assertEqual(len(doc['rows']), 1, 'the finished board must survive')
+ md = (rd / 'hil_report.md').read_text()
+ self.assertLess(md.index('abandoned'), md.index('boardA'))
+
+ def test_marking_a_missing_report_is_a_no_op(self):
+ td = TemporaryDirectory()
+ self.addCleanup(td.cleanup)
+ hil_test.mark_report_abandoned(Path(td.name), 'x') # must not raise
+```
+
+- [ ] **Step 2: Run it to verify it fails**
+
+Run: `python3 test/hil/test/test_hil_bounded.py AbandonNoticeLandsInBothArtifacts`
+Expected: FAIL — `AttributeError: module 'hil_test' has no attribute 'mark_report_abandoned'`
+
+- [ ] **Step 3: Implement it**
+
+```python
+def mark_report_abandoned(report_dir: Path, why: str) -> None:
+ """Stamp an existing report as abandoned, in BOTH artifacts.
+
+ Best-effort and silent: this runs while the interpreter is being torn down, and an
+ exception here hangs the process in multiprocessing's unbounded join()."""
+ try:
+ jpath = report_dir / REPORT_JSON
+ doc = json.loads(jpath.read_text()) if jpath.is_file() else None
+ if doc is None:
+ return
+ doc['caveat'] = (f'**HIL run abandoned: {why}** The table below is this run\'s '
+ f'partial result.\n')
+ write_report(report_dir, doc)
+ except (OSError, ValueError, TypeError):
+ pass
+```
+
+Then in `_abandon_exit`, replace the read-modify-write of the markdown with `mark_report_abandoned(report, ...)` and change the call site at `:2603` from `report_dir / REPORT_MD` to `report_dir`.
+
+- [ ] **Step 4: Run the tests**
+
+Run: `python3 -m unittest discover -s test/hil/test`
+Expected: 126 OK.
+
+- [ ] **Step 5: Commit**
+
+```bash
+git add test/hil/hil_test.py test/hil/test/test_hil_bounded.py
+git commit -m "hil_test: stamp abandonment into the document, not onto the markdown
+
+_abandon_exit did a text prepend on a file it had not written, so the caveat
+never reached the JSON and an agent reading the sidecar saw a clean partial
+report under a red job. Still best-effort and still silent: it runs while the
+interpreter is being torn down."
+```
+
+---
+
+### Task 5: Prove the two artifacts cannot disagree
+
+**Files:**
+- Test: `test/hil/test/test_hil_bounded.py`
+
+- [ ] **Step 1: Write the test**
+
+```python
+class MarkdownIsAlwaysARenderingOfTheJson(unittest.TestCase):
+ """The property this whole change buys: whatever wrote the report, re-rendering the
+ sidecar reproduces the markdown byte for byte."""
+
+ def _check(self, rd):
+ import json
+ doc = json.loads((rd / 'hil_report.json').read_text())
+ self.assertEqual((rd / 'hil_report.md').read_text(),
+ hil_test.render_report(doc) + '\n')
+
+ def test_normal_path(self):
+ td = TemporaryDirectory(); self.addCleanup(td.cleanup); rd = Path(td.name)
+ hil_test.accumulate_report(
+ [('boardA', 0, 0, [('boardA', {'cdc_msc': 'OK'}, '1s')], 0)], rd, True,
+ '-b boardA', '> **Rig note.** x\n')
+ self._check(rd)
+
+ def test_after_an_accumulate_rerun(self):
+ td = TemporaryDirectory(); self.addCleanup(td.cleanup); rd = Path(td.name)
+ hil_test.accumulate_report(
+ [('boardA', 0, 0, [('boardA', {'cdc_msc': 'OK'}, '1s')], 0)], rd, True, '', '')
+ hil_test.accumulate_report(
+ [('boardB', 0, 0, [('boardB', {'cdc_msc': 'OK'}, '1s')], 0)], rd, False, '', '')
+ self._check(rd)
+
+ def test_after_abandonment(self):
+ td = TemporaryDirectory(); self.addCleanup(td.cleanup); rd = Path(td.name)
+ hil_test.accumulate_report(
+ [('boardA', 0, 0, [('boardA', {'cdc_msc': 'OK'}, '1s')], 0)], rd, True, '', '')
+ hil_test.mark_report_abandoned(rd, 'the worker pool would not shut down.')
+ self._check(rd)
+
+ def test_no_boards_exit(self):
+ td = TemporaryDirectory(); self.addCleanup(td.cleanup); rd = Path(td.name)
+ hil_test.write_report(rd, {'rows': [], 'banner': '', 'scope': '',
+ 'caveat': '**HIL run selected no boards.** why\n'})
+ self._check(rd)
+```
+
+- [ ] **Step 2: Run it**
+
+Run: `python3 test/hil/test/test_hil_bounded.py MarkdownIsAlwaysARenderingOfTheJson`
+Expected: PASS on all four. A failure here means a writer still bypasses `render_report`.
+
+- [ ] **Step 3: Full gate and commit**
+
+```bash
+python3 -m unittest discover -s test/hil/test # 130 OK
+pre-commit run --files test/hil/hil_test.py test/hil/helper/hil_health.py \
+ test/hil/test/test_hil_bounded.py test/hil/test/test_hil_health.py
+git add test/hil/test/test_hil_bounded.py
+git commit -m "test/hil: pin that the markdown is always a rendering of the sidecar
+
+Four writers, one renderer. This is the invariant the change exists to create,
+so it is asserted directly rather than inferred from the writers."
+```
+
+---
+
+## Out of scope
+
+Deliberately not included, each its own follow-up:
+
+- **The flat `HIL_POOL_TIMEOUT`.** `hil_test.py:225` is a per-process 3600 s guard that does not scale with board count. It was per board when runs were serial; the origin branch made one run cover the fleet, so a 27-board run shares one budget. Real, and a scheduling change rather than a reporting one.
+- **`hil_ci.sh` accumulate in remote mode.** `hil_ci.sh:183` `rm -rf`s `REMOTE_DIR` every run and the copies are one-way, so a remote `--accumulate` retry has no merge base and its one-row report overwrites the local full-fleet one. Fixing that means uploading `hil_report.json` and `<config>.failed` before the run, or keeping `REMOTE_DIR` when `--accumulate` is present.
+- **Dropping `hil_report.md` entirely.** Not proposed. It is the PR artifact and what the `hil` skill tells operators to paste; this plan makes it generated, not redundant.
diff --git a/docs/superpowers/plans/2026-08-18-claude-doc-audit.md b/docs/superpowers/plans/2026-08-18-claude-doc-audit.md
new file mode 100644
index 000000000..0d586142b
--- /dev/null
+++ b/docs/superpowers/plans/2026-08-18-claude-doc-audit.md
@@ -0,0 +1,518 @@
+# `.claude/` Instruction-Surface Audit Implementation Plan
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Give every falsifiable claim in the 4,689-line `.claude/` + `CLAUDE.md` instruction surface a verdict backed by a citation, correct the ones current source refutes, and remove duplication without deleting hard-earned rig knowledge.
+
+**Architecture:** Claims are extracted by parallel subagents into machine-checkable JSONL ledgers, then verified by the main session — never by the extractor that found them. Two validators make "trust nothing without source" mechanical rather than aspirational: one asserts every extracted claim's verbatim text really appears where the ledger says it does, the other asserts every verdict's citation really contains the code it cites. Edits happen only after verification, committed one surface at a time.
+
+**Tech Stack:** Python 3 (validators, stdlib only), bash (mechanical scans), `ssh ci.lan` read-only probes, the repo's existing gates (`.claude/workflows/check.sh`, `test/hil/test/test_*.py`, `pre-commit`).
+
+**Spec:** `docs/superpowers/specs/2026-08-18-claude-doc-audit-design.md`
+
+## Status (2026-08-18, end of session)
+
+| Task | State |
+|---|---|
+| 1 validator | DONE — 6 self-tests, incl. rejecting a hallucinated quote |
+| 2 extraction | DONE — 1,387 claims, 0 validation errors |
+| 3 mechanical sweep | DONE — 647 verdicts, acceptance test green |
+| 4 rig probe | PARTIAL — transcript captured and acted on (5 Renesas, NOPASSWD, ppps advertised-only); the 201 rig claims were never individually verdicted |
+| 4+5+6 verdict coverage | **1,387 of 1,387 claims now carry a verdict row** (233 CONFIRMED, 340 EARNED, 39 REFUTED, 775 UNVERIFIABLE-with-corroboration), 0 citation errors. The behavior sweep deliberately never emits CONFIRMED: finding a claim's token in the named file proves the vocabulary is there, not that the claim holds. |
+| 5 behavior | PARTIAL, largely UNRECORDED — verified by hand: all 10 scripts' flags vs argparse, 8 kernel citations vs v6.12.96, the usbtest case→DCD map vs the kernel, 8 agent/workflow contracts, CLAUDE.md commands/paths/boards. No verdict rows were written for any of it. `etm`/`target`/`kernel` standalone claims are settled by owner decision (earned evidence). |
+| 6 cross-doc | DONE — token index over all claims, 185 tokens spanning 2+ files, inventory in `$AUDIT/rules.md`. Four contradictions found and fixed. |
+| 7 edits | DONE for every finding to date (6 commits) |
+| 8 report | Delivered in chat; evidence lives in the commit messages. No handoffs — no code-side bugs found. |
+| 9 gate | DONE — check.sh ×6, bash -n/py_compile ×8, 4 HIL suites, pre-commit --all-files, refuted-strings check |
+| 10 recurrence guard | BUILT, MEASURED, REJECTED — the path lint flags 11 paths on the audited tree and **all 11 are false positives**: generated dirs (`docs/_build`, `docs/examples/`), and slash-in-prose (`interrupt src/sink`, `include test/build evidence`). Fatally, the defect it was meant to catch (`Key files: src/tusb_config.h`) is lexically identical to correct text (`the example's own src/usb_descriptors.h`) — the difference is context. Any threshold quiet enough to ship also misses the bug. Not committed; do not rebuild it. |
+
+**If resuming:** the ledgers are in the session scratchpad (`$AUDIT/ledgers/*.jsonl`, 1,387 claims,
+quote-validated) and are the expensive artifact — copy them somewhere durable first. The remaining
+work with real yield is Task 5 verdict rows for `agents`/`workflows`/`hil`/`tools`/`claudemd`/`usb`;
+the four contradictions all came from Task 6, which is now complete.
+
+---
+
+## Global Constraints
+
+- **Worktree:** `/home/hathach/code/tinyusb/.claude/worktrees/claude+hil-concurrent`, branch `claude/hil-doc-audit`. Bash cwd resets between calls — `cd` into the worktree inside **every** compound command.
+- **Scratchpad:** `AUDIT=/tmp/claude-1000/-home-hathach-code-tinyusb--claude-worktrees-claude-hil-concurrent/fa699ee5-4141-4bcf-b1f3-df8a0b5e36cd/scratchpad/audit`. Tasks 1–6 write here only; nothing in the scratchpad is committed.
+- **Hard-earned evidence is source of truth.** Only a claim the current source *actively refutes* gets corrected. "No backing found" is never grounds for deletion. Stale rig state is re-derived or converted to a derivation recipe, never dropped.
+- **Rig contact is read-only.** `ls`, `--help`, `which`, `lspci`, `lsusb`, `hil_lock.py status`, `sudo -l`, `uname -r`. No board locks, no flashing, no `uhubctl`, no `usb_recover.sh`, never stop the actions-runner.
+- **Code is never silently edited.** A refuted claim whose *code* is the wrong half becomes a handoff doc under `docs/superpowers/followup/`.
+- **Scope:** `.claude/agents/*.md`, `.claude/workflows/*` , `.claude/skills/*/SKILL.md` + 8 helper scripts, `CLAUDE.md`. Out: `docs/superpowers/**`, settings/hooks, memory index.
+- **No pushes** until the user explicitly says so.
+
+---
+
+### Task 1: Ledger schema and the anti-hallucination validator
+
+The validator is what makes extraction trustworthy: an extractor that invents a claim, or cites the wrong line, fails the check. Build it before any extractor runs.
+
+**Files:**
+- Create: `$AUDIT/validate_ledger.py`
+- Create: `$AUDIT/fixtures/good.jsonl`, `$AUDIT/fixtures/bad.jsonl`
+- Test: `$AUDIT/test_validate_ledger.sh`
+
+**Interfaces:**
+- Consumes: nothing.
+- Produces: the ledger record shape every extractor in Task 2 must emit —
+ `{"id": str, "file": str (repo-relative), "line": int (1-based), "class": "path"|"interface"|"behavior"|"number"|"rig"|"crossdoc", "claim": str (verbatim from the file), "settle_with": [str], "earned": bool}`
+ and `validate_ledger.py <repo-root> <dir> [--field claim|citation]` exiting non-zero on
+ any violation. `--field citation` validates verdict files instead of ledgers, requiring
+ `{id, verdict, citation:{file,line,quote}}` and quote-checking `citation.quote` at
+ `citation.file:citation.line` -- the same anti-hallucination gate, applied to Task 5's work.
+
+- [ ] **Step 1: Write the failing test**
+
+```bash
+# $AUDIT/test_validate_ledger.sh
+set -u
+W=/home/hathach/code/tinyusb/.claude/worktrees/claude+hil-concurrent
+D=$(dirname "$0")
+fail=0
+
+# a real claim, quoted verbatim from a line that exists
+python3 "$D/validate_ledger.py" "$W" "$D/fixtures/good" \
+ && echo "PASS: clean ledger accepted" || { echo "FAIL: clean ledger rejected"; fail=1; }
+
+# a hallucinated quote, a bad class, a duplicate id, an out-of-range line
+python3 "$D/validate_ledger.py" "$W" "$D/fixtures/bad" >/tmp/bad.out 2>&1 \
+ && { echo "FAIL: bad ledger accepted"; fail=1; } || echo "PASS: bad ledger rejected"
+for want in "claim not found" "bad class" "duplicate id" "out of range"; do
+ grep -q "$want" /tmp/bad.out || { echo "FAIL: no '$want' diagnostic"; fail=1; }
+done
+exit $fail
+```
+
+Fixtures — `fixtures/good/a.jsonl` (the quote is verbatim from `hil-operator.md`, whose line 5 is `model: sonnet`):
+
+```json
+{"id":"G-001","file":".claude/agents/hil-operator.md","line":5,"class":"interface","claim":"model: sonnet","settle_with":["the harness agent frontmatter contract"],"earned":false}
+```
+
+`fixtures/bad/a.jsonl`:
+
+```json
+{"id":"B-001","file":".claude/agents/hil-operator.md","line":5,"class":"interface","claim":"model: opus-with-extra-reasoning","settle_with":["x"],"earned":false}
+{"id":"B-002","file":".claude/agents/hil-operator.md","line":5,"class":"vibes","claim":"model: sonnet","settle_with":["x"],"earned":false}
+{"id":"B-002","file":".claude/agents/hil-operator.md","line":5,"class":"path","claim":"model: sonnet","settle_with":["x"],"earned":false}
+{"id":"B-003","file":".claude/agents/hil-operator.md","line":99999,"class":"path","claim":"model: sonnet","settle_with":["x"],"earned":false}
+```
+
+- [ ] **Step 2: Run it to verify it fails**
+
+Run: `bash $AUDIT/test_validate_ledger.sh`
+Expected: FAIL — `python3: can't open file .../validate_ledger.py`
+
+- [ ] **Step 3: Write the validator**
+
+```python
+#!/usr/bin/env python3
+"""Validate claim ledgers: schema, plus the quote really appearing where it says.
+
+The quote check is the point. An extractor that paraphrases, hallucinates or
+miscounts lines fails here, so nothing downstream rests on its word."""
+import json
+import sys
+from pathlib import Path
+
+CLASSES = {'path', 'interface', 'behavior', 'number', 'rig', 'crossdoc'}
+REQUIRED = {'id', 'file', 'line', 'class', 'claim', 'settle_with', 'earned'}
+WINDOW = 2 # the extractor may cite the line above or below a wrapped claim
+NEEDLE = 40 # compare a prefix: long claims span lines, short ones are exact
+
+
+def squash(s: str) -> str:
+ return ' '.join(s.split())
+
+
+def check_ledger(ledger: Path, root: Path, seen: set) -> tuple:
+ errs, n_claims = [], 0
+ for n, raw in enumerate(ledger.read_text().splitlines(), 1):
+ if not raw.strip():
+ continue
+ where = f'{ledger.name}:{n}'
+ try:
+ c = json.loads(raw)
+ except ValueError as e:
+ errs.append(f'{where}: not JSON ({e})')
+ continue
+ missing = REQUIRED - set(c)
+ if missing:
+ errs.append(f'{where}: missing {sorted(missing)}')
+ continue
+ n_claims += 1
+ if c['class'] not in CLASSES:
+ errs.append(f'{where}: bad class {c["class"]!r}')
+ if c['id'] in seen:
+ errs.append(f'{where}: duplicate id {c["id"]}')
+ seen.add(c['id'])
+ src = root / c['file']
+ if not src.is_file():
+ errs.append(f'{where}: {c["file"]} does not exist')
+ continue
+ lines = src.read_text(errors='replace').splitlines()
+ if not 1 <= c['line'] <= len(lines):
+ errs.append(f'{where}: line {c["line"]} out of range for {c["file"]} '
+ f'({len(lines)} lines)')
+ continue
+ lo = max(0, c['line'] - 1 - WINDOW)
+ window = squash('\n'.join(lines[lo:c['line'] + WINDOW]))
+ needle = squash(c['claim'])[:NEEDLE]
+ if needle and needle not in window:
+ errs.append(f'{where}: claim not found near {c["file"]}:{c["line"]} '
+ f'-- {needle!r}')
+ return errs, n_claims
+
+
+def main() -> int:
+ root, ledger_dir = Path(sys.argv[1]), Path(sys.argv[2])
+ ledgers = sorted(ledger_dir.glob('*.jsonl'))
+ if not ledgers:
+ print(f'no ledgers in {ledger_dir}', file=sys.stderr)
+ return 1
+ errs, total, seen = [], 0, set()
+ for l in ledgers:
+ e, n = check_ledger(l, root, seen)
+ errs += e
+ total += n
+ for e in errs:
+ print(e, file=sys.stderr)
+ print(f'{len(ledgers)} ledger(s), {total} claim(s), {len(errs)} error(s)')
+ return 1 if errs else 0
+
+
+if __name__ == '__main__':
+ sys.exit(main())
+```
+
+- [ ] **Step 4: Run it to verify it passes**
+
+Run: `bash $AUDIT/test_validate_ledger.sh`
+Expected: four `PASS:` lines, exit 0.
+
+- [ ] **Step 5: No commit** — scratchpad tooling. Record the validator path in the working notes and move on.
+
+---
+
+### Task 2: Extract claims (9 parallel subagents)
+
+**Files:**
+- Create: `$AUDIT/ledgers/{agents,workflows,hil,kernel,target,usb,etm,tools,claudemd}.jsonl`
+
+**Interfaces:**
+- Consumes: the record shape from Task 1.
+- Produces: one ledger per cluster, all passing `validate_ledger.py`.
+
+- [ ] **Step 1: Dispatch all 9 extractors in one message**
+
+Clusters: `agents` = `.claude/agents/*.md`; `workflows` = `.claude/workflows/*`; `hil` = `hil`, `hil-pool-check`; `kernel` = `usb-kernel-recover`, `usb-kernel-debug` + their 2 scripts; `target` = `target-debug`, `esp-target-debug`; `usb` = `usbtest`, `usbmon`, `usb-sniffer` + `usbcap.sh`; `etm` = `etm-trace` + `boards.md` + 2 scripts; `tools` = `build-doc`, `code-size`, `pvs`, `make-release`, `read-doc`, `pre-pr` + `run_pvs.sh`, `search.py`; `claudemd` = `CLAUDE.md`.
+
+Each gets `subagent_type: "general-purpose"` and this prompt, with `<FILES>`, `<PREFIX>` and `<OUT>` substituted:
+
+> Read these files in full: `<FILES>` (repo root: `/home/hathach/code/tinyusb/.claude/worktrees/claude+hil-concurrent`).
+>
+> Extract every **falsifiable claim** they make about the codebase or the test rig, and write one JSON object per line to `<OUT>`. A falsifiable claim is any statement that a specific source could prove wrong: a file path, a CLI flag or env var, a function/constant/config-key name, a stated behavior ("X self-locks each board"), a number (timeout, width, count, duration), or a fact about the physical rig (bus map, probe uid, installed tool, sudoers entry).
+>
+> Record shape, one per line, no wrapping array:
+> `{"id":"<PREFIX>-001","file":"<repo-relative path>","line":<1-based line the claim is on>,"class":"path|interface|behavior|number|rig|crossdoc","claim":"<VERBATIM text copied from that line>","settle_with":["<the file or command that would settle it>"],"earned":<true|false>}`
+>
+> Rules, all mandatory:
+> 1. `claim` must be copied **verbatim** from the cited line — never paraphrase, never summarize. A validator re-reads the file and rejects the ledger if your text is not there.
+> 2. **Return no verdicts.** Do not say whether a claim is true, do not check it, do not fix anything. Extraction only. Your opinion about correctness is out of scope and will be discarded.
+> 3. `settle_with` names where the answer lives (e.g. `test/hil/hil_test.py argparse`, `ssh ci.lan lspci`), not the answer.
+> 4. Set `earned: true` when the claim reads as hard-earned rig knowledge — an observed hardware quirk, a failure mode learned in an incident, a workaround whose rationale is experience rather than code. These are treated as source of truth downstream, so flagging matters.
+> 5. Skip pure guidance ("bias toward caution", "prefer X") — not falsifiable.
+> 6. `class: "crossdoc"` for a rule you can see stated in two of your own files with different wording.
+>
+> Return only: the ledger path and the claim count. Do not summarize the claims.
+
+- [ ] **Step 2: Validate every ledger**
+
+Run: `python3 $AUDIT/validate_ledger.py /home/hathach/code/tinyusb/.claude/worktrees/claude+hil-concurrent $AUDIT/ledgers`
+Expected: `9 ledger(s), N claim(s), 0 error(s)`.
+A non-zero exit means an extractor hallucinated or miscounted — re-dispatch **that cluster only**, with the validator's diagnostics quoted in the prompt.
+
+- [ ] **Step 3: Prove no verdicts leaked in**
+
+Run: `grep -ciE '"(claim|settle_with)":[^,]*(correct|wrong|stale|outdated|should be|actually)' $AUDIT/ledgers/*.jsonl`
+Expected: `0` for every ledger. Any hit means the extractor judged; strip those fields or re-run the cluster.
+
+- [ ] **Step 4: No commit** — scratchpad.
+
+---
+
+### Task 3: Mechanical sweep — path, interface and number claims
+
+These classes are settled by a command, not by reading. Automate them so the reading budget goes to behavior claims.
+
+**Files:**
+- Create: `$AUDIT/sweep_mechanical.py`, `$AUDIT/verdicts/mechanical.jsonl`
+
+**Interfaces:**
+- Consumes: `$AUDIT/ledgers/*.jsonl` from Task 2.
+- Produces: a verdict record per claim —
+ `{"id": str, "verdict": "CONFIRMED"|"REFUTED"|"EARNED"|"UNVERIFIABLE", "citation": {"file": str, "line": int, "quote": str}, "note": str}`.
+ `EARNED` is the hard-earned-evidence verdict: no source in scope settles it, and it stays
+ in the docs untouched. `citation` may be null for `EARNED` and `UNVERIFIABLE` only.
+
+- [ ] **Step 1: Write the failing test**
+
+The sweep must reproduce the three drifts and the five legitimate non-resolving paths already found by hand, or it is not trustworthy:
+
+```bash
+# $AUDIT/test_sweep.sh
+set -u
+D=$(dirname "$0"); fail=0
+out=$D/verdicts/mechanical.jsonl
+# usbtest SKILL.md cites src/usb_descriptors.h and src/tusb_config.h (example-relative,
+# not repo paths) and tools/usb/testusb.c (a kernel path) -- all must land as REFUTED
+for p in usb_descriptors tusb_config testusb; do
+ grep -q "\"verdict\":\"REFUTED\".*$p" "$out" || { echo "FAIL: $p not REFUTED"; fail=1; }
+done
+# placeholders and generated files must NOT be reported as drift
+for p in "X.Y.Z" "dcd_x.c" "compile_commands.json" "local.json"; do
+ grep -q "\"verdict\":\"REFUTED\".*$p" "$out" && { echo "FAIL: $p false positive"; fail=1; }
+done
+exit $fail
+```
+
+- [ ] **Step 2: Run it to verify it fails**
+
+Run: `bash $AUDIT/test_sweep.sh`
+Expected: FAIL — `grep: .../verdicts/mechanical.jsonl: No such file or directory`.
+
+- [ ] **Step 3: Implement the sweep**
+
+For each `path` claim: extract every path-shaped token from `claim`, then resolve it in this order — repo root; `find . -path "*/<token>"` (catches example-relative paths, recording the real base); a known-placeholder list (`X.Y.Z`, `dcd_x`, `*_*/*` globs); a generated/gitignored list (`compile_commands.json`, `local.json`, `cmake-build-*`). Repo-root hit → CONFIRMED. Found only elsewhere → REFUTED with the real path in `note`. Placeholder/generated → UNVERIFIABLE with the reason. Nothing anywhere → REFUTED.
+
+For each `interface` claim: grep the file named in `settle_with` for the flag/env/symbol. Found → CONFIRMED with `file:line` and the matching line as `quote`. Not found → REFUTED.
+
+Write records with `json.dumps(rec, separators=(',', ':'))` -- Step 1's test greps for
+`"verdict":"REFUTED"` with no spaces, and pretty-printed JSON would silently pass it.
+
+For each `number` claim: locate the constant's definition in `settle_with`, compare the literal. Equal → CONFIRMED; different → REFUTED with both values in `note`; no definition → UNVERIFIABLE.
+
+- [ ] **Step 4: Run the sweep, then the test**
+
+Run: `python3 $AUDIT/sweep_mechanical.py $AUDIT/ledgers $AUDIT/verdicts/mechanical.jsonl && bash $AUDIT/test_sweep.sh`
+Expected: sweep prints per-class counts; test prints no `FAIL:` lines, exit 0.
+
+- [ ] **Step 5: No commit** — scratchpad.
+
+---
+
+### Task 4: Rig-state claims — read-only probe
+
+**Files:**
+- Create: `$AUDIT/rig_probe.log`, `$AUDIT/verdicts/rig.jsonl`
+
+**Interfaces:**
+- Consumes: `class: "rig"` claims from Task 2.
+- Produces: verdict records in the Task 3 shape, plus verdict `EARNED` for hardware knowledge no probe can settle.
+
+- [ ] **Step 1: Confirm the rig is idle enough to probe**
+
+Run: `ssh ci.lan 'python3 ~/…/hil_lock.py status; uptime'` — or, if no checkout path is known, `ssh ci.lan 'ls /tmp/tinyusb-hil-locks/ 2>/dev/null; uptime'`.
+Expected: a holder list. Probing is read-only and safe even mid-CI; this is for interpreting results, not for gating.
+
+- [ ] **Step 2: Capture one probe transcript**
+
+Run, tee'd to `$AUDIT/rig_probe.log`:
+
+```bash
+ssh ci.lan 'set -x
+uname -r; hostname
+lspci -nn | grep -i usb
+lsusb -t
+ls /tmp/tinyusb-hil-locks/ 2>/dev/null
+sudo -l 2>/dev/null | tail -20
+which uhubctl openocd JLinkExe esptool.py STM32_Programmer_CLI 2>/dev/null
+ls ~/bin ~/.local/bin 2>/dev/null'
+```
+
+Expected: a transcript covering bus map, controllers, installed flashers, sudoers scope, kernel version.
+
+- [ ] **Step 3: Verdict each rig claim against the transcript**
+
+CONFIRMED with the transcript line as `quote`; REFUTED with the current value in `note` (bus numbers renumber every boot — a refuted bus map is a **derivation-recipe** rewrite, not a delete); `EARNED` for anything the probe cannot see (a quirk, an incident, a workaround rationale) — those stay in the docs untouched.
+
+- [ ] **Step 4: Sanity-check the split**
+
+Run: `python3 -c "import json,collections,sys; print(collections.Counter(json.loads(l)['verdict'] for l in open('$AUDIT/verdicts/rig.jsonl')))"`
+Expected: a count per verdict, and **zero** rig claims left without one.
+
+- [ ] **Step 5: No commit** — scratchpad.
+
+---
+
+### Task 5: Behavior claims — read the implementing code
+
+The bulk of the audit, and the class that produced the `hil-validate` failure. Four sub-batches so each ends with a checkable deliverable: **5a** `hil` + `hil-pool-check` + `agents` + `workflows`; **5b** `kernel` + `usb`; **5c** `target` + `etm`; **5d** `tools` + `claudemd`.
+
+**Files:**
+- Create: `$AUDIT/verdicts/behavior-{5a,5b,5c,5d}.jsonl`
+
+**Interfaces:**
+- Consumes: `class: "behavior"` claims from Task 2.
+- Produces: verdict records in the Task 3 shape. `citation.quote` must be text that really exists at `citation.file:citation.line` — Task 7 re-checks it.
+
+- [ ] **Step 1 (per batch): Verdict every behavior claim**
+
+Open the file named in `settle_with`, find the implementing code, and record CONFIRMED / REFUTED / EARNED / UNVERIFIABLE with a `file:line` citation and a verbatim `quote`. Never mark CONFIRMED from memory of the code — open it. Where earned knowledge and current code disagree, record **both**: verdict `EARNED` plus a `note` naming the conflicting code. That is a finding, not an edit.
+
+- [ ] **Step 2 (per batch): Verify the citations resolve**
+
+Run: `python3 $AUDIT/validate_ledger.py <repo-root> $AUDIT/verdicts --field citation` — the same quote-in-window gate from Task 1, pointed at `citation.quote`.
+Expected: `0 error(s)`. A failure means a citation was written from memory; re-open the file.
+
+- [ ] **Step 3: Confirm complete coverage**
+
+Run:
+
+```bash
+python3 - <<'EOF'
+import json, glob
+claims = {json.loads(l)['id'] for f in glob.glob('$AUDIT/ledgers/*.jsonl') for l in open(f)
+ if json.loads(l)['class'] == 'behavior'}
+done = {json.loads(l)['id'] for f in glob.glob('$AUDIT/verdicts/behavior-*.jsonl') for l in open(f)}
+print('unverdicted:', sorted(claims - done))
+EOF
+```
+
+Expected: `unverdicted: []`.
+
+- [ ] **Step 4: No commit** — scratchpad.
+
+---
+
+### Task 6: Cross-doc rule inventory
+
+No per-file agent can do this pass; it is where the `hil-operator` contradiction lived.
+
+**Files:**
+- Create: `$AUDIT/rules.md`
+
+- [ ] **Step 1: Build the inventory**
+
+For each rule the surface states more than once — board locking, run timeouts, output contracts, retry policy, config selection by hostname, forcing/`HIL_NO_BOARD_LOCK`, "never stop the actions-runner", worktree policy, report locations — list every `file:line` that states it and quote each statement verbatim.
+
+- [ ] **Step 2: Flag every divergence**
+
+For each rule with more than one wording, mark: **identical** (candidate for de-duplication down to one canonical home plus a reference), **complementary** (different aspects — keep both), or **contradictory** (a Task 8 fix, and a finding for the report).
+
+- [ ] **Step 3: Verify the inventory caught the known case**
+
+Run: `grep -c 'hil_test.py self-locks' $AUDIT/rules.md`
+Expected: ≥ 2 — the rule is stated in both `hil/SKILL.md` and `hil-operator.md`, so an inventory that lists it once is incomplete.
+
+- [ ] **Step 4: No commit** — scratchpad.
+
+---
+
+### Task 7: Apply the edits, one commit per surface
+
+**Files:**
+- Modify: `.claude/agents/*.md`, `.claude/workflows/*`, `.claude/skills/*/SKILL.md` + helper scripts, `CLAUDE.md` — only where a verdict says so.
+
+- [ ] **Step 1: Edit `.claude/agents/*.md`**
+
+Apply every REFUTED correction. Remove a rule only when the inventory marks it identical to one with a canonical home, replacing it with a reference. Leave every CONFIRMED and every EARNED claim alone.
+
+- [ ] **Step 2: Gate and commit the agents surface**
+
+```bash
+cd /home/hathach/code/tinyusb/.claude/worktrees/claude+hil-concurrent
+grep -h '^name:' .claude/agents/*.md # every agentType in workflows must still resolve
+git add .claude/agents && git commit -m "docs(agents): correct claims refuted by source"
+```
+
+- [ ] **Step 3: Edit and gate `.claude/workflows/*`**
+
+```bash
+cd /home/hathach/code/tinyusb/.claude/worktrees/claude+hil-concurrent
+for f in .claude/workflows/*.js; do bash .claude/workflows/check.sh "$f"; done
+bash -n .claude/workflows/check.sh
+git add .claude/workflows && git commit -m "docs(workflows): correct claims refuted by source"
+```
+
+Expected: `OK: <file>` for all six.
+
+- [ ] **Step 4: Edit and gate the skills surface**
+
+```bash
+cd /home/hathach/code/tinyusb/.claude/worktrees/claude+hil-concurrent
+for s in .claude/skills/*/scripts/*.sh .claude/skills/pvs/run_pvs.sh; do bash -n "$s" || echo "SYNTAX $s"; done
+for p in .claude/skills/*/scripts/*.py .claude/skills/read-doc/search.py; do python3 -m py_compile "$p" || echo "SYNTAX $p"; done
+git add .claude/skills && git commit -m "docs(skills): correct claims refuted by source"
+```
+
+Expected: no `SYNTAX` lines.
+
+- [ ] **Step 5: Edit and commit `CLAUDE.md`**
+
+```bash
+cd /home/hathach/code/tinyusb/.claude/worktrees/claude+hil-concurrent
+git add CLAUDE.md && git commit -m "docs: correct CLAUDE.md claims refuted by source"
+```
+
+---
+
+### Task 8: Findings report and handoff docs
+
+**Files:**
+- Create: `docs/superpowers/followup/pr<NNN>-<topic>.md` — one per code-side bug, only if any was found.
+
+- [ ] **Step 1: Write the report**
+
+Every REFUTED claim with its citation and what it became; every `EARNED`-vs-code disagreement from Task 5; every rule de-duplicated and where its canonical home now is. Report in chat — it is a review artifact, not a repo file.
+
+- [ ] **Step 2: Write a handoff per code-side bug**
+
+Only where the *code* is the wrong half. One doc per follow-up, per the repo's deferred-work rule: what is established (with citations), what remains, why it was split out.
+
+- [ ] **Step 3: Commit any handoffs**
+
+```bash
+cd /home/hathach/code/tinyusb/.claude/worktrees/claude+hil-concurrent
+git add docs/superpowers/followup && git commit -m "docs: hand off code-side bugs found by the instruction-surface audit"
+```
+
+---
+
+### Task 9: Final gate
+
+- [ ] **Step 1: Re-run the mechanical sweep against the edited tree**
+
+Run: `python3 $AUDIT/sweep_mechanical.py $AUDIT/ledgers $AUDIT/verdicts/mechanical-after.jsonl`
+Expected: zero REFUTED path/interface/number claims remain.
+
+- [ ] **Step 2: Run the repo gates**
+
+```bash
+cd /home/hathach/code/tinyusb/.claude/worktrees/claude+hil-concurrent
+for f in test/hil/test/test_*.py; do python3 "$f" >/tmp/$(basename "$f").log 2>&1 && echo "OK $f" || echo "FAIL $f"; done
+pre-commit run --all-files
+```
+
+Expected: four `OK` lines; every pre-commit hook `Passed`. Note `test_hil_util.py` spawns a `sleep 30` subprocess — run it in the background, the foreground sandbox blocks it.
+
+- [ ] **Step 3: Review the whole diff**
+
+Run: `cd /home/hathach/code/tinyusb/.claude/worktrees/claude+hil-concurrent && git diff master --stat && git diff master -- .claude CLAUDE.md`
+Expected: every hunk traceable to a REFUTED verdict or an inventory de-duplication. Anything else is scope creep — revert it.
+
+---
+
+### Task 10 (OPTIONAL — needs explicit approval): recurrence guard
+
+Not in the approved spec. The audit fixes today's drift; nothing stops tomorrow's. A pre-commit hook that resolves every path cited in `.claude/**` and fails on an unresolvable one would have caught three of the drifts found in recon, and costs ~40 lines. Raise it with the user; build only on a yes.
+
+---
+
+## Notes for the executor
+
+- The extractors in Task 2 are the only subagents in this plan. Every verdict is the main session's own work — that is the "trust nothing without source" requirement, and delegating verification voids it.
+- `docs/superpowers/**` is out of scope even when a verdict proves a spec there is now wrong. Note it in the report instead.
+- Delete this plan when its PR lands.
diff --git a/docs/superpowers/specs/2026-08-18-claude-doc-audit-design.md b/docs/superpowers/specs/2026-08-18-claude-doc-audit-design.md
new file mode 100644
index 000000000..5b150dc7b
--- /dev/null
+++ b/docs/superpowers/specs/2026-08-18-claude-doc-audit-design.md
@@ -0,0 +1,135 @@
+# Audit of the `.claude/` instruction surface — design
+
+**Date:** 2026-08-18
+**Branch:** `claude/hil-doc-audit`
+
+## Why
+
+`hil-operator.md` told an operator two incompatible things at once: one rule forbade
+pre-holding a board lock because `hil_test.py` self-locks, while a rule added in the same
+revision made the lock the thing that keeps concurrent operators off each other's hardware —
+so an operator following the second would take a hold that made its own run fail fast against
+it. Both statements were fixed before this branch was folded, so neither survives in history;
+what survives is the lesson that nothing checks these files against the code they describe.
+
+That is not an isolated slip. A scan of the 36 repo paths cited across `.claude/` flags 8
+that do not resolve. Five are legitimate — placeholders (`docs/changelog/X.Y.Z.md`,
+`src/portable/x/dcd_x.c`, a `test_*.py` glob), a generated file
+(`examples/cmake-build-pvs/compile_commands.json`), and a per-host gitignored config
+(`test/hil/local.json`, whose absence the skill already handles). Three are drift:
+`usbtest/SKILL.md:24,50` cites `src/usb_descriptors.h` and `src/tusb_config.h`, which are
+example-relative but read as repo paths, and `:101` cites `tools/usb/testusb.c`, a Linux
+kernel path presented like a repo file.
+
+Cross-references are in better shape: every `agentType` in a workflow resolves to an agent
+in `.claude/agents/`, every `.claude/skills/<name>` referenced by an agent or workflow
+exists, and the workflow scripts call only harness functions that exist. The drift is in
+**prose claims about behavior** — the class that made `hil-validate` parallelize at the
+wrong layer, on top of a `hil_test.py` that already schedules boards across host
+controllers under per-controller permits (`hil_lock.py:7,133-134`; `hil_test.py:2249,2419`).
+
+## Scope
+
+**In:** `.claude/agents/*.md` (7), `.claude/workflows/*.js` + `check.sh` (7),
+`.claude/skills/*/SKILL.md` (16) and their 8 helper scripts, and the repo `CLAUDE.md`.
+~4,700 lines (2,874 of prose, the rest helper scripts and `etm-trace/boards.md`).
+
+**Out:** `docs/superpowers/**` (historical records — correcting them rewrites history
+rather than fixing what a future session executes), `.claude/settings*.json` and hooks, the
+memory index, and any behavior change to the scripts themselves.
+
+## Claim taxonomy
+
+Only falsifiable classes get a verdict. Guidance ("bias toward caution") is checked solely
+for contradiction with the classes below.
+
+| Class | Settled by | Example |
+|---|---|---|
+| Path | `ls`/`find`, with the base dir made explicit | `src/tusb_config.h` — example-relative, reads as repo-relative |
+| Interface | argparse/grep in the named source | `-b` is `action='append'` (`hil_test.py:2249`) |
+| Behavior | reading the implementing code, cited `file:line` | "permits are in-process semaphores" (`hil_lock.py:7`) |
+| Number | the constant's definition | `FLASH_PARALLEL=4` (`hil_lock.py:133`) |
+| Rig state | read-only `ssh ci.lan` probe | bus map, probe uids, sudoers entries, installed tools |
+| Cross-doc | diffing the same rule's two statements | `hil-operator.md:18` vs `:37` |
+
+### Verdicts
+
+- **CONFIRMED** — current source says so. Cite `file:line`. Leave alone.
+- **REFUTED** — current source says otherwise. Cite, correct the doc.
+- **EARNED** — no source in scope settles it, and it is hard-earned rig knowledge. Stays in
+ the docs untouched; see the rule below.
+- **UNVERIFIABLE** — no source in scope settles it and it is not earned knowledge either
+ (a placeholder, a generated file, a claim about something outside the repo).
+
+### Hard-earned evidence is source of truth
+
+A claim with no code backing is **not** a cut candidate when it is earned rig knowledge:
+an observed hardware quirk, a failure mode paid for in rig downtime, a workaround whose
+rationale lives only in the incident that produced it. Code is authoritative about code;
+experience is authoritative about hardware, and the hardware does not document itself.
+
+Consequences:
+
+- Only a claim the **current source actively refutes** gets corrected. "I could not find
+ backing" is never grounds for deletion.
+- Rig-state claims that have gone stale (a bus map, a probe uid) are **re-derived and
+ updated**, or converted into a derivation recipe ("buses renumber every boot — re-derive
+ with X"), never dropped.
+- Where earned knowledge and current code disagree, that is a **finding to report**, not an
+ edit to make: one of them is a bug, and deciding which is out of this audit's scope.
+
+## Passes
+
+1. **Extraction (fan-out, 9 agents, no verdicts).** One agent per cluster, each writing a
+ ledger to the scratchpad and returning only a count and the ledger path. Per claim:
+ `file:line`, verbatim claim, class, what source would settle it, and a flag for
+ suspected hard-earned evidence. Agents return no judgments, so nothing arrives as a
+ verdict that would have to be unwound.
+2. **Verification (mine).** Every claim checked against source myself: scripted checks for
+ paths/interfaces/numbers, code reading for behavior, read-only `ssh ci.lan` for rig
+ state (`ls`, `--help`, `which`, `lspci`, `lsusb`, `hil_lock.py status`, `sudo -l`,
+ `uname -r` — no locks, no flashing, no `uhubctl`, no recovery). Nothing acted on is
+ taken on an extractor's word.
+3. **Cross-doc consistency (mine).** Build a rule inventory — board locks, timeouts,
+ output contracts, retry policy, config selection, forcing — and diff every place each
+ rule is stated. No per-file agent can do this pass; it is where the `hil-operator`
+ failure lived.
+4. **Edits.** Delete only what is refuted by source, restates the command it precedes, or
+ duplicates a rule that has a canonical home elsewhere (keep one, reference it). Keep
+ every claim source confirms that changes behavior, every hard-earned observation, and
+ the "why" behind non-obvious rules. Structure stays as is.
+5. **Gate.** Re-run the path and interface scans; `check.sh` on every workflow; `bash -n`
+ and `py_compile` on all 8 helper scripts; the four `test/hil` suites;
+ `pre-commit run --all-files`.
+
+## Extraction clusters
+
+| # | Cluster | Lines |
+|---|---|---|
+| 1 | `.claude/agents/*.md` (7 files) | 313 |
+| 2 | `.claude/workflows/*.js` + `check.sh` | 659 |
+| 3 | `hil`, `hil-pool-check` | 223 |
+| 4 | `usb-kernel-recover`, `usb-kernel-debug` + 2 scripts | 253 + scripts |
+| 5 | `target-debug`, `esp-target-debug` | 496 |
+| 6 | `usbtest`, `usbmon`, `usb-sniffer` + `usbcap.sh` | 382 + script |
+| 7 | `etm-trace` + `boards.md` + 2 scripts | 203 + files |
+| 8 | `build-doc`, `code-size`, `pvs`, `make-release`, `read-doc`, `pre-pr` + 2 scripts | 345 + scripts |
+| 9 | `CLAUDE.md` | 139 |
+
+## Deliverables
+
+Commits split by surface (agents / workflows / skills / CLAUDE.md) so review stays
+tractable, on `claude/claude-doc-audit`. A findings report covering every REFUTED claim
+with its citation, and every earned-knowledge-vs-code disagreement found in pass 2.
+
+A refuted claim whose *code* is the wrong half does not get a silent code edit: it becomes
+a handoff doc under `docs/superpowers/followup/`, per the repo's deferred-work rule.
+
+## Success criteria
+
+- Every falsifiable claim in scope carries a verdict with a citation.
+- No claim that current source refutes survives in the tree.
+- No hard-earned observation is deleted; stale rig state is re-derived or turned into a
+ derivation recipe.
+- No rule is stated in two places with two different meanings.
+- The gate in pass 5 passes.
diff --git a/test/hil/helper/hil_pool_check.py b/test/hil/helper/hil_pool_check.py
index 371aff1e1..d926bbe3d 100644
--- a/test/hil/helper/hil_pool_check.py
+++ b/test/hil/helper/hil_pool_check.py
@@ -328,7 +328,8 @@ def flash(board: dict, fw, allow_recovery: bool, probe_port: str, note: list) ->
if rc == 0:
return True
if rc == 127: # flasher binary missing: retries/probe recovery can't fix env
- note.append(f'flasher tool missing ({err}) — esptool needs the ESP-IDF env (get-idf)'
+ note.append(f'flasher tool missing ({err}) — esptool needs the ESP-IDF env '
+ f'(. "$IDF_PATH/export.sh")'
if board['flasher']['name'].lower() == 'esptool' else
f'flasher tool missing: {err}')
return False
@@ -487,7 +488,8 @@ def ensure_fw(board: dict, variant: str, example: str, note: list):
rc = build_example(board, variant, example)
if rc == 127 and board['flasher']['name'].lower() == 'esptool':
_builds[key] = (None, 'no-env')
- note.append(f'cannot build {base}: ESP-IDF env missing (get-idf)')
+ note.append(f'cannot build {base}: ESP-IDF env missing '
+ f'(. "$IDF_PATH/export.sh")')
return None
if rc == 124: # hung build: a deps/cache retry cannot cure it, don't double the stall
_builds[key] = (None, 'timeout')
diff --git a/test/hil/helper/hil_summary.py b/test/hil/helper/hil_summary.py
new file mode 100644
index 000000000..e566bead0
--- /dev/null
+++ b/test/hil/helper/hil_summary.py
@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: MIT
+"""Fold hil_report.json into one machine-readable verdict per BOARD.
+
+A workflow driving hil_test.py through an operator agent has no filesystem access, so the
+agent has to carry the results across. It must carry them, not retype them: the previous
+design asked the agent to transcribe the markdown table, and every defect found in four
+review rounds came from re-parsing that prose -- variant row names vs board names,
+`board locked` vs `board-locked`, folding several variant rows into one verdict, rows that
+matched no board. All of it is a join, and the join belongs here, where the roster is.
+
+Report rows are named per VARIANT (hil_test.py builds them from `vname`), and a variant name
+is not required to start with the board name -- nanoch32v203 produces only `-fsdev`/`-usbfs`,
+ch32v307v_r1_1v0 only `-usbhs`/`-usbfs`. The config is what maps them back.
+
+Emits, on stdout:
+ {"results": [{"board", "ran", "pass", "locked", "detail"}...], "banner": str}
+
+`locked` is a field, not a prefix to grep for. `ran` false means the board produced no row at
+all, which is not the same as failing.
+
+Usage: hil_summary.py <config.json> [-b BOARD]... [--report-dir DIR]
+"""
+import argparse
+import json
+import sys
+from pathlib import Path
+
+FAIL_ICON, SKIP_ICON = '❌', '⚪' # a pass needs no icon: unmarked = pass
+LOCKED_CELL = 'board-locked'
+
+
+def cell_state(v: str) -> str:
+ """'pass' | 'fail' | 'skip' -- the EXACT classifier hil_test.py's own tally uses
+ (cell_kind in render_matrix): 'fail' or a ❌ prefix is a failure, 'skip' or a ⚪
+ prefix is a skip, and EVERYTHING ELSE is a pass. That last arm is load-bearing: a
+ passing test may return a plain metric string ('480.0 MBps') that lands in the cell
+ unprefixed, while failures are guaranteed marked -- TestFail's docstring pins that its
+ metric is icon-prefixed precisely so render/tally treat it as a failure. Classifying
+ unknown shapes as fail here would publish a green table as a red verdict."""
+ if v == 'fail' or v.startswith(FAIL_ICON):
+ return 'fail'
+ if v == 'skip' or v.startswith(SKIP_ICON):
+ return 'skip'
+ return 'pass'
+
+
+def variants_of(cfg: dict, board: str) -> list:
+ for b in cfg.get('boards', []):
+ if b['name'] == board:
+ return [v['name'] for v in (b.get('variant') or [])] or [board]
+ return [board]
+
+
+def summarize(cfg: dict, boards: list, report: dict) -> dict:
+ rows = {r['board']: r.get('cells') or {} for r in report.get('rows', [])}
+ owner = {v['name']: b['name'] for b in cfg.get('boards', [])
+ for v in (b.get('variant') or [])}
+ results = []
+ for board in boards:
+ names = variants_of(cfg, board)
+ mine = {n: rows[n] for n in names if n in rows}
+ # a variant name that is neither declared nor prefixed cannot be attributed; the
+ # `<board>-` fallback only helps ad-hoc builds, it is not the primary path. It must
+ # also never steal a row DECLARED by another board: a declared variant need not start
+ # with its own board's name, so it may happen to start with this board's name plus '-'.
+ mine.update({n: c for n, c in rows.items()
+ if n.startswith(f'{board}-') and n not in mine
+ and owner.get(n, board) == board})
+ if not mine:
+ results.append({'board': board, 'ran': False, 'pass': False, 'locked': False,
+ 'detail': 'no report row for this board'})
+ continue
+ locked = any(LOCKED_CELL in cells for cells in mine.values())
+ bad = []
+ for vname, cells in sorted(mine.items()):
+ for test, val in sorted(cells.items()):
+ if test == LOCKED_CELL:
+ continue
+ if cell_state(str(val)) == 'fail':
+ bad.append(f'{vname} {test}: {val}')
+ ok = not bad and not locked
+ if locked:
+ detail = 'held by another holder; not flashed'
+ elif bad:
+ detail = '; '.join(bad)
+ else:
+ detail = f'{len(mine)} variant(s), {sum(len(c) for c in mine.values())} cell(s) ok'
+ results.append({'board': board, 'ran': True, 'pass': ok, 'locked': locked,
+ 'detail': detail})
+ return {'results': results, 'banner': report.get('banner', '')}
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser()
+ ap.add_argument('config_file')
+ ap.add_argument('-b', '--board', action='append', default=[],
+ help='boards to report on; default: every board in the config')
+ ap.add_argument('--report-dir', default='.', help='where hil_report.json lives (default: cwd)')
+ a = ap.parse_args()
+
+ cfg = json.loads(Path(a.config_file).read_text())
+ boards = a.board or [b['name'] for b in cfg.get('boards', [])]
+ jpath = Path(a.report_dir) / 'hil_report.json'
+ if not jpath.is_file():
+ print(f'error: {jpath} not found -- did hil_test.py run in this directory?',
+ file=sys.stderr)
+ return 1
+ json.dump(summarize(cfg, boards, json.loads(jpath.read_text())), sys.stdout, indent=2)
+ print()
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main())
diff --git a/test/hil/hil_ci.sh b/test/hil/hil_ci.sh
index c7dfa95df..66f4e48d4 100644
--- a/test/hil/hil_ci.sh
+++ b/test/hil/hil_ci.sh
@@ -1,8 +1,9 @@
#!/usr/bin/env bash
# Run HIL test remotely on ci.lan
-# Usage: test/hil/hil_ci.sh [-b BOARD] [-t TEST] [extra hil_test.py args...]
+# Usage: test/hil/hil_ci.sh [-b BOARD]... [-t TEST] [extra hil_test.py args...]
# Example:
# test/hil/hil_ci.sh -b stm32f723disco
+# test/hil/hil_ci.sh -b stm32f723disco -b raspberry_pi_pico
# test/hil/hil_ci.sh -b stm32f723disco -t host/cdc_msc_hid -r 1
#
# Env overrides: REMOTE, REMOTE_DIR, CONFIG (path to HIL config json),
@@ -44,17 +45,46 @@ for a in "$@"; do
exit 1
done
-# Parse -b BOARD from arguments to know which build to copy
-BOARD=""
+# Parse -b BOARD from arguments to know which builds to copy. Repeatable: hil_test.py
+# takes the whole board set in ONE run (it schedules them across host controllers and
+# budgets the flashes itself), so every -b needs its binaries staged, not just the last.
+BOARDS=()
ARGS=()
while [[ $# -gt 0 ]]; do
case "$1" in
- -b)
- [[ $# -ge 2 ]] || { echo "error: -b requires a BOARD argument" >&2; exit 1; }
- BOARD="$2"
+ # hil_test.py declares `-b, --board` with action='append', so argparse also accepts
+ # --board=X and -bX. Recognising only the bare `-b X` forwarded the others to the rig
+ # while never staging them: the board ran with no firmware and reported a green row.
+ -b|--board)
+ [[ $# -ge 2 ]] || { echo "error: $1 requires a BOARD argument" >&2; exit 1; }
+ BOARDS+=("$2")
ARGS+=("$1" "$2")
shift 2
;;
+ --board=*)
+ BOARDS+=("${1#--board=}")
+ ARGS+=("$1")
+ shift
+ ;;
+ # -bt (--board-test) BEFORE the glued -b?* arm, mirroring argparse's longest-match: it is
+ # the form <config>.failed uses, and a bare -b?* would register a board named "t..." that
+ # the roster check below rejects -- killing every documented retry.
+ -bt|--board-test)
+ [[ $# -ge 2 ]] || { echo "error: $1 requires NAME:tests" >&2; exit 1; }
+ ARGS+=("$1" "$2")
+ shift 2
+ ;;
+ -bt?*|--board-test=*)
+ ARGS+=("$1")
+ shift
+ ;;
+ # glued short form: argparse resolves -bNAME to --board NAME, so staging must too --
+ # unparsed it fell through to the all-boards branch and silently staged everything built
+ -b?*)
+ BOARDS+=("${1#-b}")
+ ARGS+=("$1")
+ shift
+ ;;
*)
ARGS+=("$1")
shift
@@ -62,6 +92,110 @@ while [[ $# -gt 0 ]]; do
esac
done
+# Resolve a board to its build dirs: its own dir, the cmake-build-<board>-* glob (ad-hoc
+# local builds), and the variant dirs named in $CONFIG -- variant names are NOT required to
+# be prefixed with the board name, so the glob alone is not enough. Prints one dir per line.
+variant_names() {
+ python3 -c '
+import json, sys
+cfg = json.load(open(sys.argv[1]))
+for b in cfg.get("boards", []):
+ if b["name"] == sys.argv[2]:
+ for v in b.get("variant") or []:
+ print(v["name"])
+' "$CONFIG" "$1"
+}
+
+resolve_build_dirs() {
+ local board="$1" d v
+ declare -A seen=()
+ shopt -s nullglob
+ for d in "$ROOT_DIR"/examples/cmake-build-"$board" "$ROOT_DIR"/examples/cmake-build-"$board"-*; do
+ [[ -d $d && -z ${seen[$d]:-} ]] && { seen[$d]=1; printf '%s\n' "$d"; }
+ done
+ shopt -u nullglob
+ # to a file, not a process substitution: `set -e`/pipefail cannot see the exit status of
+ # the latter, so a malformed roster silently yielded zero variant dirs
+ local vf; vf=$(mktemp)
+ variant_names "$board" > "$vf" || { rm -f "$vf"; echo "Error: could not read variants for $board from $CONFIG" >&2; exit 1; }
+ while IFS= read -r v; do
+ d="$ROOT_DIR/examples/cmake-build-$v"
+ [[ -d $d && -z ${seen[$d]:-} ]] && { seen[$d]=1; printf '%s\n' "$d"; }
+ done < "$vf"
+ rm -f "$vf"
+}
+
+# Pre-flight: EVERY board must resolve to at least one build dir before anything is wiped or
+# copied. This check used to live in the copy loop, so an unbuilt board late in the list
+# aborted the run after the remote tree had been rm -rf'd and earlier boards fully rsynced --
+# zero coverage, a half-staged rig, and a stale local hil_report.md left in place. Report all
+# missing boards at once so one build round fixes them.
+MANIFEST=$(mktemp)
+trap 'rm -f "$MANIFEST"' EXIT
+# Roster membership first: hil_test.py rejects an unknown -b with sys.exit(1) for the WHOLE
+# run (hil_test.py:2297), and it does so AFTER this script has wiped REMOTE_DIR and staged
+# every board -- one typo then costs the entire batch. We already parse $CONFIG here, so
+# catch it before anything is touched. Note -b matches board names only, never variant names.
+if [ ${#BOARDS[@]} -gt 0 ]; then
+ROSTER=$(python3 -c '
+import json, sys
+print("\n".join(b["name"] for b in json.load(open(sys.argv[1])).get("boards", [])))
+' "$CONFIG") || { echo "error: could not read the board roster from $CONFIG" >&2; exit 1; }
+notinroster=()
+for b in ${BOARDS[@]+"${BOARDS[@]}"}; do
+ grep -qxF -- "$b" <<< "$ROSTER" || notinroster+=("$b")
+done
+if [ ${#notinroster[@]} -gt 0 ]; then
+ echo "error: not in $(basename "$CONFIG"): ${notinroster[*]}" >&2
+ echo " (-b takes board names, not variant names)" >&2
+ exit 1
+fi
+fi # BOARDS non-empty: nothing to validate for an all-boards run
+
+missing=()
+for b in ${BOARDS[@]+"${BOARDS[@]}"}; do
+ dirs=$(resolve_build_dirs "$b")
+ if [ -z "$dirs" ]; then
+ missing+=("$b")
+ else
+ while IFS= read -r d; do printf '%s\t%s\n' "$b" "$d" >> "$MANIFEST"; done <<< "$dirs"
+ # A declared variant with no build dir is NOT an error -- no cmake preset is
+ # variant-suffixed, so this is the normal state for e.g. the -DMA variants. It is worth
+ # saying out loud: hil_test.py logs `Skip (no binary)` and counts zero errors for it, so
+ # the run exits 0 and the operator reads a green table for cells that never ran.
+ # plain assignment, not process substitution: set -e sees a variant_names failure here,
+ # the same trap the comment in resolve_build_dirs warns about
+ vnames=$(variant_names "$b")
+ while IFS= read -r v; do
+ [ -z "$v" ] && continue
+ # whole lines: a substring match lets cmake-build-<v>-DMA silence the warning for <v>
+ grep -qxF -- "$ROOT_DIR/examples/cmake-build-$v" <<< "$dirs" \
+ || echo "warning: $b variant '$v' has no build dir -- its cells will be skipped, not tested" >&2
+ done <<< "$vnames"
+ fi
+done
+if [ ${#missing[@]} -gt 0 ]; then
+ echo "Error: no build directory under $ROOT_DIR/examples/ for: ${missing[*]}" >&2
+ for b in "${missing[@]}"; do
+ echo " cd examples && cmake --preset $b && cmake --build --preset $b" >&2
+ done
+ exit 1
+fi
+
+# The all-boards form needs its emptiness check HERE too: below the setup ssh it fired after
+# the remote tree was already rm -rf'd, destroying the previous run's report and re-run spec
+# on the rig before deciding there was nothing to do.
+if [ ${#BOARDS[@]} -eq 0 ]; then
+ shopt -s nullglob
+ allbuilds=("$ROOT_DIR"/examples/cmake-build-*/)
+ shopt -u nullglob
+ if [ ${#allbuilds[@]} -eq 0 ]; then
+ echo "error: no examples/cmake-build-* directories under $ROOT_DIR -- nothing to test" >&2
+ echo " build first, e.g.: cd examples && cmake --preset <board> && cmake --build --preset <board>" >&2
+ exit 1
+ fi
+fi
+
# Setup remote directory. `bash -s` + heredoc so REMOTE_DIR arrives as a positional
# parameter, keeping the `rm -rf` target out of the command string the heredoc runs.
echo "==> Setting up remote $REMOTE:$REMOTE_DIR"
@@ -89,6 +223,7 @@ scp -q "$ROOT_DIR/test/hil/helper/__init__.py" \
"$ROOT_DIR/test/hil/helper/hil_util.py" \
"$ROOT_DIR/test/hil/helper/hil_health.py" \
"$ROOT_DIR/test/hil/helper/hil_lock.py" \
+ "$ROOT_DIR/test/hil/helper/hil_summary.py" \
"$ROOT_DIR/test/hil/helper/hil_select.py" \
"$REMOTE:$REMOTE_DIR/test/hil/helper/"
@@ -103,52 +238,23 @@ copy_board_binaries() {
"$src" "$REMOTE:$REMOTE_DIR/examples/"
}
-if [ -n "$BOARD" ]; then
- # Copy the board's build dir plus its variant dirs. Variant names come from
- # $CONFIG (they are not required to be prefixed with the board name); the
- # cmake-build-<BOARD>-* glob is kept as a fallback for ad-hoc local builds.
- # Collect only dirs that actually exist, deduplicated.
- declare -A SEEN_DIRS=()
- BUILD_DIRS=()
- add_build_dir() {
- [[ -d "$1" && -z "${SEEN_DIRS[$1]:-}" ]] || return 0
- SEEN_DIRS[$1]=1
- BUILD_DIRS+=("$1")
- }
- shopt -s nullglob
- for d in "$ROOT_DIR"/examples/cmake-build-"$BOARD" "$ROOT_DIR"/examples/cmake-build-"$BOARD"-*; do
- add_build_dir "$d"
- done
- shopt -u nullglob
- # to a file, not a process substitution: `set -e`/pipefail cannot see the exit
- # status of the latter, so a malformed roster silently yielded zero variant dirs
- VARIANTS_FILE=$(mktemp)
- python3 -c '
-import json, sys
-cfg = json.load(open(sys.argv[1]))
-for b in cfg.get("boards", []):
- if b["name"] == sys.argv[2]:
- for v in b.get("variant") or []:
- print(v["name"])
-' "$CONFIG" "$BOARD" > "$VARIANTS_FILE" || {
- echo "Error: could not read variants for $BOARD from $CONFIG"
- rm -f "$VARIANTS_FILE"
- exit 1
- }
- while IFS= read -r v; do
- add_build_dir "$ROOT_DIR/examples/cmake-build-$v"
- done < "$VARIANTS_FILE"
- rm -f "$VARIANTS_FILE"
- if [ ${#BUILD_DIRS[@]} -eq 0 ]; then
- echo "Error: no build directory found for $BOARD under $ROOT_DIR/examples/"
- echo "Build first with: cd examples && cmake --preset $BOARD && cmake --build --preset $BOARD"
- exit 1
- fi
- echo "==> Copying binaries for $BOARD (${#BUILD_DIRS[@]} build dir(s))"
- for d in "${BUILD_DIRS[@]}"; do
- copy_board_binaries "$d"
+if [ ${#BOARDS[@]} -gt 0 ]; then
+ # Replay the pre-flight manifest: the dirs were already resolved and proved non-empty
+ # for every board, so nothing here can abort mid-staging. Plain reads of the manifest --
+ # a process substitution would hide a reader failure from set -e (the comment in
+ # resolve_build_dirs is about exactly that trap).
+ for b in "${BOARDS[@]}"; do
+ dirs=()
+ while IFS=$'\t' read -r bb d; do
+ [ "$bb" = "$b" ] && [ -n "$d" ] && dirs+=("$d")
+ done < "$MANIFEST"
+ echo "==> Copying binaries for $b (${#dirs[@]} build dir(s))"
+ for d in ${dirs[@]+"${dirs[@]}"}; do
+ copy_board_binaries "$d"
+ done
done
else
+ # emptiness was already refused in pre-flight, before the remote wipe
echo "==> Copying all built binaries"
# Use `%/` parameter expansion to strip the trailing slash from the glob —
# rsync needs the bare dir name so the per-board cmake-build-<BOARD>/ subdir
@@ -170,14 +276,38 @@ for a in ${ARGS[@]+"${ARGS[@]}"}; do ARGS_Q+=("$(printf '%q' "$a")"); done
CONFIG_Q="$(printf '%q' "test/hil/$(basename "$CONFIG")")"
echo "==> Running HIL test on $REMOTE"
rc=0
-# --retry 1 FIRST, before the user's args: this targets the same shared rig CI uses, and
-# the pool guard is a flat constant that does not scale with max_retry -- argparse's
-# default of 3 lets a few flaky boards re-pay 510s each until the 3600s guard fires,
-# abandoning the pool and holding board flocks against concurrent CI. Placed first, not
-# appended, so argparse's last-wins means `hil_ci.sh -r 3` still gets 3.
-ssh "$REMOTE" bash -s -- "$REMOTE_DIR" --retry 1 ${ARGS_Q[@]+"${ARGS_Q[@]}"} "$CONFIG_Q" <<'REMOTE' || rc=$?
+# --retry 1 FIRST, before the user's args: this targets the same shared rig CI uses, and the
+# pool guard is a flat constant that does not scale with max_retry, so a few flaky boards can
+# re-pay ~510s each until the 3600s guard fires, abandoning the pool and holding board flocks
+# against concurrent CI. hil_test.py's own default is already 1; passing it explicitly keeps
+# that true if the default ever moves. Placed first, not appended, so argparse's last-wins
+# means `hil_ci.sh -r 3` still gets 3.
+# Forward the HIL_* knobs (HIL_NO_BOARD_LOCK for an authorized force, the parallel widths,
+# HIL_POOL_TIMEOUT). ssh passes no environment and joins its argv into one string the remote
+# shell re-splits, so a bare NAME=value element would arrive as a positional argument to
+# hil_test.py and argparse would exit 2. Build `export` lines instead and hand them over as a
+# single %q-quoted word for the remote to eval.
+# Joined with '; ', NOT newlines: %q renders a newline as bash-only $'...' quoting, which the
+# remote LOGIN shell must parse from the joined command string -- under dash the force arrives
+# as garbage and silently does nothing. Backslash escaping round-trips in both shells.
+# HIL_REPORT_DIR stays local: where the report lands on the rig is this script's contract
+# (REMOTE_DIR, where all three copy-backs below look), so forwarding it would relocate the
+# report and every copy-back would come home empty.
+HIL_EXPORTS=""
+while IFS= read -r v; do
+ [ -z "$v" ] && continue
+ HIL_EXPORTS+="export $(printf '%s=%q' "$v" "${!v}"); "
+done < <(compgen -v | grep -x 'HIL_[A-Z0-9_]*' | grep -vxE 'HIL_EXPORTS|HIL_REPORT_DIR' || true)
+[ -n "$HIL_EXPORTS" ] && echo "==> Forwarding: $HIL_EXPORTS"
+# One %q-quoted word, so ssh's argv join and the remote shell's re-split hand it back
+# byte-for-byte, and the remote evals it. Empty stays `''` -- a real, shiftable argument --
+# rather than vanishing from the joined string and shifting the run's own flags out of place.
+HIL_EXPORTS_Q=$(printf '%q' "$HIL_EXPORTS")
+
+ssh "$REMOTE" bash -s -- "$REMOTE_DIR" "$HIL_EXPORTS_Q" --retry 1 ${ARGS_Q[@]+"${ARGS_Q[@]}"} "$CONFIG_Q" <<'REMOTE' || rc=$?
cd -- "$1"
shift
+eval "$1"; shift # HIL_* exports, %q-quoted locally into one word
# Flasher CLIs live in the user bin dirs on ci.lan (esptool/idf in ~/.local/bin,
# STM32CubeProgrammer's STM32_Programmer_CLI in ~/bin); the non-interactive shell
# subprocess used for flashing doesn't source profile/rc, so add them explicitly.
@@ -191,4 +321,15 @@ scp -q "$REMOTE:$REMOTE_DIR/hil_report.md" "$ROOT_DIR/hil_report.md" \
&& echo "==> Report copied to $ROOT_DIR/hil_report.md" \
|| echo "==> warning: no hil_report.md copied back" >&2
+# The re-run spec and the JSON sidecar live in the run's cwd on the rig (REMOTE_DIR), and the
+# next invocation rm -rf's it. Without copying them back, the `--accumulate` retry every doc on
+# this branch prescribes has nothing to read and nothing to merge onto. Delete the local copies
+# FIRST: a green run writes no .failed, so a silent no-op scp would leave last run's spec in
+# the checkout looking current, and "retry from the spec" would re-flash boards that passed.
+for extra in "$(basename "$CONFIG").failed" hil_report.json; do
+ rm -f "$ROOT_DIR/$extra"
+ scp -q "$REMOTE:$REMOTE_DIR/$extra" "$ROOT_DIR/$extra" 2>/dev/null \
+ && echo "==> $extra copied to $ROOT_DIR/$extra" || true
+done
+
exit $rc
diff --git a/test/hil/test/test_hil_bounded.py b/test/hil/test/test_hil_bounded.py
index 908a142d5..c6d454f0e 100644
--- a/test/hil/test/test_hil_bounded.py
+++ b/test/hil/test/test_hil_bounded.py
@@ -1364,7 +1364,14 @@ class RemoteDirIsScreened(unittest.TestCase):
rc = 0 if keep_going else 77
for tool in ('ssh', 'scp', 'rsync'):
write_script(Path(td) / tool, f'echo "stub-{tool} $*" >&2; exit {rc}')
+ # hil_ci.sh now refuses an all-boards run with nothing built, so this arg-quoting
+ # test needs a checkout stub with one build dir to reach the run invocation
+ root = Path(td) / 'root'
+ (root / 'test' / 'hil').mkdir(parents=True)
+ (root / 'test' / 'hil' / 'hil_test.py').touch()
+ (root / 'examples' / 'cmake-build-alpha').mkdir(parents=True)
env = {**os.environ, 'REMOTE_DIR': remote_dir, 'REMOTE': 'stub',
+ 'ROOT_DIR': str(root),
'PATH': td + os.pathsep + os.environ['PATH']}
return subprocess.run(
['bash', str(Path(TEST_DIR).parents[0] / 'hil_ci.sh'), *args],
@@ -1414,6 +1421,316 @@ class RemoteDirIsScreened(unittest.TestCase):
self.assertIn(r'host/cdc\ msc', run_line)
+class EveryBoardIsStaged(unittest.TestCase):
+ """One hil_test.py run takes several `-b` flags, and hil-operator hands it the whole board
+ set that way. The `-b` parse loop kept a single BOARD, so only the LAST board's binaries
+ were rsynced and every other board died on the rig with a missing firmware path -- after
+ its flash slot and lock were already spent.
+
+ Three of these five fail against the pre-fix script (the discriminating unbuilt case
+ puts the board FIRST, because the old single-BOARD parse happened to handle a trailing
+ one correctly); the run-line and variant-dir tests are characterization -- the old script
+ already forwarded ARGS whole and read variants from the config for its one board."""
+
+ def _run(self, boards, cfg_boards=None, variants=None):
+ import json
+ import subprocess
+ td = TemporaryDirectory()
+ self.addCleanup(td.cleanup)
+ root = Path(td.name) / 'root'
+ (root / 'test' / 'hil').mkdir(parents=True)
+ (root / 'test' / 'hil' / 'hil_test.py').touch()
+ built = cfg_boards if cfg_boards is not None else boards
+ (root / 'examples').mkdir(parents=True, exist_ok=True)
+ for b in built:
+ (root / 'examples' / f'cmake-build-{b}').mkdir(parents=True)
+ roster = [{'name': b} for b in boards]
+ for entry in roster:
+ for v in (variants or {}).get(entry['name'], []):
+ entry.setdefault('variant', []).append({'name': v})
+ cfg = root / 'test' / 'hil' / 'cfg.json'
+ cfg.write_text(json.dumps({'boards': roster}))
+ stubs = Path(td.name) / 'bin'
+ stubs.mkdir()
+ # real ssh/scp/rsync would reach the rig; these just record the argv
+ for tool in ('ssh', 'scp', 'rsync'):
+ write_script(stubs / tool, f'echo "stub-{tool} $*" >&2; exit 0')
+ env = {**os.environ, 'REMOTE': 'stub', 'ROOT_DIR': str(root), 'CONFIG': str(cfg),
+ 'PATH': str(stubs) + os.pathsep + os.environ['PATH']}
+ args = [a for b in boards for a in ('-b', b)]
+ r = subprocess.run(['bash', str(Path(TEST_DIR).parents[0] / 'hil_ci.sh'), *args],
+ capture_output=True, text=True, timeout=60, env=env)
+ r.rsyncs = [l for l in r.stderr.splitlines() if l.startswith('stub-rsync')]
+ # the RUN ssh is the one carrying hil_test.py's args; the setup ssh is not
+ r.run_lines = [l for l in r.stderr.splitlines() if '--retry 1' in l]
+ return r
+
+ def test_binaries_for_every_requested_board_are_copied(self):
+ r = self._run(['alpha', 'beta', 'gamma'])
+ self.assertEqual(r.returncode, 0, r.stderr)
+ for b in ('alpha', 'beta', 'gamma'):
+ self.assertTrue(any(f'cmake-build-{b} ' in l for l in r.rsyncs),
+ f'{b} binaries never staged: {r.rsyncs}')
+
+ def test_every_board_reaches_hil_test(self):
+ r = self._run(['alpha', 'beta'])
+ self.assertEqual(len(r.run_lines), 1, r.stderr)
+ self.assertIn('-b alpha', r.run_lines[0])
+ self.assertIn('-b beta', r.run_lines[0])
+
+ def test_an_unbuilt_board_aborts_before_anything_is_staged(self):
+ """The discriminating case: the unbuilt board is FIRST. The pre-fix script kept only
+ the last -b, found it built, and ran happily while silently testing one board. It also
+ has to fail BEFORE staging -- the old in-loop check fired after the remote tree was
+ wiped and earlier boards were rsynced, costing a run and leaving a half-staged rig."""
+ r = self._run(['alpha', 'beta'], cfg_boards=['beta'])
+ self.assertNotEqual(r.returncode, 0, 'unbuilt first board was accepted')
+ self.assertIn('alpha', r.stdout + r.stderr)
+ self.assertEqual(r.rsyncs, [], f'staged despite an unbuilt board: {r.rsyncs}')
+ self.assertEqual(r.run_lines, [], 'reached the run despite an unbuilt board')
+
+ def test_a_board_whose_firmware_is_only_a_variant_dir_is_accepted(self):
+ """Variant names are not required to be prefixed with the board name, so a board can
+ own no `cmake-build-<board>` dir at all. A pre-flight that only globs the board name
+ rejects it and tells the user to build firmware that is already there."""
+ r = self._run(['alpha', 'beta'], cfg_boards=['alpha', 'odd-name-v'],
+ variants={'beta': ['odd-name-v']})
+ self.assertEqual(r.returncode, 0, r.stderr)
+ self.assertTrue(any('cmake-build-odd-name-v ' in l for l in r.rsyncs),
+ f"beta's variant dir never staged: {r.rsyncs}")
+
+ def test_all_unbuilt_boards_are_named_at_once(self):
+ """One build round should fix every complaint, so the guard reports the whole set."""
+ r = self._run(['alpha', 'beta', 'gamma'], cfg_boards=['beta'])
+ self.assertNotEqual(r.returncode, 0)
+ out = r.stdout + r.stderr
+ self.assertIn('alpha', out)
+ self.assertIn('gamma', out)
+
+
+class StagingCoversEveryBoardForm(unittest.TestCase):
+ """hil_test.py declares `-b, --board` with action='append', so argparse accepts --board X,
+ --board=X and -bX too. Staging only the bare form sent boards to the rig with no firmware,
+ where every test logs `Skip (no binary)` and counts zero errors -- a green row for a board
+ that was never flashed. Also covers the roster check, which has to fire BEFORE the remote
+ tree is wiped, since hil_test.py rejects an unknown -b for the whole run."""
+
+ def _run(self, argv, built, roster=None, variants=None, env_extra=None, stale=None):
+ import json
+ import subprocess
+ td = TemporaryDirectory()
+ self.addCleanup(td.cleanup)
+ root = Path(td.name) / 'root'
+ (root / 'test' / 'hil').mkdir(parents=True)
+ (root / 'test' / 'hil' / 'hil_test.py').touch()
+ (root / 'examples').mkdir(parents=True, exist_ok=True)
+ for b in built:
+ (root / 'examples' / f'cmake-build-{b}').mkdir(parents=True)
+ entries = [{'name': b} for b in (roster if roster is not None else built)]
+ for e in entries:
+ for v in (variants or {}).get(e['name'], []):
+ e.setdefault('variant', []).append({'name': v})
+ cfg = root / 'test' / 'hil' / 'cfg.json'
+ cfg.write_text(json.dumps({'boards': entries}))
+ stubs = Path(td.name) / 'bin'
+ stubs.mkdir()
+ # ssh joins its argv into ONE string that the REMOTE shell re-splits, and feeds the
+ # heredoc on stdin. A stub that echoes "$*" hides exactly that, which is how a
+ # completely broken env-forwarding change once passed its own test -- so this stub
+ # re-splits like the real thing and reports the script body separately.
+ write_script(stubs / 'ssh', 'shift; printf "REMOTE-ARGV: %s\\n" "$*" >&2; '
+ 'body=$(cat); printf "REMOTE-BODY: %s\\n" "$body" >&2; exit 0')
+ for tool in ('scp', 'rsync'):
+ write_script(stubs / tool, f'echo "stub-{tool} $*" >&2; exit 0')
+ for name, content in (stale or {}).items():
+ (root / name).write_text(content)
+ env = {**os.environ, 'REMOTE': 'stub', 'ROOT_DIR': str(root), 'CONFIG': str(cfg),
+ 'PATH': str(stubs) + os.pathsep + os.environ['PATH'], **(env_extra or {})}
+ r = subprocess.run(['bash', str(Path(TEST_DIR).parents[0] / 'hil_ci.sh'), *argv],
+ capture_output=True, text=True, timeout=60, env=env)
+ r.rsyncs = [l for l in r.stderr.splitlines() if l.startswith('stub-rsync')]
+ r.run_lines = [l for l in r.stderr.splitlines() if '--retry 1' in l]
+ r.body = '\n'.join(l for l in r.stderr.splitlines() if l.startswith('REMOTE-BODY'))
+ r.stale_left = {name: (root / name).exists() for name in (stale or {})}
+ return r
+
+ def test_long_board_forms_are_staged_and_only_that_board(self):
+ """Two boards are built so the pre-fix 'copy all built binaries' else-branch cannot
+ stage the right one by accident -- that is what made the first version of this test
+ pass against master while the feature was broken. -balpha is the glued short form
+ argparse resolves to --board alpha; unparsed it fell through to the all-boards branch
+ and silently staged everything built with no roster check."""
+ for argv in (['--board', 'alpha'], ['--board=alpha'], ['-balpha']):
+ with self.subTest(argv=argv):
+ r = self._run(argv, built=['alpha', 'beta'], roster=['alpha', 'beta'])
+ self.assertEqual(r.returncode, 0, r.stderr)
+ self.assertTrue(any('cmake-build-alpha ' in l for l in r.rsyncs),
+ f'{argv} never staged: {r.rsyncs}')
+ self.assertFalse(any('cmake-build-beta ' in l for l in r.rsyncs),
+ f'{argv} staged an unrequested board: {r.rsyncs}')
+
+ def test_board_test_flag_is_not_mistaken_for_a_board(self):
+ """-bt is hil_test.py's --board-test and is exactly what <config>.failed contains, so
+ a glued -b?* pattern turns the documented retry into 'not in the roster: t'."""
+ for argv in (['-b', 'alpha', '-bt', 'alpha:device/cdc_msc'],
+ ['-b', 'alpha', '-btalpha:device/cdc_msc']):
+ with self.subTest(argv=argv):
+ r = self._run(argv, built=['alpha'])
+ self.assertEqual(r.returncode, 0, r.stderr)
+ self.assertNotIn('not in', r.stderr)
+ self.assertTrue(any('alpha:device/cdc_msc' in l for l in r.run_lines),
+ f'-bt never reached the rig: {r.run_lines}')
+
+ def test_a_board_outside_the_roster_is_refused_before_staging(self):
+ r = self._run(['-b', 'alpha', '-b', 'ghost'], built=['alpha', 'ghost'], roster=['alpha'])
+ self.assertNotEqual(r.returncode, 0)
+ self.assertIn('ghost', r.stdout + r.stderr)
+ self.assertEqual(r.rsyncs, [], 'staged despite an unknown board')
+ self.assertEqual(r.run_lines, [], 'reached the run despite an unknown board')
+
+ def test_a_variant_with_no_build_dir_warns_instead_of_passing_silently(self):
+ r = self._run(['-b', 'alpha'], built=['alpha'], variants={'alpha': ['alpha-DMA']})
+ self.assertEqual(r.returncode, 0, r.stderr)
+ self.assertIn('alpha-DMA', r.stderr)
+ self.assertIn('skipped, not tested', r.stderr)
+
+ def test_no_build_dirs_at_all_aborts_the_all_boards_form(self):
+ """`hil_ci.sh` with no -b stages everything built. With nothing built it used to wipe
+ the rig, stage nothing, and return a green all-skip table."""
+ r = self._run([], built=[], roster=['alpha'])
+ self.assertNotEqual(r.returncode, 0)
+ self.assertIn('nothing to test', r.stdout + r.stderr)
+ self.assertEqual(r.run_lines, [], 'reached the run with nothing staged')
+ self.assertNotIn('Setting up remote', r.stdout + r.stderr,
+ 'the guard fired only after the remote tree was already wiped')
+
+ def test_hil_env_reaches_the_rig_as_environment_not_argv(self):
+ """An authorized force is HIL_NO_BOARD_LOCK=1. Passed through ssh's argv it arrives as a
+ positional argument and argparse exits 2, so it has to travel in the script body."""
+ r = self._run(['-b', 'alpha'], built=['alpha'], env_extra={'HIL_NO_BOARD_LOCK': '1'})
+ self.assertEqual(r.returncode, 0, r.stderr)
+ # one %q-quoted word of `export NAME=value; ` fragments, evaluated by the remote —
+ # NOT a bare NAME=value element, which hil_test.py's argparse takes as a positional.
+ # %q backslash-escapes the spaces, so match the pieces rather than the plain phrase.
+ run = '\n'.join(r.run_lines)
+ self.assertIn('HIL_NO_BOARD_LOCK=1', run)
+ self.assertIn('export', run)
+ self.assertFalse(any(' HIL_NO_BOARD_LOCK=1 ' in l for l in r.run_lines),
+ 'env reached argv unquoted, where hil_test.py sees a positional')
+
+ def test_a_value_with_spaces_survives_forwarding(self):
+ r = self._run(['-b', 'alpha'], built=['alpha'],
+ env_extra={'HIL_SCRATCH': '/tmp/my scratch'})
+ self.assertEqual(r.returncode, 0, r.stderr)
+ run = '\n'.join(r.run_lines)
+ self.assertIn('HIL_SCRATCH', run)
+ self.assertIn('scratch', run)
+
+ def test_hil_report_dir_is_never_forwarded(self):
+ """Where the report lands on the rig is this script's contract (REMOTE_DIR, where all
+ three copy-backs look); forwarding a local HIL_REPORT_DIR relocates it there and every
+ copy-back comes home empty -- two of the three silently."""
+ r = self._run(['-b', 'alpha'], built=['alpha'],
+ env_extra={'HIL_REPORT_DIR': '/tmp/elsewhere'})
+ self.assertEqual(r.returncode, 0, r.stderr)
+ self.assertFalse(any('HIL_REPORT_DIR' in l for l in r.run_lines),
+ f'HIL_REPORT_DIR reached the rig: {r.run_lines}')
+
+ def test_a_stale_local_failed_spec_does_not_survive_a_green_run(self):
+ """A green run writes no .failed on the rig, so the copy-back scp no-ops; the local
+ spec from a previous FAILED run must not survive it looking current -- a later
+ "retry from the spec" would re-flash boards that already passed."""
+ r = self._run(['-b', 'alpha'], built=['alpha'],
+ stale={'cfg.json.failed': '--accumulate -b alpha'})
+ self.assertEqual(r.returncode, 0, r.stderr)
+ self.assertFalse(r.stale_left['cfg.json.failed'],
+ "last run's re-run spec survived a green run")
+
+
+class SummaryFoldsReportToBoards(unittest.TestCase):
+ """hil_summary.py replaces the agent retyping the markdown table. Report rows are named per
+ VARIANT and a variant need not start with the board name, so the config is what maps them
+ back -- the previous string-matching design produced a defect in each of four review rounds."""
+
+ def _sum(self, boards, rows, cfg_boards=None, banner=''):
+ import json
+ import subprocess
+ td = TemporaryDirectory()
+ self.addCleanup(td.cleanup)
+ d = Path(td.name)
+ (d / 'hil_report.json').write_text(json.dumps(
+ {'rows': [{'board': b, 'cells': c, 'duration': '1s'} for b, c in rows],
+ 'banner': banner}))
+ cfg = d / 'cfg.json'
+ cfg.write_text(json.dumps({'boards': cfg_boards or [{'name': b} for b in boards]}))
+ args = [a for b in boards for a in ('-b', b)]
+ r = subprocess.run(['python3', str(Path(TEST_DIR).parents[0] / 'helper' / 'hil_summary.py'),
+ str(cfg), *args, '--report-dir', str(d)],
+ capture_output=True, text=True, timeout=60)
+ self.assertEqual(r.returncode, 0, r.stderr)
+ return json.loads(r.stdout)['results']
+
+ def test_variant_rows_fold_onto_their_board(self):
+ """nanoch32v203 never produces a row named after the board."""
+ got = self._sum(['nanoch32v203'],
+ [('nanoch32v203-fsdev', {'usbtest': 'pass'}),
+ ('nanoch32v203-usbfs', {'usbtest': 'pass'})],
+ cfg_boards=[{'name': 'nanoch32v203',
+ 'variant': [{'name': 'nanoch32v203-fsdev'},
+ {'name': 'nanoch32v203-usbfs'}]}])
+ self.assertEqual([r['board'] for r in got], ['nanoch32v203'])
+ self.assertTrue(got[0]['pass'])
+ self.assertTrue(got[0]['ran'])
+
+ def test_one_failing_variant_fails_the_board(self):
+ got = self._sum(['nano'],
+ [('nano-a', {'usbtest': 'pass'}), ('nano-b', {'usbtest': '❌ 29/30'})],
+ cfg_boards=[{'name': 'nano', 'variant': [{'name': 'nano-a'},
+ {'name': 'nano-b'}]}])
+ self.assertFalse(got[0]['pass'])
+ self.assertIn('29/30', got[0]['detail'])
+
+ def test_lock_contention_is_a_field_not_a_prefix(self):
+ got = self._sum(['alpha'], [('alpha', {'board-locked': 'fail'})])
+ self.assertTrue(got[0]['locked'])
+ self.assertFalse(got[0]['pass'])
+
+ def test_a_board_with_no_row_is_marked_not_run(self):
+ got = self._sum(['alpha', 'beta'], [('alpha', {'usbtest': 'pass'})])
+ self.assertTrue(got[0]['ran'])
+ self.assertFalse(got[1]['ran'])
+ self.assertFalse(got[1]['pass'])
+
+ def test_a_metric_cell_counts_by_its_icon(self):
+ got = self._sum(['a', 'b'], [('a', {'cdc_msc_throughput': '✅ C 1.2 M 3.4'}),
+ ('b', {'cdc_msc_throughput': '❌ C 0.0 M 0.0'})])
+ self.assertTrue(got[0]['pass'])
+ self.assertFalse(got[1]['pass'])
+
+ def test_skipped_cells_do_not_fail_a_board(self):
+ got = self._sum(['a'], [('a', {'usbtest': 'skip', 'cdc_msc': 'pass'})])
+ self.assertTrue(got[0]['pass'])
+
+ def test_a_plain_metric_cell_is_a_pass(self):
+ """Mirrors hil_test.py's own tally (cell_kind): failures are ALWAYS marked -- 'fail'
+ or a ❌ prefix, per TestFail's docstring -- while a passing test may return a plain
+ metric string that lands in the cell unprefixed. Treating unknown shapes as fail
+ would publish a green table as a red verdict."""
+ got = self._sum(['a'], [('a', {'device_speed': '480.0 MBps'})])
+ self.assertTrue(got[0]['pass'])
+
+ def test_a_declared_variant_of_another_board_is_not_stolen(self):
+ """A declared variant need not start with its own board's name, so it may start with
+ a DIFFERENT board's name plus '-'. The prefix fallback must not attribute it twice."""
+ got = self._sum(['alpha', 'beta'],
+ [('beta-x', {'usbtest': 'fail'})],
+ cfg_boards=[{'name': 'alpha', 'variant': [{'name': 'beta-x'}]},
+ {'name': 'beta'}])
+ self.assertTrue(got[0]['ran'])
+ self.assertFalse(got[0]['pass'])
+ self.assertFalse(got[1]['ran'], "beta must not inherit alpha's row")
+
+
class CaveatSurvivesAccumulate(unittest.TestCase):
"""CI reruns with --accumulate: the sidecar keeps every earlier attempt's cells, but the
banner was recomputed per attempt. A first attempt on a degraded rig and a clean rerun