diff options
89 files changed, 9304 insertions, 3124 deletions
diff --git a/.claude/agents/builder.md b/.claude/agents/builder.md index 4edb7e0d4..3f06f328a 100644 --- a/.claude/agents/builder.md +++ b/.claude/agents/builder.md @@ -3,6 +3,7 @@ name: builder description: Build TinyUSB examples for one board and report structured pass/fail with first-error triage. Use for build sweeps and post-change build verification. Never edits source. tools: Bash, Read, Grep, Glob model: haiku +effort: low --- You build TinyUSB examples for exactly one board per run and report the result as machine-readable JSON. You never modify source files. diff --git a/.claude/agents/driver-reviewer.md b/.claude/agents/code-verifier.md index 9ce8b621f..7e529a641 100644 --- a/.claude/agents/driver-reviewer.md +++ b/.claude/agents/code-verifier.md @@ -1,8 +1,9 @@ --- -name: driver-reviewer +name: code-verifier description: Review one TinyUSB driver directory or one diff against one review dimension (correctness, ISR safety, datasheet/errata conformance, style) with coverage-first structured findings; or adversarially verify a single finding / fix. Read-only. tools: Bash, Read, Grep, Glob, Skill model: opus +effort: xhigh --- You review exactly the scope given in your prompt (one driver directory, or one git diff) for exactly the dimension(s) given. Read the code yourself; follow callers, headers, and macros as far as needed to judge correctly. You never modify files. diff --git a/.claude/agents/port-dev.md b/.claude/agents/code-writer.md index 77a28bafa..82e52e334 100644 --- a/.claude/agents/port-dev.md +++ b/.claude/agents/code-writer.md @@ -1,7 +1,8 @@ --- -name: port-dev +name: code-writer description: Implement one well-scoped change in one TinyUSB port or explicit file set, following repo style and .clang-format, verified by a targeted build. Use for fan-out development across ports and for fixing validated PR findings. model: opus +effort: xhigh --- You implement exactly one specified change in one assigned scope (a directory under `src/portable/`, a class driver, or an explicitly listed file set). Never touch files outside the assigned scope. diff --git a/.claude/agents/hil-operator.md b/.claude/agents/hil-operator.md index a37501211..d128f6f53 100644 --- a/.claude/agents/hil-operator.md +++ b/.claude/agents/hil-operator.md @@ -3,6 +3,7 @@ name: hil-operator description: Run TinyUSB hardware-in-the-loop actions on the physical test rig — per-board locking, firmware flash, hil_test.py runs, USB recovery. Strictly one instance at a time. Never edits source; never touches the actions-runner service. tools: Bash, Read, Grep, Glob model: sonnet +effort: high --- You operate physical USB test hardware. These repo skills are your source of truth — read the relevant one BEFORE acting: @@ -68,12 +69,12 @@ For a board run, do NOT transcribe the report table. Run the tests, then hand ba output verbatim: ```bash -python3 test/hil/helper/hil_summary.py <config> -b BOARD [-b BOARD...] # from the report dir +python3 test/hil/helper/hil_report.py <config> -b BOARD [-b BOARD...] # from the report dir ``` -`{"results": <its results array, verbatim>, "banner": <its banner, verbatim>, "wedged": ["board", ...]}` +`{"results": <its results array, verbatim>, "banner": <its banner, verbatim>, "caveat": <its caveat, verbatim>, "wedged": ["board", ...]}` -`results` and `banner` are copied, never retyped, reworded or re-ordered: report rows are named +`results`, `banner` and `caveat` are copied, never retyped, reworded or re-ordered (`caveat` is the run-level notice — abandoned, aborted, no-boards — and it can say the run failed while every row says pass): report rows are named per variant, a variant name need not start with the board name, and lock contention is a cell rather than a phrase, so re-deriving any of it by hand is how this contract broke before. `wedged` is yours — the boards your run left unresponsive, usually none — and the only field you diff --git a/.claude/agents/pr-ci-watcher.md b/.claude/agents/pr-ci-watcher.md new file mode 100644 index 000000000..10a32084b --- /dev/null +++ b/.claude/agents/pr-ci-watcher.md @@ -0,0 +1,26 @@ +--- +name: pr-ci-watcher +description: Watch one TinyUSB PR's CI — classify failures (infra flake / real / rig-side), re-run infra ones, report real ones with first error and files. CI only; never reads review comments, never edits code, never pushes. +tools: Bash, Read, Grep, Glob +model: sonnet +effort: high +--- + +You watch CI for exactly one PR (number given in your prompt) using `gh`. You never modify source files, never commit, never push, never read review comments. + +## Procedure + +1. `gh pr checks <N>`. If checks are running and your prompt says to wait, run `gh pr checks <N> --watch` as a BACKGROUND Bash task (the foreground timeout is capped at 10 min). +2. For each failing check, find its run and read the failure: `gh run view <run-id> --log-failed | head -150`. +3. Classify each failure: + - **infra/flake**: runner lost communication, network/DNS timeouts, artifact 404, docker pull/rate-limit errors, cancelled-by-timeout with no test output. Re-run once (`gh run rerun <run-id> --failed`); record run ids in `infraRerun`. + - **real**: compile/link errors, test assertions, HIL failures with device output. Extract the FIRST error line and the source files involved. + - **rigSide=true** on a real failure NOT attributable to the PR: probe/fixture faults, byte-identical reproduction on unrelated PRs, boards outside the diff. These are reported for humans, never handed to a fixer. + +## Output contract + +Your final message is parsed by a program. Return ONLY this JSON — no prose, no code fences: + +{"status": "green", "infraRerun": [], "realFailures": [{"check": "...", "firstError": "...", "files": ["..."], "rigSide": false}]} + +status: "green" (all pass), "red" (any real failure), "running" (still pending after your wait budget). diff --git a/.claude/agents/pr-monitor.md b/.claude/agents/pr-monitor.md deleted file mode 100644 index 77777b0fb..000000000 --- a/.claude/agents/pr-monitor.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -name: pr-monitor -description: Triage one TinyUSB GitHub PR — CI status + failure classification, infra re-runs, bot review harvesting (Codex/Copilot/Claude) with adversarial validation of each finding against the code. Read/triage/re-run only; never edits code, never pushes. -tools: Bash, Read, Grep, Glob -model: sonnet ---- - -You triage exactly one PR (number given in your prompt) using `gh`. You never modify source files, never commit, never push. - -## CI triage - -1. `gh pr checks <N>`. If checks are running and your prompt says to wait, run `gh pr checks <N> --watch` as a BACKGROUND Bash task (the foreground timeout is capped at 10 min). -2. For each failing check, find its run and read the failure: `gh run view <run-id> --log-failed | head -150`. -3. Classify each failure: - - **infra/flake**: runner lost communication, network/DNS timeouts, artifact 404, docker pull/rate-limit errors, cancelled-by-timeout with no test output. - - **real**: compile/link errors, test assertions, HIL failures with device output. -4. Re-run infra failures once: `gh run rerun <run-id> --failed`; record run ids in `infraRerun`. -5. For real failures extract the FIRST error line and the source files involved (from the log paths). - -## Bot review harvest - -- Inline review comments: `gh api repos/{owner}/{repo}/pulls/<N>/comments --paginate` (use `gh repo view --json nameWithOwner -q .nameWithOwner` for owner/repo). Issue comments: `gh pr view <N> --comments`. -- Known signals: Codex posts an issue comment when done — "Didn't find any major issues" means clean, not silence. Copilot is finished when it no longer appears in `requested_reviewers`. Bot logins differ across REST/GraphQL — match authors case-insensitively on substrings `codex`, `copilot`, `claude`. -- For EACH unresolved bot finding: open the file at the cited line in the current checkout and judge the claim adversarially. `valid` only if the code truly has the problem; `invalid` with a concrete refutation otherwise; `stale` if the current code already fixed it. -- Draft a courteous, technical reply for every `invalid`/`stale` finding (cite the code that refutes it). Put them in `replies` with the comment id — a later step posts the reply AND marks the inline thread resolved (via the GraphQL `resolveReviewThread` mutation); you do not post or resolve. The `commentId` must be the inline review comment's integer databaseId so the thread can be found. - -## done - -`done` = true only when CI is green (all checks pass, nothing running) AND no unresolved `valid` findings remain. - -## Output contract - -Your final message is parsed by a program. Return ONLY this JSON — no prose, no code fences: - -{"ci": {"status": "green", "infraRerun": [], "realFailures": [{"check": "...", "firstError": "...", "files": ["..."]}]}, - "findings": [{"source": "codex", "commentId": 123, "file": "...", "line": 1, "claim": "...", "verdict": "valid", "reason": "...", "fixHint": "..."}], - "replies": [{"commentId": 123, "body": "..."}], - "done": false} diff --git a/.claude/agents/pr-review-validator.md b/.claude/agents/pr-review-validator.md new file mode 100644 index 000000000..4069a856f --- /dev/null +++ b/.claude/agents/pr-review-validator.md @@ -0,0 +1,30 @@ +--- +name: pr-review-validator +description: Harvest one TinyUSB PR's bot reviews (Codex/Copilot/Claude) and adversarially validate each finding against the code — verdict valid/invalid/stale, draft replies for refuted ones. Read-only; never edits code, never posts, never pushes. +tools: Bash, Read, Grep, Glob +model: opus +effort: xhigh +--- + +You validate the bot review findings on exactly one PR (number given in your prompt) using `gh`. You never modify source files, never commit, never push, never post comments. Do not triage or classify CI failures or logs — pr-ci-watcher owns that; you may read the review bots' own check runs to see whether they concluded. + +## Procedure + +- Inline review comments: `gh api repos/{owner}/{repo}/pulls/<N>/comments --paginate` (use `gh repo view --json nameWithOwner -q .nameWithOwner` for owner/repo). Issue comments: `gh api repos/{owner}/{repo}/issues/<N>/comments --paginate` — this returns each comment's integer `id`, which `gh pr view --comments` does not print and the output contract needs. PR reviews (the Copilot/Claude verdict bodies): `gh api repos/{owner}/{repo}/pulls/<N>/reviews --paginate` — compare each review's `commit_id` to the head SHA from `gh pr view <N> --json headRefOid -q .headRefOid` to tell a review of the current push from an older one. +- Known signals: Codex posts an issue comment when done — "Didn't find any major issues" means clean, not silence. It can also signal a clean pass with no comment at all: a 👍 (`+1`) reaction on the PR description (`gh api "repos/{owner}/{repo}/issues/<N>/reactions?content=%2B1&per_page=100" --paginate`, author matching `codex`; without `--paginate` a fresh reaction can fall off the first page and Codex looks pending forever) — settled when the reaction's `created_at` postdates the head push time defined below. Its body carries a `**Reviewed commit:** <short sha>` line: Codex is settled only when that short SHA prefix-matches the head SHA, otherwise the comment is a verdict for an older push and Codex is still pending. Its "Something went wrong" comment has no Reviewed-commit line, so correlate that one by time instead — against the moment the SHA *became* the head, `gh api repos/{owner}/{repo}/commits/<headSha>/check-suites --jq '[.check_suites[].created_at] | min'` (the suites are created when the push lands; fall back to `gh api repos/{owner}/{repo}/commits/<headSha> --jq .commit.committer.date` only if the SHA has no check suites). The committer date alone is when the commit was written, which can precede the push by hours and make a leftover error comment look fresh. An error/quota comment settles Codex only when its `created_at` postdates that push time, or when it arrives as a PR review whose `commit_id` is the head SHA. An older one is a leftover from an earlier push — Codex is still pending. Copilot submits a PR review whose body opens with a verdict header (`### 🟢 Approval recommended` / `### 🟡 Changes recommended`) and leaves `requested_reviewers` once submitted. The Claude bot posts a PR review, or its `claude-review` check run for the head SHA reaches `status: completed` — ask for that check by name, `gh api "repos/{owner}/{repo}/commits/<headSha>/check-runs?check_name=claude-review"`, since the unfiltered listing is paginated and drops it on a PR with more than a page of checks. A bot reporting a usage/quota limit counts as settled once that report postdates the head push time above (the check-suite timestamp, not the committer date) — do not wait on it. Bot logins differ across REST/GraphQL — match authors case-insensitively on substrings `codex`, `copilot`, `claude`. +- For EACH unresolved bot finding: open the file at the cited line in the current checkout and judge the claim adversarially. `valid` only if the code truly has the problem; `invalid` with a concrete refutation otherwise; `stale` if the current code already fixed it. +- Draft a courteous, technical reply for every `invalid`/`stale` finding (cite the code that refutes it). Put them in `replies` with the comment id — a later step posts the reply AND resolves the thread; you do not. For a finding from an inline thread, `commentId` is the inline review comment's integer databaseId (that is how the thread is located and resolved); for one that exists only in an issue comment, use that issue comment's id — the poster falls back to a plain PR comment and skips resolving. + +## Output contract + +Your final message is parsed by a program. Return ONLY this JSON — no prose, no code fences: + +{"findings": [{"source": "codex", "commentId": 123, "file": "...", "line": 1, "claim": "...", "verdict": "valid", "reason": "...", "fixHint": "..."}], + "replies": [{"commentId": 123, "body": "..."}], + "done": false} + +done = true only when no unresolved `valid` findings remain AND every auto-reviewer +has settled for the current head SHA: its verdict is posted (Copilot review header, +Codex verdict comment, Claude review or concluded check) or it reported hitting a +usage/quota limit. A reviewer that has not reported since the last push is pending — +return done = false so the caller re-checks next cycle. diff --git a/.claude/agents/static-analyzer.md b/.claude/agents/static-analyzer.md index 0f0b2a6e1..e7da82192 100644 --- a/.claude/agents/static-analyzer.md +++ b/.claude/agents/static-analyzer.md @@ -3,6 +3,7 @@ name: static-analyzer description: Run PVS-Studio static analysis (SAST + MISRA C:2023/C++:2008) on TinyUSB for one board and report structured findings, gated on diagnostics in files changed vs a base ref. Read-only; never edits source. tools: Bash, Read, Grep, Glob model: sonnet +effort: medium --- You run PVS-Studio over the TinyUSB examples build for exactly one board per run and report machine-readable findings. You never modify source files. diff --git a/.claude/agents/target-debugger.md b/.claude/agents/target-debugger.md index 1b6931307..c7acca91c 100644 --- a/.claude/agents/target-debugger.md +++ b/.claude/agents/target-debugger.md @@ -2,6 +2,7 @@ name: target-debugger description: Root-cause one USB misbehavior on real HIL hardware by instrumenting the TinyUSB target — device or host stack — with TU_LOG/RTT, RAM ring-buffer trace, GDB autopsy, J-Link PC-sampling, correlated with capture from the link's other end (Linux PC host, another TinyUSB board, or a Linux gadget peer) and the wire. Long serial debug loop under one held board lock; strictly one instance. Produces a diagnosis with on-target evidence (plus a candidate fix when one emerges), never a merged patch. model: opus +effort: xhigh --- You debug one failing USB behavior on one physical board until you can name the diff --git a/.claude/skills/etm-trace/boards.md b/.claude/skills/etm-trace/boards.md index 044d4e0ee..f54e1d7d6 100644 --- a/.claude/skills/etm-trace/boards.md +++ b/.claude/skills/etm-trace/boards.md @@ -26,7 +26,7 @@ reference. | mimxrt1170_evkb | 996 MHz | 50 MHz (root/2) | 1 | 0 | weld 0 Ω R1881-R1886; JP4 shorted; J58 (populated) | re-weld R1884 (D3 open; D1/D2 meter-verified good) → width 4 | | ra6m5_ek (M33) | 200 MHz | 25 MHz (TRCLK/4 /2) | 4 | 0 (unset) | J9 closed; native J20 trace | — | | ra8m1_ek (M85) | 480 MHz | 60 MHz (TRCLK/4 /2) | 4 | 0 (unset) | J9 closed + Table 7 jumpers | — | -| raspberry_pi_pico2 (RP2350 M33) | 48 MHz | 24 MHz (clk_sys/2) | 4 | 0 (unset) | fly-wire GPIO1-5 → MIPI20 (map in jdebug) | 72-80 MHz per seating (re-qualify); >80 needs V3 probe + trace board | +| pico2_etm_trace (RP2350 M33) | 150 MHz | 75 MHz (clk_sys/2) | 4 | +1 ns | Pico 2 on the trace-carrier PCB (MIPI-20) | — | | same54_xplained (E54 M4F) | 120 MHz | 60 MHz (CPU/2) | 4 | 0 (unset) | none — populated 20-pin ETM header | — | | same70_xplained (E70 M7) | 300 MHz | 37.5 MHz (PCK3/2) | 1 | 0 (unset) | solder 20-pin header on J403 (bottom) | width 4 blocked: D1 (J403.16) dead at speed — probe-channel crosscheck pending | | SEGGER H7/F407 ref | demo defaults | demo | 4 | demo | probe-powered: add `--power` | — | @@ -105,42 +105,68 @@ Board caveats (beyond the table): the decoder at t≈0.05 s every run. Runs both chip maxima (120 MHz TRCLK, 60 MHz pin) clean. `ReadIntoTraceCache 0x0 0x10000` in the download hook covers runtime chip-ROM execution. ISR entry: `tusb_int_handler`. -- **raspberry_pi_pico2** (RP2350): TRACECLK is a fixed clk_sys/2, no divider - (DDR data, like every ARM TPIU pin port). **Measured cliff on this rig:** - 80 MHz core (40 MHz TRACECLK) traces idle code but dies under dense data; - 88 MHz+ dies instantly at any width/global-timing/TIF/pad setting. Cause - not pinned down: the same V2 probe samples 66 MHz TRACECLK (132 Msample/s) - on metro_m7_1011, so it is NOT a plain probe sample-rate ceiling. The - cliff at >40 MHz TRACECLK (84+ MHz core) survived a full sweep - global - AND per-pin `--trace-timing`, pad drive 2/4/8/12 mA + slew, width 4/2/1, - TIF 1-25 MHz, newer J-Link library - all flat, so it is V3-probe / real- - trace-board territory (SEGGER's Pico 2 KB requires J-Trace PRO **V3.0+** - and recommends a proper trace board; community reports fly-wires fail at - 75 MHz for everyone, PCBs work). Separately, fly-wire seating quality - sets the width-4 DENSE-data ceiling (48-72 MHz observed across seatings): - after ANY rewiring re-qualify with idle blinky at the target clock, then - cdc_msc x3. Random unknown-packet deaths KB into a clean stream = one - marginal wire; `--trace-width` 1 vs 2 vs 4 bisects which (width 1 = - CLK+D0 only; D1 = GPIO3->MIPI20 pin 16 has gone marginal twice on this - rig). Width-1 is a full-quality fallback: complete cdc_msc profiles at up - to 80 MHz core even when width 4 is broken. - **Never set a custom JLinkScript** — it - replaces J-Link's built-in RP2350 device script, which both declares the - trace component map (funnel/TPIU/ETM are not in the ROM table → "Required - trace components for pin trace not found", 0 fetches) and re-arms the whole - chip-side path via `OnTraceStart` at every resume. Firmware therefore does - no trace setup; TRACE_ETM builds only (a) pin clk_sys to 48 MHz from crt0 - (board.cmake) — the fly-wire ceiling: 96/150 MHz kill the stream in the - startup burst at any sample timing (and at 150 MHz the saturated probe - stops answering halts, "CPU could not be halted"); any post-arm clock - change steps TRACECLK mid-stream and kills the decoder — and (b) - clear TIMER0/1 DBGPAUSE (family.c): debug sessions leave cores - halted-at-reset and the default DBGPAUSE freezes the µs timer, so every - `sleep_ms()` spins forever (looks like a dead board; watchdog-scratch - breadcrumbs survive warm resets but not POR when diagnosing). UART console - is TX-only (GPIO1 = TRACECLK). Empty reset/download hooks: the bootrom - must run the IMAGE_DEF. If the chip ends up wedged/un-attachable: - J-Link `erase` + reset drops it into BOOTSEL (2e8a:000f) for picotool. +- **pico2_etm_trace** (Pico 2 / RP2350 on the carrier; board `raspberry_pi_pico2` + is the bare module and has no trace wiring): rig = **pico2 trace motherboard PCB** + (~/code/pcb/pico2_trace_motherboard: MIPI-20, 27 Ohm source-terminated, + GND-guarded). TRACECLK is a fixed clk_sys/2 (DDR), so the board traces at + the rp2350 pico-sdk default 150 MHz -> 75 MHz TRACECLK width 4, validated + 2026-08-26: cdc_msc enumeration burst 3/3, zero overflow, **data sampling + +1 ns** (committed in the reference; idle eye -1000..+2000 ps, +3000 dead; + TD aliases modulo the 6.67 ns UI); soak: cdc_msc_throughput under a live + host CDC+MSC bulk pump, 3/3 x 15 s, zero overflow, 53.7M fetches (DCD hot + path at 9% load). `TRACE_ETM` is set by the board's own board.cmake - no + build flag needed. + **Other rates need a hand-built clock**: pass `SYS_CLK_KHZ` *together with* + `PLL_SYS_VCO_FREQ_HZ`/`POSTDIV1`/`POSTDIV2` from the SDK's + `scripts/vcocalc.py` as compile definitions (a bare `-DSYS_CLK_KHZ=` only + sets a CMake cache var and is silently ignored - the BSP no longer carries + a PLL table). Measured: 180000 = 90 MHz TRACECLK, loaded eye + +4000..+5000 ps (3/3); 240000 = **the J-Trace PRO V2 ceiling** (120 MHz + TRACECLK, TD +3500) — **⚠ 240 MHz was measured with the core regulator + raised to 1.15 V, which nothing does automatically any more: add + `SYS_CLK_VREG_VOLTAGE_AUTO_ADJUST=1` and + `SYS_CLK_VREG_VOLTAGE_MIN=VREG_VOLTAGE_1_15` yourself, or the chip runs + 60% over its 150 MHz rating at stock 1.10 V.** >=125 MHz + TRACECLK is a hard probe wall at every sample delay/width (the V2 AT its + documented limit: Arm spec 100 MHz in-spec, SEGGER's tuned-V2 best is + 120; the 150 MHz on current product pages is V3/V4). **Firmware needs + almost no trace code**: J-Link's built-in RP2350 device script declares + the off-ROM-table trace components (funnel/TPIU/ETM) and re-arms the + whole chip-side path via OnTraceStart at every resume — **never set a + custom JLinkScript** (it replaces the built-in script: "Required trace + components for pin trace not found", 0 fetches). What TRACE_ETM (set by + this board's board.cmake) does in firmware: (a) clears TIMER0/1 DBGPAUSE + — J-Link does NOT clear it, and with the reset default the us-timer + freezes while a core is debug-halted, so sleep_ms() spins forever after + any debugger session (measured: DBGPAUSE reads 0x7 and TIMERAWL stands + still until the clear); (b) compile-time pin-conflict checks — #error if + the UART console lands on a trace pin GP1-5, #pragma message if the + default I2C does. The console itself is full-duplex on GP12/13 (the + carrier routes it off GP0/1; the old TX-only fallback is gone with the + fly-wire rig). PCB A/B validation did remove the 12 mA fast-slew trace + pads (default pads pass 3/3 with a wider idle eye) — do not re-add + without fresh PCB evidence. A runtime clk_sys switch **silently + truncates the capture at the switch** (no decoder error — profile just + ends; verified 3/3 with a board_init-time 120->156 step), so nothing may + re-switch the clock at runtime. + **This is the only trace-capable board in the rp2040 family** - it owns the + sole ozone reference, so `--board <any other rp2040/rp2350 board>` exits + with "cannot resolve J-Link device" (the script's board.cmake fallback + cannot help: this family sets `JLINK_DEVICE` in family.cmake). Capture with + `--board pico2_etm_trace`. + **Arm-phase flake**: an occasional instant unknown-packet death at + offset ~0x10-0x6C right at trace start — just re-run; only mid-stream + deaths indicate a real problem. **Loose MIPI-20 cable symptom ladder**: + flash "Failed to perform RAMCode-sided Prepare()" / "Download failed" + first, then "Target voltage too low" (VTref lost) — reseat the cable at + both ends before debugging software. Empty reset/download hooks in the + reference: the bootrom must run the IMAGE_DEF (setting SP/PC from the + vector table bypasses it and the pico-sdk runtime never comes up). If + the chip ends up wedged/un-attachable: J-Link `erase` + reset drops it + into BOOTSEL (2e8a:000f) for picotool. *Historical*: bring-up used a + fly-wire rig (same GPIO1-5 -> MIPI20 map) whose wire SI capped TRACECLK + at 24-40 MHz and motivated the removed workarounds; it is retired — + details in git history (the 48/80 MHz PLL rows left with it). - **same54_xplained**: the CM4 trace unit is clocked from **GCLK channel 47 (GCLK_CM4_TRACE)** — with it disabled the pins mux fine, TPIU/ETM arm fine, and the port stays perfectly silent (zero fetches, no errors); diff --git a/.claude/skills/hil/SKILL.md b/.claude/skills/hil/SKILL.md index d1e4bdcd5..03a462ce6 100644 --- a/.claude/skills/hil/SKILL.md +++ b/.claude/skills/hil/SKILL.md @@ -65,8 +65,8 @@ hooks: `test_ci_select.py` covers only selection, `test_ci_metrics.py` only the plumbing. The containment work --- bounded reads, the kill ladders, the build and pool guards --- lives in `test_hil_bounded.py`, `test_hil_health.py` and `test_hil_util.py`, so run all five when changing `test/hil`: -`for f in test/hil/test/test_*.py; do python3 "$f"; done` (~84s, of which -`test_hil_bounded.py` is ~76s of deliberate hang/timeout simulation; the two `test_ci_*` +`for f in test/hil/test/test_*.py; do python3 "$f"; done` (~48s, of which +`test_hil_bounded.py` is ~30s of deliberate hang/timeout simulation; the two `test_ci_*` suites are ~4s together). ## Pre-flight rig health check @@ -82,6 +82,8 @@ See the `usb-kernel-recover` skill for what a real wedge looks like and how to c Examples must be built for the target board(s) — see CLAUDE.md "Build" → "All examples for a board" (produces `examples/cmake-build-<board>/`). `-B examples` points `hil_test.py` at that parent folder. (This applies to `hil_test.py`; `hil_pool_check.py` builds its own missing firmware.) +A board whose flasher probe has no VCOM (or whose BSP has no UART) uses RTT as its console — "No serial device found for /dev/serial/by-id/…" on every host test is the symptom. Config: `"logger": "rtt"` (jlink flashers only) plus a self-named variant carrying the define — `"variant": [{"name": "<board>", "defines": ["LOGGER=rtt"]}]` — and prebuilt example sets must carry the same `-DLOGGER=rtt`. Caveat: the cdc/msc-fixture host tests don't speak RTT yet, so such a board cannot carry `is_cdc`/`is_msc` fixtures (the config loader rejects it; see the rtt follow-up doc). Details: the `rtt` skill. + ## Arguments - **Board:** `-b BOARD_NAME`, repeatable for a subset (`-b a -b b`); omit to run all boards in the config. Give a whole set to ONE run rather than one run per board: it schedules the boards across host controllers and budgets concurrent flashes and usbtest batteries per controller (`hil_lock.py` `FLASH_PARALLEL`/`USBTEST_PARALLEL`). Those permits are in-process semaphores — a second `hil_test.py` running alongside does not share them, it multiplies the load on the same xHCI cards. @@ -135,8 +137,8 @@ The user-facing answer to a HIL run IS the tool's summary table: paste the compl table (and footer counts) verbatim — never truncate rows or reduce it to a prose digest; at most one line of commentary below it. -**First check what sits above the table.** Seven banners can appear there; match on a -PREFIX, since each carries trailing detail and one is a blockquote: +**First check what sits above the table.** Six banners can appear there; match on a +PREFIX, since each carries trailing detail and two are blockquotes: - `**HIL run abandoned: worker pool timed out after …s.**` — no results were collected this attempt, so any table below is a PREVIOUS attempt's. Report the abandonment, never those @@ -157,9 +159,6 @@ PREFIX, since each carries trailing detail and one is a blockquote: - `> **Rig dirty.**` — a process survived SIGKILL and still holds a probe or usbfs node into the NEXT job. The table below is this run's and can be reported, but say the rig is dirty: the next job starts degraded and nothing in the harness can clear it. -- `> **Not all verdicts are evidence.**` — one or more workers went blind on sysfs, so - "device not found" from the named boards means "could not tell". Do NOT report their red - cells as broken boards. On failure, retry once with `-v` — from the `<config>.failed` spec the run just wrote, which already begins with `--accumulate` and restricts each board to its failed tests. A hand-scoped diff --git a/.claude/skills/rtt/SKILL.md b/.claude/skills/rtt/SKILL.md new file mode 100644 index 000000000..5e14ae84c --- /dev/null +++ b/.claude/skills/rtt/SKILL.md @@ -0,0 +1,201 @@ +--- +name: rtt +description: Use when you need console or printf I/O, TU_LOG capture, or a raw byte channel over a debug probe on real hardware — the board has no UART wired or its probe no VCOM, a LOGGER=rtt build needs reading or writing, "RTT Control Block not found", an RTT server won't come up or drops output, JLinkRTTLogger/JLinkRTTClient/JLinkGDBServer/openocd rtt misbehave, or another workflow (HIL console, SystemView capture) needs RTT stood up on a J-Link, ST-Link, CMSIS-DAP or WCH-Link probe. +--- + +# rtt — SEGGER RTT transport and console + +RTT is nothing but RAM: a control block `_SEGGER_RTT` (starts with the magic +string `"SEGGER RTT"`) plus per-channel ring buffers +`{sName, pBuffer, SizeOfBuffer, WrOff, RdOff, Flags}`. The target advances +`WrOff`; the host must **write `RdOff` back** to free space — a reader that +only reads never drains the ring. Channel 0 is the "Terminal" console; +SystemView claims its own `"SysView"` up-buffer on the same control block — +they coexist. The debug probe reads/writes this RAM while the core runs, so +everything here is zero-wiring: no UART, no VCOM. + +Scope: byte transport and console. Timing/profiling → `etm-trace`/`sysview`; +debugging decision flows and the wedged-target drain model → `target-debug`; +Espressif consoles → `esp-target-debug` (USB-Serial-JTAG, no SEGGER RTT). + +## Quick start — console on a J-Link probe + +Use the skill's tool `tools/rtt.py` for every route; do not hand-roll +JLinkExe/JLinkGDBServer/openocd/telnet pipelines (`--help` for all modes): + +```bash +# firmware: TU_LOG + stdio → RTT channel 0 (hw/bsp/board.c routes sys_read too) +cmake -DBOARD=<board> -DLOG=2 -DLOGGER=rtt ... # Make: LOG=2 LOGGER=rtt + +# flash + reset FIRST (the console owns the probe once open), then: +python3 tools/rtt.py --backend jlink --probe <serial> --device <JLINK_DEVICE> --seconds 20 +# -i forwards stdin to the target; --seconds 0 streams until Ctrl-C/EOF +``` + +`JLINK_DEVICE` comes from `hw/bsp/<family>/boards/<board>/board.cmake` (or +`family.cmake`). Always pass the probe serial — rigs and benches run several +probes, and the `ninja <example>-jlink` flash target grabs whichever J-Link +enumerates first: pin it (`-DJLINK_OPTION="-USB <serial>"`) or flash with +`JLinkExe -SelectEmuBySN`. The HIL harness uses the same implementation +(`hil_util.JlinkRtt`) via a board's `"logger": "rtt"` (jlink flashers +only) plus a single self-named variant carrying the define — +`"variant": [{"name": "<board>", "defines": ["LOGGER=rtt"]}]`, the roster's +one shape for always-on defines — variant defines feed `hil_test.py +--build` and the CI matrix; a prebuilt `cmake-build-<board>` set must be +configured with the same `-DLOGGER=rtt` itself. Keep harness console builds +quiet (`LOGGER=rtt` WITHOUT `LOG=2`): reset-then-attach only preserves what +fits the up-buffer (stock 1 KB, NO_BLOCK_SKIP), and a chatty boot burst +truncates at the ring boundary before the drain attaches — measured +1022-1023 B captures on ea4088 with `LOG=2`, enumeration lines falling off +the end. `BUFFER_SIZE_UP` is the knob when verbose logs are really needed. +Rig boards need `hil_lock.py` held first — see the `hil` skill. + +To validate bidirectionality end-to-end you need firmware that both polls +the console AND replies via printf. `board_test` polls `board_getchar()` +(RTT-aware via `sys_read`) but echoes through `board_putchar` → +`board_uart_write`, which is NOT LOGGER-aware — on a UART-less board the +echo hits the `-1` stub and vanishes (measured on ea4088). For a validation +run, patch its echo to `printf` locally, or drive a host example's menu +(`msc_file_explorer`, `cdc_msc_hid` — they reply via printf). Sending +keystrokes to `cdc_msc` and expecting an echo proves nothing: it never polls +the console. + +## Transport matrix + +| Transport / tool | Live read | Write | Notes | +| ----------------------------------------------- | ---------- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| ARM memory-AP (any J-Link/ST-Link/CMSIS-DAP) | yes | yes | zero intrusion; core keeps running | +| RISC-V SBA (where implemented) | yes | yes | autonomous like memory-AP | +| WCH QingKe SDI | **NO** | no | DM abstract-command reads perturb the running core: A/B-proven firmware kill ~1.9 s into USB traffic. Halt→read→resume or post-mortem dump ONLY | +| OpenOCD/jaylink on a genuine SEGGER J-Link | yes | untested | routine in the sysview campaigns (metro_m4_express, dozens of attaches, zero wedges); prefer SEGGER tools where both exist (drain rate) | +| OpenOCD/jaylink on the LPC-Link2 (J-Link OB fw) | forbidden | — | measured on ea4088's LPC-Link2 (2023 OB image): transport fails (`jaylink_swd_io`) and knocks the probe off USB; physical replug to recover — SEGGER tools only THERE. Verdict is for that probe only: other J-Link-OB firmware probes are untested — hardware-test before assuming either way | +| `JLinkRTTLogger` | unreliable | — | searches for the control block once at attach and gives up — on some parts it never finds it ("RTT Control Block not found" even with `-RTTAddress`; measured 0/6 on LPC4088). May work elsewhere, but don't build automation on a single-search tool | + +Validated boards, directions and per-board caveats: [boards.md](boards.md). + +## Capture: J-Link route + +`rtt.py` above is this route packaged. Raw form (what it runs): + +```bash +JLinkExe -USB <serial> -device <dev> -if swd -speed 4000 -NoGui 1 -AutoConnect 1 \ + -RTTTelnetPort <port> # keep stdin open; 'exit' tears it down +nc localhost <port> # JLinkRTTClient minus the banner; carries input too +``` + +Commander keeps hunting for the control block and delivers the buffered boot +burst once the target's first printf creates it. `JLinkGDBServer +-RTTTelnetPort` also serves the port but on some parts (measured: LPC4088) +never locates the control block **unless a GDB client attaches** — fine +inside a GDB session, a silent failure headless — and it briefly halts the +core on connect (measured), which matters for timing-sensitive repros; +Commander does not. One telnet client per port at a time. + +## Capture: OpenOCD route (native probes: ST-Link, CMSIS-DAP) + +This is the LIVE route — WCH-Link targets are SDI and get only the halt→dump +route (transport matrix). Same script, openocd backend (`--elf` = the +FLASHED elf; the script takes the exact control-block address from `nm` — +a full-RAM scan is slower and can match stale RAM after a soft reset): + +```bash +python3 tools/rtt.py --backend openocd --probe <serial> \ + --cfg "-f interface/stlink.cfg -f target/stm32h7x.cfg" --elf <flashed.elf> --seconds 20 +# --channel: up-buffer index (0 = "Terminal" console, 1 = SystemView's "SysView" +# buffer in TinyUSB builds); -i forwards stdin → down-buffer 0 +# --vid-pid "0x2e8a 0x000c": pin the probe by USB IDs (with or instead of --probe; +# also keeps openocd discovery off foreign usbfs nodes) +# --addr 0x2000xxxx: explicit control-block address when the flashed elf is not at hand +# --reset-before-attach: reset the target INSIDE the session (2 s settle, then +# attach — the control block must exist before `rtt start` can find it; the ring's +# NO_BLOCK_SKIP head-retention is what preserves byte 0 across the settle) — +# required for streams that only decode from byte 0 +# (SystemView emits its Init record, carrying the timestamp frequency, once at boot; +# a mid-flight attach yields a stream no decoder can lock onto). Verified on +# stm32h743nucleo: after the ring is drained, a plain attach misses the boot preamble +# entirely and this flag captures it. NOT for SAMD5x (an in-session reset via the DSU +# leaves the core held) or WCH SDI. +``` + +What it runs: `openocd <cfg> -c "adapter serial <sn>" -c init -c "rtt setup +<nm-addr> 0x800 \"SEGGER RTT\"" -c "rtt polling_interval 1" -c "rtt start" +-c "rtt server start <port> <ch>"`, then a socket on that port. + +Attach WITHOUT reset when the flash step already reset the board (on SAMD5x, +an in-session `reset run` goes through the DSU CPU Reset Extension and leaves +the core held). After any reset the target's offsets restart at zero while +the server holds stale ones, and the tool exposes no console to type into (it +launches openocd with tcl/gdb/telnet ports disabled): stop the capture and +run it again to resync — do not reset mid-capture if you can avoid it. `rtt start` +fails while the block doesn't exist yet: it appears at the firmware's first +RTT write, so reset, settle ~500 ms, then start. Read AND write validated on +the ci rig's 8 native-probe boards (ST-Link + CMSIS-DAP, incl. RP2350), +end-to-end through this script's backend on all 8 — per-board rows in +boards.md. OpenOCD polls, and host-side loss is invisible +to the target's overflow counter: at the default 100 ms interval a busy +stream loses most samples (measured 2066 of 5064 events/s delivered on +stm32f407disco) — `rtt polling_interval 1` is mandatory for quantitative +capture, not a tuning nicety. Prefer SEGGER tools where a J-Link exists. + +## Post-mortem: reading the ring without a live server + +Default log mode is `NO_BLOCK_SKIP`: with no reader draining, the ring holds +the **first KB after boot, not the tail** — interpretation rules in +`target-debug`. To keep the last N bytes instead, the firmware must log via +`SEGGER_RTT_WriteWithOverwriteNoLock` (target drags `RdOff` itself; no host +needed) — but SEGGER's own restriction comes with it: *"Do not use +SEGGER_RTT_WriteWithOverwriteNoLock if a J-Link connection reads RTT data"* +(`lib/SEGGER_RTT/RTT/SEGGER_RTT.c`), because the target moving `RdOff` races +the host reader. So it is for firmware you dump post-mortem, never for a +board that also runs a live console (every HIL rtt board does). Reading a wedged target's ring — debug-AP RAM reads don't halt the +core: + +```bash +python3 tools/rtt.py --backend jlink --dump ring.bin \ + --probe <serial> --device <JLINK_DEVICE> --elf <flashed.elf> # or --addr 0x... +# prints pBuffer/Size/WrOff/RdOff; WrOff/RdOff delimit the valid bytes +``` + +(What it runs, for hand-driving JLinkExe: `nm` the ELF for `_SEGGER_RTT`, +`mem32 <addr+0x18>, 6` = aUp[0] {sName,pBuffer,Size,WrOff,RdOff,Flags}, +then `savebin <file> <pBuffer> <SizeOfBuffer>`.) + +## Buffer modes and locking (target side) + +- Modes: `NO_BLOCK_SKIP` (default for logs — drops whole writes when full), + `NO_BLOCK_TRIM`, `BLOCK_IF_FIFO_FULL` (target spins — dangerous in ISRs). +- Throughput is drain-limited: measured 24.6 KiB/s over a J-Link console + against a saturating printf loop, with the drops happening at the target. + RTT console output is NOT lossless under load; for high-bandwidth streams + size the buffer up (SystemView needs 2048–8192) and watch for overflow. +- Non-ARM ports must supply `SEGGER_RTT_LOCK/UNLOCK`: the vendored generic + RISC-V lock uses `mstatus` CSRs that trap (mcause=2) on WCH QingKe. Worked + port on branch `claude/add-systemview-debug`: `hw/bsp/ch583/ + sysview_rtt_lock_wch.h` (brace-scoped save/restore of CSR 0x800), and the + shared `hw/bsp/sysview_rtt_conf_wch.h` that ch32v20x/ch32v30x family.cmake + force-include to win the include-guard race against the vendored conf. + +## Common mistakes + +- **Attaching before the first printf** — the control block is zeroed `.bss` + until the firmware's first RTT write; early readers see nothing (and + RTTLogger gives up for good). Commander/`rtt.py` keep hunting. +- **Sending input before the server finds the control block** — the J-Link + telnet route silently DROPS client bytes until then (measured on the rig: + an instant `ping` vanished, a delayed one echoed). `rtt.py -i` + holds stdin until target output flows (or 5 s); when driving the raw + socket yourself, wait for output before writing. +- **Resetting while a console is attached** — flash and reset first; the + console owns the probe until closed. +- **Killing servers with `pkill -f`** — the pattern matches your own shell's + cmdline (and unrelated sessions): a compound command that pkills its + wrapper then re-reads a stale log misdiagnosed a healthy probe for an + hour. Close `rtt.py` with Ctrl-C/`--seconds` (its teardown reaps + the whole process group); if you must pattern-kill, bracket a char: + `pkill -f '[J]LinkExe -USB <serial>'`. +- **Unpinned flash with several probes attached** — pin by serial, always. +- **Two probes wired to one SWD header** — wedges the target; rewire. +- **Expecting an echo from firmware that never reads the console** — only + code polling `board_getchar()` consumes down-buffer 0 (`board_test` does). +- **Full-RAM `rtt setup` scans** — can lock onto a stale pre-reset block; + use the `nm` address. diff --git a/.claude/skills/rtt/boards.md b/.claude/skills/rtt/boards.md new file mode 100644 index 000000000..5ddd072d2 --- /dev/null +++ b/.claude/skills/rtt/boards.md @@ -0,0 +1,78 @@ +# rtt — per-board validation matrix + +A row appears here only after the board was exercised on real hardware; a new +validation adds the row AND any caveat it surfaced. "Read" = console/log +capture reached the host; "Write" = the target demonstrably consumed console +input (a printf-echo `board_test` returned the sent bytes — stock +`board_test` cannot, see SKILL.md's echo-validation note). Routes match +SKILL.md's capture sections; `Device/cfg` is the J-Link `--device` string or +the openocd target cfg. Rig rows (ci.lan) were validated 2026-08-24 by a +flash→capture→`ping`-echo sweep under per-board `hil_lock` flocks, and +re-validated 2026-08-25 end-to-end through the skill's own CLI +(`tools/rtt.py`, jlink + openocd backends): 20/20 read+write — +including CONCURRENTLY at 8 parallel consoles (20 boards in 39 s, mixed +routes, no port collisions or cross-board output bleed: one server per +probe on its own ephemeral port). htpc rows on the local bench. The openocd backend's `--reset-before-attach` +is decode-validated: a channel-1 SystemView capture on stm32h743nucleo +(byte-identical boot preamble to the sysview campaign's golden reference, +49765 events decoded, ISR/task timings matching to 0.1 µs, overflow 0). + +| Board | Rig | Probe | Route | Read | Write | Device/cfg | +| ------------------------ | ---- | ---------------------- | ------- | ---- | ----- | --------------------- | +| ea4088_quickstart | htpc | LPC-Link2 J-Link fw | J-Link | yes | yes | `LPC4088` | +| raspberry_pi_pico2 | htpc | J-Trace PRO | J-Link | yes | — | `rp2350_m33_0` | +| frdm_k64f | ci | J-Link | J-Link | yes | yes | `MK64FN1M0xxx12` | +| feather_nrf52840_express | ci | J-Link | J-Link | yes | yes | `nrf52840_xxaa` | +| metro_m4_express | ci | J-Link | J-Link | yes | yes | `ATSAMD51J19` | +| lpcxpresso11u37 | ci | J-Link | J-Link | yes | yes | `LPC11U37/401` | +| lpcxpresso55s28 | ci | J-Link | J-Link | yes | yes | `LPC55S28` | +| ra4m1_ek | ci | J-Link | J-Link | yes | yes | `R7FA4M1AB` | +| stm32f072disco | ci | J-Link | J-Link | yes | yes | `stm32f072rb` | +| stm32f407disco | ci | J-Link | J-Link | yes | yes | `stm32f407vg` | +| stm32f723disco | ci | J-Link | J-Link | yes | yes | `stm32f723ie` | +| stm32l476disco | ci | J-Link | J-Link | yes | yes | `STM32L476VG` | +| mimxrt1064_evk | ci | J-Link | J-Link | yes | yes | `MIMXRT1064xxx6A` | +| nrf54lm20dk | ci | J-Link | J-Link | yes | yes | `NRF54LM20A_M33` | +| max32666fthr | ci | CMSIS-DAP | OpenOCD | yes | yes | `target/max32665.cfg` | +| raspberry_pi_pico | ci | debugprobe (CMSIS-DAP) | OpenOCD | yes | yes | `target/rp2040.cfg` | +| raspberry_pi_pico_w | ci | debugprobe (CMSIS-DAP) | OpenOCD | yes | yes | `target/rp2040.cfg` | +| raspberry_pi_pico2 | ci | debugprobe (CMSIS-DAP) | OpenOCD | yes | yes | `target/rp2350.cfg` | +| adafruit_fruit_jam | ci | debugprobe (CMSIS-DAP) | OpenOCD | yes | yes | `target/rp2350.cfg` | +| stm32h743nucleo | ci | ST-Link | OpenOCD | yes | yes | `target/stm32h7x.cfg` | +| stm32g0b1nucleo | ci | ST-Link | OpenOCD | yes | yes | `target/stm32g0x.cfg` | +| stm32u083nucleo | ci | ST-Link | OpenOCD | yes | yes | `target/stm32u0x.cfg` | + +Probe serials live in the rig configs (`test/hil/tinyusb.json`, bench +`local.json`) — always pass them (`--probe` / `adapter serial`). + +## Caveats + +- **ea4088_quickstart**: probe has no VCOM and the BSP has no UART — RTT is + the ONLY console; measured there: 6/6 JLinkExe attaches, boot burst + delivered, 24.6 KiB/s drain; the HIL suite runs over the RTT console + (device_info-class tests — the cdc/msc-fixture host tests don't speak RTT + yet, see the follow-up doc). + NEVER point OpenOCD at this J-Link-firmware probe (jaylink knocks it off + USB; physical replug). JLinkGDBServer never finds the CB headless on this + part; JLinkRTTLogger 0/6. +- **raspberry_pi_pico2 (htpc, J-Trace)**: pin the probe by serial — that + bench runs two J-Links (`-DJLINK_OPTION="-USB <sn>"` for the flash + target). Never set a custom JLinkScript for RP2350 over J-Link. Write path + untested there only because the flashed example doesn't poll the console + (the ci row's debugprobe sweep validated RP2350 writes). +- **ST-Link rows**: flashed by `STM32_Programmer_CLI`; RTT capture is a + separate openocd session (`interface/stlink.cfg` + the target cfg above), + attach without reset. + +## Excluded (recorded so absence is never read as "works") + +- `espressif_s3_devkitm`, `espressif_p4_function_ev` — no SEGGER RTT path in + our builds (console is the chip's USB-Serial-JTAG; see `esp-target-debug`). +- `ek_tm4c123gxl` — flashed by `lm4flash`; no debug-probe path configured on + the rig. +- `nanoch32v203`, `ch32v103r_r1_1v0`, `ch32v307v_r1_1v0`, `ch582m_evt` — a + `LOGGER=rtt` build traps on WCH QingKe (the vendored generic RISC-V + `SEGGER_RTT_LOCK` reads `mstatus` CSRs → mcause=2; the working lock port + `sysview_rtt_lock_wch.h` lives only on branch `claude/add-systemview-debug`), + and SDI permits no live streaming anyway (transport matrix). Revisit after + that branch merges. diff --git a/.claude/skills/target-debug/SKILL.md b/.claude/skills/target-debug/SKILL.md index 050a697b9..7ee96f48e 100644 --- a/.claude/skills/target-debug/SKILL.md +++ b/.claude/skills/target-debug/SKILL.md @@ -217,40 +217,36 @@ dump binary memory /tmp/ring.bin &dbg_ring[0] &dbg_ring[512] ## TU_LOG capture Build with `LOG=2` (`LOG=3` adds per-transfer noise and much more timing skew). -`LOGGER=rtt` routes it over the debug probe — no UART wiring. SEGGER's host -tools need a J-Link, but OpenOCD serves the same RTT buffer on ST-Link / -CMSIS-DAP / WCH-Link boards: +`LOGGER=rtt` routes it over the debug probe — no UART wiring. Stand the +channel up per the **rtt** skill (servers per probe, transport matrix, +control-block gotchas live there): ```bash -# RTT: JLinkGDBServer from CLAUDE.md "GDB Debugging" + -RTTTelnetPort, then: -timeout 20s JLinkRTTClient > /tmp/rtt.log # non-interactive capture +# RTT (J-Link probe; flash + reset first — the console owns the probe): +timeout 20s python3 tools/rtt.py --backend jlink --probe <sn> --device <JLINK_DEVICE> > /tmp/rtt.log # UART (board's debug serial, if wired): stty -F /dev/ttyACM<N> 115200 raw && timeout 20s cat /dev/ttyACM<N> | tee /tmp/uart.log ``` -```bash -# OpenOCD RTT (any probe OpenOCD drives) — in telnet :4444 (or -c equivalents): -rtt setup 0x20000000 0x8000 "SEGGER RTT" # RAM ORIGIN + LENGTH (from the .ld/map) -rtt start # after firmware booted; rerun after each reflash -rtt server start 19021 0 -# then: timeout 20s nc localhost 19021 > /tmp/rtt.log -``` - -OpenOCD polls — bursty logs can drop lines; prefer J-Link where both -exist. The drain-model warning below applies unchanged. +OpenOCD RTT (native probes: ST-Link/CMSIS-DAP): rtt skill §OpenOCD — exact +CB address from `nm`, attach-only. OpenOCD polls — bursty logs can drop +lines; prefer J-Link where both exist. The drain-model warning below +applies unchanged. An RTT-built firmware that has since wedged still holds a log tail in RAM — but ONLY what fits the drain model: the default SEGGER mode (NO_BLOCK_SKIP) **drops** writes once the ring fills with no reader, so an undrained target -holds the first KB after boot, not the wedge tail. There is no overwrite mode -in stock SEGGER RTT (only SKIP/TRIM/BLOCK): post-mortem RTT is evidence only -if a live drain was running — otherwise instrument with the RAM ring above. -Use `JLinkGDBServer -RTTTelnetPort 19021` + `JLinkRTTClient` for the drain -(proven; note the server briefly halts the core on connect). `JLinkRTTLogger` -fails to find the control block on some parts (LPC4088) even when it exists -and even given `-RTTAddress`; don't fight it — `nm` the ELF for `_SEGGER_RTT`, -read the aUp[0] descriptor (`mem32`), `savebin` the buffer — debug-AP RAM -reads don't halt the target. +holds the first KB after boot, not the wedge tail. The buffer flags have no +overwrite mode (only SKIP/TRIM/BLOCK); keeping the tail instead requires the +firmware-side overwrite write call (rtt skill §post-mortem). So post-mortem +RTT from a default-mode build is evidence only if a live drain was running — +otherwise instrument with the RAM ring above. +Stand up the drain per the **rtt** skill: JLinkExe's `-RTTTelnetPort` (what +`rtt.py` wraps) is the headless-proven route; JLinkGDBServer's needs +a GDB client attached on some parts (LPC4088), and JLinkRTTLogger fails to +find the control block on some parts (measured LPC4088, 0/6). The manual +ring read for a wedged target (`nm`/`mem32`/`savebin` — debug-AP reads don't +halt the core) lives there too. ## GDB — state autopsy and watchpoints @@ -331,7 +327,7 @@ Linux gadget peer): ```bash .claude/skills/usbmon/scripts/usbcap.sh cafe: 30 /tmp/host.pcapng & # host URBs (usbmon skill) -timeout 30s JLinkRTTClient > /tmp/target.rtt & # target (or ring dump after) +timeout 30s python3 tools/rtt.py --backend jlink --probe <sn> --device <dev> > /tmp/target.rtt & # target (rtt skill; or ring dump after) wait ``` diff --git a/.claude/skills/update-sponsor/SKILL.md b/.claude/skills/update-sponsor/SKILL.md new file mode 100644 index 000000000..8c4987c8f --- /dev/null +++ b/.claude/skills/update-sponsor/SKILL.md @@ -0,0 +1,28 @@ +--- +name: update-sponsor +description: Use when a GitHub sponsor joins, upgrades, cancels, or switches between public and private; when the README sponsor sections are stale or show "be the first!" despite active sponsors; or when new issues, PRs, or discussions still need sponsor, priority, or Adafruit triage labels. +--- + +# Update Sponsors + +Rewrite `README.rst`'s sponsor blocks and backfill triage labels from live GitHub Sponsors data. + +```bash +S=.claude/skills/update-sponsor/update_sponsor.py +python3 $S --rules # tier -> README section -> labels, and the privacy rules +python3 $S --help # flags +python3 $S --dry-run # preview; every run previews and asks before applying +``` + +Run from the repo root as `hathach` — the script refuses any other account, whose sponsors are not +the ones this README lists. Hand-edited data lives in `config.json`, documented in that file. + +**Agents:** the confirmation prompt needs a terminal and a tool-call shell has none, so the script +refuses to apply rather than guessing. Run `--dry-run`, show the maintainer the preview, get their +answer, then re-run with `--yes`. Never `--yes` on the first call — the preview is the point. +Applying dirties tracked files (`README.rst`, sometimes `tools/codespell/ignore-words.txt`); +leave them unstaged for the maintainer, as `make-release` does. + +**`.github/workflows/labeler.yml` owns the label rules.** It applies the same labels when a ticket is +opened; this script only backfills what that workflow cannot reach — tickets older than it, and +private sponsors its `GITHUB_TOKEN` cannot see. Changing the policy means changing both. diff --git a/.claude/skills/update-sponsor/config.json b/.claude/skills/update-sponsor/config.json new file mode 100644 index 000000000..32d535be6 --- /dev/null +++ b/.claude/skills/update-sponsor/config.json @@ -0,0 +1,15 @@ +{ + "_comment": "Curated data for update_sponsor.py. Committed. Edit by hand.", + "_exclude": "Logins that never get labels, however they qualify. The maintainer is a public member of the adafruit org, so without this every self-authored ticket would be tagged 'Reported by an Adafruit member'.", + "_org_members": "Sponsoring ORGANIZATIONS only. A GitHub org never opens tickets itself - its people do. List the logins that should inherit that org's tier. Curated on purpose: the public-members API misses private members and over-counts uninvolved ones.", + "org_members": { + "8086net": [ + "burtyb" + ] + }, + "_adafruit_members_extra": "Adafruit logins the org's PUBLIC member list omits. Unioned with `gh api orgs/adafruit/public_members`. Never use /members: it returns concealed members to an org admin, and the Adafruit label would then publish an affiliation those people deliberately hid.", + "adafruit_members_extra": [], + "exclude": [ + "hathach" + ] +} diff --git a/.claude/skills/update-sponsor/update_sponsor.py b/.claude/skills/update-sponsor/update_sponsor.py new file mode 100644 index 000000000..d09c8b294 --- /dev/null +++ b/.claude/skills/update-sponsor/update_sponsor.py @@ -0,0 +1,505 @@ +#!/usr/bin/env python3 +"""Sync README.rst sponsor sections and sponsor/priority labels from GitHub Sponsors. + +Reads live sponsorship data (including private sponsors) via `gh api graphql`, +rewrites the four marker-delimited blocks in README.rst, and applies triage +labels to OPEN issues / PRs / discussions authored by entitled logins. + +Every run plans first and prints what it would change, then asks before +touching anything. --yes skips the prompt (needed when stdin is not a tty), +--dry-run stops after the preview. + +State lives in state.json next to this file (gitignored): the highest ticket +number already scanned, so later runs skip old tickets. +""" + +import argparse +import difflib +import hashlib +import json +import re +import subprocess +import tempfile +import sys +from datetime import date +from pathlib import Path + +HERE = Path(__file__).resolve().parent +REPO = HERE.parents[2] +README = REPO / "README.rst" +CONFIG = HERE / "config.json" +IGNORE_WORDS = REPO / "tools" / "codespell" / "ignore-words.txt" +STATE = HERE / "state.json" + +OWNER, NAME = "hathach", "tinyusb" + +L_SPONSOR = "Sponsor \U0001f496" +L_PRIO = "Prio \U0001f4cc" +L_PRIO_TOP = "Prio Top \U0001f6a8" +L_ADAFRUIT = "Adafruit \U0001f338" + +# Label rules mirror .github/workflows/labeler.yml, which applies the same set when +# a ticket is opened. Keep the two in step or a ticket's labels start depending on +# which mechanism happened to touch it. +TIER_LABELS = { + "QWORD": {L_SPONSOR, L_PRIO_TOP}, + "DWORD": {L_SPONSOR, L_PRIO_TOP}, + "WORD": {L_SPONSOR, L_PRIO}, + "BYTE": {L_SPONSOR}, + "BIT": {L_SPONSOR}, +} +ADAFRUIT_LABELS = {L_ADAFRUIT, L_SPONSOR, L_PRIO_TOP} + +# tier key -> (min $/month, README marker, placeholder when empty, avatar px) +TIERS = [ + ("QWORD", 512, "QWORD-SPONSORS", "*No QWORD sponsors yet — be the first!*", 120), + ("DWORD", 128, "DWORD-BACKERS", "*No backers yet — be the first!*", 80), + ("WORD", 32, "WORD-SUPPORTERS", "*No supporters yet — be the first!*", 40), + ("BYTE", 8, "BYTE-THANKS", "*No names listed yet — be the first!*", 0), + ("BIT", 2, None, None, 0), # no README listing +] + + +def gh(*args, **kw): + out = subprocess.run(["gh", *args], capture_output=True, text=True, **kw) + if out.returncode: + sys.exit(f"gh {' '.join(args[:2])} failed:\n{out.stderr.strip()}") + return out.stdout + + +def graphql(query, **variables): + args = ["api", "graphql", "-f", f"query={query}"] + for k, v in variables.items(): + args += ["-F", f"{k}={'null' if v is None else v}"] + data = json.loads(gh(*args)) + if "errors" in data: + sys.exit("GraphQL errors:\n" + json.dumps(data["errors"], indent=2)) + return data["data"] + + +# ---------------------------------------------------------------- sponsors + +SPONSOR_Q = """ +query($cursor:String){ viewer{ login sponsorshipsAsMaintainer(first:100, includePrivate:true, activeOnly:true, after:$cursor){ + pageInfo{hasNextPage endCursor} + nodes{ privacyLevel createdAt tier{monthlyPriceInDollars} + sponsorEntity{ __typename ... on User{login name} ... on Organization{login name} } } } } } +""" + + +def tier_of(dollars): + for key, floor, *_ in TIERS: + if dollars >= floor: + return key + return "BIT" # below the lowest published tier, but still a sponsor + + +def fetch_sponsors(): + """Active sponsorships, oldest first (chronological README order).""" + sponsors, cursor = [], None + while True: + viewer = graphql(SPONSOR_Q, cursor=cursor)["viewer"] + if not viewer: + sys.exit("gh is authenticated with a token that has no user identity - " + "it cannot see sponsorships") + if viewer["login"].lower() != OWNER.lower(): + sys.exit(f"gh is authenticated as {viewer['login']}, not {OWNER} - " + f"its sponsors are not the ones this README lists") + page = viewer["sponsorshipsAsMaintainer"] + for n in page["nodes"]: + entity = n["sponsorEntity"] + if not entity: # private sponsor we somehow cannot resolve + continue + tier = tier_of((n["tier"] or {}).get("monthlyPriceInDollars") or 0) + sponsors.append({ + "login": entity["login"], + "name": (entity["name"] or "").strip() or entity["login"], + "is_org": entity["__typename"] == "Organization", + "private": n["privacyLevel"] == "PRIVATE", + "since": n["createdAt"], + "tier": tier, + }) + if not page["pageInfo"]["hasNextPage"]: + break + cursor = page["pageInfo"]["endCursor"] + sponsors.sort(key=lambda s: s["since"]) + return sponsors + + +# ------------------------------------------------------------------ README + +def mask(login): + """Private sponsor display name: first 3 chars, rest hidden behind a fixed + 4 stars so the real length does not leak. A login of 3 chars or fewer has no + `rest` to hide, so it is withheld entirely.""" + return login[:3] + "****" if len(login) > 3 else "a private supporter" + + +def rst_escape(text): + """A GitHub display name is free-form: backticks/angle brackets would break out + of the inline-link markup and could point the link anywhere.""" + return re.sub(r"([*`<>|_\\])", r"\\\1", text) + + +def render(sponsors, size, use_company_name, seen): + """One line of comma-separated entries, plus any avatar substitution defs.""" + entries, defs = [], [] + for s in sponsors: + if s["login"].lower() in seen: # duplicate |av-x| defs are an RST error + continue + seen.add(s["login"].lower()) + if s["private"]: + entries.append(rst_escape(mask(s["login"]))) # no avatar, no link: both would out them + continue + # DWORD/QWORD perks promise a company name; Byte/Word promise a username. + label = rst_escape(s["name"]) if use_company_name else "@" + s["login"] + link = f"`{label} <https://github.com/{s['login']}>`__" + if size: + entries.append(f"|av-{s['login']}| {link}") + defs += [f".. |av-{s['login']}| image:: https://github.com/{s['login']}.png?size={size}", + f" :target: https://github.com/{s['login']}", + f" :alt: {s['login']}", ""] + else: + entries.append(link) + body = ", ".join(entries) + return body + ("\n\n" + "\n".join(defs).rstrip() if defs else "") + + +def render_readme(sponsors, original): + """Return README.rst with every marker block regenerated, or None if unchanged.""" + text, seen = original, set() + for key, _floor, marker, placeholder, size in TIERS: + if marker is None: + continue + members = [s for s in sponsors if s["tier"] == key] + block = render(members, size, key in ("DWORD", "QWORD"), seen) if members else placeholder + pattern = re.compile(rf"(^\.\. {re.escape(marker)}-START$\n)(.*?)(^\.\. {re.escape(marker)}-END$)", + re.M | re.S) + if not pattern.search(text): + sys.exit(f"README.rst: marker {marker}-START/-END not found") + text = pattern.sub(lambda m: m.group(1) + "\n" + block + "\n\n" + m.group(3), text) + return None if text == original else text + + +def codespell_collisions(original, new_text): + """Logins/names the repo's auto-fixing codespell hook would rewrite in place.""" + added = [l for l in difflib.unified_diff(original.splitlines(), + new_text.splitlines(), n=0) if l.startswith("+")] + if not added: + return [] + with tempfile.NamedTemporaryFile("w", suffix=".txt", delete=True) as probe: + # outside the repo and not dot-prefixed: codespell skips hidden files unless + # .codespellrc is picked up from cwd, which would make this guard fail open + probe.write("\n".join(added) + "\n") + probe.flush() + try: + run = subprocess.run(["codespell", "--ignore-words", str(IGNORE_WORDS), probe.name], + capture_output=True, text=True) + except FileNotFoundError: + print(" note: codespell not on PATH - generated block is UNCHECKED") + return [] + if run.returncode not in (0, 65): # 65 = typos found; anything else is a tool error + print(f" note: codespell failed (rc={run.returncode}) - generated block is UNCHECKED") + return [] + out = run.stdout + # v2.2.4 (the pinned hook) lowercases dictionary keys before testing ignore-words, + # so a cased entry would never match and -w would rewrite the login anyway. + return sorted({l.split(":", 2)[2].split("==>")[0].strip().lower() + for l in out.splitlines() if l.count(":") >= 2}) + + +def readme_diff(original, new_text): + return "".join(difflib.unified_diff( + original.splitlines(keepends=True), new_text.splitlines(keepends=True), + fromfile="README.rst", tofile="README.rst (new)", n=2)) + + +# ------------------------------------------------------------------ labels + +def as_logins(value, where): + """config.json is hand-edited: a bare string here would iterate as characters and + label the single-letter accounts it spells.""" + if not isinstance(value, list) or not all(isinstance(x, str) for x in value): + sys.exit(f"config.json: {where} must be a list of logins, got {value!r}") + return [x for x in value if x] + + +def entitlements(sponsors, config): + """login -> sorted labels, first matching rule only (Adafruit, then sponsor tier).""" + ent = {} + + # GitHub logins are case-insensitive and config.json is hand-edited, so + # normalise everywhere. labeler.yml compares with .toLowerCase() for this reason. + skip = {x.lower() for x in as_logins(config.get("exclude", []), "exclude")} + + def grant(login, labels): + login = login.lower() + if login and login not in skip: # the maintainer does not triage their own tickets + ent.setdefault(login, set(labels)) # setdefault: first rule wins, never a union + + # Adafruit is evaluated FIRST, matching labeler.yml's branch order. + # public_members ONLY: /members returns concealed members to an org admin, and + # labelling one "Reported by an Adafruit member" publishes what they hid. + members = set(gh("api", "orgs/adafruit/public_members", "--paginate", "-q", ".[].login").split()) + members |= set(as_logins(config.get("adafruit_members_extra", []), "adafruit_members_extra")) + for m in members: + grant(m, ADAFRUIT_LABELS) + + # Rules mirror .github/workflows/labeler.yml, which applies these same labels + # when a ticket is opened. This pass backfills what the workflow cannot reach: + # tickets older than it, and private sponsors its GITHUB_TOKEN cannot see. + for s in sponsors: + # A private sponsor gets NO label. Every candidate set was measured against the + # live repo and each one identifies them: withholding `Sponsor 💖` leaves a bare + # `Prio Top 🚨`, which nothing else in the repo emits; and a bare `Prio 📌` + # appears on 1 of 204 open issues and 0 of 873 discussions. A label applied only + # to private sponsors IS the disclosure, whichever label it is. The triage perk + # cannot ride on a public label - honour it off-ticket. + if s["private"]: + continue + labels = set(TIER_LABELS[s["tier"]]) + if not labels: + continue + if s["is_org"]: + members = as_logins({k.lower(): v for k, v in config.get("org_members", {}).items()} + .get(s["login"].lower(), []), f"org_members[{s['login']}]") + if not members: + who = mask(s["login"]) if s["private"] else s["login"] + print(f"note: org sponsor {who} ({s['tier']}) has no members in config.json - " + f"nothing to label") + for m in members: + grant(m, labels) + else: + grant(s["login"], labels) + + return {k: sorted(v) for k, v in sorted(ent.items())} + + +SCAN_Q = """ +query($cursor:String){ repository(owner:"%s",name:"%s"){ %s(first:100, %sorderBy:{field:CREATED_AT,direction:DESC}, after:$cursor){ + pageInfo{hasNextPage endCursor} + nodes{ number id %s author{login} labels(first:40){nodes{name}} } } } } +""" + + +def scan(kind, since): + """Open tickets with number > since, newest first; stops at the watermark.""" + states = "" if kind == "discussions" else "states:OPEN, " + closed = "closed" if kind == "discussions" else "" + query = SCAN_Q % (OWNER, NAME, kind, states, closed) + cursor, found = None, [] + while True: + page = graphql(query, cursor=cursor)["repository"][kind] + for n in page["nodes"]: + if n["number"] <= since: + return found + if n.get("closed"): + continue + found.append({"number": n["number"], "id": n["id"], + "author": (n["author"] or {}).get("login"), + "labels": {x["name"] for x in n["labels"]["nodes"]}}) + if not page["pageInfo"]["hasNextPage"]: + return found + cursor = page["pageInfo"]["endCursor"] + + +def label_ids(): + q = '{repository(owner:"%s",name:"%s"){labels(first:100){nodes{name id}}}}' % (OWNER, NAME) + return {n["name"]: n["id"] for n in graphql(q)["repository"]["labels"]["nodes"]} + + +def highest_number(): + q = ('{repository(owner:"%s",name:"%s"){' + 'issues(first:1,orderBy:{field:CREATED_AT,direction:DESC}){nodes{number}}' + 'pullRequests(first:1,orderBy:{field:CREATED_AT,direction:DESC}){nodes{number}}' + 'discussions(first:1,orderBy:{field:CREATED_AT,direction:DESC}){nodes{number}}}}') % (OWNER, NAME) + r = graphql(q)["repository"] + return max((v["nodes"][0]["number"] for v in r.values() if v["nodes"]), default=0) + + +def plan_labels(ent, since, private): + """Open tickets above the watermark whose author is owed labels they lack.""" + actions = [] + # path segment differs per type, and doubles as the "which kind is this?" hint + for kind, path in (("issues", "issues"), ("pullRequests", "pull"), ("discussions", "discussions")): + for t in scan(kind, since): + want = sorted(set(ent.get((t["author"] or "").lower(), [])) - t["labels"]) + if want: + actions.append({"number": t["number"], "id": t["id"], "add": want, + "author": mask(t["author"]) if (t["author"] or "").lower() in private else t["author"], + "url": f"https://github.com/{OWNER}/{NAME}/{path}/{t['number']}"}) + return sorted(actions, key=lambda a: a["number"]) + + +def print_label_table(actions): + rows = [(a["url"], a["author"], ", ".join(a["add"])) for a in actions] + head = ("Ticket", "Author", "Labels to add") + # emoji render double-width, so pad by display width, not len() + width = lambda t: len(t) + sum(c > "\u2100" for c in t) + w = [max(width(r[i]) for r in rows + [head]) for i in range(3)] + pad = lambda t, i: t + " " * (w[i] - width(t)) + print(" " + " ".join(pad(head[i], i) for i in range(3))) + print(" " + " ".join("-" * w[i] for i in range(3))) + for r in rows: + print(" " + " ".join(pad(r[i], i) for i in range(3))) + + +def apply_label_actions(actions, ids): + for i, a in enumerate(actions, 1): + # IDs inlined: `gh api graphql -F` cannot pass a list variable. + graphql("mutation{addLabelsToLabelable(input:{labelableId:%s,labelIds:%s})" + "{clientMutationId}}" % (json.dumps(a["id"]), + json.dumps([ids[w] for w in a["add"]]))) + print(f" [{i}/{len(actions)}] {a['url']}") # per ticket: a mid-run failure must be legible + print(f"labels: {len(actions)} ticket(s) updated") + + +def confirm(question): + if not sys.stdin.isatty(): + sys.exit("stdin is not a terminal - re-run with --yes to apply, or --dry-run to plan only") + return input(f"{question} [y/N] ").strip().lower() in ("y", "yes") + + +# -------------------------------------------------------------------- main + +def print_rules(): + """The applied policy, read out of the constants above so it cannot drift.""" + print("Label rules mirror .github/workflows/labeler.yml (the source of truth for new " + "tickets).\nThis script backfills what that workflow cannot reach: tickets older " + "than it, and\nprivate sponsors, which its GITHUB_TOKEN cannot see at all.\n") + row = " {:<9} {:>5} {:<17} {:<22} {}" + print(row.format("Tier", "$/mo", "README section", "Listed as", "Labels")) + for key, floor, marker, _placeholder, size in TIERS: + company = key in ("DWORD", "QWORD") + listed = ("not listed" if marker is None else + (("logo + " if company else "avatar + ") if size else "") + + ("company name" if company else "@username")) + print(row.format(key, floor, marker or "-", listed, " ".join(sorted(TIER_LABELS[key])))) + print(row.format("Adafruit", "-", "hand-written", "-", " ".join(sorted(ADAFRUIT_LABELS)))) + print("\nAdafruit membership comes from orgs/adafruit/public_members, never /members:" + "\n an org admin sees concealed members too, and the Adafruit label would publish" + "\n an affiliation those people deliberately hid." + "\nA private sponsor is masked in the README (first 3 chars, no avatar, no link) and" + "\n gets NO label at all: any label applied only to private sponsors is itself the" + "\n disclosure. Measured live - a bare Prio Top 🚨 is emitted by nothing else in the" + "\n repo, and a bare Prio 📌 by 1 of 204 open issues. Honour their perk off-ticket." + "\nRules are first-match-only (Adafruit, then tier), never a union: two priority" + "\n labels on one ticket double-count it in triage." + "\nOnly OPEN tickets are labelled, labels are only ever added, and a ticket reopened" + "\n below the watermark needs --full-rescan.") + + +def main(): + p = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--rules", action="store_true", + help="print the tier/label policy and exit") + p.add_argument("--dry-run", action="store_true", + help="preview and stop; writes nothing, not even state.json") + p.add_argument("--yes", action="store_true", + help="skip the confirmation prompt (required when stdin is not a terminal)") + p.add_argument("--full-rescan", action="store_true", + help="ignore the watermark and scan every open ticket") + only = p.add_mutually_exclusive_group() + only.add_argument("--readme-only", action="store_true", help="skip the label pass") + only.add_argument("--labels-only", action="store_true", help="skip the README pass") + args = p.parse_args() + + if args.rules: + return print_rules() + + if REPO != Path.cwd().resolve() and REPO not in Path.cwd().resolve().parents: + sys.exit(f"run from inside {REPO} - this script writes that checkout, not the cwd") + + config = json.loads(CONFIG.read_text(encoding="utf-8")) + unknown = {k for k in config if not k.startswith("_")} - {"exclude", "org_members", + "adafruit_members_extra"} + if unknown: # a mistyped key reads as absent, and `exclude` failing open means + sys.exit(f"config.json: unknown key(s) {sorted(unknown)}") # labelling our own tickets + state = json.loads(STATE.read_text(encoding="utf-8")) if STATE.exists() else {} + original = README.read_text(encoding="utf-8") # one snapshot, re-checked before the write + + sponsors = fetch_sponsors() + print(f"{len(sponsors)} active sponsor(s):") + for s in sponsors: + who = mask(s["login"]) + " (private)" if s["private"] else s["login"] + print(f" {s['since'][:10]} {s['tier']:<5} {who}") + + # ---------------------------------------------------------------- plan + if not args.labels_only and not sponsors: + # Rewriting every section back to "be the first!" is indistinguishable from a + # token that cannot see the sponsorships. Refuse rather than wipe. + sys.exit("no active sponsorships returned - refusing to rewrite README.rst") + new_readme = None if args.labels_only else render_readme(sponsors, original) + + actions, ids, watermark, fingerprint = [], {}, None, None + if not args.readme_only: + ent = entitlements(sponsors, config) + fingerprint = hashlib.sha256(json.dumps(ent, sort_keys=True).encode()).hexdigest()[:16] + changed = bool(state) and fingerprint != state.get("fingerprint") + since = 0 if args.full_rescan or changed else state.get("last_ticket", 0) + if changed: + print("entitlements changed since last run - rescanning all open tickets") + elif args.full_rescan: + print("--full-rescan - ignoring the watermark") + print(f"scanning open tickets above #{since}") + ids = label_ids() + watermark = highest_number() + # checked before the prompt: an unknown name must not KeyError mid-apply + unknown = {n for n in (L_SPONSOR, L_PRIO, L_PRIO_TOP, L_ADAFRUIT) if n not in ids} + if unknown: + sys.exit(f"labels missing from the repo: {sorted(unknown)}") + private = {s["login"].lower() for s in sponsors if s["private"]} + actions = plan_labels(ent, since, private) + + # ------------------------------------------------------------- preview + collisions = codespell_collisions(original, new_readme) if new_readme else [] + if not args.labels_only: + print("\nREADME.rst") + print(readme_diff(original, new_readme) if new_readme else " no change\n") + if collisions: + print(f" note: codespell (-w) would rewrite {', '.join(collisions)} in the generated block;\n" + f" adding them to {IGNORE_WORDS.relative_to(REPO)} on apply\n") + if not args.readme_only: + print("Tickets") + if actions: + print_label_table(actions) + else: + print(" no change") + print() + + if not new_readme and not actions: + print("nothing to do") + return + if args.dry_run: + print("dry run - nothing applied") + return + if not args.yes and not confirm("Apply these changes?"): + sys.exit("aborted - nothing applied") + + # --------------------------------------------------------------- apply + # Remote label writes go FIRST: they cannot be undone by git, so if they fail + # partway the local README edit has not happened and `git status` stays honest. + if actions: + apply_label_actions(actions, ids) + if new_readme: + if README.read_text(encoding="utf-8") != original: # edited during the prompt + sys.exit("README.rst changed while this run was in progress - " + "labels are applied, re-run for the README") + if collisions: # before the write, so the hook cannot mangle it first + have = [w for w in IGNORE_WORDS.read_text(encoding="utf-8").splitlines() if w.strip()] + IGNORE_WORDS.write_text("\n".join(sorted(set(have) | set(collisions))) + "\n", encoding="utf-8") + print(f"ignore-words.txt: added {', '.join(collisions)}") + README.write_text(new_readme, encoding="utf-8") + print("README.rst: updated") + # Written only here: a dry run or an aborted confirmation must leave the + # watermark alone, or the next run would skip tickets it never labelled. + if watermark is not None: + STATE.write_text(json.dumps( + {"last_ticket": watermark, "fingerprint": fingerprint, "updated": date.today().isoformat()}, + indent=2) + "\n", encoding="utf-8") + print(f"state.json: last_ticket={watermark}") + + +if __name__ == "__main__": + main() diff --git a/.claude/skills/usbtest/SKILL.md b/.claude/skills/usbtest/SKILL.md index 4d7c6e5ab..606b379b5 100644 --- a/.claude/skills/usbtest/SKILL.md +++ b/.claude/skills/usbtest/SKILL.md @@ -30,7 +30,7 @@ python3 test/hil/usbtest.py --serial <uid> --keep-binding --tests 29 # one case ``` - **Always `--keep-binding`**: the cleanup unbind path has wedged host xHCIs (`usb_hcd_alloc_bandwidth`). -- CI (`hil_test.py`) additionally passes `--budget`, `--outer-timeout` and +- CI (`hil_test.py`) additionally passes `--budget` and `--recover-board`/`--recover-fw`: on a HUNG case the battery aborts, RESETS the DUT through its roster probe (non-destructive, ~130 ms) and reflashes only if that does not clear the wedge (see usb-kernel-recover). Manual runs without those flags leave a HUNG diff --git a/.claude/workflows/driver-review.js b/.claude/workflows/driver-review.js index 255b8ac74..3d380c7e0 100644 --- a/.claude/workflows/driver-review.js +++ b/.claude/workflows/driver-review.js @@ -1,9 +1,9 @@ export const meta = { name: 'driver-review', - description: 'Review driver directories across dimensions with driver-reviewer scanners, then adversarially verify every finding; returns only confirmed findings', + description: 'Review driver directories across dimensions with code-verifier scanners, then adversarially verify every finding; returns only confirmed findings', whenToUse: 'Auditing dcd/hcd drivers for a bug class (pass question) or a full-dimension review (default dimensions)', phases: [ - { title: 'Scan', detail: 'driver-reviewer per (dir x dimension)' }, + { title: 'Scan', detail: 'code-verifier per (dir x dimension)' }, { title: 'Verify', detail: 'adversarial refutation per finding' }, ], } @@ -58,7 +58,7 @@ const results = await pipeline( p => agent( `Review ${p.dir} for exactly one dimension: ${p.dim}. Read the sources yourself. Coverage-first — report everything, a verifier filters.`, - { label: `scan:${short(p.dir)}`, phase: 'Scan', agentType: 'driver-reviewer', effort: 'xhigh', schema: FINDINGS }, + { label: `scan:${short(p.dir)}`, phase: 'Scan', agentType: 'code-verifier', schema: FINDINGS }, ), (scan, p) => { @@ -69,7 +69,8 @@ const results = await pipeline( `Adversarially verify ONE review finding about ${p.dir}.\nDimension: ${p.dim}\nFinding: ${JSON.stringify(f)}\n` + 'Read the cited code plus enough context (callers, ISR paths, macros, and the datasheet if register-related) to judge. ' + 'Try to REFUTE it; real=true only if it survives your best attempt. Return {"real": bool, "reason": string}.', - { label: `verify:${short(p.dir)}:${f.line}`, phase: 'Verify', agentType: 'driver-reviewer', effort: 'xhigh', schema: VERDICT }, + // max: one judgment-dense call per finding decides what survives - worth the top tier + { label: `verify:${short(p.dir)}:${f.line}`, phase: 'Verify', agentType: 'code-verifier', effort: 'max', schema: VERDICT }, ).then(v => v && { ...f, verdict: v }) )).then(vs => { const alive = vs.filter(Boolean) diff --git a/.claude/workflows/fanout-dev.js b/.claude/workflows/fanout-dev.js index e98a86f1d..386f6df46 100644 --- a/.claude/workflows/fanout-dev.js +++ b/.claude/workflows/fanout-dev.js @@ -1,11 +1,11 @@ export const meta = { name: 'fanout-dev', - description: 'Implement one described change across many ports/file-sets: one port-dev worker per item, independent builder verification, optional review', + description: 'Implement one described change across many ports/file-sets: one code-writer worker per item, independent builder verification, optional review', whenToUse: 'Applying a fix or pattern across multiple TinyUSB ports (e.g. the same DCD bug in several drivers)', phases: [ - { title: 'Implement', detail: 'port-dev per item (opus xhigh)' }, + { title: 'Implement', detail: 'code-writer per item (opus xhigh)' }, { title: 'Verify', detail: 'builder single-example check' }, - { title: 'Review', detail: 'optional driver-reviewer pass' }, + { title: 'Review', detail: 'optional code-verifier pass' }, ], } @@ -71,7 +71,7 @@ const results = await pipeline( : ' Pick a verification board from hw/bsp whose family uses this scope.'), { label: `dev:${short(item)}`, phase: 'Implement', - agentType: 'port-dev', effort: 'xhigh', schema: DEV, + agentType: 'code-writer', schema: DEV, ...(args.worktree ? { isolation: 'worktree' } : {}), }, ), @@ -96,7 +96,7 @@ const results = await pipeline( return agent( `Review the uncommitted change in ${item} (inspect with: git diff -- ${item}) against this task:\n${args.task}\n` + 'Dimension: does the diff correctly and completely implement the task with no unintended side effects? Coverage-first findings.', - { label: `review:${short(item)}`, phase: 'Review', agentType: 'driver-reviewer', effort: 'xhigh', schema: FINDINGS }, + { label: `review:${short(item)}`, phase: 'Review', agentType: 'code-verifier', schema: FINDINGS }, ).then(f => { // review: array = findings; null = reviewer died; absent = not requested if (!f) log(`review:${short(item)}: reviewer agent died`) diff --git a/.claude/workflows/hil-validate.js b/.claude/workflows/hil-validate.js index bc0bda8b9..741ba481d 100644 --- a/.claude/workflows/hil-validate.js +++ b/.claude/workflows/hil-validate.js @@ -11,10 +11,10 @@ if (!args || !Array.isArray(args.boards) || args.boards.length === 0) { throw new Error('args must be { boards: string[], force? } with the boards already built') } -// The operator returns hil_summary.py's JSON verbatim plus its own observations. It does NOT +// The operator returns hil_report.py's JSON verbatim plus its own observations. It does NOT // retype the report table: rows are named per variant, a variant need not start with the board // name, and lock contention is a cell rather than a phrase — rebuilding board identity from -// prose produced a defect in each of four review rounds. hil_summary.py does that join against +// prose produced a defect in each of four review rounds. hil_report.py does that join against // the roster, so `locked` and `ran` arrive as fields and nothing here parses a detail string. const BOARD = { type: 'object', additionalProperties: false, @@ -26,12 +26,16 @@ const BOARD = { } const HIL = { type: 'object', additionalProperties: false, - required: ['results', 'wedged'], + required: ['results', 'wedged', 'caveat'], properties: { results: { type: 'array', items: BOARD }, // the operator's own observation — not derivable from the report wedged: { type: 'array', items: { type: 'string' } }, banner: { type: 'string' }, + // the run-level caveat: abandoned / aborted / selected-no-boards. `banner` carries rig + // HEALTH across an --accumulate retry; `caveat` carries how THIS run ended, and every + // row can still say pass while it failed — so it gates `pass` in summarize() below. + caveat: { type: 'string' }, }, } @@ -51,12 +55,12 @@ const runBoards = (boards, isRetry = false) => agent( (args.force ? 'THE USER HAS EXPLICITLY AUTHORIZED FORCING: run hil_test.py with HIL_NO_BOARD_LOCK=1 in the environment (bypasses the board lock check; do NOT release or kill the existing holder). ' : 'A board whose lock is held (a dev session or concurrent CI job) fails fast inside the run without blocking the others — never force the lock. ') + - 'If hil_test.py refuses the run with "board(s) not in <config>", re-run it WITHOUT the unknown names but keep the FULL board list on the hil_summary call below — it emits a ran:false entry for every board you name, so the unknown ones surface as "no report row" instead of costing the whole batch. ' + + 'If hil_test.py refuses the run with "board(s) not in <config>", re-run it WITHOUT the unknown names but keep the FULL board list on the hil_report call below — it emits a ran:false entry for every board you name, so the unknown ones surface as "no report row" instead of costing the whole batch. ' + 'Use the config for this host (hostname first). Run hil_test.py as a BACKGROUND Bash task and wait for it (a stuck fleet runs to its pool guard, 60 min by default — beyond any foreground timeout); never cancel it early. ' + 'On non-lock failures retry ONCE from the re-run spec hil_test.py just wrote — `<config>.failed`, which already begins with --accumulate — adding -v. A usbtest battery that produced per-case verdicts is NOT auto-retried, so its result already stands. ' + 'THEN, from the directory the run wrote its report to, produce the results with:\n' + - ` python3 test/hil/helper/hil_summary.py <the config you used> ${boards.map((b) => `-b ${b}`).join(' ')}\n` + - 'Return its `results` array and `banner` EXACTLY as printed — do not retype, reword, re-order or "correct" them, and never transcribe the markdown table instead. ' + + ` python3 test/hil/helper/hil_report.py <the config you used> ${boards.map((b) => `-b ${b}`).join(' ')}\n` + + 'Return its `results` array, `banner` and `caveat` EXACTLY as printed — do not retype, reword, re-order or "correct" them, and never transcribe the markdown table instead. ' + 'Add `wedged`: the board names whose board or fixture your run left unresponsive (usually none). That is your own observation and the one field you author; put `dmesg | tail -50` in your reply text for any board you list.', { label: boards.length === 1 ? `hil:${boards[0]}` : `hil:${boards.length} boards`, @@ -64,7 +68,7 @@ const runBoards = (boards, isRetry = false) => agent( }, ) -// A lookup, not a reconciliation: hil_summary.py emits exactly one entry per requested board, +// A lookup, not a reconciliation: hil_report.py emits exactly one entry per requested board, // so a missing entry means the operator dropped it rather than that the names disagree. const byBoard = (out) => new Map((out?.results || []) .filter((r) => r && typeof r.board === 'string') @@ -91,7 +95,9 @@ const results = args.boards.map((b) => { }) for (const r of results) log(`${r.board}: ${r.pass ? 'PASS' : r.locked ? 'LOCKED' : 'FAIL'}`) if (first?.banner) log(`report banner: ${first.banner.trim().split('\n')[0]}`) +if (first?.caveat) log(`report caveat: ${first.caveat.trim().split('\n')[0]}`) +let runCaveat = first?.caveat || '' // A concurrent CI job may have held some boards (its hil_test.py flock). // CI finishes a board in minutes — retry locked boards once, at the end. if (!args.force) { @@ -116,19 +122,26 @@ if (!args.force) { } log(`${b}: retry ${results[i].pass ? 'PASS' : 'FAIL'}`) } + // the retry's own run-level verdict, not the first attempt's: a retry that abandoned + // or aborted must sink the run even though its rows may all say pass. + if (again?.caveat) runCaveat = again.caveat } } -// pass/wedged/locked in one place so it can be exercised without running an agent -const summarize = (rs, force) => ({ - pass: rs.every((r) => r.pass), +// pass/wedged/locked in one place so it can be exercised without running an agent. +// `caveat` is a RUN-level verdict and must gate `pass`: on the abandon and no-boards +// paths every row can legitimately say pass while the run itself failed (hil_test.py +// os._exit(1) -- a red job), so per-row agreement alone published those runs as green. +const summarize = (rs, force, caveat) => ({ + pass: rs.every((r) => r.pass) && !/^\*\*HIL run (abandoned|aborted|selected no boards)/m + .test(caveat || ''), wedged: rs.filter((r) => r.wedged).map((r) => r.board), locked: force ? [] : rs.filter((r) => !r.pass && r.locked).map((r) => r.board), }) -const { pass, wedged, locked } = summarize(results, args.force) +const { pass, wedged, locked } = summarize(results, args.force, runCaveat) if (wedged.length) log(`WEDGED boards needing usb-kernel-recover: ${wedged.join(', ')}`) // Workers cannot prompt the user — surface still-locked boards for the main // session to ask: force (re-invoke with force: true), wait, or accept. if (locked.length) log(`still locked after retry: ${locked.join(', ')} — ask the user: force / keep waiting / accept`) -return { pass, results, wedged, locked } +return { pass, results, wedged, locked, caveat: runCaveat } diff --git a/.claude/workflows/pr-babysit.js b/.claude/workflows/pr-babysit.js index 406213a5f..8438d4d04 100644 --- a/.claude/workflows/pr-babysit.js +++ b/.claude/workflows/pr-babysit.js @@ -1,47 +1,55 @@ export const meta = { name: 'pr-babysit', - description: 'Drive a PR to green: pr-monitor triage (CI + bot reviews), port-dev fixes for validated findings, driver-reviewer verification, one commit+push per cycle', + description: 'Drive a PR to green: a fast review lane (validate bot findings, fix, push without waiting on CI) overlapped with a CI-watch lane; code-writer fixes, code-verifier verification, at most one push per lane per cycle', whenToUse: 'After opening a PR, from a checkout of the PR branch. Default is a dry run (fixes left uncommitted, nothing posted); passing autoPush: true is the explicit authorization for pushes and PR comments.', phases: [{ title: 'Triage' }, { title: 'Fix' }, { title: 'Verify' }, { title: 'Push' }], } -// args: { pr: number, maxCycles?: number, autoPush?: boolean (default false = dry run) } +// args: { pr: number, maxCycles?: number, autoPush?: boolean (default false = dry run), +// checkoutDir?: string (PR branch checkout; default: the session working dir) } if (typeof args === 'string') { try { args = JSON.parse(args) } catch { /* not JSON: shape check below reports it */ } } if (!args || !args.pr) { - throw new Error('args must be { pr: number, maxCycles?, autoPush? }; run from a checkout of the PR branch') + throw new Error('args must be { pr: number, maxCycles?, autoPush?, checkoutDir? }; run from the PR branch checkout or point checkoutDir at it') } args.pr = Number(args.pr) if (!Number.isInteger(args.pr) || args.pr <= 0) { throw new Error('args.pr must be a positive integer PR number') } +const checkoutDir = args.checkoutDir || '.' +if (typeof checkoutDir !== 'string' || checkoutDir.includes("'")) { + throw new Error('checkoutDir must be a plain path string') +} +const IN_CHECKOUT = checkoutDir === '.' ? 'The working tree IS the PR checkout. ' + : `The PR branch checkout is at ${checkoutDir} - run every git/build/file command there, not in the session directory. ` const maxCycles = args.maxCycles ?? 3 if (!Number.isInteger(maxCycles) || maxCycles < 1) { throw new Error('maxCycles must be an integer >= 1') } -const TRIAGE = { +const CI = { type: 'object', additionalProperties: false, - required: ['ci', 'findings', 'replies', 'done'], + required: ['status', 'infraRerun', 'realFailures'], properties: { - ci: { - type: 'object', additionalProperties: false, - required: ['status', 'infraRerun', 'realFailures'], - properties: { - status: { type: 'string', enum: ['green', 'red', 'running'] }, - infraRerun: { type: 'array', items: { type: 'string' } }, - realFailures: { - type: 'array', - items: { - type: 'object', additionalProperties: false, - required: ['check', 'firstError', 'files'], - properties: { - check: { type: 'string' }, firstError: { type: 'string' }, - files: { type: 'array', items: { type: 'string' } }, - }, - }, + status: { type: 'string', enum: ['green', 'red', 'running'] }, + infraRerun: { type: 'array', items: { type: 'string' } }, + realFailures: { + type: 'array', + items: { + type: 'object', additionalProperties: false, + required: ['check', 'firstError', 'files', 'rigSide'], + properties: { + check: { type: 'string' }, firstError: { type: 'string' }, + files: { type: 'array', items: { type: 'string' } }, + rigSide: { type: 'boolean' }, }, }, }, + }, +} +const REVIEWS = { + type: 'object', additionalProperties: false, + required: ['findings', 'replies', 'done'], + properties: { findings: { type: 'array', items: { @@ -84,6 +92,19 @@ const OP = { required: ['pass', 'detail'], properties: { pass: { type: 'boolean' }, detail: { type: 'string' } }, } +const SCOPE = { + type: 'object', additionalProperties: false, + required: ['files'], + properties: { files: { type: 'array', items: { type: 'string' } } }, +} +const OPIDS = { + type: 'object', additionalProperties: false, + required: ['pass', 'detail', 'doneIds'], + properties: { + pass: { type: 'boolean' }, detail: { type: 'string' }, + doneIds: { type: 'array', items: { type: 'integer' } }, + }, +} // Marking a review thread resolved has no REST endpoint — it needs the // GraphQL resolveReviewThread mutation. Shared recipe handed to the posting @@ -107,120 +128,282 @@ const postReplyRecipe = (noun) => const history = [] const repliedIds = new Set() // issue comments can't be thread-resolved, so they re-harvest every cycle — never reply twice -for (let cycle = 1; cycle <= maxCycles; cycle++) { - const t = await agent( - `Triage PR #${args.pr}. If checks are still running, wait for them first (gh pr checks ${args.pr} --watch as a BACKGROUND Bash task; the foreground timeout is capped at 10 min). ` + - 'Then follow your triage procedure: classify CI failures, re-run infra ones, harvest and adversarially validate bot review findings, draft replies for invalid/stale ones.', - { label: `triage#${cycle}`, phase: 'Triage', agentType: 'pr-monitor', schema: TRIAGE }, - ) - if (!t) { - history.push({ cycle, error: 'pr-monitor died' }) - return { pass: false, cycles: cycle, history, reason: 'pr-monitor-died' } - } - const entry = { cycle, triage: t } - history.push(entry) - // Post drafted replies to REFUTED findings as soon as triage produces them — - // decoupled from fixing/pushing so done/unactionable cycles still post. - // Reply AND resolve the thread. Outward-facing, so gated on autoPush. - const freshReplies = t.replies.filter(r => !repliedIds.has(r.commentId)) - if (freshReplies.length > 0 && args.autoPush === true) { - const posted = await agent( - `Reply to and resolve these refuted review comments on PR #${args.pr}. For each: ${postReplyRecipe('reply')}` + - `Replies: ${JSON.stringify(freshReplies)}. pass=true only if every reply was posted and every inline thread resolved; detail = what went where.`, - { label: `replies#${cycle}`, phase: 'Push', model: 'sonnet', schema: OP }, - ) - // attempted counts as replied: better to drop a failed reply than spam duplicates - freshReplies.forEach(r => repliedIds.add(r.commentId)) - if (!posted || !posted.pass) log(`cycle ${cycle}: refuted reply/resolve incomplete — ${posted ? posted.detail : 'agent died'}`) - } +// Backoff between cycles that have nothing to do but wait. Degrades to a no-op +// rather than throwing if the workflow host has no timer. +const nap = (ms) => new Promise(res => { if (typeof setTimeout === 'function') setTimeout(res, ms); else res() }) - if (t.done) { - log(`cycle ${cycle}: PR is green with no unresolved valid findings`) - return { pass: true, cycles: cycle, history } +// Canonicalize a repo-relative path for set/collision comparison: resolve ./.. +// segments, unify separators; '' for anything that escapes the repo or uses +// characters no repo path does (also makes the path shell-safe to interpolate). +const canon = (p) => { + const s = String(p).trim().replace(/\\/g, '/') + // Absolute (CI-runner) paths: reject rather than corrupt into a bogus relative + // path — the file-less group then routes through the scoper, which recovers the + // real repo path and is existence-checked. + if (s.startsWith('/')) return '' + const out = [] + for (const seg of s.split('/')) { + if (!seg || seg === '.') continue + if (seg === '..') { if (out.pop() === undefined) return '' } else out.push(seg) } + const c = out.join('/') + return /^[A-Za-z0-9._+/-]+$/.test(c) ? c : '' +} - // Group actionable work by top-level scope (plain JS — no model tokens). +// Group actionable notes by top-level scope (plain JS — no model tokens). +const groupWork = (notes) => { const groups = new Map() - const groupOf = (key) => { + for (const n of notes) { + const key = (canon(n.scopeFile) || n.scopeFile).split('/').slice(0, 3).join('/') if (!groups.has(key)) groups.set(key, { key, files: new Set(), notes: [] }) - return groups.get(key) + const g = groups.get(key) + n.files.forEach(f => { const c = canon(f); if (c) g.files.add(c) }) + g.notes.push(n.text) } - for (const f of t.findings.filter(x => x.verdict === 'valid')) { - const g = groupOf(f.file.split('/').slice(0, 3).join('/')) - g.files.add(f.file) - g.notes.push(`${f.file}:${f.line} [${f.source}] ${f.claim} — hint: ${f.fixHint}`) + return [...groups.values()] +} + +// Fix + verify one work list; returns { ok, fixes } — ok only if every group +// was scoped, fixed by a live worker, AND passed code-verifier verification. +const fixAndVerify = async (workIn) => { + // code-writer's contract needs an explicit file set: a group whose notes named no + // files (a CI failure whose log yielded no paths) is scoped by a dedicated agent + // first; if that fails too, the group is withheld (ok=false → human review) rather + // than dispatched with an invalid scope. + const fileless = workIn.filter(w => w.files.size === 0) + await parallel(fileless.map(w => () => + agent( + `${IN_CHECKOUT}Determine which repo files must change to address these notes (read the code; if a note is a CI failure, read its CI log too):\n- ${w.notes.join('\n- ')}\n` + + 'files = repo-relative paths; empty only if genuinely undeterminable.', + { label: `scope:${w.key}`, phase: 'Fix', model: 'sonnet', schema: SCOPE }, + ).then(s => s && s.files.forEach(f => { const c = canon(f); if (c) w.files.add(c) })))) + // Scoped paths are model output: keep only what git ls-files confirms exists. + // The check is executed (by a mechanical agent) and intersected here — a dead + // checker drops every candidate, so unconfirmed groups fall through to withheld. + const candidates = [...new Set(fileless.flatMap(w => [...w.files]))] + if (candidates.length > 0) { + const v = await agent( + `${IN_CHECKOUT}Run exactly: git ls-files -- ${candidates.join(' ')}\nReturn files = the paths that command printed, verbatim — no additions, no substitutions.`, + { label: 'scope:verify', phase: 'Fix', model: 'haiku', schema: SCOPE }, + ) + const exists = new Set((v ? v.files : []).map(canon)) + for (const w of fileless) for (const f of [...w.files]) + if (!exists.has(f)) { w.files.delete(f); log(`scope:${w.key}: dropped ${f} — not confirmed as a repo file`) } } - for (const rf of t.ci.realFailures) { - const g = groupOf((rf.files[0] || rf.check).split('/').slice(0, 3).join('/')) - rf.files.forEach(x => g.files.add(x)) - g.notes.push(`CI ${rf.check}: ${rf.firstError}`) + const unscoped = workIn.filter(w => w.files.size === 0) + for (const w of unscoped) log(`fix for ${w.key}: no file scope determinable — withheld for human review`) + // Scoping can make groups overlap (two checks resolving to the same file); merge + // intersecting groups (to closure) so two fixers never edit one file concurrently. + const work = [] + for (let g of workIn.filter(w => w.files.size > 0)) { + for (let i; (i = work.findIndex(m => [...g.files].some(f => m.files.has(f)))) >= 0;) { + const [m] = work.splice(i, 1) + g.files.forEach(f => m.files.add(f)); m.notes.push(...g.notes); m.key = `${m.key}+${g.key}` + g = m + } + work.push(g) } - const work = [...groups.values()] - - if (work.length === 0) { - if (t.ci.status === 'running' || t.ci.infraRerun.length > 0) { - log(`cycle ${cycle}: only infra re-runs in flight — next cycle waits on them`) - continue + // HIL rig rosters (test/hil/*.json) describe physical hardware the user owns: + // never edit them autonomously — skipping/reshaping tests there papers over a + // failing fixture. A failure that needs hardware swapped or re-cabled stays RED + // for the user; roster edits happen only with the user's explicit approval. + const withheld = [] + for (const w of work) { + for (const f of [...w.files]) if (/^test\/hil\/[^/]+\.json$/.test(f)) { + w.files.delete(f) + log(`fix for ${w.key}: ${f} is a HIL rig config — edits need user approval, dropped from scope`) + } + if (w.files.size === 0) { + withheld.push(w) + log(`fix for ${w.key}: only a HIL rig config edit would address it — leaving red for the user`) } - log(`cycle ${cycle}: nothing actionable`) - return { pass: false, cycles: cycle, history, reason: 'unactionable' } } - + for (const w of withheld) work.splice(work.indexOf(w), 1) + const scopeOf = (w) => [...w.files].join(', ') const fixes = await pipeline( work, w => agent( - `Fix the following issues on the current PR branch (the working tree IS the PR checkout).\n` + - `Scope: ${[...w.files].join(', ')}\nIssues:\n- ${w.notes.join('\n- ')}`, - { label: `fix:${w.key}`, phase: 'Fix', agentType: 'port-dev', effort: 'xhigh', schema: DEV }, + `Fix the following issues on the PR branch. ${IN_CHECKOUT}\n` + + 'Constraint: never modify test/hil/*.json (HIL rig hardware config) — a failure that needs hardware swapped/changed stays red for the user.\n' + + `Scope: ${scopeOf(w)}\nIssues:\n- ${w.notes.join('\n- ')}`, + { label: `fix:${w.key}`, phase: 'Fix', agentType: 'code-writer', schema: DEV }, ), (fix, w) => fix && agent( - `Verify the uncommitted changes for ${[...w.files].join(', ')} (use git diff -- <files>, and read any newly created untracked files directly) address these issues:\n- ${w.notes.join('\n- ')}\n` + + `${IN_CHECKOUT}Verify the uncommitted changes for ${scopeOf(w)} (use git diff -- <the files above>, and read any newly created untracked files directly) address these issues:\n- ${w.notes.join('\n- ')}\n` + 'Return {"addresses": bool, "reason": string}.', - { label: `check:${w.key}`, phase: 'Verify', agentType: 'driver-reviewer', effort: 'xhigh', schema: CHECK }, + { label: `check:${w.key}`, phase: 'Verify', agentType: 'code-verifier', schema: CHECK }, ).then(v => ({ ...fix, addresses: !!(v && v.addresses), checkReason: v ? v.reason : 'verifier died' })), ) - const aliveFixes = fixes.filter(Boolean) - if (aliveFixes.length < work.length) log(`${work.length - aliveFixes.length} fix group(s) lost to dead workers`) - entry.fixes = aliveFixes - - if (args.autoPush !== true) { - log('autoPush not set: fixes left uncommitted in the working tree (dry run)') - return { pass: false, cycles: cycle, history, dryRun: true } - } - - // Verification gates the push: never push a cycle containing an unverified - // fix or the partial edits of a dead worker. - const unverified = aliveFixes.filter(f => f.addresses !== true) - if (aliveFixes.length < work.length || unverified.length > 0) { - for (const f of unverified) log(`fix for ${f.item}: failed verification — ${f.checkReason}`) - log(`cycle ${cycle}: fixes left uncommitted for human review — not pushing unverified changes`) - return { pass: false, cycles: cycle, history, reason: 'fix-verification-failed' } - } + const alive = fixes.filter(Boolean) + if (alive.length < work.length) log(`${work.length - alive.length} fix group(s) lost to dead workers`) + const unverified = alive.filter(f => f.addresses !== true) + for (const f of unverified) log(`fix for ${f.item}: failed verification — ${f.checkReason}`) + return { ok: unscoped.length === 0 && withheld.length === 0 && alive.length === work.length && unverified.length === 0, fixes: alive } +} +// Verification gates every push: never push unverified or partial edits. +const commitAndPush = async (cycle, what) => { const push = await agent( - `On the current PR branch: commit ALL working-tree changes as ONE commit (imperative message summarizing the cycle-${cycle} fixes for PR #${args.pr}, repo commit conventions), ` + + `${IN_CHECKOUT}On the PR branch: commit ALL working-tree changes as ONE commit (imperative message summarizing the cycle-${cycle} ${what} fixes for PR #${args.pr}, repo commit conventions), ` + "then push to the PR's remote branch. pass=true only if commit AND push succeeded; detail = pushed SHA.", - { label: `push#${cycle}`, phase: 'Push', model: 'sonnet', schema: OP }, + { label: `push#${cycle}-${what}`, phase: 'Push', model: 'sonnet', schema: OP }, + ) + return push && push.pass ? push : null +} + +for (let cycle = 1; cycle <= maxCycles; cycle++) { + // Two independent lanes, launched together. The review lane never waits on + // CI: it validates, fixes, and pushes while the CI lane is still watching. + const ciPromise = agent( + `Watch CI for PR #${args.pr} per your procedure; wait for pending checks.`, + { label: `ci#${cycle}`, phase: 'Triage', agentType: 'pr-ci-watcher', schema: CI }, + ).catch(e => { log(`cycle ${cycle}: pr-ci-watcher errored — ${e && e.message}`); return null }) + // Every early return below leaves the loop while the CI lane is still + // running: settle it first so no CI agent outlives the workflow. + const stopWith = async (result) => { await ciPromise; return result } + + const r = await agent( + `Validate the bot review findings on PR #${args.pr} per your procedure. ${IN_CHECKOUT}`, + { label: `reviews#${cycle}`, phase: 'Triage', agentType: 'pr-review-validator', schema: REVIEWS }, ) - if (!push || !push.pass) { - log(`cycle ${cycle}: push failed — stopping`) - return { pass: false, cycles: cycle, history, reason: 'push-failed' } + if (!r) { + history.push({ cycle, error: 'pr-review-validator died' }) + return await stopWith({ pass: false, cycles: cycle, history, reason: 'review-validator-died' }) } + const entry = { cycle, reviews: r } + history.push(entry) + // Outward reply/resolve attempts this cycle that did not fully complete; a green + // PR must not terminate the loop while any remain, or the retry never happens. + let pendingReplies = 0 - // The valid bot findings were fixed and pushed — answer each inline comment - // with what changed and resolve its thread. CI-failure work has no comment. - const fixed = t.findings.filter(x => x.verdict === 'valid') - if (fixed.length > 0) { + // Post drafted replies to REFUTED findings immediately. Outward-facing, + // so gated on autoPush. + const freshReplies = r.replies.filter(x => !repliedIds.has(x.commentId)) + if (freshReplies.length > 0 && args.autoPush === true) { + const posted = await agent( + `Reply to and resolve these refuted review comments on PR #${args.pr}. For each: ${postReplyRecipe('reply')}` + + 'If a thread already carries an identical reply of ours (a prior attempt that posted but failed to resolve), do not repost — just resolve it. ' + + `Replies: ${JSON.stringify(freshReplies)}. pass=true only if every reply was posted and every inline thread resolved; detail = what went where. ` + + 'doneIds = the commentIds fully handled: reply posted (or already present) AND (thread resolved, or an issue comment with no thread to resolve).', + { label: `replies#${cycle}`, phase: 'Push', model: 'sonnet', schema: OPIDS }, + ) + // Per-id accounting, matching the resolve path: only fully handled ids are marked + // replied; a failed reply/resolve stays fresh and retries next cycle (the prompt's + // already-present check keeps the retry from duplicating the reply). + for (const id of (posted && posted.doneIds) || []) repliedIds.add(id) + pendingReplies += freshReplies.filter(x => !repliedIds.has(x.commentId)).length + if (!posted || !posted.pass) log(`cycle ${cycle}: refuted reply/resolve incomplete — ${posted ? posted.detail : 'agent died'}`) + } + + // ---- review lane: fix + push without waiting for CI ---- + const validFindings = r.findings.filter(x => x.verdict === 'valid') + let reviewPushed = false + if (validFindings.length > 0) { + const work = groupWork(validFindings.map(f => ({ + scopeFile: f.file, files: [f.file], + text: `${f.file}:${f.line} [${f.source}] ${f.claim} — hint: ${f.fixHint}`, + }))) + const { ok, fixes } = await fixAndVerify(work) + entry.reviewFixes = fixes + if (args.autoPush !== true) { + log('autoPush not set: review-lane fixes left uncommitted (dry run)') + return await stopWith({ pass: false, cycles: cycle, history, dryRun: true }) + } + if (!ok) { + log(`cycle ${cycle}: review-lane fixes left uncommitted for human review — not pushing unverified changes`) + return await stopWith({ pass: false, cycles: cycle, history, reason: 'fix-verification-failed' }) + } + const push = await commitAndPush(cycle, 'review') + if (!push) { + log(`cycle ${cycle}: review-lane push failed — stopping`) + return await stopWith({ pass: false, cycles: cycle, history, reason: 'push-failed' }) + } + reviewPushed = true const resolved = await agent( `The fixes for PR #${args.pr}'s valid review findings were just committed and pushed (${push.detail}). ` + `For each finding below: ${postReplyRecipe('fix note')}` + 'Each reply states the finding is fixed in the pushed commit, with one line on the change. ' + - `Findings: ${JSON.stringify(fixed.map(f => ({ commentId: f.commentId, file: f.file, line: f.line, claim: f.claim, fixHint: f.fixHint })))}. ` + - 'pass=true only if every reply was posted and every thread resolved; detail = what went where.', - { label: `resolve#${cycle}`, phase: 'Push', model: 'sonnet', schema: OP }, + `Findings: ${JSON.stringify(validFindings.map(f => ({ commentId: f.commentId, file: f.file, line: f.line, claim: f.claim, fixHint: f.fixHint })))}. ` + + 'pass=true only if every reply was posted and every thread resolved; detail = what went where. ' + + 'doneIds = the commentIds fully handled: reply posted AND (thread resolved, or an issue comment with no thread to resolve).', + { label: `resolve#${cycle}`, phase: 'Push', model: 'sonnet', schema: OPIDS }, ) + // Per-id accounting: a fully handled finding never re-replies (an issue comment + // has no thread to resolve, so it re-harvests as stale next cycle and would get + // a duplicate "fixed" note); an unfinished one stays out of repliedIds so its + // reply/resolve is retried next cycle instead of silently abandoned. + for (const id of (resolved && resolved.doneIds) || []) repliedIds.add(id) + pendingReplies += validFindings.filter(f => !repliedIds.has(f.commentId)).length if (!resolved || !resolved.pass) log(`cycle ${cycle}: fixed reply/resolve incomplete — ${resolved ? resolved.detail : 'agent died'}`) } + + // ---- CI lane result ---- + const c = await ciPromise + entry.ci = c + if (!c) { + log(`cycle ${cycle}: pr-ci-watcher died — re-arming`) + continue + } + if (reviewPushed) { + // The push restarted CI: this cycle's CI verdict is superseded. Re-arm; + // next cycle's ci#N watches the fresh run. + log(`cycle ${cycle}: review-lane push superseded the CI run — re-arming`) + continue + } + const rigSide = c.realFailures.filter(rf => rf.rigSide) + for (const rf of rigSide) log(`cycle ${cycle}: rig-side CI failure (not fixing): ${rf.check} — ${rf.firstError.slice(0, 120)}`) + const fixable = c.realFailures.filter(rf => !rf.rigSide) + if (fixable.length > 0) { + const work = groupWork(fixable.map(rf => ({ + scopeFile: rf.files[0] || rf.check, files: rf.files, + text: `CI ${rf.check}: ${rf.firstError}`, + }))) + const { ok, fixes } = await fixAndVerify(work) + entry.ciFixes = fixes + if (args.autoPush !== true) { + log('autoPush not set: CI-lane fixes left uncommitted (dry run)') + return { pass: false, cycles: cycle, history, dryRun: true } + } + if (!ok) { + log(`cycle ${cycle}: CI-lane fixes left uncommitted for human review — not pushing unverified changes`) + return { pass: false, cycles: cycle, history, reason: 'fix-verification-failed' } + } + if (!(await commitAndPush(cycle, 'ci'))) { + log(`cycle ${cycle}: CI-lane push failed — stopping`) + return { pass: false, cycles: cycle, history, reason: 'push-failed' } + } + continue // pushed: fresh CI run next cycle + } + if (r.done && c.status === 'green') { + if (pendingReplies > 0) { + log(`cycle ${cycle}: PR green but ${pendingReplies} reply/resolve unfinished — re-arming to retry`) + continue + } + log(`cycle ${cycle}: PR is green with no unresolved valid findings`) + return { pass: true, cycles: cycle, history } + } + if (r.done && rigSide.length > 0 && fixable.length === 0 && c.infraRerun.length === 0 && c.status !== 'running') { + log(`cycle ${cycle}: CI red only from rig-side failures — human/rig attention needed, nothing to fix in the PR`) + return { pass: false, cycles: cycle, history, reason: 'ci-red-rig-side' } + } + if (c.status === 'running' || c.infraRerun.length > 0) { + log(`cycle ${cycle}: CI still settling (${c.infraRerun.length} infra re-run(s)) — re-arming`) + continue + } + if (!r.done) { + // A bot has not reported for this head SHA yet. With CI already green there is + // nothing else to wait on, so back off before re-arming or the cycle budget + // burns on back-to-back re-harvests of the same unchanged PR. + if (cycle < maxCycles) { + log(`cycle ${cycle}: auto-review still pending — re-arming after a wait`) + await nap(60000 * cycle) // no wait on the last cycle: nothing would re-check after it + } else { + log(`cycle ${cycle}: auto-review still pending — cycle budget exhausted`) + } + continue + } + log(`cycle ${cycle}: nothing actionable`) + return { pass: false, cycles: cycle, history, reason: 'unactionable' } } return { pass: false, cycles: maxCycles, history, reason: 'maxCycles reached' } diff --git a/.claude/workflows/test-hil-validate.mjs b/.claude/workflows/test-hil-validate.mjs index db73095f6..e8be57cfc 100644 --- a/.claude/workflows/test-hil-validate.mjs +++ b/.claude/workflows/test-hil-validate.mjs @@ -4,7 +4,7 @@ // `board locked` out of a prose detail, folding rows, keeping a wedged flag alive -- produced // a defect in each of four review rounds, including a test that asserted an invariant using // the one input shape that could not break it. That logic now lives in -// test/hil/helper/hil_summary.py, where the roster is, and arrives here as fields. What is +// test/hil/helper/hil_report.py, where the roster is, and arrives here as fields. What is // left is a lookup and a verdict, and this pins both. // // Run: node .claude/workflows/test-hil-validate.mjs @@ -23,6 +23,11 @@ const cut = (start, end) => { } const body = cut('const byBoard =', 'const first = await runBoards') + cut('const summarize =', 'const { pass, wedged, locked } =') +// more than one schema declares `required:`; pick the HIL one by its contents +const HIL_REQUIRED = (src.match(/required: \[[^\]]*\]/g) || []) + .map((m) => m.replace('required: ', '').replace(/'/g, '"')) + .map((m) => JSON.parse(m)) + .find((a) => a.includes('wedged')) || [] const { byBoard, summarize, wedgedFor } = new Function(`${body}; return { byBoard, summarize, wedgedFor }`)() let failed = 0 @@ -65,5 +70,27 @@ check('wedged surfaces', summarize([R('a', false, false, true)], false).wedged, check('a wedged board that passed still surfaces', summarize([R('a', true, false, true)], false).wedged, ['a']) +// A run-level caveat outranks per-row agreement: on the abandon and no-boards paths every +// row can legitimately pass while hil_test.py exits non-zero. Row agreement alone published +// those runs green. +check('all rows pass and no caveat is a pass', + summarize([R('a', true), R('b', true)], false, '').pass, true) +check('an abandoned run is not a pass', + summarize([R('a', true)], false, + '**HIL run abandoned: the worker pool would not shut down.** x').pass, false) +check('an aborted run is not a pass', + summarize([R('a', true)], false, '**HIL run aborted: a worker raised RuntimeError**').pass, + false) +check('a no-boards run is not a pass', + summarize([R('a', true)], false, '**HIL run selected no boards.** filters emptied').pass, + false) +check('a rig-health note is NOT a caveat and does not fail the run', + summarize([R('a', true)], false, '> **Rig note.** 2 process(es) in D state').pass, true) +check('a retry that abandoned sinks the run even with all rows passing', + summarize([R('a', true)], false, + '**HIL run abandoned: the worker pool would not shut down.** retry').pass, false) +check('an omitted caveat cannot silently disable the gate (schema requires it)', + HIL_REQUIRED.includes('caveat'), true) + console.log(failed ? `\n${failed} FAILED` : '\nall checks passed') process.exit(failed ? 1 : 0) diff --git a/.claude/workflows/validate.js b/.claude/workflows/validate.js index 522548ee6..773dd47ad 100644 --- a/.claude/workflows/validate.js +++ b/.claude/workflows/validate.js @@ -1,18 +1,33 @@ export const meta = { name: 'validate', - description: 'Pre-PR software validation: unit tests + per-board build sweeps + code-size compare + PVS, in parallel, joined into one verdict', + description: 'Pre-PR software validation loop: unit tests + per-board build sweeps + code-size compare + PVS + diff reviews (claude + codex) in parallel; a red verdict dispatches a fix agent for the confirmed findings, then the affected stages re-run — up to maxCycles (default 5) validation passes; a fix that edits a workflow file stops with restartRequired so the caller re-invokes it', whenToUse: 'Before opening or updating a PR, after any non-trivial change', - phases: [{ title: 'Validate', detail: 'unit + builds + size + pvs in parallel' }], + phases: [ + { title: 'Validate', detail: 'unit + builds + size + pvs + reviews in parallel' }, + { title: 'Fix', detail: 'one fix agent per red cycle; commits, then affected stages re-run' }, + ], } -// args: { boards: string[], examples?: string, base?: string, skip?: ('unit'|'size'|'pvs')[] } +// args: { boards: string[], examples?: string, base?: string, +// skip?: ('unit'|'size'|'pvs'|'review'|'codex')[], maxCycles?: number } if (typeof args === 'string') { try { args = JSON.parse(args) } catch { /* not JSON: shape check below reports it */ } } if (!args || !Array.isArray(args.boards) || args.boards.length === 0) { - throw new Error('args must be { boards: string[], examples?, base?, skip? }') + throw new Error('args must be { boards: string[], examples?, base?, skip?, maxCycles? }') } +if (args.maxCycles !== undefined && (!Number.isInteger(args.maxCycles) || args.maxCycles < 1)) { + throw new Error('maxCycles must be an integer >= 1') +} +const maxCycles = args.maxCycles ?? 5 const skip = args.skip || [] for (const s of skip) log(`stage skipped by request: ${s}`) const base = args.base || 'master' +// Every stage agent re-resolves `base` in every cycle, and the fixer commits +// between cycles: a moving expression (HEAD~1, @{u}, a ^/~ walk) would advance +// with each fix commit, so cycle 2 would review only the fix and silently drop +// the original branch changes. Accept stationary refs only. +if (/(^|[^\w/-])HEAD/.test(base) || /[~^]/.test(base) || base.includes('@{')) { + throw new Error(`base must be a fixed ref (sha or branch name), not the moving expression "${base}" — resolve it with git rev-parse first`) +} const clip = (s, n = 800) => s.length > n ? s.slice(0, n) + ` …[truncated ${s.length - n} chars]` : s @@ -56,45 +71,268 @@ const PVS = { }, } -const thunks = [] +const REVIEW = { + type: 'object', additionalProperties: false, + required: ['pass', 'findings', 'detail'], + properties: { + pass: { type: 'boolean' }, + findings: { + type: 'array', + items: { + type: 'object', additionalProperties: false, + required: ['file', 'line', 'severity', 'summary'], + properties: { + file: { type: 'string' }, line: { type: 'integer' }, + severity: { type: 'string' }, summary: { type: 'string' }, + }, + }, + }, + detail: { type: 'string' }, + }, +} + +const FIX = { + type: 'object', additionalProperties: false, + required: ['changed', 'commit', 'files', 'summary'], + properties: { + changed: { type: 'boolean' }, commit: { type: 'string' }, + files: { type: 'array', items: { type: 'string' } }, + summary: { type: 'string' }, + }, +} -if (!skip.includes('unit')) thunks.push(() => - agent( +// read back out of git what the fix commit actually touched +const PATHS = { + type: 'object', additionalProperties: false, + required: ['paths', 'isHead'], + properties: { + paths: { type: 'array', items: { type: 'string' } }, + isHead: { type: 'boolean' }, + }, +} + +// gate helpers — enforced here, never trusted from the agents +const confirmedReview = f => + /^confirmed/i.test(f.severity) && !/quality|simplification|style/i.test(f.severity) +const codexBlocking = f => /\bP[01]\b/i.test(f.severity) + +// --------------------------------------------------------------------------- +// Stage builders, parameterized so later cycles can re-run a subset. Stage +// names: 'unit', 'build:<board>', 'size', 'pvs', 'review', 'codex'. +// --------------------------------------------------------------------------- +const stageNames = [] +if (!skip.includes('unit')) stageNames.push('unit') +for (const b of args.boards) stageNames.push(`build:${b}`) +if (!skip.includes('size')) stageNames.push('size') +if (!skip.includes('pvs')) stageNames.push('pvs') +if (!skip.includes('review')) stageNames.push('review') +if (!skip.includes('codex')) stageNames.push('codex') + +function stageThunk(name, cycle) { + const label = (cycle > 1 ? `c${cycle}:` : '') + name + // findings: [] so a dead review/codex stage flows through fixerEvidence() + // instead of throwing on f.findings inside the fix dispatch's catch + const died = { stage: name, pass: false, findings: [], detail: 'stage agent died' } + + if (name === 'unit') return () => agent( 'Run the TinyUSB unit tests: cd test/unit-test && ceedling test:all. ' + 'pass=true only if every test passes. detail = the ceedling summary line, or the first failing test output.', - { label: 'unit', phase: 'Validate', model: 'haiku', schema: STAGE }, - ).then(r => r && { stage: 'unit', ...r })) + { label, phase: 'Validate', model: 'haiku', schema: STAGE }, + ).then(r => r ? { stage: name, ...r } : died).catch(() => died) -for (const b of args.boards) thunks.push(() => - agent( - `Build TinyUSB examples for board ${b}` + (args.examples ? ` (only: ${args.examples})` : ' (full example set)') + '.', - { label: `build:${b}`, phase: 'Validate', agentType: 'builder', schema: BUILD }, - ).then(r => r && { - stage: `build:${b}`, pass: r.pass, - detail: r.pass ? `${r.builtCount} examples built` : clip(JSON.stringify(r.failures)), - })) + if (name.startsWith('build:')) { + const b = name.slice('build:'.length) + return () => agent( + `Build TinyUSB examples for board ${b}` + (args.examples ? ` (only: ${args.examples})` : ' (full example set)') + '.', + { label, phase: 'Validate', agentType: 'builder', schema: BUILD }, + ).then(r => r ? { + stage: name, pass: r.pass, + detail: r.pass ? `${r.builtCount} examples built` : clip(JSON.stringify(r.failures)), + } : died).catch(() => died) + } -if (!skip.includes('size')) thunks.push(() => - agent( + if (name === 'size') return () => agent( `Compare TinyUSB code size against ${base}: python3 tools/metrics_compare_base.py --base-branch ${base} -b ${args.boards[0]} -e device/cdc_msc (exactly this command — no extra positional args). ` + 'The report lands in cmake-metrics/<board>/metrics_compare.md. pass=false only if the tool itself errors; ' + 'detail = the flash/RAM delta summary from the report (mention any example that grew).', - { label: 'size', phase: 'Validate', model: 'haiku', schema: STAGE }, - ).then(r => r && { stage: 'size', ...r })) + { label, phase: 'Validate', model: 'haiku', schema: STAGE }, + ).then(r => r ? { stage: name, ...r } : died).catch(() => died) -if (!skip.includes('pvs')) thunks.push(() => - agent( + if (name === 'pvs') return () => agent( `Run PVS-Studio static analysis for board ${args.boards[0]}, gating on files changed vs ${base}. ` + 'Parallel build agents are running — use your dedicated build dir, never cmake-build-<board>.', - { label: 'pvs', phase: 'Validate', agentType: 'static-analyzer', effort: 'low', schema: PVS }, - ).then(r => r && { - stage: 'pvs', pass: r.pass, + { label, phase: 'Validate', agentType: 'static-analyzer', effort: 'low', schema: PVS }, + ).then(r => r ? { + stage: name, pass: r.pass, detail: r.pass ? r.detail : clip(`${r.detail} ${JSON.stringify(r.changedFindings)}`), - })) + } : died).catch(() => died) + + if (name === 'review') return () => agent( + `Code-review this branch's diff vs ${base} (git diff ${base}...HEAD), coverage-first: walk every hunk, no spot checks. ` + + 'Find pass — candidate defects across all dimensions: correctness/logic, ISR & concurrency safety, ' + + 'memory/resource handling (bounds, leaks, no dynamic alloc), API contract & spec conformance, ' + + 'security of untrusted input parsing, behavior regressions; plus quality/simplification notes. ' + + 'Verify pass — adversarially check each candidate against the surrounding code: verdict CONFIRMED ' + + '(failing scenario constructed) or PLAUSIBLE (could not refute); report both, drop only refuted ones. ' + + 'Read-only: never apply fixes. severity = verdict plus category (e.g. "CONFIRMED correctness"). ' + + 'pass=false if any CONFIRMED correctness/safety/security bug survives; PLAUSIBLE and quality findings keep pass=true. ' + + 'detail = one-line review summary.', + { label, phase: 'Validate', model: 'opus', effort: 'high', schema: REVIEW }, + ).then(r => r ? { + stage: name, + pass: r.pass && !r.findings.some(confirmedReview), + findings: r.findings, detail: r.detail, + } : died).catch(() => died) + + if (name === 'codex') return () => agent( + `Run a Codex review of this branch's diff vs ${base}: ` + + `codex review --base ${base} -c model="gpt-5.6-sol" -c model_reasoning_effort="high" ` + + '(Bash timeout 600000; run from the repo root). Parse its output into findings; severity = Codex\'s priority label. ' + + 'pass=false only if Codex reports a correctness bug (P0/P1); style-level items keep pass=true. ' + + 'detail = Codex\'s overall verdict line. If the codex CLI is missing or the run errors, pass=false with the error in detail.', + { label, phase: 'Validate', model: 'haiku', schema: REVIEW }, + ).then(r => r ? { + stage: name, + pass: r.pass && !r.findings.some(codexBlocking), + findings: r.findings, detail: r.detail, + } : died).catch(() => died) + + throw new Error(`unknown stage ${name}`) +} + +// Only the material that FAILED the gate reaches the fixer: confirmed review +// findings, codex P0/P1, and failed unit/build/size/pvs stage evidence. +// PLAUSIBLE and quality findings stay report-only — fixing them here would +// churn style on an otherwise green branch. Bounded at the leaves (per-stage +// finding cap, clipped summaries/details) so the serialized JSON stays valid +// and every failed stage is represented — a document-level clip could cut +// mid-JSON and silently drop trailing stages. +function fixerEvidence(failures) { + return failures.map(f => { + const findings = (f.findings || []) + .filter(f.stage === 'review' ? confirmedReview : codexBlocking) + .slice(0, 10) + .map(x => ({ ...x, summary: clip(x.summary, 300) })) + if (f.stage === 'review' || f.stage === 'codex') + return { stage: f.stage, detail: clip(f.detail, 300), findings } + return { stage: f.stage, detail: clip(f.detail) } + }) +} + +function fixThunkPrompt(cycle, failures) { + return 'You are the fix agent of the validate loop, cycle ' + cycle + ', in this TinyUSB repo (work from the repo root). ' + + 'Failed stages: ' + failures.map(f => f.stage).join(', ') + '. ' + + 'The JSON below carries their evidence (review findings are pre-verified CONFIRMED, codex ones are P0/P1):\n' + + JSON.stringify(fixerEvidence(failures), null, 1) + '\n\n' + + 'For each item: verify it against the actual code first; fix the real ones with the smallest correct change, matching surrounding style. ' + + 'Skip anything that is an infrastructure failure rather than a code defect (missing CLI, tool crash, dead stage agent) and anything you can refute with evidence — say which and why in summary. ' + + 'Run the tests/suites covering what you changed. ' + + 'BEFORE editing anything, run git status --porcelain and record every path already dirty in either column ' + + '(staged or unstaged — those are someone else\'s in-flight edits, and git add <path> would sweep them into your commit). ' + + 'If any file you need to modify is in that set, edit nothing at all: return changed=false, naming the file in summary. ' + + 'Commit as ONE commit, staging ONLY the files you changed (git add <paths> — never git add -A or git commit -a). ' + + 'Message: imperative mood, subject like "validate: fix cycle ' + cycle + ' findings", ' + + 'NO trailers of any kind (no Co-Authored-By, no Claude-Session). Do NOT push. Never spawn subagents. ' + + 'Return: changed=true only if you committed; commit = the new sha (empty string if none); ' + + 'files = the commit\'s own paths, verbatim from git show --name-only --format= HEAD (empty if you did not commit); ' + + 'summary = one paragraph of what was fixed/skipped and why.' +} + +// --------------------------------------------------------------------------- +// The loop: validate → (red) fix → re-run affected stages, up to maxCycles +// validation passes. Reviews always re-run after a fix (their input is the +// diff, which just changed); unit/builds/size/pvs re-run only when the fix +// touched code they consume, or when they failed themselves. +// --------------------------------------------------------------------------- +const latest = new Map() // stage name -> most recent result +const history = [] +let toRun = new Set(stageNames) + +for (let cycle = 1; cycle <= maxCycles; cycle++) { + log(`cycle ${cycle}/${maxCycles}: running ${toRun.size}/${stageNames.length} stage(s)`) + const results = await parallel([...toRun].map(n => stageThunk(n, cycle))) + for (const r of results.filter(Boolean)) latest.set(r.stage, r) + const failures = [...latest.values()].filter(r => !r.pass) + const entry = { cycle, ran: [...toRun], failed: failures.map(f => f.stage), fix: null } + history.push(entry) + + if (failures.length === 0) { + log(`cycle ${cycle}: all stages green`) + return { pass: true, cycles: history, stages: [...latest.values()], failures: [] } + } + log(`cycle ${cycle}: ${failures.length} stage(s) failing: ${entry.failed.join(', ')}`) + if (cycle === maxCycles) break + + const fix = await (async () => { + try { + return await agent(fixThunkPrompt(cycle, failures), + { label: `c${cycle}:fix`, phase: 'Fix', model: 'sonnet', schema: FIX }) + } catch { return null } + })() + if (!fix) { entry.fix = 'fix agent died'; break } + entry.fix = { changed: fix.changed, commit: fix.commit, files: fix.files, summary: clip(fix.summary) } + if (!fix.changed) { + // Nothing fixable in code. A dead stage agent is still worth retrying — + // that failure is transient infrastructure, and a retry is the only + // useful action for it. Everything else (refuted findings, missing CLI) + // would just spin, so stop with the report. + const deadStages = failures.filter(f => f.detail === 'stage agent died').map(f => f.stage) + if (deadStages.length > 0) { + log(`cycle ${cycle}: fix agent changed nothing — retrying dead stage(s): ${deadStages.join(', ')}`) + toRun = new Set(deadStages) + continue + } + log(`cycle ${cycle}: fix agent changed nothing — stopping`) + break + } + + // What re-runs is gated on what the commit actually contains, never on the + // fixer's self-report (the FIX schema lets `files` be empty or wrong, and a + // code fix reported as a doc path would keep stale-green results). Read the + // paths back out of git; if that read fails, fall back to the self-report and + // grant no exemption below. + const verified = await (async () => { + if (!fix.commit) return null + try { + return await agent( + `In this repo run: git show --name-only --format= ${fix.commit} and git rev-parse HEAD. ` + + 'paths = the repo-relative paths that commit touched, verbatim, one per array entry; ' + + `isHead = true only if git rev-parse HEAD is exactly ${fix.commit}. ` + + 'Read-only: edit nothing, commit nothing, never spawn subagents.', + { label: `c${cycle}:fix-paths`, phase: 'Fix', model: 'haiku', schema: PATHS }) + } catch { return null } + })() + const trusted = !!(verified && verified.isHead && verified.paths.length > 0) + const files = trusted ? verified.paths : (fix.files || []) + entry.fix.files = files + entry.fix.verified = trusted + if (!trusted) log(`cycle ${cycle}: could not confirm the fix commit's paths — treating the fix as touching everything`) + + // The fixer rewrote this workflow, but the stage thunks, the gates and this + // loop are the old file — already loaded in memory. Re-running here would + // validate the corrected workflow with superseded orchestration and could + // report green off it, so hand the restart back to the caller instead. + const workflowFiles = files.filter(f => /^\.claude\/workflows\//.test(f)) + if (workflowFiles.length > 0) { + log(`cycle ${cycle}: the fix commit edits ${workflowFiles.join(', ')} — stopping. ` + + 'Re-invoke validate so the committed workflow is loaded fresh; this run\'s verdict is not final.') + return { pass: false, restartRequired: true, cycles: history, stages: [...latest.values()], failures } + } + + // A fix can invalidate any stage: builds/unit consume src|hw|examples|test, + // and size/pvs run tools the fixer may have edited. Only a pure-docs fix + // is safe to exempt — everything else re-runs the full stage set. Agent and + // skill instructions are Markdown but drive the stage agents themselves, so + // they are operational, not documentation: editing them must re-run + // everything, or the loop reports green on results the old instructions produced. + const docsOnly = trusted && files.length > 0 && files.every(f => + !f.startsWith('.claude/') && f !== 'CLAUDE.md' && f !== 'AGENTS.md' && + (/^docs\//.test(f) || f.endsWith('.md') || f.endsWith('.rst'))) + toRun = new Set(failures.map(f => f.stage)) + if (!skip.includes('review')) toRun.add('review') + if (!skip.includes('codex')) toRun.add('codex') + if (!docsOnly) for (const n of stageNames) toRun.add(n) +} -const results = (await parallel(thunks)).filter(Boolean) -const dead = thunks.length - results.length -if (dead > 0) log(`${dead} stage agent(s) died — counted as failures`) -const failures = results.filter(r => !r.pass) -log(`${results.length}/${thunks.length} stages completed, ${failures.length} failing`) -return { pass: failures.length === 0 && dead === 0, stages: results, failures } +const failures = [...latest.values()].filter(r => !r.pass) +return { pass: false, cycles: history, stages: [...latest.values()], failures } diff --git a/.github/scripts/ci_set_matrix.py b/.github/scripts/ci_set_matrix.py index 79f466893..409e6dbc1 100755 --- a/.github/scripts/ci_set_matrix.py +++ b/.github/scripts/ci_set_matrix.py @@ -131,7 +131,13 @@ def set_matrix_json(select=None): # a family this file does not list builds on no toolchain, so it contributes no # leg. hw/bsp holds several CI has never built (efm32, py32f0, same7x, ...) plus # espressif, whose boards hil-build-esp builds by name. - unbuilt = sorted(f for f in sel_fams if f not in family_list) + # espressif is not a gap: its examples need the ESP-IDF environment + # (CLAUDE.md: `. "$IDF_PATH/export.sh"` before any build), which the cmake legs + # do not have - that is why it is commented out of family_list above. Its + # coverage comes from hil-build-esp, which builds those boards BY NAME in an IDF + # container, so an espressif-only PR is already validated and falling open to the + # full matrix would add 74 legs, none of which can compile espressif. + unbuilt = sorted(f for f in sel_fams if f not in family_list and f != 'espressif') if unbuilt and not any(matrix.values()): # NONE of the selected families is buildable here, so every leg would skip # and the PR would go green from a build job that ran no compiler. That is diff --git a/.github/scripts/hil_ci_set_matrix.py b/.github/scripts/hil_ci_set_matrix.py index bf50061dd..b567f347c 100644 --- a/.github/scripts/hil_ci_set_matrix.py +++ b/.github/scripts/hil_ci_set_matrix.py @@ -1,5 +1,6 @@ import argparse import json +import shlex import os import sys @@ -112,14 +113,20 @@ def main(): # Each variant builds into cmake-build-<variant.name> with its own cmake # -D defines and raw CFLAGS. No 'variant' -> a single build named after - # the board. + # the board; an always-on define (MAX3421_HOST=1, LOGGER=rtt) is a single + # self-named variant carrying it. variants = board.get('variant') or [{'name': name, 'flags': ''}] for v in variants: arg = build_board if v['name'] != name: arg += f' --build-name {v["name"]}' + # build_util.yml's Build step splices this string into bash source, + # so the quoting round-trips a spaced value into one argv item like + # build_board's argv path. The SAME string also reaches the get_deps + # env expansion and the artifact-name charset, where spaced/quoted + # values still fail (loudly) -- keep defines space-free for d in v.get('defines', []): - arg += f' -D{d}' + arg += f' -D{shlex.quote(d)}' for tok in v.get('flags', '').split(): arg += f' --cflag={tok}' append_build_arg(toolchain, arg) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 39a4e7afd..70555b111 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,6 +41,7 @@ jobs: - 'tools/ci_select.py' - 'tools/get_deps.py' - 'tools/metrics.py' + - 'tools/rtt.py' - '.github/actions/**' - '.github/workflows/build.yml' - '.github/workflows/build_util.yml' @@ -68,14 +69,9 @@ jobs: with: fetch-depth: 0 - # The `ci-full` PR label turns the scoping off for one PR: no selection file is - # written, so both matrices and every rig job fall back to the unscoped behaviour. - # An escape hatch is the point - a selector bug under-selects SILENTLY, and without - # a label the only routes back to a full matrix are accidental (touch an - # unclassified path, or break the selector badly enough that it falls open). - name: CI selection (PR only) id: hil-select - if: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ci-full') + if: github.event_name == 'pull_request' env: BASE_REF: ${{ github.base_ref }} run: | @@ -179,29 +175,43 @@ jobs: # treats false like null, so .build.full is compared explicitly. EXAMPLE_MAP='{}' BUILD_FILTERED='false' - FAM_REGEX='' + FAMILY_REGEX='' if [ -n "$BUILD_SELECT_FILE" ]; then EXAMPLE_MAP=$(jq -c '.build.family_examples // {}' "$BUILD_SELECT_FILE") || EXAMPLE_MAP='{}' BUILD_FILTERED=$(jq -r 'if (.build? | type) == "object" and .build.full == false then "true" else "false" end' "$BUILD_SELECT_FILE") || BUILD_FILTERED='false' if [ "$BUILD_FILTERED" = "true" ]; then - FAM_REGEX=$(jq -r '.build.families | join("|")' "$BUILD_SELECT_FILE") || FAM_REGEX='' + FAMILY_COUNT=$(jq -r '.build.families | length' "$BUILD_SELECT_FILE") || FAMILY_COUNT=0 + FAMILY_REGEX=$(jq -r '.build.families | join("|")' "$BUILD_SELECT_FILE") || FAMILY_REGEX='' # family names come from hw/bsp dir names, which rule 6 reads straight out # of the PR's diff path - and this is interpolated raw into a # `name_is_regexp` artifact pattern, so a regex metacharacter there would # silently match another family's baseline - case "$FAM_REGEX" in + FAMILY_REJECTED=0 + case "$FAMILY_REGEX" in *[!-A-Za-z0-9_\|]*) echo "::warning::unexpected characters in the family list - dropping the scoping" - FAM_REGEX='' ;; + FAMILY_REGEX=''; FAMILY_REJECTED=1 ;; esac - if [ -z "$FAM_REGEX" ]; then - # all three drop together, as CircleCI's fall-open does. Resetting only + # An EMPTY families list and a REJECTED one both leave FAMILY_REGEX empty and + # mean opposite things, so branch on which happened. Testing `-z` alone sent + # every nothing-selected PR down the fall-open path: a docs/.gitignore diff + # (#3842) and a test/hil-only diff (#3840) each rebuilt all 74 cmake legs + # after the selector had correctly chosen none. + if [ "$FAMILY_REJECTED" = "1" ]; then + # unusable: fall open, and all three drop together. Resetting only # build_filtered leaves the build scoped while code-metrics takes the # UNSCOPED branch, diffing a 1-family run against the full averaged # baseline and publishing that as the PR's code-size impact. BUILD_FILTERED='false' EXAMPLE_MAP='{}' MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py) + elif [ "$FAMILY_COUNT" = "0" ]; then + # legitimate nothing-selected. MATRIX_JSON already holds the all-empty + # matrix ci_set_matrix produced from this selection - keep it, so every + # leg skips. Nothing is built, so there is nothing to compare a baseline + # against: build_filtered goes false to keep code-metrics off the scoped + # path, and EXAMPLE_MAP stays '{}' (family_examples is empty anyway). + BUILD_FILTERED='false' fi fi fi @@ -210,7 +220,7 @@ jobs: echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT echo "example_map=$EXAMPLE_MAP" >> $GITHUB_OUTPUT echo "build_filtered=$BUILD_FILTERED" >> $GITHUB_OUTPUT - echo "build_families_regex=$FAM_REGEX" >> $GITHUB_OUTPUT + echo "build_families_regex=$FAMILY_REGEX" >> $GITHUB_OUTPUT # HIL matrix (merged from tinyusb + hifiphile configs), scoped on PRs. # Scoping is best-effort too: fall back to the unscoped (full) matrix. diff --git a/.github/workflows/build_util.yml b/.github/workflows/build_util.yml index 52999616d..407ed1e71 100644 --- a/.github/workflows/build_util.yml +++ b/.github/workflows/build_util.yml @@ -126,16 +126,11 @@ jobs: MEMBROWSE_API_KEY: ${{ secrets.MEMBROWSE_API_KEY }} run: | # if code-changed is false --> there is no elf -> membrowse target upload with --identical flag - # $EX_ARGS is passed for the BOARD it picks, not to scope the targets: - # --one-first now chooses a board that can build the -e set (tools/build.py), - # so omitting it here would configure a DIFFERENT, empty build dir and upload - # --identical for a board that was never compiled. The target list is not - # scoped by it - `examples-membrowse-upload` is not `all`, so it passes - # through as the aggregate, which has no DEPENDS (hw/bsp/family_support.cmake): - # it rebuilds nothing and still records every example, --identical for the - # ones without an elf. + # deliberately unscoped by $EX_ARGS: keeps the size history on a stable board + # per family, at the cost of an --identical-only upload where that board is not + # the one the Build step picked (test_ci_metrics pins which families those are) BUILD_PY_ARGS="-s ${{ inputs.build-system }} ${{ steps.setup-toolchain.outputs.build_option }} ${{ inputs.build-options }}" - python tools/build.py $BUILD_PY_ARGS --target examples-membrowse-upload -j 1 ${{ matrix.arg }} $EX_ARGS + python tools/build.py $BUILD_PY_ARGS --target examples-membrowse-upload -j 1 ${{ matrix.arg }} shell: bash - name: Upload Artifacts for Metrics diff --git a/.gitignore b/.gitignore index f6c8702b6..14bc22b61 100644 --- a/.gitignore +++ b/.gitignore @@ -62,6 +62,7 @@ README_processed.rst docs/examples/ .worktrees .claude/worktrees/ +.claude/skills/update-sponsor/state.json cmake-metrics/ # Directories fetched by tools/get_deps.py - not to be committed lib/CMSIS_5/ @@ -95,3 +96,4 @@ hw/mcu/sony/cxd56/spresense-exported-sdk/ hw/mcu/st/ hw/mcu/ti/ hw/mcu/wch/ +test/hil/local.json diff --git a/.idea/cmake.xml b/.idea/cmake.xml index 23e8af7ea..f5f1fda2d 100644 --- a/.idea/cmake.xml +++ b/.idea/cmake.xml @@ -9,6 +9,8 @@ <configuration PROFILE_NAME="raspberry_pi_pico-pio_host" ENABLED="false" CONFIG_NAME="Debug" GENERATION_OPTIONS="-DBOARD=raspberry_pi_pico -DLOG=1 -DCFLAGS_CLI="-DCFG_TUH_RPI_PIO_USB=1"" /> <configuration PROFILE_NAME="raspberry_pi_pico2" ENABLED="false" CONFIG_NAME="Debug" GENERATION_OPTIONS="-DBOARD=raspberry_pi_pico2 -DLOG=1" /> <configuration PROFILE_NAME="raspberry_pi_pico2-pio_host" ENABLED="false" CONFIG_NAME="Debug" GENERATION_OPTIONS="-DBOARD=raspberry_pi_pico2 -DLOG=1 -DCFLAGS_CLI="-DCFG_TUH_RPI_PIO_USB=1"" /> + <configuration PROFILE_NAME="pico2_etm_trace" ENABLED="false" CONFIG_NAME="Debug" GENERATION_OPTIONS="-DBOARD=pico2_etm_trace -DLOG=1" /> + <configuration PROFILE_NAME="pico2_etm_trace-pio_host" ENABLED="false" CONFIG_NAME="Debug" GENERATION_OPTIONS="-DBOARD=pico2_etm_trace -DLOG=1 -DCFLAGS_CLI="-DCFG_TUH_RPI_PIO_USB=1"" /> <configuration PROFILE_NAME="feather_rp2040" ENABLED="false" CONFIG_NAME="Debug" GENERATION_OPTIONS="-DBOARD=pico_sdk -DPICO_BOARD=adafruit_feather_rp2040 -DLOG=1" /> <configuration PROFILE_NAME="feather_rp2040_max3421" ENABLED="false" CONFIG_NAME="Debug" GENERATION_OPTIONS="-DBOARD=feather_rp2040_max3421 -DLOG=1" /> <configuration PROFILE_NAME="metro_rp2040" ENABLED="false" CONFIG_NAME="Debug" GENERATION_OPTIONS="-DBOARD=pico_sdk -DPICO_BOARD=adafruit_metro_rp2040 -DLOG=1 -DMAX3421_HOST=1" /> diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7a29dc89a..537ed3bc4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -51,7 +51,7 @@ repos: # Two hooks, split by what each suite RUNS, not by what it reads: discovery is # disjoint (test_hil*.py vs the two named suites) so nothing runs twice, but the # file patterns overlap where both suites care. hil-test runs test_hil*.py only - # (~80s: deliberate hang and timeout simulations) and is scoped to the rig harness + # (~45s: deliberate hang and timeout simulations) and is scoped to the rig harness # that owns them. The one part of it the selector depends on - the BottomLayer # stdlib-closure AST guard over tools/ci_select.py and its imports - is named # explicitly by ci-select-test instead, so a tools/ or workflow edit costs 4s @@ -69,7 +69,7 @@ repos: # and md5-checks the logo header from there as its MTP fixtures. - id: hil-test name: hil-test - files: ^(test/hil/|examples/device/mtp/src/) + files: ^(test/hil/|examples/device/mtp/src/|tools/rtt\.py$) entry: python3 -m unittest discover -s test/hil/test -p 'test_hil*.py' pass_filenames: false language: system @@ -84,7 +84,7 @@ repos: language: system - id: ci-select-test name: ci-select-test - files: ^(hw/bsp/|hw/mcu/|src/|examples/|test/hil/|tools/(ci_select|build|build_utils|get_deps|metrics)\.py$|\.github/(scripts|workflows)/|\.circleci/) + files: ^(hw/bsp/|hw/mcu/|src/|examples/|test/hil/|tools/(ci_select|build|build_utils|get_deps|metrics|rtt)\.py$|\.github/(scripts|workflows)/|\.circleci/) entry: sh -c "python3 test/hil/test/test_ci_select.py && python3 test/hil/test/test_ci_metrics.py && cd test/hil/test && python3 -m unittest -q test_hil_util.BottomLayer" pass_filenames: false language: system @@ -74,7 +74,7 @@ Terminal 2 — connect (`<port>`: 2331 JLink, 3333 OpenOCD): arm-none-eabi-gdb build/your_app.elf (gdb) target remote :<port> # then: monitor reset halt → load → continue ``` -**RTT:** build `LOG=2 LOGGER=rtt`, run JLinkGDBServer with `-RTTTelnetPort 19021`, then `JLinkRTTClient` (`timeout 20s JLinkRTTClient > rtt.log` for non-interactive capture). +**RTT:** build `LOG=2 LOGGER=rtt`; capture/console via the `rtt` skill (`.claude/skills/rtt/SKILL.md`). ## Testing diff --git a/README.rst b/README.rst index dea352532..7fa8d70f1 100644 --- a/README.rst +++ b/README.rst @@ -60,7 +60,7 @@ Supporters (Word) .. WORD-SUPPORTERS-START -*No supporters yet — be the first!* +cee\*\*\*\* .. WORD-SUPPORTERS-END @@ -69,7 +69,7 @@ Thanks (Byte) .. BYTE-THANKS-START -*No names listed yet — be the first!* +`@8086net <https://github.com/8086net>`__, `@GCRev <https://github.com/GCRev>`__ .. BYTE-THANKS-END diff --git a/docs/reference/boards.rst b/docs/reference/boards.rst index 8b0f798ba..794c4fa51 100644 --- a/docs/reference/boards.rst +++ b/docs/reference/boards.rst @@ -236,19 +236,20 @@ nrf54lm20dk Nordic nRF54LM20 DK nrf ht Raspberry Pi ------------ -================================ ============================================ ============== ========================================================== ====== -Board Name Family URL Note -================================ ============================================ ============== ========================================================== ====== +================================ ============================================ ============== ================================================================ ====== +Board Name Family URL Note +================================ ============================================ ============== ================================================================ ====== raspberrypi_zero Raspberry Pi Zero broadcom_32bit https://www.raspberrypi.org/products/raspberry-pi-zero/ raspberrypi_cm4 Raspberry CM4 broadcom_64bit https://www.raspberrypi.org/products/compute-module-4 raspberrypi_zero2 Raspberry Zero2 broadcom_64bit https://www.raspberrypi.org/products/raspberry-pi-zero-2-w adafruit_feather_rp2040_usb_host Adafruit Feather RP2040 with USB Type A Host rp2040 https://www.adafruit.com/product/5723 adafruit_fruit_jam Adafruit Fruit Jam - Mini RP2350 rp2040 https://www.adafruit.com/product/6200 adafruit_metro_rp2350 Adafruit Metro RP2350 rp2040 https://www.adafruit.com/product/6003 +pico2_etm_trace Pico 2 ETM Trace Carrier rp2040 https://github.com/hathach/pcb/tree/main/pico2_trace_motherboard raspberry_pi_pico Pico rp2040 https://www.raspberrypi.com/products/raspberry-pi-pico/ raspberry_pi_pico2 Pico2 rp2040 https://www.raspberrypi.com/products/raspberry-pi-pico-2/ raspberry_pi_pico_w Pico rp2040 https://www.raspberrypi.com/products/raspberry-pi-pico/ -================================ ============================================ ============== ========================================================== ====== +================================ ============================================ ============== ================================================================ ====== Renesas ------- diff --git a/docs/reference/hil_boards.md b/docs/reference/hil_boards.md index e8f364646..678f7f0ed 100644 --- a/docs/reference/hil_boards.md +++ b/docs/reference/hil_boards.md @@ -12,7 +12,7 @@ | espressif_s3_devkitm | device, host | esptool | espressif_s3_devkitm, espressif_s3_devkitm-DMA | Use TS3USB30 mux to test both device and host | | feather_nrf52840_express | device | jlink | | | | max32666fthr | device | openocd | | | -| metro_m4_express | device, dual | jlink | | pl23x; audio_test_freertos skipped: samd51 iso-IN capture fails (arecord EIO) | +| metro_m4_express | device, dual | jlink | metro_m4_express | pl23x; audio_test_freertos skipped: samd51 iso-IN capture fails (arecord EIO) | | lpcxpresso11u37 | device | jlink | | | | lpcxpresso55s28 | device | jlink | | | | ra4m1_ek | device | jlink | | | diff --git a/docs/superpowers/followup/pr3803-hil-blindness-reporting.md b/docs/superpowers/followup/pr3803-hil-blindness-reporting.md deleted file mode 100644 index 69ff939b0..000000000 --- a/docs/superpowers/followup/pr3803-hil-blindness-reporting.md +++ /dev/null @@ -1,185 +0,0 @@ -# Blindness Reporting Gaps Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make a HIL worker's sysfs blindness reach the report in the two cases where it -currently does not — an untested producer, and a board that raises. - -**Architecture:** A worker returns `hil_util.sysfs_blind()` as the last field of its result -tuple; `_blind_note()` turns that into a report banner. Two holes: nothing tests the -producer, and a board that raises returns no tuple at all, so its blindness is lost. - -**Tech Stack:** Python 3.13 stdlib, multiprocessing Pool with `maxtasksperchild=1`. - -## Global Constraints - -- A blind worker answers `SYSFS_UNKNOWN` for every attribute, so its "device not found" - means "could not tell". The report must say so or a red cell reads as a broken board. -- `maxtasksperchild=1`: one worker per board, so the flag is per-board and must not be - smeared across boards. -- Tests: `cd test/hil && python3 test/test_hil_bounded.py`. - -## What is already established - -- `hil_test.test_board` returns `(..., hil_util.sysfs_blind(), stray)`; `_blind_note(mret)` - renders the banner; wired into all three report paths. -- **The producer is provably untested**: replacing `hil_util.sysfs_blind()` with `False` in - the return leaves all tests green. Nothing drives `test_board` — it needs a board dict, a - real flock, a flasher and `test_example` per test. -- Blindness fired for real on ci.lan: four workers went blind in one run, and cells failed - *because* of it (`Printer device not found ... (this worker is blind)`). - -**Why this is a separate PR:** closing it means making `test_board` testable, which is a -refactor of the harness's orchestration layer — a different scope from the containment -work, and the reason the gap was accepted rather than papered over. - -## File Structure - -- `test/hil/hil_test.py` — extract the result-tuple assembly from `test_board` so it can be - built and asserted without running a board; carry blindness out of the raise path. -- `test/hil/test/test_hil_bounded.py` — tests for both. - ---- - -### Task 1: Make the result tuple assembly testable - -**Files:** -- Modify: `test/hil/hil_test.py` (`test_board`, the `return (name, err_count, ...)` at the - end of the try block) -- Test: `test/hil/test/test_hil_bounded.py` - -**Interfaces:** -- Produces: `_board_result(name, err_count, failed_tests, rows, t_total, board_wide_fail)` - returning the 7-tuple `(name, err_count, failed, rows, t_total, blind, stray)`, reading - `hil_util.sysfs_blind()` and `hil_health.kill_own_children()` itself. - -- [ ] **Step 1: Write the failing test** - -```python -class BoardResultCarriesBlindness(unittest.TestCase): - def test_a_blind_worker_reports_it(self): - from helper import hil_util, hil_health - self.addCleanup(setattr, hil_util, 'sysfs_blind', hil_util.sysfs_blind) - self.addCleanup(setattr, hil_health, 'kill_own_children', hil_health.kill_own_children) - hil_util.sysfs_blind = lambda: True - hil_health.kill_own_children = lambda: 0 - row = hil_test._board_result('b', 0, [], [], 1.0, False) - self.assertTrue(row[5], 'blindness did not reach the result tuple') - self.assertIn('b', hil_test._blind_note([row])) - - def test_a_sighted_worker_does_not(self): - from helper import hil_util, hil_health - self.addCleanup(setattr, hil_util, 'sysfs_blind', hil_util.sysfs_blind) - self.addCleanup(setattr, hil_health, 'kill_own_children', hil_health.kill_own_children) - hil_util.sysfs_blind = lambda: False - hil_health.kill_own_children = lambda: 0 - row = hil_test._board_result('b', 0, [], [], 1.0, False) - self.assertFalse(row[5]) - self.assertEqual(hil_test._blind_note([row]), '') -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `cd test/hil && python3 test/test_hil_bounded.py BoardResultCarriesBlindness -v` -Expected: FAIL — `module 'hil_test' has no attribute '_board_result'` - -- [ ] **Step 3: Write minimal implementation** - -```python -def _board_result(name, err_count, failed_tests, rows, t_total, board_wide_fail): - """Assemble a worker's result tuple. Separate from test_board so the two fields only - the WORKER can answer -- its process-global blindness latch and what it could not kill - -- are testable without running a board.""" - stray = hil_health.kill_own_children() - return (name, err_count, [] if board_wide_fail else sorted(set(failed_tests)), - rows, t_total, hil_util.sysfs_blind(), stray) -``` - -Replace the tail of `test_board` with: - -```python - return _board_result(name, err_count, failed_tests, rows, t_total, board_wide_fail) -``` - -- [ ] **Step 4: Run test to verify it passes** - -Run: `cd test/hil && python3 test/test_hil_bounded.py -v` -Expected: PASS, and the existing `BlindWorkerReachesTheReport` tests still pass. - -- [ ] **Step 5: Verify the mutation is now caught** - -Replace `hil_util.sysfs_blind()` with `False` inside `_board_result` and re-run; the suite -MUST fail. Restore it. - -- [ ] **Step 6: Commit** - -```bash -git add test/hil/hil_test.py test/hil/test/test_hil_bounded.py -git commit -m "test/hil: make the worker result tuple testable, covering blindness" -``` - ---- - -### Task 2: Carry blindness out of the worker-raise path - -**Files:** -- Modify: `test/hil/hil_test.py` (`test_board`'s except/finally, and `main`'s worker-raise - handler that builds synthetic rows) -- Test: `test/hil/test/test_hil_bounded.py` - -**Interfaces:** -- Consumes: `_board_result` from Task 1. -- Produces: a board that raises still contributes a row whose blindness field is accurate. - -- [ ] **Step 1: Write the failing test** - -```python - def test_a_board_that_raises_still_reports_blindness(self): - """The result tuple is returned inside a try whose finally only releases the lock, - so a board that dies by exception contributed nothing -- and its blindness, the - thing that most explains its failure, was lost with it.""" - from helper import hil_util - self.addCleanup(setattr, hil_util, 'sysfs_blind', hil_util.sysfs_blind) - hil_util.sysfs_blind = lambda: True - row = hil_test._board_result_on_error('b', RuntimeError('boom')) - self.assertTrue(row[5]) - self.assertIn('b', hil_test._blind_note([row])) -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `cd test/hil && python3 test/test_hil_bounded.py BoardResultCarriesBlindness -v` -Expected: FAIL — no `_board_result_on_error` - -- [ ] **Step 3: Write minimal implementation** - -```python -def _board_result_on_error(name, exc): - """A row for a board that died by exception. err_count 1, no per-test detail, but the - blindness and stray fields are still accurate -- they explain the failure more often - than the exception text does.""" - rows = [(name, {BOUNDARY_CELL: f'{REPORT_CELL["fail"]} {type(exc).__name__}'}, None)] - return _board_result(name, 1, [], rows, 0.0, True) -``` - -Wrap the body of `test_board` so the exception path returns it instead of propagating. - -- [ ] **Step 4: Run test to verify it passes** - -Run: `cd test/hil && python3 test/test_hil_bounded.py -v` -Expected: PASS - -- [ ] **Step 5: Commit** - -```bash -git add test/hil/hil_test.py test/hil/test/test_hil_bounded.py -git commit -m "test/hil: keep a raising board's blindness in the report" -``` - ---- - -## Caution - -`test_board`'s `finally` releases the board flock. Any restructuring MUST keep that -release on every path, including the new error path — a leaked flock locks the board until -the host reboots. diff --git a/docs/superpowers/followup/pr3803-usbtest-recovery-reserve.md b/docs/superpowers/followup/pr3803-usbtest-recovery-reserve.md deleted file mode 100644 index eb8959520..000000000 --- a/docs/superpowers/followup/pr3803-usbtest-recovery-reserve.md +++ /dev/null @@ -1,175 +0,0 @@ -# usbtest Recovery Reserve Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make the post-hang recovery reserve a derived, asserted property instead of an -accident of four independently-set constants. - -**Architecture:** `hil_test` passes `--budget` and `--outer-timeout` to `usbtest.py`, which -decides at runtime whether a recovery still fits. Today the reserve survives only because -the four numbers happen to line up; nothing ties them together or fails when they stop. - -**Tech Stack:** Python 3.13 stdlib. - -## Global Constraints - -- `usbtest.py`: `RECOVER_FLASH_TIMEOUT = 90`, `RECOVER_RESET_TIMEOUT = 30`. -- `hil_test.py`: `USBTEST_BATTERY_BUDGET = 260`, `USBTEST_RECOVERY_BUDGET = 250`, - `USBTEST_OVERSHOOT = 120`; `outer = BATTERY_BUDGET + (RECOVERY_BUDGET if recovery else - OVERSHOOT)`, used for both the child's `--outer-timeout` and the parent's `run_cmd` bound. -- All five are env-overridable via `hil_util.pos_int_env`, so a rig can change them. -- Tests: `cd test/hil && python3 test/test_hil_health.py` and `test_hil_bounded.py`. - -## What is already established - -The reserve holds at the shipped values, checked by hand: - -- The battery checks its budget BEFORE dispatching a case, so it can overshoot by one - case — worst case `260 + 60 + 5 = 325 s`. -- Recovery is gated on `_time_left() >= RECOVER_RESET_TIMEOUT`, where - `_time_left() = outer_timeout - elapsed - 35`; with `outer = 510` that allows recovery - until `elapsed = 445 s`, and the reflash until `385 s`. -- So ~60 s of margin survives, and recovery does fire. - -**The defect is structural, not arithmetic:** lower `--outer-timeout`, raise `--timeout`, or -raise `USBTEST_BATTERY_BUDGET` via the env and the reserve silently disappears. The failure -mode is a skipped reflash that leaves the D-state holder for the next job — the exact thing -the containment exists to prevent — with no error anywhere. - -**Why this is a separate PR:** it changes the timing contract between `hil_test` and -`usbtest.py`, which affects every board's run duration, so it wants its own review and a -full rig run. - -## File Structure - -- `test/hil/usbtest.py` — a `reserve_ok()` predicate plus a startup assertion. -- `test/hil/hil_test.py` — derive the battery budget from the outer bound rather than - setting both independently. -- `test/hil/test/test_hil_health.py` — tests. - ---- - -### Task 1: Assert the reserve at startup - -**Files:** -- Modify: `test/hil/usbtest.py` (constants block, and `main()` after argparse) -- Test: `test/hil/test/test_hil_health.py` - -**Interfaces:** -- Produces: `usbtest.reserve_ok(budget, outer, case_timeout)` returning bool. - -- [ ] **Step 1: Write the failing test** - -```python -class RecoveryReserveIsChecked(unittest.TestCase): - """The battery may overshoot its budget by ONE already-started case, so the outer bound - must leave room for that overshoot AND a bounded recovery afterwards.""" - - def setUp(self): - import usbtest - self.u = usbtest - - def test_the_shipped_numbers_leave_room(self): - self.assertTrue(self.u.reserve_ok(budget=260, outer=510, case_timeout=60)) - - def test_a_tighter_outer_bound_is_rejected(self): - self.assertFalse(self.u.reserve_ok(budget=260, outer=380, case_timeout=60)) - - def test_a_longer_case_timeout_is_rejected(self): - self.assertFalse(self.u.reserve_ok(budget=260, outer=510, case_timeout=200)) -``` - -- [ ] **Step 2: Run test to verify it fails** - -Run: `cd test/hil && python3 test/test_hil_health.py RecoveryReserveIsChecked -v` -Expected: FAIL — `module 'usbtest' has no attribute 'reserve_ok'` - -- [ ] **Step 3: Write minimal implementation** - -```python -def reserve_ok(budget: int, outer: int, case_timeout: int) -> bool: - """Does `outer` leave room for the battery's worst case AND a bounded recovery? - - The budget is checked BEFORE dispatch, so the battery can run to - `budget + case_timeout + 5` (the +5 is run_case's reap). _time_left() subtracts a - further 35 s of fixed tail. A reflash needs RECOVER_FLASH_TIMEOUT beyond that. - """ - worst_case_end = budget + case_timeout + 5 - return outer - worst_case_end - 35 >= RECOVER_FLASH_TIMEOUT -``` - -In `main()`, after parsing args: - -```python - if args.budget and args.outer_timeout and not reserve_ok( - args.budget, args.outer_timeout, args.timeout): - print(f'warning: --outer-timeout {args.outer_timeout} leaves no room for a bounded ' - f'recovery after a --budget {args.budget} battery with --timeout ' - f'{args.timeout} cases; a HUNG board will be left wedged', file=sys.stderr) -``` - -Warn, do not exit: a caller that deliberately runs without recovery is legitimate. - -- [ ] **Step 4: Run test to verify it passes** - -Run: `cd test/hil && python3 test/test_hil_health.py RecoveryReserveIsChecked -v` -Expected: PASS - -- [ ] **Step 5: Commit** - -```bash -git add test/hil/usbtest.py test/hil/test/test_hil_health.py -git commit -m "usbtest: check the recovery reserve instead of assuming it" -``` - ---- - -### Task 2: Derive the outer bound from one place - -**Files:** -- Modify: `test/hil/hil_test.py` (constants block ~line 227, and `test_device_usbtest`) -- Test: `test/hil/test/test_hil_bounded.py` - -**Interfaces:** -- Consumes: `usbtest.reserve_ok` semantics (duplicate the arithmetic, do not import - usbtest — `hil_test` must not import it). -- Produces: an assertion at module import that the shipped constants satisfy the reserve. - -- [ ] **Step 1: Write the failing test** - -```python - def test_the_shipped_constants_satisfy_the_reserve(self): - """Whatever the env overrides, the pair hil_test computes must leave recovery room: - outer - (budget + case_timeout + 5) - 35 >= 90.""" - outer = hil_test.USBTEST_BATTERY_BUDGET + hil_test.USBTEST_RECOVERY_BUDGET - self.assertGreaterEqual(outer - (hil_test.USBTEST_BATTERY_BUDGET + 60 + 5) - 35, 90) -``` - -- [ ] **Step 2: Run test to verify it fails** - -Temporarily set `HIL_USBTEST_RECOVERY_BUDGET=100` and run; expect FAIL. Unset. - -- [ ] **Step 3: Add the guard** - -```python -# The recovery reserve is a PROPERTY of these two, not a coincidence: the battery may -# overshoot its budget by one already-started case (checked before dispatch), and a bounded -# reflash needs 90 s after a 35 s fixed tail. Env overrides make this checkable at import -# rather than discoverable when a wedge is left unrecovered. -if USBTEST_RECOVERY_BUDGET - 60 - 5 - 35 < 90: - print(f'warning: HIL_USBTEST_RECOVERY_BUDGET={USBTEST_RECOVERY_BUDGET} leaves no room ' - f'for a bounded reflash after a one-case overshoot; HUNG boards will stay wedged', - file=sys.stderr) -``` - -- [ ] **Step 4: Run tests to verify they pass** - -Run: `cd test/hil && python3 test/test_hil_bounded.py -v` -Expected: PASS - -- [ ] **Step 5: Commit** - -```bash -git add test/hil/hil_test.py test/hil/test/test_hil_bounded.py -git commit -m "hil: warn when the timeout constants leave no recovery reserve" -``` diff --git a/docs/superpowers/followup/pr3836-report-single-source.md b/docs/superpowers/followup/pr3836-report-single-source.md deleted file mode 100644 index f4ccc77c2..000000000 --- a/docs/superpowers/followup/pr3836-report-single-source.md +++ /dev/null @@ -1,470 +0,0 @@ -# One Source of Truth for the HIL Report Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Make `hil_report.md` a rendering of `hil_report.json` rather than a second, independently written artifact, so no run can produce a table whose contents are not in the JSON. - -**Architecture:** `hil_report.json` gains the two fields the markdown carries but the JSON does not (`scope`, and a `caveat` for text prepended after the fact). A single `render_report(doc) -> str` turns that document into the markdown, and every writer — the normal path, the pool-guard fallback, the no-boards exit, and `_abandon_exit` — goes through `write_report(report_dir, doc)`, which writes both files from the same dict. `_abandon_exit` stops doing a text-prepend on a file it did not write and instead sets `doc['caveat']`. - -**Tech Stack:** Python 3.13 stdlib only (`json`, `pathlib`); existing unit suites under `test/hil/test/` run with plain `unittest`. - -**Spec:** none — this is a follow-up split out of the `claude/hil-doc-audit` branch. The evidence it argues from is inline below. - -**Origin:** split out of PR #3836 (the HIL one-run rework + `.claude` instruction audit). Delete this file when its own PR lands. - -## Global Constraints - -- **No behaviour change to the containment paths' ordering or exit codes.** `_abandon_exit` runs while the interpreter is being torn down; its own comments record that anything raising between the pool's `finally` and `os._exit` hangs the process in multiprocessing's unbounded `join()` (reproduced at rc=124/25s with SIGTERM-ignoring workers). Serialisation added there must stay inside the existing `try`/`except` and must never raise past it. -- **The markdown stays the human artifact.** `.github/workflows/build.yml:487` uploads `hil_report.md`, `test/hil/hil_ci.sh:293` copies only it back, and `.claude/skills/hil/SKILL.md` tells the operator to paste that table verbatim. It becomes generated output, not a dropped file. -- **Banner outranks the scope note outranks the table.** Preserve the existing order (`hil_test.py:2153-2161`): the caveat is outermost because that is where `hil/SKILL.md` tells an agent to look. -- **`--accumulate` merges from the JSON** (`hil_test.py:2102-2115`), including carrying the prior banner forward. Adding fields must not break that merge for a sidecar written by an older version. -- Run `python3 -m unittest discover -s test/hil/test` (115 tests, ~78 s) before each commit; `pre-commit run --files <changed>` before pushing. - -## Why this is worth doing - -Four writers produce `hil_report.md`, and three of them write no JSON at all: - -| Writer | JSON? | Line | -|---|---|---| -| `accumulate_report` — the normal path | yes | `hil_test.py:2149`, `:2162` | -| `**HIL run selected no boards.**` | **no** | `hil_test.py:2317` | -| pool-guard fallback → `hil_health.write_timeout_report(...)` | **no** | `hil_test.py:2469`, `hil_health.py:346` | -| `_abandon_exit` — prepends to whatever `.md` exists | **no** | `hil_test.py:2603` | - -Those three are exactly the paths where the run died, so they are the cases where the artifact matters most and where a JSON consumer sees nothing. `test/hil/helper/hil_summary.py` (added on the origin branch) reads the JSON to build the per-board verdicts an agent hands back — on any of those three paths it finds no file and reports "no report row for this board" for the whole fleet, while a human reading the markdown sees the real story. - -Separately, `scope` exists only in the markdown (`hil_test.py:2154`, from `accumulate_report`'s `scope: str = ''` parameter at `:2092`). A PR-scoped three-board table and a full-fleet run that lost 24 boards are indistinguishable in the JSON. - ---- - -### Task 1: Put `scope` in the JSON - -**Files:** -- Modify: `test/hil/hil_test.py:2092-2163` (`accumulate_report`) -- Test: `test/hil/test/test_hil_bounded.py` (new class beside `CaveatSurvivesAccumulate`) - -**Interfaces:** -- Produces: `hil_report.json` gains a top-level `"scope": str` (empty string when unscoped). Existing keys `rows` and `banner` are unchanged. - -- [ ] **Step 1: Write the failing test** - -```python -class ScopeSurvivesInTheJson(unittest.TestCase): - """A scoped run's small table is indistinguishable from a full run that lost boards. - The markdown says so; the JSON did not, so any JSON consumer could not tell.""" - - def _rows(self, board, cell): - return [(board, 0, 0, [(board, {cell: 'OK'}, '1s')], 0)] - - def test_scope_is_recorded_in_the_sidecar(self): - import json - td = TemporaryDirectory() - self.addCleanup(td.cleanup) - rd = Path(td.name) - hil_test.accumulate_report(self._rows('boardA', 'cdc_msc'), rd, True, - '-b boardA', '') - doc = json.loads((rd / 'hil_report.json').read_text()) - self.assertEqual(doc['scope'], '-b boardA') - - def test_an_unscoped_run_records_an_empty_scope(self): - import json - td = TemporaryDirectory() - self.addCleanup(td.cleanup) - rd = Path(td.name) - hil_test.accumulate_report(self._rows('boardA', 'cdc_msc'), rd, True, '', '') - self.assertEqual(json.loads((rd / 'hil_report.json').read_text())['scope'], '') -``` - -- [ ] **Step 2: Run it to verify it fails** - -Run: `python3 test/hil/test/test_hil_bounded.py ScopeSurvivesInTheJson` -Expected: FAIL — `KeyError: 'scope'` - -- [ ] **Step 3: Add the field** - -In `accumulate_report`, change the `jpath.write_text(...)` call at `hil_test.py:2149`: - -```python - jpath.write_text(json.dumps({'rows': [{'board': k, 'cells': c, 'duration': d} - for k, (c, d) in acc.items()], - 'banner': banner, - 'scope': scope}, indent=2) + '\n') -``` - -- [ ] **Step 4: Run the tests** - -Run: `python3 test/hil/test/test_hil_bounded.py ScopeSurvivesInTheJson` → PASS -Run: `python3 -m unittest discover -s test/hil/test` → 117 tests OK (the merge at `:2102` reads only `rows` and `banner`, so an older sidecar without `scope` still loads). - -- [ ] **Step 5: Commit** - -```bash -git add test/hil/hil_test.py test/hil/test/test_hil_bounded.py -git commit -m "hil_test: record the run's scope in hil_report.json - -The markdown says a scoped table is scoped; the JSON did not, so a consumer -could not tell a three-board PR run from a full run that lost 24 boards." -``` - ---- - -### Task 2: Render the markdown from the document - -**Files:** -- Modify: `test/hil/hil_test.py:1921` (`render_matrix`), `:2149-2163` (`accumulate_report`'s tail) -- Test: `test/hil/test/test_hil_bounded.py` - -**Interfaces:** -- Consumes: the `scope` key from Task 1. -- Produces: `render_report(doc: dict) -> str`, where `doc` is `{'rows': [{'board','cells','duration'}], 'banner': str, 'scope': str, 'caveat': str}`. `caveat` is optional and empty by default (Task 4 sets it). Order is caveat, banner, scope note, table. - -- [ ] **Step 1: Write the failing test** - -```python -class RenderReportIsPureFunctionOfTheDocument(unittest.TestCase): - def _doc(self, **kw): - d = {'rows': [{'board': 'boardA', 'cells': {'cdc_msc': 'pass'}, 'duration': '1s'}], - 'banner': '', 'scope': '', 'caveat': ''} - d.update(kw) - return d - - def test_table_comes_from_rows(self): - md = hil_test.render_report(self._doc()) - self.assertIn('boardA', md) - self.assertIn('cdc_msc', md) - - def test_scope_note_appears_above_the_table(self): - md = hil_test.render_report(self._doc(scope='-b boardA')) - self.assertLess(md.index('Scoped run'), md.index('boardA')) - - def test_banner_outranks_the_scope_note(self): - md = hil_test.render_report(self._doc(scope='-b boardA', - banner='> **Rig dirty.** x\n')) - self.assertLess(md.index('Rig dirty'), md.index('Scoped run')) - - def test_caveat_is_outermost(self): - md = hil_test.render_report(self._doc(banner='> **Rig dirty.** x\n', - caveat='**HIL run abandoned.**\n')) - self.assertLess(md.index('abandoned'), md.index('Rig dirty')) - - def test_a_document_with_no_rows_still_renders(self): - md = hil_test.render_report(self._doc(rows=[])) - self.assertIn('No tests were run.', md) -``` - -- [ ] **Step 2: Run it to verify it fails** - -Run: `python3 test/hil/test/test_hil_bounded.py RenderReportIsPureFunctionOfTheDocument` -Expected: FAIL — `AttributeError: module 'hil_test' has no attribute 'render_report'` - -- [ ] **Step 3: Add `render_report` and route `accumulate_report` through it** - -Add beside `render_matrix` (after `hil_test.py:1919`): - -```python -def render_report(doc: dict) -> str: - """The markdown IS a rendering of the sidecar. Every writer goes through here, so a - table can never contain something the JSON does not.""" - md = render_matrix([(r['board'], r['cells'], r.get('duration')) - for r in doc.get('rows', [])]) - if doc.get('scope'): - # a scoped run's small table is otherwise indistinguishable from a full one, and - # it replaces the previous full table in the sticky PR comment - md = f'_Scoped run: {doc["scope"]}. Boards/tests not listed were not run._\n\n' + md - # banner, then caveat: a rig-health caveat outranks the table AND the scope note, and an - # abandon notice outranks even that -- the top of the report is where hil/SKILL.md tells - # the agent to look - if doc.get('banner'): - md = doc['banner'] + '\n' + md - if doc.get('caveat'): - md = doc['caveat'] + '\n' + md - return md -``` - -Then replace `accumulate_report`'s tail (`hil_test.py:2153-2163`) with: - -```python - doc = {'rows': [{'board': k, 'cells': c, 'duration': d} for k, (c, d) in acc.items()], - 'banner': banner, 'scope': scope, 'caveat': ''} - jpath.write_text(json.dumps(doc, indent=2) + '\n') - md = render_report(doc) - (report_dir / REPORT_MD).write_text(md + '\n', encoding='utf-8') - return md -``` - -- [ ] **Step 4: Run the tests** - -Run: `python3 -m unittest discover -s test/hil/test` -Expected: 122 OK. `CaveatSurvivesAccumulate` must still pass — it asserts the banner survives a rerun, which is now the `banner` key round-tripping through the document. - -- [ ] **Step 5: Commit** - -```bash -git add test/hil/hil_test.py test/hil/test/test_hil_bounded.py -git commit -m "hil_test: render the markdown from the report document - -One function turns the sidecar into the table, so the markdown cannot carry -anything the JSON lacks. Ordering (caveat > banner > scope > table) is pinned -by tests rather than by the order of three string concatenations." -``` - ---- - -### Task 3: Give the two early-exit paths a document - -**Files:** -- Modify: `test/hil/hil_test.py:2313-2320` (no-boards exit), `test/hil/helper/hil_health.py:346` (`write_timeout_report`) -- Test: `test/hil/test/test_hil_health.py` (beside `WriteTimeoutReport`), `test/hil/test/test_hil_bounded.py` - -**Interfaces:** -- Consumes: `render_report(doc)` from Task 2. -- Produces: `write_report(report_dir: Path, doc: dict) -> None`, which writes `hil_report.json` and `hil_report.md` from one dict. Both early-exit paths call it. - -- [ ] **Step 1: Write the failing test** - -```python -class EveryExitPathLeavesBothArtifacts(unittest.TestCase): - """hil_summary.py builds an agent's verdicts from the JSON. A path that writes only - markdown reports the whole fleet as 'no report row' while a human sees the real story.""" - - def test_the_no_boards_exit_writes_json_too(self): - import json - td = TemporaryDirectory() - self.addCleanup(td.cleanup) - rd = Path(td.name) - hil_test.write_report(rd, {'rows': [], 'banner': '', 'scope': '', - 'caveat': '**HIL run selected no boards.** why\n'}) - self.assertIn('selected no boards', (rd / 'hil_report.md').read_text()) - doc = json.loads((rd / 'hil_report.json').read_text()) - self.assertEqual(doc['rows'], []) - self.assertIn('selected no boards', doc['caveat']) -``` - -and, in `test_hil_health.py`: - -```python - def test_timeout_report_writes_the_sidecar(self): - import json - td = TemporaryDirectory() - self.addCleanup(td.cleanup) - rd = Path(td.name) - hil_health.write_timeout_report(rd, [{'name': 'boardA'}], 3600, 'hil_report.md') - self.assertTrue((rd / 'hil_report.json').is_file()) - self.assertIn('boardA', (rd / 'hil_report.json').read_text()) -``` - -- [ ] **Step 2: Run them to verify they fail** - -Run: `python3 test/hil/test/test_hil_bounded.py EveryExitPathLeavesBothArtifacts` -Expected: FAIL — `AttributeError: module 'hil_test' has no attribute 'write_report'` -Run: `python3 test/hil/test/test_hil_health.py WriteTimeoutReport` -Expected: FAIL — `hil_report.json` is not a file - -- [ ] **Step 3: Add `write_report` and use it in both paths** - -Beside `render_report`: - -```python -def write_report(report_dir: Path, doc: dict) -> None: - """Write both artifacts from one document. Best-effort by design: every caller is on a - failure path where an OSError must not replace the failure being reported.""" - try: - report_dir.mkdir(parents=True, exist_ok=True) - (report_dir / REPORT_JSON).write_text(json.dumps(doc, indent=2) + '\n') - (report_dir / REPORT_MD).write_text(render_report(doc) + '\n', encoding='utf-8') - except OSError: - pass -``` - -Replace the no-boards block at `hil_test.py:2315-2320` with: - -```python - rd = Path(os.environ.get('HIL_REPORT_DIR', '.')) - write_report(rd, {'rows': [], 'banner': '', 'scope': '', - 'caveat': f'**HIL run selected no boards.** {msg}\n'}) -``` - -In `hil_health.write_timeout_report`, after the markdown is composed, write the sidecar next to it with a row per stuck board: - -```python - json_path = report_dir / 'hil_report.json' - json_path.write_text(json.dumps( - {'rows': [{'board': b['name'], 'cells': {'pool-timeout': 'fail'}, - 'duration': None} for b in boards], - 'banner': banner, 'scope': '', 'caveat': prefix}, indent=2) + '\n') -``` - -Keep it inside the function's existing broad `try` — a roster entry without `name` must not escape, which is what that handler exists to prevent. - -- [ ] **Step 4: Run the tests** - -Run: `python3 -m unittest discover -s test/hil/test` -Expected: 124 OK. - -- [ ] **Step 5: Commit** - -```bash -git add test/hil/hil_test.py test/hil/helper/hil_health.py test/hil/test/ -git commit -m "hil_test, hil_health: write the sidecar on the early-exit paths too - -The no-boards exit and the pool-guard fallback wrote markdown only, so a JSON -consumer saw nothing on exactly the runs that failed. hil_summary.py reported -the whole fleet as 'no report row' while the markdown told the real story." -``` - ---- - -### Task 4: Make `_abandon_exit` set a field instead of prepending text - -**Files:** -- Modify: `test/hil/hil_test.py` (`_abandon_exit`, the `if report is not None:` block near `:2622`), and its call site at `:2603` -- Test: `test/hil/test/test_hil_bounded.py` - -**Interfaces:** -- Consumes: `write_report`/`render_report` from Tasks 2–3. -- Produces: `_abandon_exit(pool, mgr, abandoned, err_count, report_dir: Path | None = None)` — the parameter becomes the **directory**, not the markdown path. - -- [ ] **Step 1: Write the failing test** - -```python -class AbandonNoticeLandsInBothArtifacts(unittest.TestCase): - def test_abandon_sets_the_caveat_not_just_the_markdown(self): - import json - td = TemporaryDirectory() - self.addCleanup(td.cleanup) - rd = Path(td.name) - hil_test.accumulate_report( - [('boardA', 0, 0, [('boardA', {'cdc_msc': 'OK'}, '1s')], 0)], rd, True, '', '') - hil_test.mark_report_abandoned(rd, 'the worker pool would not shut down.') - doc = json.loads((rd / 'hil_report.json').read_text()) - self.assertIn('abandoned', doc['caveat']) - self.assertEqual(len(doc['rows']), 1, 'the finished board must survive') - md = (rd / 'hil_report.md').read_text() - self.assertLess(md.index('abandoned'), md.index('boardA')) - - def test_marking_a_missing_report_is_a_no_op(self): - td = TemporaryDirectory() - self.addCleanup(td.cleanup) - hil_test.mark_report_abandoned(Path(td.name), 'x') # must not raise -``` - -- [ ] **Step 2: Run it to verify it fails** - -Run: `python3 test/hil/test/test_hil_bounded.py AbandonNoticeLandsInBothArtifacts` -Expected: FAIL — `AttributeError: module 'hil_test' has no attribute 'mark_report_abandoned'` - -- [ ] **Step 3: Implement it** - -```python -def mark_report_abandoned(report_dir: Path, why: str) -> None: - """Stamp an existing report as abandoned, in BOTH artifacts. - - Best-effort and silent: this runs while the interpreter is being torn down, and an - exception here hangs the process in multiprocessing's unbounded join().""" - try: - jpath = report_dir / REPORT_JSON - doc = json.loads(jpath.read_text()) if jpath.is_file() else None - if doc is None: - return - doc['caveat'] = (f'**HIL run abandoned: {why}** The table below is this run\'s ' - f'partial result.\n') - write_report(report_dir, doc) - except (OSError, ValueError, TypeError): - pass -``` - -Then in `_abandon_exit`, replace the read-modify-write of the markdown with `mark_report_abandoned(report, ...)` and change the call site at `:2603` from `report_dir / REPORT_MD` to `report_dir`. - -- [ ] **Step 4: Run the tests** - -Run: `python3 -m unittest discover -s test/hil/test` -Expected: 126 OK. - -- [ ] **Step 5: Commit** - -```bash -git add test/hil/hil_test.py test/hil/test/test_hil_bounded.py -git commit -m "hil_test: stamp abandonment into the document, not onto the markdown - -_abandon_exit did a text prepend on a file it had not written, so the caveat -never reached the JSON and an agent reading the sidecar saw a clean partial -report under a red job. Still best-effort and still silent: it runs while the -interpreter is being torn down." -``` - ---- - -### Task 5: Prove the two artifacts cannot disagree - -**Files:** -- Test: `test/hil/test/test_hil_bounded.py` - -- [ ] **Step 1: Write the test** - -```python -class MarkdownIsAlwaysARenderingOfTheJson(unittest.TestCase): - """The property this whole change buys: whatever wrote the report, re-rendering the - sidecar reproduces the markdown byte for byte.""" - - def _check(self, rd): - import json - doc = json.loads((rd / 'hil_report.json').read_text()) - self.assertEqual((rd / 'hil_report.md').read_text(), - hil_test.render_report(doc) + '\n') - - def test_normal_path(self): - td = TemporaryDirectory(); self.addCleanup(td.cleanup); rd = Path(td.name) - hil_test.accumulate_report( - [('boardA', 0, 0, [('boardA', {'cdc_msc': 'OK'}, '1s')], 0)], rd, True, - '-b boardA', '> **Rig note.** x\n') - self._check(rd) - - def test_after_an_accumulate_rerun(self): - td = TemporaryDirectory(); self.addCleanup(td.cleanup); rd = Path(td.name) - hil_test.accumulate_report( - [('boardA', 0, 0, [('boardA', {'cdc_msc': 'OK'}, '1s')], 0)], rd, True, '', '') - hil_test.accumulate_report( - [('boardB', 0, 0, [('boardB', {'cdc_msc': 'OK'}, '1s')], 0)], rd, False, '', '') - self._check(rd) - - def test_after_abandonment(self): - td = TemporaryDirectory(); self.addCleanup(td.cleanup); rd = Path(td.name) - hil_test.accumulate_report( - [('boardA', 0, 0, [('boardA', {'cdc_msc': 'OK'}, '1s')], 0)], rd, True, '', '') - hil_test.mark_report_abandoned(rd, 'the worker pool would not shut down.') - self._check(rd) - - def test_no_boards_exit(self): - td = TemporaryDirectory(); self.addCleanup(td.cleanup); rd = Path(td.name) - hil_test.write_report(rd, {'rows': [], 'banner': '', 'scope': '', - 'caveat': '**HIL run selected no boards.** why\n'}) - self._check(rd) -``` - -- [ ] **Step 2: Run it** - -Run: `python3 test/hil/test/test_hil_bounded.py MarkdownIsAlwaysARenderingOfTheJson` -Expected: PASS on all four. A failure here means a writer still bypasses `render_report`. - -- [ ] **Step 3: Full gate and commit** - -```bash -python3 -m unittest discover -s test/hil/test # 130 OK -pre-commit run --files test/hil/hil_test.py test/hil/helper/hil_health.py \ - test/hil/test/test_hil_bounded.py test/hil/test/test_hil_health.py -git add test/hil/test/test_hil_bounded.py -git commit -m "test/hil: pin that the markdown is always a rendering of the sidecar - -Four writers, one renderer. This is the invariant the change exists to create, -so it is asserted directly rather than inferred from the writers." -``` - ---- - -## Out of scope - -Deliberately not included, each its own follow-up: - -- **The flat `HIL_POOL_TIMEOUT`.** `hil_test.py:225` is a per-process 3600 s guard that does not scale with board count. It was per board when runs were serial; the origin branch made one run cover the fleet, so a 27-board run shares one budget. Real, and a scheduling change rather than a reporting one. -- **`hil_ci.sh` accumulate in remote mode.** `hil_ci.sh:183` `rm -rf`s `REMOTE_DIR` every run and the copies are one-way, so a remote `--accumulate` retry has no merge base and its one-row report overwrites the local full-fleet one. Fixing that means uploading `hil_report.json` and `<config>.failed` before the run, or keeping `REMOTE_DIR` when `--accumulate` is present. -- **Dropping `hil_report.md` entirely.** Not proposed. It is the PR artifact and what the `hil` skill tells operators to paste; this plan makes it generated, not redundant. diff --git a/docs/superpowers/followup/pr3840-mret-board-result.md b/docs/superpowers/followup/pr3840-mret-board-result.md new file mode 100644 index 000000000..77b76b605 --- /dev/null +++ b/docs/superpowers/followup/pr3840-mret-board-result.md @@ -0,0 +1,99 @@ +# Give the HIL worker result a name + +**Origin:** split out of PR #3840 (making `hil_report.md` a rendering of `hil_report.json`). +Delete this file when its own PR lands. + +> **SUPERSEDED IN PART (2026-08-26).** Written against a 7-field tuple whose index 5 was +> `blind`. The sysfs blindness subsystem is gone: `test_board` now returns **6** fields with +> `stray` at index 5, and its board-locked early return is 5 wide. The problem described +> below is unchanged and still worth fixing — three producers, three widths, and +> `len(r) > 5 and r[5]` reads a WRONG SLOT rather than raising. But drop the `blind` field +> from the proposed NamedTuple and re-derive every index from `hil_test.test_board` before +> executing, or `_stray_note` starts reading a duration as a stray count. +> `StrayNoteSurvivesTheTupleWidth` pins the current shape. + +## What is established + +`test_board()` returns a bare tuple that three producers build and fourteen call sites read +positionally. It has grown 5 → 6 → 7 fields, and the code already works around its own +shape: + +```python +hil_test.py:1992 dirty = [(r[0], r[6]) for r in mret if len(r) > 6 and r[6]] +hil_test.py:2014 blind = [r[0] for r in mret if len(r) > 5 and r[5]] +hil_test.py:2386 for name, _, _, _, dur, *_ in mret: +hil_report.py:306 for name, _, _, rows, *_ in mret: +``` + +Two facts make this worth closing rather than tolerating: + +- **The declared type is already wrong.** `hil_test.py:1711` says + `tuple[str, int, list[str], list, float]` — five fields — while the main return at `:1872` + yields seven (`+ sysfs_blind(), stray`). +- **A wrong slot is a wrong verdict, not a crash.** Field 5 is `blind`, which decides whether + a board's red cells are reported as broken hardware or as "could not tell". Inserting a + field mid-tuple makes `r[5]` read the wrong slot and keep running. + +It has bitten once already: `test_hil_bounded.py`'s +`test_both_row_widths_survive_the_report_writers` exists because the blindness flag widened +the tuple to 6 while the pool-timeout path still synthesised 5-field rows, and *"a +fixed-width unpack in either one raises INSIDE the containment path, which is where a raise +costs every board's results."* That is why the unpacks end in `*_`. + +## What remains + +A `NamedTuple` with defaults. Verified to pickle across the pool boundary and to stay +fully tuple-compatible — existing `r[0]`, `e[1]`, `for name, _, _, rows, *_` and `len(r)` +all keep working, so it lands without touching the fourteen consumers: + +```python +class BoardResult(NamedTuple): + """What one worker returns. Field ORDER is load-bearing: it is unpacked positionally + in a dozen places, and the pool-timeout path synthesises one by hand.""" + name: str + err_count: int + failed_tests: list[str] + rows: list | None # None from the pool-timeout synthesis, never [] + duration: float + blind: bool = False # defaults, so a synthesised result is full-width + stray: int = 0 +``` + +Then a second, smaller step removes the coupling itself: `accumulate_report` takes +`[(name, rows)]` pairs instead of `mret`, and `hil_test` does the extraction because it owns +the shape. One line at each end; the subtle merge logic — stale lock clearing, +`BOUNDARY_CELL`, `duration=None` preservation — is untouched. + +## Sizing + +| | Sites | +|---|---| +| Producers to convert | 4 (`hil_test.py:1724`, `:1872`, `:2283`, `:2327`) | +| Arity guards deleted | 2 (`:1992`, `:2014`) | +| Wrong annotation fixed | 1 (`:1711`) | +| `hil_report`'s coupled line | 1 (`:306`) | +| Positional consumers (optional migration) | 14 | +| **Test fixtures building tuples by hand** | **34** | + +Production code is roughly ten changed lines. **The work is dominated by the test +fixtures**, which is also the risk. + +## Do this first, or the refactor is unverifiable + +`test_hil_report.py` (27 sites) and `test_hil_bounded.py` (7) construct plain tuples by +hand — `('boardA', 0, [], [], 1.0, True)`. A producer that forgot to switch to +`BoardResult`, or a pickling regression, **passes the entire 310-test suite** and surfaces +only on the rig. Convert the fixtures to build `BoardResult` as task 1, before touching any +producer. This ordering is not optional. + +Second trap: `rows` is `None` on the pool-timeout path (`hil_test.py:2283`), never `[]`, and +`accumulate_report` guards with `if rows and ...`. A well-meaning `rows: list = []` default +silently changes that path. Pin it with a test before the conversion. + +## Why it was split out + +PR #3840 touches the report document. This touches `test_board`'s return and the containment +paths, where a raise costs every board's results rather than one board's — a different blast +radius, needing its own review and its own rig run. #3840 is twice-reviewed and dogfooded +ten times on hardware; folding this in would reset that surface for a latent-trap cleanup +that is not causing bugs today. diff --git a/docs/superpowers/followup/pr3840-skill-md-no-boards-drift.md b/docs/superpowers/followup/pr3840-skill-md-no-boards-drift.md new file mode 100644 index 000000000..a039a8c12 --- /dev/null +++ b/docs/superpowers/followup/pr3840-skill-md-no-boards-drift.md @@ -0,0 +1,38 @@ +# `SKILL.md` contradicts the code on no-boards tables + +**Origin:** split out of PR #3840, surfaced by its second review round. Delete this file +when its own PR lands. + +`.claude/skills/hil/SKILL.md:150-151` tells the reading agent: + +> `**HIL run selected no boards.**` — the filters intersected to nothing, so there is **no +> table at all**. Report that (and the filter shown), never `"pass": true`. + +That was true when the no-boards exit wrote a bare notice. It no longer is. An +`--accumulate` no-boards run keeps the accumulated rows — deliberately, because wiping them +destroyed real results — so the artifact now reads: + +``` +**HIL run selected no boards.** filters emptied + +**✅ 1 passed · ❌ 0 failed · ⚪ 0 skipped · blank not run** + +| Board | t | duration | +... +``` + +The behaviour is correct; the documentation is wrong, and wrong in the direction that +matters. An agent is told to expect no table, sees one, and has no rule for whether those +rows are reportable. **They are not this run's** — they are a previous attempt's, carried +forward. + +**What remains:** update that bullet to describe both cases — a fresh run has no table, an +`--accumulate` run shows the previous attempt's rows under the notice and they must not be +reported as this run's. Add a test asserting the fresh case renders no matrix, so the two +halves cannot drift again. + +## Why it was split out + +PR #3840 fixed the findings that changed a verdict. This is a documentation drift: the +behaviour is correct and the doc describing it is not, so it is better reviewed on its own +than appended to a branch already carrying a module consolidation. diff --git a/docs/superpowers/followup/pr3840-write-report-atomicity.md b/docs/superpowers/followup/pr3840-write-report-atomicity.md new file mode 100644 index 000000000..2094207bb --- /dev/null +++ b/docs/superpowers/followup/pr3840-write-report-atomicity.md @@ -0,0 +1,30 @@ +# `write_report` commits the two artifacts non-atomically + +**Origin:** split out of PR #3840, surfaced by its second review round. Delete this file +when its own PR lands. + +```python +md = render_report(doc) + '\n' +report_dir.mkdir(parents=True, exist_ok=True) +(report_dir / REPORT_JSON).write_text(json.dumps(doc, indent=2) + '\n') +(report_dir / REPORT_MD).write_text(md, encoding='utf-8') +``` + +Rendering before writing closed the *render-failure* case: a raise can no longer commit a +sidecar the markdown contradicts. It does not close the *interrupted-between-writes* case. A +kill between those two lines leaves the pair disagreeing — and this runs on the containment +path, on the way to `os._exit`, on a rig whose jobs get cancelled by the GitHub job ceiling. + +**What remains:** write both to temp files, then `os.replace` both. The window shrinks from +two full writes to two renames, and neither file is ever observed half-written. `os.replace` +is atomic per file on POSIX; the pair is still not transactional, which is acceptable and +should be said in the docstring rather than implied away. + +Worth pairing with a test that kills between the writes — or, more practically, one that +asserts no partial file is ever visible by checking the temp-then-rename shape directly. + +## Why it was split out + +A durability edge, not a wrong verdict. PR #3840 closed the render-failure half of this +(nothing is written until the markdown renders); the interrupted-between-writes half needs +a temp-then-rename and is better reviewed on its own. diff --git a/docs/superpowers/followup/pr3851-msc-host-tur-retry.md b/docs/superpowers/followup/pr3851-msc-host-tur-retry.md new file mode 100644 index 000000000..462b90fdf --- /dev/null +++ b/docs/superpowers/followup/pr3851-msc-host-tur-retry.md @@ -0,0 +1,149 @@ +# MSC host: bound the Test Unit Ready retry loop and act on sense data + +> Split out of PR #3851 (`etmtrace-rp2350`, rp2350 ETM trace + stock clocks): +> a host-stack MSC bug with no relation to that branch's scope. + +**Goal:** stop `msch_open`'s enumeration retry from spinning forever when a +device answers Test Unit Ready with CHECK CONDITION, and use the sense data the +driver already fetches to decide whether to keep waiting, give up, or report. + +--- + +## What is already established + +### The loop is unbounded, and the source says so + +`src/class/msc/msc_host.c:445-472` is a two-function cycle with no counter: + +```c +static bool config_test_unit_ready_complete(...) { + if (csw->status == 0) { + ... tuh_msc_read_capacity(...); // ready -> proceed to mount + } else { + // Note: During enumeration, some device fails Test Unit Ready and require a few retries + // with Request Sense to start working !! + // TODO limit number of retries <-- :459, pre-existing + TU_LOG_DRV("SCSI Request Sense\r\n"); + TU_ASSERT(tuh_msc_request_sense(dev_addr, cbw->lun, enum_buf, + config_request_sense_complete, 0)); + } + return true; +} + +static bool config_request_sense_complete(...) { + TU_ASSERT(csw->status == 0); + TU_ASSERT(tuh_msc_test_unit_ready(dev_addr, cbw->lun, + config_test_unit_ready_complete, 0)); // :472 + return true; +} +``` + +Two defects, independent of each other: + +1. **No bound.** TUR fail -> Request Sense -> TUR -> ... forever. `tuh_msc_mount_cb()` + is never called and the application is never told anything; the device sits + enumerated-but-unmounted indefinitely. +2. **Sense data is fetched and discarded.** `config_request_sense_complete` + checks only the CSW status. `enum_buf` holds a `scsi_sense_fixed_resp_t` + whose `sense_key` / ASC / ASCQ distinguish "Not Ready — becoming ready" + (retry is correct) from "Not Ready — medium not present" (a card reader with + no card; retrying can never succeed) from a hard error. The driver cannot + currently tell these apart because it never looks. + +### Measured on hardware (2026-08-25) + +Rig: `raspberry_pi_pico` (RP2040) + Pico-PIO-USB host on GP20/21, probe +`E6614103E719612F`, console over the probe's CDC. Build: +`-DCFG_TUH_RPI_PIO_USB=1 -DLOG=2`. + +- `examples/host/msc_file_explorer` never mounts. Debug log over ~25 s: + **1× `SCSI Test Unit Ready`, 350× `SCSI Request Sense`**, zero + `SCSI Read Capacity`, zero mount callbacks. `dd` reports + `no MSC device mounted`. +- **The transfers themselves all succeed** — every CBW/CSW pair logs `OK` + (`Queue EP 02 with 31 bytes ... OK`, `Queue EP 81 with 13 bytes ... OK`), so + this is a SCSI-state-machine problem, not a bulk-transfer or PIO-USB timing + problem. +- Reproduced with **two different drives** (`24a9:1802` "STORAGE DEVICE" and the + drive swapped in after it), so it is not one device's quirk. +- Control transfers on the same target are fine: `examples/host/device_info` + reads full descriptors from the same drive on the same board + (`bcdUSB 0210`, `bMaxPacketSize0 64`, i.e. full-speed). +- **The very same drive mounts and sustains I/O on RP2350** + (`pico2_etm_trace` carrier): `msc_file_explorer` + `dd` returns + `dd: 524288 bytes in 8448 ms = 62 KB/s`. Confirmed by the maintainer at the + bench, so the device is healthy and the "not ready" answer is provoked by + something specific to the RP2040 setup. +- **Bumping Pico-PIO-USB does not fix it.** Retested with upstream HEAD + `5a37a66` (10 commits ahead of the pinned `675543b`, including + `512d3a2` "Place calc_usb_crc16 in RAM like calc_usb_crc5 and the CRC + tables", which looked like a promising RP2040 timing fix, and `cbf055d` + transaction-length clamp) via `-DPICO_PIO_USB_PATH=<clone>`: identical + failure, no mount. +- Clock is **not** a factor: identical failure at 120 MHz, 133 MHz and + 156 MHz on RP2040 (and on RP2350 all of 120/125/126/138/150/156/162/174/186/240 MHz + behave identically). + +### What is NOT established + +- The actual sense key/ASC/ASCQ the failing drives return — the driver never + logs it. **Task 1 below exists to capture it**, and its answer decides whether + a bounded retry is sufficient or a "medium not present" path is also needed. +- **Why the RP2040 setup provokes the not-ready state.** Leading suspect is + VBUS quality rather than firmware: the RP2350 carrier feeds J5 through a + proper load switch, while the RP2040 rig is a bare Pico whose GP22 "VBUS + enable" drives nothing (no load switch on a bare Pico), so the drive is fed + directly off the VBUS pin through hookup wire. A bus-powered drive that + cannot spin up answers exactly this "not ready" forever. Measure VBUS at the + device under load, or retest with a powered hub / self-powered device, + BEFORE attributing the stall to the host stack. +- The actual sense key (Task 1) — still the gate for any policy change. + +--- + +## What remains + +### Task 1: Log the sense response (diagnostic, ship-able on its own) + +**Files:** `src/class/msc/msc_host.c` (`config_request_sense_complete`, ~:467) + +Add a `TU_LOG_DRV` of `sense_key`, `add_sense_code`, `add_sense_qualifier` from +the fixed-format response in `usbh_get_enum_buf()`. `scsi_sense_fixed_resp_t` is +already declared in `src/class/msc/msc.h`. + +Verify on the rig above: rebuild `msc_file_explorer` with `-DLOG=2`, flash, read +the probe CDC, and record the triple. Expected candidates: +`0x02/0x04/0x01` (becoming ready) or `0x02/0x3A/0x00` (medium not present). + +### Task 2: Bound the retry + +**Files:** `src/class/msc/msc_host.c`, `msch_interface_t` (add a retry counter), +`src/class/msc/msc_host.h` (a `CFG_TUH_MSC_TUR_RETRY_COUNT`-style knob with a +sane default; follow the existing `CFG_TUH_MSC_*` naming in +`src/tusb_option.h`). + +On exhaustion, stop the cycle and surface the failure rather than silently +looping — the application currently has no way to learn the device is stuck. + +### Task 3: Decide behaviour per sense key + +Gated on Task 1's measurement. At minimum: keep retrying on "becoming ready", +stop immediately on "medium not present". Do not invent policy for sense keys +that were not observed. + +### Task 4: Regression coverage + +`test/unit-test/` has no MSC host suite today; adding one means mocking +`tuh_msc_*` completions. Confirm with the maintainer whether a unit test or a +HIL case on a known not-ready device (an empty card reader is the cheap +reproducer) is the wanted evidence before building either. + +--- + +## Why it was split out + +Found while sweeping PIO-USB clocks on the `etmtrace-rp2350` branch, which +touches only rp2040/rp2350 clock pinning and ETM trace config. This bug is in +the class-driver layer, affects every MCU running the MSC host, and predates +that branch (the `// TODO limit number of retries` is already in master). It +deserves its own PR and its own hardware evidence. diff --git a/docs/superpowers/followup/pr3853-board-putchar-logger.md b/docs/superpowers/followup/pr3853-board-putchar-logger.md new file mode 100644 index 000000000..46a4417bd --- /dev/null +++ b/docs/superpowers/followup/pr3853-board-putchar-logger.md @@ -0,0 +1,57 @@ +# `board_putchar` is not LOGGER-aware + +**Origin:** surfaced while validating the RTT console in PR #3853 (the `rtt` skill +promotion), which is harness-only scope. This is a src-level fix to `hw/bsp/board.c` +that touches every board/logger combination, so it needs its own build sweep rather +than a drive-by. Delete this file when its own PR lands. + +## Established (with evidence) + +`hw/bsp/board.c` retargets stdio through `sys_write`/`sys_read`, which are compiled +per logger: `SEGGER_RTT_Write`/`SEGGER_RTT_Read` under `LOGGER_RTT`, ITM under +`LOGGER_SWO`, `board_uart_write`/`board_uart_read` by default. The two board-level +character helpers do not agree: + +```c +168: int board_getchar(void) { +169: char c; +170: return (sys_read(0, &c, 1) > 0) ? (int) c : (-1); +171: } +172: +173: int board_putchar(int c) { +174: if (board_uart_write((const char *)&c, 1) > 0) { +``` + +`board_getchar` follows the logger; `board_putchar` always goes to the UART. So with +`LOGGER=rtt` console input arrives over RTT while the echo goes out the UART. + +Measured on ea4088_quickstart (`LOGGER=rtt`, `board_uart_write` is a `-1` stub on +lpc40): the `board_test` echo vanishes entirely while a `printf` echo — same console, +same keystroke — comes back byte-for-byte. `LOGGER=swo` has the same asymmetry by +construction (ITM out of `sys_write`, UART out of `board_putchar`), unverified on +hardware. + +## What remains + +Candidate fix: route `board_putchar` through `sys_write(0, ...)` for symmetry with +`board_getchar`. Two things to settle while doing it: + +- `board_putchar` currently passes `&c` of an `int` to a `const char*` — it writes + the low byte only on little-endian. Narrow to a `char` local as part of the change. +- The default (UART) path must keep its current return contract: `board_uart_write` + returns negative when the UART is a stub, and the default `sys_write` breaks out of + its retry loop on that, returning a short count — so `board_putchar` still has to + map "wrote nothing" to `-1`. + +## Validation + +Build sweep across loggers and families — at minimum one UART board, one +`LOGGER=rtt` board and one `LOGGER=swo` board — plus a hardware check that the +`board_test` echo comes back on an RTT board (ea4088_quickstart reproduces the bug +today) and that a plain UART board's echo is unchanged. + +## Why it was split out + +PR #3853 promotes a debug-tooling skill and touches `test/hil/*.py` and +`tools/rtt.py`. A `hw/bsp/board.c` change lands in every example on every board and +belongs in a review that carries the build evidence for it. diff --git a/docs/superpowers/followup/pr3853-rtt-harness-adoption.md b/docs/superpowers/followup/pr3853-rtt-harness-adoption.md new file mode 100644 index 000000000..8f3eae16b --- /dev/null +++ b/docs/superpowers/followup/pr3853-rtt-harness-adoption.md @@ -0,0 +1,62 @@ +# Follow-up: finish RTT-console adoption in the HIL harness + +Split out of the `rtt` skill-promotion PR #3853. That PR deliberately ships the skill + CLI and leaves the harness's remaining +VCOM assumptions in place — converting them is separate test-infra scope that +deserves its own review and HIL runs. Scope here is `test/hil/*.py` only; the +src-level `board_putchar` asymmetry this work surfaced has its own handoff +(`pr3853-board-putchar-logger.md`). + +## Established (with evidence) + +- `hil_util.JlinkRtt` + `open_board_console()` work end-to-end: + ea4088_quickstart runs its host suite over RTT (16 passed / 0 failed / 3 + skipped, the 'hil: read the host console over RTT when the probe has no VCOM' commit), and the `rtt` skill's boards.md carries the + validated matrix. +- `test_host_device_info` honors `"logger": "rtt"` (hil_test.py, `test_host_device_info`; the eof fail-fast assert sits in its read loop): + in RTT mode it resets via the flasher BEFORE opening the console (which + then owns the probe; Commander delivers the buffered boot burst) and its + read loop fails fast on `JlinkRtt.eof` instead of blaming the board. + +## Remaining gaps + +1. **`test_host_cdc_msc_hid` and `test_host_msc_file_explorer` (hil_test.py) still call `hil_util.get_serial_dev(flasher["uid"], ...)` + directly** — on a `logger: rtt` board with `is_cdc`/`is_msc` fixtures they + would fail with the same "No serial device found" the console work fixed + for device_info (an interim load-time gate in `hil_test.py` now rejects + that combination up front; delete the gate when this lands). Fix: route + both through `open_board_console(board)` — but design the conversion + reset-aware rather than hand-copying device_info's dual branch: hoist a + `reset=` parameter into `open_board_console` that does the per-console + ordering itself (RTT: reset via flasher BEFORE opening — the console owns + the probe; VCOM: reset after open to catch the banner), and REMOVE the + existing post-open `# reset device to catch mount messages` blocks in both + tests (grep the marker — line numbers churn) — kept as-is on an RTT board they reset + while the console holds the probe. `JlinkRtt` carries input for their + menus and implements the `reset_input_buffer()` those tests call. +2. **`hil_pool_check.check_host_serial` carries its own inline RTT branch** + (reset → `JlinkRtt` → poll through `hil_util.strip_banner`) — RTT boards + ARE health-checkable today, but the console-opening logic now lives in + two places (`open_board_console` in hil_test.py and this branch), each + with its own reset-ordering. Fix: hoist `open_board_console()` into + `hil_util.py` with the `reset=` parameter from item 1 and collapse + pool_check's branch onto it; keep the `do_reset` flush semantics for the + VCOM path intact. +3. **OpenOCD console backend in the harness**: the skill's CLI + (`tools/rtt.py --backend openocd`, class + `OpenocdRtt` in the same module) is built, deduplicated behind a shared + base class next to `JlinkRtt` in `tools/rtt.py`, re-exported by + `hil_util`, and hardware-validated (all 20 rig boards through the CLI on + both backends, incl. the 8 native-probe ones). What remains is only the + `open_board_console` plumbing: choosing `OpenocdRtt` for a + `"logger": "rtt"` board with an openocd/stlink flasher needs the per-test + flashed-ELF path (for the control-block address) and, for stlink + flashers, an openocd target-cfg mapping the roster doesn't carry — until + then the config-load gate keeps rejecting non-jlink rtt boards. + +## Validation for this follow-up + +Run the ea4088 local host suite (a board with a `is_cdc`+`is_msc` capable +device attached to J3, or the rig's frdm_k64f/mimxrt1064 with a temporary +`logger: rtt` entry) so cdc_msc_hid and msc_file_explorer actually execute +over RTT; then a `hil_pool_check.py` pass on a no-VCOM board. Delete this doc +when the follow-up PR lands. diff --git a/docs/superpowers/plans/2026-08-21-hil-report-module.md b/docs/superpowers/plans/2026-08-21-hil-report-module.md new file mode 100644 index 000000000..5a7c832d8 --- /dev/null +++ b/docs/superpowers/plans/2026-08-21-hil-report-module.md @@ -0,0 +1,658 @@ +# hil_report.py Module Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Fold every function that produces, renders, merges or reads `hil_report.json`/`hil_report.md` into one module, `test/hil/helper/hil_report.py`, and take the two fixes that consolidation enables. + +**Architecture:** A new leaf-ish module owns the report document. `hil_test.py` and `hil_health.py` both import it, which dissolves the circular-import constraint that forced `write_timeout_report` to compose its own markdown. The duplicated cell classifier (`cell_kind` in `hil_test`, `cell_state` in `hil_summary`) collapses into one. `hil_summary.py` is deleted and its CLI moves in. + +**Tech Stack:** Python 3.13 stdlib only (`json`, `argparse`, `pathlib`); existing unit suites under `test/hil/test/` run with plain `unittest`. + +**Spec:** `docs/superpowers/specs/2026-08-21-hil-report-module-design.md` + +## Global Constraints + +- **Behaviour-preserving motion.** `hil_test.py`'s CLI, arguments, output and report format stay byte-identical. The two intended exceptions are named in the spec: the `hil_summary.py` → `hil_report.py` CLI path, and `write_timeout_report` rendering instead of concatenating. +- **`hil_report.py` must work in two modes.** It is imported as `helper.hil_report` by `hil_test.py`, and run as a script by the operator (`python3 test/hil/helper/hil_report.py <config> -b BOARD`). A script run puts `test/hil/helper/` on `sys.path`, *not* `test/hil/`, so `from helper import hil_health` fails in that mode. Task 1 pins both modes with tests. +- **Containment paths must never raise.** `mark_report_abandoned` and `write_timeout_report` run while the interpreter is being torn down or on the way to `os._exit`; anything escaping hangs the process in multiprocessing's unbounded `join()`. Their existing broad handlers move with them unchanged. +- **`hil_ci.sh` stages helpers by an explicit list** (`test/hil/hil_ci.sh:222-228`). A helper module missing from it reaches the rig absent, and the run dies with `ImportError` *after* `REMOTE_DIR` has been wiped. `RemoteStaging.test_import_closure_is_staged_to_the_rig` in `test_hil_bounded.py` already enforces this from the AST import closure; Task 1 only has to add the file to the list. +- Run `python3 -m unittest discover -s test/hil/test` (~82 s) before each commit; `pre-commit run --files <changed>` before pushing. + +--- + +### Task 1: The module, the vocabulary, one classifier, and the render half + +**Files:** +- Create: `test/hil/helper/hil_report.py` +- Create: `test/hil/test/test_hil_report.py` +- Modify: `test/hil/hil_test.py:110` (`REPORT_CELL`), `:1715` (`BOUNDARY_CELL`), `:1902-1903` (`REPORT_MD`/`REPORT_JSON`), `:1921-1978` (`render_matrix`), `:1981-2003` (`render_report`), `:67` (imports) +- Modify: `test/hil/hil_ci.sh:222-228` (scp list) +- Modify: `test/hil/test/test_hil_bounded.py` (move `RenderReportIsPureFunctionOfTheDocument` out) + +**Interfaces:** +- Produces: `helper.hil_report` exposing `REPORT_MD`, `REPORT_JSON`, `REPORT_CELL`, `BOUNDARY_CELL`, `LOCKED_CELL`, `cell_state(v) -> str`, `render_matrix(rows_all) -> str`, `render_report(doc) -> str`. +- `hil_test.py` re-exports nothing: call sites become `hil_report.NAME`. + +- [ ] **Step 1: Write the failing tests** + +Create `test/hil/test/test_hil_report.py`: + +```python +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests for the report document: the vocabulary, the one cell classifier, rendering, +# the four writers, and the fold to per-board verdicts. Split out of test_hil_bounded.py +# and test_hil_health.py when the report code moved into helper/hil_report.py. +# Run directly: +# python3 test/hil/test/test_hil_report.py +import json +import os +import subprocess +import sys +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory + +TEST_DIR = os.path.dirname(os.path.abspath(__file__)) +HIL_DIR = os.path.dirname(TEST_DIR) +sys.path.insert(0, HIL_DIR) + +from helper import hil_report + + +class OneClassifierForBothArtifacts(unittest.TestCase): + """The markdown tally and the agent's verdict used to classify cells with two separate + copies of one rule -- hil_test's cell_kind against REPORT_CELL, and hil_summary's + cell_state against its own re-typed '❌'/'⚪' literals. Change the icons and the table + and the verdict silently disagree.""" + + def test_bare_states(self): + self.assertEqual(hil_report.cell_state('fail'), 'fail') + self.assertEqual(hil_report.cell_state('skip'), 'skip') + self.assertEqual(hil_report.cell_state('pass'), 'pass') + + def test_icon_prefixed_metrics_carry_their_verdict(self): + self.assertEqual(hil_report.cell_state(f'{hil_report.REPORT_CELL["fail"]} 29/30'), 'fail') + self.assertEqual(hil_report.cell_state(f'{hil_report.REPORT_CELL["skip"]} board wedged'), + 'skip') + + def test_an_unprefixed_metric_is_a_pass(self): + """Load-bearing: a passing test may return a plain metric string. Classifying + unknown shapes as fail would publish a green table as a red verdict.""" + self.assertEqual(hil_report.cell_state('480.0 MBps'), 'pass') + self.assertEqual(hil_report.cell_state('1103 KB/s'), 'pass') + + def test_a_non_string_cell_does_not_raise(self): + """render_matrix's copy guarded with isinstance; hil_summary's did not, because its + caller str()'d first. The merged one keeps the guard -- it is the safer superset.""" + self.assertEqual(hil_report.cell_state(None), 'pass') + + def test_the_icons_come_from_REPORT_CELL(self): + """No second copy of the emoji anywhere in the module.""" + src = (Path(HIL_DIR) / 'helper' / 'hil_report.py').read_text(encoding='utf-8') + for icon in ('❌', '⚪', '✅'): + self.assertEqual(src.count(f"'{icon}'"), 1, + f'{icon} is spelled as a literal more than once') + + +class ModuleWorksImportedAndAsAScript(unittest.TestCase): + """It is imported as helper.hil_report by hil_test, and run as a script by the operator + (.claude/agents/hil-operator.md). A script run puts helper/ on sys.path, NOT test/hil, + so a plain `from helper import hil_health` breaks the CLI and only the CLI.""" + + def test_importable_as_a_package_module(self): + r = subprocess.run( + [sys.executable, '-c', + f'import sys; sys.path.insert(0, {HIL_DIR!r}); ' + f'from helper import hil_report; print(hil_report.REPORT_JSON)'], + capture_output=True, text=True, timeout=60) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn('hil_report.json', r.stdout) + + def test_runnable_as_a_script(self): + r = subprocess.run( + [sys.executable, str(Path(HIL_DIR) / 'helper' / 'hil_report.py'), '--help'], + capture_output=True, text=True, timeout=60) + self.assertEqual(r.returncode, 0, r.stderr) + + +class HilCiStagesEveryHelperTheRunImports(unittest.TestCase): + """hil_ci.sh copies helper modules by an EXPLICIT list. One missing module reaches the + rig absent and the run dies with ImportError -- after REMOTE_DIR has already been + rm -rf'd, so the previous run's report and re-run spec are gone too.""" + + def test_the_scp_list_covers_what_hil_test_imports(self): + sh = (Path(HIL_DIR) / 'hil_ci.sh').read_text(encoding='utf-8') + staged = {line.split('helper/')[1].rstrip('" \\\n') + for line in sh.splitlines() if '/test/hil/helper/' in line and '.py' in line} + imported = set() + for mod in (Path(HIL_DIR) / 'hil_test.py', Path(HIL_DIR) / 'helper' / 'hil_report.py'): + src = mod.read_text(encoding='utf-8') + for raw in src.splitlines(): + line = raw.strip() # hil_report's own import is indented in a try + if line.startswith('from helper import '): + imported |= {f'{n.strip()}.py' for n in line.split('import', 1)[1].split(',')} + elif line.startswith('from helper.'): + imported.add(line.split('.')[1].split(' ')[0] + '.py') + missing = imported - staged + self.assertEqual(missing, set(), + f'hil_ci.sh does not stage {missing}; a remote run will ImportError') + + +if __name__ == '__main__': + unittest.main() +``` + +Then **move** the class `RenderReportIsPureFunctionOfTheDocument` from `test/hil/test/test_hil_bounded.py` into this file verbatim, changing only `hil_test.render_report` → `hil_report.render_report` throughout. + +- [ ] **Step 2: Run them to verify they fail** + +Run: `python3 test/hil/test/test_hil_report.py` +Expected: FAIL — `ModuleNotFoundError: No module named 'helper.hil_report'` + +- [ ] **Step 3: Create the module** + +Create `test/hil/helper/hil_report.py`: + +```python +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""The HIL report document: one owner for hil_report.json and hil_report.md. + +The markdown IS a rendering of the sidecar -- every writer goes through render_report(), +so a table can never contain something the JSON does not. This module owns the whole life +of that document: the cell vocabulary, the one classifier both artifacts share, rendering, +the four writers, and the fold to one machine-readable verdict per board. + +Dual-mode by design: imported as `helper.hil_report` by hil_test.py, and run as a script by +the operator (see .claude/agents/hil-operator.md). A script run puts test/hil/helper on +sys.path rather than test/hil, hence the guarded hil_health import below. +""" +import argparse +import json +import sys +from pathlib import Path + +try: # imported as part of the helper package + from helper.hil_health import _p +except ImportError: # run as a script: helper/ is sys.path[0] + from hil_health import _p + +REPORT_MD = 'hil_report.md' +REPORT_JSON = 'hil_report.json' +# The status vocabulary, shared by the code that WRITES a cell (hil_test's test runners) and +# the code that reads one back (cell_state). One dict, so the human's table and the agent's +# verdict cannot drift apart. +REPORT_CELL = {'pass': '✅', 'fail': '❌', 'skip': '⚪'} +BOUNDARY_CELL = 'same-PID boundary' +LOCKED_CELL = 'board-locked' + + +def cell_state(v) -> str: + """'pass' | 'fail' | 'skip' for one report cell. + + THE classifier -- the markdown tally and the per-board verdict both call this, so they + cannot disagree. 'fail' or a ❌ prefix is a failure, 'skip' or a ⚪ prefix is a skip, and + EVERYTHING ELSE is a pass. That last arm is load-bearing: a passing test may return a + plain metric string ('480.0 MBps') that lands in the cell unprefixed, while failures are + guaranteed marked -- TestFail's docstring pins that its metric is icon-prefixed precisely + so render and tally treat it as a failure. Classifying unknown shapes as fail here would + publish a green table as a red verdict. + + isinstance-guarded: cells are usually str but a caller may hand over None or a number, + and .startswith on those raises inside a report writer that must not raise.""" + if v == 'fail' or (isinstance(v, str) and v.startswith(REPORT_CELL['fail'])): + return 'fail' + if v == 'skip' or (isinstance(v, str) and v.startswith(REPORT_CELL['skip'])): + return 'skip' + return 'pass' +``` + +Then move, verbatim, from `hil_test.py`: +- `render_matrix` (`hil_test.py:1921-1978`) — with one change: delete its nested `cell_kind` + definition and call the module-level `cell_state` instead. The line + `kinds = [cell_kind(v) for _, cells, _ in rows_all for v in cells.values()]` becomes + `kinds = [cell_state(v) for _, cells, _ in rows_all for v in cells.values()]`. +- `render_report` (`hil_test.py:1981-2003`) — unchanged. + +Add a placeholder CLI so `--help` works (Task 4 fills in `summarize`): + +```python +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument('config_file') + ap.add_argument('-b', '--board', action='append', default=[], + help='boards to report on; default: every board in the config') + ap.add_argument('--report-dir', default='.', help=f'where {REPORT_JSON} lives (default: cwd)') + ap.parse_args() + raise SystemExit('hil_report: summarize() lands in Task 4') + + +if __name__ == '__main__': + sys.exit(main()) +``` + +- [ ] **Step 4: Point `hil_test.py` at the module** + +In `hil_test.py:67`, extend the import: + +```python +from helper import hil_health, hil_lock, hil_report, hil_util +``` + +Delete `REPORT_CELL` (`:110`), `BOUNDARY_CELL` (`:1715`), `REPORT_MD`/`REPORT_JSON` +(`:1902-1903`), `render_matrix` and `render_report` from `hil_test.py`. Then rewrite every +reference to the moved names as `hil_report.<name>`. Find them all with: + +```bash +grep -n "REPORT_CELL\|BOUNDARY_CELL\|REPORT_MD\|REPORT_JSON\|render_matrix\|render_report" \ + test/hil/hil_test.py +``` + +Known sites: `:876`, `:1369`, `:1459`, `:1490`, `:1492`, `:1508`, `:1818`, `:1834`, `:2162`, +`:2191-2192`, `:2209-2210`, `:2403`, `:2592`. + +- [ ] **Step 5: Stage the new module for remote runs** + +In `test/hil/hil_ci.sh:222-228`, add the module to the scp list (keep alphabetical-ish order +with the rest): + +```bash +scp -q "$ROOT_DIR/test/hil/helper/__init__.py" \ + "$ROOT_DIR/test/hil/helper/hil_util.py" \ + "$ROOT_DIR/test/hil/helper/hil_health.py" \ + "$ROOT_DIR/test/hil/helper/hil_lock.py" \ + "$ROOT_DIR/test/hil/helper/hil_report.py" \ + "$ROOT_DIR/test/hil/helper/hil_summary.py" \ + "$ROOT_DIR/test/hil/helper/hil_select.py" \ + "$REMOTE:$REMOTE_DIR/test/hil/helper/" +``` + +- [ ] **Step 6: Run the tests** + +Run: `python3 test/hil/test/test_hil_report.py` → OK +Run: `python3 -m unittest discover -s test/hil/test` → 274 OK (266 + 8 new: 5 classifier, +2 dual-mode, 1 scp guard; `RenderReport…` moves rather than adds) + +- [ ] **Step 7: Commit** + +```bash +git add test/hil/helper/hil_report.py test/hil/hil_test.py test/hil/hil_ci.sh \ + test/hil/test/test_hil_report.py test/hil/test/test_hil_bounded.py +git commit -m "hil_report: new module for the report vocabulary, classifier and rendering + +The markdown tally and the agent's verdict classified cells with two separate +copies of one rule, the second documented as 'the EXACT classifier hil_test.py's +own tally uses'. One cell_state now serves both, keyed off the one REPORT_CELL." +``` + +--- + +### Task 2: Move the three writers + +**Files:** +- Modify: `test/hil/helper/hil_report.py` (add the writers) +- Modify: `test/hil/hil_test.py:2005-2036` (`write_report`, `mark_report_abandoned`), `:2149-2212` (`accumulate_report`) +- Modify: `test/hil/test/test_hil_bounded.py` (move three classes out), `test/hil/test/test_hil_report.py` + +**Interfaces:** +- Consumes: `render_report`, `REPORT_MD`, `REPORT_JSON`, `BOUNDARY_CELL` from Task 1. +- Produces: `hil_report.write_report(report_dir, doc)`, `hil_report.mark_report_abandoned(report_dir, why)`, `hil_report.accumulate_report(mret, report_dir, fresh, scope='', banner='') -> str`. + +- [ ] **Step 1: Move the tests** + +Move these classes from `test/hil/test/test_hil_bounded.py` into `test/hil/test/test_hil_report.py`, +verbatim except `hil_test.<name>` → `hil_report.<name>` for the three moved functions: + +- `ScopeSurvivesInTheJson` +- `EveryExitPathLeavesBothArtifacts` +- `AbandonNoticeLandsInBothArtifacts` +- `CaveatSurvivesAccumulate` +- `MarkdownIsAlwaysARenderingOfTheJson` + +`AbandonNoticeLandsInBothArtifacts.test_an_existing_abandon_caveat_is_not_overwritten` calls +`hil_health.write_timeout_report`; leave that call as-is — Task 3 moves it. + +- [ ] **Step 2: Run them to verify they fail** + +Run: `python3 test/hil/test/test_hil_report.py` +Expected: FAIL — `AttributeError: module 'helper.hil_report' has no attribute 'write_report'` + +- [ ] **Step 3: Move the functions** + +Cut `write_report` (`hil_test.py:2005-2014`), `mark_report_abandoned` (`:2016-2036`) and +`accumulate_report` (`:2149-2212`) from `hil_test.py` and paste them into `hil_report.py` +below `render_report`, unchanged. + +Add to `accumulate_report`'s docstring, after the existing text, so the wart is recorded +where a reader meets it: + +``` + `mret` is hil_test.py's worker-result shape (name, err, fts, rows, ...), so this one + function knows something about its caller that the rest of the module does not. Folding + mret into rows could live in hil_test and only the merge here, but that would rewrite + the subtle parts -- stale board-locked clearing, BOUNDARY_CELL dropping, duration=None + preservation -- for a tidier seam. Data-shape coupling, not an import cycle. +``` + +- [ ] **Step 4: Update the call sites** + +In `hil_test.py`, the three call sites become `hil_report.*`: + +```bash +grep -n "accumulate_report(\|write_report(\|mark_report_abandoned(" test/hil/hil_test.py +``` + +Known sites: `:2260` (inside `_abandon_exit`), `:2351` (no-boards exit), `:2486`, `:2525`, +`:2618`. + +- [ ] **Step 5: Run the tests** + +Run: `python3 -m unittest discover -s test/hil/test` → 274 OK (motion only, no count change) + +- [ ] **Step 6: Commit** + +```bash +git add test/hil/helper/hil_report.py test/hil/hil_test.py \ + test/hil/test/test_hil_report.py test/hil/test/test_hil_bounded.py +git commit -m "hil_report: move the report writers off hil_test + +write_report, mark_report_abandoned and accumulate_report join the renderer they +already call. Pure motion; accumulate_report's knowledge of mret's tuple shape +moves with it and is now documented rather than implicit." +``` + +--- + +### Task 3: `write_timeout_report` renders like everyone else + +**Files:** +- Modify: `test/hil/helper/hil_report.py` (receive the function) +- Modify: `test/hil/helper/hil_health.py:347-398` (remove it), `:19` (drop `import json`) +- Modify: `test/hil/hil_test.py:2498` (call site) +- Modify: `test/hil/test/test_hil_health.py` (move `WriteTimeoutReport` out), `test/hil/test/test_hil_report.py` + +**Interfaces:** +- Consumes: `render_report`, `write_report` from Tasks 1-2. +- Produces: `hil_report.write_timeout_report(report_dir, boards, secs, banner='', prefix='')`. The `md_name` parameter is **gone** — the module owns `REPORT_MD`. + +- [ ] **Step 1: Write the failing tests** + +Move `WriteTimeoutReport` from `test/hil/test/test_hil_health.py` into +`test/hil/test/test_hil_report.py`, changing `hil_health.write_timeout_report` → +`hil_report.write_timeout_report` and dropping the `md_name` argument from every call. Two +of its tests change substantively: + +```python + def test_the_prior_attempts_rows_survive(self): + """Was: the prior MARKDOWN TEXT survives below the banner. It now re-renders from + the merged sidecar, so the guarantee is stated against rows -- one table with the + stuck boards in it, rather than a banner stapled above a duplicate table.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('done', 0, 0, [('done', {'cdc_msc': 'OK'}, '1s')], 0)], rd, True, '', '') + hil_report.write_timeout_report(rd, [{'name': 'stuck'}], 3600) + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertEqual([r['board'] for r in doc['rows']], ['done', 'stuck']) + md = (rd / hil_report.REPORT_MD).read_text() + self.assertIn('done', md) + self.assertIn('stuck', md) + self.assertIn('abandoned', md) + self.assertLess(md.index('abandoned'), md.index('done')) + self.assertEqual(md.count('| Board'), 1, 'the prior table was duplicated, not merged') + + def test_prefix_carries_the_preflight_diagnosis(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.write_timeout_report(rd, [{'name': 'b1'}], 4200, + prefix='> **wedged usb_hub_wq worker.**\n') + out = (rd / hil_report.REPORT_MD).read_text() + self.assertTrue(out.startswith('> **wedged usb_hub_wq worker.**')) + self.assertIn('timed out after 4200s', out) + self.assertIn('b1', out) +``` + +And in `MarkdownIsAlwaysARenderingOfTheJson`, **delete** +`test_the_pool_guard_fallback_agrees_even_if_it_does_not_render` and add the fifth case in +its place: + +```python + def test_the_pool_guard_fallback(self): + """The last writer to join the invariant: it composed its own markdown only because + hil_health could not import the renderer.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('done', 0, 0, [('done', {'cdc_msc': 'OK'}, '1s')], 0)], rd, True, '', '') + hil_report.write_timeout_report(rd, [{'name': 'stuck'}], 3600, + prefix='> **wedged usb_hub_wq worker.**\n') + self._check(rd) +``` + +- [ ] **Step 2: Run them to verify they fail** + +Run: `python3 test/hil/test/test_hil_report.py` +Expected: FAIL — `AttributeError: module 'helper.hil_report' has no attribute 'write_timeout_report'` + +- [ ] **Step 3: Move it and make it render** + +Add to `hil_report.py`, and delete `hil_health.py:347-398` plus its now-unused +`import json` at `hil_health.py:19`: + +```python +def write_timeout_report(report_dir: Path, boards, secs: int, + banner: str = '', prefix: str = '') -> None: + """Leave a report behind when the worker pool has to be abandoned. + + map_async is all-or-nothing, so a timeout loses every per-board result and the report + dir would stay empty with no reason for the failure. Any prior attempt's rows are kept + and the stuck boards are merged in beside them. + + `prefix` carries the preflight rig-health verdict: the timeout aborts before + accumulate_report, so without it the report loses the one line saying WHY the pool never + finished.""" + try: + # Built INSIDE the try: a roster entry without a 'name' key raises while assembling + # the board list, and outside the try that escaped and stranded the runner -- which + # is exactly what the broad handler below exists to prevent. + caveat = (prefix + '\n' if prefix else '') + (banner or ( + f'**HIL run abandoned: worker pool timed out after {secs}s.**\n\n' + f'No per-board results could be collected for this attempt, so any rows below ' + f'are from an earlier one. Boards dispatched:\n\n' + + '\n'.join(f'- {b.get("name", "?")}' for b in boards) + '\n')) + # Rows MERGE rather than replace: an earlier attempt's finished boards are real + # results and this attempt has none of its own. Own handler, because a torn sidecar + # must not cost the stuck rows -- losing the old table is a nicety, losing the + # caveat is the failure. + jpath = report_dir / REPORT_JSON + try: + doc = json.loads(jpath.read_text()) if jpath.is_file() else {} + rows = list(doc.get('rows', [])) + except (OSError, ValueError, TypeError, AttributeError): + doc, rows = {}, [] + done = {r.get('board') for r in rows if isinstance(r, dict)} + rows += [{'board': b.get('name', '?'), 'cells': {'pool-timeout': 'fail'}, + 'duration': None} for b in boards if b.get('name', '?') not in done] + write_report(report_dir, {'rows': rows, 'banner': doc.get('banner', ''), + 'scope': doc.get('scope', ''), 'caveat': caveat}) + except Exception as e: # noqa: BLE001 + # Deliberately broad: this is the first statement of the pool-abandon path, so ANY + # escape skips kill_pool_children and os._exit and strands the runner. + _p(f'warning: cannot write {REPORT_MD} to {report_dir}: {e}', flush=True) +``` + +Update `hil_health.py`'s module docstring: its first line reads "Shutting a wedged HIL run +down: kill what the workers spawned, then report." — drop ", then report". + +- [ ] **Step 4: Update the call site** + +`hil_test.py:2498` becomes: + +```python + hil_report.write_timeout_report( + report_dir, [b for b in config_boards + if b['name'] in stuck], POOL_TIMEOUT, + prefix=health_banner) +``` + +- [ ] **Step 5: Run the tests** + +Run: `python3 -m unittest discover -s test/hil/test` → 274 OK (one deleted, one added) + +- [ ] **Step 6: Commit** + +```bash +git add test/hil/helper/hil_report.py test/hil/helper/hil_health.py test/hil/hil_test.py \ + test/hil/test/test_hil_report.py test/hil/test/test_hil_health.py +git commit -m "hil_report: the pool-guard fallback renders like every other writer + +It composed its own markdown for one reason: hil_health cannot import hil_test +back, so it could not reach render_report. With the renderer in a module both +import, that constraint is gone and all five writers are byte-identical -- +MarkdownIsAlwaysARenderingOfTheJson covers the fifth, and the weaker +'agrees even if it does not render' promise is deleted. + +hil_health goes back to doing one thing: killing wedged processes." +``` + +--- + +### Task 4: Fold `hil_summary.py` in and delete it + +**Files:** +- Modify: `test/hil/helper/hil_report.py` (real `summarize` + CLI) +- Delete: `test/hil/helper/hil_summary.py` +- Modify: `test/hil/hil_ci.sh` (drop `hil_summary.py` from the scp list) +- Modify: `.claude/agents/hil-operator.md:71`, `.claude/workflows/hil-validate.js:14,17,54,58,67`, `.claude/workflows/test-hil-validate.mjs:7` +- Modify: `test/hil/test/test_hil_bounded.py` (move `SummaryFoldsReportToBoards` out), `test/hil/test/test_hil_report.py` + +**Interfaces:** +- Consumes: `cell_state`, `LOCKED_CELL`, `REPORT_JSON` from Task 1. +- Produces: `hil_report.variants_of(cfg, board) -> list`, `hil_report.summarize(cfg, boards, report) -> dict` returning `{'results': [...], 'banner': str, 'caveat': str}`; CLI `python3 test/hil/helper/hil_report.py <config> [-b BOARD]... [--report-dir DIR]`. + +- [ ] **Step 1: Move the tests** + +Move `SummaryFoldsReportToBoards` from `test/hil/test/test_hil_bounded.py` into +`test/hil/test/test_hil_report.py`, changing the subprocess target from +`helper/hil_summary.py` to `helper/hil_report.py` in both places (`test_hil_bounded.py:1675` +and `:1757`). Add one test pinning that the old entry point is gone: + +```python + def test_the_old_entry_point_is_gone(self): + """hil_summary.py's CLI moved here. A leftover file would keep working while + drifting from the module that now owns the fold.""" + self.assertFalse((Path(HIL_DIR) / 'helper' / 'hil_summary.py').exists()) +``` + +- [ ] **Step 2: Run them to verify they fail** + +Run: `python3 test/hil/test/test_hil_report.py` +Expected: FAIL — the subprocess exits non-zero with `hil_report: summarize() lands in Task 4` + +- [ ] **Step 3: Move `summarize` in and delete the old file** + +Copy `variants_of` (`hil_summary.py:47-52`) and `summarize` (`:54-92`) into `hil_report.py` +verbatim, with two changes: `cell_state(str(val))` becomes `cell_state(val)` (the merged +classifier is isinstance-guarded, so the `str()` is dead), and the module's own +`FAIL_ICON`/`SKIP_ICON`/`LOCKED_CELL`/`cell_state` definitions are NOT copied — Task 1's +already serve. + +Replace the Task 1 placeholder `main()` with the real one from `hil_summary.py:94-115`, +changing `Path(a.report_dir) / 'hil_report.json'` to `Path(a.report_dir) / REPORT_JSON`. + +Then: + +```bash +git rm test/hil/helper/hil_summary.py +``` + +- [ ] **Step 4: Update the consumers** + +`test/hil/hil_ci.sh` — remove the `hil_summary.py` line from the scp list added in Task 1. + +`.claude/agents/hil-operator.md:71`: + +```bash +python3 test/hil/helper/hil_report.py <config> -b BOARD [-b BOARD...] # from the report dir +``` + +`.claude/workflows/hil-validate.js:58`: + +```javascript + ` python3 test/hil/helper/hil_report.py <the config you used> ${boards.map((b) => `-b ${b}`).join(' ')}\n` + +``` + +In `.claude/workflows/hil-validate.js` lines 14, 17, 54 and 67, and +`.claude/workflows/test-hil-validate.mjs` line 7, replace the prose mentions of +`hil_summary.py` with `hil_report.py`. Change nothing else in those files — the operator's +return contract (`{results, banner, wedged}`) is untouched. + +- [ ] **Step 5: Run the tests** + +Run: `python3 test/hil/test/test_hil_report.py` → OK +Run: `python3 -m unittest discover -s test/hil/test` → 275 OK +Run: `node .claude/workflows/test-hil-validate.mjs` → OK +Run: `grep -rn "hil_summary" . --include=*.py --include=*.sh --include=*.js --include=*.mjs --include=*.md | grep -v docs/superpowers` → no hits + +- [ ] **Step 6: Commit** + +```bash +git add test/hil/helper/hil_report.py test/hil/hil_ci.sh test/hil/test/ \ + .claude/agents/hil-operator.md .claude/workflows/hil-validate.js \ + .claude/workflows/test-hil-validate.mjs +git rm --cached test/hil/helper/hil_summary.py 2>/dev/null || true +git commit -m "hil_report: fold hil_summary in; one module owns the document end to end + +The fold to per-board verdicts is the read half of the artifact the rest of this +module writes, and it carried the second copy of the cell classifier. The CLI +keeps its arguments; only its path changes, which the two harness docs that +invoke it by name follow." +``` + +--- + +## Validation + +- [ ] **Full gate** + +```bash +python3 -m unittest discover -s test/hil/test # 275 OK +pre-commit run --all-files +``` + +- [ ] **Prove the motion changed no behaviour.** Re-render the real fleet report captured + before the refactor and diff it against what the branch produces now: + +```bash +python3 - <<'EOF' +import json, sys +sys.path.insert(0, 'test/hil') +from helper import hil_report +doc = json.load(open('hil_report.json')) # the pair the rig produced pre-refactor +assert open('hil_report.md').read() == hil_report.render_report(doc) + '\n', 'render drifted' +print('render is byte-identical to the pre-refactor artifact') +EOF +``` + +- [ ] **Rig re-check.** `hil_report.py` must reach the rig and the CLI must run there: + +```bash +bash test/hil/hil_ci.sh -b stm32f407disco -b nanoch32v203 +ssh [email protected] 'cd /tmp/tinyusb-hil && python3 test/hil/helper/hil_report.py \ + test/hil/tinyusb.json -b stm32f407disco -b nanoch32v203' +``` + +Expect a two-board table, `md == render_report(json)`, and a `summarize` verdict naming both +boards — `nanoch32v203` proving the variant fold still works through the moved code. + +## Out of scope + +Each its own follow-up, unchanged from the spec: + +- Splitting `accumulate_report`'s `mret` folding from its merge. +- The flat `HIL_POOL_TIMEOUT` that does not scale with board count. +- Carrying `caveat` through the operator/workflow return contract (`hil-validate.js:34`). diff --git a/docs/superpowers/plans/2026-08-24-rtt-skill.md b/docs/superpowers/plans/2026-08-24-rtt-skill.md new file mode 100644 index 000000000..e2a40c448 --- /dev/null +++ b/docs/superpowers/plans/2026-08-24-rtt-skill.md @@ -0,0 +1,423 @@ +# `rtt` Skill Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Promote SEGGER RTT to a standalone skill `.claude/skills/rtt/` (transport core + console layer) with a versioned CLI, validated first on the local htpc bench, then across the ci.lan rig. + +**Architecture:** Knowledge lives in `.claude/skills/rtt/SKILL.md` + `boards.md`; the single code implementation is `test/hil/helper/hil_util.py::RttConsole` (cherry-picked from branch `hil-add-ea4088qs`) exposed via a thin CLI `test/hil/helper/rtt.py`. Existing docs (target-debug, CLAUDE.md, hil) shrink their RTT recipes to pointers. + +**Tech Stack:** Python 3 (stdlib only, matching hil_util), JLinkExe, OpenOCD, TinyUSB `LOGGER=rtt` builds, TDD-for-skills (superpowers:writing-skills). + +> **Historical record — EXECUTED 2026-08-24/25.** The shipped shape evolved past +> this plan during review rounds: the implementation is `tools/rtt.py` (classes +> `JlinkRtt`/`OpenocdRtt`, `--backend` required), not `test/hil/helper/`. The +> spec's "Tooling home" section is the current truth; do not re-execute this plan. + +**Spec:** `docs/superpowers/specs/2026-08-24-rtt-skill-design.md` — read it first; every content decision below argues from it. + +## Global Constraints + +- Branch: `rttconsole-skill`, worktree `/home/hathach/.herdr/worktrees/tinyusb/rttconsole-skill`. Never touch the primary checkout's branch. +- Commit messages: imperative mood, **no `Co-Authored-By:`/`Claude-Session:` trailers, no footers of any kind** (user's standing authorship rule — overrides harness defaults). +- **Never push.** Commit locally; final report says "ready to push". +- Curated-skills rule: smallest possible diffs to existing skills/agents/CLAUDE.md; anything beyond the pointer edits listed here must be proposed to the user first. +- Iron Law (superpowers:writing-skills): no SKILL.md content and no edit to an existing skill without a failing/baseline test first. +- Hardware rules: **never point OpenOCD at a J-Link-firmware probe** (LPC-Link2 611000000, the J-Trace (nickname `jtrace`; its serial is private — read it with ShowEmuList on the bench) — it drops them off USB; each attempt costs the user a physical replug). J-Trace is wired to raspberry_pi_pico2 (never set a custom JLinkScript for RP2350). Prefix any step needing the user's hands with **[ACTION]**. +- ci.lan rig work: hold per-board locks per `.claude/skills/hil/SKILL.md` §Board locks; the actions-runner keeps running. Use the hil-operator agent for rig sweeps (strictly one instance). +- Scratch files go in the session scratchpad, never `/tmp`, never committed. +- `pre-commit run --all-files` must pass before declaring done. + +--- + +### Task 1: Bring the tooling onto this branch + +**Files:** +- Modify: `test/hil/helper/hil_util.py` (via cherry-pick + docstring fix) +- Modify: `test/hil/hil_test.py` (via cherry-pick) + +**Interfaces:** +- Produces: `hil_util.RttConsole(board: dict, timeout: float = 0.1)` where `board = {'flasher': {'uid': '<probe-serial>', 'args': '-device <JLINK_DEVICE>'}}`; methods `read(size)->bytes`, `write(bytes)->int`, `in_waiting->int`, `close()`, attr `timeout`. Also `hil_test.open_board_console(board)`. + +- [ ] **Step 1: Symlink missing deps** (worktree has `lib/SEGGER_RTT` but not the MCU SDKs): + +```bash +cd /home/hathach/.herdr/worktrees/tinyusb/rttconsole-skill +python3 - <<'EOF' +import os, sys +sys.path.insert(0, 'tools'); import get_deps +main = os.path.expanduser('~/code/tinyusb') +for dep in get_deps.deps_all: + src, dst = os.path.join(main, dep), dep + if not os.path.exists(dst) and os.path.isdir(src): + os.makedirs(os.path.dirname(dst), exist_ok=True); os.symlink(src, dst); print('link', dep) +EOF +``` + +- [ ] **Step 2: Cherry-pick the console commit** (object store is shared across worktrees): + +```bash +git cherry-pick d98e77bac +``` + +Expected: clean pick of `hil: read the host console over RTT when the probe has no VCOM` (touches only hil_util.py + hil_test.py). If it conflicts, resolve keeping d98e77bac's hunks verbatim — master has not touched these regions. + +- [ ] **Step 3: Fix the stale docstring.** `RttConsole`'s docstring opens with "JLinkGDBServer owns the probe and serves RTT channel 0 over TCP" but the code launches `JLinkExe` (J-Link Commander). Edit the docstring's first paragraph to: + +``` + J-Link Commander (JLinkExe) owns the probe and serves RTT channel 0 on -RTTTelnetPort -- + what JLinkRTTClient talks to, minus its banner. Exposes the slice of pyserial the tests + use (read, in_waiting, write, close, timeout) so a caller does not care which console it got. +``` + +- [ ] **Step 4: Import smoke test:** + +```bash +python3 -c "import sys; sys.path.insert(0,'test/hil/helper'); import hil_util; print(hil_util.RttConsole.__doc__.splitlines()[1].strip()[:20])" +``` + +Expected: `J-Link Commander (JL` + +- [ ] **Step 5: Commit** + +```bash +git add test/hil/helper/hil_util.py +git commit -m "hil: RttConsole docstring names the tool it actually runs (JLinkExe)" +``` + +--- + +### Task 2: RED — baseline scenarios without the skill + +Per superpowers:writing-skills, run the failing test before writing any skill text. These are **plan-only** subagents (they must output the exact commands they would run and MUST NOT execute anything against hardware — a wrong baseline attempt costs a probe replug). The lpc4088 session's real lost hour is the primary RED datapoint; these probes map the gap precisely. + +**Files:** +- Create: `<scratchpad>/rtt-baselines.md` (verbatim findings; not committed) + +- [ ] **Step 1: Scenario S1 (console/harness routing + technique).** Dispatch a general-purpose subagent, no mention of RTT: + +> In the TinyUSB repo at /home/hathach/.herdr/worktrees/tinyusb/rttconsole-skill: board ea4088_quickstart is flashed via an LPC-Link2 running J-Link firmware (serial 611000000). The probe exposes no VCOM and hw/bsp/lpc40/family.c's board_uart_read/write return -1. PLAN ONLY — do not run any hardware command. First list which repo skill(s) (.claude/skills/) you would load for this task and why. Then produce the exact commands to (a) get the firmware's printf/TU_LOG output on this PC headlessly and (b) send keystrokes to the firmware. State every failure mode you anticipate. + +- [ ] **Step 2: Scenario S2 (capture technique, OpenOCD/ST-Link).** Same rules: + +> PLAN ONLY. TinyUSB repo, board stm32h743nucleo flashed over an ST-Link. The firmware was built with LOG=2 LOGGER=rtt. Produce the exact commands to capture 20 seconds of its RTT log headlessly on Linux, and explain how you locate the RTT control block and what can go wrong right after a reset. + +- [ ] **Step 3: Record baseline verbatim** in `<scratchpad>/rtt-baselines.md`: which skills each agent said it would load (expected gap: nothing routes, or target-debug loaded for a non-debugging task), which tool each picked (expected: JLinkRTTLogger or bare JLinkGDBServer for S1; full-RAM `rtt setup` scan for S2), which known gotchas each missed (control-block-after-first-printf, probe-by-serial, exact CB address via nm, attach-only after flash-reset, drain-limited/lossy, probe ownership). Every missed item becomes required SKILL.md content; every wrong routing becomes description-keyword input. + +- [ ] **Step 4: Gate.** If a baseline agent nails everything (no gaps), STOP and tell the user — the skill may not be needed in that area and the plan's GREEN content shrinks. (Do not expect this; the lpc4088 session is an existence proof of the failure.) + +--- + +### Task 3: `rtt.py` CLI (TDD) + +**Files:** +- Create: `test/hil/helper/rtt.py` +- Test: fake-probe harness in `<scratchpad>/fakejlink/` (not committed) + +**Interfaces:** +- Consumes: `hil_util.RttConsole` from Task 1. +- Produces: CLI `python3 test/hil/helper/rtt.py --probe <serial> --device <JLINK_DEVICE> [--seconds N] [-i]` — streams channel-0 bytes to stdout; `--seconds 0` (default) runs until Ctrl-C/EOF; `-i` forwards stdin to the target. Exit 0 on clean close, 1 on connect failure. + +- [ ] **Step 1: Write the fake probe** `<scratchpad>/fakejlink/JLinkExe` (`chmod +x`): + +```python +#!/usr/bin/env python3 +# Stands in for J-Link Commander: serves -RTTTelnetPort, greets, echoes input back +# uppercased, exits when stdin says exit (mirrors RttConsole's close() contract). +import socket, sys, threading +port = int(sys.argv[sys.argv.index('-RTTTelnetPort') + 1]) +srv = socket.socket(); srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +srv.bind(('127.0.0.1', port)); srv.listen(1) +def serve(): + conn, _ = srv.accept() + conn.sendall(b'hello from target\r\n') + while True: + d = conn.recv(4096) + if not d: return + conn.sendall(d.upper()) +threading.Thread(target=serve, daemon=True).start() +for line in sys.stdin: + if line.strip() == 'exit': break +``` + +- [ ] **Step 2: Run the failing test:** + +```bash +cd /home/hathach/.herdr/worktrees/tinyusb/rttconsole-skill +PATH=<scratchpad>/fakejlink:$PATH timeout 15 python3 test/hil/helper/rtt.py --probe 000 --device FAKE --seconds 2 +``` + +Expected: FAIL — `No such file or directory` (rtt.py does not exist). + +- [ ] **Step 3: Implement** `test/hil/helper/rtt.py`: + +```python +#!/usr/bin/env python3 +"""Stream a board's RTT channel-0 console to stdout over a J-Link probe. + +Thin CLI over hil_util.RttConsole -- the same implementation the HIL harness uses. +The probe is owned for the whole run: flash and reset BEFORE starting this, never +reset the target while it is attached. Select the probe by serial; rigs run several. +""" +import argparse +import sys +import threading +import time + +import hil_util # same directory when run by path + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument('--probe', required=True, help='J-Link probe serial (JLinkExe -USB value)') + ap.add_argument('--device', required=True, help='JLINK_DEVICE string from the board.cmake/family.cmake') + ap.add_argument('--seconds', type=float, default=0, help='capture duration; 0 = until Ctrl-C/EOF') + ap.add_argument('-i', '--interactive', action='store_true', help='forward stdin to the target') + args = ap.parse_args() + + board = {'flasher': {'uid': args.probe, 'args': f'-device {args.device}'}} + try: + con = hil_util.RttConsole(board, timeout=0.1) + except RuntimeError as e: + print(e, file=sys.stderr) + return 1 + + if args.interactive: + def pump_stdin(): + for line in sys.stdin: + con.write(line.encode()) + threading.Thread(target=pump_stdin, daemon=True).start() + + deadline = time.monotonic() + args.seconds if args.seconds else None + try: + while deadline is None or time.monotonic() < deadline: + chunk = con.read(con.in_waiting or 1) + if chunk: + sys.stdout.buffer.write(chunk) + sys.stdout.buffer.flush() + except KeyboardInterrupt: + pass + finally: + con.close() + return 0 + + +if __name__ == '__main__': + sys.exit(main()) +``` + +- [ ] **Step 4: Run the tests, verify they pass:** + +```bash +P=<scratchpad>/fakejlink +PATH=$P:$PATH timeout 15 python3 test/hil/helper/rtt.py --probe 000 --device FAKE --seconds 2 # expect: hello from target +echo hi | PATH=$P:$PATH timeout 15 python3 test/hil/helper/rtt.py --probe 000 --device FAKE --seconds 2 -i # expect: hello from target + HI +pgrep -f '[J]LinkExe -USB 000' && echo LEAK || echo CLEAN # expect: CLEAN (bracket: else pgrep matches its own shell) +``` + +- [ ] **Step 5: Commit** + +```bash +git add test/hil/helper/rtt.py +git commit -m "hil: add rtt.py, a CLI over RttConsole" +``` + +--- + +### Task 4: GREEN — write `.claude/skills/rtt/SKILL.md` + `boards.md` skeleton + +Write the skill addressing Task 2's recorded failures — nothing more (minimal GREEN). All facts below are established in the spec; the drafting job is assembling them into the sibling-skill shape (structure model: `sysview` SKILL.md; ~150–200 lines). + +**Files:** +- Create: `.claude/skills/rtt/SKILL.md` +- Create: `.claude/skills/rtt/boards.md` + +- [ ] **Step 1: Frontmatter.** Name `rtt`. Description (trigger-only, third person, no workflow — superpowers:writing-skills SDO; extend with keywords from Task 2's routing misses): + +```yaml +--- +name: rtt +description: Use when you need console or printf I/O, TU_LOG capture, or a raw byte channel over a debug probe on real hardware — the board has no UART wired or its probe no VCOM, a LOGGER=rtt build needs reading or writing, "RTT Control Block not found", an RTT server won't come up or drops output, JLinkRTTLogger/JLinkRTTClient/JLinkGDBServer/openocd rtt misbehave, or another workflow (HIL console, SystemView capture) needs RTT stood up on a J-Link, ST-Link, CMSIS-DAP or WCH-Link probe. +--- +``` + +- [ ] **Step 2: Body sections**, each carrying exactly this content (wording final at execution, facts verbatim from the spec): + 1. **Overview** — RTT is nothing but RAM (control block `_SEGGER_RTT`, magic "SEGGER RTT", up/down rings `{sName,pBuffer,SizeOfBuffer,WrOff,RdOff,Flags}`); host must write RdOff back to drain; channel 0 = console, SystemView's "SysView" buffer coexists. + 2. **When to use / when not** — console & capture here; timing/profiling → etm-trace/sysview; debugging decision flows → target-debug; Espressif console → esp-target-debug. + 3. **Transport matrix (quick reference table)** — spec §v1 backend matrix verbatim, per-TRANSPORT rows: ARM memory-AP (live, zero intrusion) / RISC-V SBA (live where implemented) / WCH SDI (**dump only, never live** — DM reads kill USB ~1.9 s in) / OpenOCD-on-J-Link-fw-probe (forbidden, USB drop + physical replug). + 4. **Console (bidirectional)** — `LOGGER=rtt` builds route TU_LOG + `sys_read` to channel 0 (`hw/bsp/board.c`); tooling `test/hil/helper/rtt.py` (CLI) / `hil_util.RttConsole` (harness, `"logger": "rtt"` board switch); flash+reset BEFORE opening, console owns the probe. + 5. **Capture: J-Link route** — `JLinkExe -USB <sn> -device <dev> -if swd -speed 4000 -NoGui 1 -AutoConnect 1 -RTTTelnetPort <port>` + socket/`nc`; proven standalone. `JLinkGDBServer -RTTTelnetPort` locates the block on some parts only with a GDB client attached (LPC4088 measured) — per-part variance, use JLinkExe when headless. `JLinkRTTLogger`: never (single search at attach, 0/6 measured). + 6. **Capture: OpenOCD route (native probes)** — exact CB address first (`arm-none-eabi-nm <elf> | grep _SEGGER_RTT`), then `-c 'rtt setup <addr> 0x1000 "SEGGER RTT"' -c 'rtt polling_interval 1' -c 'rtt start' -c 'rtt server start <port> 0'`; attach without reset when the flash step already reset (SAMD5x DSU `reset run` leaves the core held); read path validated on 13 boards (sysview campaign), write path per boards.md. + 7. **Post-mortem** — undrained NO_BLOCK_SKIP ring holds the FIRST KB after boot, not the wedge tail; overwrite mode (`SEGGER_RTT_WriteWithOverwriteNoLock`) keeps the last N bytes with no live host; manual ring read: `nm` the ELF for `_SEGGER_RTT`, `mem32` the aUp[0] descriptor, `savebin` the buffer — debug-AP reads don't halt the target (moved here from target-debug). + 8. **Buffer modes & locking** — SKIP/TRIM/BLOCK (BLOCK spins the target — dangerous in ISRs); non-ARM ports must supply `SEGGER_RTT_LOCK/UNLOCK` (worked example: `hw/bsp/ch583/sysview_rtt_lock_wch.h` on branch `claude/add-systemview-debug` — generic RISC-V lock traps mcause=2 on QingKe). + 9. **Common mistakes** — attach before first printf (block doesn't exist yet); reset while attached; probe not pinned by serial; two probes on one SWD header; treating RTT as lossless (24.6 KiB/s drain measured, drops at the target); full-RAM scan matching stale RAM after soft reset. + 10. **Per-board notes** → pointer to `boards.md`. + +- [ ] **Step 3: `boards.md` skeleton** — header modeled on sysview's boards.md (row = board, probe/transport, backend+direction validated, JLINK_DEVICE/openocd cfg, caveats), plus the two measured rows seeded from the spec: `ea4088_quickstart` (J-Link/LPC-Link2 611000000, read+write-accepted, `LPC4088`, "probe has no VCOM; BSP has no UART; never OpenOCD on this probe") and a placeholder-free note that all further rows land during Tasks 7–8 validation (no unvalidated rows allowed). + +- [ ] **Step 4: Length check:** `wc -l .claude/skills/rtt/SKILL.md` — expect ≤ ~200 (siblings: hil 168, etm-trace 203). + +- [ ] **Step 5: Commit** + +```bash +git add .claude/skills/rtt/ +git commit -m "skills: add rtt - RTT transport and console reference" +``` + +--- + +### Task 5: GREEN verification + REFACTOR + +- [ ] **Step 1: Re-run S1 and S2** (Task 2 prompts verbatim, still plan-only) with fresh subagents. Success criteria: S1 routes to the `rtt` skill, picks `rtt.py`/JLinkExe route, names probe-by-serial + flash-before-attach; S2 uses exact CB address via `nm`, attach-only, and the openocd command block. +- [ ] **Step 2: REFACTOR.** Any missed item or new wrong turn → tighten the specific SKILL.md section (form per writing-skills "Match the Form to the Failure": these are technique/reference failures → recipes and required table slots, not prohibitions) → re-run that scenario until it passes. +- [ ] **Step 3: Commit** (`git add .claude/skills/rtt/SKILL.md && git commit -m "skills: rtt - close gaps found in scenario verification"`) — only if Step 2 changed anything. + +--- + +### Task 6: Pointer edits in existing docs + +Iron Law for skill edits: the failing test is S3 below, run BEFORE editing. + +**Files:** +- Modify: `.claude/skills/target-debug/SKILL.md:224-253` +- Modify: `CLAUDE.md:77` +- Modify: `.claude/skills/hil/SKILL.md` (one added line) + +- [ ] **Step 1: S3 baseline (failing test).** Plan-only subagent: + +> PLAN ONLY. In this TinyUSB repo, a HIL host test on a board whose flasher probe has no VCOM fails with "No serial device found for /dev/serial/by-id/usb-*_<uid>-if*". Which repo skill(s) would you load, and what is the fix path? + +Expected FAIL today: the agent loads `hil` (correct routing) but `hil` says nothing about RTT consoles, so the fix path is rediscovery. Record verbatim. + +- [ ] **Step 2: Edit `hil/SKILL.md`** — add one line under its Prerequisites section (placement judgment at execution; content fixed): + +``` +- A board whose probe has no VCOM (or whose BSP has no UART) uses RTT as its console: `"logger": "rtt"` + `"build": {"args": ["LOGGER=rtt"]}` in its config entry — see the rtt skill. +``` + +- [ ] **Step 3: Edit `target-debug/SKILL.md`.** (a) Replace the two RTT lines of the capture block at 224-226 with: + +```bash +# RTT (probe console; details, servers, gotchas: rtt skill): +timeout 20s python3 test/hil/helper/rtt.py --probe <sn> --device <JLINK_DEVICE> > /tmp/rtt.log +``` + +(b) Replace the OpenOCD RTT block (232-237) with the single line: `` OpenOCD RTT (native probes): rtt skill §OpenOCD — exact CB address from `nm`, attach-only. `` Keep the drain-preference sentence that follows. (c) Keep the drain-model paragraph (242-247) unchanged; replace 248-253 (GDBServer/RTTLogger/manual-ring-read) with: + +``` +Stand up the drain per the **rtt** skill: JLinkExe's `-RTTTelnetPort` is the +headless-proven route; GDBServer's needs a GDB client on some parts, and +JLinkRTTLogger never works. The manual ring read for a wedged target +(`nm`/`mem32`/`savebin`) lives there too. +``` + +(d) Line 334's correlation one-liner: swap `JLinkRTTClient` for the `rtt.py` invocation from (a). Keep the capture-channel table rows 64-65 unchanged. + +- [ ] **Step 4: Edit `CLAUDE.md:77`** to: + +``` +**RTT:** build `LOG=2 LOGGER=rtt`; capture/console via the `rtt` skill (`.claude/skills/rtt/SKILL.md`). +``` + +- [ ] **Step 5: GREEN for the edits.** Re-run S3 (expect: hil → rtt route, `logger: rtt` fix path) AND re-run S1 once more (expect: unchanged pass — the removed target-debug text must be reachable through the pointers). Also grep for dangling references: `grep -rn "JLinkRTTClient\|RTTTelnetPort" CLAUDE.md .claude/ | grep -v skills/rtt` — every remaining hit must be a deliberate pointer or the sysview branch's own copy. + +- [ ] **Step 6: Commit** + +```bash +git add .claude/skills/target-debug/SKILL.md .claude/skills/hil/SKILL.md CLAUDE.md +git commit -m "docs: route RTT recipes through the rtt skill" +``` + +--- + +### Task 7: Dogfood on the local htpc bench + +Follow ONLY the SKILL.md text (dogfood discipline: gaps found here are REFACTOR input, fixed in SKILL.md before moving on). **[ACTION]-gate with the user before first hardware touch**: confirm LPC-Link2 (611000000) is back on USB and J-Trace (`jtrace`) is on pico2 with pico2 powered. + +**Files:** +- Modify: `.claude/skills/rtt/boards.md` (validated rows) +- Modify: `.claude/skills/rtt/SKILL.md` (only if dogfood exposes gaps) +- Create: `test/hil/local.json` (untracked — copy from the lpc4088 worktree) + +- [ ] **Step 1: Probe roster check:** `JLinkExe -CommandFile <(echo -e 'ShowEmuList\nexit')` (or `lsusb`) — expect 611000000 and the jtrace probe. Missing probe → **[ACTION]** ask the user, do not improvise. + +- [ ] **Step 2: ea4088 bidirectional echo (board_test).** Build + flash + echo, exactly as SKILL.md describes it: + +```bash +cd examples/device/board_test && mkdir -p build-ea4088 && cd build-ea4088 +cmake -DBOARD=ea4088_quickstart -DLOG=2 -DLOGGER=rtt -G Ninja -DCMAKE_BUILD_TYPE=MinSizeRel .. && cmake --build . +ninja board_test-jlink # flashes via the LPC-Link2; resets the target +cd ../../../.. +(sleep 1; echo ping) | timeout 15 python3 test/hil/helper/rtt.py --probe 611000000 --device LPC4088 --seconds 8 -i | tee <scratchpad>/ea4088-echo.log +``` + +Expected: board_test's periodic print lines AND the echoed `ping` (board_test echoes `board_getchar()`). This is the first true validation of target-side console INPUT consumption (the 8550-byte measurement only proved the socket accepted the bytes). + +- [ ] **Step 3: ea4088 HIL host suite over RTT.** Copy the untracked config: `cp /home/hathach/.herdr/worktrees/tinyusb/hil-add-ea4088qs/test/hil/local.json test/hil/local.json`. Build the full example set (`cd examples && cmake -B cmake-build-ea4088_quickstart -DBOARD=ea4088_quickstart -G Ninja -DCMAKE_BUILD_TYPE=MinSizeRel . && cmake --build cmake-build-ea4088_quickstart` — LOGGER=rtt comes from local.json's `build.args`; verify the harness applies it, else add `-DLOGGER=rtt -DLOG=2`). Run per `.claude/skills/hil/SKILL.md` §Local execution against `local.json`. Expected: ≥ 16 passed / 0 failed (parity with d98e77bac's measured result). + +- [ ] **Step 4: pico2 second-probe/second-architecture capture.** Two J-Links are attached — the flash target MUST pin the probe: + +```bash +cd examples/device/cdc_msc && mkdir -p build-pico2 && cd build-pico2 +cmake -DBOARD=raspberry_pi_pico2 -DLOG=2 -DLOGGER=rtt -DJLINK_OPTION="-USB <jtrace-serial>" -G Ninja -DCMAKE_BUILD_TYPE=MinSizeRel .. && cmake --build . +ninja cdc_msc-jlink +cd ../../../.. +timeout 15 python3 test/hil/helper/rtt.py --probe <jtrace-serial> --device rp2350_m33_0 --seconds 8 | tee <scratchpad>/pico2-rtt.log +``` + +(Verify `-DJLINK_OPTION` is the pin mechanism in `hw/bsp/rp2040/family.cmake` before flashing; if the variable differs, use the family's actual one — do NOT flash with an unpinned `-jlink` target.) Expected: TinyUSB init/TU_LOG lines. Silence → check SKILL.md's own troubleshooting first (block-after-first-printf, wrong device string); if it doesn't resolve the silence, that's a dogfood gap → REFACTOR. + +- [ ] **Step 5: Record boards.md rows** for ea4088_quickstart (upgrade: write path VALIDATED via echo) and raspberry_pi_pico2 (J-Trace, `rp2350_m33_0`, "pin probe by serial — bench runs two J-Links; never a custom JLinkScript"). Apply any SKILL.md refactors the dogfood forced. + +- [ ] **Step 6: Commit** + +```bash +git add .claude/skills/rtt/ +git commit -m "skills: rtt - htpc dogfood rows (ea4088 bidirectional, pico2 capture)" +``` + +--- + +### Task 8: ci.lan rig sweep — all applicable boards + +Goal: a boards.md row per rig board, per its transport. Drive hardware through the hil-operator agent (one instance), locks per hil skill. Builds: `LOGGER=rtt LOG=2` `board_test` per board (echo validates both directions where the backend supports writes). Firmware left on boards is fine — CI reflashes every run. + +**Files:** +- Modify: `.claude/skills/rtt/boards.md` +- Create: `<scratchpad>/rtt_sweep/` (per-board logs; not committed) + +- [ ] **Step 1: Build matrix.** From `test/hil/tinyusb.json` take all boards; groups: jlink×12, openocd×9, stlink×3; excluded with reasons recorded in boards.md: esptool×2 (no SEGGER-RTT path in our builds — USB-Serial-JTAG console), ek_tm4c123gxl (lm4flash only, no probe path configured on the rig). For each included board build `examples/device/board_test` with `-DLOG=2 -DLOGGER=rtt` locally where the toolchain exists (arm-none-eabi covers all but WCH); WCH boards (nanoch32v203, ch32v103, ch32v307, ch582m): build only if the riscv toolchain is present locally or on ci.lan — otherwise record `skipped: no riscv toolchain` rather than silently dropping (no silent caps). + +- [ ] **Step 2: Stage on ci.lan:** `scp` each ELF/bin + `test/hil/helper/{hil_util.py,rtt.py}` to `[email protected]:~/rtt-sweep/`. + +- [ ] **Step 3: Per-board procedure** (hil-operator executes on ci.lan; lock → flash → capture → echo → release): + - **jlink boards:** flash with the board's rig flasher recipe (uid + `-device` from tinyusb.json `flasher.args`), then `(sleep 1; echo ping) | timeout 15 python3 ~/rtt-sweep/rtt.py --probe <uid> --device <dev> --seconds 8 -i`. PASS = periodic board_test output + `ping` echoed. + - **stlink + openocd boards (native probes):** CB address from the local ELF (`arm-none-eabi-nm board_test.elf | grep _SEGGER_RTT`, computed before scp, carried in the sweep table). Then on ci.lan, one session per board using the board's existing openocd args from tinyusb.json plus: `-c 'adapter serial <uid>' -c 'rtt setup <addr> 0x1000 "SEGGER RTT"' -c 'rtt polling_interval 1' -c 'rtt start' -c 'rtt server start <port> 0'`; attach WITHOUT reset (flash already reset it). Read: `timeout 8 nc localhost <port>`. Write test: `(sleep 1; echo ping; sleep 3) | nc localhost <port>` — PASS/FAIL per direction recorded separately; a write failure here is a finding, not a blocker (spec: OpenOCD write path is the open question this phase answers). + - **WCH boards (WCH-Link, SDI):** NO live streaming, NO rtt server during USB traffic. Validation = post-mortem-style read only: flash, let it run 5 s, then `halt; read the ring via nm address + mdw/dump_image; resume` in one short openocd/wlink session. PASS = ring contains board_test's boot output. Any anomaly → stop, quiesce the DM (rig standing rule), record. +- [ ] **Step 4: Per-board rows into boards.md** — board, transport, read/write verdicts, device string / cfg, caveat. Every board in tinyusb.json appears: validated, failed (with symptom), or skipped (with reason). If OpenOCD write path validated, update SKILL.md's transport matrix row; if not, matrix row says "read-only validated; write untested/failed on <boards>". +- [ ] **Step 5: Restore rig state:** release all locks; run a normal single-board HIL smoke (`stm32f407disco`) per hil skill to confirm the rig is healthy for CI. +- [ ] **Step 6: Commit** + +```bash +git add .claude/skills/rtt/ +git commit -m "skills: rtt - ci.lan rig validation matrix" +``` + +--- + +### Task 9: Follow-up doc, final validation, report + +**Files:** +- Create: `docs/superpowers/followup/pr-rtt-pool-check.md` (rename to `pr<NNN>-…` once the PR number exists) + +- [ ] **Step 1: Follow-up handoff doc** (superpowers:writing-plans style, per CLAUDE.md "Deferred work"): adopting `RttConsole` in `hil_pool_check.check_host_serial` (`test/hil/helper/hil_pool_check.py:354` — bidirectional, VCOM-assuming; needs `open_board_console` hoisted from `hil_test.py` into `hil_util.py`), citing the ea4088 validation as established ground. Also note the deferred sysview SKILL.md pointer (that branch owns its file; propose to user when it merges). +- [ ] **Step 2: `pre-commit run --all-files`** — expect pass (~55 s; HIL hooks exercise real timeouts). +- [ ] **Step 3: Commit follow-up doc:** `git add docs/superpowers/followup/ && git commit -m "docs: follow-up - pool-check adoption of RttConsole"` +- [ ] **Step 4: Report** to the user: commit list, validation matrix summary (htpc + rig, per-direction verdicts), open findings (e.g. OpenOCD write path), and **ready to push — not pushed**. + +--- + +## Self-Review (completed at planning time) + +- Spec coverage: scoring→spec only; scope/sections→Task 4; tooling→Tasks 1,3; measured-evidence carriage→Task 4 step 2; doc edits→Task 6; validation strategy→Tasks 7,8; non-goals→Task 4 §2 + exclusions in Task 8. Deferred sysview pointer→Task 9. No gaps. +- Placeholder scan: `<scratchpad>` is the session scratchpad path (known at execution); `<port>/<addr>/<uid>` are computed per-board by given commands; Task 4 prose is assembled from enumerated facts (TDD forbids pre-writing final skill text before RED completes). No TBDs. +- Type consistency: `RttConsole(board, timeout)` board-dict shape identical in Tasks 1, 3; CLI flags identical in Tasks 3, 6, 7, 8; skill name `rtt` throughout. diff --git a/docs/superpowers/specs/2026-07-30-hil-usbtest-fleet-wedge-design.md b/docs/superpowers/specs/2026-07-30-hil-usbtest-fleet-wedge-design.md index 3ed0c1519..a34848f06 100644 --- a/docs/superpowers/specs/2026-07-30-hil-usbtest-fleet-wedge-design.md +++ b/docs/superpowers/specs/2026-07-30-hil-usbtest-fleet-wedge-design.md @@ -1,9 +1,9 @@ # HIL fleet-wedge containment Date: 2026-07-30 -Status: implemented, then superseded in part — addendum last checked 2026-08-12 -against the shipped code; where they disagree the CODE and the usb-kernel-recover -skill win, never this document. +Status: implemented, then superseded in part, then TRIMMED (2026-08-25 — see the +addendum at the end). Last checked against the shipped code 2026-08-25; where they +disagree the CODE and the usb-kernel-recover skill win, never this document. - **Pool guard.** A single constant, not the flat 4200s below and not a derivation: `POOL_TIMEOUT = pos_int_env('HIL_POOL_TIMEOUT', 3600)`. A per-controller model briefly @@ -17,7 +17,9 @@ skill win, never this document. - **Job ceilings.** 90/90/120 min (build.yml), not 60/60/90 and not the 85/115 below. They must clear the 3600s guard plus the pre-pool checkout/artifact merge and the post-guard sweep and report upload. No job pins `HIL_POOL_TIMEOUT`. -- **Battery budgets.** `USBTEST_BATTERY_BUDGET` 260s, `USBTEST_RECOVERY_BUDGET` 250s. +- **Battery budgets.** `USBTEST_BATTERY_BUDGET` 260s. The recovery reserve is no longer a + constant: `usbtest.recovery_reserve(flasher)` derives it per flasher (RP-target openocd 390s, + other openocd/jlink/stlink 190s, esptool/lm4flash 150s) — see the trim addendum. The 200s-with-a-197s-floor derivation recorded here was never shipped; the floor assertion was removed with it. - **HUNG recovery.** Reflash of the DUT through its roster flasher @@ -234,3 +236,108 @@ a stuck run and explain it without anyone touching the rig. the board or flashing `board_test` by hand resolves it without any code change. - **An unattended PVE watchdog** that detects the wedge and power-cycles the host. Declined: more moving parts, and it can cut a running CI job. + +--- + +## Trim addendum — 2026-08-25 + +The containment above grew past what one maintainer could hold. This records what was +removed and, more importantly, the rule that decided it, so the next reader does not +re-derive the deleted layers from the incident above. + +### The dividing principle + +**The CI job ceiling bounds how long a run can burn. It does nothing about state that +outlives the run.** Cut what the ceiling contains; keep what it does not. + +- Contained by the ceiling: a worker blocked on a wedged device. `drain_pool` keeps the + boards that finished, `_write_failed_spec` names the one in flight, `_abandon_exit` + writes and uploads the report, and the job dies at `timeout-minutes` regardless. The + cost is one pool slot. +- **Not** contained: a D-state holder left on a usbfs node, or an unswept stray still + holding a probe. The job dies and those survive it, on a self-hosted runner, into the + next run. That is the original incident. + +### Removed + +- **The sysfs blindness subsystem.** `SYSFS_UNKNOWN`, the `_SysfsUnknown` sentinel, the + path→inode strand memo with its `_STRAND_MISS` miss-sentinel, the four-credit blindness + cap, `sysfs_blind()`/`sysfs_blind_note()`, `note_sysfs_strand()`, `bounded_open()`, + `usb_scan`'s `(list, bool)` return, usbtest's `inconclusive` abort, and `_blind_note`'s + report banner. `read_sysfs` is an ordinary `open().read()` returning `str | None`. + + It was a three-valued contract five files had to reason about, and misreading unknown as + absence was silent — a healthy board reported as a firmware regression. It existed for + exactly one attribute that can block. Verified against v6.12.96 `sysfs.c`: only + `usb_string_attr` (`product`/`manufacturer`/`serial`, sysfs.c:141-143) takes + `usb_lock_device_interruptible`; `idVendor`, `idProduct`, `bcdDevice`, `busnum`, + `devnum` and `speed` are lock-free `sysfs_emit` from cached fields. Two of the five + `read_sysfs` call sites read attributes that cannot block at all. + + **The bound stayed, and it is not opt-in.** An early cut of this trim made `read_sysfs` + unbounded on the theory that a blocked worker costs one pool slot. That is false: + `usb_scan` reads `serial` on every device matching the VID to find the one it wants, and + `hil_lock.controller_of` does exactly that from `controller_permit`, on essentially every + board — so one wedged DUT would stall *every* worker and the pool guard would take the + whole run. `read_sysfs` and `usb_scan` are bounded by `SYSFS_READ_GRACE` by default; + three call sites forgot an opt-in version within a single sitting, and a unit test now + pins the default. + + What is gone is the *contract*, not the bound: no third value, no process-wide blindness + latch, no `(list, bool)` return, no report banner. A give-up reads as None like any + unreadable attribute, and the cost is confined to the device that is actually wedged. + + **`hil_pool_check` is why the memo has to be exact.** It is a standalone + ThreadPoolExecutor tool with no guard behind it, run precisely when a device is suspected + wedged, and it polls (`wait_device` re-scans every 0.5 s). The bounded read gives up and + remembers + the path so a poll loop cannot leak a thread and an fd per pass. That memo is keyed by + **kernfs inode, not by path**: a busport does not change when a board returns to the same + physical port, so a path-only blacklist would outlive the wedge and make the tool's own + recovery flow (reset/reflash → `wait_device` polls for the new inode) never see the board + again. A changed inode is the all-clear; `os.stat` is safe on a wedged device because it + does not invoke `->show()`. A give-up reads as None + — the same as unreadable — and `sysfs_stranded()` lets the footer warn that a "missing" + row may be the tool losing sight of healthy hardware. One local bound with a warning + line, not the five-file three-valued contract that was removed. + +- **The recovery budget arithmetic.** `recovery_steps()`, `_time_left()` and its three + per-step gates. The reserve was an independent 250s — one number for the whole fleet — + that could not contain the ladder it + reserved for (reset 30 + reflash 90 + Rescue-DP POR 90 + retry 90 + settles), which is + why the child re-decided before every step — with a bare `- 35` for downstream costs + that nobody could re-derive. Between them they produced a recovery that skipped its own + steps for most real hangs. The reserve now counts `hil_util.REAP_GRACE` **per bounded + step** — `run_cmd` spends that reaping a child it had to SIGKILL, on top of the step's own + timeout — which is what the `- 35` was standing in for. Undersizing it is worse than not + recovering at all: the outer killpg lands mid-reflash and orphans the flasher on the + probe. A unit test asserts the reserve covers the ladder. `USBTEST_RECOVERY_BUDGET` is now derived from + `usbtest.RECOVER_*` **per flasher and per target**: the Rescue-DP legs are openocd-only + (`rescue_openocd` refuses anything else) and a stub reset is screened out, so an esptool + board no longer reserves 200s it can never spend. The child runs the ladder straight + through, and `--outer-timeout` — parsed but unused once the gates went — is deleted. + +### Deliberately kept + +- The pool guard, `drain_pool`, the re-run spec, `_abandon_exit`, the CI ceilings. +- `hil_health`'s sweep **including** `_kill_and_confirm`. SIGKILL is queued, not delivered, + for a task in uninterruptible sleep, and a healthy in-flight testusb sits in exactly that + state — so `os.kill` returning success proves nothing, and the recheck is the only honest + answer to "is the rig dirty for the next job?". +- usbtest's reset→check→reflash ladder and the `convoy_safe` gate. This is the only thing + that unpoisons the rig mid-run, and PR #3832 extends it from 11 to 18 of 27 boards. +- `mtp_test.py` as a separate process — one job, a clean boundary, and runnable by hand + against a board while debugging. + +### Structural changes with no behaviour change + +- Blocking device IO now runs in a child process everywhere, not just where it was noticed + first. The printer WRITE half joined the read half (`usblp_open` ignores `O_NONBLOCK` and + stalls in `usb_autopm_get_interface()` holding the driver-global `usblp_mutex`), and the + HID echo followed (`hid.enumerate()` reads `manufacturer`/`product` for every HID device + it lists, both under the device lock). `test_device_midi_test` is NOT in that set: ALSA + rawmidi honours `O_NONBLOCK` on open (v6.12.96 rawmidi.c:489), unlike usblp. +- `main()`'s two abort paths were near-identical 40-line blocks; `_abort_report` holds that + shape once. The controller-hint cache and pool construction moved to their own helpers. +- The unit suite stopped sleeping 54 of its 78 seconds — mostly one named-and-zeroable + post-flash settle paid by ten tests against a fake rig. diff --git a/docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md b/docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md index b10f5b4ae..799c83c23 100644 --- a/docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md +++ b/docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md @@ -49,8 +49,8 @@ never inflates one axis with another's breadth. | # | Changed path | Build families | Build examples | HIL boards → tests | | --- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------- | ----------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | | 1 | `docs/`, `.claude/`, `*.md`, `*.rst`, `LICENSE` | — | — | — | -| 1b | `.gitignore`, `.clang-format`, `.idea/**`, `test/{fuzz,unit-test}/**`, non-build `.github/**`, packaging manifests | — | — | — | -| 2 | `test/hil/**` | — | — | all boards → all tests | +| 1b | `.gitignore`, `.clang-format`, `.idea/**`, `test/{fuzz,unit-test}/**`, `test/hil/test/**`, non-build `.github/**`, packaging manifests | — | — | — | +| 2 | `test/hil/**` (not `test/hil/test/**`) | — | — | all boards → all tests | | 2b | `tools/metrics.py`, `.github/scripts/metrics_*.py` | `ALL` (unchanged — `tinyusb_metrics` runs `metrics.py` as a build target) | `ALL` | — (nothing on the rig runs it) | | 3 | `src/portable/<port>/dcd_*`, `*_device.[ch]` | `FAM` | `DEV`+`DUAL` | `FAM`'s device-role boards → device+dual tests | | 4 | `src/portable/<port>/hcd_*`, `*_host.[ch]` | `FAM` | `HOST`+`DUAL` | `FAM`'s host-role boards → host+dual tests | @@ -75,6 +75,12 @@ never inflates one axis with another's breadth. **Rule 2 is deliberately asymmetric.** A `test/hil/**` change is invisible to the family matrix but is exactly what the rig exercises, so it builds nothing and runs everything. +`test/hil/test/**` is carved out to rule 1b: it holds the harness's own unit tests, which +nothing on the rig runs (pre-commit does, and `build.yml` runs `test_ci_select.py` as the +gate before trusting a selection). A bare `test/hil/` prefix was booking the full 27-board +rig for diffs that cannot reach it. The carve-out is a claim about that directory's +contents, so a test pins its file list: add anything the rig reads and it fails. + **Rule 7 is the one HIL-side behaviour change in this design.** Today `hw/mcu/` sits in `hil_select`'s `_FULL_RE` and forces the full HIL matrix. Since the build axis now resolves those paths to a family through the same scan, forcing full on the rig is inconsistent. The @@ -146,6 +152,19 @@ the existing `test_hil_util.BottomLayer` structural tests. Fail-open survives where it belongs: an *unclassified* path or any exception widens to `ALL` on every axis. +### A class no example enables selects nothing + +`src/class/bth` is the live instance: no example's `tusb_config.h` sets `CFG_TUD_BTH`, so +rules 8-10 resolve to no examples and a bth-only PR builds nothing and runs nothing. That is +the empty-means-empty ruling applied to classes, and it is deliberate — nothing compiles the +file, so nothing can validate it, and the master-push build is the net. + +Worth stating plainly because the exposure changed: GHA used to rebuild everything for such +a PR by accident, through the empty-`families` bug in `build.yml`. With that fixed, both +providers now correctly build nothing, so `tud_bt_*` can be broken by a green PR. +`TestClassesWithNoEnablingExample` pins the set to `{bth}` so a second class cannot enter +this state unnoticed. + ### Why `hw/mcu/**` is rule 7 and not "full" `hw/mcu` is overwhelmingly dependency territory — `tools/get_deps.py` has 87 entries under it, diff --git a/docs/superpowers/specs/2026-08-21-hil-report-module-design.md b/docs/superpowers/specs/2026-08-21-hil-report-module-design.md new file mode 100644 index 000000000..41e7000b7 --- /dev/null +++ b/docs/superpowers/specs/2026-08-21-hil-report-module-design.md @@ -0,0 +1,144 @@ +# hil_report.py: one owner for the HIL report document + +**Date:** 2026-08-21 +**Branch:** `hil-report` (continues the report-unification work already on it) + +## Motivation + +`hil_report.json` and `hil_report.md` are now one document rendered two ways, but the code that +produces, renders, merges and reads that document is spread across three modules: + +| Module | Report-related content | +|---|---| +| `hil_test.py` | `REPORT_CELL`, `BOUNDARY_CELL`, `REPORT_MD`, `REPORT_JSON`, `render_matrix`, `render_report`, `write_report`, `mark_report_abandoned`, `accumulate_report` | +| `helper/hil_health.py` | `write_timeout_report` — composes its own markdown | +| `helper/hil_summary.py` | `cell_state`, `variants_of`, `summarize`, CLI | + +Two concrete defects follow from that spread. + +**One classifier, two copies.** `hil_test.py:1966` (`cell_kind`, keyed off `REPORT_CELL`) and +`hil_summary.py:34` (`cell_state`, with its own re-typed `FAIL_ICON, SKIP_ICON = '❌', '⚪'`) +implement the same rule. The latter's docstring says it is *"the EXACT classifier hil_test.py's own +tally uses"* — the duplication was noticed and documented as an obligation to keep in sync, rather +than removed. Change `REPORT_CELL` and the human's table and the agent's verdict silently disagree: +the markdown says ❌ where the JSON says `pass`. That is the same class of defect this branch +exists to eliminate, one layer up. + +**A writer that cannot render.** `hil_test.py` imports `hil_health`, so `hil_health` cannot import +`hil_test` back. That is the only reason `write_timeout_report` composes its own markdown instead of +calling `render_report`, and the only reason the pool-guard fallback is held to a weaker promise +(same boards and caveat in both artifacts, not byte-identical) while the other four writers are +exact. The constraint is structural, not essential: a leaf module both can import dissolves it. + +## Goal / non-goals + +**Goal:** `test/hil/helper/hil_report.py` becomes the single owner of the report document. + +**This is NOT purely code motion, and the distinction matters for review.** Measured against +`master`, `hil_test.py` contains only `render_matrix` and `accumulate_report`. Everything else in +the new module — `render_report`, `write_report`, `mark_report_abandoned`, `mark_report_no_boards`, +`_load`, `_write_stuck_over_prior_md`, `cell_state`, and the `scope`/`caveat` plumbing — is NEW +code, roughly 150 lines of it, and two rounds of review found most of their defects there. Read +those functions as new, not as relocated. `hil_test.py`'s CLI, arguments and table format do stay +unchanged. + +**Deliberate user-visible changes:** +1. `hil_summary.py` is deleted; its CLI moves to `hil_report.py`. The documented command becomes + `python3 test/hil/helper/hil_report.py <config> -b BOARD [-b BOARD…]`. +2. `write_timeout_report` re-renders from the merged sidecar instead of stapling its banner above + the previous attempt's markdown text. Output improves — one table containing the stuck boards, + rather than a fresh banner above a duplicate table — but it is a change (see Testing). + +**Non-goals (explicit follow-ups, not this change):** +- Splitting `accumulate_report`'s `mret` folding from its merge (see "Deliberate wart"). +- The flat `HIL_POOL_TIMEOUT` that does not scale with board count (`hil_test.py:225`). + +## Resulting layout (`test/hil/`) + +| File | ~Lines | Role | +|---|---|---| +| `hil_test.py` | 2390 (−250) | tests + orchestration + CLI | +| `helper/hil_report.py` (new) | ~400 | the report document: vocabulary, render, write, merge, fold, CLI | +| `helper/hil_health.py` | ~345 (−53) | killing wedged processes only | +| `helper/hil_summary.py` | deleted | superseded by `hil_report.py` | + +Import graph: `hil_health` is a leaf; `hil_report` → `hil_health` (for `_p`, the +BrokenPipeError-safe print used on containment paths); `hil_test` → both. No cycles. + +## hil_report.py + +Stdlib only (`json`, `argparse`, `pathlib`) beyond that one `_p` import. Sections, in order: + +**Vocabulary.** `REPORT_MD`, `REPORT_JSON`, `REPORT_CELL`, `BOUNDARY_CELL`, `LOCKED_CELL`. +`REPORT_CELL` becomes the single source of the status icons; `hil_summary.py`'s `FAIL_ICON`/ +`SKIP_ICON` literals are deleted. + +**Classifier.** One `cell_state(v) -> 'pass' | 'fail' | 'skip'`, replacing both `cell_kind` and the +old `cell_state`. Keeps the surviving docstring's warning that the `pass` arm is load-bearing: a +passing test may return an unprefixed metric string (`'480.0 MBps'`), while failures are guaranteed +icon-marked, so classifying unknown shapes as `fail` would publish a green table as a red verdict. + +**Render.** `render_matrix(rows_all)`, `render_report(doc)`. Unchanged; `render_matrix`'s inline +`cell_kind` is replaced by a call to the module-level `cell_state`. + +**Write.** `write_report`, `accumulate_report`, `mark_report_abandoned`, `write_timeout_report`. +Moved verbatim except `write_timeout_report`, which loses its `md_name` parameter (the module owns +`REPORT_MD`) and renders instead of concatenating. + +**Fold.** `variants_of`, `summarize`, and the `main()` CLI from `hil_summary.py`. + +## Deliberate wart + +`accumulate_report` moves wholesale, keeping its knowledge of `mret`'s worker-result tuple shape. +The cleaner boundary would split "fold `mret` → rows" (`hil_test`'s domain) from "merge rows → doc" +(`hil_report`'s), but that rewrites subtle, well-tested logic — stale `board-locked` clearing, +`BOUNDARY_CELL` dropping, `duration=None` preservation — for a tidier seam. It is a data-shape +coupling, not an import cycle. Moving it verbatim keeps the motion reviewable as motion. + +## The sharp edge + +`hil_ci.sh:222-228` stages helper modules by an **explicit scp list**. A new `helper/hil_report.py` +that is not added there reaches the rig missing, and the run dies with `ImportError` *after* +`REMOTE_DIR` has already been wiped — so the previous run's report and re-run spec are gone too. + +This is already guarded: `test_hil_bounded.py`'s `RemoteStaging.test_import_closure_is_staged_to_the_rig` +walks the AST import closure from `hil_test.py`, `usbtest.py` and `mtp_test.py` and requires an exact +scp entry for each file. Adding the module to the list is all this change needs; no new guard is +warranted, and an earlier draft of this document wrongly claimed none existed. + +## Consumers to update + +| File | Change | +|---|---| +| `test/hil/hil_ci.sh:226` | `hil_summary.py` → `hil_report.py` in the scp list | +| `.claude/agents/hil-operator.md:71` | the documented command | +| `.claude/workflows/hil-validate.js:58` | the command the operator is told to run | +| `.claude/workflows/hil-validate.js:14,17,54,67`, `test-hil-validate.mjs:7` | stale `hil_summary.py` mentions in comments | + +No logic in the `.claude` files changes — the operator's return contract +(`{results, banner, wedged}`) is untouched. + +## Testing + +New `test/hil/test/test_hil_report.py`. The report-specific classes move there from +`test_hil_bounded.py` (`CaveatSurvivesAccumulate`, `SummaryFoldsReportToBoards`, +`ScopeSurvivesInTheJson`, `RenderReportIsPureFunctionOfTheDocument`, +`EveryExitPathLeavesBothArtifacts`, `AbandonNoticeLandsInBothArtifacts`, +`MarkdownIsAlwaysARenderingOfTheJson`) and from `test_hil_health.py` (`WriteTimeoutReport`). + +Three test changes are substantive rather than mechanical: + +1. `WriteTimeoutReport.test_keeps_a_previous_attempts_table` asserts the prior **markdown text** + survives. It becomes an assertion that the prior attempt's **rows** survive — the same guarantee + against the new representation. +2. `MarkdownIsAlwaysARenderingOfTheJson` gains a fifth case for the pool-guard fallback, which now + satisfies the byte-identical invariant like the other four. +3. `test_the_pool_guard_fallback_agrees_even_if_it_does_not_render` — the weaker promise — is + deleted, because the promise it encoded no longer applies. + +Gate: `python3 -m unittest discover -s test/hil/test` at 275 — the current 266, minus the one +deleted test, plus the fifth invariant case, the scp-list guard, two dual-mode import tests, +five classifier tests and one pinning that the old entry point is gone — then +`pre-commit run --all-files`. Because this lands on a +branch already validated on hardware, it closes with a rig re-check: the invariant check against a +real report pair and a scoped `--accumulate` run, not the full fleet. diff --git a/docs/superpowers/specs/2026-08-24-rtt-skill-design.md b/docs/superpowers/specs/2026-08-24-rtt-skill-design.md new file mode 100644 index 000000000..7a621726f --- /dev/null +++ b/docs/superpowers/specs/2026-08-24-rtt-skill-design.md @@ -0,0 +1,164 @@ +# `rtt` skill — design & decision record + +Date: 2026-08-24. Branch: `rttconsole-skill`. Author sessions: lpc4088 handoff +(measurements), sysview handoff (mechanics + probe matrix), this session +(verification + decision). User approved promotion and the name `rtt` on +2026-08-24. + +## Decision + +Promote SEGGER RTT from an inline technique in `.claude/skills/target-debug/` +to a standalone skill `.claude/skills/rtt/`, scoped as **transport core + +console layer**: getting bytes on/off RTT channels over any debug probe, plus +the bidirectional console tooling the HIL harness ships. Consumer-specific +layers (SystemView encode/decode/licensing, TU_LOG conventions, debugging +methodology) stay in their skills and cross-reference. + +## Scoring against the promotion criteria + +Criteria: `docs/superpowers/specs/2026-07-09-claude-agents-workflows-design.md` +§"Skill vs technique — promotion criteria" (exists only on branch +`claude/add-systemview-debug`; read via `git show`). Two or more of four +required. Score: **3/4**. + +1. **Ships tooling — yes.** `hil_util.JlinkRtt` (commit d98e77bac: probe + selection by serial, dynamic port allocation, non-blocking bidirectional + socket, process-group teardown) plus a thin CLI added by this plan. + Precedent: `hil` and `code-size` are skills wrapping repo-versioned tools; + "recipes over already-installed tools" is what RTT was *before* this code + existed (why SWO stayed a technique at 1.5/4 — see `SWO_SKILL_HANDOFF.md`). +2. **Answers its own routed question — yes.** "Give this board a console / + printf I/O with no UART and no VCOM" is asked from harness and bring-up + contexts that never load target-debug (whose trigger is *misbehaving + firmware*). Measured cost of the missing route: the lpc4088 session burned + an hour rediscovering a gotcha already written at target-debug + SKILL.md:249-253. +3. **Carries validation state — yes.** Measured tool matrix (below), 13-board + OpenOCD read-path campaign from the sysview cycle, WCH SDI A/B proof, + SAMD5x DSU gotcha, lock-porting example, per-probe constraints. +4. **Long but conditionally relevant — yes.** The transport knowledge is a + page+ that most target-debug sessions don't need and harness sessions + can't find there. + +## Measured evidence the skill must carry + +From the lpc4088 session (LPC4088 + LPC-Link2 J-Link fw 611000000, SWD 4 MHz; +single board — re-verify on more hardware during validation): + +- `JLinkExe -RTTTelnetPort <port> -AutoConnect 1`: 6/6 reliable; delivers the + buffered boot burst; accepted an 8550-byte write in one call. **The proven + standalone path.** +- Drain rate 24.6 KiB/s (253,127 B / 10.0 s) against a saturating printf + firmware that produced 689,896 lines — 0.6 % delivered. RTT console is + **drain-limited and lossy under saturation; drops happen at the target** + (NO_BLOCK_SKIP, 1 KB default buffer). +- `JLinkRTTLogger`: 0/6 — "RTT Control Block not found" even given + `-RTTAddress`, block plainly readable over SWD. Searches once at attach, + never retries. **Never build on it.** +- `JLinkGDBServer -RTTTelnetPort` with **no GDB client attached**: served the + port, never located the control block (this board). target-debug's + GDBServer+JLinkRTTClient recipe was proven in flows where GDB attaches, and + CLAUDE.md's recipe worked on other parts — treat as per-part variance, + document both; do not "correct" either into a flat contradiction. +- OpenOCD (jaylink) driving this J-Link-firmware probe: transport failure + (`LIBUSB_ERROR_TIMEOUT`, `jaylink_swd_io() failed`), probe drops off USB, + **physical replug needed** — twice, reproducible. Standing rule: never + point OpenOCD at that class of probe (J-Link OB firmware on a debug-probe + board like the LPC-Link2). Genuine SEGGER J-Links work under jaylink — + routine in the sysview campaigns (metro_m4_express). + +From the sysview cycle (branch `claude/add-systemview-debug`, 13-board +campaign 2026-08-12): + +- OpenOCD `rtt setup <exact CB addr> … ; rtt start; rtt server start <port> + <ch>` **read path validated** on ST-Link, CMSIS-DAP and J-Link probes + (`test/hil/sysview_ci.py`). Exact CB address from + `arm-none-eabi-nm <elf> | grep _SEGGER_RTT` beats a full-RAM scan (slower, + can mis-hit stale RAM after soft reset). +- The real transport requirement is **autonomous memory access while the core + runs**: ARM memory-AP (zero intrusion), RISC-V SBA where implemented. + **WCH QingKe SDI has neither** — Debug Module abstract commands perturb the + running core; A/B-proven kill ~1.9 s into USB traffic. Per-transport rule: + SDI = halt→read→resume / post-mortem dump only, never live streaming. +- SAMD5x + OpenOCD: in-session `reset run` via the DSU CPU Reset Extension + leaves the core held — attach without reset when the flash step already + reset the board (general preference: attach-only capture). +- Lock porting example: `hw/bsp/ch583/sysview_rtt_lock_wch.h` (QingKe CSR + 0x800 brace-scoped save/restore; generic RISC-V lock traps mcause=2). +- Drain hierarchy: J-Link native > OpenOCD polling; matters only at + SystemView bandwidths (workable buffers 2048–8192); console logs never + overflow the drain in practice. +- RTT mechanics for the concepts section: control block `_SEGGER_RTT` (magic + "SEGGER RTT") + ring buffers {sName, pBuffer, SizeOfBuffer, WrOff, RdOff, + Flags}; the HOST must write RdOff back to drain; modes NO_BLOCK_SKIP (log + default) / NO_BLOCK_TRIM / BLOCK_IF_FIFO_FULL (target spins — dangerous in + ISRs); post-mortem mode = `SEGGER_RTT_WriteWithOverwriteNoLock` (target + drags RdOff, ring holds last N bytes, no live host needed); channel 0 = + "Terminal" console, SystemView claims its own "SysView" up-buffer — + coexist on one control block. + +## Gotchas the skill centralises + +Control block exists only after the target's first printf (early reader sees +nothing; Logger gives up). The console owns the probe: flash and reset before +opening it; never reset while attached. An undrained NO_BLOCK_SKIP ring holds +the FIRST KB after boot, not the wedge tail. Always select probes by serial +(`-USB <sn>` / `adapter serial`) — rigs run several. Two probes wired to one +SWD header wedge the target. + +## v1 backend matrix + +| Backend | Read (capture) | Write (console input) | +| ----------------------------------------------------- | ---------------------------- | ------------------------------------------ | +| J-Link native (`JLinkExe -RTTTelnetPort`) | validated | validated (8.5 KB writes) | +| OpenOCD on native probes (ST-Link/CMSIS-DAP/WCH-Link) | validated (sysview campaign) | unvalidated — validate in the ci-rig phase | +| OpenOCD on the LPC-Link2 (J-Link OB fw, measured) | forbidden (USB drop) | forbidden | +| WCH SDI (any tool) | halt→dump only | n/a | + +`JlinkRtt`/CLI are J-Link-only in v1; OpenOCD console-write support is +added only if the ci-rig phase validates it. + +## Tooling home + +Single implementation in `tools/rtt.py`: a stdlib-only importable module +(shared socket-console base + `JlinkRtt` + `OpenocdRtt`) that doubles as +the CLI. `hil_util` imports and re-exports the classes (the harness keeps +addressing `hil_util.JlinkRtt`), so the dependency points harness → tools, +never tools → harness. Because `hil_util` loads it at import time, the file +is harness-critical: it is classified with `test/hil/` in `ci_select`'s full +rule and covered by the pre-commit `hil-test` hook (test_hil_rtt.py). +Precedent: `code-size` wrapping `tools/metrics_compare_base.py` — the skill +is md-only and points at the tool. `open_board_console()` stays in +`hil_test.py` for now; pool-check adoption is a follow-up doc, not this PR. + +## Doc edits (curated-skills rule: smallest possible diffs) + +- `target-debug/SKILL.md`: capture-channel rows and the drain-model warning + stay; the two capture recipe blocks and the RTTLogger/GDBServer paragraph + shrink to one-liners pointing at `rtt`; the manual ring-read recipe + (`nm`/`mem32`/`savebin`) moves into `rtt` §post-mortem. +- `CLAUDE.md` GDB section RTT line becomes build flag + pointer. +- `hil/SKILL.md` gains one routing line (the fix that would have prevented + the lost hour). +- `sysview/SKILL.md` pointer is **deferred** until that branch merges, and + proposed to the user first. No edits to `sysview_ci.py` or the sysview + skill now. + +## Validation strategy (user-directed) + +1. **Dogfood on the local htpc bench first**: ea4088_quickstart via LPC-Link2 + (replugged; OpenOCD attempts on it are skipped outright) and + raspberry_pi_pico2 via the J-Trace (nickname `jtrace`, serial private; now wired to pico2; RP2350 = + `rp2350_m33_0`, never a custom JLinkScript). Follow only the SKILL.md + text (dogfood = REFACTOR input). +2. **Then all boards on the ci.lan rig**, per-transport smoke capture, rows + recorded in `.claude/skills/rtt/boards.md`. Exclusions recorded honestly + (esptool boards: no SEGGER-RTT path in our builds — USB-Serial-JTAG + console instead; tm4c: no probe path configured on the rig). + +## Non-goals + +Timing/profiling (etm-trace, sysview, parked swo-trace), SystemView +encode/decode/licensing, TU_LOG conventions, debugging decision flows +(target-debug), Espressif USB-Serial-JTAG console (esp-target-debug), WCH SDI +live streaming (impossible — see matrix). diff --git a/examples/device/audio_4_channel_mic/skip.txt b/examples/device/audio_4_channel_mic/skip.txt index 3ca433c08..e5e74cd60 100644 --- a/examples/device/audio_4_channel_mic/skip.txt +++ b/examples/device/audio_4_channel_mic/skip.txt @@ -1,5 +1,4 @@ mcu:SAMD11 -mcu:SAME5X mcu:SAMG family:broadcom_64bit family:espressif diff --git a/examples/device/audio_4_channel_mic_freertos/skip.txt b/examples/device/audio_4_channel_mic_freertos/skip.txt index 1fd6b4b8a..cfde51051 100644 --- a/examples/device/audio_4_channel_mic_freertos/skip.txt +++ b/examples/device/audio_4_channel_mic_freertos/skip.txt @@ -7,7 +7,6 @@ mcu:CXD56 mcu:F1C100S mcu:GD32VF103 mcu:MCXA15 -mcu:MKL25ZXX mcu:MSP430x5xx mcu:FT90X mcu:SAMD11 diff --git a/examples/device/audio_test/skip.txt b/examples/device/audio_test/skip.txt index 42394bb11..862c91c6f 100644 --- a/examples/device/audio_test/skip.txt +++ b/examples/device/audio_test/skip.txt @@ -1,5 +1,4 @@ mcu:SAMD11 -mcu:SAME5X mcu:SAMG family:espressif mcu:CH583 diff --git a/examples/device/audio_test_freertos/skip.txt b/examples/device/audio_test_freertos/skip.txt index 660bacd25..3d8d43286 100644 --- a/examples/device/audio_test_freertos/skip.txt +++ b/examples/device/audio_test_freertos/skip.txt @@ -7,7 +7,6 @@ mcu:CXD56 mcu:F1C100S mcu:GD32VF103 mcu:MCXA15 -mcu:MKL25ZXX mcu:MSP430x5xx mcu:FT90X mcu:SAMD11 diff --git a/examples/device/audio_test_multi_rate/skip.txt b/examples/device/audio_test_multi_rate/skip.txt index 42394bb11..862c91c6f 100644 --- a/examples/device/audio_test_multi_rate/skip.txt +++ b/examples/device/audio_test_multi_rate/skip.txt @@ -1,5 +1,4 @@ mcu:SAMD11 -mcu:SAME5X mcu:SAMG family:espressif mcu:CH583 diff --git a/examples/device/cdc_msc_freertos/skip.txt b/examples/device/cdc_msc_freertos/skip.txt index 48781de84..095e350c9 100644 --- a/examples/device/cdc_msc_freertos/skip.txt +++ b/examples/device/cdc_msc_freertos/skip.txt @@ -7,7 +7,6 @@ mcu:CXD56 mcu:F1C100S mcu:GD32VF103 mcu:MCXA15 -mcu:MKL25ZXX mcu:MSP430x5xx mcu:FT90X mcu:SAMD11 diff --git a/examples/device/cdc_uac2/skip.txt b/examples/device/cdc_uac2/skip.txt index db1d5b80b..3159cb176 100644 --- a/examples/device/cdc_uac2/skip.txt +++ b/examples/device/cdc_uac2/skip.txt @@ -2,7 +2,6 @@ mcu:LPC11UXX mcu:LPC13XX mcu:NUC121 mcu:SAMD11 -mcu:SAME5X mcu:SAMG board:stm32l052dap52 family:espressif diff --git a/examples/device/hid_composite_freertos/skip.txt b/examples/device/hid_composite_freertos/skip.txt index 97d8e168b..0e8415d3b 100644 --- a/examples/device/hid_composite_freertos/skip.txt +++ b/examples/device/hid_composite_freertos/skip.txt @@ -7,7 +7,6 @@ mcu:CXD56 mcu:F1C100S mcu:GD32VF103 mcu:MCXA15 -mcu:MKL25ZXX mcu:MSP430x5xx mcu:FT90X mcu:SAMD11 diff --git a/examples/device/midi_test_freertos/skip.txt b/examples/device/midi_test_freertos/skip.txt index 97d8e168b..0e8415d3b 100644 --- a/examples/device/midi_test_freertos/skip.txt +++ b/examples/device/midi_test_freertos/skip.txt @@ -7,7 +7,6 @@ mcu:CXD56 mcu:F1C100S mcu:GD32VF103 mcu:MCXA15 -mcu:MKL25ZXX mcu:MSP430x5xx mcu:FT90X mcu:SAMD11 diff --git a/examples/device/msc_dual_lun/skip.txt b/examples/device/msc_dual_lun/skip.txt index a9e3a99b1..833fd072c 100644 --- a/examples/device/msc_dual_lun/skip.txt +++ b/examples/device/msc_dual_lun/skip.txt @@ -1,3 +1,2 @@ mcu:SAMD11 -mcu:MKL25ZXX family:espressif diff --git a/examples/device/uac2_headset/skip.txt b/examples/device/uac2_headset/skip.txt index db1d5b80b..3159cb176 100644 --- a/examples/device/uac2_headset/skip.txt +++ b/examples/device/uac2_headset/skip.txt @@ -2,7 +2,6 @@ mcu:LPC11UXX mcu:LPC13XX mcu:NUC121 mcu:SAMD11 -mcu:SAME5X mcu:SAMG board:stm32l052dap52 family:espressif diff --git a/examples/device/uac2_speaker_fb/skip.txt b/examples/device/uac2_speaker_fb/skip.txt index 0c7339c65..88df3e549 100644 --- a/examples/device/uac2_speaker_fb/skip.txt +++ b/examples/device/uac2_speaker_fb/skip.txt @@ -2,7 +2,6 @@ mcu:LPC11UXX mcu:LPC13XX mcu:NUC121 mcu:SAMD11 -mcu:SAME5X mcu:SAMG board:stm32l052dap52 family:broadcom_64bit diff --git a/hw/bsp/BoardPresets.json b/hw/bsp/BoardPresets.json index a480efc3e..280d6d592 100644 --- a/hw/bsp/BoardPresets.json +++ b/hw/bsp/BoardPresets.json @@ -507,6 +507,10 @@ "inherits": "default" }, { + "name": "pico2_etm_trace", + "inherits": "default" + }, + { "name": "pico_sdk", "inherits": "default" }, @@ -1641,6 +1645,11 @@ "configurePreset": "nutiny_sdk_nuc505" }, { + "name": "pico2_etm_trace", + "description": "Build preset for the pico2_etm_trace board", + "configurePreset": "pico2_etm_trace" + }, + { "name": "pico_sdk", "description": "Build preset for the pico_sdk board", "configurePreset": "pico_sdk" @@ -3901,6 +3910,19 @@ ] }, { + "name": "pico2_etm_trace", + "steps": [ + { + "type": "configure", + "name": "pico2_etm_trace" + }, + { + "type": "build", + "name": "pico2_etm_trace" + } + ] + }, + { "name": "pico_sdk", "steps": [ { diff --git a/hw/bsp/rp2040/boards/pico2_etm_trace/board.cmake b/hw/bsp/rp2040/boards/pico2_etm_trace/board.cmake new file mode 100644 index 000000000..53f9132b5 --- /dev/null +++ b/hw/bsp/rp2040/boards/pico2_etm_trace/board.cmake @@ -0,0 +1,35 @@ +set(PICO_PLATFORM rp2350-arm-s) +set(PICO_BOARD pico2) + +# ETM trace is wired on this carrier only (GP1-5 -> MIPI-20), so the trace +# build flag lives here rather than being a global -D anyone can pass: on a +# board whose PIO-USB D+ sits on GP1 (e.g. adafruit_fruit_jam) it would fight +# the trace clock. +set(TRACE_ETM 1) + +# Point the pico-sdk's own defaults at the carrier's wiring: pico2.h guards +# every PICO_DEFAULT_* with #ifndef, so these win. Without them anything that +# talks to the SDK directly instead of the TinyUSB BSP (e.g. stdio_init_all() +# in examples/device/cdc_uac2) would mux GP0/GP1 for UART - and GP1 is +# TRACECLK, so it would silently kill the trace clock mid-capture. +add_compile_definitions( + PICO_DEFAULT_UART_TX_PIN=12 + PICO_DEFAULT_UART_RX_PIN=13 + PICO_DEFAULT_LED_PIN=10 + PICO_DEFAULT_I2C=0 # STEMMA-QT / Qwiic port on GP8/9; + PICO_DEFAULT_I2C_SDA_PIN=8 # the sdk default GP4/5 is TRACEDATA2/3 + PICO_DEFAULT_I2C_SCL_PIN=9 +) + +# the carrier's MIPI-20 is driven by a J-Trace; uncomment (or pass +# -DJLINK_OPTION=...) to pin one probe by USB nickname/serial when several +# J-Links are attached during hardware validation +#set(JLINK_OPTION "-USB jtrace") + +# Clock: the rp2350 pico-sdk default, 150 MHz -> 75 MHz TRACECLK (clk_sys/2), +# validated on the trace motherboard: cdc_msc enumeration burst 3/3, zero +# overflow, +1 ns data sampling (idle eye -1000..+2000 ps; committed in the +# ozone reference). Nothing may switch clk_sys at runtime - that truncates a +# capture at the switch. Other validated rates (156000, 180000, and 240000 = +# the J-Trace PRO V2 ceiling) need PLL_SYS_* from the SDK's vcocalc.py; see +# the etm-trace skill's boards.md. diff --git a/hw/bsp/rp2040/boards/pico2_etm_trace/board.h b/hw/bsp/rp2040/boards/pico2_etm_trace/board.h new file mode 100644 index 000000000..863d0e6b9 --- /dev/null +++ b/hw/bsp/rp2040/boards/pico2_etm_trace/board.h @@ -0,0 +1,80 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2025 Ha Thach (tinyusb.org) + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * This file is part of the TinyUSB stack. + */ + +/* metadata: + name: Pico 2 ETM Trace Carrier + url: https://github.com/hathach/pcb/tree/main/pico2_trace_motherboard +*/ + +// Raspberry Pi Pico 2 seated on the "pico2 trace motherboard" carrier: a +// MIPI-20 Cortex Debug+ETM adapter (SWD + 4-bit trace) plus a TinyUSB test +// bench. Same RP2350 module as raspberry_pi_pico2, different pin map: the +// carrier keeps GP1-5 free for TRACECLK/TRACEDATA0-3 and moves the console, +// LED, button and USB control pins out of the way. +// +// Carrier pin map (only the pins the BSP uses are defined below): +// 0 GND guard (JP2) 1 TRACECLK +// 2-5 TRACEDATA0-3 6 GND guard (JP3) +// 8/9 I2C0 SDA/SCL (STEMMA-QT) 10 user LED +// 11 device D+ pull-up enable 12/13 UART0 TX/RX (console) +// 14 user button (to GND, unused - BSP uses BOOTSEL) +// 15 host VBUS fault +// 16 native VBUS-detect tap 17 host VBUS enable +// 18/19 PIO-USB device D+/D- (J9) 20/21 PIO-USB host D+/D- (J5) +// 26 VBUS current sense (ADC) 27 J9 device VBUS-detect + +#ifndef TUSB_BOARD_H +#define TUSB_BOARD_H + +#ifdef __cplusplus + extern "C" { +#endif + +//--------------------------------------------------------------------+ +// LED, UART (button: the family BSP uses BOOTSEL, like every rp2040 board) +//--------------------------------------------------------------------+ +#define LED_PIN 10 +#define LED_STATE_ON 1 + +// console is on GP12/13, NOT the pico default GP0/1: GP1 is TRACECLK, so the +// console stays full-duplex while tracing +#define UART_DEV 0 // uart0 (index, see uart_get_instance) +#define UART_TX_PIN 12 +#define UART_RX_PIN 13 + +//--------------------------------------------------------------------+ +// PIO_USB +//--------------------------------------------------------------------+ +// host port J5 (USB-A): D+ = GP20, D- = GP21, load switch enable = GP17 +#define PICO_DEFAULT_PIO_USB_DP_PIN 20 +#define PICO_DEFAULT_PIO_USB_VBUSEN_PIN 17 +#define PICO_DEFAULT_PIO_USB_VBUSEN_STATE 1 + +#ifdef __cplusplus + } +#endif + +#endif diff --git a/hw/bsp/rp2040/boards/raspberry_pi_pico2/ozone/rp2350.jdebug b/hw/bsp/rp2040/boards/pico2_etm_trace/ozone/rp2350.jdebug index ff48eb673..fd5d589f2 100644 --- a/hw/bsp/rp2040/boards/raspberry_pi_pico2/ozone/rp2350.jdebug +++ b/hw/bsp/rp2040/boards/pico2_etm_trace/ozone/rp2350.jdebug @@ -6,17 +6,22 @@ * Project load routine. Required. * * Notes -* Pico 2 has no trace connector - fly-wire GPIO1-5 to the MIPI20: +* Board pico2_etm_trace = a Pico 2 seated on the pico2 trace motherboard +* carrier (MIPI-20, source-terminated), GPIO1-5 to the MIPI20: * TRACECLK=GPIO1->12, D0=GPIO2->14, D1=GPIO3->16, D2=GPIO4->18, -* D3=GPIO5->20 (SEGGER validates this board the same way). Firmware must -* be built with TRACE_ETM=1: it pins clk_sys to 48 MHz (board.cmake) so -* the 4-bit port never saturates and the clock never steps mid-stream, -* and keeps the us-timer free of TIMER DBGPAUSE (family.c). The whole -* chip-side trace path (ETM/funnel/TPIU/pin mux) is armed by J-Link's -* built-in RP2350 script at every resume - do NOT set a custom -* JLinkScript here: it would replace that script and J-Link then fails -* with "Required trace components for pin trace not found". -* GPIO1 is the default UART0 RX: console TX still works, RX is lost. +* D3=GPIO5->20. Firmware needs NO trace-specific code: J-Link's +* built-in RP2350 device script declares the off-ROM-table trace +* components (funnel/TPIU/ETM) and re-arms the whole chip-side path +* via OnTraceStart at every resume - do NOT set a custom JLinkScript +* here (it replaces that built-in script and J-Link then fails with +* "Required trace components for pin trace not found"). TRACE_ETM (set +* by this board's own board.cmake) clears TIMER0/1 DBGPAUSE - J-Link +* does not, and the reset default freezes the us-timer while a core is +* debug-halted - and adds compile-time checks that no console/I2C pin +* lands on the trace pins GP1-5; the console is full-duplex on GP12/13. +* clk_sys is the rp2350 pico-sdk default +* 150 MHz (75 MHz TRACECLK) and nothing may re-switch it at runtime: +* a mid-stream step silently truncates the capture. * ********************************************************************** */ @@ -24,7 +29,11 @@ void OnProjectLoad (void) { Project.SetTraceSource ("Trace Pins"); Project.SetTracePortWidth (4); Project.SetSWO (0); - Edit.SysVar (VAR_TRACE_CORE_CLOCK, 48000000); + // +1 ns data sampling: at 75 MHz TRACECLK (DDR) on the trace motherboard + // the idle eye spans -1000..+2000 ps and cdc_msc passes 3/3 at +1000 + // (+3000 dead; TD aliases modulo the 6.67 ns UI) + Project.SetTraceTiming (1000, 1000, 1000, 1000); + Edit.SysVar (VAR_TRACE_CORE_CLOCK, 150000000); Project.AddSvdFile ("$(InstallDir)/Config/CPU/Cortex-M33F.svd"); Project.SetDevice ("RP2350_M33_0"); @@ -32,7 +41,7 @@ void OnProjectLoad (void) { Project.SetTargetIF ("SWD"); Project.SetTIFSpeed ("25 MHz"); - File.Open ("../../../../../../examples/cmake-build-raspberry_pi_pico2/device/cdc_msc/cdc_msc.elf"); + File.Open ("../../../../../../examples/cmake-build-pico2_etm_trace/device/cdc_msc/cdc_msc.elf"); } /********************************************************************* diff --git a/hw/bsp/rp2040/boards/raspberry_pi_pico2/board.cmake b/hw/bsp/rp2040/boards/raspberry_pi_pico2/board.cmake index 08384b0cd..0a7dd4d23 100644 --- a/hw/bsp/rp2040/boards/raspberry_pi_pico2/board.cmake +++ b/hw/bsp/rp2040/boards/raspberry_pi_pico2/board.cmake @@ -1,17 +1,3 @@ set(PICO_PLATFORM rp2350-arm-s) set(PICO_BOARD pico2) #set(OPENOCD_SERIAL E6614103E77C5A24) - -if (TRACE_ETM STREQUAL "1") - # TRACECLK is clk_sys/2 and must stay constant once trace is armed (a step - # desyncs the decoder), so the trace clock is pinned from crt0 onwards. - # 48 MHz (24 MHz TRACECLK) holds full-width trace on a typical fly-wire - # seating; a fresh, tight seating supports up to 72-80 MHz (re-qualify per - # the etm-trace skill), and >80 MHz needs a V3 probe + real trace board. - add_compile_definitions( - SYS_CLK_KHZ=48000 - PLL_SYS_VCO_FREQ_HZ=1440000000 - PLL_SYS_POSTDIV1=6 - PLL_SYS_POSTDIV2=5 - ) -endif () diff --git a/hw/bsp/rp2040/family.c b/hw/bsp/rp2040/family.c index e12f51b14..9de1658eb 100644 --- a/hw/bsp/rp2040/family.c +++ b/hw/bsp/rp2040/family.c @@ -158,15 +158,34 @@ static void stdio_rtt_init(void) { } #endif -//--------------------------------------------------------------------+ -// -//--------------------------------------------------------------------+ #if defined(TRACE_ETM) && defined(PICO_RP2350) && PICO_RP2350 == 1 -// J-Link's built-in RP2350 device script re-arms the whole chip-side trace -// path (ETM/funnel/TPIU/pins) via OnTraceStart at every resume, so firmware -// must NOT touch it - it only keeps the us-timer running while cores sit -// debug-halted (default TIMER DBGPAUSE freezes it, and sleep_ms() then spins -// forever after any debugger session). +// ETM trace owns GP1-5 (GP1 = TRACECLK, GP2-5 = TRACEDATA0-3): muxing any of +// them away - even briefly - gaps the trace clock/data and desyncs the probe. +#define TRACE_PIN_CONFLICT(pin) ((pin) >= 1 && (pin) <= 5) +// board_init() muxes UART_TX_PIN/UART_RX_PIN, which are defined whenever UART_DEV is +#ifdef UART_DEV + #if TRACE_PIN_CONFLICT(UART_TX_PIN) || TRACE_PIN_CONFLICT(UART_RX_PIN) + #error "TRACE_ETM: UART TX/RX sits on a trace pin (GP1-5) - route the console elsewhere (pico2_etm_trace uses GP12/13)" + #endif +#endif +// stdio_init_all() muxes the sdk defaults even when the BSP console is elsewhere +#if defined(LIB_PICO_STDIO_UART) && defined(PICO_DEFAULT_UART_TX_PIN) && \ + (TRACE_PIN_CONFLICT(PICO_DEFAULT_UART_TX_PIN) || TRACE_PIN_CONFLICT(PICO_DEFAULT_UART_RX_PIN)) + #error "TRACE_ETM: pico-sdk default UART (stdio_init_all) sits on a trace pin (GP1-5)" +#endif +#if defined(PICO_DEFAULT_I2C_SDA_PIN) && (TRACE_PIN_CONFLICT(PICO_DEFAULT_I2C_SDA_PIN) || TRACE_PIN_CONFLICT(PICO_DEFAULT_I2C_SCL_PIN)) + // #pragma message, not #warning: examples build with -Werror, and this is + // only a hazard if the app actually uses i2c_default + #pragma message("TRACE_ETM: default I2C SDA/SCL sits on a trace pin (GP1-5) - using i2c_default will corrupt the trace stream (pico2_etm_trace routes I2C to GP8/9)") +#endif + +// A debugger session leaves a core halted (Ozone captures halt at the end, +// openocd halts both cores to flash), and TIMER's reset default pauses the +// us-timer whenever EITHER core is debug-halted - J-Link's RP2350 script does +// NOT clear it (verified: DBGPAUSE still reads 0x7, TIMERAWL frozen while +// halted). tusb_time_millis_api()/sleep_ms() then spin forever and the board +// looks dead, so free the timer for trace builds, which always run under a +// probe. static void trace_etm_init(void) { *(volatile uint32_t*) 0x400B002Cu = 0; // TIMER0 DBGPAUSE *(volatile uint32_t*) 0x400B802Cu = 0; // TIMER1 DBGPAUSE @@ -177,20 +196,15 @@ static void trace_etm_init(void) { void board_init(void) { + trace_etm_init(); + #if (CFG_TUH_ENABLED && CFG_TUH_RPI_PIO_USB) || (CFG_TUD_ENABLED && CFG_TUD_RPI_PIO_USB) - // Set the system clock to a multiple of 12mhz for bit-banging USB with pico-usb - #if defined(PICO_RP2350) && PICO_RP2350 == 1 - #ifdef TRACE_ETM - #error "TRACE_ETM pins clk_sys to 48 MHz (board.cmake) - too slow for PIO-USB, and a runtime clock switch desyncs the trace stream" - #endif - set_sys_clock_khz(156000, true); // rp2350 default is 150Mhz - #else + // rp2350 runs the pico-sdk stock 150 MHz (a runtime switch also truncates ETM + // capture). rp2040 keeps 120 MHz: soak-tested — the stock 125 MHz collapses + // PIO-USB bulk-OUT (device NAKs ~600:1, wire-measured, zero CRC errors). + #if !(defined(PICO_RP2350) && PICO_RP2350 == 1) set_sys_clock_khz(120000, true); // rp2040 default is 125Mhz #endif - // set_sys_clock_khz(180000, true); - // set_sys_clock_khz(192000, true); - // set_sys_clock_khz(240000, true); - // set_sys_clock_khz(264000, true); #ifdef PICO_DEFAULT_PIO_USB_VBUSEN_PIN gpio_init(PICO_DEFAULT_PIO_USB_VBUSEN_PIN); @@ -217,17 +231,9 @@ void board_init(void) #ifdef UART_DEV uart_inst = uart_get_instance(UART_DEV); -#if defined(TRACE_ETM) && defined(PICO_RP2350) && PICO_RP2350 == 1 - // GPIO1 (default UART RX) is TRACECLK: TX-only console, and never touch - // GPIO1 - even a brief re-mux gaps the trace clock and desyncs the probe - bi_decl(bi_1pin_with_name(UART_TX_PIN, "UART TX")); - stdio_uart_init_full(uart_inst, CFG_BOARD_UART_BAUDRATE, UART_TX_PIN, -1); -#else bi_decl(bi_2pins_with_func(UART_TX_PIN, UART_RX_PIN, GPIO_FUNC_UART)); stdio_uart_init_full(uart_inst, CFG_BOARD_UART_BAUDRATE, UART_TX_PIN, UART_RX_PIN); #endif -#endif - trace_etm_init(); #if defined(LOGGER_RTT) stdio_rtt_init(); diff --git a/test/hil/helper/hil_health.py b/test/hil/helper/hil_health.py index 92f0accc8..d78d0f220 100644 --- a/test/hil/helper/hil_health.py +++ b/test/hil/helper/hil_health.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: MIT -"""Shutting a wedged HIL run down: kill what the workers spawned, then report. +"""Shutting a wedged HIL run down: kill what the workers spawned. A device whose usbfs node is held by a D-state process cannot be freed -- SIGKILL is not delivered in uninterruptible sleep -- so the goal is never to fix the rig from here. It is @@ -214,9 +214,6 @@ def _kill_kids(kids: dict, seen: set) -> int: own = os.getpgid(0) except OSError: own = None # cannot tell our own group apart: never killpg, signal pids only - # One list: every pid here is a DESCENDANT of one of our own workers, so it is ours by - # construction -- no argv identity check needed, because we never signal anything we - # did not discover through our own ppid tree. touched: list = [] for children in kids.values(): for cpid, cpgid in children: @@ -249,10 +246,8 @@ def _kill_kids(kids: dict, seen: set) -> int: if denied: _p(f'warning: could not kill {sorted(denied)}; they still hold whatever they ' f'had open (probe, usbfs node) into the next job', flush=True) - # SURVIVORS, not the signalled-child count: the caller needs to know the rig is dirty - # for the next job, and a count of what we successfully signalled cannot tell it that. - # (They are different units anyway -- a killpg is counted once per child sharing the - # group -- so the old return was never comparable to anything.) + # SURVIVORS, not the count we signalled: the caller needs to know the rig is dirty for + # the next job, and a killpg is counted once per child sharing the group anyway. return len(denied) @@ -341,40 +336,3 @@ def kill_pool_children(pool, *extra) -> int: # SIGKILL is asynchronous and a D-state task ignores it: only a confirmed survivor # justifies the caller's power-cycle wording return len(_kill_and_confirm(killed_pids)) if killed_pids else 0 - - -def write_timeout_report(report_dir: Path, boards, secs: int, md_name: str, - banner: str = '', prefix: str = '') -> None: - """Leave a report behind when the worker pool has to be abandoned. - - map_async is all-or-nothing, so a timeout loses every per-board result and the report - dir would stay empty with no reason for the failure. Any prior attempt's markdown is - kept below the banner.""" - # `prefix` carries the preflight rig-health verdict: the timeout aborts before - # accumulate_report, so without it the report loses the one line saying WHY the pool - # never finished. The '\n' stops Markdown lazy continuation pulling the banner into - # the blockquote. - try: - # Built INSIDE the try: a roster entry without a 'name' key raises KeyError while - # assembling the board list, and outside the try that escaped and stranded the - # runner -- which is exactly what the broad handler below exists to prevent. - head = (prefix + '\n' if prefix else '') + (banner or ( - f'**HIL run abandoned: worker pool timed out after {secs}s.**\n\n' - f'No per-board results could be collected for this attempt, so the ' - f'table below (if any) is from an earlier one. Boards dispatched:\n\n' - + '\n'.join(f'- {b.get("name", "?")}' for b in boards) + '\n')) - report_dir.mkdir(parents=True, exist_ok=True) - md_path = report_dir / md_name - # Its own handler so it cannot take the write down with it: a report torn by an - # attempt killed mid-write raises UnicodeDecodeError (a ValueError, and prior - # reports always contain status emoji), which under a shared try skipped the write - # entirely. Losing the old table is a nicety; losing the banner is the failure. - try: - prior = md_path.read_text(encoding='utf-8') if md_path.is_file() else '' - except (OSError, ValueError): - prior = '' - md_path.write_text(head + (f'\n{prior}' if prior else ''), encoding='utf-8') - except Exception as e: # noqa: BLE001 - # Deliberately broad: this is the first statement of the pool-abandon path, so ANY - # escape skips kill_pool_children and os._exit and strands the runner. - _p(f'warning: cannot write {md_name} to {report_dir}: {e}', flush=True) diff --git a/test/hil/helper/hil_lock.py b/test/hil/helper/hil_lock.py index 7757ef17d..91f05ca86 100755 --- a/test/hil/helper/hil_lock.py +++ b/test/hil/helper/hil_lock.py @@ -175,13 +175,12 @@ def controller_of(uid: str): if cached: return cached # vid='cafe' first: the target is always a TinyUSB DUT, and the VID is a lock-free - # descriptor field. Without it this read every probe's and hub's `serial` -- the - # attribute served under device_lock -- so a HEALTHY peer mid-usbtest would strand a - # reader here and spend one of this worker's four blindness credits. - devs, _ = hil_util.usb_scan(vid='cafe', serial=uid) + # descriptor field. Without it this reads every probe's and hub's `serial` -- the one + # attribute served under device_lock -- so a wedged peer would block us here. + devs = hil_util.usb_scan(vid='cafe', serial=uid) for dev in devs: busnum = hil_util.read_sysfs(os.path.join(dev['dir'], 'busnum')) - if busnum is None or busnum is hil_util.SYSFS_UNKNOWN: + if busnum is None: continue try: root = os.path.realpath(f'/sys/bus/usb/devices/usb{int(busnum)}') @@ -212,7 +211,7 @@ def controller_slot(pci: str) -> int: # Unresolved boards budget in a slot of their OWN, one past the real ones, and that slot # holds exactly ONE permit whatever the per-controller width is. Neither neighbour works: # a permit on every slot (the old fail-closed rule) serialized the whole fleet the moment -# a worker went blind, while a full private budget let unknown boards run a second +# one board could not be resolved, while a full private budget let unknown boards run a second # controller's worth of batteries on top of the resolved ones -- doubling the load on # whichever physical controller they actually sit on, which is the saturation the # uPD720201 deaths above are attributed to. Width 1 caps the over-subscription at +1. @@ -252,8 +251,8 @@ class controller_permit: if pci is None: pci = controller_of(uid) if pci is None and warn_unknown: - log(f'warning: cannot resolve {uid} to a host controller' - f'{hil_util.sysfs_blind_note()}; budgeting it in the unknown bucket') + log(f'warning: cannot resolve {uid} to a host controller; ' + f'budgeting it in the unknown bucket') self.slots = [controller_slot(pci) if pci else UNKNOWN_SLOT] def __enter__(self): diff --git a/test/hil/helper/hil_pool_check.py b/test/hil/helper/hil_pool_check.py index 179a417ed..d98b92bd4 100644 --- a/test/hil/helper/hil_pool_check.py +++ b/test/hil/helper/hil_pool_check.py @@ -54,7 +54,7 @@ ENUM_WAIT_RETRY = 8 # s, uid wait after a recovery reset/re-flash SERIAL_WAIT = 6 # s, host-board serial-output wait print_mutex = threading.Lock() -_UNKNOWN_WARNED = False # scan_usb's caveat: once per process, not once per poll +_STRANDED_WARNED = False # scan_usb's caveat: once per process, not once per poll t0 = time.monotonic() @@ -72,20 +72,20 @@ def scan_usb() -> dict: USB-Serial-JTAG bridge and the cafe device it flashes both derive it from the same MAC), and one dict slot would silently drop whichever lost the race.""" found = {} - # `unknown` matters BEFORE the blindness latch trips: one wedged device is the normal - # reason this tool is run, and its serial read stranding makes it absent from `devs`. - # Reported as fact, that is "probe MISSING" for hardware that is physically present. - devs, unknown = hil_util.usb_scan() - # ONCE per process: this is called from 0.5s poll loops across 4 worker threads and - # ~26 boards, so warning per call buried the table it exists to qualify under 600+ - # identical lines. The memo in read_sysfs makes the condition sticky, so one line is - # as true as six hundred. - global _UNKNOWN_WARNED - if unknown and not _UNKNOWN_WARNED: - _UNKNOWN_WARNED = True - say('WARNING: at least one device did not answer a bounded read; rows below that ' - 'say a probe or board is missing may be this scan losing sight of healthy ' - 'hardware. Find the wedged device (usb-kernel-recover) and re-run.') + # usb_scan's `serial` read is bounded by default (see hil_util.read_sysfs) -- this tool + # has no pool guard behind it and is run exactly when a device is suspected wedged. A + # device that will not answer is simply absent from the table; the footer says so. + devs = hil_util.usb_scan() + # ONCE per process, at SCAN time, not only in the footer: this tool prints rows as it + # goes over minutes, so a board dropped from the scan says "probe MISSING" within + # seconds while the only qualification would arrive after the final counts -- and an + # operator acting on the streaming output, or a run cut short by ^C, never sees it. + global _STRANDED_WARNED + if hil_util.sysfs_stranded() and not _STRANDED_WARNED: + _STRANDED_WARNED = True + say('WARNING: a bounded sysfs read gave up; rows below that say a probe or board ' + 'is missing may be this scan losing sight of healthy hardware. Find the ' + 'wedged device (usb-kernel-recover) and re-run.') for dev in devs: try: found[dev['busport']] = { @@ -360,7 +360,47 @@ def check_host_serial(board: dict, do_reset: bool = True, want_hello: bool = Fal do_reset=False listens to the firmware as-is: used right after a flash whose own reset already started it — a second openocd/JLink session back-to-back on - the same probe can fail transiently and leave the target halted.""" + the same probe can fail transiently and leave the target halted. + + "logger": "rtt" boards have no VCOM: the same check runs over the probe's RTT + console instead. The reset happens BEFORE the console opens (it owns the probe), + which also zeroes the .bss ring — so pre-reset backlog cannot count as life, and + without a reset Commander delivers the boot burst the preceding flash left.""" + if board.get('logger') == 'rtt': + if do_reset: + # a failed reset leaves the previous run's ring intact: attaching anyway would + # score stale output as life, so bail to host_alive's board_test reflash ladder + rc, err = call_flasher(getattr(hil_flash, f'reset_{board["flasher"]["name"].lower()}'), board) + if rc: + say(f'{board["name"]:26} reset failed: {err}') + return None + try: + ser = hil_util.JlinkRtt(board, timeout=0.3) + except hil_util.RttError as e: + say(f'{board["name"]:26} no RTT console: {e}') + return None + try: + data = b'' + deadline = time.monotonic() + SERIAL_WAIT + while time.monotonic() < deadline: + ser.write(b'U') + data += ser.read(256) + # JLinkExe's banner arrives whether or not the target is alive -- + # judged unfiltered it scores a dead board 'alive'. Same shared filter + # as test_host_device_info; complete_only drops a trailing partial + # line, so a banner FRAGMENT split by this read boundary cannot count + # as target output either. + td = hil_util.strip_banner(data, complete_only=True) + if want_hello: + if b'Hello from TinyUSB' in td: + return td + elif td and not boardtest_output(td): + return td + return hil_util.strip_banner(data) + except hil_util.RttError: + return None # console died mid-poll (server exited, probe dropped) + finally: + ser.close() import serial try: port = hil_util.get_serial_dev(board['flasher']['uid'], None, None, 0) @@ -983,9 +1023,13 @@ def main() -> None: headers = ['Board', 'Probe', 'Flash', 'Device', 'Status', 'Note'] cells = [[r['name'], r['probe'], r['flash'], r['device'], status_mark.get(r['status'], r['status']), '; '.join(r['note'])] for r in rows] - widths = [max(len(h), *(len(c[i]) for c in cells)) if cells else len(h) + # display_width, not len(): ✅ / ❌ / 🔒 / ⚠ are one character and two columns, so + # len() pads every row holding one a column short of the header rule + _w = hil_util.display_width + widths = [max(_w(h), *(_w(c[i]) for c in cells)) if cells else _w(h) for i, h in enumerate(headers)] - line = lambda vals: '| ' + ' | '.join(v.ljust(w) for v, w in zip(vals, widths)) + ' |' + line = lambda vals: ('| ' + ' | '.join(hil_util.pad(v, w) + for v, w in zip(vals, widths)) + ' |') print() print(line(headers)) print('|' + '|'.join('-' * (w + 2) for w in widths) + '|') @@ -1001,17 +1045,16 @@ def main() -> None: counts[r.get('status', 'failed')] += 1 print(f'\n{counts["ok"]} ok · {counts["flash-failed"]} flash-failed · {counts["failed"]} failed ' f'· {counts["locked"]} locked · in {time.monotonic() - t0:.0f}s') - if hil_util.sysfs_blind(): - # Without this the table is the worst kind of wrong: once the process latches - # blind, every read answers SYSFS_UNKNOWN, scan_usb() returns {}, and EVERY board - # prints "probe MISSING"/"off bus" -- a clean-looking report declaring the whole - # fleet dead, produced during exactly the incident this tool is run to diagnose, - # and it sends the operator to power-cycle a rig where one device is wedged. - print('WARNING: this scan lost sight of the bus' - f'{hil_util.sysfs_blind_note()}. Rows above that say a probe or board is ' - f'missing may be this tool losing sight of healthy hardware, not absent ' - f'hardware. Find the wedged device (see the usb-kernel-recover skill) and ' - f're-run before acting on the table.') + if hil_util.sysfs_stranded(): + # Without this the table is the worst kind of wrong: a device whose `serial` never + # answered is absent from the scan, which prints as "probe MISSING"/"off bus" for + # hardware that is physically present -- during exactly the incident this tool is + # run to diagnose, and it sends the operator to power-cycle a healthy rig. + print('WARNING: at least one sysfs read did not answer within ' + f'{hil_util.SYSFS_READ_GRACE:.0f}s, so rows above that say a probe or board ' + f'is missing may be this tool losing sight of healthy hardware rather than ' + f'absent hardware. Find the wedged device (see the usb-kernel-recover ' + f'skill) and re-run before acting on the table.') sys.exit(min(counts['flash-failed'] + counts['failed'], 125)) diff --git a/test/hil/helper/hil_report.py b/test/hil/helper/hil_report.py new file mode 100644 index 000000000..c93c8e6a1 --- /dev/null +++ b/test/hil/helper/hil_report.py @@ -0,0 +1,578 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""The HIL report document: one owner for hil_report.json and hil_report.md. + +The markdown IS a rendering of the sidecar -- every writer goes through render_report(), so +a table can never contain something the JSON does not. This module owns the whole life of +that document: the cell vocabulary, the one classifier both artifacts share, rendering, the +writers, and the fold to one machine-readable verdict per board. + +Dual-mode by design: imported as `helper.hil_report` by hil_test.py, and run as a script by +the operator (see .claude/agents/hil-operator.md). A script run puts test/hil/helper on +sys.path rather than test/hil, so this module imports no sibling helper at all -- +_p and the width helpers below are defined locally for that reason. +""" +import argparse +import json +import sys +import unicodedata +from pathlib import Path + + +def _w(s: str) -> int: + """Terminal COLUMNS, not characters. Every status mark in REPORT_CELL is one Python + character and TWO columns wide, so len() pads a cell holding one a column short and + the pipes drift out of line with the header rule for the whole table. + + Local, like _p above and for the same reason: this module is also run as a script, and + under PYTHONSAFEPATH=1 a sibling import dies before argparse runs. hil_util carries the + same pair for callers that can import it. + """ + return sum(2 if unicodedata.east_asian_width(c) in 'WF' else 1 for c in s) + + +def _pad(s: str, width: int, center: bool = False) -> str: + """str.ljust/center, measured in display columns. See _w.""" + room = max(0, width - _w(s)) + if not center: + return s + ' ' * room + left = room // 2 + return ' ' * left + s + ' ' * (room - left) + + +def _p(*args, **kwargs) -> None: + """Print that cannot raise. Defined here rather than imported from hil_health: this + module is ALSO run as a script (hil-operator.md invokes it by path), and under + PYTHONSAFEPATH=1 -- which the suite's own MTP fixtures set -- sys.path[0] is not the + script dir, so any sibling import dies before argparse runs. Five lines beat that.""" + try: + print(*args, **kwargs) + except (OSError, ValueError): + # ValueError too: printing to a CLOSED stream raises "I/O operation on closed + # file", and escaping here skips the containment path's os._exit. + pass + +REPORT_MD = 'hil_report.md' +REPORT_JSON = 'hil_report.json' +# The status vocabulary, shared by the code that WRITES a cell (hil_test's test runners) and +# the code that reads one back (cell_state). One dict, so the human's table and the agent's +# verdict cannot drift apart. +REPORT_CELL = {'pass': '✅', 'fail': '❌', 'skip': '⚪'} +BOUNDARY_CELL = 'same-PID boundary' +LOCKED_CELL = 'board-locked' +# A pseudo-test column, not a real one: write_timeout_report marks the boards that were +# still dispatched when the pool guard fired. accumulate_report clears it on a retry. +POOL_TIMEOUT_CELL = 'pool-timeout' +# The other way a board can fail to report: the pool did not expire, a worker RAISED. Same +# shape, different cause, and naming the cause is the whole point of the column -- a board +# marked pool-timeout by an abort that never timed out sends the reader after the guard. +RUN_ABORTED_CELL = 'run-aborted' + + +def _load(report_dir: Path) -> tuple: + """(doc, readable) for the sidecar, coerced to the canonical shape. + + hil_ci.sh uploads a sidecar as the --accumulate merge base, so a non-conforming one is + reachable from OUTSIDE the harness -- and every writer here runs on a path where a + TypeError costs the whole report. Coerce once, at the boundary, instead of guarding + each use: `banner: null` used to kill a fully successful run with a traceback and no + artifact at all, and `cells: null` sent write_timeout_report down its fallback so a + board that ate the whole pool guard was published as a pass. + + `readable` is False only when a sidecar EXISTS but could not be parsed, or is absent -- + both mean its rows are unrecoverable, which callers use to avoid destroying a markdown + that may still hold them.""" + jpath = report_dir / REPORT_JSON + if not jpath.is_file(): + return {'rows': [], 'banner': '', 'scope': '', 'caveat': ''}, False + try: + raw = json.loads(jpath.read_text()) + if not isinstance(raw, dict): + raise ValueError('sidecar is not an object') + except (OSError, ValueError, TypeError): + return {'rows': [], 'banner': '', 'scope': '', 'caveat': ''}, False + rows = [] + # isinstance, not `or []`: a sidecar with `rows: 1` iterates an int and raises outside + # the parse handler above. + for r in (raw.get('rows') if isinstance(raw.get('rows'), list) else []): + if not isinstance(r, dict) or 'board' not in r: + continue + cells = r.get('cells') + dur = r.get('duration') + # VALUES as well as keys: render_matrix does REPORT_CELL.get(v, v), which raises + # TypeError on an unhashable value, and cell_state does v.startswith. A non-str + # cell is corrupt, and dropping it renders blank -- "not run" -- which is the + # honest reading. Coercing it to str would make it classify as a PASS. + rows.append({'board': str(r['board']), + 'cells': {str(k): v for k, v in cells.items() if isinstance(v, str)} + if isinstance(cells, dict) else {}, + 'duration': dur if isinstance(dur, str) else None}) + text = lambda k: raw[k] if isinstance(raw.get(k), str) else '' + return {'rows': rows, 'banner': text('banner'), 'scope': text('scope'), + 'caveat': text('caveat')}, True + + +def cell_state(v) -> str: + """'pass' | 'fail' | 'skip' for one report cell. + + THE classifier -- the markdown tally and the per-board verdict both call this, so they + cannot disagree. 'fail' or a fail-icon prefix is a failure, 'skip' or a skip-icon prefix + is a skip, and EVERYTHING ELSE is a pass. That last arm is load-bearing: a passing test + may return a plain metric string ('480.0 MBps') that lands in the cell unprefixed, while + failures are guaranteed marked -- TestFail's docstring pins that its metric is + icon-prefixed precisely so render and tally treat it as a failure. Classifying unknown + shapes as fail here would publish a green table as a red verdict. + + isinstance-guarded: cells are usually str but a caller may hand over None or a number, + and .startswith on those raises inside a report writer that must not raise.""" + if v == 'fail' or (isinstance(v, str) and v.startswith(REPORT_CELL['fail'])): + return 'fail' + if v == 'skip' or (isinstance(v, str) and v.startswith(REPORT_CELL['skip'])): + return 'skip' + return 'pass' + + +def render_matrix(rows_all: list) -> str: + """Render rows (list of (row_label, {example: status}, duration)) as an aligned + markdown matrix: columns = tests (bare names) centered, boards left-aligned, + per-row duration as the trailing column.""" + seen = set() + for _, cells, _ in rows_all: + seen.update(cells) + if not seen: + return 'No tests were run.' + + # metric-bearing columns pinned first, the rest alphabetical: stable regardless of the + # shuffled execution order + pinned = ['usbtest', 'cdc_msc_throughput', 'msc_file_explorer', 'msc_file_explorer_freertos'] + + def col_key(t): + name = t.rsplit('/', 1)[-1] + return (pinned.index(name) if name in pinned else len(pinned), name, t) + + columns = sorted(seen, key=col_key) + headers = [c.rsplit('/', 1)[-1] for c in columns] + ['duration'] # bare example names + + def cell(cells, col): + v = cells.get(col) + if v is None: + return '' + return REPORT_CELL.get(v, v) # status symbol, or a metric string (e.g. speed) verbatim + + rows_vals = [(lbl, [cell(cells, c) for c in columns] + [dur or '']) + for lbl, cells, dur in rows_all] + board_hdr = 'Board' + # display_width, not len(): the ✅/❌/⚪ marks are one character and two columns + board_w = max([_w(board_hdr)] + [_w(lbl) for lbl, _ in rows_vals]) + col_w = [max([_w(h)] + [_w(vals[i]) for _, vals in rows_vals]) + for i, h in enumerate(headers)] + + def line(label, values): + padded = [_pad(label, board_w)] + [_pad(v, w, center=True) + for v, w in zip(values, col_w)] + return '| ' + ' | '.join(padded) + ' |' + + header = line(board_hdr, headers) + sep = '| ' + '-' * board_w + ' | ' + ' | '.join(':' + '-' * (w - 2) + ':' for w in col_w) + ' |' + body = [line(lbl, vals) for lbl, vals in rows_vals] + + # tally run cells (not-run cells are absent from the dicts). A cell is a bare status or + # a metric string carrying its own icon ("❌ 29/30"), so classify by the leading icon -- + # through cell_state, the same call the per-board verdict makes. + kinds = [cell_state(v) for _, cells, _ in rows_all for v in cells.values()] + failed = kinds.count('fail') + skipped = kinds.count('skip') + passed = kinds.count('pass') + summary = (f'**{REPORT_CELL["pass"]} {passed} passed · {REPORT_CELL["fail"]} {failed} failed · ' + f'{REPORT_CELL["skip"]} {skipped} skipped · blank not run**') + + return summary + '\n\n' + '\n'.join([header, sep] + body) + + +def render_report(doc: dict) -> str: + """The markdown IS a rendering of the sidecar. Every writer goes through here, so a + table can never contain something the JSON does not.""" + # .get throughout, not subscripts: mark_report_abandoned renders a sidecar it did NOT + # write (hil_ci.sh reuses a persistent REMOTE_DIR, so it may be an older version's or + # a torn one) on the way to os._exit, and a KeyError there is not in its handler -- + # it would unwind into multiprocessing's unbounded join and hang the runner it is + # trying to free. Same reason summarize() below reads cells as `r.get('cells') or {}`. + md = render_matrix([(r.get('board', '?'), r.get('cells') or {}, r.get('duration')) + for r in doc.get('rows') or [] if isinstance(r, dict)]) + if doc.get('scope'): + # a scoped run's small table is otherwise indistinguishable from a full one, and + # it replaces the previous full table in the sticky PR comment + md = f'_Scoped run: {doc["scope"]}. Boards/tests not listed were not run._\n\n' + md + # banner, then caveat: a rig-health caveat outranks the table AND the scope note, and an + # abandon notice outranks even that -- the top of the report is where hil/SKILL.md tells + # the agent to look + if doc.get('banner'): + md = doc['banner'] + '\n' + md + if doc.get('caveat'): + md = doc['caveat'] + '\n' + md + return md + + +def write_report(report_dir: Path, doc: dict) -> None: + """Write both artifacts from one document. + + RAISES on failure, deliberately: every caller is on a path whose own handler exists to + report exactly this (write_timeout_report's _p warning, hil_test's fallback-of-the- + fallback). Swallowing OSError here made both of those dead code, so an unwritable or + root-owned report dir produced no artifact AND no message. + + Renders BEFORE writing anything: committing the JSON first and then raising in + render_report left a sidecar saying "abandoned" beside a markdown still reading as a + clean green table -- the one invariant this module exists to hold.""" + md = render_report(doc) + '\n' + report_dir.mkdir(parents=True, exist_ok=True) + (report_dir / REPORT_JSON).write_text(json.dumps(doc, indent=2) + '\n') + (report_dir / REPORT_MD).write_text(md, encoding='utf-8') + + +def _abandon_notice(why: str) -> str: + # Wording is a CONTRACT: .claude/skills/hil/SKILL.md pins this banner as the case where + # "the table below IS this run's ... Report the results AND the abandonment". Calling + # the table partial would send the reading agent to re-run boards that already passed. + return (f'**HIL run abandoned: {why}** The table below was collected before the ' + f'abandon; treat board results as unverified.\n') + + +def _already_abandoned(doc: dict) -> bool: + """Whether THIS attempt already recorded how it ended. + + `caveat` only. It used to check `banner` too, because hil_test.py folded its abandon + notices in there -- but banner is carried across an --accumulate retry by design, so a + stale notice from an earlier attempt silenced a genuinely new abandon and the run's own + failure went unrecorded. banner now carries rig HEALTH (which describes the conditions + the cells were collected under, and so must persist); caveat carries the run's OUTCOME + (which must not).""" + return '**HIL run ab' in doc.get('caveat', '') + + +def _stamp_markdown(report_dir: Path, notice: str) -> None: + """Last line of defence: prepend the notice to the markdown itself. + + pr_comment.yml cats only hil_report.md, so a path that gives up here publishes a clean + green table under an abandoned, non-zero job. Master did this unconditionally.""" + mpath = report_dir / REPORT_MD + if not mpath.is_file(): + return + # errors='replace' and catch ValueError: a torn report or a LANG=C locale raises + # UnicodeDecodeError -- NOT an OSError -- straight past os._exit. + body = mpath.read_text(encoding='utf-8', errors='replace') + if '**HIL run ab' not in body[:2000]: + mpath.write_text(notice + '\n' + body, encoding='utf-8') + + +def mark_report_abandoned(report_dir: Path, why: str) -> None: + """Stamp an existing report as abandoned, in BOTH artifacts. + + Best-effort and silent: this runs while the interpreter is being torn down, and an + exception here hangs the process in multiprocessing's unbounded join().""" + notice = _abandon_notice(why) + try: + doc, readable = _load(report_dir) + if readable: + if _already_abandoned(doc): + return # whoever got there first wins, WRITE included + doc['caveat'] = notice + write_report(report_dir, doc) + return + except (OSError, ValueError, TypeError, AttributeError): + pass # fall through -- a failure here must not cost the stamp entirely + # Unreadable sidecar, or the document write failed. Either way the markdown is what + # the PR comment reads, so stamp it directly rather than giving up. + try: + _stamp_markdown(report_dir, notice) + except (OSError, ValueError, TypeError, AttributeError): + pass + + +def mark_report_no_boards(report_dir: Path, msg: str, fresh: bool = True) -> None: + """Record that the board filters intersected to nothing. + + `fresh` mirrors hil_test's own flag, because this runs BEFORE the fresh wipe: without + it a fresh run whose filter emptied re-published the PREVIOUS run's green rows under + this run's red job -- the stale-table failure it exists to prevent. An --accumulate run + keeps them, since nothing this attempt did invalidates them.""" + try: + doc, _ = _load(report_dir) + if not fresh and _already_abandoned(doc): + # SKILL.md gives the two notices OPPOSITE rules, and an abandon outranks a + # filter that matched nothing -- do not overwrite the record of a failed run. + # Only while ACCUMULATING, though: this runs before the fresh wipe, so guarding + # a fresh run would leave the previous attempt's rows AND its abandon notice + # published as this run's. + return + # A fresh run carries NOTHING from the prior sidecar -- rows, banner and scope + # alike, matching accumulate_report, which builds from an empty prior when fresh. + # Resetting only rows republished a stale rig-health note and a stale scope line + # under this run's notice, from a leftover or uploaded sidecar. + prior = {'rows': [], 'banner': '', 'scope': ''} if fresh else doc + write_report(report_dir, {'rows': prior['rows'], 'banner': prior['banner'], + 'scope': prior['scope'], + 'caveat': f'**HIL run selected no boards.** {msg}\n'}) + except (OSError, ValueError, TypeError, AttributeError): + pass # loud on stdout already; the exit code is what the job reads + + +def accumulate_report(mret: list, report_dir: Path, fresh: bool, scope: str = '', + banner: str = '', caveat: str = '') -> str: + """Merge this run's results into json in report_dir, then (re)write + the markdown matrix to md. `fresh` (a first run, no --accumulate) + starts a new report; otherwise a re-run accumulates so boards/tests that + already passed are preserved while re-run cells are updated. `scope` names the + board filter, if any, so a scoped table is not mistaken for a full one. + Returns the md. + + `mret` is hil_test.py's worker-result shape (name, err, fts, rows, ...), so this one + function knows something about its caller that the rest of the module does not. Folding + mret into rows could live in hil_test and only the merge here, but that would rewrite + the subtle parts -- stale board-locked clearing, BOUNDARY_CELL dropping, duration=None + preservation -- for a tidier seam. Data-shape coupling, not an import cycle.""" + # ONE canonical load: a sidecar reaching here may have been uploaded by hil_ci.sh as + # the merge base, so it is untrusted input. `banner` carries forward -- it describes + # the conditions the earlier cells were collected under, and the .failed spec re-runs + # only FAILURES so those passes are never re-earned. `caveat` does NOT: it records how + # a RUN ENDED, and this attempt has not ended yet. Carrying it made a clean retry + # publish "HIL run abandoned" over a run where nothing was abandoned. + prior = {'rows': [], 'banner': ''} + if not fresh: + prior, _ = _load(report_dir) + acc = {r['board']: [dict(r['cells']), r['duration']] for r in prior['rows']} + prior_banner = prior['banner'] + + # current cells override prior for boards/tests that ran; a filtered run reports + # duration None, keeping the previous full-run value + for name, _, _, rows, *_ in mret: + if rows and not any(LOCKED_CELL in cells for _, cells, _ in rows): + # board ran for real: clear a stale lock-failure cell (its row is keyed by + # board name; test rows may be variant names) + stale = acc.get(name) + if stale is not None: + stale[0].pop(LOCKED_CELL, None) + # and the pool-timeout mark: write_timeout_report stamps it on a board that + # never reported, and update() below MERGES, so without this a board that + # passed clean on the retry kept a red cell for ever. + stale[0].pop(POOL_TIMEOUT_CELL, None) + stale[0].pop(RUN_ABORTED_CELL, None) + if not stale[0]: + # variant-keyed boards never repopulate the board-name row, so drop it + # or it renders as a blank ghost row + del acc[name] + for row_label, cells, dur in rows: + row = acc.setdefault(row_label, [{}, None]) + # a row that ran is no longer pool-timed-out, whatever it is keyed by + row[0].pop(POOL_TIMEOUT_CELL, None) + row[0].pop(RUN_ABORTED_CELL, None) + # the boundary cell is only ever written on failure, so a re-run of this + # variant that cleared the boundary must drop the previous attempt's ❌ + if BOUNDARY_CELL not in cells: + row[0].pop(BOUNDARY_CELL, None) + row[0].update(cells) + if dur is not None: + row[1] = dur + + report_dir.mkdir(parents=True, exist_ok=True) + # by LINE, deduped: attempts repeat the same caveat far more often than they add a new + # one, and three copies of the D-state note reads as three incidents + seen, merged = set(), [] + for line in (prior_banner + banner).splitlines(): + if line.strip() and line not in seen: + seen.add(line) + merged.append(line) + banner = '\n'.join(merged) + '\n' if merged else '' + doc = {'rows': [{'board': k, 'cells': c, 'duration': d} for k, (c, d) in acc.items()], + 'banner': banner, 'scope': scope, 'caveat': caveat} + # through write_report, not hand-rolled: writing the JSON and only then rendering is + # the ordering write_report exists to forbid -- a render failure left the sidecar ahead + # of the markdown, which is the one invariant this module holds. + write_report(report_dir, doc) + return render_report(doc) + + +def _write_stuck_over_prior_md(report_dir: Path, doc: dict) -> None: + """Sidecar unrecoverable: rebuild it from the stuck rows alone, but leave the + markdown's existing table beneath the caveat rather than throwing real results away. + + The one place the md-is-a-rendering-of-the-json invariant is deliberately suspended, + because there is no readable json left for it to be a rendering of.""" + try: + prior = (report_dir / REPORT_MD).read_text(encoding='utf-8') + except (OSError, ValueError): + prior = '' + # Say so explicitly: those rows exist only as rendered text, so no later --accumulate + # can merge them back. Claiming the sidecar represents them would be false. + note = ('_The table below is a previous attempt\'s rendered output. The sidecar could ' + 'not be read, so those rows are NOT in it and will not survive another run._\n') + head = (doc['banner'] + '\n' if doc['banner'] else '') + doc['caveat'] + '\n' + note + body = prior if prior.strip() else render_matrix( + [(r['board'], r['cells'], r['duration']) for r in doc['rows']]) + report_dir.mkdir(parents=True, exist_ok=True) + (report_dir / REPORT_JSON).write_text(json.dumps(doc, indent=2) + '\n') + (report_dir / REPORT_MD).write_text(head + '\n' + body, encoding='utf-8') + + +def write_timeout_report(report_dir: Path, boards, secs: int, + banner: str = '', prefix: str = '', + cell: str = POOL_TIMEOUT_CELL) -> None: + """Leave a report behind when the worker pool has to be abandoned. + + map_async is all-or-nothing, so a timeout loses every per-board result and the report + dir would stay empty with no reason for the failure. Any prior attempt's rows are kept + and each stuck board is marked with a POOL_TIMEOUT_CELL beside them. + + `prefix` is the preflight rig-health verdict and goes to the BANNER, where rig health + lives and where an --accumulate retry carries it forward; the abandon notice goes to + the caveat, which does not carry. Folding both into the caveat is what made a clean + retry report an abandonment that had not happened.""" + try: + # names INSIDE the try: a roster entry that is not a dict raises here, and outside + # it that escaped and stranded the runner. + names = [b.get('name', '?') if isinstance(b, dict) else '?' for b in boards] + caveat = banner or ( + f'**HIL run abandoned: worker pool timed out after {secs}s.**\n\n' + f'No per-board results could be collected for this attempt. Rows other than ' + f'the {cell} cells below are from an earlier attempt. Boards ' + f'dispatched:\n\n' + '\n'.join(f'- {n}' for n in names) + '\n') + doc, readable = _load(report_dir) + rows = doc['rows'] + by_board = {r['board']: r for r in rows} + for name in names: + row = by_board.get(name) + if row is None: + rows.append({'board': name, 'cells': {cell: 'fail'}, + 'duration': None}) + else: + # _load guarantees `cells` is a dict, so a null-cells row from an uploaded + # sidecar can no longer send this down the fallback and publish a board + # that ate the whole pool guard as a pass. + row['cells'][cell] = 'fail' + out = {'rows': rows, 'scope': doc['scope'], 'caveat': caveat, + 'banner': ((doc['banner'] + prefix) if prefix not in doc['banner'] + else doc['banner'])} + if not readable and (report_dir / REPORT_MD).is_file(): + # `readable` covers ABSENT as well as torn: an absent sidecar beside an intact + # markdown used to re-render from the stuck row alone and destroy real results. + _write_stuck_over_prior_md(report_dir, out) + return + write_report(report_dir, out) + except Exception as e: # noqa: BLE001 + # Deliberately broad: this is the first statement of the pool-abandon path, so ANY + # escape skips kill_pool_children and os._exit and strands the runner. + _p(f'warning: cannot write {REPORT_MD} to {report_dir}: {e}', flush=True) + try: + # Same wording as above and the same guarded name extraction -- the fallback + # used to re-derive b.get("name") outside any try and raise identically, so a + # malformed roster left NO artifact at all. + names = [b.get('name', '?') if isinstance(b, dict) else '?' for b in boards] + head = (prefix + '\n' if prefix else '') + (banner or ( + f'**HIL run abandoned: worker pool timed out after {secs}s.**\n\n' + f'No per-board results could be collected for this attempt, so the table ' + f'below (if any) is from an earlier one. Boards dispatched:\n\n' + + '\n'.join(f'- {n}' for n in names) + '\n')) + try: + prior = (report_dir / REPORT_MD).read_text(encoding='utf-8') + except (OSError, ValueError): + prior = '' + report_dir.mkdir(parents=True, exist_ok=True) + (report_dir / REPORT_MD).write_text( + head + (f'\n{prior}' if prior else ''), encoding='utf-8') + except Exception as e2: # noqa: BLE001 + _p(f'warning: fallback {REPORT_MD} write failed too: {e2}', flush=True) + + +def variants_of(cfg: dict, board: str) -> list: + for b in cfg.get('boards', []): + if b['name'] == board: + return [v['name'] for v in (b.get('variant') or [])] or [board] + return [board] + + +def summarize(cfg: dict, boards: list, report: dict) -> dict: + # .get, not a subscript: this is the one reader an agent's verdict depends on, and a + # row without 'board' used to kill the CLI with a traceback and no results at all -- + # hil-validate.js then reports every board as "hil-operator returned no entry". + rows = {r['board']: r.get('cells') or {} + for r in (report.get('rows') or []) + if isinstance(r, dict) and 'board' in r} + owner = {v['name']: b['name'] for b in cfg.get('boards', []) + for v in (b.get('variant') or [])} + results = [] + for board in boards: + names = variants_of(cfg, board) + mine = {n: rows[n] for n in names if n in rows} + # a variant name that is neither declared nor prefixed cannot be attributed; the + # `<board>-` fallback only helps ad-hoc builds, it is not the primary path. It must + # also never steal a row DECLARED by another board: a declared variant need not start + # with its own board's name, so it may happen to start with this board's name plus '-'. + mine.update({n: c for n, c in rows.items() + if n.startswith(f'{board}-') and n not in mine + and owner.get(n, board) == board}) + # the BOARD-name row too: hil_test writes lock contention and pool timeouts keyed + # by board name, but variants_of returns only DECLARED variant names -- and + # nanoch32v203 / ch32v307v_r1_1v0 declare none equal to their board name. Without + # this those rows are invisible, so a lock held by concurrent CI is published as a + # hardware FAIL and hil-validate.js never retries it. + if board in rows and board not in mine: + mine[board] = rows[board] + if not mine: + results.append({'board': board, 'ran': False, 'pass': False, 'locked': False, + 'detail': 'no report row for this board'}) + continue + # a wedge outranks lock contention: `locked` short-circuits `detail` below, so a + # stale board-locked cell from an earlier attempt used to mask the pool-timeout + # cell the retry added -- publishing a board that hung the rig as LOCKED, which + # hil-validate.js then RE-RUNS, paying another pool guard on it. RUN_ABORTED_CELL + # is written by the same _abort_report path for a board the guard never reached, + # and must outrank it for the same reason. + wedged = any(POOL_TIMEOUT_CELL in cells or RUN_ABORTED_CELL in cells + for cells in mine.values()) + locked = not wedged and any(LOCKED_CELL in cells for cells in mine.values()) + bad = [] + for vname, cells in sorted(mine.items()): + for test, val in sorted(cells.items()): + if test == LOCKED_CELL: + continue + if cell_state(val) == 'fail': + bad.append(f'{vname} {test}: {val}') + ok = not bad and not locked + if locked: + detail = 'held by another holder; not flashed' + elif bad: + detail = '; '.join(bad) + else: + detail = f'{len(mine)} variant(s), {sum(len(c) for c in mine.values())} cell(s) ok' + results.append({'board': board, 'ran': True, 'pass': ok, 'locked': locked, + 'detail': detail}) + # `caveat` too: an abandoned or no-boards run says so THERE, and this JSON is all + # an agent gets -- leaving it in the sidecar puts it back where only a human looks. + return {'results': results, 'banner': report.get('banner', ''), + 'caveat': report.get('caveat', '')} + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument('config_file') + ap.add_argument('-b', '--board', action='append', default=[], + help='boards to report on; default: every board in the config') + ap.add_argument('--report-dir', default='.', help=f'where {REPORT_JSON} lives (default: cwd)') + a = ap.parse_args() + + cfg = json.loads(Path(a.config_file).read_text()) + boards = a.board or [b['name'] for b in cfg.get('boards', [])] + jpath = Path(a.report_dir) / REPORT_JSON + if not jpath.is_file(): + print(f'error: {jpath} not found -- did hil_test.py run in this directory?', + file=sys.stderr) + return 1 + # through _load, like every writer: feeding raw JSON to summarize left the one reader an + # agent's verdict depends on crashing on the malformed sidecars the writers tolerate. + doc, _ = _load(Path(a.report_dir)) + json.dump(summarize(cfg, boards, doc), sys.stdout, indent=2) + print() + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/test/hil/helper/hil_summary.py b/test/hil/helper/hil_summary.py deleted file mode 100644 index e566bead0..000000000 --- a/test/hil/helper/hil_summary.py +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MIT -"""Fold hil_report.json into one machine-readable verdict per BOARD. - -A workflow driving hil_test.py through an operator agent has no filesystem access, so the -agent has to carry the results across. It must carry them, not retype them: the previous -design asked the agent to transcribe the markdown table, and every defect found in four -review rounds came from re-parsing that prose -- variant row names vs board names, -`board locked` vs `board-locked`, folding several variant rows into one verdict, rows that -matched no board. All of it is a join, and the join belongs here, where the roster is. - -Report rows are named per VARIANT (hil_test.py builds them from `vname`), and a variant name -is not required to start with the board name -- nanoch32v203 produces only `-fsdev`/`-usbfs`, -ch32v307v_r1_1v0 only `-usbhs`/`-usbfs`. The config is what maps them back. - -Emits, on stdout: - {"results": [{"board", "ran", "pass", "locked", "detail"}...], "banner": str} - -`locked` is a field, not a prefix to grep for. `ran` false means the board produced no row at -all, which is not the same as failing. - -Usage: hil_summary.py <config.json> [-b BOARD]... [--report-dir DIR] -""" -import argparse -import json -import sys -from pathlib import Path - -FAIL_ICON, SKIP_ICON = '❌', '⚪' # a pass needs no icon: unmarked = pass -LOCKED_CELL = 'board-locked' - - -def cell_state(v: str) -> str: - """'pass' | 'fail' | 'skip' -- the EXACT classifier hil_test.py's own tally uses - (cell_kind in render_matrix): 'fail' or a ❌ prefix is a failure, 'skip' or a ⚪ - prefix is a skip, and EVERYTHING ELSE is a pass. That last arm is load-bearing: a - passing test may return a plain metric string ('480.0 MBps') that lands in the cell - unprefixed, while failures are guaranteed marked -- TestFail's docstring pins that its - metric is icon-prefixed precisely so render/tally treat it as a failure. Classifying - unknown shapes as fail here would publish a green table as a red verdict.""" - if v == 'fail' or v.startswith(FAIL_ICON): - return 'fail' - if v == 'skip' or v.startswith(SKIP_ICON): - return 'skip' - return 'pass' - - -def variants_of(cfg: dict, board: str) -> list: - for b in cfg.get('boards', []): - if b['name'] == board: - return [v['name'] for v in (b.get('variant') or [])] or [board] - return [board] - - -def summarize(cfg: dict, boards: list, report: dict) -> dict: - rows = {r['board']: r.get('cells') or {} for r in report.get('rows', [])} - owner = {v['name']: b['name'] for b in cfg.get('boards', []) - for v in (b.get('variant') or [])} - results = [] - for board in boards: - names = variants_of(cfg, board) - mine = {n: rows[n] for n in names if n in rows} - # a variant name that is neither declared nor prefixed cannot be attributed; the - # `<board>-` fallback only helps ad-hoc builds, it is not the primary path. It must - # also never steal a row DECLARED by another board: a declared variant need not start - # with its own board's name, so it may happen to start with this board's name plus '-'. - mine.update({n: c for n, c in rows.items() - if n.startswith(f'{board}-') and n not in mine - and owner.get(n, board) == board}) - if not mine: - results.append({'board': board, 'ran': False, 'pass': False, 'locked': False, - 'detail': 'no report row for this board'}) - continue - locked = any(LOCKED_CELL in cells for cells in mine.values()) - bad = [] - for vname, cells in sorted(mine.items()): - for test, val in sorted(cells.items()): - if test == LOCKED_CELL: - continue - if cell_state(str(val)) == 'fail': - bad.append(f'{vname} {test}: {val}') - ok = not bad and not locked - if locked: - detail = 'held by another holder; not flashed' - elif bad: - detail = '; '.join(bad) - else: - detail = f'{len(mine)} variant(s), {sum(len(c) for c in mine.values())} cell(s) ok' - results.append({'board': board, 'ran': True, 'pass': ok, 'locked': locked, - 'detail': detail}) - return {'results': results, 'banner': report.get('banner', '')} - - -def main() -> int: - ap = argparse.ArgumentParser() - ap.add_argument('config_file') - ap.add_argument('-b', '--board', action='append', default=[], - help='boards to report on; default: every board in the config') - ap.add_argument('--report-dir', default='.', help='where hil_report.json lives (default: cwd)') - a = ap.parse_args() - - cfg = json.loads(Path(a.config_file).read_text()) - boards = a.board or [b['name'] for b in cfg.get('boards', [])] - jpath = Path(a.report_dir) / 'hil_report.json' - if not jpath.is_file(): - print(f'error: {jpath} not found -- did hil_test.py run in this directory?', - file=sys.stderr) - return 1 - json.dump(summarize(cfg, boards, json.loads(jpath.read_text())), sys.stdout, indent=2) - print() - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/test/hil/helper/hil_util.py b/test/hil/helper/hil_util.py index 0a2a13fca..6f84c143d 100644 --- a/test/hil/helper/hil_util.py +++ b/test/hil/helper/hil_util.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: MIT # Bottom layer of the HIL harness: the bounded command runner plus the shared helpers and -# data every other module needs. Stays stdlib-only and imports nothing local -- everything +# data every other module needs. Stays stdlib-only; its one local dependency is +# tools/rtt.py (the RTT console, loaded by path below) -- everything # else imports this, including the unit tests on GitHub's bare runner; never import them # from here. Callers set the module global `verbose`. @@ -11,6 +12,7 @@ import glob import os import signal import subprocess +import unicodedata import threading import sys from pathlib import Path @@ -88,10 +90,33 @@ def pos_float_env(name: str, default: float) -> float: CMD_TIMEOUT = pos_int_env('HIL_CMD_TIMEOUT', 180) +# Post-SIGKILL reap, spent ON TOP of a run_cmd timeout whenever the child has to be killed. +# A caller budgeting several bounded steps must add one of these PER STEP, or its own outer +# bound fires mid-step -- for a flasher, orphaning it on the probe. +REAP_GRACE = 10 TINYUSB_ROOT = Path(__file__).resolve().parents[3] # test/hil/helper/ -> repo root +def display_width(s: str) -> int: + """Terminal COLUMNS, not characters. + + The status marks the reports use -- ✅ ❌ ⚪ ⚠ 🔒 -- are one Python character and TWO + columns wide. Measuring with len() pads every cell containing one a column short, so + the pipes drift out of line against the header rule for the whole table. + """ + return sum(2 if unicodedata.east_asian_width(c) in 'WF' else 1 for c in s) + + +def pad(s: str, width: int, center: bool = False) -> str: + """str.ljust/center, measured in display columns. See display_width.""" + room = max(0, width - display_width(s)) + if not center: + return s + ' ' * room + left = room // 2 + return ' ' * left + s + ' ' * (room - left) + + def cmd_stdout_text(out: Any) -> str: if out is None: return '' @@ -137,74 +162,119 @@ def _print_banner(title: str, out: Any, err: Any) -> None: print(_banner_body(out, err)) -SYSFS_READ_GRACE = 2.0 # bound on one attribute read of a possibly-wedged device -SYSFS_STUCK_MAX = 4 # stranded readers tolerated before read_sysfs goes blind -_sysfs_stuck = 0 # each costs a thread + an fd for the life of the process -_sysfs_stuck_lock = threading.Lock() -_sysfs_blind_logged = False +SYSFS_READ_GRACE = 2.0 # default bound on one attribute read; see read_sysfs +# path -> the kernfs inode the node had when its bounded read gave up. Keyed by INODE, not +# by path alone: a busport does not change when a board returns to the same physical port, +# so a path-only blacklist outlives the wedge -- hil_pool_check resets or reflashes the +# board, wait_device polls that busport for the new inode, and the scan it polls through +# would never look at the device again. A re-enumeration destroys the kernfs node and makes +# a new one, so a CHANGED inode is the all-clear. os.stat is safe on a wedged device: it +# does not call ->show(), so it cannot block on the lock the reader is stuck behind. +_stranded: dict = {} +_strand_hits: dict = {} # path -> how many times it has stranded, ever +_refused: set = set() # paths answered None WITHOUT reading, once past _STRAND_MAX +_strand_lock = threading.Lock() +_ever_stranded = False -class _SysfsUnknown: - """Sentinel: the read did not answer. NOT "the attribute is absent" -- reading it as - absence turns a healthy board into a firmware regression in the report.""" - __slots__ = () +# Each strand costs a thread AND an fd for the life of the process -- on sysfs the open() +# SUCCEEDS and only the read blocks. Two ceilings, because they bound different things: +# +# _PATH_STRAND_MAX -- a device that FLAPS while still wedged re-enumerates, clears the +# inode memo, and strands again. Per path, so one sick board cannot leak without bound. +# After this many it stays memoised whatever its inode says. +# _STRAND_MAX -- a whole-process backstop against RLIMIT_NOFILE or the thread ceiling, +# which would raise inside a worker and lose every board's result. Counted PER PATH, not +# per reader: hil_pool_check runs four poll threads over one bus, and counting each +# reader let four threads on ONE wedged device spend four credits between them. With +# per-path counting a 27-board rig cannot approach this. +_PATH_STRAND_MAX = 4 +_STRAND_MAX = 64 - def __bool__(self) -> bool: - return False - def __repr__(self) -> str: - return 'SYSFS_UNKNOWN' +def sysfs_stranded() -> bool: + """True once any bounded read has given up, and it STAYS true. + + A sticky, process-wide fact, so it answers exactly one question: "could anything in + this process's output be the tool losing sight of healthy hardware?" -- which is what + hil_pool_check's footer needs. It canNOT answer "is THIS device unreadable" for a + caller deciding what a single missing device means; use path_stranded() for that. + """ + return _ever_stranded -SYSFS_UNKNOWN = _SysfsUnknown() +def strand_note() -> str: + """Suffix for an absence claim, so "not found" never reads as proven absence. + Lives here because every caller that can say "not found" needs the same sentence, and + the one that had to re-invent it got missed: a wedged-but-enumerated printer was + reported as an enumeration failure, sending a maintainer after firmware. + """ + return (' (a bounded sysfs read gave up, so "not found" here means "could not tell"' + ' -- see the usb-kernel-recover skill)') if sysfs_stranded() else '' -def sysfs_blind() -> bool: - """True once this process has stranded SYSFS_STUCK_MAX readers: every later read - answers SYSFS_UNKNOWN, so nothing it reports about a device is a fact any more.""" - return _sysfs_stuck >= SYSFS_STUCK_MAX +def path_stranded(path: str) -> bool: + """Whether THIS attribute is currently memoised as unreadable. -def sysfs_blind_note() -> str: - """Suffix for a failure message, so a blind worker's verdict never reads as hardware.""" - return (f' (this worker is blind: {SYSFS_STUCK_MAX} sysfs reads stranded on a wedged ' - f'device, so the check could not see the bus)') if sysfs_blind() else '' + The per-device question sysfs_stranded() cannot answer. usbtest uses it to tell a DUT + whose `serial` is held under device_lock from one that genuinely left the bus, because + the difference decides whether it performs driver-registry writes that take the + UNINTERRUPTIBLE device_lock. + """ + with _strand_lock: + return path in _stranded or path in _refused -def read_sysfs(path: str, grace: float = SYSFS_READ_GRACE) -> str | None | _SysfsUnknown: - """Read a sysfs attribute with a WALL-CLOCK bound. +def read_sysfs(path: str, timeout: float = SYSFS_READ_GRACE) -> str | None: + """A sysfs attribute's value, or None when it did not answer. - The value, None when the attribute is genuinely unreadable (OSError), or SYSFS_UNKNOWN - when the read did not answer -- it timed out, or this process is already blind. Callers - MUST keep those apart: absence is a fact, unknown is not. + BOUNDED BY DEFAULT, and it has to be. `serial` is served by usb_string_attr, which + takes usb_lock_device_interruptible (v6.12.96 sysfs.c:141-143) -- the same lock a + wedged usbfs ioctl holds. Every OTHER attribute the harness reads (idVendor, idProduct, + bcdDevice, busnum, devnum, speed) is a lock-free sysfs_emit from a cached field and + cannot block. - usb_string_attr (serial/product/manufacturer) is served under the device lock a wedged - usbfs ioctl holds, so a plain open().read() blocks for as long as the wedge lasts, on - exactly the board an incident is about. The reader sleeps INTERRUPTIBLY (every read - takes usb_lock_device_interruptible, v6.12.96 sysfs.c:124-139 -- uninterruptible is the - ioctl holder, not us), so it dies with a SIGKILLed worker; what it costs meanwhile is a - thread and an fd for this process's life, because on sysfs the open() SUCCEEDS and only - the read blocks. Measured: 20 blocking reads leave 20 live threads. + "Only the wedged board's own worker pays" is FALSE, which is why the bound is not + opt-in: usb_scan reads `serial` on every device matching the VID to find the one it + wants, so resolving MY board touches every peer's locked attribute. hil_lock's + controller_of does that from controller_permit, on essentially every board -- one + wedged DUT would stall every worker, not one. hil_pool_check has no guard at all. - Hence the cap: callers rescan (hil_lock's controller_of re-reads every unresolved - device on EVERY permit), and hitting RLIMIT_NOFILE or the thread ceiling raises inside - the worker and loses every board's result -- worse than the hang this prevents. + A give-up reads as None, the same as unreadable: there is no third value and no + per-attribute blindness. The memo is keyed by inode so the cost stays on the device + that is actually wedged; path_stranded() tells a caller which device that was. """ - if sysfs_blind(): - return SYSFS_UNKNOWN - # Known-stranded? Re-reading costs another permanent thread+fd and a blindness credit - # to learn what we already know. Lives HERE, not at the call sites: a call-site memo - # has to be remembered by every new scanner, and twice it was not. - was = _sysfs_stranded.get(path, _STRAND_MISS) - if was is not _STRAND_MISS: - if was is None: - return SYSFS_UNKNOWN # stranded, inode unknown: never re-read it + with _strand_lock: + was = _stranded.get(path) + stuck_for_good = _strand_hits.get(path, 0) >= _PATH_STRAND_MAX + budget_spent = len(_stranded) >= _STRAND_MAX + if was is not None: try: if os.stat(path).st_ino == was: - return SYSFS_UNKNOWN # same node, still wedged + return None # same kernfs node, still wedged except OSError: - pass # gone: fall through, the read reports it - _sysfs_stranded.pop(path, None) # replaced or gone -> re-read it + pass # gone: let the read below report it + if stuck_for_good: + return None # flapped too many times; see _PATH_STRAND_MAX + with _strand_lock: + _stranded.pop(path, None) # a different inode is the all-clear + elif budget_spent: + # see _STRAND_MAX. Recorded, not just returned: usbtest fails CLOSED on + # path_stranded() before the lock-taking cleanup, and a path we declined to read + # is exactly the case it must not be told is readable-and-absent. + with _strand_lock: + _refused.add(path) + return None + + # BEFORE the read, not after: a node that re-enumerates DURING the grace would + # otherwise have its brand-new HEALTHY inode recorded as the wedged one, and only a + # second re-enumeration could ever clear it. If it cannot be stat'd there is no key to + # memoise against, so the path is simply re-read next time -- the open fails fast. + try: + ino = os.stat(path).st_ino + except OSError: + ino = None out: dict = {} def _read(): @@ -212,96 +282,70 @@ def read_sysfs(path: str, grace: float = SYSFS_READ_GRACE) -> str | None | _Sysf with open(path) as f: out['v'] = f.read().strip() except (OSError, ValueError): - pass # no such attribute, or not text: unreadable, and that IS a fact + pass t = threading.Thread(target=_read, daemon=True) t.start() - t.join(grace) - # `out` FIRST, not is_alive() alone: a reader can deposit its value and still be alive - # for a moment afterwards, and counting that as a strand memoises a healthy attribute as - # unreadable and spends one of four blindness credits. bounded_open has always checked - # its box for the same reason. + t.join(timeout) + # `out` FIRST: a reader can deposit its value and still be alive for a moment + # afterwards, and counting that as a strand blacklists a healthy attribute forever + if 'v' in out: + # a path that answered is not refused any more: _refused feeds path_stranded(), + # and a stale entry makes usbtest read a LATER genuine disconnect as "cannot tell" + with _strand_lock: + _refused.discard(path) if t.is_alive() and 'v' not in out: - # Count the PATH once, not once per reader. hil_pool_check runs -j4 by default, - # which equals SYSFS_STUCK_MAX, so four threads hitting ONE wedged device used to - # spend the entire blindness budget between them -- latching blind on the single - # wedge the tool was run to find. The strand is real for each thread, but the - # DEVICE is what the cap is about. - # Under the SAME lock as the counter: check-then-act here is a race, and - # hil_pool_check runs a ThreadPoolExecutor of exactly SYSFS_STUCK_MAX workers in - # ONE process, so four threads on one wedged path could each see `first` before any - # of them recorded it -- spending the whole blindness budget on a single device, - # which is what this memo exists to prevent. note_sysfs_strand takes the lock - # itself, so call it after releasing. - with _sysfs_stuck_lock: - first = path not in _sysfs_stranded - if first: - try: - # stat, never the thread's own open(): stat does not call ->show(), so - # it cannot block on the device lock the reader is stuck behind - _sysfs_stranded[path] = os.stat(path).st_ino - except OSError: - _sysfs_stranded[path] = None # unstattable, but still known-stranded - if first: - note_sysfs_strand() - return SYSFS_UNKNOWN + global _ever_stranded + announce = False + if ino is None: + # the pre-read stat lost a race the open then won -- the node was replaced + # between them. Re-stat now: the reader is blocked on whatever node exists, + # so this is the key it is stuck on. Without a key nothing is memoised and + # every later poll starts another permanent thread and fd for this path. + try: + ino = os.stat(path).st_ino + except OSError: + pass + with _strand_lock: + _ever_stranded = True + if ino is not None: + first = path not in _stranded # count the PATH once, not each reader + _stranded[path] = ino + if first: + _strand_hits[path] = _strand_hits.get(path, 0) + 1 + announce = len(_stranded) == _STRAND_MAX + else: + _refused.add(path) # unkeyable: at least do not vouch for it + if announce: + print(f'warning: {_STRAND_MAX} devices have unreadable sysfs attributes; ' + f'refusing to start more bounded readers, so later reads answer None ' + f'without looking. Find the wedged device (usb-kernel-recover skill).', + file=sys.stderr, flush=True) + return None return out.get('v') -def note_sysfs_strand() -> None: - """Record ONE stranded sysfs reader. Shared by read_sysfs and bounded_open so both - account against a single counter -- the report caveat keys off it.""" - global _sysfs_stuck, _sysfs_blind_logged - with _sysfs_stuck_lock: - _sysfs_stuck += 1 - announce = sysfs_blind() and not _sysfs_blind_logged - _sysfs_blind_logged = _sysfs_blind_logged or announce - if announce: - # once per process, on stderr: a worker's stdout is compacted into one report - # row, where this would be lost among the test output - print(f'warning: {SYSFS_STUCK_MAX} sysfs reads stranded on a wedged device; ' - f'this process is now blind and answers SYSFS_UNKNOWN for every ' - f'attribute -- its verdicts about device presence are not evidence', - file=sys.stderr, flush=True) - - -# path -> the inode it had when its read stranded. A stranded attribute stays -# stranded until the DEVICE is replaced, and a re-enumeration destroys the kernfs -# node and makes a new one -- so a changed inode is the all-clear. Keyed by path -# alone it would outlive the wedge: a busport does not change when a board comes -# back on the same port, so the HUNG reflash this branch performs would recover a -# board the harness could then never see again. -_sysfs_stranded: dict = {} -# A stranded path whose inode could not be read is stored as None, so a plain .get() cannot -# tell 'known stranded, inode unknown' from 'never seen' -- and treating the first as the -# second re-reads it, stranding another permanent thread and fd every call. Distinct miss -# sentinel, so None keeps its own meaning. -_STRAND_MISS = object() - - -def usb_scan(vid_pid=None, serial=None, vid=None) -> tuple[list, bool]: - """Enumerated USB devices matching the filters, and whether anything is unknown. - - Returns ([{busport, dir, vid, pid, serial}], unknown). `unknown` True means a bounded - read did not answer, so absence is NOT proven -- the same contract as read_sysfs. +def usb_scan(vid_pid=None, serial=None, vid=None, timeout=SYSFS_READ_GRACE) -> list: + """Enumerated USB devices matching the filters: [{busport, dir, vid, pid, serial}]. Three rules, one implementation for every caller: * Root hubs excluded (glob `*-*`): no DUT is one, and scans including them measured seconds slower (observation, no mechanism -- the "autosuspend wake" explanation was - wrong; usb_string_attr reads a cached string, sysfs.c:124-139). + wrong; usb_string_attr reads a cached string, sysfs.c:141-143). * idVendor/idProduct first: lock-free `sysfs_emit` from udev->descriptor (sysfs.c:688-705), so they rule out nearly every device for free. - * `serial` last and bounded: it is served under the lock a wedged ioctl holds, and a - path that already stranded is never re-read (each strand costs a thread and an fd - for this process's life). + * `serial` LAST and BOUNDED: it is the only attribute here served under the device + lock, so it is the only one that can block. Filtering on the lock-free pair first + keeps most devices out of it, but a scan for ONE board still reads the serial of + every peer that shares its VID -- so the bound is what stops one wedged DUT from + stalling every caller (see read_sysfs). """ out = [] - unknown = False for d in glob.glob('/sys/bus/usb/devices/*-*'): - # Interfaces are '<busport>:<cfg>.<ifnum>' (e.g. 2-4:1.0) -- they CONTAIN the - # colon, they do not end with it, so the original endswith() never fired and every - # scan opened idVendor/idProduct on all of them (measured: 31 of 44 matches). + # `in`, not endswith: an interface is '<busport>:<cfg>.<ifnum>' (2-4:1.0), which + # CONTAINS the colon rather than ending with it. Screening them out here is worth + # real time -- they were 31 of 44 matches on this rig. if ':' in os.path.basename(d): continue try: @@ -315,106 +359,14 @@ def usb_scan(vid_pid=None, serial=None, vid=None) -> tuple[list, bool]: continue # ruled out for free, without touching the locked attribute if vid is not None and dev_vid != vid: continue # same, for callers that know the VID but not the PID - sn = read_sysfs(os.path.join(d, 'serial')) - if sn is SYSFS_UNKNOWN: - unknown = True # read_sysfs memoises it; a repeat scan costs nothing - continue + sn = read_sysfs(os.path.join(d, 'serial'), timeout) if sn is None: - continue # no serial attribute: a fact + continue # no serial attribute if serial is not None and sn.lower() != serial.lower(): continue out.append({'busport': os.path.basename(d), 'dir': d, 'vid': dev_vid, 'pid': dev_pid, 'serial': sn}) - return out, unknown - - -def bounded_open(path: str, flags: int, timeout: float = SYSFS_READ_GRACE): - """os.open() with a wall-clock bound. - - The fd, None when the open genuinely FAILED (OSError: EBUSY, ENOENT, EACCES), or - SYSFS_UNKNOWN when it did not answer -- the same three-valued contract as read_sysfs, - and for the same reason: folding a fact into an unknown made an ordinary EBUSY read as - a wedged device and sent the operator hunting hardware that is healthy. - - An open CAN block on a wedged device -- not on O_NONBLOCK, which usblp_open never - consults, but on usb_autopm_get_interface(), a runtime-PM resume that does I/O - (v6.12.96 drivers/usb/class/usblp.c). It holds usblp_mutex while it waits, and that - mutex is driver-GLOBAL, so one wedged printer blocks opens of every usblp node. - - Unlike read_sysfs the stranded thread cleans up after itself: if we have given up it - closes the fd it eventually got, so only the thread leaks. Both sides take `handoff` - -- "store or close" and "abandon and drain" are a check-then-act pair that can - interleave into an fd stored after the box was drained, which would leak it into a - node that allows a SINGLE opener (usblp_open returns -EBUSY when usblp->used). - """ - # Same short-circuit as read_sysfs: once blind, another stranded thread buys nothing - # and the cap exists precisely to stop them accumulating. - if sysfs_blind(): - return SYSFS_UNKNOWN - # Known-stranded? Re-opening costs another thread, another fd and another blindness - # credit to learn what we already know -- and the printer test re-opens ONE lp node on - # every retry. Same memo and same inode check as read_sysfs. - was = _sysfs_stranded.get(path, _STRAND_MISS) - if was is not _STRAND_MISS: - if was is None: - return SYSFS_UNKNOWN # stranded, inode unknown: never re-read it - try: - if os.stat(path).st_ino == was: - return SYSFS_UNKNOWN - except OSError: - pass - _sysfs_stranded.pop(path, None) - box: dict = {} - done, abandoned = threading.Event(), threading.Event() - handoff = threading.Lock() - - def _open(): - try: - fd = os.open(path, flags) - except OSError: - done.set() - return - with handoff: - stored = not abandoned.is_set() - if stored: - box['fd'] = fd - if not stored: - try: - os.close(fd) - except OSError: - pass - done.set() - - threading.Thread(target=_open, daemon=True).start() - if not done.wait(timeout): - with handoff: - abandoned.set() - fd = box.pop('fd', None) # completed in the gap between timeout and flag - if fd is not None: - # It DID open, just after our deadline -- the thread finished, so nothing is - # stranded. Report unknown (we already gave up on it) but do not spend a - # blindness credit, and do not call a merely-slow node wedged. - try: - os.close(fd) - except OSError: - pass - return SYSFS_UNKNOWN - # counted like a stranded read_sysfs: the thread and (eventually) its fd are gone - # for the life of the process, and the cap exists to stop that reaching the - # thread/fd ceiling -- an exception there escapes the worker and loses every board. - # Memoised by inode so a retry of the same node does not pay again. - # same lock as read_sysfs, same reason - with _sysfs_stuck_lock: - first = path not in _sysfs_stranded - if first: - try: - _sysfs_stranded[path] = os.stat(path).st_ino - except OSError: - _sysfs_stranded[path] = None - if first: - note_sysfs_strand() - return SYSFS_UNKNOWN - return box.get('fd') + return out def _close_pipes(p: subprocess.Popen) -> None: @@ -457,7 +409,7 @@ def run_alongside(argv: list, work, timeout: int) -> subprocess.CompletedProcess except OSError: p.kill() try: - out, err = p.communicate(timeout=5) + out, err = p.communicate(timeout=REAP_GRACE) except subprocess.TimeoutExpired: # Outlasted SIGKILL: uninterruptible, still holding whatever it opened. # Abandoned like any other stray -- but as a real child in its own @@ -479,9 +431,49 @@ def run_alongside(argv: list, work, timeout: int) -> subprocess.CompletedProcess return _reap() -def run_cmd(cmd: str, cwd: str | None = None, timeout: int | None = None, +# The RTT console implementation lives in tools/rtt.py (importable classes + CLI, +# stdlib-only, harness-critical — see its module docstring). Loaded by file path so +# no sys.path entry for tools/ can shadow other imports; re-exported here so the +# harness keeps addressing hil_util.JlinkRtt. +import importlib.util as _ilu + +_rtt_path = TINYUSB_ROOT / 'tools' / 'rtt.py' +if not _rtt_path.exists(): + # name the real cause: a bare FileNotFoundError out of an exec_module here reads + # as a harness bug, when the actual problem is an incompletely staged tree + raise ImportError(f'{_rtt_path} is missing — the RTT console lives there and the ' + f'harness depends on it; stage it alongside test/hil (hil_ci.sh does)') +_rtt_spec = _ilu.spec_from_file_location('tinyusb_tools_rtt', _rtt_path) +_rtt = _ilu.module_from_spec(_rtt_spec) +sys.modules[_rtt_spec.name] = _rtt # registered: RttError must be picklable across the fork Pool +_rtt_spec.loader.exec_module(_rtt) +JlinkRtt = _rtt.JlinkRtt +OpenocdRtt = _rtt.OpenocdRtt +RttError = _rtt.RttError +RTT_BANNER_RE = _rtt.RTT_BANNER_RE +strip_banner = _rtt.strip_banner + + +def _cmd_label(cmd) -> str: + """A one-line name for a banner. An argv whose payload is a `python3 -c` program would + otherwise dump the whole body into the CI log, where run_cmd's banners are already the + noisiest thing in a failing row.""" + if isinstance(cmd, str): + return cmd + parts = [a if len(a) <= 60 else f'<{len(a)}-char program>' for a in cmd] + return ' '.join(parts) + + +def run_cmd(cmd: str | list, cwd: str | None = None, timeout: int | None = None, binary: bool = False, split_stderr: bool = False, quiet: bool = False) -> subprocess.CompletedProcess: + """Bounded subprocess: own session, killpg on expiry, rc 124 when it had to be killed. + + `cmd` is a shell STRING or an argv LIST. argv exists for a program that cannot survive + a trip through the shell -- a multi-line `python3 -c` body -- which is how the harness + runs a library call that no in-process bound can contain. A daemon thread cannot bound + a C call that holds the GIL, so for those the child process IS the bound. + """ if timeout is None: timeout = CMD_TIMEOUT # binary: raw bytes (text mode's errors='replace' mangles non-UTF-8 file content). @@ -490,34 +482,31 @@ def run_cmd(cmd: str, cwd: str | None = None, timeout: int | None = None, # still print: a killed child is always noteworthy). popen_kwargs = { 'cwd': cwd, - 'shell': True, + # a list goes straight to execve; only a string needs a shell to parse it + 'shell': isinstance(cmd, str), 'stdout': subprocess.PIPE, 'stderr': subprocess.PIPE if split_stderr else subprocess.STDOUT, } if not binary: popen_kwargs.update({'text': True, 'encoding': 'utf-8', 'errors': 'replace'}) - if os.name != 'nt': - # C-level setsid, same process-group semantics as preexec_fn=os.setsid but - # safe when called from threads (pool_check runs flashes from a thread pool) - popen_kwargs['start_new_session'] = True + # C-level setsid, same process-group semantics as preexec_fn=os.setsid but safe when + # called from threads (pool_check runs flashes from a thread pool) + popen_kwargs['start_new_session'] = True p = subprocess.Popen(cmd, **popen_kwargs) try: out, err = p.communicate(timeout=timeout) r = subprocess.CompletedProcess(args=cmd, returncode=p.returncode, stdout=out, stderr=err) except subprocess.TimeoutExpired as ex: - if os.name != 'nt': - try: - os.killpg(p.pid, signal.SIGKILL) - except OSError: - # ProcessLookupError: already gone. PermissionError: an all-root group - # refuses the group kill -- letting either escape would skip the bounded - # reap, the pipe close and the rc-124 return this handler exists for. - pass - else: - p.kill() try: - out, err = p.communicate(timeout=10) + os.killpg(p.pid, signal.SIGKILL) + except OSError: + # ProcessLookupError: already gone. PermissionError: an all-root group refuses + # the group kill -- letting either escape would skip the bounded reap, the pipe + # close and the rc-124 return this handler exists for. + pass + try: + out, err = p.communicate(timeout=REAP_GRACE) except subprocess.TimeoutExpired: # Something in the group outlived SIGKILL: D state (truly unkillable), or # root-owned because sudo FORKS rather than execs, so the wrapper dies and its @@ -543,7 +532,7 @@ def run_cmd(cmd: str, cwd: str | None = None, timeout: int | None = None, timeout_err = _typed(err if err is not None else ex.stderr) if split_stderr and timeout_err is None: timeout_err = b'' if binary else '' - _print_banner(f'COMMAND TIMEOUT ({timeout}s): {cmd}', timeout_out, timeout_err) + _print_banner(f'COMMAND TIMEOUT ({timeout}s): {_cmd_label(cmd)}', timeout_out, timeout_err) return subprocess.CompletedProcess(args=cmd, returncode=124, stdout=timeout_out, stderr=timeout_err) except BaseException: # BaseException, not Exception (as in CPython's own subprocess.run): @@ -551,18 +540,15 @@ def run_cmd(cmd: str, cwd: str | None = None, timeout: int | None = None, # its OWN group, so it never got the terminal's SIGINT -- without this, Ctrl-C # leaves the flasher or testusb holding the probe and its usbfs node. Kill and # close, never wait: this path must not add a hang of its own. - if os.name != 'nt': - try: - os.killpg(p.pid, signal.SIGKILL) - except OSError: - pass - else: - p.kill() + try: + os.killpg(p.pid, signal.SIGKILL) + except OSError: + pass _close_pipes(p) raise if r.returncode != 0 and not quiet: - _print_banner(f'COMMAND FAILED: {cmd}', r.stdout, r.stderr) + _print_banner(f'COMMAND FAILED: {_cmd_label(cmd)}', r.stdout, r.stderr) elif verbose: print(cmd) print(cmd_stdout_text(r.stdout)) diff --git a/test/hil/hil_ci.sh b/test/hil/hil_ci.sh index 514b0f174..43ede5795 100644 --- a/test/hil/hil_ci.sh +++ b/test/hil/hil_ci.sh @@ -210,6 +210,69 @@ rm -rf -- "$1" mkdir -p -- "$1/test/hil/helper" "$1/examples" REMOTE +# The --accumulate merge base. The wipe above just cleared REMOTE_DIR, and +# accumulate_report merges onto the sidecar in the RUN's cwd (hil_test.py:2193 sets +# `fresh = not args.accumulate`, and only a non-fresh run reads it) -- so without this a +# remote retry starts from nothing and its one-row table REPLACES the full-fleet one it was +# meant to extend. The copy-back at the end of this script has always existed; this is the +# other half of it. +# +# Gated, not unconditional: a fresh run unlinks the sidecar anyway (hil_test.py:2244), so +# uploading there is wasted work that also obscures what the wipe means. +# +# <config>.failed is deliberately NOT uploaded: hil_test.py only ever writes it, never +# reads it -- the retry spec reaches the rig as the -b/-bt arguments the caller expanded +# from it (`hil_ci.sh $(cat <config>.failed)`). +# argparse decides, not a case arm: hil_test.py declares `-a, --accumulate`, so argparse +# also accepts `-av`, `-va`, `--accum` and `--acc` -- and hil-validate.js tells the +# operator to retry "adding -v", which makes `-av` the natural spelling. A hand-rolled +# match missed all four: no upload, and the else-branch warning never fired either, so the +# one-row table replaced the full-fleet one in silence. +ACCUMULATE=$(python3 - ${ARGS[@]+"${ARGS[@]}"} <<'PY' +import argparse, sys +p = argparse.ArgumentParser(add_help=False) +p.add_argument('-a', '--accumulate', action='store_true') +p.add_argument('-v', '--verbose', action='store_true') # so -av/-va bundle as they do there +print(1 if p.parse_known_args(sys.argv[1:])[0].accumulate else 0) +PY +) || ACCUMULATE=0 +if [ "$ACCUMULATE" = 1 ]; then + if [ -f "$ROOT_DIR/hil_report.json" ]; then + # Provenance: hil_report.json is not namespaced by CONFIG or REMOTE (build.yml and + # pr_comment.yml read that exact name), so a `REMOTE=hifiphile CONFIG=.../hfp.json` + # run leaves an hfp sidecar behind that a later ci.lan retry would merge, publishing + # boards that never ran here. Require at least one row to belong to THIS roster. + if python3 - "$ROOT_DIR/hil_report.json" "$CONFIG" <<'PY' +import json, sys +try: + rows = json.load(open(sys.argv[1])).get('rows') or [] + cfg = json.load(open(sys.argv[2])).get('boards') or [] +except Exception: + sys.exit(1) +known = set() +for b in cfg: + known.add(b.get('name')) + known.update(v.get('name') for v in (b.get('variant') or [])) +sys.exit(0 if not rows or any(r.get('board') in known for r in rows if isinstance(r, dict)) + else 1) +PY + then + echo "==> Uploading hil_report.json as the --accumulate merge base" + scp -q "$ROOT_DIR/hil_report.json" "$REMOTE:$REMOTE_DIR/" + else + echo "==> warning: $ROOT_DIR/hil_report.json holds no board from $(basename "$CONFIG")" \ + "-- it is from another rig or config, so it is NOT being uploaded; this run's" \ + "table will REPLACE rather than extend" >&2 + fi + else + # Loud, because this is the failure mode: the run still succeeds, and quietly + # publishes a small table where a full one used to be. + echo "==> warning: --accumulate was requested but $ROOT_DIR/hil_report.json does not" \ + "exist, so there is nothing to merge onto -- this run's table will REPLACE the" \ + "previous one rather than extend it" >&2 + fi +fi + # Copy HIL test script and config echo "==> Copying test scripts" scp -q "$ROOT_DIR/test/hil/hil_test.py" \ @@ -223,8 +286,11 @@ scp -q "$ROOT_DIR/test/hil/helper/__init__.py" \ "$ROOT_DIR/test/hil/helper/hil_util.py" \ "$ROOT_DIR/test/hil/helper/hil_health.py" \ "$ROOT_DIR/test/hil/helper/hil_lock.py" \ - "$ROOT_DIR/test/hil/helper/hil_summary.py" \ + "$ROOT_DIR/test/hil/helper/hil_report.py" \ "$REMOTE:$REMOTE_DIR/test/hil/helper/" +# the rtt console/capture tool (rtt skill), harness-critical: hil_util imports it +ssh "$REMOTE" mkdir -p "$REMOTE_DIR/tools" +scp -q "$ROOT_DIR/tools/rtt.py" "$REMOTE:$REMOTE_DIR/tools/" # Copy only firmware binaries (elf/bin/hex) plus esptool metadata # (config.env + flash_args needed by the esptool flasher), preserving structure @@ -316,19 +382,41 @@ REMOTE # Copy the generated report back to the local checkout (best-effort; the run's # exit code is preserved regardless of whether a report was produced). -scp -q "$REMOTE:$REMOTE_DIR/hil_report.md" "$ROOT_DIR/hil_report.md" \ - && echo "==> Report copied to $ROOT_DIR/hil_report.md" \ - || echo "==> warning: no hil_report.md copied back" >&2 +# rm -f FIRST, exactly as the sidecar loop below does: the markdown and the JSON are two +# halves of ONE document now, so leaving a stale table behind when the copy fails -- beside +# a sidecar that was correctly removed -- publishes last run's green results under this +# run's red job, and the operator's hil_report.py call exits 1 against the missing sidecar. +# Fetch BOTH halves to temps and commit them as a pair. Separate fetch/rename meant a +# markdown that arrived beside a sidecar that did not left the local pair failing the +# rendering invariant, and the next --accumulate retry merging the wrong base. Deleting +# first and then scp'ing was worse still: an ssh drop at the end of a 60-minute run +# destroyed the report outright. +md_ok=0; json_ok=0 +scp -q "$REMOTE:$REMOTE_DIR/hil_report.md" "$ROOT_DIR/hil_report.md.tmp" 2>/dev/null \ + && [ -f "$ROOT_DIR/hil_report.md.tmp" ] && md_ok=1 +scp -q "$REMOTE:$REMOTE_DIR/hil_report.json" "$ROOT_DIR/hil_report.json.tmp" 2>/dev/null \ + && [ -f "$ROOT_DIR/hil_report.json.tmp" ] && json_ok=1 +if [ "$md_ok" = 1 ] && [ "$json_ok" = 1 ]; then + mv -f "$ROOT_DIR/hil_report.md.tmp" "$ROOT_DIR/hil_report.md" + mv -f "$ROOT_DIR/hil_report.json.tmp" "$ROOT_DIR/hil_report.json" + echo "==> Report copied to $ROOT_DIR/hil_report.md (+ sidecar)" +else + rm -f "$ROOT_DIR/hil_report.md.tmp" "$ROOT_DIR/hil_report.json.tmp" + # All or nothing: a half-copied pair is worse than none. The stale local markdown goes + # because that is what gets pasted into a PR as this run's results; the stale sidecar + # goes with it so the two cannot disagree. + rm -f "$ROOT_DIR/hil_report.md" "$ROOT_DIR/hil_report.json" + echo "==> warning: report copy-back incomplete (md=$md_ok json=$json_ok); removed the" \ + "stale local pair -- an --accumulate retry has no merge base until a run succeeds" >&2 +fi -# The re-run spec and the JSON sidecar live in the run's cwd on the rig (REMOTE_DIR), and the -# next invocation rm -rf's it. Without copying them back, the `--accumulate` retry every doc on -# this branch prescribes has nothing to read and nothing to merge onto. Delete the local copies -# FIRST: a green run writes no .failed, so a silent no-op scp would leave last run's spec in -# the checkout looking current, and "retry from the spec" would re-flash boards that passed. -for extra in "$(basename "$CONFIG").failed" hil_report.json; do - rm -f "$ROOT_DIR/$extra" - scp -q "$REMOTE:$REMOTE_DIR/$extra" "$ROOT_DIR/$extra" 2>/dev/null \ - && echo "==> $extra copied to $ROOT_DIR/$extra" || true -done +# The re-run spec lives in the run's cwd on the rig and the next invocation rm -rf's it. +# Delete the local copy first: a green run writes no .failed, so a silent no-op scp would +# leave last run's spec looking current and "retry from the spec" would re-flash boards +# that passed. +spec="$(basename "$CONFIG").failed" +rm -f "$ROOT_DIR/$spec" +scp -q "$REMOTE:$REMOTE_DIR/$spec" "$ROOT_DIR/$spec" 2>/dev/null \ + && echo "==> $spec copied to $ROOT_DIR/$spec" || true exit $rc diff --git a/test/hil/hil_flash.py b/test/hil/hil_flash.py index c4d4e6552..15f476ccd 100755 --- a/test/hil/hil_flash.py +++ b/test/hil/hil_flash.py @@ -237,7 +237,7 @@ def convoy_safe(flasher: dict) -> bool: return True # EXACT, not startswith: rescue_openocd and usbtest's # getattr(hil_flash, f'flash_{name}') both require the exact name, so an - # 'openocd_wch'-style entry would pass this gate, reserve USBTEST_RECOVERY_BUDGET, + # 'openocd_wch'-style entry would pass this gate, reserve the Rescue-DP legs, # and then find no recovery path at all -- paying for a path that cannot fire, which # is the precise cost this gate exists to avoid. if name != 'openocd': @@ -270,7 +270,7 @@ def flash_esptool(board: Board, firmware: str, timeout=None) -> subprocess.Compl def reset_esptool(board): # NO-OP, and marked as one: esptool's reset would be `--after hard_reset`, which is not # wired here. Returning rc 0 without resetting is why callers must never read the exit - # code as proof -- recovery_steps skips a primitive carrying `no_op`. + # code as proof -- usbtest's recovery skips a primitive carrying `no_op`. return subprocess.CompletedProcess(args=['dummy'], returncode=0) diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py index fcd7c7e6f..b2b74b13c 100755 --- a/test/hil/hil_test.py +++ b/test/hil/hil_test.py @@ -44,7 +44,6 @@ import itertools import os import random import re -import select import signal import shlex import sys @@ -64,14 +63,15 @@ from multiprocessing import TimeoutError as MpTimeoutError sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # PYTHONSAFEPATH drops it import hil_flash -from helper import hil_health, hil_lock, hil_util +import usbtest # for the recovery bounds only; hil_test runs it as a subprocess +from helper import hil_health, hil_lock, hil_report, hil_util from helper.hil_util import device_tests, dual_tests, host_test # Raw Lock/Semaphore objects in Pool initargs are inheritable only under fork # (spawn/forkserver pickle them and fail at Pool creation), so pin it against an -# interpreter default change. Windows has no fork: fall back so it still IMPORTS there. +# interpreter default change. -_mp = multiprocessing.get_context('fork') if os.name != 'nt' else multiprocessing.get_context() +_mp = multiprocessing.get_context('fork') Pool, Lock, Semaphore, Manager = _mp.Pool, _mp.Lock, _mp.Semaphore, _mp.Manager import string @@ -106,9 +106,6 @@ STATUS_OK = "\033[32mOK\033[0m" STATUS_FAILED = "\033[31mFailed\033[0m" STATUS_SKIPPED = "\033[33mSkipped\033[0m" -# Plain (non-ANSI) cell symbols for hil_report.md; a missing binary counts as skipped. -REPORT_CELL = {'pass': '✅', 'fail': '❌', 'skip': '⚪'} - class TestFail(AssertionError): """Fail a test but still surface a metric string in its report cell (e.g. usbtest's '❌ 29/30' @@ -209,6 +206,7 @@ class Board(TypedDict): # needs one carries a single variant named after itself (metro_m4_express / # MAX3421_HOST=1), which is exactly what the `or [...]` default below synthesises variant: NotRequired[list[VariantCfg]] + logger: NotRequired[str] # "rtt": console = the debug probe's RTT channel 0, not a VCOM (rtt skill) toolchain: NotRequired[str] # CI build bucket override, e.g. "riscv-gcc" (consumed by hil_ci_set_matrix.py) @@ -223,13 +221,17 @@ class HilConfig(TypedDict): POOL_TIMEOUT = hil_util.pos_int_env('HIL_POOL_TIMEOUT', 3600) -# Headroom on top of a battery's own budget so ONE HUNG recovery (case timeout, SIGKILL -# wait, bounded reflash, settle) can finish. Only spent when cases actually time out. -USBTEST_RECOVERY_BUDGET = hil_util.pos_int_env('HIL_USBTEST_RECOVERY_BUDGET', 250) +# The post-hang recovery reserve is PER BOARD and lives in usbtest.recovery_reserve(), +# derived from the ladder that file itself declares. Reserved whole, which is what lets the +# child run the ladder straight through instead of asking "does the next step still fit?" +# before each step. It only ELAPSES when cases actually time out; a healthy battery returns +# in ~200s and never touches it. + # How long usbtest.py may keep starting new cases (--budget). The outer run_cmd timeout is -# always this PLUS the recovery headroom, never a separate literal, or lowering one eats -# the reserve the recovery needs. 0 is refused (usbtest.py reads it as "no limit"); the -# margin over a healthy battery (~200s) keeps contention from becoming BUDGET entries. +# always this PLUS the overshoot PLUS the recovery reserve when one can run, never a +# separate literal, or lowering one eats the room the other needs. 0 is refused (usbtest.py +# reads it as "no limit"); the margin over a healthy battery (~200s) keeps contention from +# becoming BUDGET entries. USBTEST_BATTERY_BUDGET = hil_util.pos_int_env('HIL_USBTEST_BATTERY_BUDGET', 260) # The battery checks its budget BEFORE dispatching a case, so it can overshoot by one @@ -237,10 +239,13 @@ USBTEST_BATTERY_BUDGET = hil_util.pos_int_env('HIL_USBTEST_BATTERY_BUDGET', 260) # as it goes to print its JSON, turning ~29 real per-case verdicts into "usbtest did not # run" and re-paying the whole battery on retry. # Worst case, from usbtest.py: --timeout 60 (the case) + 5s post-SIGKILL reap + -# dmesg_tail(), which is bounded by HELPER_TIMEOUT=30 and runs on BOTH the FAIL and HUNG -# timeout paths = 95s. 120 leaves a margin; 75 (my first estimate, taken before checking -# dmesg_tail) was 20s SHORT and would have killed the battery mid-print. +# dmesg_tail(), bounded by HELPER_TIMEOUT=30 and run on BOTH the FAIL and HUNG timeout +# paths = 95s. 120 leaves a margin. Re-derive it if any of those three moves -- dmesg_tail +# is the one easily missed, and without it the estimate lands 20s short. USBTEST_OVERSHOOT = 120 +# Named, not a literal, so the unit tests can zero it: every test that drives +# test_device_usbtest against a fake rig otherwise pays a real 3s (ten of them, 30s a run). +USBTEST_SETTLE = 3 SERIAL_READ_TIMEOUT = hil_util.pos_float_env('HIL_SERIAL_READ_TIMEOUT', 5) SERIAL_WRITE_TIMEOUT = hil_util.pos_float_env('HIL_SERIAL_WRITE_TIMEOUT', 10) @@ -292,6 +297,25 @@ def open_serial_dev(port: str): return ser +def open_board_console(board: Board): + """The board's log console: its probe's VCOM, or RTT when the probe has none. + + Both ends expose the same read/in_waiting/write/close surface, so the tests read one + the same way they read the other.""" + if board.get('logger') == 'rtt': + # JlinkRtt speaks JLinkExe only; an openocd/stlink flasher would yield + # `-device ''` and fail 15 s later with a misleading port error. The OpenOCD + # RTT route is validated manually on native probes but has no harness backend + # yet (rtt skill; followup doc) — and never point it at ea4088's LPC-Link2 + # (measured: knocks that probe off USB; other J-Link-OB probes untested) + assert board['flasher']['name'].lower() == 'jlink', \ + f'{board["name"]}: "logger": "rtt" needs a jlink flasher, not {board["flasher"]["name"]}' + return hil_util.JlinkRtt(board) + ser = open_serial_dev(hil_util.get_serial_dev(board['flasher']["uid"], None, None, 0)) + ser.timeout = 0.1 + return ser + + def serial_write_all(ser: serial.Serial, data: bytes): # write_timeout is a deadline for the whole call. A timeout means the device stopped # draining, and it is fatal: pyserial loses the partial-write count on raise, so @@ -300,7 +324,18 @@ def serial_write_all(ser: serial.Serial, data: bytes): ser.write(data) except serial.SerialTimeoutException: raise AssertionError(f'Serial write timeout after {SERIAL_WRITE_TIMEOUT:.1f}s') + except hil_util.RttError as e: + # the RTT console's failure contract (stall/closed/peer death): same + # drain-stopped meaning as the serial timeout -- a test failure, not a harness + # crash. Deliberately NOT bare RuntimeError: NotImplementedError and CPython's + # own 'dictionary changed size during iteration' are RuntimeErrors too, and a + # harness bug must not be reported as this board misbehaving. + raise AssertionError(f'Console write failed: {e}') + +# J-Link Commander's telnet greeting: never target output (defined with the console +# in tools/rtt.py; hil_pool_check strips it through the same object) +RTT_BANNER_RE = hil_util.RTT_BANNER_RE LP_OPEN_TIMEOUT = 5 # bound on opening the printer lp node; see test_device_printer_to_cdc # Runs under hil_util.run_alongside as `python3 -c`. Inline rather than a file so hil_ci.sh's @@ -320,6 +355,87 @@ LP_READER = ( ' buf += chunk\n' 'sys.stdout.buffer.write(buf)\n' ) +# Runs under hil_util.run_cmd as `python3 -c`, argv so the body needs no shell quoting. +# A PROCESS, not a thread, and not optional: cython-hidapi wraps hid_enumerate in +# `with nogil` but calls hid_open and hid_close BARE (hidapi 0.15.0 hid.pyx), so those hold +# the GIL for their whole blocking call. A daemon thread cannot bound that -- the waiter +# parks off-GIL but must reacquire the GIL to return, which the stuck thread never yields +# -- so an in-process bound is inert exactly where it is needed, and the whole worker +# freezes rather than just the call. killpg reaches a child regardless. +# +# What blocks: hidapi's hidraw backend reads `manufacturer` and `product` via udev for each +# device that reaches create_device_info_for_device, via copy_udev_string(usb_dev, +# "manufacturer"/"product") -- both usb_string_attr, served under the device lock a wedged +# usbfs ioctl holds (v6.12.96 sysfs.c:141-143). +# +# Passing BOTH ids is what keeps a wedged peer out of that path, and it does more than skip +# non-matches: hidapi only runs the cheap pre-check `if (vendor_id != 0 || product_id != 0)` +# (0.15.0 linux/hid.c:962), so an unfiltered walk sends EVERY device straight to the locked +# reads. The pre-check itself is free -- parse_hid_vid_pid_from_sysfs parses +# <sysfs_path>/device/uevent (:532) -- and both `continue`s precede +# create_device_info_for_device (:966-970 before :976). Six examples in this tree expose a +# HID interface under VID cafe, so a VID-only walk would stall on any of them wedged on a +# peer. hid_open passes the same ids through to hid_enumerate internally (:1030), so the +# filter narrows that walk too -- but a peer running THIS example still matches both ids, +# which is why the child process, not the filter, is what bounds this. +HID_ECHO = r""" +import hid, random, sys, time + +uid, budget, want_pid = sys.argv[1], float(sys.argv[2]), int(sys.argv[3], 16) +deadline = time.monotonic() + budget + +dev = None +while dev is None: + for d in hid.enumerate(0xCafe, want_pid): + if d["serial_number"] == uid: + dev = d + break + if dev is not None or time.monotonic() >= deadline: + break + time.sleep(1) +if dev is None: + sys.exit(f"HID device not found for {uid}") + +h = hid.device() +h.open(dev["vendor_id"], dev["product_id"], uid) +try: + for size in (8, 32, 63): + # Report ID (0) + payload, padded to 64 bytes + payload = bytes(random.randint(1, 255) for _ in range(size)) + h.write(bytes([0]) + payload + bytes(64 - size)) + echo = h.read(64, 2000) + if not echo or len(echo) < size: + sys.exit(f"HID echo timeout or short read ({size} bytes)") + if bytes(echo[:size]) != payload: + sys.exit(f"HID echo wrong data ({size} bytes): " + f"sent {payload.hex()} received {bytes(echo[:size]).hex()}") +finally: + h.close() +""" +# The write half, same shape and same reason: usblp_open() ignores O_NONBLOCK and stalls in +# usb_autopm_get_interface() on a wedged device, holding the driver-global usblp_mutex. A +# blocked THREAD cannot be abandoned without keeping the fd, and usblp allows a single opener +# (v6.12.96 usblp.c), so the next open of this node returns -EBUSY for the life of the worker. +# A killed process takes its fd with it. O_NONBLOCK is kept because usblp DOES honour it on +# write, which is what the select()/partial-write loop below relies on. +LP_WRITER = ( + 'import os, random, select, sys\n' + 'lp, payload_path, ready = sys.argv[1], sys.argv[2], sys.argv[3]\n' + 'data = open(payload_path, "rb").read()\n' + 'fd = os.open(lp, os.O_WRONLY | os.O_NONBLOCK)\n' + # readiness marker, as in LP_READER: the parent must not read CDC before the node is open + 'open(ready, "w").close()\n' + 'off = 0\n' + 'while off < len(data):\n' + ' n = min(random.randint(1, 64), len(data) - off)\n' + ' buf, w = data[off:off + n], 0\n' + ' while w < len(buf):\n' + ' _, wr, _ = select.select([], [fd], [], 5.0)\n' + ' if not wr:\n' + ' sys.exit("printer write timeout (firmware not draining OUT endpoint)")\n' + ' w += os.write(fd, buf[w:])\n' + ' off += n\n' +) MTYPE_TIMEOUT = 30 # a README-sized read is <1 s; bounds a D-state hang on a wedged device @@ -358,6 +474,13 @@ def read_disk_file(uid: str, lun: int, fname: str) -> bytes: # ~5 KB of transfers plus libmtp setup takes seconds, not minutes; a larger value makes a # wedged MTP board cost that much on every retry, all charged to the pool guard. MTP_SESSION_MARGIN = 30 # transfer budget after enumeration; past it the session is killed +# room past the child's OWN enumeration budget for the echo exchange (3 x write + a 2000ms +# hidapi read) and interpreter start-up, so the outer kill only fires on a real stall +HID_ECHO_MARGIN = 30 +# hid_generic_inout's own idProduct. Pinned against the example's descriptor by +# HidEchoRunsInAChild.test_the_pid_matches_the_example, because a silent drift here would +# widen the walk back to every cafe: HID device without failing anything. +HID_INOUT_PID = 0x4012 def get_printer_dev(id: str, vendor_str, product_str, ifnum: int): @@ -366,12 +489,8 @@ def get_printer_dev(id: str, vendor_str, product_str, ifnum: int): product_str = product_str.replace(' ', '_') if product_str else '' for lp in glob.glob('/sys/class/usbmisc/lp*'): try: - # bounded: same device_lock() exposure as the sibling reads (see read_sysfs) sn = hil_util.read_sysfs(f'{lp}/device/../serial') - # UNKNOWN is not None: the sentinel has no __eq__, so an unanswered read - # would fall through both tests and read as 'not this board' -- the exact - # absence/unknown conflation read_sysfs exists to prevent. - if sn is None or sn is hil_util.SYSFS_UNKNOWN: + if sn is None: continue if sn == id: return f'/dev/usb/{os.path.basename(lp)}' @@ -388,7 +507,7 @@ def open_printer_dev(id: str, vendor_str, product_str, ifnum: int) -> str: lp_dev = wait_until(try_find) assert lp_dev, (f'Printer device not found for {id} if{ifnum:02d}' - + hil_util.sysfs_blind_note()) + + hil_util.strand_note()) return lp_dev @@ -444,34 +563,53 @@ def test_host_device_info(board): flasher = board['flasher'] declared_devs = [f'{d["vid_pid"]}_{d["serial"]}' for d in board['tests']['dev_attached']] - port = hil_util.get_serial_dev(flasher["uid"], None, None, 0) - ser = open_serial_dev(port) - ser.timeout = 0.1 - - # reset device since we can miss the first line - ret = getattr(hil_flash, f'reset_{flasher["name"].lower()}')(board) - assert ret.returncode == 0, 'Failed to reset device' + if board.get('logger') == 'rtt': + # The RTT console owns the probe, so reset BEFORE opening it (Commander then + # delivers the buffered boot burst). Unconditional, not only under --skip-flash: + # a previous run's console drained the ring, and the enumeration lines print + # only once — without this a re-run on unchanged firmware reads an empty ring. + ret = getattr(hil_flash, f'reset_{flasher["name"].lower()}')(board) + assert ret.returncode == 0, 'Failed to reset device' + ser = open_board_console(board) + try: + if board.get('logger') != 'rtt': + # reset device since we can miss the first line; on the VCOM the console + # survives the reset, so resetting after open catches the boot banner. + ret = getattr(hil_flash, f'reset_{flasher["name"].lower()}')(board) + assert ret.returncode == 0, 'Failed to reset device' - data = b'' - timeout = enum_timeout() - while timeout > 0: - new_data = ser.read(ser.in_waiting or 1) - if new_data: - data += new_data - enum_dev_sn = [] - for l in data.decode('utf-8', errors='ignore').splitlines(): - vid_pid_sn = re.search(r'ID ([0-9a-fA-F]+):([0-9a-fA-F]+) SN (\w+)', l) - if vid_pid_sn: - enum_dev_sn.append(f'{vid_pid_sn.group(1)}_{vid_pid_sn.group(2)}_{vid_pid_sn.group(3)}') - if set(declared_devs).issubset(set(enum_dev_sn)): - break - time.sleep(0.1) - timeout -= 0.1 - ser.close() + data = b'' + timeout = enum_timeout() + while timeout > 0: + # infra death is not a board failure: without this a dead JLinkExe/probe + # would burn the whole timeout and report as 'No data from device' + assert not getattr(ser, 'eof', False), \ + 'RTT console died (its server exited or the probe dropped off USB)' + new_data = ser.read(ser.in_waiting or 1) + if new_data: + data += new_data + enum_dev_sn = [] + for l in data.decode('utf-8', errors='ignore').splitlines(): + vid_pid_sn = re.search(r'ID ([0-9a-fA-F]+):([0-9a-fA-F]+) SN (\w+)', l) + if vid_pid_sn: + enum_dev_sn.append(f'{vid_pid_sn.group(1)}_{vid_pid_sn.group(2)}_{vid_pid_sn.group(3)}') + if set(declared_devs).issubset(set(enum_dev_sn)): + break + time.sleep(0.1) + timeout -= 0.1 + finally: + ser.close() - if len(data) == 0: - assert False, 'No data from device' lines = data.decode('utf-8', errors='ignore').splitlines() + if board.get('logger') == 'rtt': + # JLinkExe's telnet banner is delivered at connect, whether or not it ever + # finds the control block, so len(data) alone cannot tell "board said nothing" + # from "console never attached to the ring" -- drop the banner first + target_lines = hil_util.strip_banner(data).splitlines() + assert target_lines, ('No data from device: the RTT console attached but the target ' + 'produced nothing -- firmware built without LOGGER=rtt, or SWD lost') + elif len(data) == 0: + assert False, 'No data from device' enum_dev_sn = [] for l in lines: @@ -768,9 +906,9 @@ def test_device_cdc_msc_freertos(board): def link_is_fs(speed) -> bool: - """Payload scaling from a `speed` attribute. Anything not positively read as high speed - counts as FS -- including None and SYSFS_UNKNOWN: the FS payload merely tests an HS - board less, while the HS payload hard-fails a healthy FS board.""" + """Payload scaling from a `speed` attribute. Anything not positively read as high + speed counts as FS, None included: the FS payload merely tests an HS board less, while + the HS payload hard-fails a healthy FS board.""" return speed not in ('480', '5000', '10000') @@ -809,16 +947,15 @@ def test_device_cdc_msc_throughput(board): # Detect speed (12 Mbps FS / 480 Mbps HS) for payload scaling; a device we never find # keeps the FS payload (see link_is_fs) - # usb_scan, not a private glob: it skips root hubs and remembers paths that already - # stranded, so one wedged peer cannot spend this worker's blindness budget four reads - # at a time. + # usb_scan, not a private glob: it skips root hubs and filters on the lock-free + # descriptor pair before touching `serial`. is_fs = True speed_known = False - devs, _ = hil_util.usb_scan(vid='cafe', serial=uid) + devs = hil_util.usb_scan(vid='cafe', serial=uid) if devs: speed = hil_util.read_sysfs(os.path.join(devs[0]['dir'], 'speed')) is_fs = link_is_fs(speed) - speed_known = speed not in (None, hil_util.SYSFS_UNKNOWN) + speed_known = speed is not None # Put tty in raw mode so dd sees pure binary throughput. rs = hil_util.run_cmd(f'timeout 30 stty -F {tty} raw -echo') @@ -871,7 +1008,7 @@ def test_device_cdc_msc_throughput(board): # payload, so an HS board reads as suspiciously slow. Say so rather than publish a green # cell whose scale is a guess. scale = '' if speed_known else ' FS?' - return f'{REPORT_CELL["pass"]} C {pair(cdc_r, cdc_w)} M {pair(msc_r, msc_w)}{scale}' + return f'{hil_report.REPORT_CELL["pass"]} C {pair(cdc_r, cdc_w)} M {pair(msc_r, msc_w)}{scale}' def test_device_dfu(board): @@ -991,45 +1128,81 @@ def test_device_printer_to_cdc(board): ser.reset_input_buffer() # Test 1: Printer -> CDC with multiple sizes, write in random 1-64 byte chunks - LP_WRITE_TIMEOUT = 5.0 # seconds; firmware may stall draining the printer OUT endpoint + # The write runs in a PROCESS for the same reason the read below does: see LP_WRITER. for size in sizes: test_data = rand_ascii(size) ser.reset_input_buffer() - rd = b'' - offset = 0 - # bounded: O_NONBLOCK does NOT save us -- usblp_open() takes the device mutex - # first -- and this open runs on the worker itself, with no thread to abandon - lp_fd = hil_util.bounded_open(lp_dev, os.O_WRONLY | os.O_NONBLOCK, 5) - # Three-valued on purpose: an OSError here is a FACT about the node (EBUSY from - # usblp's single-opener rule, ENOENT from a re-enumeration race, EACCES from a - # udev gap) and must not be reported as a wedge -- that sends the operator to - # usb-kernel-recover for hardware that is fine. - assert lp_fd is not hil_util.SYSFS_UNKNOWN, ( - f'printer: opening {lp_dev} for write blocked (device wedged)' - f'{hil_util.sysfs_blind_note()}') - assert lp_fd is not None, f'printer: {lp_dev} could not be opened for write' + rd = bytearray() + + payload = Path(tempfile.gettempdir()) / f'hil-lp-tx-{os.getpid()}-{size}' + ready = Path(tempfile.gettempdir()) / f'hil-lp-ready-{os.getpid()}-{size}' + payload.write_bytes(test_data) + ready.unlink(missing_ok=True) + # +5 like write_cdc's sibling wait below: the bound is on the OPEN, and the child + # must first fork, exec and boot CPython, which on a loaded rig routinely exceeds + # LP_OPEN_TIMEOUT on its own. A tighter wait here reports a slow interpreter start + # as a wedged node. + open_deadline = time.monotonic() + LP_OPEN_TIMEOUT + 5 + saw_ready = False + + def read_cdc(): + # WAIT for the writer to have the node open, as Test 2's write_cdc does: the + # child has to fork, exec and boot CPython, and reading before it starts just + # burns the serial timeout. + # ONE deadline, shared with the child's bound below. Two different ones let + # the writer open after the parent gave up: it writes the whole payload with + # nobody reading, exits 0, and the byte-compare reports FIRMWARE DATA + # CORRUPTION for a board whose only problem was a slow open. + nonlocal saw_ready + while not ready.exists(): + if time.monotonic() > open_deadline: + return # never opened; the assert below reports THAT, not data + time.sleep(0.02) + saw_ready = True + # fullspeed devices may need extra time; ser.read is bounded by + # SERIAL_READ_TIMEOUT, so an empty return means the stream went quiet + while len(rd) < size: + chunk = ser.read(size - len(rd)) + if not chunk: + break + rd.extend(chunk) # in place: `rd +=` would rebind it as a local + try: - while offset < size: - chunk_size = min(random.randint(1, 64), size - offset) - buf = test_data[offset:offset + chunk_size] - written = 0 - while written < len(buf): - _, wr, _ = select.select([], [lp_fd], [], LP_WRITE_TIMEOUT) - assert wr, f'Printer write timeout after {LP_WRITE_TIMEOUT}s (firmware not draining OUT endpoint)' - n = os.write(lp_fd, buf[written:]) - written += n - rd += ser.read(chunk_size) - offset += chunk_size + r = hil_util.run_alongside( + [sys.executable, '-c', LP_WRITER, lp_dev, str(payload), str(ready)], + read_cdc, LP_OPEN_TIMEOUT + 12) finally: - os.close(lp_fd) - # read any remaining bytes (fullspeed devices may need extra time) - while len(rd) < size: - remaining = ser.read(size - len(rd)) - if not remaining: - break - rd += remaining - assert rd == test_data, (f'Printer->CDC wrong data ({size} bytes):\n' - f' expected: {test_data[:64]}\n received: {rd[:64]}') + ready.unlink(missing_ok=True) + payload.unlink(missing_ok=True) + # rc 124 is run_alongside's kill, i.e. the open blocked -- and stderr is EMPTY + # there, so without the fallback the cell reads 'failed (32 bytes, rc 124):' and + # nothing, for the one failure this conversion exists to contain. An OSError is a + # FACT about the node (EBUSY from usblp's single-opener rule, ENOENT from a + # re-enumeration race) and must not send the operator to usb-kernel-recover. + # The bound covers the open AND the whole write, so rc 124 alone does not mean a + # wedged node. `ready` is written on the line after os.open() returns, so its + # ABSENCE is what says the open never completed -- the case that sends an operator + # to usb-kernel-recover. Anything else killed on the bound was a slow drain. + detail = hil_util.cmd_stdout_text(r.stderr).strip()[:200] + # FIRST: a child that exited on its OWN carries the concrete errno, and only one + # we KILLED (rc 124) can be diagnosed as an open that never completed. Asserting + # the marker before this reported EBUSY/ENOENT as a wedged node -- the conflation + # the comment above exists to prevent. rc is in the message because a child killed + # by a signal leaves `detail` empty. + assert r.returncode in (0, 124), ( + f'Printer->CDC writer failed ({size} bytes, rc {r.returncode}): {detail}') + # saw_ready, not ready.exists(): a marker that appeared AFTER read_cdc gave up + # means the child wrote with nobody reading, and the byte-compare below would call + # that firmware data corruption. Report the slow open instead. + assert saw_ready, (f'printer: {lp_dev} was not opened for write within ' + f'{LP_OPEN_TIMEOUT + 5}s (device wedged, or the writer never ' + f'started); rc {r.returncode}') + assert r.returncode == 0, ( + f'Printer->CDC writer killed on its bound after opening {lp_dev} ' + f'(rc {r.returncode}): the firmware stopped draining the OUT endpoint') + assert bytes(rd) == test_data, (f'Printer->CDC wrong data ({size} bytes):\n' + f' expected: {test_data[:64]}\n' + f' received: {bytes(rd)[:64]}') # Test 2: CDC -> Printer with multiple sizes, write in random 1-64 byte chunks. # The lp read runs in a PROCESS, not a thread: /dev/usb/lp* blocks on read, usblp @@ -1069,8 +1242,13 @@ def test_device_printer_to_cdc(board): ready.unlink(missing_ok=True) # stderr, not stdout: run_alongside keeps the payload stream clean, so a traceback # from the reader now arrives on its own pipe - assert r.returncode == 0, (f'CDC->Printer reader failed ({size} bytes, rc ' - f'{r.returncode}): {hil_util.cmd_stdout_text(r.stderr)[:200]}') + # rc 124 is run_alongside's kill -- a blocked usblp_open leaves stderr EMPTY, so + # without the fallback this renders as 'failed (32 bytes, rc 124):' and nothing + rdetail = hil_util.cmd_stdout_text(r.stderr).strip()[:200] + assert r.returncode == 0, ( + f'CDC->Printer reader failed ({size} bytes): {rdetail}' if rdetail else + f'printer: reading {lp_dev} blocked (device wedged): the reader was killed on ' + f'its bound (rc {r.returncode})') assert r.stdout == test_data, (f'CDC->Printer wrong data ({size} bytes):\n' f' expected: {test_data[:64]}\n received: {r.stdout[:64]}') time.sleep(0.2) @@ -1237,9 +1415,6 @@ def test_device_midi_test(board): def test_device_audio_test_freertos(board): uid = board['uid'] - if os.name == 'nt': - return 'skipped' - pcm = None timeout = enum_timeout() while timeout > 0: @@ -1299,38 +1474,19 @@ def test_device_audio_test_freertos(board): def test_device_hid_generic_inout(board): + # The whole exchange runs in a child (see HID_ECHO): hidapi's blocking calls hold the + # GIL, so nothing in-process can bound them. run_cmd's killpg can. uid = board['uid'] - import hid # cython-hidapi (pip: hidapi, apt: python3-hid) - - timeout = enum_timeout() - dev = None - while timeout > 0: - for d in hid.enumerate(0xCafe): - if d['serial_number'] == uid: - dev = d - break - if dev: - break - time.sleep(1) - timeout -= 1 - assert dev is not None, f'HID device not found for {uid}' - - h = hid.device() - h.open(dev['vendor_id'], dev['product_id'], uid) - try: - for size in [8, 32, 63]: - # Report ID (0) + payload, padded to 64 bytes - payload = bytes([random.randint(1, 255) for _ in range(size)]) - report = bytes([0]) + payload + bytes(64 - size) - h.write(report) - echo = h.read(64, 2000) - assert echo and len(echo) >= size, ( - f'HID echo timeout or short read ({size} bytes)') - assert bytes(echo[:size]) == payload, ( - f'HID echo wrong data ({size} bytes):\n' - f' expected: {payload.hex()}\n received: {bytes(echo[:size]).hex()}') - finally: - h.close() + r = hil_util.run_cmd( + [sys.executable, '-c', HID_ECHO, uid, str(enum_timeout()), f'{HID_INOUT_PID:#06x}'], + timeout=enum_timeout() + HID_ECHO_MARGIN, split_stderr=True) + # rc 124 is run_cmd's kill: the child was still inside a hidapi call, which is the + # wedge this runs in a child FOR -- and stderr is empty there, so say so rather than + # render a bare trailing colon + detail = hil_util.cmd_stdout_text(r.stderr).strip()[:300] + assert r.returncode == 0, (f'hid_generic_inout: {detail}' if detail else + f'hid_generic_inout: the child was killed on its bound ' + f'(rc {r.returncode}) -- a hidapi call did not return') def test_device_usbtest(board): @@ -1340,35 +1496,30 @@ def test_device_usbtest(board): uid = board['uid'] def usbtest_enumerated(): - """True, False, or None when a bounded read did not answer -- absence unproven.""" # vid_pid FIRST: right after flashing, the previous example's enumeration (same # serial, different PID) can linger and would fail usbtest.py's lookup -- and # filtering on the two lock-free descriptor fields rules out every other device - # on the bus before the one read that can block. usb_scan memoises paths that - # already stranded, so one wedged peer cannot spend the blindness budget here. - devs, unknown = hil_util.usb_scan(vid_pid=('cafe', '4010'), serial=uid) - if devs: - return True - return None if unknown else False + # on the bus before the one read that can block. + return bool(hil_util.usb_scan(vid_pid=('cafe', '4010'), serial=uid)) end = time.monotonic() + enum_timeout() seen = usbtest_enumerated() - while time.monotonic() < end and seen is not True: + while time.monotonic() < end and not seen: time.sleep(0.2) seen = usbtest_enumerated() # fail before usbtest_permit: an absent device would otherwise queue on the battery # mutex for minutes behind real batteries just to have usbtest.py report "no device" - if seen is not True: + if not seen: # 0/30 rather than a bare cell: the battery never ran (30 = standard case count) - raise TestFail( - f'no cafe:4010 device with serial {uid}' if seen is False else - f'cannot tell whether cafe:4010 {uid} is present: the bounded sysfs reads did ' - f'not answer{hil_util.sysfs_blind_note()}', - metric=f'{REPORT_CELL["fail"]} 0/30') + # maxtasksperchild=1, so this worker only ever handled THIS board: a give-up here + # is about this device. Without the caveat a wedged-but-present DUT reads as a + # positive absence claim -- the conflation this whole path exists to avoid. + raise TestFail(f'no cafe:4010 device with serial {uid}{hil_util.strand_note()}', + metric=f'{hil_report.REPORT_CELL["fail"]} 0/30') # settle: right after flashing the enumeration can bounce once (and on dual-port parts # the other port's stale node — same serial and PID — lingers), and testusb run into # that gap sees the device drop mid-case - time.sleep(3) + time.sleep(USBTEST_SETTLE) # --keep-binding is required for concurrent batteries: usbtest.py's cleanup unbinds # EVERY usbtest-bound interface, killing a peer battery under USBTEST_PARALLEL > 1, and @@ -1384,8 +1535,9 @@ def test_device_usbtest(board): # Post-hang recovery reflashes the DUT through its own probe, NEVER a root-port cycle # (one board reached instead of every fixture under the port; see usb-kernel-recover). # _current_fw is the artifact test_example flashed for THIS test: re-deriving it from - # board['name'] reflashes the wrong build on variant-only boards. --outer-timeout lets - # usbtest skip a reflash it cannot finish before our run_cmd kill, which would orphan + # board['name'] reflashes the wrong build on variant-only boards. Our run_cmd bound + # below RESERVES the whole ladder (usbtest.recovery_reserve), which is what lets the + # child run it straight through without an outer kill landing mid-flash and orphaning # the flasher (own session) on the probe. Never under --skip-flash -- and say so: a # HUNG case then holds the DUT's usbfs lock for the rest of the run, and a probe reset # is no substitute (the DWC2 pullup survives a core halt). @@ -1397,14 +1549,12 @@ def test_device_usbtest(board): # same probe convoy-safely without changing how the board is normally flashed. _rec_flasher = hil_flash.recover_flasher(board) recovery = bool(_current_fw and not skip_flash and hil_flash.convoy_safe(_rec_flasher)) - # ONE bound, computed here and used for BOTH the child's --outer-timeout and our own - # run_cmd kill below. Three separate expressions disagreed: --skip-flash appended no - # --outer-timeout at all (usbtest reads 0 as "no limit"), and the no-recovery branch - # narrowed only the CHILD's view while run_cmd still waited the full reserve -- so a - # board that cannot recover held a pool worker AND its battery permit idle for - # USBTEST_RECOVERY_BUDGET it had no way to spend, under a usbtest width of 2. - outer = USBTEST_BATTERY_BUDGET + (USBTEST_RECOVERY_BUDGET if recovery - else USBTEST_OVERSHOOT) + # ONE bound: run_cmd's kill below. It carries the recovery reserve only when a + # recovery can actually run, and only what THIS flasher's ladder can spend -- a board + # that cannot recover used to hold a pool worker AND its battery permit idle for a + # reserve it had no way to spend, under a usbtest width of 2. + outer = USBTEST_BATTERY_BUDGET + USBTEST_OVERSHOOT + ( + usbtest.recovery_reserve(_rec_flasher) if recovery else 0) if _current_fw and skip_flash: print('note: --skip-flash disables usbtest hang recovery; a HUNG case will leave ' 'the device wedged until it is reflashed', flush=True) @@ -1413,12 +1563,11 @@ def test_device_usbtest(board): f'usbfs node, so usbtest hang recovery is disabled for {board["name"]}; a ' f'HUNG case will leave it wedged for the rest of the run', flush=True) if recovery: - # ship the RECOVERY flasher as `flasher`: usbtest.py, recovery_steps and - # convoy_safe all read board['flasher'], so substituting here keeps the entire - # child side unaware that a second roster entry exists + # ship the RECOVERY flasher as `flasher`: usbtest.py and convoy_safe both read + # board['flasher'], so substituting here keeps the entire child side unaware that + # a second roster entry exists rb = json.dumps({'name': board['name'], 'flasher': _rec_flasher}) cmd += f' --recover-board {shlex.quote(rb)} --recover-fw {shlex.quote(_current_fw)}' - cmd += f' --outer-timeout {outer}' # The reserve above USBTEST_BATTERY_BUDGET exists because the battery can overrun by # one already-started case, and a hang there needs room for the recovery (whose reflash # is bounded by usbtest.RECOVER_FLASH_TIMEOUT, not HIL_CMD_TIMEOUT). Without it run_cmd @@ -1454,8 +1603,22 @@ def test_device_usbtest(board): board_wedged = (f'{board["name"]}: usbtest reported a hang and was killed ' f'before it could report a verdict') raise TestFail(f'usbtest did not run: {detail}', - metric=f'{REPORT_CELL["fail"]} 0/30') + metric=f'{hil_report.REPORT_CELL["fail"]} 0/30') + + return _usbtest_verdict(board, data, out, passed, failed, recovery, + _rec_flasher) + +def _usbtest_verdict(board: Board, data: dict, out: str, passed: int, failed: int, + recovery: bool, rec_flasher: dict) -> str: + """The report cell for a battery that produced JSON, or a TestFail carrying one. + + Also latches board_wedged, which stops the REST of this board's examples: each would + flash THROUGH the poisoned usbfs node, block, survive SIGKILL and add another stray -- + one wedge becoming one stray per remaining example, which is the convoy this whole + containment path exists to prevent. + """ + global board_wedged # A HUNG case that recovery could not clear leaves a D-state holder on this board's # usbfs node. Latch it: the remaining examples would each flash THROUGH that node, # block, survive SIGKILL and add another stray -- turning one wedge into one stray per @@ -1464,30 +1627,30 @@ def test_device_usbtest(board): # the reflash worked, so a convoy-safe board whose recovery failed used to come back # unlatched and flash every remaining example through the poisoned node. if data.get('wedged') or (not recovery and 'HUNG' in out): - # _rec_flasher, NOT board['flasher']: recovery was decided against recover_flasher() - # at the top of this function, and the two diverge as soon as a roster carries the + # rec_flasher, NOT board['flasher']: recovery was decided against recover_flasher() + # in the caller, and the two diverge as soon as a roster carries the # optional `flasher_recover` key -- naming the wrong one sends the operator to the # wrong probe. The wording stays on what usbtest actually reported ("still wedged"), - # because unrecovered_hang is also set by the ambiguous/inconclusive aborts, where + # because unrecovered_hang is also set by the ambiguous abort, where # nothing hung and the old text was false on both clauses. board_wedged = (f'{board["name"]}: usbtest reports the device still wedged ' - + (f'after a recovery reflash via {_rec_flasher["name"]}' if recovery - else f'and {_rec_flasher["name"]} cannot deliver a recovery reflash')) + + (f'after a recovery reflash via {rec_flasher["name"]}' if recovery + else f'and {rec_flasher["name"]} cannot deliver a recovery reflash')) # notrun counts toward the denominator but is NOT a failure: listing cases that never # ran as failures sends a maintainer bisecting one of them. notrun = int(data.get('notrun', 0)) total = passed + failed + notrun if board_wedged and failed == 0 and notrun == 0: - # Every case passed and the device STILL wedged -- usbtest's inconclusive/ambiguous + # Every case passed and the device STILL wedged -- usbtest's ambiguous # abort fires after the last case, so nothing back-fills a BUDGET entry. Reporting # the pass would exit 0 with a D-state holder on the rig and the board absent from # the re-run spec. parsed=True: a retry re-pays the whole battery to re-observe a # wedge, and flashes through the poisoned node to do it. raise TestFail(f'usbtest {passed}/{total} but the device wedged ({board_wedged})', - metric=f'{REPORT_CELL["fail"]} {passed}/{total}', parsed=True) + metric=f'{hil_report.REPORT_CELL["fail"]} {passed}/{total}', parsed=True) if failed == 0 and notrun == 0 and total > 0: - return f'{REPORT_CELL["pass"]} {passed}/{total}' + return f'{hil_report.REPORT_CELL["pass"]} {passed}/{total}' bad = [c.get('num') for c in data.get('cases', []) if c.get('status') not in ('PASS', 'BUDGET')] why = f'usbtest {passed}/{total}' @@ -1503,7 +1666,7 @@ def test_device_usbtest(board): why += f'; {notrun} case(s) never ran ({reason}), so this says nothing about them' # parsed ONLY when every case ran: an aborted battery (budget expiry, kernel hang, bus # drop) leaves BUDGET entries, and those are exactly what a reflash retry can fix. - raise TestFail(why, metric=f'{REPORT_CELL["fail"]} {passed}/{total}', + raise TestFail(why, metric=f'{hil_report.REPORT_CELL["fail"]} {passed}/{total}', parsed=(notrun == 0)) @@ -1668,7 +1831,7 @@ def test_example(board: Board, variant: str, example: str) -> tuple[int, str, st def build_board(board: Board) -> tuple[str, int]: """Build firmware for this board via tools/build.py. - Honors board config's variant list (name, defines, flags). + Honors board config's variant list. Output goes to cmake-build/cmake-build-<variant>/ (tools/build.py layout). Unbounded on purpose: --build is a local convenience (no CI workflow passes it), so @@ -1705,11 +1868,48 @@ def build_board(board: Board) -> tuple[str, int]: return name, failed -# pseudo-test column for a variant boundary the park-flash could not clear (see below) -BOUNDARY_CELL = 'same-PID boundary' +def _tests_for(board: Board) -> list: + """Which examples this board runs, in roster order. + + Three sources, most specific first: an explicit -bt list for this board, a global -t + list filtered against what the board can actually do, or the roster's own capability + flags. The -t filter is not cosmetic -- without it a device-only board runs host/dual + tests whose `dev_attached` roster entry does not exist. + """ + name = board['name'] + if name in board_test: + return list(board_test[name]) + + board_tests = board.get('tests', {}) + if test_only: + if 'only' in board_tests: + allowed = set(board_tests['only']) + return [t for t in test_only if t in allowed] + return [t for t in test_only + if board_tests.get(t.split('/', 1)[0]) is True] + + if 'tests' not in board: + return [] + test_list: list = [] + if board_tests.get('device') is True: + test_list += list(device_tests) + if board_tests.get('dual') is True: + test_list += dual_tests + if board_tests.get('host') is True: + test_list += host_test + if 'only' in board_tests: + test_list = list(board_tests['only']) + for skip in board_tests.get('skip', []): + if skip in test_list: + test_list.remove(skip) + log_line(f'{name:25} {skip:30} ... Skip') + return test_list -def test_board(board: Board) -> tuple[str, int, list[str], list, float]: +def test_board(board: Board) -> tuple: + # (name, err_count, failed_tests, rows, duration[, strays]) -- the board-LOCKED early + # return is 5 wide, the normal one 6. _stray_note reads index 5 behind a len() guard, + # so a field inserted anywhere before it silently reports a duration as a stray count. swept = False name = board['name'] flasher = board['flasher'] @@ -1722,42 +1922,11 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: log_line(f'{name:25} {STATUS_FAILED}: {e}') # visible report row so the ❌ matches the exit code; failed-tests stays empty so a # re-run repeats the whole board (no bogus -bt filter) - return name, 1, [], [(name, {'board-locked': 'fail'}, None)], 0.0 + return name, 1, [], [(name, {hil_report.LOCKED_CELL: 'fail'}, None)], 0.0 # after the lock: flock wait behind a concurrent run is not board cost t_board = time.monotonic() try: - test_list = [] - - if name in board_test: - test_list = board_test[name] - elif len(test_only) > 0: - # Explicit -t: filter against the board's capabilities, or a device-only board - # runs host/dual tests whose `dev_attached` config entry does not exist. - board_tests = board.get('tests', {}) - if 'only' in board_tests: - allowed = set(board_tests['only']) - test_list = [t for t in test_only if t in allowed] - else: - for t in test_only: - category = t.split('/', 1)[0] - if board_tests.get(category) is True: - test_list.append(t) - else: - if 'tests' in board: - board_tests = board['tests'] - if board_tests.get('device') is True: - test_list += list(device_tests) - if board_tests.get('dual') is True: - test_list += dual_tests - if board_tests.get('host') is True: - test_list += host_test - if 'only' in board_tests: - test_list = board_tests['only'] - if 'skip' in board_tests: - for skip in board_tests['skip']: - if skip in test_list: - test_list.remove(skip) - log_line(f'{name:25} {skip:30} ... Skip') + test_list = _tests_for(board) err_count = 0 failed_tests = [] @@ -1809,7 +1978,7 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: # charging again would double-count one incident in the exit code if not wedge_skip: err_count += 1 - cells[BOUNDARY_CELL] = 'fail' + cells[hil_report.BOUNDARY_CELL] = 'fail' # blaming run_list[0] would re-run an innocent test that then passes, # leaving the boundary unretested; re-run the whole board instead board_wide_fail = True @@ -1825,7 +1994,7 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: # Do NOT flash through a poisoned node: each attempt enumerates into # it, blocks uninterruptibly and leaves another stray behind. Report # the skip so the cell is not mistaken for a pass. - cells[test] = f'{REPORT_CELL["skip"]} board wedged' + cells[test] = f'{hil_report.REPORT_CELL["skip"]} board wedged' # ...and re-run the WHOLE board, like the boundary-failure path above: # these tests never executed, so naming them individually in the .failed # spec is not enough -- an --accumulate re-run that fixes only the wedged @@ -1866,12 +2035,10 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: stray = hil_health.kill_own_children() swept = True - # LAST fields: whether this worker ran out of bounded-read budget, and what it could - # not kill. Only the worker can answer either -- the blindness latch is - # process-global and this is a separate process -- and the result tuple already - # crosses back, so no Manager round-trip. + # LAST field: what this worker could not kill. Only the worker can answer it, and + # the result tuple already crosses back, so no Manager round-trip. return (name, err_count, [] if board_wide_fail else sorted(set(failed_tests)), - rows, t_total, hil_util.sysfs_blind(), stray) + rows, t_total, stray) finally: # A raise skips the sweep above, and maxtasksperchild=1 retires this process # immediately afterwards -- reparenting its flasher to init and erasing the ppid @@ -1893,8 +2060,6 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: _lock_fh.close() -REPORT_MD = 'hil_report.md' -REPORT_JSON = 'hil_report.json' # controller hints from previous runs: uid -> {'name', 'pci', 'duration'}. Only 'pci' is # consumed (dispatch order and first-flash budgeting, never battery serialization). PCI # addresses are boot-stable, so the cache survives reboots and goes stale on re-cabling. @@ -1912,66 +2077,6 @@ def schedule_boards(boards: list, pci_of_uid: dict) -> list: return [b for grp in itertools.zip_longest(*buckets.values()) for b in grp if b is not None] -def render_matrix(rows_all: list) -> str: - """Render rows (list of (row_label, {example: status}, duration)) as an aligned - markdown matrix: columns = tests (bare names) centered, boards left-aligned, - per-row duration as the trailing column.""" - seen = set() - for _, cells, _ in rows_all: - seen.update(cells) - if not seen: - return 'No tests were run.' - - # metric-bearing columns pinned first, the rest alphabetical: stable regardless of the - # shuffled execution order - pinned = ['usbtest', 'cdc_msc_throughput', 'msc_file_explorer', 'msc_file_explorer_freertos'] - - def col_key(t): - name = t.rsplit('/', 1)[-1] - return (pinned.index(name) if name in pinned else len(pinned), name, t) - - columns = sorted(seen, key=col_key) - headers = [c.rsplit('/', 1)[-1] for c in columns] + ['duration'] # bare example names - - def cell(cells, col): - v = cells.get(col) - if v is None: - return '' - return REPORT_CELL.get(v, v) # status symbol, or a metric string (e.g. speed) verbatim - - rows_vals = [(lbl, [cell(cells, c) for c in columns] + [dur or '']) - for lbl, cells, dur in rows_all] - board_hdr = 'Board' - board_w = max([len(board_hdr)] + [len(lbl) for lbl, _ in rows_vals]) - col_w = [max([len(h)] + [len(vals[i]) for _, vals in rows_vals]) - for i, h in enumerate(headers)] - - def line(label, values): - padded = [label.ljust(board_w)] + [v.center(w) for v, w in zip(values, col_w)] - return '| ' + ' | '.join(padded) + ' |' - - header = line(board_hdr, headers) - sep = '| ' + '-' * board_w + ' | ' + ' | '.join(':' + '-' * (w - 2) + ':' for w in col_w) + ' |' - body = [line(lbl, vals) for lbl, vals in rows_vals] - - # tally run cells (not-run cells are absent from the dicts). A cell is a bare status or - # a metric string carrying its own icon ("❌ 29/30"), so classify by the leading icon. - def cell_kind(v): - if v == 'fail' or (isinstance(v, str) and v.startswith(REPORT_CELL['fail'])): - return 'fail' - if v == 'skip' or (isinstance(v, str) and v.startswith(REPORT_CELL['skip'])): - return 'skip' - return 'pass' - kinds = [cell_kind(v) for _, cells, _ in rows_all for v in cells.values()] - failed = kinds.count('fail') - skipped = kinds.count('skip') - passed = kinds.count('pass') - summary = (f'**{REPORT_CELL["pass"]} {passed} passed · {REPORT_CELL["fail"]} {failed} failed · ' - f'{REPORT_CELL["skip"]} {skipped} skipped · blank not run**') - - return summary + '\n\n' + '\n'.join([header, sep] + body) - - def _write_failed_spec(failed_fname: Path, report_dir: Path, mret: list) -> None: """Re-run spec: only the failed boards (-b), each restricted to its own failed tests (-bt); a board with failures but no test list re-runs entirely. @@ -2052,7 +2157,7 @@ def _stray_note(mret: list) -> str: runs AFTER accumulate_report on both abort paths, so a banner appended there was written to a variable nobody read again. """ - dirty = [(r[0], r[6]) for r in mret if len(r) > 6 and r[6]] + dirty = [(r[0], r[5]) for r in mret if len(r) > 5 and r[5]] if not dirty: return '' total = sum(n for _, n in dirty) @@ -2061,109 +2166,13 @@ def _stray_note(mret: list) -> str: f'{", ".join(f"{b} ({n})" for b, n in dirty)}.\n') -def _blind_note(mret: list) -> str: - """Name the boards whose worker went blind, for the report banner. - - A blind worker answers SYSFS_UNKNOWN for every attribute, so its "device not found" is - "could not tell". That already reaches the log and the per-cell failure text, but the - TABLE is what gets quoted -- and a red cell there is read as a broken board. Seen live - (run 31794359407): four workers blind, several cells red because of it, and a report - that said nothing. - - Per-board, not global: maxtasksperchild=1 gives every board a fresh worker, so a board - that ran on a healthy one is not smeared by a neighbour's wedge. Rows synthesised by - the timeout path are 5 fields wide and have nothing to report. - """ - blind = [r[0] for r in mret if len(r) > 5 and r[5]] - if not blind: - return '' - return (f'> **Not all verdicts are evidence.** {len(blind)} board(s) ran on a worker ' - f'that went blind on sysfs -- too many bounded reads stranded on a wedged ' - f'device -- so "not found" from them means "could not tell": ' - f'{", ".join(blind)}. See the usb-kernel-recover skill.\n') - - -def accumulate_report(mret: list, report_dir: Path, fresh: bool, scope: str = '', - banner: str = '') -> str: - """Merge this run's results into hil_report.json in report_dir, then (re)write - the markdown matrix to hil_report.md. `fresh` (a first run, no --accumulate) - starts a new report; otherwise a re-run accumulates so boards/tests that - already passed are preserved while re-run cells are updated. `scope` names the - board filter, if any, so a scoped table is not mistaken for a full one. - Returns the md.""" - acc = {} # ordered {row_label: [cells dict, duration str|None]} - prior_banner = '' - jpath = report_dir / REPORT_JSON - if not fresh and jpath.is_file(): - try: - saved = json.loads(jpath.read_text()) - # CI keys the report dir by run id, so the sidecar is from an earlier attempt - for entry in saved.get('rows', []): - acc[entry['board']] = [dict(entry['cells']), entry.get('duration')] - # ... and so is the caveat those cells were collected under. A rerun on a rig - # that has since recovered contributes no banner, and the .failed spec reruns - # only FAILURES -- so the earlier attempt's passes are never re-earned and - # would be published as clean results of a rig that was not. - prior_banner = saved.get('banner', '') - except (ValueError, KeyError, TypeError): - pass # corrupt/old sidecar: start fresh - - # current cells override prior for boards/tests that ran; a filtered run reports - # duration None, keeping the previous full-run value - for name, _, _, rows, *_ in mret: - if rows and not any('board-locked' in cells for _, cells, _ in rows): - # board ran for real: clear a stale lock-failure cell (its row is keyed by - # board name; test rows may be variant names) - stale = acc.get(name) - if stale is not None: - stale[0].pop('board-locked', None) - if not stale[0]: - # variant-keyed boards never repopulate the board-name row, so drop it - # or it renders as a blank ghost row - del acc[name] - for row_label, cells, dur in rows: - row = acc.setdefault(row_label, [{}, None]) - # the boundary cell is only ever written on failure, so a re-run of this - # variant that cleared the boundary must drop the previous attempt's ❌ - if BOUNDARY_CELL not in cells: - row[0].pop(BOUNDARY_CELL, None) - row[0].update(cells) - if dur is not None: - row[1] = dur - - report_dir.mkdir(parents=True, exist_ok=True) - # by LINE, deduped: attempts repeat the same caveat far more often than they add a new - # one, and three copies of the D-state note reads as three incidents - seen, merged = set(), [] - for line in (prior_banner + banner).splitlines(): - if line.strip() and line not in seen: - seen.add(line) - merged.append(line) - banner = '\n'.join(merged) + '\n' if merged else '' - jpath.write_text(json.dumps({'rows': [{'board': k, 'cells': c, 'duration': d} - for k, (c, d) in acc.items()], - 'banner': banner}, indent=2) + '\n') - - md = render_matrix([(k, c, d) for k, (c, d) in acc.items()]) - if scope: - # a scoped run's small table is otherwise indistinguishable from a full one, and - # it replaces the previous full table in the sticky PR comment - md = f'_Scoped run: {scope}. Boards/tests not listed were not run._\n\n' + md - # LAST, so it is outermost: a rig-health caveat outranks the table AND the scope note, - # and the top of the report is where hil/SKILL.md tells the agent to look for it. - if banner: - md = banner + '\n' + md - (report_dir / REPORT_MD).write_text(md + '\n', encoding='utf-8') - return md - - # containment paths print through hil_health._p: stdout may already be a dead pipe (a # dropped ssh session), and a BrokenPipeError there would skip os._exit _p = hil_health._p def _abandon_exit(pool, mgr, abandoned: bool, err_count: int, - report: Path | None = None) -> None: + report_dir: Path | None = None) -> None: """Free the runner when the pool could not be shut down. Returns only if not abandoned. Must run even while an exception is propagating: multiprocessing's atexit handler @@ -2199,24 +2208,11 @@ def _abandon_exit(pool, mgr, abandoned: bool, err_count: int, 'stay locked.', flush=True) # A report already written by accumulate_report says nothing about the abandon, and a # green table under a red job is how an agent ends up pasting it as this run's result. - # Prepend the caveat; best-effort, never at the cost of exiting. - if report is not None: - try: - if report.exists(): - # utf-8 explicitly (the cells are ✅/❌/⚪) and catch ValueError too: a torn - # report or a LANG=C locale raises UnicodeDecodeError -- NOT an OSError -- - # straight past os._exit, stranding the runner. - body = report.read_text(encoding='utf-8', errors='replace') - # Only when no banner is there yet, searched anywhere in the head rather - # than at char 0: write_timeout_report's banner must stay FIRST (its table - # is a PREVIOUS attempt's) and it puts the rig-health quote above itself. - if '**HIL run ab' not in body[:2000]: - report.write_text( - '**HIL run abandoned: the worker pool would not shut down.** The ' - 'table below was collected before the abandon; treat board ' - 'results as unverified.\n\n' + body, encoding='utf-8') - except (OSError, ValueError): - pass + # Set the caveat in the DOCUMENT -- prepending to the markdown alone left the sidecar, + # which is all hil_report.summarize() and therefore an agent ever sees, saying nothing. + # Best-effort, never at the cost of exiting. + if report_dir is not None: + hil_report.mark_report_abandoned(report_dir, 'the worker pool would not shut down.') try: sys.stdout.flush() except OSError: @@ -2226,6 +2222,135 @@ def _abandon_exit(pool, mgr, abandoned: bool, err_count: int, os._exit(min(err_count, 125) if err_count else 1) +def _load_controller_hints() -> tuple[dict, dict]: + """The uid -> {name, pci, duration} cache, plus the uid -> pci view scheduling wants. + + Best effort throughout: a missing, hand-edited or torn cache costs dispatch ORDER, + never the run. + """ + hints: dict = {} + try: + with CONTROLLER_CACHE.open() as f: + loaded = json.load(f) + if isinstance(loaded, dict): # keep only the expected uid -> dict shape + hints = {k: v for k, v in loaded.items() if isinstance(v, dict)} + except (OSError, ValueError): + pass + return hints, {uid: h['pci'] for uid, h in hints.items() if h.get('pci')} + + +def _save_controller_hints(hints: dict, mret: list, uid_of: dict, cmap) -> None: + """Fold this run's PCI resolutions and durations back into the cache, atomically. + + Merge-on-write: another HIL job (the esp split) may have finished since our startup + read, so overlay only this run's boards rather than publishing our whole view. + """ + for name, _, _, _, dur, *_ in mret: + uid = uid_of.get(name) + if uid is None: + continue + h = dict(hints.get(uid) or {}) + h['name'] = name # informational: the cache is keyed by uid + h['pci'] = cmap.get(f'uid:{uid}') or h.get('pci') + if dur > 0: # test_board reports 0.0 for filtered (partial) runs + h['duration'] = round(dur, 1) + hints[uid] = h + merged: dict = {} + try: + with CONTROLLER_CACHE.open() as f: + cur = json.load(f) + if isinstance(cur, dict): + merged = {k: v for k, v in cur.items() if isinstance(v, dict)} + except (OSError, ValueError): + pass + # overlay onto what the CACHE now holds, not onto our startup snapshot: another HIL + # job may have written a newer duration/pci for these boards since we read it + for name, *_ in mret: + uid = uid_of.get(name) + if uid is not None and uid in hints: + merged[uid] = {**merged.get(uid, {}), **hints[uid]} + CONTROLLER_CACHE.parent.mkdir(parents=True, exist_ok=True) + tmp = CONTROLLER_CACHE.with_suffix('.json.tmp') + with tmp.open('w') as f: + json.dump(merged, f, indent=1, sort_keys=True) + tmp.replace(CONTROLLER_CACHE) + + +def _abort_report(reason: str, mret: list, config_boards: list, failed_fname: Path, + report_dir: Path, fresh: bool, health_banner: str, + timeout_secs: int | None = None) -> None: + """Keep what finished, name what did not, and get a report on disk. Never raises. + + Both abort paths -- the pool guard expiring and a worker raising -- need exactly this, + and in this order. The re-run spec goes FIRST: a fresh run already unlinked it, and + leaving it unwritten is what made a GitHub re-run repeat the whole fleet. Only the + boards that never reported go in it. + + The report follows, before anything that can block, and the caller raises afterwards + into the one containment path. `timeout_secs` adds the pool-guard fallback: when + accumulate_report itself fails -- an unwritable report dir, a torn JSON -- + _abandon_exit can only stamp a report that EXISTS, so without it the artifact upload + finds nothing and the sticky PR comment keeps the previous push's green table under a + red job. + """ + stuck = [b['name'] for b in config_boards if b['name'] not in {r[0] for r in mret}] + try: + _write_failed_spec(failed_fname, report_dir, + [(n, 1, [], None, 0) for n in stuck] + + [r for r in mret if r[1] > 0]) + except Exception as werr: # noqa: BLE001 - it mkdir()s and open()s the very report dir + # the fallback below is FOR an unwritable/root-owned report dir; letting the spec + # raise here replaces the caller's RuntimeError, so the operator never sees the + # 'pool timed out' line and no report is written at all + print(f'warning: re-run spec failed: {type(werr).__name__}: {werr}', flush=True) + banner = (f"**HIL run {reason}.** {len(mret)} board(s) below finished and are this " + f"run's; {len(stuck)} never reported and are NOT in the table: " + f"{', '.join(stuck)}. The re-run spec covers those.\n") + try: + hil_report.accumulate_report(mret, report_dir, fresh, '', + health_banner + _stray_note(mret), caveat=banner) + return + except Exception as rerr: # noqa: BLE001 - the caller's raise must still happen + print(f'warning: partial report failed: {type(rerr).__name__}: {rerr}' + + '; falling back to the board list', flush=True) + try: + # banner=, or write_timeout_report's default caveat publishes 'No per-board + # results could be collected' onto a report where mret DID hold finished rows + # the CELL names the cause: a board the pool guard never reached did not + # "pool-timeout", and marking it so sends the reader after a guard that did not fire + hil_report.write_timeout_report( + report_dir, [b for b in config_boards if b['name'] in stuck], + timeout_secs or 0, banner=banner, prefix=health_banner, + cell=(hil_report.POOL_TIMEOUT_CELL if timeout_secs + else hil_report.RUN_ABORTED_CELL)) + except Exception as re2: # noqa: BLE001 + print(f'warning: fallback report failed too: {type(re2).__name__}: {re2}', + flush=True) + + +def _start_pool(mgr, seed: str, hints_by_uid: dict): + """(cmap, pool). Split out so main()'s try/finally reads as one shape. + + The Manager is created by the CALLER and passed in: Pool() forks, and after a convoy + that fork is what hits EAGAIN/ENOMEM. Creating the Manager here too would leave main() + with `mgr` still None while a live SyncManager child exists -- os._exit skips its + finalizer and the orphan holds the runner's stdout, so the job step never completes. + + maxtasksperchild=1: a fresh worker per board makes cross-board contamination + structural rather than dependent on every module global being reset by hand + (board_wedged, _current_fw, hil_flash's warn-once sets). The extra fork is noise + against a flash+test cycle. + """ + cmap = mgr.dict() + initargs = (Lock(), seed, + hil_lock.make_permit_sems(Semaphore, hil_lock.USBTEST_PARALLEL), + hil_lock.make_permit_sems(Semaphore, hil_lock.FLASH_PARALLEL), + cmap, Lock(), hints_by_uid) + pool = Pool(processes=os.cpu_count() or 1, initializer=init_worker, + initargs=initargs, maxtasksperchild=1) + return cmap, pool + + def main() -> None: """ Hardware test on specified boards @@ -2297,6 +2422,56 @@ def main() -> None: config_boards = [e for e in config['boards'] if e['name'] in boards] config_boards = [e for e in config_boards if e['flasher']['name'] not in args.exclude_flasher and (not args.flasher or e['flasher']['name'] in args.flasher)] + + # fail rtt misconfigurations before the first flash cycle -- but only for boards + # this run actually touches: one bad roster entry must not abort other runs' subsets + def _rtt_config_abort(msg: str): + # loud AND leaving evidence, like the no-boards branch below: exiting with no + # report at all lets the PR comment keep the previous push's stale table + print(f'ERROR: {msg}', flush=True) + rd = Path(os.environ.get('HIL_REPORT_DIR', '.')) + hil_report.mark_report_no_boards(rd, f'config error: {msg}', fresh=not args.accumulate) + sys.exit(1) + + bad_logger = [e['name'] for e in config_boards if e.get('logger') not in (None, 'rtt')] + if bad_logger: + # only the exact string activates RTT handling; anything else would silently + # mean VCOM and reproduce the misleading 'No serial device found' failure + _rtt_config_abort(f'unknown "logger" value (only "rtt" is supported): {", ".join(bad_logger)}') + bad_rtt = [e['name'] for e in config_boards + if e.get('logger') == 'rtt' and e['flasher']['name'].lower() != 'jlink'] + if bad_rtt: + # JlinkRtt speaks JLinkExe only (the OpenOCD RTT route is manual — rtt skill) + _rtt_config_abort(f'"logger": "rtt" needs a jlink flasher: {", ".join(bad_rtt)}') + rtt_no_logger_def = [e['name'] for e in config_boards + if e.get('logger') == 'rtt' + and any('LOGGER=rtt' not in (v.get('defines') or []) + for v in (e.get('variant') or [{}]))] + if rtt_no_logger_def: + # a prebuilt cmake-build-<board> configured with -DLOGGER=rtt is a legitimate + # build path the roster need not describe, so warn there -- but when this run is + # responsible for the firmware (--build, or CI where the hil-build job compiled + # the artifact from these same defines) the flashed image is UART-logger and every + # test times out as 'the target produced nothing'. An always-on define is + # expressed as a single self-named variant (see the Board comment). + msg = (f'"logger": "rtt" board has a variant without LOGGER=rtt in its defines ' + f'({", ".join(rtt_no_logger_def)})') + if args.build or os.environ.get('GITHUB_ACTIONS'): + _rtt_config_abort(f'{msg} -- the firmware built for this run cannot serve the ' + f'configured RTT console') + print(f'warning: {msg} -- fine for prebuilt example sets, wrong for --build/CI ' + f'builds', flush=True) + rtt_fixture = [e['name'] for e in config_boards + if e.get('logger') == 'rtt' + and any(d.get('is_cdc') or d.get('is_msc') + for d in e.get('tests', {}).get('dev_attached', []))] + if rtt_fixture: + # interim guard, removed when the followup lands: cdc_msc_hid/msc_file_explorer + # still open the flasher VCOM directly and would die mid-run on an rtt board + _rtt_config_abort(f'"logger": "rtt" boards cannot carry is_cdc/is_msc fixtures yet ' + f'(host cdc/msc tests bypass the RTT console — see ' + f'the rtt harness-adoption doc in docs/superpowers/followup/): {", ".join(rtt_fixture)}') + if not config_boards: # same reason the unknown -b board exits 1: 'No tests were run.' with rc 0 reads as # a green HIL leg, so a roster edit emptying a leg's filter stops testing silently @@ -2305,13 +2480,11 @@ def main() -> None: print(msg, flush=True) # loud AND leaving evidence: exiting with no report at all lets the PR comment # keep the previous push's stale table under a red job - try: - rd = Path(os.environ.get('HIL_REPORT_DIR', '.')) - rd.mkdir(parents=True, exist_ok=True) - (rd / REPORT_MD).write_text(f'**HIL run selected no boards.** {msg}\n', - encoding='utf-8') - except OSError: - pass + rd = Path(os.environ.get('HIL_REPORT_DIR', '.')) + # fresh must be threaded through: this runs BEFORE the `if fresh:` wipe below, so + # defaulting it here wiped an --accumulate run's accumulated rows -- the exact + # regression the parameter exists to prevent. + hil_report.mark_report_no_boards(rd, msg, fresh=not args.accumulate) sys.exit(1) @@ -2346,9 +2519,6 @@ def main() -> None: report_dir = Path(os.environ.get('HIL_REPORT_DIR', '.')) failed_fname = report_dir / (config_file.name + '.failed') fresh = not args.accumulate - # The unlink is DEFERRED to inside the pool try/except below: wiping here leaves - # Manager() and Pool() running with the old report gone and no report-writing path - # armed, so an EAGAIN/ENOMEM on fork gives CI an EMPTY report dir with no reason. seed = os.getenv('HIL_SHUFFLE_SEED') or str(int(time.time())) log_line(f'test-order shuffle seed: {seed} (HIL_SHUFFLE_SEED={seed} to replay); ' @@ -2358,16 +2528,7 @@ def main() -> None: # unattributable from the log alone f'pool guard: {POOL_TIMEOUT}s') - hints = {} - try: - with CONTROLLER_CACHE.open() as f: - loaded = json.load(f) - # tolerate a hand-edited/torn cache: keep only the expected uid -> dict shape - if isinstance(loaded, dict): - hints = {k: v for k, v in loaded.items() if isinstance(v, dict)} - except (OSError, ValueError): - pass - hints_by_uid = {uid: h['pci'] for uid, h in hints.items() if h.get('pci')} + hints, hints_by_uid = _load_controller_hints() config_boards = schedule_boards(config_boards, hints_by_uid) log_line('dispatch order: ' + ', '.join(b['name'] for b in config_boards)) @@ -2387,31 +2548,22 @@ def main() -> None: # BEFORE Manager()/Pool(), not inside the try: hil_ci.sh reuses a persistent REMOTE_DIR # and scp's the report back unconditionally, so if a fork failure (OSError/EAGAIN right # after a convoy -- the case this whole block guards) skipped the wipe, the finally's - # _abandon_exit would prepend "HIL run abandoned" to the PREVIOUS run's table and + # _abandon_exit would stamp "HIL run abandoned" onto the PREVIOUS run's report and # publish last night's board results as this run's. Nothing is live yet here, so an # OSError from the wipe itself just exits with its traceback -- it cannot strand the # interpreter in multiprocessing's unbounded atexit join, which is what deferring it # was protecting against. if fresh: report_dir.mkdir(parents=True, exist_ok=True) - for f in (REPORT_JSON, REPORT_MD): + for f in (hil_report.REPORT_JSON, hil_report.REPORT_MD): (report_dir / f).unlink(missing_ok=True) failed_fname.unlink(missing_ok=True) try: + # BOUND FIRST, in main's own scope: a Pool fork failure inside _start_pool must + # still leave a live Manager reachable by the finally below, or its child is + # orphaned holding the runner's stdout. mgr = Manager() - cmap = mgr.dict() - initargs = (Lock(), seed, - hil_lock.make_permit_sems(Semaphore, hil_lock.USBTEST_PARALLEL), - hil_lock.make_permit_sems(Semaphore, hil_lock.FLASH_PARALLEL), - cmap, Lock(), hints_by_uid) - # maxtasksperchild=1: the sysfs blindness latch is process-global and permanent - # (no decrement anywhere -- see hil_util.SYSFS_STUCK_MAX), so a worker that goes - # blind on ONE wedged board would report 0/30 and "probe missing" for the 2-3 - # healthy boards it picked up afterwards. A fresh worker per board confines the - # damage to the board that caused it; the extra fork is noise against a - # flash+test cycle. - pool = Pool(processes=os.cpu_count() or 1, initializer=init_worker, - initargs=initargs, maxtasksperchild=1) + cmap, pool = _start_pool(mgr, seed, hints_by_uid) # OUTER: encloses the pool block too, not just the reporting below. An exception # escaping async_ret.get() (a worker exception, a Ctrl-C) runs the pool finally and # then propagates straight out of main(); with _abandon_exit in a sibling try it @@ -2428,43 +2580,13 @@ def main() -> None: try: mret = drain_pool(it, config_boards, deadline, out=mret) except MpTimeoutError as te: - mret = te.finished - stuck = [b['name'] for b in config_boards - if b['name'] not in {r[0] for r in mret}] - # The re-run spec FIRST and before the raise: a fresh run already unlinked - # it, so leaving it unwritten is what made the GitHub re-run repeat the - # whole fleet. Only the boards that never reported go in it. - _write_failed_spec(failed_fname, report_dir, - [(n, 1, [], None, 0) for n in stuck] - + [r for r in mret if r[1] > 0]) - # Then the report, with the rows that DID finish, before anything that can - # block. Then RAISE into the ONE containment path: the inner finally runs + # RAISE afterwards into the ONE containment path: the inner finally runs # the ordered sweep (kill_worker_children BEFORE terminate, or a reaped # worker's flasher reparents out of reach), the outer one os._exit's. - banner = (f'**HIL run abandoned: worker pool timed out after ' - f'{POOL_TIMEOUT}s.** {len(mret)} board(s) below finished and ' - f'are this run\'s; {len(stuck)} never reported and are NOT in ' - f'the table: {", ".join(stuck)}. Re-run covers those.\n') - try: - accumulate_report(mret, report_dir, fresh, '', - health_banner + _blind_note(mret) - + _stray_note(mret) + banner) - except Exception as rerr: # noqa: BLE001 - the raise below must still happen - # FALL BACK, do not just warn: accumulate_report can raise on an - # unwritable/root-owned report dir or a torn JSON, and _abandon_exit - # only PREPENDS to a report that exists. Without this the artifact - # upload finds nothing (if-no-files-found: ignore) and the sticky PR - # comment keeps the previous push's green table under a red job. - print(f'warning: partial report failed: {type(rerr).__name__}: {rerr}; ' - f'falling back to the board list', flush=True) - try: - hil_health.write_timeout_report( - report_dir, [b for b in config_boards - if b['name'] in stuck], POOL_TIMEOUT, REPORT_MD, - prefix=health_banner) - except Exception as re2: # noqa: BLE001 - print(f'warning: fallback report failed too: ' - f'{type(re2).__name__}: {re2}', flush=True) + mret = te.finished + _abort_report(f'abandoned: worker pool timed out after {POOL_TIMEOUT}s', + mret, config_boards, failed_fname, report_dir, fresh, + health_banner, timeout_secs=POOL_TIMEOUT) _p(f'HIL worker pool timed out after {POOL_TIMEOUT}s; sweeping and ' f'shutting it down (abandoning it if a worker is unkillable)', flush=True) @@ -2472,44 +2594,26 @@ def main() -> None: except Exception as e: # A worker RAISED -- e.g. a flasher adapter dropping off the bus makes # get_serial_dev raise in the worker's flash section, which no per-test - # handler guards. Same treatment as the timeout path: the drain means - # `mret` already holds every board that finished, so keep those rows and - # name only the ones still in flight. (Under map_async they were all lost, - # which is what the old banner here claimed.) - done = {r[0] for r in mret} - stuck = [b['name'] for b in config_boards if b['name'] not in done] - _write_failed_spec(failed_fname, report_dir, - [(n, 1, [], None, 0) for n in stuck] - + [r for r in mret if r[1] > 0]) - banner = (f'**HIL run aborted: a worker raised {type(e).__name__}: {e}.** ' - f'{len(mret)} board(s) below finished and are this run\'s; ' - f'{len(stuck)} did not report: {", ".join(stuck)}.\n') - try: - accumulate_report(mret, report_dir, fresh, '', - health_banner + _blind_note(mret) - + _stray_note(mret) + banner) - except Exception as re2: # noqa: BLE001 - the raise below must still happen - print(f'warning: partial report failed: {type(re2).__name__}: {re2}', - flush=True) + # handler guards. The drain means `mret` already holds every board that + # finished, so keep those rows and name only the ones still in flight. + _abort_report(f'aborted: a worker raised {type(e).__name__}: {e}', + mret, config_boards, failed_fname, report_dir, fresh, + health_banner) raise err_count = build_err + sum(e[1] for e in mret) _write_failed_spec(failed_fname, report_dir, mret) finally: - # Not `with Pool(...)`: its __exit__ joins the workers unbounded, hanging on + # Not `with Pool(...)`: its __exit__ joins the workers unbounded and hangs on # any worker in uninterruptible sleep. shutdown_pool bounds the same terminate() - # by a grace period, so the pool is NOT cleanly closed/joined when it returns - # False. Record the outcome but never exit here: the report below is the only - # record of a run that otherwise passed. + # and returns False when the pool is NOT cleanly closed. # - # Same ordering as the timeout path: what the workers spawned must be - # snapshotted and killed while its parent is alive, or terminate() reparents it - # out of reach. + # Sweep BEFORE shutdown: what the workers spawned must be snapshotted and + # killed while its parent is alive, or terminate() reparents it out of reach. # - # Both calls must stay guarded: a raise here skips accumulate_report(), so a run - # whose boards ALL passed publishes an empty report dir -- and both can raise - # for reasons unrelated to the results. pool_abandoned stays fail-CLOSED, so - # _abandon_exit still arms. + # Both calls stay guarded and neither exits: a raise here would skip + # accumulate_report and publish an empty report dir for a run whose boards all + # passed. pool_abandoned is fail-CLOSED, so _abandon_exit still arms. try: # Still worth running for the TIMEOUT path, where the workers are # genuinely stuck mid-task and their children are still reachable through @@ -2537,33 +2641,8 @@ def main() -> None: report_dir.mkdir(parents=True, exist_ok=True) with (report_dir / 'hil_profile_ctrl.json').open('w') as f: json.dump(dict(cmap), f, indent=1, sort_keys=True) - uid_of = {b['name']: b['uid'] for b in config['boards']} - for name, _, _, _, dur, *_ in mret: - uid = uid_of.get(name) - if uid is None: - continue - h = dict(hints.get(uid) or {}) - h['name'] = name # informational: cache is keyed by uid - h['pci'] = cmap.get(f'uid:{uid}') or h.get('pci') - if dur > 0: # test_board reports 0.0 for filtered (partial) runs - h['duration'] = round(dur, 1) - hints[uid] = h - # merge-on-write: another HIL job (e.g. the esp split) may have finished since - # our startup read, so overlay only this run's boards and replace atomically - merged = {} - try: - with CONTROLLER_CACHE.open() as f: - cur = json.load(f) - if isinstance(cur, dict): - merged = {k: v for k, v in cur.items() if isinstance(v, dict)} - except (OSError, ValueError): - pass - merged.update({uid_of[n]: hints[uid_of[n]] for n, *_ in mret if n in uid_of}) - CONTROLLER_CACHE.parent.mkdir(parents=True, exist_ok=True) - tmp = CONTROLLER_CACHE.with_suffix('.json.tmp') - with tmp.open('w') as f: - json.dump(merged, f, indent=1, sort_keys=True) - tmp.replace(CONTROLLER_CACHE) + _save_controller_hints( + hints, mret, {b['name']: b['uid'] for b in config['boards']}, cmap) except Exception as e: # Deliberately broad, and it must stay that way: this best-effort refresh makes # Manager proxy RPCs that raise EOFError / BrokenPipeError / RemoteError when @@ -2578,12 +2657,11 @@ def main() -> None: # looks exactly like a full run that happened to be small scoped = sorted(set(args.board) | set(board_test)) scope = f'{len(scoped)} board(s) — {", ".join(scoped)}' if scoped else '' - report = accumulate_report(mret, report_dir, fresh, scope, - health_banner + _blind_note(mret) - + _stray_note(mret)) + report = hil_report.accumulate_report(mret, report_dir, fresh, scope, + health_banner + _stray_note(mret)) print() print(report) - print(f'\nReport written to {(report_dir / REPORT_MD).resolve()}') + print(f'\nReport written to {(report_dir / hil_report.REPORT_MD).resolve()}') duration = time.time() - duration print() @@ -2594,7 +2672,7 @@ def main() -> None: # In the finally, not after: any raise above (accumulate_report sits outside the # OSError handler) would skip the abandon path and unwind into multiprocessing's # unbounded atexit join, hanging the runner. - _abandon_exit(pool, mgr, pool_abandoned, err_count, report_dir / REPORT_MD) + _abandon_exit(pool, mgr, pool_abandoned, err_count, report_dir) # Same clamp: exit status is a byte either way, so 256 failures would report green. sys.exit(min(err_count, 125)) diff --git a/test/hil/test/stubs/hid.py b/test/hil/test/stubs/hid.py new file mode 100644 index 000000000..20a6cccef --- /dev/null +++ b/test/hil/test/stubs/hid.py @@ -0,0 +1,76 @@ +# SPDX-License-Identifier: MIT +"""Scripted stand-in for cython-hidapi, for the HID_ECHO child tests. + +A real wedge cannot be manufactured on demand, so the failure modes are scripted here and +selected with FAKE_HID_MODE. Mirrors test/stubs/pymtp.py, which does the same for libmtp. +""" +import ctypes +import ctypes.util +import os +import time + +_MODE = os.environ.get('FAKE_HID_MODE', 'ok') +_UID = os.environ.get('FAKE_HID_UID', 'CAFE01') + + +def _gil_stall(): + """Block forever WITHOUT releasing the GIL -- the shape cython-hidapi's bare + hid_open()/hid_close() calls have, and the one an in-process bound cannot touch. + + PyDLL, not CDLL: CDLL releases the GIL around the call, which would make this the + easy case instead of the hard one. Resolved through find_library so a non-glibc libc + still works; PyDLL(None) is not usable here (its `sleep` returns immediately). + """ + ctypes.PyDLL(ctypes.util.find_library('c') or 'libc.so.6').sleep(3600) +_PID = int(os.environ.get('FAKE_HID_PID', '0x4012'), 16) + + +def enumerate(vid=0, pid=0): + """Real hid.enumerate(vid, pid) filters on both ids -- 0 means "any" -- and returns a + 'path' key too. The filters are applied BEFORE the locked manufacturer/product reads, + which is why passing both narrows what a wedged peer can stall.""" + if _MODE == 'wedged_enumerate': + # hidapi's hidraw backend reads `manufacturer`/`product` for every device it + # lists, both served under the device lock -- this is that stall. + while True: + time.sleep(3600) + if _MODE == 'absent': + return [] + if vid not in (0, 0xCafe) or pid not in (0, _PID): + return [] + return [{'serial_number': _UID, 'vendor_id': 0xCafe, 'product_id': _PID, + 'path': b'/dev/hidraw0'}] + + +class device: + def __init__(self): + self._last = b'' + + def open(self, vid, pid, serial): + # HID_ECHO really does call this, and usb_autopm/hidraw can block in it, so the + # child must be bounded here too -- exercised by test_a_wedged_open_is_killed. + if _MODE == 'wedged_open': + while True: + time.sleep(3600) + if _MODE == 'wedged_open_gil': + # a thread-based bound is inert against this; only killing the process works + _gil_stall() + + def write(self, report): + self._last = bytes(report) + + def read(self, size, timeout_ms): + if _MODE == 'wedged_read': + while True: + time.sleep(3600) + if _MODE == 'short_read': + return list(self._last[1:4]) + if _MODE == 'wrong_data': + return list(bytes(b ^ 0xFF for b in self._last[1:])) + return list(self._last[1:]) # the device echoes the payload, minus report ID + + def close(self): + if _MODE == 'wedged_close': + # also GIL-holding in cython-hidapi, and it runs in HID_ECHO's finally on + # every failure path + _gil_stall() diff --git a/test/hil/test/test_ci_metrics.py b/test/hil/test/test_ci_metrics.py index a76b6e3a0..6f1511913 100644 --- a/test/hil/test/test_ci_metrics.py +++ b/test/hil/test/test_ci_metrics.py @@ -447,16 +447,134 @@ class TestWorkflowSelectionHandOff(unittest.TestCase): self.assertIn('UNSCOPED', flat[max(0, i - 200):i], 'a fall-open path without the marker build.yml greps for') - def test_membrowse_upload_sees_the_same_board_as_the_build(self): - # $EX_ARGS is passed for the BOARD it selects: --one-first picks a board that can - # build the -e set, so without it membrowse configures a different, empty build - # dir and uploads --identical for a board that was never compiled. It does NOT - # scope the targets - `examples-membrowse-upload` is not `all`, so it passes - # through as the aggregate, which has no DEPENDS and still records every example. + def _run_extras_block(self, sel): + """Extract the build-extras shell block from build.yml and run it for real. + Nothing else exercises it, which is why the empty/rejected conflation shipped.""" + import re as _re, shlex, subprocess, tempfile, json as _json + repo = os.path.dirname(CIRCLECI) + i = self.build.index("EXAMPLE_MAP='{}'\n BUILD_FILTERED='false'") + i = self.build.rindex('\n', 0, i) + 1 + j = self.build.index(' echo "matrix=$MATRIX_JSON"', i) + block = _re.sub(r'^ {10}', '', self.build[i:j], flags=_re.M) + with tempfile.TemporaryDirectory() as d: + selp = os.path.join(d, 'sel.json') + with open(selp, 'w') as fh: + _json.dump(sel, fh) + matrix = subprocess.run( + [sys.executable, os.path.join(repo, '.github/scripts/ci_set_matrix.py'), + '--select-file', selp], capture_output=True, text=True, cwd=repo).stdout.strip() + self.assertTrue(matrix, 'ci_set_matrix produced nothing') + sh = os.path.join(d, 'probe.sh') + with open(sh, 'w') as fh: + # shlex.quote, not hand-rolled quoting: a TMPDIR with a space in it + # made this fail for a reason that had nothing to do with the block + fh.write('BUILD_SELECT_FILE=' + shlex.quote(selp) + '\n') + fh.write('MATRIX_JSON=' + shlex.quote(matrix) + '\n') + fh.write(block) + # sentinel + newline separated: the block itself writes ::warning:: to + # stdout, and '|' would collide with the regex's own separator + fh.write('\nprintf "@@R@@\\n%s\\n%s\\n%s" "$MATRIX_JSON" "$BUILD_FILTERED" "$FAMILY_REGEX"\n') + r = subprocess.run(['bash', sh], capture_output=True, text=True, cwd=repo) + self.assertEqual(r.returncode, 0, r.stderr) + mj, filtered, regex = r.stdout.split('@@R@@\n', 1)[1].split('\n', 2) + return sum(len(v) for v in _json.loads(mj).values()), filtered, regex + + def test_an_empty_family_list_is_not_treated_as_unusable(self): + """.build.families is read twice - as a count and as a `|`-joined regex. An EMPTY + list and one REJECTED by the charset guard both leave the regex empty and mean + opposite things, so the block has to branch on which happened. + + Testing `-z "$FAMILY_REGEX"` alone sent every nothing-selected PR down the + fall-open path and discarded the correct all-empty matrix: #3842 (docs + + .gitignore) and #3840 (test/hil only) each rebuilt all 74 cmake legs after the + selector had correctly chosen none.""" + legs, filtered, regex = self._run_extras_block( + {'build': {'full': False, 'families': [], 'family_examples': {}}}) + self.assertEqual(legs, 0, 'an empty families list must keep the all-empty matrix') + self.assertEqual(filtered, 'false', 'nothing was built, so nothing to compare') + self.assertEqual(regex, '') + + def test_a_real_family_list_stays_scoped(self): + legs, filtered, regex = self._run_extras_block( + {'build': {'full': False, 'families': ['stm32f4', 'rp2040'], + 'family_examples': {}}}) + self.assertGreater(legs, 0) + self.assertEqual(filtered, 'true') + self.assertEqual(regex, 'stm32f4|rp2040') + + def test_a_regex_metacharacter_in_a_family_name_falls_open(self): + # the name is interpolated raw into a name_is_regexp artifact pattern, so a + # metacharacter would match another family's baseline - reject and widen + legs, filtered, regex = self._run_extras_block( + {'build': {'full': False, 'families': ['stm32f4.*'], 'family_examples': {}}}) + self.assertGreater(legs, 100, 'a rejected family list must fall open to full') + self.assertEqual(filtered, 'false') + self.assertEqual(regex, '') + + def test_membrowse_upload_is_not_scoped_by_the_pr_filter(self): + # by decision, the upload runs unfiltered so the size history stays keyed on the + # family's preferred board whatever the PR touched. $EX_ARGS would not have + # scoped the targets either way - `examples-membrowse-upload` is not `all`, so + # resolve_example_target_groups passes it through as the aggregate - but it DID + # move the board, because --one-first picks one that can build the -e set. + # + # The accepted cost: on a family whose preferred board cannot build that set, + # the upload lands on a board the Build step never compiled and every example + # goes up --identical. test_the_upload_board_can_diverge_from_the_built_board + # keeps that consequence measured rather than assumed. line = [l for l in self.util.splitlines() if '--target examples-membrowse-upload' in l][0] - self.assertIn('$EX_ARGS', line) - self.assertNotIn('-e ', line.replace('$EX_ARGS', '')) + self.assertNotIn('$EX_ARGS', line) + self.assertNotIn('-e ', line) + + def test_the_upload_board_can_diverge_from_the_built_board(self): + """Pins the SIZE of what the removal gave up, so it cannot grow unnoticed. + + --one-first with no -e returns preferred_list[0]; with one it returns the first + preferred board that can build it. Where those differ, the Membrowse Upload step + configures a build dir the Build step never wrote. + + ci=True unconditionally, as _prune_buildable does and for the same reason: the + answer must be the runner's, not the developer's. The CI skip lists are off by + default locally, which moves the pick on three families - this test asserted the + local set and went red on its first CI run.""" + sys.path.insert(0, os.path.join(REPO, 'tools')) + import build as build_py + roles = ('device', 'host', 'dual') + exs = sorted(f'{r}/{n}' for r in roles + for n in os.listdir(os.path.join(REPO, 'examples', r)) + if os.path.isdir(os.path.join(REPO, 'examples', r, n))) + fams = sorted(d for d in os.listdir(os.path.join(REPO, 'hw/bsp')) + if os.path.isdir(os.path.join(REPO, 'hw/bsp', d, 'boards'))) + cwd = os.getcwd() + os.chdir(REPO) + try: + diverging = set() + for fam in fams: + try: + base = build_py.get_family_boards(fam, False, True, None, 'cmake', + (), ci=True) + except Exception: + continue + if not base: + continue + for e in exs: + try: + one = build_py.get_family_boards(fam, False, True, [e], 'cmake', + (), ci=True) + except Exception: + continue + if one and one[0] != base[0]: + diverging.add(fam) + break + finally: + os.chdir(cwd) + self.assertEqual(diverging, {'imxrt', 'lpc11', 'lpc18', 'lpc54', 'mcx', 'rx', + 'samd11', 'samd2x_l2x', 'samd5x_e5x', 'stm32l0', + 'stm32l4', 'tm4c'}, + 'the set of families whose membrowse upload can land on an ' + 'uncompiled board changed; re-check whether dropping $EX_ARGS ' + 'from the upload step is still the right trade') if __name__ == '__main__': diff --git a/test/hil/test/test_ci_select.py b/test/hil/test/test_ci_select.py index a19392bde..22fbde17b 100644 --- a/test/hil/test/test_ci_select.py +++ b/test/hil/test/test_ci_select.py @@ -302,7 +302,11 @@ class TestArgsEmission(unittest.TestCase): out = j.loads(r.stdout) self.assertFalse(out['full']) self.assertIn('tinyusb.json', out['args']) - self.assertTrue(any('cdc_device' in line for line in out['reasons'])) + # reasons are a stderr diagnostic, deliberately NOT in the payload: they were + # 97% of a 9.8 MB JSON on a dep bump, and every consumer re-parses that file + self.assertNotIn('reasons', out, 'reasons must not ride in the machine-read JSON') + self.assertNotIn('reasons', out['build']) + self.assertIn('cdc_device', r.stderr) # A core-class diff must select boards THROUGH THE CLI: the in-process tests # inject their own repo root, so only this subprocess path catches a broken # repo_root derivation -- which once made every repo-relative glob match @@ -495,6 +499,8 @@ class TestPortAndCoreRoleUseExtras(unittest.TestCase): self.assertFalse(s['full']) for board in boards: tests = s['boards'][board] + if tests == 'all': + continue # a board whose whole allowed set is selected collapses to 'all' self.assertIn('device/hid_composite_freertos', tests) self.assertIn('device/cdc_msc_freertos', tests) self.assertIn('device/audio_test_freertos', tests) @@ -506,6 +512,8 @@ class TestPortAndCoreRoleUseExtras(unittest.TestCase): self.assertFalse(s['full']) for board in boards: tests = s['boards'][board] + if tests == 'all': + continue # a board whose whole allowed set is selected collapses to 'all' self.assertIn('device/hid_composite_freertos', tests) self.assertIn('device/cdc_msc_freertos', tests) self.assertIn('device/audio_test_freertos', tests) @@ -938,6 +946,232 @@ class TestClassesWithNoEnablingExample(unittest.TestCase): 'both axes, so nothing compiles it until the next master push') +class TestTheHarnessTestsAreNotTheHarness(unittest.TestCase): + """test/hil/test/ selects nothing; test/hil/ itself still selects everything. + + Rule 2 is a bare `test/hil/` prefix, so the harness's own unit tests were booking + the full 27-board rig - ~11 minutes of exclusive hardware for a diff that cannot + reach it. Nothing on the rig runs them: pre-commit does, and build.yml runs + test_ci_select.py as the gate before trusting a selection at all. + + The carve-out is only safe while that directory holds nothing rig-affecting, which + is what the second test pins.""" + + def test_the_harness_own_tests_select_nothing_on_either_axis(self): + for p in ('test/hil/test/test_ci_select.py', 'test/hil/test/test_ci_metrics.py', + 'test/hil/test/test_hil_bounded.py', 'test/hil/test/stubs/pymtp.py', + 'test/hil/test/stubs/hid.py'): + s = ci_select.classify([p], REPO, ROSTERS) + self.assertFalse(s['full'], p) + self.assertFalse(s['boards'], p) + b = ci_select.classify_build([p], REPO) + self.assertFalse(b['full'], p) + self.assertFalse(b['families'], p) + + def test_the_harness_itself_still_takes_the_whole_rig(self): + # the thing rule 2 exists for: these decide what the rig does, so they cannot be + # trusted to narrow their own blast radius + for p in ('test/hil/hil_test.py', 'test/hil/tinyusb.json', + 'test/hil/helper/hil_ci_set_matrix.py'): + s = ci_select.classify([p], REPO, ROSTERS) + self.assertTrue(s['full'], f'{p} must still force the full rig') + + def test_nothing_rig_affecting_has_moved_into_the_carve_out(self): + """The carve-out is a claim about that directory's contents; pin them. + + A new file there that the rig DOES read would silently stop selecting the rig. + Listing them costs one line per file and makes that a failing test instead.""" + out = subprocess.run(['git', 'ls-files', 'test/hil/test'], cwd=REPO, + capture_output=True, text=True, check=True) + self.assertEqual(sorted(out.stdout.split()), [ + 'test/hil/test/stubs/hid.py', + 'test/hil/test/stubs/pymtp.py', + 'test/hil/test/test_ci_metrics.py', + 'test/hil/test/test_ci_select.py', + 'test/hil/test/test_hil_bounded.py', + 'test/hil/test/test_hil_health.py', + 'test/hil/test/test_hil_report.py', + 'test/hil/test/test_hil_rtt.py', + 'test/hil/test/test_hil_util.py', + ], 'test/hil/test/ gained or lost a file; it is carved out of rule 2, so confirm ' + 'the rig still does not read anything in there before updating this list') + + +class TestExampleMapOmitsFullFamilies(unittest.TestCase): + """A family whose selection is ALREADY everything it can build carries no -e list. + + Sixth of the same shape as the class below, found the same way: a perf rewrite of + _prune_buildable dropped the `set(kept) != set(buildable)` test and all 216 tests + stayed green. The build outcome is identical either way -- build.py applies the same + skip_example the pruner just did -- so nothing compiled differently and only the + payload grew (22 families x 33 examples on one dcd_dwc2.c diff). That is exactly the + kind of drift no build failure ever reports.""" + + def test_a_device_only_port_diff_still_omits_families_it_cannot_narrow(self): + # dcd_dwc2.c selects device+dual examples only, but a family whose host examples + # are all unbuildable anyway ends up wanting its entire buildable set + b = ci_select.classify_build(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO) + self.assertFalse(b['full']) + self.assertTrue(b['families']) + omitted = [f for f in b['families'] if f not in b['family_examples']] + self.assertTrue(omitted, 'no family omitted its -e list; the "already everything ' + 'this family builds" case stopped being detected') + for fam in omitted: + self.assertNotIn(fam, b['family_examples']) + + def test_a_family_that_can_build_more_than_the_diff_wants_keeps_its_list(self): + # the other direction: one example selects itself and nothing else, so every + # family it lands on must carry an explicit -e or CI builds all 46 + b = ci_select.classify_build(['examples/device/cdc_msc/src/main.c'], REPO) + self.assertFalse(b['full']) + for fam in b['families']: + self.assertEqual(b['family_examples'].get(fam), ['device/cdc_msc'], fam) + + +class TestSelectionBehavioursThatHadNoTest(unittest.TestCase): + """Five behaviours a reviewer's mutation pass proved were unpinned: break each one + and the whole suite stayed green. Each test here fails against its mutant. + + They are grouped because they share a shape - every one is a small expression whose + removal silently NARROWS the selection, which is the failure direction that merges a + regression rather than wasting a runner.""" + + def test_build_defines_reach_the_prefilter(self): + # mutant: `defines = ()` in build.py's build_boards_list. metro_m4_express gets + # MAX3421_HOST=1 from its roster variant, never from its BSP, so without the + # defines the -e prefilter drops the rig's only MAX3421 firmware and hil-tinyusb + # has nothing to flash. + import build as build_py, build_utils, inspect + src = inspect.getsource(build_py.build_boards_list) + self.assertIn('defines = tuple(sorted(build_defines))', src, + 'the -D tokens must reach cmake_board/skip_example') + old = os.getcwd() + os.chdir(REPO) + try: + ex, board = 'dual/host_info_to_device_cdc', 'metro_m4_express' + self.assertTrue(build_utils.skip_example(ex, board), + 'without the define this example is correctly skipped') + self.assertFalse(build_utils.skip_example(ex, board, ('MAX3421_HOST=1',)), + 'with it, it must build - that is what the roster passes') + finally: + os.chdir(old) + + def test_one_first_prefers_a_board_that_can_build_the_filter(self): + # mutant: buildable() -> True, i.e. back to all_boards[0]. lpc54's first board + # skips every msc_file_explorer example, so the leg would compile nothing. + import build as build_py + old_env, old = os.environ.get('GITHUB_ACTIONS'), os.getcwd() + os.environ['GITHUB_ACTIONS'] = 'true' + os.chdir(REPO) + try: + unfiltered = build_py.get_family_boards('lpc54', False, True) + filtered = build_py.get_family_boards('lpc54', False, True, + ['host/msc_file_explorer']) + self.assertEqual(unfiltered, ['lpcxpresso54114'], 'unfiltered pick must not move') + self.assertNotEqual(filtered, unfiltered, + 'the -e pick must avoid a board that skips the whole filter') + import build_utils + self.assertFalse(build_utils.skip_example('host/msc_file_explorer', filtered[0]), + f'{filtered[0]} must actually build the filtered example') + finally: + os.chdir(old) + if old_env is None: + os.environ.pop('GITHUB_ACTIONS', None) + else: + os.environ['GITHUB_ACTIONS'] = old_env + + def test_a_class_file_selects_its_own_macro_not_just_the_directory(self): + # mutant: delete the _CLS_STEM_RE block. src/class/midi holds MIDI 1.0 AND 2.0; + # examples/device/midi2_device is the only example enabling CFG_TUD_MIDI2 and the + # only one that compiles midi2_device.c, but the directory macro alone misses it. + got = ci_select._build_class_examples('midi', 'midi2_device.c', {'device'}, REPO) + self.assertIn('device/midi2_device', got, + 'a midi2 change must select the example that compiles it') + host = ci_select._build_class_examples('midi', 'midi2_host.c', {'host'}, REPO) + self.assertIn('host/midi2_host', host) + # and the plain midi files must NOT drag midi2 in + plain = ci_select._build_class_examples('midi', 'midi_device.c', {'device'}, REPO) + self.assertNotIn('device/midi2_device', plain) + + def test_a_port_change_selects_the_dual_examples(self): + # mutant: drop `+ ('dual',)`. A dcd/hcd change must build the dual examples - + # they exercise both stacks on one board, so a dwc2 break lands there first. + s = ci_select.classify_build(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO) + duals = {e for exs in s['family_examples'].values() for e in exs + if e.startswith('dual/')} + self.assertTrue(duals, 'a dcd change selected no dual example') + + def test_the_selector_answers_the_same_with_and_without_ci_env(self): + # mutant: drop ci=True from _prune_buildable. ci_skip_boards/ci_preferred_boards + # only apply when GITHUB_ACTIONS/CIRCLECI is set, so without the pin a laptop and + # a runner disagree - and /pre-pr would report a family list CI will not build. + files = ['examples/host/cdc_msc_hid_freertos/src/main.c'] + old = os.environ.get('GITHUB_ACTIONS') + os.environ.pop('GITHUB_ACTIONS', None) + try: + local = ci_select.classify_build(files, REPO)['families'] + os.environ['GITHUB_ACTIONS'] = 'true' + import importlib + importlib.reload(ci_select) + runner = ci_select.classify_build(files, REPO)['families'] + finally: + if old is None: + os.environ.pop('GITHUB_ACTIONS', None) + else: + os.environ['GITHUB_ACTIONS'] = old + import importlib + importlib.reload(ci_select) + self.assertEqual(local, runner, 'the selector must not depend on the CI env vars') + + +class TestRuleTableIsCarbonOfTheSpec(unittest.TestCase): + """ci_select's module docstring carries the rule table so a reader landing in the + code does not have to open the spec to learn what rule 6 is. Both are maintained by + hand, so this pins them cell-for-cell: edit one without the other and this fails. + + It also pins the table against the CODE - every rule id the docstring claims must + appear as a `# rule N` marker on a branch of _classify_build_one, so a row cannot be + documented without a branch, or a branch renumbered without the table.""" + + @staticmethod + def _rows(text): + import re as _re + out = [] + for l in text.splitlines(): + if not l.startswith('| '): + continue + c = [x.strip() for x in l.strip().strip('|').split('|')] + if len(c) == 5 and _re.fullmatch(r'\d+[a-z]?', c[0]): + out.append(c) + return out + + def test_docstring_table_matches_the_spec(self): + spec = open(os.path.join( + REPO, 'docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md')).read() + doc, spec_rows = self._rows(ci_select.__doc__), self._rows(spec) + self.assertTrue(spec_rows, 'no rule table found in the spec') + self.assertEqual([r[0] for r in doc], [r[0] for r in spec_rows], + 'rule ids differ between ci_select.__doc__ and the spec') + for d, s in zip(doc, spec_rows): + self.assertEqual(d, s, f'rule {d[0]} differs between the docstring and the spec') + + def test_every_documented_rule_has_a_branch(self): + import re as _re + src = open(os.path.join(REPO, 'tools/ci_select.py')).read() + marked = set() + # handles `# rule 6`, `# rules 1, 1b` and `# rules 8-10` + for m in _re.finditer(r'#\s*rules?\s+([0-9a-z, -]+)', src): + for tok in _re.split(r',\s*', m.group(1).strip()): + rng = _re.fullmatch(r'(\d+)\s*-\s*(\d+)', tok.strip()) + if rng: + marked.update(str(n) for n in range(int(rng.group(1)), int(rng.group(2)) + 1)) + elif _re.fullmatch(r'\d+[a-z]?', tok.strip()): + marked.add(tok.strip()) + documented = {r[0] for r in self._rows(ci_select.__doc__)} + missing = sorted(documented - marked, key=lambda s: (int(_re.match(r'\d+', s).group()), s)) + self.assertEqual(missing, [], f'documented rules with no `# rule N` branch marker: {missing}') + + class TestNoTrackedFileIsUnclassified(unittest.TestCase): """Rule 17 (unclassified -> full on both axes) is the fail-open net for paths nobody anticipated. It must stay that way - a wrong `full` costs runner minutes and is @@ -1477,10 +1711,17 @@ class TestBuildPostFilter(unittest.TestCase): # the accepted net for a break outside its #if guard). src/class/bth is the # live instance of this state today; TestClassesWithNoEnablingExample pins the # whole set, so a new one cannot appear unnoticed. - s = ci_select.classify_build(['src/class/vendor/vendor_host.c'], REPO) + # src/class/bth/bth_device.c, a file that EXISTS: the old assertion named + # src/class/vendor/vendor_host.c, deleted by the same branch, so any made-up + # path reached the same branch and the test passed vacuously. + real = os.path.join(REPO, 'src/class/bth/bth_device.c') + self.assertTrue(os.path.isfile(real), 'the case needs a file that exists') + s = ci_select.classify_build(['src/class/bth/bth_device.c'], REPO) self.assertFalse(s['full']) self.assertEqual(s['families'], []) self.assertTrue(any('no contribution' in r for r in s['reasons']), s['reasons']) + # and the reason must name the class, not just any empty answer + self.assertTrue(any('bth' in r for r in s['reasons']), s['reasons']) def test_class_source_with_examples_still_scopes(self): s = ci_select.classify_build(['src/class/cdc/cdc_device.c'], REPO) @@ -2040,14 +2281,14 @@ class TestMcuTokensResolve(unittest.TestCase): # produce, or a rename nobody followed through. `family:samd21` was one of these # until the nine examples/host/*/only.txt files were corrected to samd2x_l2x. # - # The `mcu:` entries are NOT all harmless. MIMXRT10XX/MIMXRT11XX and LPC177X_8X sit - # beside a live token in the same file, so they gate nothing either way. MKL25ZXX - # (device/msc_dual_lun) and SAME5X (device/audio_test) do not: those skips are dead, - # and both examples are built today on the boards their skip file meant to exclude - - # successfully, which is why nobody noticed. Correcting them REMOVES working build - # coverage, so it is a maintainer call, not a drive-by fix. + # The remaining `mcu:` entries sit beside a live token in the same file, so they gate + # nothing either way. MKL25ZXX (7 files) and SAME5X (1) were dead too, but unlike + # these they were the ONLY token for their board - the examples were already being + # built on the very boards those lines meant to exclude. Dropping them is a no-op for + # the build (verified per example) and was chosen over re-pointing, which would have + # removed working coverage. UNREACHABLE_TOKENS = { - 'mcu': {'LPC177X_8X', 'MIMXRT10XX', 'MIMXRT11XX', 'MKL25ZXX', 'SAME5X', 'STM32U3'}, + 'mcu': {'LPC177X_8X', 'MIMXRT10XX', 'MIMXRT11XX', 'STM32U3'}, 'family': set(), 'board': set(), } diff --git a/test/hil/test/test_hil_bounded.py b/test/hil/test/test_hil_bounded.py index c6d454f0e..c30c58cbd 100644 --- a/test/hil/test/test_hil_bounded.py +++ b/test/hil/test/test_hil_bounded.py @@ -46,6 +46,17 @@ def write_script(path: Path, body: str) -> None: path.chmod(path.stat().st_mode | stat.S_IEXEC) +def no_settle(case): + """Zero test_device_usbtest's post-flash settle for one test. + + Real hardware needs it -- the enumeration can bounce once after a flash, and on + dual-port parts the stale same-serial node lingers. A fake rig has neither, and ten + tests drive that path, so leaving it real cost 30s of every suite run. + """ + case.addCleanup(setattr, hil_test, 'USBTEST_SETTLE', hil_test.USBTEST_SETTLE) + hil_test.USBTEST_SETTLE = 0 + + def run_bounded(fn, timeout: float): """Run fn in a daemon thread; return (finished, exception). A still-running thread is the hang under test — leave it to die with the interpreter.""" @@ -63,7 +74,6 @@ def run_bounded(fn, timeout: float): return not t.is_alive(), exc[0] if exc else None [email protected](os.name == 'nt', 'POSIX shell fakes') class ReadDiskFile(unittest.TestCase): def setUp(self): self.tmp = TemporaryDirectory() @@ -77,7 +87,7 @@ class ReadDiskFile(unittest.TestCase): for name in ('get_disk_dev', '_enum_timeout', 'MTYPE_TIMEOUT'): self.addCleanup(setattr, hil_test, name, getattr(hil_test, name)) hil_test.get_disk_dev = lambda uid, vendor, lun: str(self.dev) - hil_test._enum_timeout = 2 + hil_test._enum_timeout = 1 # the wait these tests must outlast; keep it small self.bin = tmp / 'bin' self.bin.mkdir() self.addCleanup(os.environ.__setitem__, 'PATH', os.environ['PATH']) @@ -112,7 +122,11 @@ class ReadDiskFile(unittest.TestCase): t0 = time.monotonic() with self.assertRaises(AssertionError) as cm: hil_test.read_disk_file('uid0', 0, 'README.TXT') - self.assertLess(time.monotonic() - t0, 1.5) + # BELOW one full _enum_timeout wait, not above it: "fails immediately" is the + # claim, and a bound of 1.5 against a 1s budget passes for code that spun the + # whole budget -- which is the regression this test exists to catch. + self.assertLess(time.monotonic() - t0, hil_test._enum_timeout, + 'read_disk_file spun the enumeration budget on a real answer') self.assertIn('README.TXT', str(cm.exception)) def test_hung_mtype_cannot_hang_the_worker(self): @@ -136,37 +150,60 @@ class CompactOutput(unittest.TestCase): class UsbtestRecovery(unittest.TestCase): def test_recovery_flags_and_flash_bound_fit_the_reserve(self): """The post-hang reflash plumbing: the CLI flags exist, and the bounded reflash - plus the fixed recovery costs (60s case timeout + 5s kill wait + 5s settle) - fits inside USBTEST_RECOVERY_BUDGET -- otherwise the outer run_cmd kill lands - mid-flash and orphans the flasher (own session) on the probe.""" + and the reserve that pays for them is derived per flasher (see the two tests + below), not pinned.""" import subprocess hil_dir = Path(TEST_DIR).parents[0] r = subprocess.run([sys.executable, str(hil_dir / 'usbtest.py'), '--help'], capture_output=True, text=True, timeout=30) self.assertEqual(r.returncode, 0, r.stderr) - for flag in ('--recover-board', '--recover-fw', '--outer-timeout'): + for flag in ('--recover-board', '--recover-fw'): self.assertIn(flag, r.stdout) - def test_the_bounded_reflash_actually_fits_the_reserve(self): - """The arithmetic the docstring above claims but never checked -- the two - constants never met in any test, so bumping either silently broke the promise. - Overrun means run_cmd's outer kill lands MID-FLASH and orphans the flasher - (start_new_session, so killpg misses it) holding the probe.""" - import re + def test_the_reserve_covers_every_step_of_its_own_ladder(self): + """Enumerated from the SIDE EFFECTS usbtest performs, so dropping a step from + recovery_reserve() fails here. Overrun means run_cmd's outer kill lands MID-FLASH + and orphans the flasher (start_new_session, so killpg misses it) on the probe. + """ import usbtest - hil_dir = Path(TEST_DIR).parents[0] - # read the case timeout hil_test actually passes, so this cannot drift silently - src = (hil_dir / 'hil_test.py').read_text() - m = re.search(r'--timeout (\d+) --budget', src) - self.assertIsNotNone(m, 'usbtest invocation changed shape; re-derive this bound') - case_timeout = int(m.group(1)) - kill_wait, settle, time_left_reserve = 5, 5, 35 # usbtest.py's fixed costs - worst = (case_timeout + kill_wait + usbtest.RECOVER_FLASH_TIMEOUT - + settle + time_left_reserve) - self.assertLessEqual( - worst, hil_test.USBTEST_RECOVERY_BUDGET, - f'a HUNG case needs {worst}s to recover but only ' - f'{hil_test.USBTEST_RECOVERY_BUDGET}s is reserved') + from helper import hil_util as _hu + # each bounded step costs its timeout PLUS run_cmd's post-SIGKILL reap + flash = usbtest.RECOVER_FLASH_TIMEOUT + _hu.REAP_GRACE + reset = usbtest.RECOVER_RESET_TIMEOUT + _hu.REAP_GRACE + fixed = 2 * usbtest.RECOVER_SETTLE + usbtest.RECOVER_OVERHEAD + rp = {'name': 'openocd', 'args': '-f target/rp2040.cfg'} + for flasher, steps in ( + # an RP openocd board: reset, reflash, then Rescue-DP POR + one retry + (rp, reset + flash + 2 * flash + fixed), + # openocd on a NON-RP target: rescue_openocd has no RESCUE_CFG entry for + # it, so its two legs are time the board can never spend + ({'name': 'openocd', 'args': '-f target/wch-riscv.cfg'}, + reset + flash + fixed), + # esptool: reset_esptool is a stub (no_op) and rescue refuses a + # non-openocd flasher, so ONE reflash is all it can ever spend + ({'name': 'esptool', 'args': ''}, flash + fixed)): + self.assertEqual(usbtest.recovery_reserve(flasher), steps, + f'{flasher} reserves time it cannot spend, or too little') + + def test_the_reserve_leaves_room_for_the_work_no_step_bounds(self): + """The ladder's step timeouts do not cover the two /proc walks, the roster + json.loads, the child's first import, or the JSON print. With zero margin any + env-overridable bound moving up puts the outer killpg inside the reflash.""" + import usbtest + self.assertGreater(usbtest.RECOVER_OVERHEAD, 0) + rp = {'name': 'openocd', 'args': '-f target/rp2350.cfg'} + bounded = (usbtest.RECOVER_RESET_TIMEOUT + 3 * usbtest.RECOVER_FLASH_TIMEOUT) + self.assertGreaterEqual(usbtest.recovery_reserve(rp) - bounded, + usbtest.RECOVER_OVERHEAD, + 'the reserve equals its own worst case with no margin') + + def test_a_flasher_reserves_nothing_for_a_rescue_it_cannot_run(self): + """rescue_openocd returns False for anything but openocd, so reserving its two + legs elsewhere holds a pool worker AND a usbtest permit for 200s of dead time.""" + import usbtest + self.assertLess(usbtest.recovery_reserve({'name': 'esptool', 'args': ''}), + usbtest.recovery_reserve({'name': 'openocd', + 'args': '-f target/rp2040.cfg'})) class UsbtestRunHelper(unittest.TestCase): @@ -313,7 +350,7 @@ class _MtpFakeRig: os.environ['PYTHONSAFEPATH'] = '1' for name in ('_enum_timeout', 'MTP_SESSION_MARGIN'): self.addCleanup(setattr, hil_test, name, getattr(hil_test, name)) - hil_test._enum_timeout = 2 + hil_test._enum_timeout = 1 # the wait these tests must outlast; keep it small # the session scratch files land in cwd self.addCleanup(os.chdir, os.getcwd()) os.chdir(tmp) @@ -326,7 +363,6 @@ class _MtpFakeRig: os.environ[k] = v [email protected](os.name == 'nt', 'POSIX shell fakes') @unittest.skipIf(sys.version_info < (3, 11), 'fake-pymtp steering needs PYTHONSAFEPATH') class DeviceMtp(_MtpFakeRig, unittest.TestCase): """test_device_mtp end to end: the real mtp_test.py subprocess under run_cmd, @@ -400,188 +436,9 @@ class ConvoySafeFlasher(unittest.TestCase): self.assertFalse(self.f(flasher)) -class BoundedOpen(unittest.TestCase): - """hil_util.bounded_open must return rather than block, and must not leak the fd if - the open completes after we gave up (usblp_open takes the device mutex before it - consults O_NONBLOCK, so a wedged node blocks the open uninterruptibly).""" - - def setUp(self): - from helper import hil_util - self.hil_util = hil_util - self.tmp = TemporaryDirectory() - self.addCleanup(self.tmp.cleanup) - # bounded_open counts its stranded threads now, and the counter is process-global - # with no decrement: three wedged-FIFO tests here reach SYSFS_STUCK_MAX and every - # later test in this file reads SYSFS_UNKNOWN for perfectly good attributes - self.addCleanup(setattr, hil_util, '_sysfs_stuck', hil_util._sysfs_stuck) - - def test_opens_a_normal_file(self): - f = Path(self.tmp.name) / 'plain' - f.write_text('x') - fd = self.hil_util.bounded_open(str(f), os.O_RDONLY, 5) - self.assertIsNotNone(fd) - os.close(fd) - - def test_missing_path_returns_none_without_raising(self): - self.assertIsNone(self.hil_util.bounded_open( - str(Path(self.tmp.name) / 'nope'), os.O_RDONLY, 5)) - - @unittest.skipIf(os.name == 'nt', 'POSIX fifo') - def test_blocking_open_gives_up_and_does_not_leak_fds(self): - """A reader-less FIFO blocks open(O_WRONLY) forever -- the closest portable - stand-in for a wedged usblp node.""" - fifo = Path(self.tmp.name) / 'fifo' - os.mkfifo(fifo) - before = len(os.listdir('/proc/self/fd')) - t0 = time.monotonic() - for _ in range(5): - self.assertIs(self.hil_util.bounded_open(str(fifo), os.O_WRONLY, 0.2), - self.hil_util.SYSFS_UNKNOWN) - self.assertLess(time.monotonic() - t0, 10, 'bounded_open did not bound') - self.assertLessEqual(len(os.listdir('/proc/self/fd')) - before, 1, - 'bounded_open leaked fds on the blocking path') - - @unittest.skipIf(os.name == 'nt', 'POSIX fifo') - def test_open_completing_during_the_abandon_does_not_leak(self): - """The window the handoff lock exists for: the worker is at its store-or-close - decision when the caller gives up and drains the box. - - The `abandoned` Event is instrumented to park the worker there, because timing - alone never reaches that window -- 1500 tries against the unlocked version leaked - nothing, so a test that merely completes the open late proves nothing. An empty - `hit` means the instrumentation no longer bites and the window is untested.""" - hil_util = self.hil_util - fifo = Path(self.tmp.name) / 'fifo' - os.mkfifo(fifo) - caller = threading.current_thread() - drained, hit = threading.Event(), [] - - class RacingEvent(threading.Event): - def is_set(self): - v = super().is_set() - if not v and not hit and threading.current_thread() is not caller: - hit.append(True) - # bounded: the fixed bounded_open holds the lock across this call, so - # the caller cannot reach its abandon (and set drained) until we return - drained.wait(0.3) - return v - - shim = types.ModuleType('threading_shim') - shim.__dict__.update(threading.__dict__) - shim.Event = RacingEvent - hil_util.threading = shim - self.addCleanup(setattr, hil_util, 'threading', threading) - - before = len(os.listdir('/proc/self/fd')) - rd = os.open(fifo, os.O_RDONLY | os.O_NONBLOCK) # the O_WRONLY open completes at once - try: - self.assertIs(hil_util.bounded_open(str(fifo), os.O_WRONLY, 0.05), - hil_util.SYSFS_UNKNOWN) - drained.set() - time.sleep(0.1) # let an abandoned worker act on what it saw - self.assertTrue(hit, 'the abandon window was never entered') - self.assertLessEqual(len(os.listdir('/proc/self/fd')) - before, 1, - 'bounded_open stored the fd after the caller drained the box') - finally: - drained.set() - os.close(rd) - - -class SysfsUnknownIsNotAbsent(unittest.TestCase): - """read_sysfs must tell "no such attribute" (a fact) from "the read did not answer" - (not a fact). Every caller that concluded absence from the latter reported a healthy - board as a firmware regression.""" - - def setUp(self): - from helper import hil_util - self.hil_util = hil_util - self.saved = (hil_util._sysfs_stuck, hil_util._sysfs_blind_logged) - self.tmp = TemporaryDirectory() - self.addCleanup(self.tmp.cleanup) - - def tearDown(self): - # a blocked read strands a counted daemon thread; leaving the count raised would - # blind every later test in this process - self.hil_util._sysfs_stuck, self.hil_util._sysfs_blind_logged = self.saved - - def test_readable_attribute_returns_its_value(self): - p = Path(self.tmp.name) / 'serial' - p.write_text('CAFE01\n') - self.assertEqual(self.hil_util.read_sysfs(str(p)), 'CAFE01') - - def test_missing_attribute_is_none(self): - self.assertIsNone(self.hil_util.read_sysfs(str(Path(self.tmp.name) / 'nope'))) - - @unittest.skipIf(os.name == 'nt', 'POSIX fifo') - def test_blocking_read_is_unknown_not_absent(self): - """A reader-less FIFO stands in for the wedged device whose sysfs read never - returns; None here would read as "the board is gone".""" - fifo = Path(self.tmp.name) / 'fifo' - os.mkfifo(fifo) - t0 = time.monotonic() - v = self.hil_util.read_sysfs(str(fifo), grace=0.3) - self.assertLess(time.monotonic() - t0, 10, 'read_sysfs did not bound') - self.assertIs(v, self.hil_util.SYSFS_UNKNOWN) - self.assertIsNotNone(v) - - def test_blind_process_answers_unknown_for_a_readable_attribute(self): - p = Path(self.tmp.name) / 'serial' - p.write_text('CAFE01') - self.hil_util._sysfs_stuck = self.hil_util.SYSFS_STUCK_MAX - self.assertTrue(self.hil_util.sysfs_blind()) - self.assertIs(self.hil_util.read_sysfs(str(p)), self.hil_util.SYSFS_UNKNOWN) - self.assertIn('blind', self.hil_util.sysfs_blind_note()) - - def test_unknown_is_falsy_but_not_none(self): - # call sites use `(v or '')` idioms; the sentinel must keep working there while - # still being distinguishable from a real absence - self.assertFalse(self.hil_util.SYSFS_UNKNOWN) - self.assertIsNotNone(self.hil_util.SYSFS_UNKNOWN) - - -class UsbtestEnumerationVerdict(unittest.TestCase): - """test_device_usbtest must not report a healthy board as "no cafe:4010 device" just - because its own sysfs reads stopped answering.""" - - def setUp(self): - from helper import hil_util - self.hil_util = hil_util - self.td = TemporaryDirectory() - self.addCleanup(self.td.cleanup) - # a real device dir: usb_scan reads idVendor/idProduct with a plain open (they are - # lock-free descriptor fields), and only `serial` through the bounded reader - dev = Path(self.td.name) / '1-2' - dev.mkdir() - (dev / 'idVendor').write_text('cafe\n') - (dev / 'idProduct').write_text('4010\n') - (dev / 'serial').write_text('CAFE01\n') - for obj, name, val in ((hil_util, 'read_sysfs', hil_util.read_sysfs), - (hil_util, 'glob', hil_util.glob), - (hil_util, '_sysfs_stranded', {}), - (hil_test, '_enum_timeout', 1)): - self.addCleanup(setattr, obj, name, getattr(obj, name)) - setattr(obj, name, val) - hil_util.glob = types.SimpleNamespace(glob=lambda pat: [str(dev)]) - - def _fail(self, reader): - self.hil_util.read_sysfs = reader - with self.assertRaises(hil_test.TestFail) as cm: - hil_test.test_device_usbtest({'uid': 'CAFE01', 'name': 'fake', 'flasher': {}}) - return str(cm.exception) - - def test_unknown_reads_do_not_claim_the_device_is_absent(self): - msg = self._fail(lambda p, *a, **kw: self.hil_util.SYSFS_UNKNOWN) - self.assertNotIn('no cafe:4010 device', msg) - self.assertIn('did not answer', msg) - - def test_a_readable_bus_without_the_device_still_says_absent(self): - msg = self._fail(lambda p, *a, **kw: 'OTHERUID') - self.assertIn('no cafe:4010 device', msg) - - class UnresolvedControllerBucket(unittest.TestCase): """An unresolved controller must budget in ONE bucket. Taking a permit on every slot - serialized the whole fleet the moment a worker went blind.""" + serialized the whole fleet the moment a single board could not be resolved.""" def setUp(self): import threading @@ -641,8 +498,7 @@ class ThroughputPayloadBound(unittest.TestCase): payload actually requested.""" def test_only_a_read_high_speed_gets_the_big_payload(self): - from helper import hil_util - for speed in (None, hil_util.SYSFS_UNKNOWN, '12', '1.5'): + for speed in (None, '12', '1.5'): self.assertTrue(hil_test.link_is_fs(speed), f'{speed!r} must scale as FS') for speed in ('480', '5000', '10000'): self.assertFalse(hil_test.link_is_fs(speed)) @@ -699,61 +555,6 @@ class FindDeviceCache(unittest.TestCase): self.assertEqual(self.usbtest.find_device('BBBB')['sysname'], '1-3') -class EnumPollDoesNotReReadAWedgedPath(unittest.TestCase): - """usbtest_enumerated re-globs every device each 0.2 s pass. One wedged peer therefore - strands a fresh bounded reader thread per pass, and SYSFS_STUCK_MAX=4 of those blind - the WHOLE worker for the rest of the run -- measured at 8 s of polling. A path that - already stranded is known-unknown; reading it again buys nothing and costs the - blindness budget.""" - - def test_a_stranded_path_is_read_at_most_once(self): - from contextlib import contextmanager - from helper import hil_lock, hil_util - - td = TemporaryDirectory() - self.addCleanup(td.cleanup) - # A REAL device dir: usb_scan reads idVendor/idProduct with a plain open and - # `continue`s on OSError, so a bare FIFO is skipped before the bounded read is ever - # reached -- this test passed identically with the memo deleted until the ids were - # added. The FIFO must be the `serial` of a device that survives the cheap filter. - devdir = Path(td.name) / '1-2' - devdir.mkdir() - (devdir / 'idVendor').write_text('cafe\n') - (devdir / 'idProduct').write_text('4010\n') - wedged = devdir / 'serial' - os.mkfifo(wedged) # open() blocks forever: no writer, ever - - def patch(obj, name, value): - self.addCleanup(setattr, obj, name, getattr(obj, name)) - setattr(obj, name, value) - - def _permit(uid): - yield - - from helper import hil_util as _hu2 - patch(_hu2, 'glob', types.SimpleNamespace(glob=lambda p: [str(devdir)])) - patch(_hu2, '_sysfs_stranded', {}) - patch(hil_lock, 'usbtest_permit', contextmanager(_permit)) - # Long enough for several 2 s reads, but under the blindness cap -- past the cap - # sysfs_blind() short-circuits reads on its own and would mask the memo entirely. - patch(hil_test, '_enum_timeout', 8) - # the blindness counter is process-global and never decrements: restore it or this - # test blinds every test that runs after it - patch(hil_util, '_sysfs_stuck', hil_util._sysfs_stuck) - - # Count LEAKED THREADS, not _sysfs_stuck: a strand is booked only the first time a - # path is seen, so the counter is deduped by the memo's own bookkeeping and stays 1 - # even when the memo is broken. Each re-read blocks a fresh thread on the FIFO - # forever and leaks its fd -- which is the cost the memo exists to avoid, and the - # only thing here that actually moves when it regresses. - before = threading.active_count() - with self.assertRaises(hil_test.TestFail): # never enumerates, by construction - hil_test.test_device_usbtest({'name': 'b', 'uid': 'UID1', - 'flasher': {'name': 'openocd'}}) - self.assertLessEqual(threading.active_count() - before, 1, - 'the poll re-read a path it already knew was stranded') - - class ReRunSpecNamesOnlyWhatFailed(unittest.TestCase): """The pool-guard path used to leave this unwritten -- and a fresh run has already unlinked it -- so build.yml's re-run step found nothing and GitHub re-tested all ~26 @@ -806,45 +607,6 @@ class WedgedPidsFailsClosed(unittest.TestCase): self.assertFalse(complete, 'a hidden holder was reported as absent') [email protected](os.name == 'nt', 'POSIX shell fakes') [email protected](sys.version_info < (3, 11), 'fake-pymtp steering needs PYTHONSAFEPATH') -class StrandMemoRemembersUnstattablePaths(unittest.TestCase): - """A stranded path whose inode could not be read is stored as None -- which dict.get() - also returns for a MISS. Testing `is not None` therefore treats 'known stranded' as - 'never seen', and every later call strands ANOTHER permanent thread and fd on a path we - already know is wedged. That is the exact unbounded growth SYSFS_STUCK_MAX exists to - stop, and it is invisible: `first = path not in _sysfs_stranded` is False, so the - blindness counter does not advance either.""" - - def setUp(self): - from helper import hil_util - self.hil_util = hil_util - self.addCleanup(hil_util._sysfs_stranded.clear) - hil_util._sysfs_stranded.clear() - self.addCleanup(setattr, hil_util, '_sysfs_stuck', hil_util._sysfs_stuck) - hil_util._sysfs_stuck = 0 - self.td = TemporaryDirectory(); self.addCleanup(self.td.cleanup) - self.fifo = os.path.join(self.td.name, 'serial') - os.mkfifo(self.fifo) # open() succeeds, read() never returns - - def test_an_unstattable_strand_is_not_re_read(self): - self.hil_util._sysfs_stranded[self.fifo] = None # as the record path stores it - before = threading.active_count() - self.assertIs(self.hil_util.read_sysfs(self.fifo, grace=0.5), - self.hil_util.SYSFS_UNKNOWN) - self.assertEqual(threading.active_count(), before, - 'a known-stranded path was re-read, stranding another thread') - - def test_a_live_strand_is_still_re_read_when_the_node_is_replaced(self): - """The memo must not become permanent blindness: a NEW inode at the same path is a - different device and has to be read.""" - self.hil_util._sysfs_stranded[self.fifo] = 999999999 # inode that is not this one - with open(os.path.join(self.td.name, 'other'), 'w') as f: - f.write('ok\n') - os.replace(os.path.join(self.td.name, 'other'), self.fifo) - self.assertEqual(self.hil_util.read_sysfs(self.fifo, grace=0.5), 'ok') - - class MtpGioOrdering(_MtpFakeRig, unittest.TestCase): """gio must not run until the device is READY. @@ -899,16 +661,16 @@ class MtpGioFallthrough(unittest.TestCase): t0 = time.monotonic() r = subprocess.run([sys.executable, str(Path(TEST_DIR).parents[0] / 'mtp_test.py'), - '--uid', 'CAFE01', '--timeout', '3'], + '--uid', 'CAFE01', '--timeout', '1'], capture_output=True, text=True, timeout=60, env=env) elapsed = time.monotonic() - t0 - self.assertLess(elapsed, 30, f'did not honour --timeout 3 ({elapsed:.1f}s)') + self.assertLess(elapsed, 30, f'did not honour --timeout 1 ({elapsed:.1f}s)') self.assertNotEqual(r.returncode, 0) # The assertions above are satisfied by an immediate CRASH, which is exactly what # shipped through this test once: `pass` left gio unbound and the next line # dereferenced it. Assert the behaviour the docstring names -- it POLLED for the # device (so it spent its budget) and did not die on a traceback. - self.assertGreater(elapsed, 2.0, + self.assertGreater(elapsed, 0.8, f'exited without polling ({elapsed:.1f}s) -- it crashed') self.assertNotIn('Traceback', r.stderr) self.assertIn('MTP device not found', r.stdout + r.stderr) @@ -936,11 +698,17 @@ class RunWhileContract(unittest.TestCase): def boom(): raise AssertionError('x') + # a duration no other process would plausibly pick: `pgrep -f` searches the WHOLE + # machine, so a bare `sleep 20` matched an unrelated background job -- another + # agent session's retry loop, in the case that exposed this -- and failed a test + # about our own child. Observed failing 3/3 in isolation while that loop ran. + sentinel = '20.0451' with self.assertRaises(AssertionError): - self.hil_util.run_alongside(['sleep', '20'], boom, 1) + self.hil_util.run_alongside(['sleep', sentinel], boom, 1) # nothing of ours is left running: the reap ran on the error path too import subprocess - out = subprocess.run(['pgrep', '-f', '^sleep 20'], capture_output=True, text=True) + out = subprocess.run(['pgrep', '-f', f'^sleep {sentinel}'], + capture_output=True, text=True) seen['strays'] = [p for p in out.stdout.split() if p] self.assertEqual(seen['strays'], [], 'work() raising leaked the child') @@ -966,55 +734,6 @@ class RunWhileContract(unittest.TestCase): self.assertNotEqual(pgids['child'], os.getpgid(0)) -class StrandedPathMemoInvalidates(unittest.TestCase): - """The memo lives in read_sysfs, so every bounded reader gets it -- call-site memos - meant each new scanner had to remember (get_printer_dev and the throughput probe did - not). And it MUST expire on re-enumeration: the key is a bus path, which does not - change when a device comes back on the same port, so a memo that never invalidates - makes a board the branch's own HUNG reflash just recovered permanently invisible.""" - - def setUp(self): - from helper import hil_util - self.hil_util = hil_util - self.td = TemporaryDirectory() - self.addCleanup(self.td.cleanup) - for name in ('_sysfs_stranded', '_sysfs_stuck'): - self.addCleanup(setattr, hil_util, name, getattr(hil_util, name)) - hil_util._sysfs_stranded = {} - hil_util._sysfs_stuck = 0 - - def test_a_stranded_path_is_not_re_read(self): - f = Path(self.td.name) / 'serial' - os.mkfifo(f) # never answers - self.assertIs(self.hil_util.read_sysfs(str(f), 0.3), self.hil_util.SYSFS_UNKNOWN) - after_first = self.hil_util._sysfs_stuck - t0 = time.monotonic() - for _ in range(3): - self.assertIs(self.hil_util.read_sysfs(str(f), 0.3), - self.hil_util.SYSFS_UNKNOWN) - self.assertLess(time.monotonic() - t0, 0.3, 'the memo did not short-circuit') - self.assertEqual(self.hil_util._sysfs_stuck, after_first, - 'repeat reads spent more of the blindness budget') - - def test_re_enumeration_clears_it(self): - """A new device on the same busport gets a fresh sysfs node, hence a fresh inode. - Without this the memo outlives the wedge it recorded.""" - f = Path(self.td.name) / 'serial' - os.mkfifo(f) - self.assertIs(self.hil_util.read_sysfs(str(f), 0.3), self.hil_util.SYSFS_UNKNOWN) - f.unlink() - f.write_text('CAFE01\n') # same path, new inode = re-enumerated - self.assertEqual(self.hil_util.read_sysfs(str(f), 0.3), 'CAFE01', - 'a recovered device stayed invisible') - - def test_a_vanished_path_is_not_remembered_as_stranded(self): - f = Path(self.td.name) / 'serial' - os.mkfifo(f) - self.assertIs(self.hil_util.read_sysfs(str(f), 0.3), self.hil_util.SYSFS_UNKNOWN) - f.unlink() - self.assertIsNone(self.hil_util.read_sysfs(str(f), 0.3)) - - class UsbScanIsTheOneWalk(unittest.TestCase): """Three call sites each had a different subset of the three things this must get right; none had all three. The expensive read is `serial` -- served under the device @@ -1035,19 +754,13 @@ class UsbScanIsTheOneWalk(unittest.TestCase): return real(path, *a, **k) self.addCleanup(setattr, hil_util, 'read_sysfs', real) hil_util.read_sysfs = counting - self.addCleanup(setattr, hil_util, '_sysfs_stranded', - dict(hil_util._sysfs_stranded)) - self.addCleanup(setattr, hil_util, '_sysfs_stuck', hil_util._sysfs_stuck) - def _dev(self, name, vid, pid, serial='S1', fifo=False): + def _dev(self, name, vid, pid, serial='S1'): d = self.root / name d.mkdir() (d / 'idVendor').write_text(vid + '\n') (d / 'idProduct').write_text(pid + '\n') - if fifo: - os.mkfifo(d / 'serial') # a read that never answers - else: - (d / 'serial').write_text(serial + '\n') + (d / 'serial').write_text(serial + '\n') return d def _scan(self, **kw): @@ -1058,85 +771,16 @@ class UsbScanIsTheOneWalk(unittest.TestCase): return self.hil_util.usb_scan(**kw) def test_a_mismatched_vid_pid_costs_no_serial_read(self): + """`serial` is the ONE attribute here served under the device lock, so it is the + one that can block on a wedged device. Filtering on the lock-free descriptor pair + first is what keeps a scan for our board off every other board's locked read.""" self._dev('1-1', '1234', '5678') self._dev('1-2', 'cafe', '4010', serial='UID1') - devs, unknown = self._scan(vid_pid=('cafe', '4010')) + devs = self._scan(vid_pid=('cafe', '4010')) self.assertEqual([d['serial'] for d in devs], ['UID1']) - self.assertFalse(unknown) # the ruled-out device's locked attribute was never touched self.assertNotIn(str(self.root / '1-1' / 'serial'), self.reads) - def test_a_wedged_device_stays_unproven_on_every_scan(self): - """The memo lives in read_sysfs now, so usb_scan still CALLS it each pass -- what - must not repeat is the cost. StrandedPathMemoInvalidates covers the short-circuit; - here the invariant is that the device stays out of the results and absence stays - unproven, however many times we look.""" - from helper import hil_util - self._dev('1-1', 'cafe', '4010', fifo=True) - first = None - t0 = time.monotonic() - for _ in range(3): - devs, unknown = self._scan() - self.assertTrue(unknown, 'a stranded read must leave absence unproven') - self.assertEqual(devs, []) - if first is None: - first = hil_util._sysfs_stuck - self.assertEqual(hil_util._sysfs_stuck, first, - 'repeat scans spent more of the blindness budget') - self.assertLess(time.monotonic() - t0, 3.0, 'repeat scans re-paid the grace') - - -class BoundedOpenTellsAbsentFromUnknown(unittest.TestCase): - """Same contract as read_sysfs, in the sibling function of the same file: a real - OSError is a FACT (EBUSY, ENOENT, EACCES), a blocked open is UNKNOWN. Folding both - into None made an ordinary EBUSY report as a USB wedge, sending the operator to - usb-kernel-recover for healthy hardware -- and left the stranded thread uncounted, - so the cap that exists to stop the fd/thread ceiling never saw it.""" - - def setUp(self): - from helper import hil_util - self.hil_util = hil_util - self.td = TemporaryDirectory() - self.addCleanup(self.td.cleanup) - - def test_a_real_oserror_is_a_fact(self): - missing = str(Path(self.td.name) / 'nope') - self.assertIsNone(self.hil_util.bounded_open(missing, os.O_RDONLY, 1)) - - def test_a_blocked_open_is_unknown_and_counted(self): - fifo = Path(self.td.name) / 'fifo' - os.mkfifo(fifo) # no reader: O_WRONLY blocks forever - self.addCleanup(setattr, self.hil_util, '_sysfs_stuck', - self.hil_util._sysfs_stuck) - before = self.hil_util._sysfs_stuck - got = self.hil_util.bounded_open(str(fifo), os.O_WRONLY, 0.3) - self.assertIs(got, self.hil_util.SYSFS_UNKNOWN) - self.assertEqual(self.hil_util._sysfs_stuck, before + 1, - 'a stranded open is invisible to the blindness budget') - - -class UsbtestSysfsReadIsCapped(unittest.TestCase): - """find_device re-scans every cafe:4010 peer after EVERY case, so the local twin -- - which had no SYSFS_STUCK_MAX -- stranded a thread and an fd per wedged peer per case. - Delegating to hil_util gets the cap, and the deferred import keeps usbtest.py - importable standalone.""" - - def test_a_stranded_read_counts_against_the_shared_cap(self): - import usbtest - from helper import hil_util - td = TemporaryDirectory() - self.addCleanup(td.cleanup) - wedged = Path(td.name) / 'serial' - os.mkfifo(wedged) # no writer: open() never returns - self.addCleanup(setattr, hil_util, '_sysfs_stuck', hil_util._sysfs_stuck) - before = hil_util._sysfs_stuck - # UNKNOWN, not None: folding them made a blinded scan read as "device dropped - # off the bus", which aborts past the HUNG reflash - self.assertIs(usbtest._read_sysfs_bounded(wedged, grace=0.5), - hil_util.SYSFS_UNKNOWN) - self.assertEqual(hil_util._sysfs_stuck, before + 1, - 'usbtest reads are invisible to the blindness budget') - class AbandonExitSurvivesAFailedFork(unittest.TestCase): """Pool() forks, and after a convoy -- every stranded read holding a thread and an fd -- @@ -1146,10 +790,16 @@ class AbandonExitSurvivesAFailedFork(unittest.TestCase): def test_none_pool_and_manager_still_write_the_banner(self): # a subprocess, because _abandon_exit ends in os._exit: in-process it would take # the test runner with it, before any assertion could run + import json import subprocess with TemporaryDirectory() as td: - report = Path(td) / 'hil_report.md' - report.write_text('| board | test |\n|---|---|\n', encoding='utf-8') + rd = Path(td) + # it takes the report DIRECTORY now and re-renders both artifacts from the + # sidecar, so seed the sidecar -- the markdown is output, not input + (rd / 'hil_report.json').write_text(json.dumps( + {'rows': [{'board': 'boardA', 'cells': {'cdc_msc': 'pass'}, + 'duration': '1s'}], + 'banner': '', 'scope': '', 'caveat': ''})) src = ( 'import sys, types\n' f'sys.path.insert(0, {str(Path(TEST_DIR).parents[0])!r})\n' @@ -1160,12 +810,14 @@ class AbandonExitSurvivesAFailedFork(unittest.TestCase): 'sys.modules.setdefault("serial", st)\n' 'import hil_test\n' f'hil_test._abandon_exit(None, None, True, 1, __import__("pathlib")' - f'.Path({str(report)!r}))\n') + f'.Path({str(rd)!r}))\n') r = subprocess.run([sys.executable, '-c', src], capture_output=True, text=True, timeout=120) self.assertEqual(r.returncode, 1, r.stderr) - self.assertTrue(report.read_text().startswith('**HIL run abandoned'), - 'the abandon banner never reached the report') + self.assertTrue((rd / 'hil_report.md').read_text().startswith( + '**HIL run abandoned'), 'the abandon banner never reached the report') + self.assertIn('abandoned', + json.loads((rd / 'hil_report.json').read_text())['caveat']) def test_kill_pool_children_tolerates_a_pool_that_never_existed(self): from helper import hil_health @@ -1174,11 +826,9 @@ class AbandonExitSurvivesAFailedFork(unittest.TestCase): class UsbtestOuterBoundIsOneValue(unittest.TestCase): - """The bound usbtest is TOLD and the bound run_cmd ENFORCES must be the same number. - Three separate expressions disagreed: --skip-flash appended no --outer-timeout at all - (usbtest reads 0 as no limit), and the no-recovery branch narrowed only the CHILD's - view while run_cmd still waited for a recovery reserve nothing on that path can - spend -- a pool worker and its battery permit idle for the difference.""" + """run_cmd's kill is the ONE bound, and it must carry a recovery reserve only when a + recovery can actually run. Otherwise a board on a path that cannot recover holds a pool + worker and its battery permit idle for the difference, under a usbtest width of 2.""" def _invoke(self, flasher, skip_flash=False): from contextlib import contextmanager @@ -1207,10 +857,7 @@ class UsbtestOuterBoundIsOneValue(unittest.TestCase): from helper import hil_util as _hu patch(_hu, 'glob', types.SimpleNamespace(glob=lambda p: [str(dev)])) - # the blindness latch and the stranded memo are process-global: another class's - # wedged-FIFO test would otherwise make every read here answer SYSFS_UNKNOWN - patch(_hu, '_sysfs_stuck', 0) - patch(_hu, '_sysfs_stranded', {}) + patch(hil_test, 'USBTEST_SETTLE', 0) # see no_settle patch(hil_lock, 'usbtest_permit', contextmanager(_permit)) patch(hil_test, 'skip_flash', skip_flash) patch(hil_test, '_current_fw', '/tmp/fw.elf') @@ -1219,24 +866,30 @@ class UsbtestOuterBoundIsOneValue(unittest.TestCase): hil_test.test_device_usbtest({'name': 'b', 'uid': 'UID1', 'flasher': flasher}) return seen - def _outer_flag(self, cmd): - toks = cmd.split() - self.assertIn('--outer-timeout', toks, 'usbtest reads a missing bound as UNLIMITED') - return int(toks[toks.index('--outer-timeout') + 1]) - def test_a_recoverable_board_reserves_the_recovery_budget(self): - seen = self._invoke({'name': 'openocd', 'vid_pid': '0x1366 0x1024'}) - want = hil_test.USBTEST_BATTERY_BUDGET + hil_test.USBTEST_RECOVERY_BUDGET - self.assertEqual(self._outer_flag(seen['cmd']), want) + import usbtest + flasher = {'name': 'openocd', 'vid_pid': '0x1366 0x1024', + 'args': '-f target/rp2040.cfg'} + seen = self._invoke(flasher) + want = (hil_test.USBTEST_BATTERY_BUDGET + hil_test.USBTEST_OVERSHOOT + + usbtest.recovery_reserve(flasher)) self.assertEqual(seen['timeout'], want) + def test_the_reserve_follows_the_board_not_a_fleet_constant(self): + """Two convoy-safe openocd boards, one RP and one not: the non-RP board cannot + run rescue_openocd, so reserving its two legs holds a pool worker and a usbtest + permit for 200s of dead time.""" + rp = self._invoke({'name': 'openocd', 'vid_pid': '0x1366 0x1024', + 'args': '-f target/rp2040.cfg'}) + wch = self._invoke({'name': 'openocd', 'vid_pid': '0x1366 0x1024', + 'args': '-f target/wch-riscv.cfg'}) + self.assertLess(wch['timeout'], rp['timeout']) + def test_a_board_with_no_recovery_does_not_pay_for_one(self): seen = self._invoke({'name': 'stlink', 'uid': 'X'}) # never convoy_safe - outer = self._outer_flag(seen['cmd']) - self.assertEqual(seen['timeout'], outer, 'the two bounds disagree') - # It does not carry the RECOVERY reserve it cannot spend... - self.assertLess(outer, hil_test.USBTEST_BATTERY_BUDGET - + hil_test.USBTEST_RECOVERY_BUDGET) + # It does not carry the RECOVERY reserve it cannot spend + self.assertEqual(seen['timeout'], + hil_test.USBTEST_BATTERY_BUDGET + hil_test.USBTEST_OVERSHOOT) # ...but it MUST still exceed the child's own --budget. The battery checks the # budget before dispatching, so it can overshoot by one already-started case; an # equal bound SIGKILLs it just as it goes to print, turning ~29 real per-case @@ -1244,12 +897,15 @@ class UsbtestOuterBoundIsOneValue(unittest.TestCase): toks = seen['cmd'].split() budget = int(toks[toks.index('--budget') + 1]) case_timeout = int(toks[toks.index('--timeout') + 1]) - self.assertGreaterEqual(outer - budget, case_timeout, + self.assertGreaterEqual(seen['timeout'] - budget, case_timeout, 'the outer kill can land mid-case, before the JSON') def test_skip_flash_still_bounds_the_child(self): + """--skip-flash disables recovery, so the child must not be given a reserve it + cannot spend -- but it MUST still be bounded.""" seen = self._invoke({'name': 'openocd', 'vid_pid': '0x1366 0x1024'}, skip_flash=True) - self.assertEqual(self._outer_flag(seen['cmd']), seen['timeout']) + self.assertEqual(seen['timeout'], + hil_test.USBTEST_BATTERY_BUDGET + hil_test.USBTEST_OVERSHOOT) class UsbtestRetryPolicy(unittest.TestCase): @@ -1318,6 +974,7 @@ class UsbtestOuterKillStaysRetryable(unittest.TestCase): from helper import hil_util as _hu patch(_hu, 'glob', types.SimpleNamespace(glob=lambda p: [str(dev)])) + patch(hil_test, 'USBTEST_SETTLE', 0) # see no_settle def _permit(uid): # a real generator: a lambda returning an iterator has yield # no .throw(), so any raise inside the `with` would # surface as an AttributeError from contextlib instead @@ -1647,178 +1304,6 @@ class StagingCoversEveryBoardForm(unittest.TestCase): "last run's re-run spec survived a green run") -class SummaryFoldsReportToBoards(unittest.TestCase): - """hil_summary.py replaces the agent retyping the markdown table. Report rows are named per - VARIANT and a variant need not start with the board name, so the config is what maps them - back -- the previous string-matching design produced a defect in each of four review rounds.""" - - def _sum(self, boards, rows, cfg_boards=None, banner=''): - import json - import subprocess - td = TemporaryDirectory() - self.addCleanup(td.cleanup) - d = Path(td.name) - (d / 'hil_report.json').write_text(json.dumps( - {'rows': [{'board': b, 'cells': c, 'duration': '1s'} for b, c in rows], - 'banner': banner})) - cfg = d / 'cfg.json' - cfg.write_text(json.dumps({'boards': cfg_boards or [{'name': b} for b in boards]})) - args = [a for b in boards for a in ('-b', b)] - r = subprocess.run(['python3', str(Path(TEST_DIR).parents[0] / 'helper' / 'hil_summary.py'), - str(cfg), *args, '--report-dir', str(d)], - capture_output=True, text=True, timeout=60) - self.assertEqual(r.returncode, 0, r.stderr) - return json.loads(r.stdout)['results'] - - def test_variant_rows_fold_onto_their_board(self): - """nanoch32v203 never produces a row named after the board.""" - got = self._sum(['nanoch32v203'], - [('nanoch32v203-fsdev', {'usbtest': 'pass'}), - ('nanoch32v203-usbfs', {'usbtest': 'pass'})], - cfg_boards=[{'name': 'nanoch32v203', - 'variant': [{'name': 'nanoch32v203-fsdev'}, - {'name': 'nanoch32v203-usbfs'}]}]) - self.assertEqual([r['board'] for r in got], ['nanoch32v203']) - self.assertTrue(got[0]['pass']) - self.assertTrue(got[0]['ran']) - - def test_one_failing_variant_fails_the_board(self): - got = self._sum(['nano'], - [('nano-a', {'usbtest': 'pass'}), ('nano-b', {'usbtest': '❌ 29/30'})], - cfg_boards=[{'name': 'nano', 'variant': [{'name': 'nano-a'}, - {'name': 'nano-b'}]}]) - self.assertFalse(got[0]['pass']) - self.assertIn('29/30', got[0]['detail']) - - def test_lock_contention_is_a_field_not_a_prefix(self): - got = self._sum(['alpha'], [('alpha', {'board-locked': 'fail'})]) - self.assertTrue(got[0]['locked']) - self.assertFalse(got[0]['pass']) - - def test_a_board_with_no_row_is_marked_not_run(self): - got = self._sum(['alpha', 'beta'], [('alpha', {'usbtest': 'pass'})]) - self.assertTrue(got[0]['ran']) - self.assertFalse(got[1]['ran']) - self.assertFalse(got[1]['pass']) - - def test_a_metric_cell_counts_by_its_icon(self): - got = self._sum(['a', 'b'], [('a', {'cdc_msc_throughput': '✅ C 1.2 M 3.4'}), - ('b', {'cdc_msc_throughput': '❌ C 0.0 M 0.0'})]) - self.assertTrue(got[0]['pass']) - self.assertFalse(got[1]['pass']) - - def test_skipped_cells_do_not_fail_a_board(self): - got = self._sum(['a'], [('a', {'usbtest': 'skip', 'cdc_msc': 'pass'})]) - self.assertTrue(got[0]['pass']) - - def test_a_plain_metric_cell_is_a_pass(self): - """Mirrors hil_test.py's own tally (cell_kind): failures are ALWAYS marked -- 'fail' - or a ❌ prefix, per TestFail's docstring -- while a passing test may return a plain - metric string that lands in the cell unprefixed. Treating unknown shapes as fail - would publish a green table as a red verdict.""" - got = self._sum(['a'], [('a', {'device_speed': '480.0 MBps'})]) - self.assertTrue(got[0]['pass']) - - def test_a_declared_variant_of_another_board_is_not_stolen(self): - """A declared variant need not start with its own board's name, so it may start with - a DIFFERENT board's name plus '-'. The prefix fallback must not attribute it twice.""" - got = self._sum(['alpha', 'beta'], - [('beta-x', {'usbtest': 'fail'})], - cfg_boards=[{'name': 'alpha', 'variant': [{'name': 'beta-x'}]}, - {'name': 'beta'}]) - self.assertTrue(got[0]['ran']) - self.assertFalse(got[0]['pass']) - self.assertFalse(got[1]['ran'], "beta must not inherit alpha's row") - - -class CaveatSurvivesAccumulate(unittest.TestCase): - """CI reruns with --accumulate: the sidecar keeps every earlier attempt's cells, but the - banner was recomputed per attempt. A first attempt on a degraded rig and a clean rerun - therefore published the degraded attempt's PASSES with no caveat on them -- and the - generated .failed spec reruns only failures, so those cells are never re-earned.""" - - def _rows(self, board, cell): - return [(board, 0, 0, [(board, {cell: 'OK'}, '1s')], 0)] - - def test_an_earlier_attempts_caveat_is_still_on_the_report(self): - td = TemporaryDirectory() - self.addCleanup(td.cleanup) - rd = Path(td.name) - banner = '> **Rig note.** 2 process(es) in D state at start.\n' - - hil_test.accumulate_report(self._rows('boardA', 'cdc_msc'), rd, True, '', banner) - self.assertIn('Rig note', (rd / hil_test.REPORT_MD).read_text()) - - # the rerun: clean rig, so this attempt contributes no banner of its own - md = hil_test.accumulate_report(self._rows('boardB', 'cdc_msc'), rd, False, '', '') - self.assertIn('boardA', md) # the earlier cells are kept ... - self.assertIn('Rig note', md, - 'the caveat the earlier cells were collected under was dropped') - - def test_the_same_caveat_twice_is_not_stacked(self): - td = TemporaryDirectory() - self.addCleanup(td.cleanup) - rd = Path(td.name) - banner = '> **Rig note.** 2 process(es) in D state at start.\n' - hil_test.accumulate_report(self._rows('boardA', 'cdc_msc'), rd, True, '', banner) - md = hil_test.accumulate_report(self._rows('boardB', 'cdc_msc'), rd, False, '', banner) - self.assertEqual(md.count('Rig note'), 1) - - -class BlindWorkerReachesTheReport(unittest.TestCase): - """A worker that exhausts its bounded-read budget answers SYSFS_UNKNOWN for every - attribute, so its "device not found" means "could not tell". That reached the log and - the per-cell failure text but NOT the table -- and the table is what gets pasted into - the PR. Seen live: run 31794359407 went blind in 4 workers and published 26 red cells - with no mention of it, several of them caused by the blindness rather than the board.""" - - def test_no_note_when_every_worker_could_see(self): - mret = [('boardA', 0, [], [], 1.0, False), ('boardB', 0, [], [], 1.0, False)] - self.assertEqual(hil_test._blind_note(mret), '') - - def test_the_note_names_the_boards_whose_verdicts_are_not_evidence(self): - mret = [('boardA', 0, [], [], 1.0, True), ('boardB', 0, [], [], 1.0, False), - ('boardC', 1, [], [], 1.0, True)] - note = hil_test._blind_note(mret) - self.assertIn('boardA', note) - self.assertIn('boardC', note) - self.assertNotIn('boardB', note) # it could see; do not smear its result - self.assertTrue(note.endswith('\n'), 'banners are line-oriented') - - def test_both_row_widths_survive_the_report_writers(self): - """The blindness flag widened the worker's result tuple to 6, but the pool-timeout - path still synthesises 5-field rows for boards that never reported and feeds them - to the same two writers. A fixed-width unpack in either one raises INSIDE the - containment path, which is where a raise costs every board's results.""" - td = TemporaryDirectory() - self.addCleanup(td.cleanup) - rd = Path(td.name) - wide = ('boardA', 1, ['device/cdc_msc'], [('boardA', {'cdc_msc': '❌'}, '2s')], 2.0, True) - narrow = ('stuck', 1, [], None, 0) # what the timeout path builds - hil_test._write_failed_spec(rd / 'x.failed', rd, [wide, narrow]) - md = hil_test.accumulate_report([wide], rd, True, '', hil_test._blind_note([wide])) - self.assertIn('boardA', md) - self.assertIn('not all verdicts are evidence', md.lower()) - - def test_the_stray_note_names_the_board_and_survives_narrow_rows(self): - """Survivors ride back on the result tuple because main()'s own sweep runs after - the report is written on both abort paths -- the banner appended there was - computed and discarded.""" - wide = ('boardA', 0, [], [], 1.0, False, 2) - clean = ('boardB', 0, [], [], 1.0, False, 0) - note = hil_test._stray_note([wide, clean]) - self.assertIn('boardA', note) - self.assertNotIn('boardB', note) - self.assertIn('2', note) - self.assertEqual(hil_test._stray_note([clean]), '') - self.assertEqual(hil_test._stray_note([('stuck', 1, [], None, 0)]), '') - - def test_the_timeout_paths_synthetic_rows_do_not_crash_it(self): - """The pool-timeout path builds (name, 1, [], None, 0) for boards that never - reported -- five fields, no blindness to report -- and hands those around.""" - self.assertEqual(hil_test._blind_note([('stuck', 1, [], None, 0)]), '') - - class PoolGuardKeepsWhatFinished(unittest.TestCase): """The guard's 30-minute predecessor fired on 5 of the last 8 HIL jobs, so this is the common failure, not an edge case: map_async discarded every board that had finished and @@ -1923,13 +1408,14 @@ class WedgedBoardCosts(unittest.TestCase): class WedgeVerdictReachesTheLatch(unittest.TestCase): """usbtest computes `unrecovered_hang` but never reported it, so hil_test inferred the latch from `not recovery and 'HUNG' in out` and missed three cases: recovery ran and - FAILED (convoy-safe boards -- max32666fthr HUNG in the 08-14 run), the `inconclusive` + FAILED (convoy-safe boards -- max32666fthr HUNG in the 08-14 run), the `ambiguous` abort (which sets the flag but leaves no case at status HUNG), and an unparsable JSON, which is the outer-timeout kill and the case where a wedge is most likely.""" def setUp(self): self.addCleanup(setattr, hil_test, 'board_wedged', hil_test.board_wedged) hil_test.board_wedged = '' + no_settle(self) def _run(self, stdout, rc=0): from helper import hil_lock, hil_util @@ -1972,12 +1458,13 @@ class WedgedBoardCannotReportAPass(unittest.TestCase): battery that still wedged returned `PASS 30/30`. That board then contributes 0 to err_count, is omitted from the .failed re-run spec (which keys on err > 0), and the job exits 0 with a D-state holder on the rig -- the exact silence this branch exists to end. - usbtest's `inconclusive` and `ambiguous` aborts fire AFTER the last case, so nothing + usbtest's `ambiguous` abort fires AFTER the last case, so nothing back-fills a BUDGET entry to make failed/notrun non-zero.""" def setUp(self): self.addCleanup(setattr, hil_test, 'board_wedged', hil_test.board_wedged) hil_test.board_wedged = '' + no_settle(self) def _cell(self, js): """Returns ('pass', cell) or ('fail', message).""" @@ -2014,5 +1501,321 @@ class WedgedBoardCannotReportAPass(unittest.TestCase): self.assertIn('30/30', cell) +def _gil_stall_available() -> bool: + """Whether the hid stub can simulate a GIL-HOLDING stall on this host. + + It needs a libc with sleep(3) loaded through ctypes.PyDLL. Everywhere the HIL harness + actually runs that is present; where it is not, the two tests that depend on it skip + rather than fail, because their subject is the bound, not ctypes. + """ + import ctypes + import ctypes.util + try: + ctypes.PyDLL(ctypes.util.find_library('c') or 'libc.so.6') + return True + except OSError: + return False + + +class HidEchoRunsInAChild(unittest.TestCase): + """hidapi's blocking calls hold the GIL -- cython-hidapi wraps hid_enumerate in + `with nogil` but calls hid_open and hid_close bare -- so a daemon thread cannot bound + them: the waiter parks off-GIL but must reacquire the GIL to return, which the stuck + thread never yields. Only a child process can be killed regardless, which is what + run_cmd's killpg does.""" + + def _run(self, mode, uid='CAFE01', budget='0', timeout=20, pid=None): + saved = {k: os.environ.get(k) for k in ('FAKE_HID_MODE', 'FAKE_HID_UID', + 'FAKE_HID_PID', 'PYTHONPATH', + 'PYTHONSAFEPATH')} + + def restore(): + for k, v in saved.items(): + os.environ.pop(k, None) if v is None else os.environ.__setitem__(k, v) + self.addCleanup(restore) + os.environ['FAKE_HID_MODE'] = mode + os.environ['FAKE_HID_UID'] = uid + stubs = os.path.join(TEST_DIR, 'stubs') + pp = saved['PYTHONPATH'] + os.environ['PYTHONPATH'] = stubs if not pp else f'{stubs}:{pp}' + # `python3 -c` puts the cwd at sys.path[0], AHEAD of PYTHONPATH, so any hid.py + # reachable from the suite's cwd would displace the stub and every mode-driven + # test below would pass or fail for the wrong reason. Safe-path mode drops it -- + # the same practice _MtpFakeRig documents. + os.environ['PYTHONSAFEPATH'] = '1' + from helper import hil_util + want = pid or f'{hil_test.HID_INOUT_PID:#06x}' + return hil_util.run_cmd( + [sys.executable, '-c', hil_test.HID_ECHO, uid, budget, want], + timeout=timeout, split_stderr=True, quiet=True) + + def _stderr(self, r): + from helper import hil_util + return hil_util.cmd_stdout_text(r.stderr) + + def test_a_healthy_device_passes(self): + r = self._run('ok') + self.assertEqual(r.returncode, 0, self._stderr(r)) + + def test_the_pid_matches_the_example(self): + """The walk filters on BOTH ids, and hidapi applies them before the locked + manufacturer/product reads. Six examples in this tree expose a HID interface under + VID cafe, so a stale PID here silently widens the walk back to all of them -- and + nothing else would fail. Pinned against the descriptor rather than restated.""" + import re + src = (Path(TEST_DIR).parents[2] + / 'examples/device/hid_generic_inout/src/usb_descriptors.c').read_text() + m = re.search(r'#define\s+USB_PID\s+(0x[0-9a-fA-F]+)', src) + self.assertIsNotNone(m, 'hid_generic_inout no longer defines USB_PID') + self.assertEqual(hil_test.HID_INOUT_PID, int(m.group(1), 16), + 'HID_INOUT_PID drifted from the example descriptor') + + def test_a_peer_running_another_example_is_filtered_out(self): + """The point of the PID filter: a wedged sibling on a different example never + reaches the locked reads at all.""" + r = self._run('ok', pid='0x400f') # hid_composite, not ours + self.assertNotEqual(r.returncode, 0) + self.assertIn('HID device not found', self._stderr(r)) + + @unittest.skipUnless(_gil_stall_available(), 'no libc for a GIL-holding stall') + def test_a_gil_holding_stall_is_still_killed(self): + """THE case an in-process bound cannot cover. hid_open is not `with nogil`, so a + thread-based guard is inert there; the child is killed anyway.""" + t0 = time.monotonic() + r = self._run('wedged_open_gil', timeout=2) + self.assertEqual(r.returncode, 124, + 'a GIL-holding hidapi stall must still be killed on the bound') + self.assertLess(time.monotonic() - t0, 20, 'run_cmd did not bound the child') + + def test_a_wedged_enumerate_is_killed_on_the_bound(self): + r = self._run('wedged_enumerate', timeout=2) + self.assertEqual(r.returncode, 124) + + @unittest.skipUnless(_gil_stall_available(), 'no libc for a GIL-holding stall') + def test_a_wedged_close_is_killed_on_the_bound(self): + """close() runs in the child's finally on EVERY failure path and is also + GIL-holding; hidraw_release takes the same rwsem hidraw_open needs.""" + r = self._run('wedged_close', timeout=3) + self.assertEqual(r.returncode, 124) + + def test_an_absent_device_reports_why(self): + r = self._run('absent') + self.assertNotEqual(r.returncode, 0) + self.assertIn('HID device not found', self._stderr(r)) + + def test_a_bad_echo_reports_both_payloads(self): + r = self._run('wrong_data') + self.assertNotEqual(r.returncode, 0) + msg = self._stderr(r) + self.assertIn('wrong data', msg) + self.assertIn('sent', msg) + self.assertIn('received', msg) + + def test_a_short_echo_is_not_read_as_a_pass(self): + r = self._run('short_read') + self.assertNotEqual(r.returncode, 0) + self.assertIn('short read', self._stderr(r)) + + +class StrayNoteSurvivesTheTupleWidth(unittest.TestCase): + """_stray_note reads r[5] -- and three producers build this tuple at three widths, so + `len(r) > 5 and r[5]` reads a WRONG SLOT rather than raising if a field is ever + inserted. The live handoff pr3840-mret-board-result.md proposes exactly that, and the + report would then say "no strays" while probes and usbfs nodes stay held into the next + job. The index changed once already in this branch (r[6] -> r[5]).""" + + def test_it_names_the_board_and_the_count(self): + wide = ('dirty', 1, [], [], 9.0, 2) + clean = ('fine', 0, [], [], 8.0, 0) + note = hil_test._stray_note([wide, clean]) + self.assertIn('dirty (2)', note) + self.assertIn('2 process(es)', note) + self.assertNotIn('fine', note, 'a clean board must not appear in the note') + + def test_a_narrow_row_from_the_timeout_path_is_not_misread(self): + """The abort paths synthesise 5-field rows for boards that never reported.""" + self.assertEqual(hil_test._stray_note([('stuck', 1, [], None, 0)]), '') + self.assertEqual(hil_test._stray_note([('fine', 0, [], [], 8.0, 0)]), '') + + def test_the_slot_it_reads_is_the_slot_test_board_writes(self): + """Pins the index against the producer, so inserting a field fails HERE rather + than silently reporting a duration as a stray count.""" + import ast + src = (Path(TEST_DIR).parents[0] / 'hil_test.py').read_text() + fn = next(n for n in ast.walk(ast.parse(src)) + if isinstance(n, ast.FunctionDef) and n.name == 'test_board') + widths = sorted({len(n.value.elts) for n in ast.walk(fn) + if isinstance(n, ast.Return) and isinstance(n.value, ast.Tuple)}) + # the board-LOCKED early return is 5 wide and carries no stray count; the normal + # one is 6, with strays last + self.assertEqual(widths, [5, 6], + 'the result tuple changed width; _stray_note reads index 5') + + +class MixedWidthRowsSurviveTheReportWriters(unittest.TestCase): + """_abort_report hands `[(n, 1, [], None, 0) for n in stuck] + [r for r in mret ...]` + to both writers -- 5-field synthetic rows mixed with 6-field worker rows. Every other + test uses uniform widths, so replacing either `*_` unpack with a fixed-width one keeps + the suite green and raises only INSIDE the containment path, where a raise costs every + board's results.""" + + def _mixed(self): + return [('stuck', 1, [], None, 0), # synthetic, 5 wide + ('ran', 1, ['device/dfu'], + [('ran', {'device/dfu': '❌ boom'}, '8s')], 8.0, 2)] # worker, 6 wide + + def test_the_rerun_spec_accepts_both_widths(self): + with TemporaryDirectory() as td: + rd = Path(td) + hil_test._write_failed_spec(rd / 'c.json.failed', rd, self._mixed()) + spec = (rd / 'c.json.failed').read_text() + self.assertIn('stuck', spec) + self.assertIn('ran', spec) + + def test_the_cell_names_the_cause_of_the_abort(self): + """A board the pool guard never reached did not "pool-timeout". Marking it so + sends whoever reads the table after a guard that never fired.""" + from helper import hil_report + real = hil_report.accumulate_report + + def render(reason, secs): + hil_report.accumulate_report = lambda *a, **k: (_ for _ in ()).throw( + OSError('report dir unwritable')) + try: + with TemporaryDirectory() as td: + rd = Path(td) + hil_test._abort_report(reason, [], [{'name': 'boardA'}], + rd / 'c.failed', rd, True, '', + timeout_secs=secs) + return (rd / hil_report.REPORT_MD).read_text() + finally: + hil_report.accumulate_report = real + + guard = render('abandoned: worker pool timed out after 3600s', 3600) + self.assertIn(hil_report.POOL_TIMEOUT_CELL, guard) + raised = render('aborted: a worker raised ValueError: x', None) + self.assertIn(hil_report.RUN_ABORTED_CELL, raised) + self.assertNotIn(hil_report.POOL_TIMEOUT_CELL, raised, + 'a run that aborted on a raise is not a pool timeout') + # and the fallback must still fire on BOTH paths -- that is what it is for + for md in (guard, raised): + self.assertIn('boardA', md) + + def test_only_the_rerun_spec_sees_the_synthetic_rows(self): + """accumulate_report gets `mret` alone -- worker rows, always 4th field a real + list. Widening _abort_report to hand it the synthetic list too would crash the + containment path: those rows carry rows=None and render_matrix iterates it.""" + import ast + src = (Path(TEST_DIR).parents[0] / 'hil_test.py').read_text() + fn = next(n for n in ast.walk(ast.parse(src)) + if isinstance(n, ast.FunctionDef) and n.name == '_abort_report') + calls = {ast.unparse(n.func): ast.unparse(n) + for n in ast.walk(fn) if isinstance(n, ast.Call) + and ast.unparse(n.func).endswith(('_write_failed_spec', + 'accumulate_report'))} + self.assertEqual( + ast.unparse(ast.parse(calls['hil_report.accumulate_report']).body[0] + ).split('(', 1)[1].split(',')[0], 'mret', + 'accumulate_report must receive worker rows only -- the synthetic rows carry ' + 'rows=None and render_matrix iterates that field') + self.assertIn('stuck', calls['_write_failed_spec'], + 'the re-run spec must still name the boards that never reported') + + +class UsbtestAbsentDeviceVerdict(unittest.TestCase): + """The arm that fails BEFORE usbtest_permit: an absent device must not queue on the + battery mutex for minutes just to have usbtest.py report "no device", and the cell + needs the 0/30 denominator or the row reads as a bare failure.""" + + def setUp(self): + self.addCleanup(setattr, hil_test, 'board_wedged', hil_test.board_wedged) + hil_test.board_wedged = '' + no_settle(self) + from helper import hil_lock, hil_util + self.addCleanup(setattr, hil_util, 'usb_scan', hil_util.usb_scan) + hil_util.usb_scan = lambda **k: [] # a readable bus, no such device + self.addCleanup(setattr, hil_test, '_enum_timeout', hil_test._enum_timeout) + hil_test._enum_timeout = 0 + self.addCleanup(setattr, hil_lock, 'usbtest_permit', hil_lock.usbtest_permit) + from contextlib import contextmanager + + def boom(uid): + raise AssertionError('took the battery permit for an absent device') + yield + hil_lock.usbtest_permit = contextmanager(boom) + + def test_a_readable_bus_without_the_device_says_absent_with_a_denominator(self): + with self.assertRaises(hil_test.TestFail) as cm: + hil_test.test_device_usbtest({'name': 'b', 'uid': 'NOPE', + 'flasher': {'name': 'stlink', 'uid': 'X'}}) + self.assertIn('no cafe:4010 device', str(cm.exception)) + self.assertIn('0/30', cm.exception.metric) + + def test_a_scan_that_gave_up_says_could_not_tell_instead(self): + """The conflation this whole path exists to avoid: an unreadable DUT is not an + absent one, and the bare string sends a maintainer after a firmware regression on + hardware that is merely wedged.""" + from helper import hil_util + self.addCleanup(setattr, hil_util, '_ever_stranded', hil_util._ever_stranded) + hil_util._ever_stranded = True + with self.assertRaises(hil_test.TestFail) as cm: + hil_test.test_device_usbtest({'name': 'b', 'uid': 'NOPE', + 'flasher': {'name': 'stlink', 'uid': 'X'}}) + self.assertIn('could not tell', str(cm.exception)) + + +class UsbtestStartupDoesNotClaimAbsenceBlind(unittest.TestCase): + """usbtest.py's own startup lookup, the sibling of the arm above. hil_test relays its + stderr verbatim into the report cell, so a positive 'no cafe:4010 device' from a scan + that gave up is the same conflation one process further out. Structural because the + exit sits mid-main(), behind argparse and the testusb probe.""" + + def test_the_sysfs_backed_absence_claims_carry_the_note(self): + """Both claims that a bounded read can turn into a false absence. The printer one + was missed: read_sysfs folds a timed-out `serial` into None, so a wedged-but- + enumerated printer read as 'Printer device not found' -- an enumeration verdict for + hardware that is merely unreadable. The MIDI lookup is deliberately NOT here: it + globs /dev/snd/by-id and readlinks it, so no bounded read can blind it.""" + import ast + tree = ast.parse(Path(hil_test.__file__).read_text()) + claims = [ast.unparse(n) for n in ast.walk(tree) + if isinstance(n, (ast.Assert, ast.Raise)) + and ('Printer device not found' in ast.unparse(n) + or 'no cafe:4010 device' in ast.unparse(n))] + self.assertEqual(len(claims), 2, 'a sysfs-backed absence claim moved or was added') + for c in claims: + self.assertIn('strand_note', c, f'absence claimed without the note: {c[:70]}') + + def test_the_absence_exit_carries_the_stranded_caveat(self): + import ast + import usbtest + tree = ast.parse(Path(usbtest.__file__).read_text()) + exits = [n for n in ast.walk(tree) + if isinstance(n, ast.Call) and ast.unparse(n.func) == 'sys.exit' + and 'no {VID}:{PID} device' in ast.unparse(n)] + self.assertEqual(len(exits), 1, 'the absence exit moved; retarget this test') + self.assertIn('strand_note', ast.unparse(exits[0]), + 'usbtest claims absence without consulting sysfs_stranded()') + + +class UsbtestGlobalCleanupStaysProcessWide(unittest.TestCase): + """The strand flag has TWO consumers at different scopes. The per-case verdict is + per-DUT -- a peer that stranded must not make OUR board report wedged. But the finally + block's cleanup is GLOBAL: remove_id plus an unbind of every interface under the + usbtest driver, including that peer's. Those writes take the uninterruptible + device_lock, so the global path has to stay gated on the process-wide question.""" + + def test_the_global_unbind_consults_the_process_wide_flag(self): + import ast + import usbtest + tree = ast.parse(Path(usbtest.__file__).read_text()) + fins = [n for n in ast.walk(tree) if isinstance(n, ast.Try) and n.finalbody + and 'remove_id' in ast.unparse(ast.Module(body=n.finalbody, type_ignores=[]))] + self.assertEqual(len(fins), 1, 'the cleanup finally moved; retarget this test') + body = ast.unparse(ast.Module(body=fins[0].finalbody, type_ignores=[])) + self.assertIn('sysfs_stranded', body, + 'global remove_id/unbind runs without the process-wide strand gate') + + if __name__ == '__main__': unittest.main() diff --git a/test/hil/test/test_hil_health.py b/test/hil/test/test_hil_health.py index 5695cad6d..ceecc8d37 100644 --- a/test/hil/test/test_hil_health.py +++ b/test/hil/test/test_hil_health.py @@ -467,49 +467,6 @@ class KillPoolChildren(PatchCase): self.assertEqual(hil_health.kill_pool_children(NoPool()), 0) -class WriteTimeoutReport(unittest.TestCase): - def test_prefix_carries_the_preflight_diagnosis(self): - """The timeout aborts before accumulate_report, so without the prefix the artifact - and the PR comment lose the one line saying WHY the pool never finished.""" - with TemporaryDirectory() as td: - d = Path(td) - hil_health.write_timeout_report(d, [{'name': 'b1'}], 4200, 'r.md', - prefix='> **wedged usb_hub_wq worker.**\n') - out = (d / 'r.md').read_text() - self.assertTrue(out.startswith('> **wedged usb_hub_wq worker.**')) - self.assertIn('timed out after 4200s', out) - self.assertIn('- b1', out) - - def test_writes_a_report_where_there_would_be_none(self): - with TemporaryDirectory() as td: - hil_health.write_timeout_report(Path(td), [{'name': 'ra6m5_ek'}], 4200, - 'hil_report.md') - md = (Path(td) / 'hil_report.md').read_text() - self.assertIn('4200s', md) - self.assertIn('ra6m5_ek', md) - - def test_keeps_a_previous_attempts_table(self): - with TemporaryDirectory() as td: - path = Path(td) / 'hil_report.md' - path.write_text('| board | cdc_msc |\n') - hil_health.write_timeout_report(Path(td), [{'name': 'b1'}], 4200, 'hil_report.md') - md = path.read_text() - self.assertIn('abandoned', md) - self.assertIn('| board | cdc_msc |', md) - self.assertLess(md.index('abandoned'), md.index('| board |')) - - def test_custom_banner_is_used(self): - with TemporaryDirectory() as td: - hil_health.write_timeout_report(Path(td), [], 0, 'hil_report.md', - banner='**refused to start.**\n') - self.assertIn('refused to start', (Path(td) / 'hil_report.md').read_text()) - - def test_unwritable_dir_does_not_raise(self): - """The caller may be about to os._exit; losing the report must not also lose the - exit path.""" - hil_health.write_timeout_report(Path('/proc/nonexistent/nope'), [], 0, 'x.md') - - class WorkerSweepsItsOwnChildren(unittest.TestCase): """maxtasksperchild=1 makes a worker exit the moment its task returns, so by the time main()'s finally sweeps, the strays have been reparented to init and are off the pool's @@ -558,51 +515,54 @@ class PermitReleasesOnlyWhatItTook(unittest.TestCase): 'the permit released a slot it never acquired: width grew') -class RecoveryPrefersResetOverReflash(unittest.TestCase): - """Probe reset is the preferred cure: non-destructive (the wedged firmware survives for - autopsy), no flash wear, no risk of a bad park image (a wfe/wfi park has bricked SWD on - mimxrt1064_evk and max32666fthr through a power cycle), and measured at 128-129 ms - against a full erase+program. It also fits in budgets a reflash does not.""" +class RecoveryUsesAResetOnlyWhenThereIsARealOne(unittest.TestCase): + """usbtest's recovery runs the reset unconditionally before the reflash -- it is + non-destructive (the wedged firmware survives for autopsy), writes no flash, cannot + brick SWD the way a bad park image has (mimxrt1064_evk, max32666fthr), and is measured + at 128-129 ms against a full erase+program. + + Two things still gate it, and both are what this pins: a flasher may have no reset + primitive at all, and reset_esptool/reset_lm4flash return rc 0 WITHOUT resetting + anything. Running those makes the log say "resetting <board> via <flasher>" for a step + that did nothing. wedged_pids() arbitrates either way, so behaviour was always right -- + the record was not, and a false record is what keeps having to be unpicked.""" def setUp(self): import usbtest # test/hil is already on sys.path (see top of file) - self.u = usbtest - - def test_reset_is_attempted_before_the_reflash(self): - steps = self.u.recovery_steps('openocd', time_left=600) - self.assertEqual([s[0] for s in steps], ['reset', 'flash']) - - def test_a_budget_too_small_to_reflash_still_gets_the_reset(self): - """The old gate skipped recovery whole when a reflash did not fit, leaving the - holder in place; a reset needs a fraction of the budget.""" - steps = self.u.recovery_steps('openocd', time_left=self.u.RECOVER_FLASH_TIMEOUT - 1) - self.assertEqual([s[0] for s in steps], ['reset']) + # PRODUCTION, not a copy: re-implementing the screen here let the real gate be + # deleted with the suite still green, which is the failure mode this pins. + self._reset_fn = usbtest.reset_primitive - def test_no_budget_at_all_yields_nothing(self): - self.assertEqual(self.u.recovery_steps('openocd', time_left=1), []) + def test_a_stub_that_resets_nothing_is_not_claimed(self): + for name in ('esptool', 'lm4flash'): + self.assertIsNone(self._reset_fn(name), + f'reset_{name} returns rc 0 without resetting; claiming it ' + f'puts a step that did nothing in the record') - def test_a_flasher_with_no_reset_primitive_goes_straight_to_reflash(self): - steps = self.u.recovery_steps('nosuchflasher', time_left=600) - self.assertEqual([s[0] for s in steps], ['flash']) + def test_a_real_reset_primitive_is_used(self): + for name in ('openocd', 'jlink', 'stlink'): + self.assertIsNotNone(self._reset_fn(name)) + def test_a_flasher_with_no_reset_primitive_goes_straight_to_the_reflash(self): + self.assertIsNone(self._reset_fn('nosuchflasher')) -class RecoveryDoesNotClaimAResetItDidNotDo(unittest.TestCase): - """reset_esptool and reset_lm4flash return rc 0 without resetting anything, so a plan - that includes them makes the log say "resetting <board> via <flasher>" for a step that - did nothing. wedged_pids() arbitrates, so behaviour was already right -- the record was - not, and a false record is what this branch keeps having to unpick.""" - - def setUp(self): + def test_the_reset_is_attempted_before_the_reflash(self): + """Order matters and now lives only in main()'s inline ladder, where no test + reaches it -- swapping the two blocks kept the suite green. Reset first is + non-destructive: the firmware under test survives for autopsy, no flash is + written, and it cannot brick SWD the way a bad park image has on mimxrt1064_evk + and max32666fthr.""" + import ast import usbtest - self.u = usbtest - - def test_a_no_op_reset_primitive_is_not_scheduled(self): - self.assertEqual([k for k, _ in self.u.recovery_steps('esptool', 600)], ['flash']) - self.assertEqual([k for k, _ in self.u.recovery_steps('lm4flash', 600)], ['flash']) + src = Path(usbtest.__file__).read_text() + fn = next(n for n in ast.walk(ast.parse(src)) + if isinstance(n, ast.FunctionDef) and n.name == 'main') + seg = ast.get_source_segment(src, fn) + reset_at = seg.index('reset_fn = reset_primitive(') + flash_at = seg.index("flash_fn(board, args.recover_fw") + self.assertLess(reset_at, flash_at, + 'the reflash is attempted before the non-destructive reset') - def test_a_real_reset_primitive_still_is(self): - self.assertEqual([k for k, _ in self.u.recovery_steps('openocd', 600)], - ['reset', 'flash']) class SudoSoftNeverRaises(unittest.TestCase): diff --git a/test/hil/test/test_hil_report.py b/test/hil/test/test_hil_report.py new file mode 100644 index 000000000..7c7a097ef --- /dev/null +++ b/test/hil/test/test_hil_report.py @@ -0,0 +1,1173 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests for the report document: the vocabulary, the one cell classifier, rendering, +# the four writers, and the fold to per-board verdicts. Split out of test_hil_bounded.py +# and test_hil_health.py when the report code moved into helper/hil_report.py. +# Run directly: +# python3 test/hil/test/test_hil_report.py +import json +import os +import subprocess +import sys +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory + +TEST_DIR = os.path.dirname(os.path.abspath(__file__)) +HIL_DIR = os.path.dirname(TEST_DIR) +sys.path.insert(0, HIL_DIR) + +from helper import hil_report + + +class OneClassifierForBothArtifacts(unittest.TestCase): + """The markdown tally and the agent's verdict used to classify cells with two separate + copies of one rule -- hil_test's cell_kind against REPORT_CELL, and hil_summary's + cell_state against its own re-typed '❌'/'⚪' literals. Change the icons and the table + and the verdict silently disagree.""" + + def test_bare_states(self): + self.assertEqual(hil_report.cell_state('fail'), 'fail') + self.assertEqual(hil_report.cell_state('skip'), 'skip') + self.assertEqual(hil_report.cell_state('pass'), 'pass') + + def test_icon_prefixed_metrics_carry_their_verdict(self): + self.assertEqual(hil_report.cell_state(f'{hil_report.REPORT_CELL["fail"]} 29/30'), 'fail') + self.assertEqual(hil_report.cell_state(f'{hil_report.REPORT_CELL["skip"]} board wedged'), + 'skip') + + def test_an_unprefixed_metric_is_a_pass(self): + """Load-bearing: a passing test may return a plain metric string. Classifying + unknown shapes as fail would publish a green table as a red verdict.""" + self.assertEqual(hil_report.cell_state('480.0 MBps'), 'pass') + self.assertEqual(hil_report.cell_state('1103 KB/s'), 'pass') + + def test_a_non_string_cell_does_not_raise(self): + """render_matrix's copy guarded with isinstance; hil_summary's did not, because its + caller str()'d first. The merged one keeps the guard -- it is the safer superset.""" + self.assertEqual(hil_report.cell_state(None), 'pass') + + def test_the_icons_come_from_REPORT_CELL(self): + """No second copy of the emoji anywhere in the module.""" + src = (Path(HIL_DIR) / 'helper' / 'hil_report.py').read_text(encoding='utf-8') + # CODE only: prose may quote an icon to explain a rule. The old assertion counted + # the single-quoted spelling `'❌'`, which a second copy written as "❌" would have + # sailed past. + code = '\n'.join(line.split('#', 1)[0] for line in src.splitlines()) + for icon in ('❌', '⚪', '✅'): + self.assertEqual(code.count(icon), 1, + f'{icon} is spelled in code more than once; REPORT_CELL is' + f' meant to be the one source') + + +class ModuleWorksImportedAndAsAScript(unittest.TestCase): + """It is imported as helper.hil_report by hil_test, and run as a script by the operator + (.claude/agents/hil-operator.md). A script run puts helper/ on sys.path, NOT test/hil, + so a plain `from helper import hil_health` breaks the CLI and only the CLI.""" + + def test_importable_as_a_package_module(self): + r = subprocess.run( + [sys.executable, '-c', + f'import sys; sys.path.insert(0, {HIL_DIR!r}); ' + f'from helper import hil_report; print(hil_report.REPORT_JSON)'], + capture_output=True, text=True, timeout=60) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn('hil_report.json', r.stdout) + + def test_runnable_as_a_script(self): + r = subprocess.run( + [sys.executable, str(Path(HIL_DIR) / 'helper' / 'hil_report.py'), '--help'], + capture_output=True, text=True, timeout=60) + self.assertEqual(r.returncode, 0, r.stderr) + + +class RenderReportIsPureFunctionOfTheDocument(unittest.TestCase): + """Four writers used to compose the markdown independently, so a table could carry + something the sidecar did not. One renderer, and the ordering it guarantees, is what + stops that -- pinned here rather than left to the order of three concatenations.""" + + def _doc(self, **kw): + d = {'rows': [{'board': 'boardA', 'cells': {'cdc_msc': 'pass'}, 'duration': '1s'}], + 'banner': '', 'scope': '', 'caveat': ''} + d.update(kw) + return d + + def test_table_comes_from_rows(self): + md = hil_report.render_report(self._doc()) + self.assertIn('boardA', md) + self.assertIn('cdc_msc', md) + + def test_scope_note_appears_above_the_table(self): + md = hil_report.render_report(self._doc(scope='-b boardA')) + self.assertLess(md.index('Scoped run'), md.index('boardA')) + + def test_banner_outranks_the_scope_note(self): + md = hil_report.render_report(self._doc(scope='-b boardA', + banner='> **Rig dirty.** x\n')) + self.assertLess(md.index('Rig dirty'), md.index('Scoped run')) + + def test_caveat_is_outermost(self): + md = hil_report.render_report(self._doc(banner='> **Rig dirty.** x\n', + caveat='**HIL run abandoned.**\n')) + self.assertLess(md.index('abandoned'), md.index('Rig dirty')) + + def test_a_document_with_no_rows_still_renders(self): + md = hil_report.render_report(self._doc(rows=[])) + self.assertIn('No tests were run.', md) + + def test_a_malformed_row_does_not_raise(self): + """mark_report_abandoned renders a sidecar it did not write -- hil_ci.sh reuses a + persistent REMOTE_DIR, so it can be an older version's or a torn one -- and it runs + on the way to os._exit, where a KeyError hangs the runner in multiprocessing's + unbounded join() instead of freeing it.""" + md = hil_report.render_report(self._doc( + rows=[{'board': 'boardA', 'cells': {'cdc_msc': 'pass'}}, {'board': 'half'}, + {}])) + self.assertIn('boardA', md) # the intact row still renders ... + self.assertIn('half', md) # ... and a cell-less one becomes a blank row + + +class ScopeSurvivesInTheJson(unittest.TestCase): + """A scoped run's small table is indistinguishable from a full run that lost boards. + The markdown says so; the JSON did not, so any JSON consumer could not tell.""" + + def _rows(self, board, cell): + return [(board, 0, 0, [(board, {cell: 'OK'}, '1s')], 0)] + + def test_scope_is_recorded_in_the_sidecar(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report(self._rows('boardA', 'cdc_msc'), rd, True, + '-b boardA', '') + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertEqual(doc['scope'], '-b boardA') + + def test_an_unscoped_run_records_an_empty_scope(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report(self._rows('boardA', 'cdc_msc'), rd, True, '', '') + self.assertEqual(json.loads((rd / hil_report.REPORT_JSON).read_text())['scope'], '') + + +class EveryExitPathLeavesBothArtifacts(unittest.TestCase): + """summarize() builds an agent's verdicts from the JSON. A path that writes only + markdown reports the whole fleet as 'no report row' while a human sees the real story.""" + + def test_the_no_boards_exit_writes_json_too(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.write_report(rd, {'rows': [], 'banner': '', 'scope': '', + 'caveat': '**HIL run selected no boards.** why\n'}) + self.assertIn('selected no boards', (rd / hil_report.REPORT_MD).read_text()) + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertEqual(doc['rows'], []) + self.assertIn('selected no boards', doc['caveat']) + + def test_write_report_raises_so_its_callers_can_report_it(self): + """write_report is NOT best-effort. Swallowing the OSError made + write_timeout_report's _p warning and hil_test's fallback-of-the-fallback dead + code -- an unwritable report dir produced no artifact and no message.""" + with self.assertRaises(OSError): + hil_report.write_report(Path('/proc/nonexistent/nope'), + {'rows': [], 'banner': '', 'scope': '', 'caveat': 'x\n'}) + + def test_the_guarded_callers_still_do_not_raise(self): + """They are the ones on the way to os._exit, where a raise hangs the interpreter + in multiprocessing's unbounded join().""" + bad = Path('/proc/nonexistent/nope') + hil_report.mark_report_abandoned(bad, 'the worker pool would not shut down.') + hil_report.mark_report_no_boards(bad, 'filters intersected to nothing') + import io + from contextlib import redirect_stdout + with redirect_stdout(io.StringIO()): + hil_report.write_timeout_report(bad, [{'name': 'b1'}], 3600) + + +class AbandonNoticeLandsInBothArtifacts(unittest.TestCase): + """_abandon_exit did a text prepend on a file it had not written, so the caveat never + reached the JSON and an agent reading the sidecar saw a clean partial report under a + red job.""" + + def test_abandon_sets_the_caveat_not_just_the_markdown(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('boardA', 0, 0, [('boardA', {'cdc_msc': 'OK'}, '1s')], 0)], rd, True, '', '') + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertIn('abandoned', doc['caveat']) + self.assertEqual(len(doc['rows']), 1, 'the finished board must survive') + md = (rd / hil_report.REPORT_MD).read_text() + self.assertLess(md.index('abandoned'), md.index('boardA')) + + def test_marking_a_missing_report_is_a_no_op(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + hil_report.mark_report_abandoned(Path(td.name), 'x') # must not raise + + def test_a_sidecar_with_a_malformed_row_still_gets_stamped(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_JSON).write_text(json.dumps( + {'rows': [{'board': 'boardA'}], 'banner': '', 'scope': '', 'caveat': ''})) + hil_report.mark_report_abandoned(rd, 'x') + self.assertIn('abandoned', + json.loads((rd / hil_report.REPORT_JSON).read_text())['caveat']) + self.assertIn('abandoned', (rd / hil_report.REPORT_MD).read_text()) + + def test_a_torn_sidecar_is_a_no_op(self): + """This runs while the interpreter is being torn down: a raise here hangs the + process in multiprocessing's unbounded join().""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_JSON).write_text('{ truncated mid-') + hil_report.mark_report_abandoned(rd, 'x') # must not raise + + def test_an_existing_abandon_caveat_is_not_overwritten(self): + """The pool-timeout path names the stuck boards and the rig-health verdict; this + one only knows the pool would not shut down. Whoever got there first wins -- + the guard _abandon_exit used to spell as "'**HIL run ab' not in body[:2000]".""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.write_timeout_report(rd, [{'name': 'stuck'}], 3600, + prefix='> **wedged usb_hub_wq worker.**\n') + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertIn('timed out after 3600s', doc['caveat']) + self.assertIn('wedged usb_hub_wq worker', doc['banner']) # rig health, not outcome + self.assertNotIn('would not shut down', doc['caveat']) + + +class CaveatSurvivesAccumulate(unittest.TestCase): + """CI reruns with --accumulate: the sidecar keeps every earlier attempt's cells, but the + banner was recomputed per attempt. A first attempt on a degraded rig and a clean rerun + therefore published the degraded attempt's PASSES with no caveat on them -- and the + generated .failed spec reruns only failures, so those cells are never re-earned.""" + + def _rows(self, board, cell): + return [(board, 0, 0, [(board, {cell: 'OK'}, '1s')], 0)] + + def test_an_earlier_attempts_caveat_is_still_on_the_report(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + banner = '> **Rig note.** 2 process(es) in D state at start.\n' + + hil_report.accumulate_report(self._rows('boardA', 'cdc_msc'), rd, True, '', banner) + self.assertIn('Rig note', (rd / hil_report.REPORT_MD).read_text()) + + # the rerun: clean rig, so this attempt contributes no banner of its own + md = hil_report.accumulate_report(self._rows('boardB', 'cdc_msc'), rd, False, '', '') + self.assertIn('boardA', md) # the earlier cells are kept ... + self.assertIn('Rig note', md, + 'the caveat the earlier cells were collected under was dropped') + + def test_the_same_caveat_twice_is_not_stacked(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + banner = '> **Rig note.** 2 process(es) in D state at start.\n' + hil_report.accumulate_report(self._rows('boardA', 'cdc_msc'), rd, True, '', banner) + md = hil_report.accumulate_report(self._rows('boardB', 'cdc_msc'), rd, False, '', banner) + self.assertEqual(md.count('Rig note'), 1) + + +class MarkdownIsAlwaysARenderingOfTheJson(unittest.TestCase): + """The property this whole change buys: whatever wrote the report, re-rendering the + sidecar reproduces the markdown byte for byte. Four writers, one renderer -- asserted + directly rather than inferred from the writers.""" + + def _check(self, rd): + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertEqual((rd / hil_report.REPORT_MD).read_text(), + hil_report.render_report(doc) + '\n') + + def test_normal_path(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('boardA', 0, 0, [('boardA', {'cdc_msc': 'OK'}, '1s')], 0)], rd, True, + '-b boardA', '> **Rig note.** x\n') + self._check(rd) + + def test_after_an_accumulate_rerun(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('boardA', 0, 0, [('boardA', {'cdc_msc': 'OK'}, '1s')], 0)], rd, True, '', '') + hil_report.accumulate_report( + [('boardB', 0, 0, [('boardB', {'cdc_msc': 'OK'}, '1s')], 0)], rd, False, '', '') + self._check(rd) + + def test_after_abandonment(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('boardA', 0, 0, [('boardA', {'cdc_msc': 'OK'}, '1s')], 0)], rd, True, '', '') + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + self._check(rd) + + def test_no_boards_exit(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.write_report(rd, {'rows': [], 'banner': '', 'scope': '', + 'caveat': '**HIL run selected no boards.** why\n'}) + self._check(rd) + + def test_the_pool_guard_fallback(self): + """The last writer to join the invariant: it composed its own markdown only because + hil_health could not import the renderer.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('done', 0, 0, [('done', {'cdc_msc': 'OK'}, '1s')], 0)], rd, True, '', '') + hil_report.write_timeout_report(rd, [{'name': 'stuck'}], 3600, + prefix='> **wedged usb_hub_wq worker.**\n') + self._check(rd) + +class WriteTimeoutReport(unittest.TestCase): + def test_prefix_carries_the_preflight_diagnosis(self): + """The timeout aborts before accumulate_report, so without the prefix the artifact + and the PR comment lose the one line saying WHY the pool never finished.""" + with TemporaryDirectory() as td: + d = Path(td) + hil_report.write_timeout_report(d, [{'name': 'b1'}], 4200, + prefix='> **wedged usb_hub_wq worker.**\n') + out = (d / hil_report.REPORT_MD).read_text() + # the abandon notice leads (run outcome), the rig-health prefix follows in the + # banner -- prefix used to be folded INTO the caveat, which is what made a clean + # --accumulate retry inherit an abandonment that had not happened + self.assertTrue(out.startswith('**HIL run abandoned: worker pool timed out'), out[:80]) + self.assertIn('> **wedged usb_hub_wq worker.**', out) + self.assertIn('timed out after 4200s', out) + self.assertIn('- b1', out) + + def test_writes_a_report_where_there_would_be_none(self): + with TemporaryDirectory() as td: + hil_report.write_timeout_report(Path(td), [{'name': 'ra6m5_ek'}], 4200) + md = (Path(td) / hil_report.REPORT_MD).read_text() + self.assertIn('4200s', md) + self.assertIn('ra6m5_ek', md) + + def test_the_prior_attempts_rows_survive(self): + """Was: the prior MARKDOWN TEXT survives below the banner. It now re-renders from + the merged sidecar, so the guarantee is stated against rows -- one table with the + stuck boards in it, rather than a banner stapled above a duplicate table.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('done', 0, 0, [('done', {'cdc_msc': 'OK'}, '1s')], 0)], rd, True, '', '') + hil_report.write_timeout_report(rd, [{'name': 'stuck'}], 3600) + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertEqual([r['board'] for r in doc['rows']], ['done', 'stuck']) + md = (rd / hil_report.REPORT_MD).read_text() + self.assertIn('done', md) + self.assertIn('stuck', md) + self.assertIn('abandoned', md) + self.assertLess(md.index('abandoned'), md.index('done')) + self.assertEqual(md.count('| Board'), 1, 'the prior table was duplicated, not merged') + + def test_custom_banner_is_used(self): + with TemporaryDirectory() as td: + hil_report.write_timeout_report(Path(td), [], 0, + banner='**refused to start.**\n') + self.assertIn('refused to start', (Path(td) / hil_report.REPORT_MD).read_text()) + + def test_timeout_report_writes_the_sidecar(self): + """This path used to write markdown only, so summarize() -- which is all an + agent gets -- reported the whole fleet as 'no report row' on exactly the runs + that failed.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.write_timeout_report(rd, [{'name': 'boardA'}], 3600) + self.assertTrue((rd / hil_report.REPORT_JSON).is_file()) + self.assertIn('boardA', (rd / hil_report.REPORT_JSON).read_text()) + + def test_the_sidecar_keeps_a_previous_attempts_rows(self): + """An earlier attempt's finished boards are real results and this attempt has none + of its own, so the rows merge rather than replace.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_JSON).write_text(json.dumps( + {'rows': [{'board': 'done', 'cells': {'cdc_msc': 'pass'}, 'duration': '1s'}], + 'banner': '', 'scope': '', 'caveat': ''})) + hil_report.write_timeout_report(rd, [{'name': 'stuck'}], 3600) + rows = json.loads((rd / hil_report.REPORT_JSON).read_text())['rows'] + self.assertEqual([r['board'] for r in rows], ['done', 'stuck']) + + def test_a_torn_sidecar_does_not_lose_the_stuck_boards(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_JSON).write_text('{ truncated mid-') + hil_report.write_timeout_report(rd, [{'name': 'stuck'}], 3600) + rows = json.loads((rd / hil_report.REPORT_JSON).read_text())['rows'] + self.assertEqual([r['board'] for r in rows], ['stuck']) + + def test_a_roster_entry_without_a_name_does_not_escape(self): + """The broad handler exists to stop a KeyError here stranding the runner, but a + report that silently loses its only board is worse than one saying '?'.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.write_timeout_report(rd, [{}], 3600) + rows = json.loads((rd / hil_report.REPORT_JSON).read_text())['rows'] + self.assertEqual([r['board'] for r in rows], ['?']) + + def test_unwritable_dir_does_not_raise(self): + """The caller may be about to os._exit; losing the report must not also lose the + exit path.""" + hil_report.write_timeout_report(Path('/proc/nonexistent/nope'), [], 0) + + +class SummaryFoldsReportToBoards(unittest.TestCase): + """summarize() replaces the agent retyping the markdown table. Report rows are named per + VARIANT and a variant need not start with the board name, so the config is what maps them + back -- the previous string-matching design produced a defect in each of four review rounds.""" + + def _sum(self, boards, rows, cfg_boards=None, banner=''): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + d = Path(td.name) + (d / 'hil_report.json').write_text(json.dumps( + {'rows': [{'board': b, 'cells': c, 'duration': '1s'} for b, c in rows], + 'banner': banner})) + cfg = d / 'cfg.json' + cfg.write_text(json.dumps({'boards': cfg_boards or [{'name': b} for b in boards]})) + args = [a for b in boards for a in ('-b', b)] + r = subprocess.run(['python3', str(Path(TEST_DIR).parents[0] / 'helper' / 'hil_report.py'), + str(cfg), *args, '--report-dir', str(d)], + capture_output=True, text=True, timeout=60) + self.assertEqual(r.returncode, 0, r.stderr) + return json.loads(r.stdout)['results'] + + def test_variant_rows_fold_onto_their_board(self): + """nanoch32v203 never produces a row named after the board.""" + got = self._sum(['nanoch32v203'], + [('nanoch32v203-fsdev', {'usbtest': 'pass'}), + ('nanoch32v203-usbfs', {'usbtest': 'pass'})], + cfg_boards=[{'name': 'nanoch32v203', + 'variant': [{'name': 'nanoch32v203-fsdev'}, + {'name': 'nanoch32v203-usbfs'}]}]) + self.assertEqual([r['board'] for r in got], ['nanoch32v203']) + self.assertTrue(got[0]['pass']) + self.assertTrue(got[0]['ran']) + + def test_one_failing_variant_fails_the_board(self): + got = self._sum(['nano'], + [('nano-a', {'usbtest': 'pass'}), ('nano-b', {'usbtest': '❌ 29/30'})], + cfg_boards=[{'name': 'nano', 'variant': [{'name': 'nano-a'}, + {'name': 'nano-b'}]}]) + self.assertFalse(got[0]['pass']) + self.assertIn('29/30', got[0]['detail']) + + def test_lock_contention_is_a_field_not_a_prefix(self): + got = self._sum(['alpha'], [('alpha', {'board-locked': 'fail'})]) + self.assertTrue(got[0]['locked']) + self.assertFalse(got[0]['pass']) + + def test_a_board_with_no_row_is_marked_not_run(self): + got = self._sum(['alpha', 'beta'], [('alpha', {'usbtest': 'pass'})]) + self.assertTrue(got[0]['ran']) + self.assertFalse(got[1]['ran']) + self.assertFalse(got[1]['pass']) + + def test_a_metric_cell_counts_by_its_icon(self): + got = self._sum(['a', 'b'], [('a', {'cdc_msc_throughput': '✅ C 1.2 M 3.4'}), + ('b', {'cdc_msc_throughput': '❌ C 0.0 M 0.0'})]) + self.assertTrue(got[0]['pass']) + self.assertFalse(got[1]['pass']) + + def test_skipped_cells_do_not_fail_a_board(self): + got = self._sum(['a'], [('a', {'usbtest': 'skip', 'cdc_msc': 'pass'})]) + self.assertTrue(got[0]['pass']) + + def test_a_plain_metric_cell_is_a_pass(self): + """Mirrors hil_test.py's own tally (cell_kind): failures are ALWAYS marked -- 'fail' + or a ❌ prefix, per TestFail's docstring -- while a passing test may return a plain + metric string that lands in the cell unprefixed. Treating unknown shapes as fail + would publish a green table as a red verdict.""" + got = self._sum(['a'], [('a', {'device_speed': '480.0 MBps'})]) + self.assertTrue(got[0]['pass']) + + def test_a_declared_variant_of_another_board_is_not_stolen(self): + """A declared variant need not start with its own board's name, so it may start with + a DIFFERENT board's name plus '-'. The prefix fallback must not attribute it twice.""" + got = self._sum(['alpha', 'beta'], + [('beta-x', {'usbtest': 'fail'})], + cfg_boards=[{'name': 'alpha', 'variant': [{'name': 'beta-x'}]}, + {'name': 'beta'}]) + self.assertTrue(got[0]['ran']) + self.assertFalse(got[0]['pass']) + self.assertFalse(got[1]['ran'], "beta must not inherit alpha's row") + + + def test_the_caveat_reaches_the_agents_verdict(self): + """The abandon/no-boards notice lives in the document now, and this JSON is all an + agent gets -- dropping it here puts the caveat back where only a human sees it.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + d = Path(td.name) + (d / 'hil_report.json').write_text(json.dumps( + {'rows': [{'board': 'boardA', 'cells': {'cdc_msc': 'pass'}, 'duration': '1s'}], + 'banner': '', 'scope': '', + 'caveat': '**HIL run abandoned: the worker pool would not shut down.**\n'})) + cfg = d / 'cfg.json' + cfg.write_text(json.dumps({'boards': [{'name': 'boardA'}]})) + r = subprocess.run( + ['python3', str(Path(TEST_DIR).parents[0] / 'helper' / 'hil_report.py'), + str(cfg), '-b', 'boardA', '--report-dir', str(d)], + capture_output=True, text=True, timeout=60) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn('abandoned', json.loads(r.stdout)['caveat']) + + def test_an_older_sidecar_without_a_caveat_still_summarises(self): + got = self._sum(['boardA'], [('boardA', {'cdc_msc': 'pass'})]) + self.assertTrue(got[0]['pass']) + + def test_the_old_entry_point_is_gone(self): + """hil_summary.py's CLI moved here. A leftover file would keep working while + drifting from the module that now owns the fold.""" + self.assertFalse((Path(HIL_DIR) / 'helper' / 'hil_summary.py').exists()) + + +class AbandonStampIsNotDestructive(unittest.TestCase): + """mark_report_abandoned runs on the way to os._exit, on a report it did not write. + Every case here was a live regression found by review.""" + + def _doc(self, **kw): + d = {'rows': [{'board': 'OLD', 'cells': {'t': 'pass'}, 'duration': '9s'}], + 'banner': '', 'scope': '', 'caveat': ''} + d.update(kw) + return d + + def test_declining_to_stamp_does_not_republish_the_markdown(self): + """The guard skipped the caveat assignment but write_report ran anyway, so a + no-op call still overwrote THIS run's table with a re-render of an older sidecar.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_JSON).write_text(json.dumps(self._doc( + caveat='**HIL run abandoned: worker pool timed out after 3600s.**\n'))) + (rd / hil_report.REPORT_MD).write_text('THIS RUN table with boardX\n') + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + self.assertEqual((rd / hil_report.REPORT_MD).read_text(), + 'THIS RUN table with boardX\n') + + def test_a_banner_borne_abandon_notice_also_wins(self): + """The pool-timeout path puts its notice in `banner` (hil_test.py:2300), not + `caveat`. SKILL.md gives the two notices OPPOSITE rules, so stamping the vaguer + one on top tells the agent to publish rows it is meant to discard.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('a', 0, 0, [('a', {'t': 'OK'}, '1s')], 0)], rd, True, '', '', + caveat='**HIL run abandoned: worker pool timed out after 3600s.** 2 never' + ' reported.\n') + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + md = (rd / hil_report.REPORT_MD).read_text() + self.assertTrue(md.startswith('**HIL run abandoned: worker pool timed out'), md[:80]) + self.assertNotIn('would not shut down', md) + + def test_a_missing_sidecar_still_stamps_the_markdown(self): + """Master read the MARKDOWN and prepended unconditionally, so it always stamped. + pr_comment.yml cats only hil_report.md -- giving up here publishes a clean green + table under an abandoned, non-zero job.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_MD).write_text('**✅ 27 passed · ❌ 0 failed**\n') + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + md = (rd / hil_report.REPORT_MD).read_text(encoding='utf-8') + self.assertIn('abandoned', md) + self.assertIn('27 passed', md) + + def test_a_torn_sidecar_still_stamps_the_markdown(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_JSON).write_text('{ truncated mid-') + (rd / hil_report.REPORT_MD).write_text('**✅ 27 passed**\n') + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + self.assertIn('abandoned', (rd / hil_report.REPORT_MD).read_text(encoding='utf-8')) + + def test_the_wording_matches_the_skill_contract(self): + """SKILL.md pins this banner as 'the table below IS this run's ... Report the + results AND the abandonment'. Calling it 'partial' sends the agent to re-run + boards that already passed.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_JSON).write_text(json.dumps(self._doc())) + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + caveat = json.loads((rd / hil_report.REPORT_JSON).read_text())['caveat'] + self.assertNotIn('partial', caveat) + self.assertIn('unverified', caveat) + + +class WriteReportFailsLoudly(unittest.TestCase): + def test_a_render_failure_does_not_leave_a_committed_json(self): + """It wrote the JSON, then rendered. A render raise left the sidecar saying + 'abandoned' beside a markdown that still read as a clean green table -- breaking + the one invariant this module exists to hold.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_MD).write_text('STALE GREEN TABLE\n') + (rd / hil_report.REPORT_JSON).write_text(json.dumps( + {'rows': ['boardA'], 'banner': '', 'scope': '', 'caveat': ''})) + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + md = (rd / hil_report.REPORT_MD).read_text() + # either both moved or neither did -- never a sidecar the markdown contradicts + self.assertEqual('abandoned' in doc.get('caveat', ''), 'abandoned' in md, + 'the sidecar was committed without its markdown') + + def test_a_non_dict_row_does_not_cost_the_abandon_stamp(self): + """A row that is a bare string raised out of render_report, so the stamp was lost + entirely -- the failure mode this whole function exists to prevent.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_JSON).write_text(json.dumps( + {'rows': ['boardA', {'board': 'good', 'cells': {'t': 'pass'}}], + 'banner': '', 'scope': '', 'caveat': ''})) + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + md = (rd / hil_report.REPORT_MD).read_text() + self.assertIn('abandoned', md) + self.assertIn('good', md) + + def test_an_unwritable_dir_reaches_the_callers_warning(self): + """write_report swallowing OSError made write_timeout_report's broad handler -- + and hil_test's fallback-of-the-fallback -- dead code: no artifact, no message.""" + import io + from contextlib import redirect_stdout + buf = io.StringIO() + with redirect_stdout(buf): + hil_report.write_timeout_report(Path('/proc/nonexistent/nope'), + [{'name': 'b1'}], 3600, prefix='x\n') + self.assertIn('warning', buf.getvalue().lower(), 'the failure was silent') + + +class PoolTimeoutCellIsHonest(unittest.TestCase): + def test_a_stuck_board_with_a_prior_row_still_gets_the_cell(self): + """`not in done` skipped the cell for any board carrying an earlier attempt's row, + so a board that just ate the 60-minute guard summarized as pass:true.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('stm32f4', 0, 0, [('stm32f4', {'cdc_msc': 'OK'}, '1s')], 0)], rd, True, '', '') + hil_report.write_timeout_report(rd, [{'name': 'stm32f4'}], 3600) + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + verdict = hil_report.summarize({'boards': [{'name': 'stm32f4'}]}, ['stm32f4'], doc) + self.assertFalse(verdict['results'][0]['pass'], + 'a board that hung the pool was published as a pass') + + def test_a_clean_retry_clears_the_cell(self): + """accumulate_report clears stale board-locked and BOUNDARY_CELL cells but not + this one, so a board that passed clean on the retry stayed red forever.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.write_timeout_report(rd, [{'name': 'stuck'}], 3600) + hil_report.accumulate_report( + [('stuck', 0, 0, [('stuck', {'cdc_msc': 'OK'}, '2s')], 0)], rd, False, '', '') + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertNotIn('pool-timeout', doc['rows'][0]['cells']) + verdict = hil_report.summarize({'boards': [{'name': 'stuck'}]}, ['stuck'], doc) + self.assertTrue(verdict['results'][0]['pass']) + + def test_a_torn_sidecar_does_not_destroy_an_intact_markdown(self): + """Re-rendering from an unusable sidecar threw away real results the human copy + still had. Master concatenated below its banner and kept them.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_MD).write_text('| Board | t |\n| a | OK |\n| b | OK |\n') + (rd / hil_report.REPORT_JSON).write_text('{ truncated') + hil_report.write_timeout_report(rd, [{'name': 'stuck'}], 3600) + md = (rd / hil_report.REPORT_MD).read_text() + self.assertIn('| a | OK |', md, "an earlier attempt's real results were destroyed") + self.assertIn('stuck', md) + + +class SummarizeSeesEveryRow(unittest.TestCase): + def test_board_name_rows_reach_a_variant_boards_verdict(self): + """hil_test writes lock-contention and pool-timeout rows keyed by BOARD name, but + variants_of returns only declared variant names -- so for nanoch32v203 and + ch32v307v_r1_1v0 those rows were invisible and a held lock published as a + hardware FAIL that hil-validate.js never retried.""" + cfg = {'boards': [{'name': 'nano', + 'variant': [{'name': 'nano-fsdev'}, {'name': 'nano-usbfs'}]}]} + doc = {'rows': [{'board': 'nano', 'cells': {'board-locked': 'fail'}, + 'duration': None}], 'banner': '', 'scope': '', 'caveat': ''} + r = hil_report.summarize(cfg, ['nano'], doc)['results'][0] + self.assertTrue(r['ran']) + self.assertTrue(r['locked'], 'a held lock was published as a hardware failure') + + def test_a_malformed_row_does_not_kill_the_cli(self): + """summarize is the one reader with no defense, and it is the only one an agent's + verdict depends on.""" + out = hil_report.summarize({'boards': [{'name': 'a'}]}, ['a'], + {'rows': [{'cells': {}}, {'board': 'a', + 'cells': {'t': 'pass'}}]}) + self.assertTrue(out['results'][0]['pass']) + + +class NoBoardsExitKeepsWhatRan(unittest.TestCase): + def test_it_does_not_wipe_an_accumulated_sidecar(self): + """Master wrote only markdown here, so the sidecar survived. Writing rows:[] + unconditionally makes an --accumulate rerun whose filters empty erase every + board that had already passed.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('a', 0, 0, [('a', {'t': 'OK'}, '1s')], 0)], rd, True, '', '') + hil_report.mark_report_no_boards(rd, 'No boards left after the flasher filter', + fresh=False) + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertEqual([r['board'] for r in doc['rows']], ['a']) + self.assertIn('selected no boards', doc['caveat']) + self.assertIn('selected no boards', (rd / hil_report.REPORT_MD).read_text()) + + +class TheMergeBehavioursAreActuallyPinned(unittest.TestCase): + """accumulate_report's docstring cites these three as the reason not to split it, yet + deleting any of them left the whole suite green. Mutation-verified.""" + + def test_a_cleared_boundary_drops_the_previous_attempts_mark(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('b', 0, 0, [('b-v', {hil_report.BOUNDARY_CELL: 'fail'}, '1s')], 0)], + rd, True, '', '') + hil_report.accumulate_report( + [('b', 0, 0, [('b-v', {'cdc_msc': 'OK'}, '2s')], 0)], rd, False, '', '') + cells = json.loads((rd / hil_report.REPORT_JSON).read_text())['rows'][0]['cells'] + self.assertNotIn(hil_report.BOUNDARY_CELL, cells) + + def test_a_board_that_really_ran_drops_its_stale_lock_cell(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('b', 0, 0, [('b', {hil_report.LOCKED_CELL: 'fail'}, None)], 0)], rd, True, '', '') + hil_report.accumulate_report( + [('b', 0, 0, [('b', {'cdc_msc': 'OK'}, '2s')], 0)], rd, False, '', '') + rows = json.loads((rd / hil_report.REPORT_JSON).read_text())['rows'] + self.assertEqual([r['board'] for r in rows], ['b']) + self.assertNotIn(hil_report.LOCKED_CELL, rows[0]['cells']) + + def test_a_filtered_rerun_keeps_the_previous_duration(self): + """A -t-filtered re-run reports duration None; blanking the column loses the only + record of how long the full run took.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('b', 0, 0, [('b', {'cdc_msc': 'OK'}, '119s')], 0)], rd, True, '', '') + hil_report.accumulate_report( + [('b', 0, 0, [('b', {'cdc_msc': 'OK'}, None)], 0)], rd, False, '', '') + self.assertEqual(json.loads( + (rd / hil_report.REPORT_JSON).read_text())['rows'][0]['duration'], '119s') + + +class TheFooterCountsAreNotSwapped(unittest.TestCase): + """SKILL.md tells the operator to paste the footer counts verbatim, and swapping the + failed/skipped tallies left the suite green.""" + + def test_each_kind_is_counted_under_its_own_label(self): + md = hil_report.render_matrix([ + ('b', {'p1': 'pass', 'p2': 'pass', 'f1': 'fail', + 's1': f'{hil_report.REPORT_CELL["skip"]} board wedged'}, '1s')]) + self.assertIn(f'{hil_report.REPORT_CELL["pass"]} 2 passed', md) + self.assertIn(f'{hil_report.REPORT_CELL["fail"]} 1 failed', md) + self.assertIn(f'{hil_report.REPORT_CELL["skip"]} 1 skipped', md) + + +class HilCiUploadsTheAccumulateMergeBase(unittest.TestCase): + """hil_ci.sh rm -rf's REMOTE_DIR at the start of every run, and accumulate_report + merges onto the sidecar in the run's cwd -- so without an upload a remote + `--accumulate` retry silently starts from nothing and its one-row table REPLACES the + full-fleet one. The copy-back at the end has always existed; the upload did not.""" + + def _gate(self, *args): + """Run the real gate block out of hil_ci.sh and return its ACCUMULATE verdict. + + Executed, not grepped: the previous pair of tests searched the source text and + stayed green when `if [ "$ACCUMULATE" = 1 ]` was mutated to `if true`, because the + comment block above it mentions --accumulate five times.""" + sh = (Path(HIL_DIR) / 'hil_ci.sh').read_text(encoding='utf-8') + a = sh.index('ACCUMULATE=$(python3 -') + b = sh.index(') || ACCUMULATE=0', a) + len(') || ACCUMULATE=0') + script = 'ARGS=("$@")\n' + sh[a:b] + '\necho "$ACCUMULATE"' + r = subprocess.run(['bash', '-c', script, '_', *args], + capture_output=True, text=True, timeout=60) + self.assertEqual(r.returncode, 0, r.stderr) + return r.stdout.strip() + + def test_every_spelling_argparse_accepts_is_detected(self): + """hil_test.py declares `-a, --accumulate`, so argparse also takes -av, -va, + --accum and --acc; hil-validate.js tells the operator to retry 'adding -v'.""" + for spelling in ('--accumulate', '-a', '-av', '-va', '--accum', '--acc'): + self.assertEqual(self._gate(spelling), '1', f'{spelling} was not detected') + + def test_a_run_without_it_is_not_treated_as_accumulate(self): + for spelling in ('-b', '-v', '--retry'): + self.assertEqual(self._gate(spelling), '0', f'{spelling} falsely detected') + + def test_the_sidecar_is_uploaded_and_gated(self): + sh = (Path(HIL_DIR) / 'hil_ci.sh').read_text(encoding='utf-8') + up = [ln for ln in sh.splitlines() + if 'scp' in ln and 'hil_report.json' in ln and '$REMOTE:' in ln] + self.assertTrue(up, 'nothing uploads hil_report.json; --accumulate has no merge base') + self.assertIn('if [ "$ACCUMULATE" = 1 ]', sh, 'the upload is not gated') + + def test_a_missing_merge_base_is_loud(self): + """The damage: --accumulate with nothing to merge onto succeeds and quietly + publishes a small table where a full one used to be.""" + warn = [ln for ln in (Path(HIL_DIR) / 'hil_ci.sh').read_text().splitlines() + if 'warning' in ln.lower() and 'accumulate' in ln.lower()] + self.assertTrue(warn, 'no warning when --accumulate has no local sidecar') + + +class RunOutcomeAndRigHealthAreSeparate(unittest.TestCase): + """`banner` describes the CONDITIONS cells were collected under, so it carries across a + retry. `caveat` describes how a RUN ENDED, so it must not: a clean retry that reports + an earlier attempt's abandonment tells the agent a green run failed.""" + + def test_a_clean_retry_drops_the_previous_abandon_notice(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report([('a', 0, 0, [('a', {'t': 'OK'}, '1s')], 0)], + rd, True, '', '') + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + hil_report.accumulate_report([('a', 0, 0, [('a', {'t': 'OK'}, '2s')], 0)], + rd, False, '', '') + self.assertEqual(json.loads((rd / hil_report.REPORT_JSON).read_text())['caveat'], '') + + def test_rig_health_still_carries_across_the_retry(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.write_timeout_report(rd, [{'name': 's'}], 3600, + prefix='> **Rig note.** wedged\n') + hil_report.accumulate_report([('a', 0, 0, [('a', {'t': 'OK'}, '1s')], 0)], + rd, False, '', '') + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertIn('Rig note', doc['banner']) + + def test_a_second_attempts_abandon_is_recorded(self): + """_already_abandoned matched a notice carried forward from an EARLIER attempt, so + a genuinely new abandon wrote nothing and the run's own failure vanished.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report( + [('a', 0, 0, [('a', {'t': 'OK'}, '1s')], 0)], rd, True, '', + '> **Rig note.** x\n', + caveat='**HIL run abandoned: worker pool timed out after 3600s.**\n') + hil_report.accumulate_report([('a', 0, 0, [('a', {'t': 'OK'}, '2s')], 0)], + rd, False, '', '') + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + self.assertIn('would not shut down', + json.loads((rd / hil_report.REPORT_JSON).read_text())['caveat']) + + +class AMalformedSidecarNeverCostsTheReport(unittest.TestCase): + """hil_ci.sh now uploads a sidecar as the merge base, so a non-conforming one is + reachable from outside the harness.""" + + def _write(self, rd, doc): + (rd / hil_report.REPORT_JSON).write_text(json.dumps(doc)) + + def test_a_null_banner_does_not_kill_a_successful_run(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + self._write(rd, {'rows': [{'board': 'a', 'cells': {'t': 'pass'}, 'duration': '1s'}], + 'banner': None, 'caveat': None, 'scope': ''}) + hil_report.accumulate_report([('b', 0, 0, [('b', {'t': 'OK'}, '1s')], 0)], + rd, False, '', '') + self.assertTrue((rd / hil_report.REPORT_MD).is_file()) + + def test_a_null_cells_row_still_gets_its_pool_timeout_cell(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + self._write(rd, {'rows': [{'board': 'boardA', 'cells': None, 'duration': '61s'}], + 'banner': '', 'caveat': '', 'scope': ''}) + hil_report.write_timeout_report(rd, [{'name': 'boardA'}], 3600) + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + v = hil_report.summarize({'boards': [{'name': 'boardA'}]}, ['boardA'], doc) + self.assertFalse(v['results'][0]['pass'], + 'a board that ate the whole pool guard was published as a pass') + + def test_an_awkward_sidecar_still_gets_the_abandon_stamp(self): + """Any raise inside the dict branch was swallowed and the markdown fallback was + unreachable, so the stamp was lost from BOTH artifacts.""" + for bad in ({'rows': [{'board': 'a', 'cells': {'t': 'p'}, 'duration': 120}], + 'banner': '', 'caveat': '', 'scope': ''}, + {'rows': [{'board': 'a', 'cells': {'t': ['x']}, 'duration': '1s'}], + 'banner': None, 'caveat': '', 'scope': ''}): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + self._write(rd, bad) + (rd / hil_report.REPORT_MD).write_text('**✅ 27 passed**\n') + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + self.assertIn('abandoned', (rd / hil_report.REPORT_MD).read_text(encoding='utf-8'), + f'no stamp for {bad}') + + def test_a_malformed_roster_entry_still_leaves_an_artifact(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + import io + from contextlib import redirect_stdout + with redirect_stdout(io.StringIO()): + hil_report.write_timeout_report(rd, ['plainstring'], 3600) + self.assertTrue((rd / hil_report.REPORT_MD).is_file(), 'no artifact at all') + + +class PoolTimeoutOutranksAStaleLock(unittest.TestCase): + def test_a_wedge_is_not_published_as_lock_contention(self): + """locked was computed across every cell and short-circuited detail, so a board + that wedged the rig on the retry was reported as LOCKED -- and hil-validate.js + re-runs those, paying another pool guard on a board that just hung it.""" + doc = {'rows': [{'board': 'boardX', + 'cells': {'board-locked': 'fail', 'pool-timeout': 'fail'}, + 'duration': None}], 'banner': '', 'caveat': '', 'scope': ''} + r = hil_report.summarize({'boards': [{'name': 'boardX'}]}, ['boardX'], doc)['results'][0] + self.assertFalse(r['locked'], 'a wedge was published as lock contention') + self.assertFalse(r['pass']) + + def test_a_run_aborted_board_is_not_published_as_lock_contention(self): + """run-aborted is written by the same _abort_report path as pool-timeout, for a + board the guard never reached. It has to outrank a stale lock cell for the same + reason -- otherwise hil-validate.js re-runs a board whose worker RAISED.""" + doc = {'rows': [{'board': 'boardX', + 'cells': {'board-locked': 'fail', 'run-aborted': 'fail'}, + 'duration': None}], 'banner': '', 'caveat': '', 'scope': ''} + r = hil_report.summarize({'boards': [{'name': 'boardX'}]}, ['boardX'], doc)['results'][0] + self.assertFalse(r['locked'], 'an aborted run was published as lock contention') + self.assertFalse(r['pass']) + + +class NoBoardsExitRespectsFreshness(unittest.TestCase): + def test_a_fresh_run_does_not_republish_the_previous_rows(self): + """It is called BEFORE the fresh wipe, so it re-published last run's green table + under this run's red job -- the stale-table failure it exists to prevent.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report([('a', 0, 0, [('a', {'t': 'OK'}, '1s')], 0)], + rd, True, '', '') + hil_report.mark_report_no_boards(rd, 'filters emptied', fresh=True) + self.assertEqual(json.loads((rd / hil_report.REPORT_JSON).read_text())['rows'], []) + + def test_an_accumulate_run_keeps_them(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report([('a', 0, 0, [('a', {'t': 'OK'}, '1s')], 0)], + rd, True, '', '') + hil_report.mark_report_no_boards(rd, 'filters emptied', fresh=False) + self.assertEqual([r['board'] for r in json.loads( + (rd / hil_report.REPORT_JSON).read_text())['rows']], ['a']) + + def test_it_does_not_overwrite_an_abandon_notice(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report([('a', 0, 0, [('a', {'t': 'OK'}, '1s')], 0)], + rd, True, '', '') + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + hil_report.mark_report_no_boards(rd, 'filters emptied', fresh=False) + self.assertIn('abandoned', + json.loads((rd / hil_report.REPORT_JSON).read_text())['caveat']) + + +class TheNoBoardsCallSiteIsWired(unittest.TestCase): + """The fresh/accumulate branches of mark_report_no_boards were tested by calling it + DIRECTLY, so both passed while hil_test.py's one real call site never passed the flag + at all -- an --accumulate run whose filter emptied still wiped the accumulated rows. + This drives hil_test.py itself; the no-boards exit needs only a config and a filter + that matches nothing, so it costs no hardware.""" + + def _run(self, *extra): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / 'cfg.json').write_text(json.dumps( + {'boards': [{'name': 'alpha', 'uid': '1', 'flasher': {'name': 'jlink', 'uid': '2'}}]})) + (rd / hil_report.REPORT_JSON).write_text(json.dumps( + {'rows': [{'board': 'earlier', 'cells': {'t': 'pass'}, 'duration': '1s'}], + 'banner': '', 'scope': '', 'caveat': ''})) + r = subprocess.run( + [sys.executable, str(Path(HIL_DIR) / 'hil_test.py'), + '--flasher', 'nonexistent', *extra, str(rd / 'cfg.json')], + capture_output=True, text=True, timeout=120, + env={**os.environ, 'HIL_REPORT_DIR': str(rd)}) + self.assertEqual(r.returncode, 1, r.stdout + r.stderr) + return json.loads((rd / hil_report.REPORT_JSON).read_text()) + + def test_an_accumulate_run_keeps_the_accumulated_rows(self): + doc = self._run('--accumulate') + self.assertEqual([r['board'] for r in doc['rows']], ['earlier'], + "the call site did not pass fresh=not args.accumulate") + self.assertIn('selected no boards', doc['caveat']) + + def test_a_fresh_run_does_not_republish_them(self): + doc = self._run() + self.assertEqual(doc['rows'], []) + self.assertIn('selected no boards', doc['caveat']) + + +class EveryWriterRendersBeforeItCommits(unittest.TestCase): + def test_accumulate_report_does_not_commit_json_then_fail_to_render(self): + """accumulate_report hand-rolled the write instead of calling write_report, so a + render failure left the sidecar ahead of the markdown -- the exact ordering + write_report's docstring forbids.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_JSON).write_text(json.dumps( + {'rows': [{'board': 'boardA', 'cells': {'t': 'pass'}, 'duration': 119.0}], + 'banner': '', 'caveat': '', 'scope': ''})) + hil_report.accumulate_report([('boardB', 0, 0, [('boardB', {'t': 'OK'}, '1s')], 0)], + rd, False, '', '') + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertEqual((rd / hil_report.REPORT_MD).read_text(), + hil_report.render_report(doc) + '\n') + + +class MissingSidecarDoesNotDestroyTheMarkdown(unittest.TestCase): + def test_an_absent_sidecar_keeps_the_prior_table(self): + """`recovered` was only cleared when the sidecar was TORN, not when it was absent + -- reachable from hil_ci.sh's asymmetric copy-back and build.yml's skip marker.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / hil_report.REPORT_MD).write_text('| Board | t |\n| a | OK |\n| b | OK |\n') + hil_report.write_timeout_report(rd, [{'name': 'stuck'}], 3600) + self.assertIn('| a | OK |', (rd / hil_report.REPORT_MD).read_text()) + + +class LoadIsTheOnlyTrustBoundary(unittest.TestCase): + """hil_ci.sh uploads a sidecar as the merge base, so these shapes arrive from OUTSIDE + the harness. Every one of these raised past a handler before.""" + + def _seed(self, rd, raw): + (rd / hil_report.REPORT_JSON).write_text(raw if isinstance(raw, str) + else json.dumps(raw)) + + def test_a_non_list_rows_does_not_raise(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + self._seed(rd, {'rows': 1, 'banner': '', 'caveat': '', 'scope': ''}) + hil_report.accumulate_report([('a', 0, 0, [('a', {'t': 'OK'}, '1s')], 0)], + rd, False, '', '') + self.assertTrue((rd / hil_report.REPORT_MD).is_file()) + + def test_an_unhashable_cell_value_does_not_raise(self): + """render_matrix does REPORT_CELL.get(v, v); an unhashable value raised TypeError + on the NORMAL accumulate path.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + self._seed(rd, {'rows': [{'board': 'a', 'cells': {'t': ['x'], 'u': 'pass'}, + 'duration': '1s'}], + 'banner': '', 'caveat': '', 'scope': ''}) + hil_report.accumulate_report([('b', 0, 0, [('b', {'t': 'OK'}, '1s')], 0)], + rd, False, '', '') + cells = {r['board']: r['cells'] + for r in json.loads((rd / hil_report.REPORT_JSON).read_text())['rows']} + self.assertNotIn('t', cells['a'], 'a corrupt cell must drop, not become a pass') + self.assertIn('u', cells['a']) + + def test_summarize_survives_a_malformed_sidecar_via_load(self): + """The CLI is the one reader an agent's verdict depends on.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + (rd / 'cfg.json').write_text(json.dumps({'boards': [{'name': 'a'}]})) + self._seed(rd, {'rows': [{'board': 1, 'cells': 'notadict'}, + {'board': 'a', 'cells': {'t': 'pass'}}], + 'banner': '', 'caveat': '', 'scope': ''}) + r = subprocess.run( + [sys.executable, str(Path(HIL_DIR) / 'helper' / 'hil_report.py'), + str(rd / 'cfg.json'), '-b', 'a', '--report-dir', str(rd)], + capture_output=True, text=True, timeout=60) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertTrue(json.loads(r.stdout)['results'][0]['pass']) + + +class NoBoardsGuardOnlyAppliesWhenAccumulating(unittest.TestCase): + def test_a_fresh_run_carries_nothing_from_the_prior_sidecar(self): + """rows were reset on fresh but banner and scope were not, so a leftover or + uploaded sidecar republished a stale rig-health note and a stale scope line under + this run's notice.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report([('old', 0, 0, [('old', {'t': 'OK'}, '1s')], 0)], + rd, True, '3 board(s) — a, b, c', + '> **Rig note.** stale D-state holder\n') + hil_report.mark_report_no_boards(rd, 'filters emptied', fresh=True) + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertEqual(doc['rows'], []) + self.assertEqual(doc['banner'], '', 'a stale rig-health banner was republished') + self.assertEqual(doc['scope'], '', 'a stale scope note was republished') + self.assertNotIn('Rig note', (rd / hil_report.REPORT_MD).read_text()) + + def test_an_accumulate_run_keeps_banner_and_scope(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report([('old', 0, 0, [('old', {'t': 'OK'}, '1s')], 0)], + rd, True, '3 board(s) — a, b, c', + '> **Rig note.** real\n') + hil_report.mark_report_no_boards(rd, 'filters emptied', fresh=False) + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertIn('Rig note', doc['banner']) + self.assertEqual([r['board'] for r in doc['rows']], ['old']) + + def test_a_fresh_run_is_not_blocked_by_a_prior_abandon(self): + """The guard runs BEFORE the fresh wipe, so guarding a fresh run left the previous + attempt's rows AND its abandon notice published as this run's.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + hil_report.accumulate_report([('old', 0, 0, [('old', {'t': 'OK'}, '1s')], 0)], + rd, True, '', '') + hil_report.mark_report_abandoned(rd, 'the worker pool would not shut down.') + hil_report.mark_report_no_boards(rd, 'filters emptied', fresh=True) + doc = json.loads((rd / hil_report.REPORT_JSON).read_text()) + self.assertEqual(doc['rows'], []) + self.assertIn('selected no boards', doc['caveat']) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/hil/test/test_hil_rtt.py b/test/hil/test/test_hil_rtt.py new file mode 100644 index 000000000..3a07f13ec --- /dev/null +++ b/test/hil/test/test_hil_rtt.py @@ -0,0 +1,506 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests for hil_util.JlinkRtt and the rtt.py CLI against a fake JLinkExe +# on PATH — real subprocesses and sockets, no hardware, stdlib only, so the pre-commit +# hil-test hook can run this on GitHub's bare runner. Run directly: +# python3 test/hil/test/test_hil_rtt.py +import os +import subprocess +import sys +import tempfile +import time +import unittest +from contextlib import suppress as contextlib_suppress +from pathlib import Path + +# the module under test lives in the parent dir's helper/ package +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from helper import hil_util + +CLI = Path(__file__).resolve().parents[3] / 'tools' / 'rtt.py' + +# Serves -RTTTelnetPort like J-Link Commander: greets, echoes input uppercased, exits on +# stdin 'exit' (JlinkRtt.close()'s contract). FAKE_JLINK_MODE=die_after_greet sends the +# greeting then drops the connection and exits — the probe-unplug/crash case; +# FAKE_JLINK_MODE=tick also streams a line every 50 ms — the continuous-capture case. +FAKE_JLINK = '''#!/usr/bin/env python3 +import os, socket, sys, threading, time +port = int(sys.argv[sys.argv.index('-RTTTelnetPort') + 1]) +srv = socket.socket(); srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +srv.bind(('127.0.0.1', port)); srv.listen(1) +mode = os.environ.get('FAKE_JLINK_MODE', '') +def serve(): + conn, _ = srv.accept() + # the real server sends its banner AT CONNECT, before the control block is + # found — target data only flows later; the CLI's -i gate must not release + # on the banner + conn.sendall(b'SEGGER J-Link fake - Real time terminal output\\r\\n' + b'J-Link FakeProbe V1.0, SN=000\\r\\nProcess: JLinkExe\\r\\n') + if mode == 'banner_only': + while True: + if not conn.recv(4096): os._exit(0) + if mode == 'rst': + import struct + conn.recv(4096) # wait for the client to speak, then reset the connection + conn.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) + conn.close(); os._exit(0) + if mode == 'late_cb': + # models JLinkExe before it finds the control block: client bytes sent in + # this window are silently dropped, output starts only after the "attach" + end = time.time() + 1.0 + conn.setblocking(False) + while time.time() < end: + try: + conn.recv(4096) # discard early input like the real server + except OSError: + pass + time.sleep(0.05) + conn.setblocking(True) + conn.sendall(b'hello from target\\r\\n') + if mode == 'die_after_greet': + conn.close(); os._exit(0) + if mode == 'tick': + def tick(): + try: + while True: + time.sleep(0.05); conn.sendall(b'tick\\r\\n') + except OSError: + pass + threading.Thread(target=tick, daemon=True).start() + while True: + d = conn.recv(4096) + if not d: return + conn.sendall(d.upper()) +threading.Thread(target=serve, daemon=True).start() +for line in sys.stdin: + if line.strip() == 'exit': break +''' + +BOARD = {'flasher': {'uid': '000', 'args': '-device FAKE'}} + + [email protected](os.name == 'nt', 'POSIX PATH/exec semantics') +class JlinkRttFakeProbe(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls._dir = tempfile.TemporaryDirectory() + fake = Path(cls._dir.name) / 'JLinkExe' + fake.write_text(FAKE_JLINK) + fake.chmod(0o755) + cls._path = f'{cls._dir.name}{os.pathsep}{os.environ["PATH"]}' + + @classmethod + def tearDownClass(cls): + cls._dir.cleanup() + + def _fake_path(self): + # register the restore BEFORE mutating, then prepend the fake tool dir + self.addCleanup(os.environ.__setitem__, 'PATH', os.environ['PATH']) + os.environ['PATH'] = self._path + + def _console(self, mode=''): + self._fake_path() + if mode: + os.environ['FAKE_JLINK_MODE'] = mode + self.addCleanup(os.environ.pop, 'FAKE_JLINK_MODE', None) + con = hil_util.JlinkRtt(BOARD, timeout=0.1) + self.addCleanup(con.close) + return con + + def _read_until(self, con, want, timeout=3): + out = b'' + end = time.monotonic() + timeout + while want not in out and time.monotonic() < end: + out += con.read(con.in_waiting or 1) + return out + + def test_read_and_echo_write(self): + con = self._console() + self.assertIn(b'hello from target', self._read_until(con, b'hello from target')) + self.assertEqual(con.write(b'ping'), 4) + self.assertIn(b'PING', self._read_until(con, b'PING')) + + def test_eof_latched_when_server_dies(self): + con = self._console(mode='die_after_greet') + self._read_until(con, b'hello from target') + end = time.monotonic() + 3 + while not con.eof and time.monotonic() < end: + time.sleep(0.05) + self.assertTrue(con.eof) # dead server is detected, not spun on + t0 = time.monotonic() + self.assertEqual(con.read(64), b'') # empty, paced like a serial timeout + elapsed = time.monotonic() - t0 + self.assertLess(elapsed, 0.5) # bounded by the 0.1 s timeout, not hung + self.assertGreater(elapsed, 0.02) # ...but not a busy-spin fast return + con.timeout = None # pyserial's block-forever mode must + t0 = time.monotonic() # ALSO pace (0.1 s default), not spin + self.assertEqual(con.read(64), b'') + elapsed = time.monotonic() - t0 + self.assertLess(elapsed, 0.5) + self.assertGreater(elapsed, 0.02) + con.timeout = 0.1 + + def test_reset_input_buffer(self): + con = self._console() + self._read_until(con, b'hello from target') + con.write(b'x') + time.sleep(0.3) + con.reset_input_buffer() + self.assertEqual(con.in_waiting, 0) + + def test_write_after_close_raises_runtimeerror(self): + con = self._console() + con.close() + with self.assertRaises(RuntimeError): + con.write(b'x') + + def test_write_after_server_death_raises(self): + # TCP accepts one send after peer death — write() must refuse instead of + # "succeeding" into the void + con = self._console(mode='die_after_greet') + self._read_until(con, b'hello from target') + end = time.monotonic() + 3 + while not con.eof and time.monotonic() < end: + time.sleep(0.05) + with self.assertRaises(RuntimeError): + con.write(b'ping') + + def test_read_after_close_raises_runtimeerror(self): + con = self._console() + self._read_until(con, b'hello from target') + con.close() + with self.assertRaises(RuntimeError): + con.read(1) + + def test_missing_jlinkexe_raises_runtimeerror(self): + self._fake_path() + os.environ['PATH'] = self._dir.name # no python3 either, but JLinkExe fails first + os.rename(f'{self._dir.name}/JLinkExe', f'{self._dir.name}/JLinkExe.off') + self.addCleanup(os.rename, f'{self._dir.name}/JLinkExe.off', f'{self._dir.name}/JLinkExe') + with self.assertRaises(RuntimeError): + hil_util.JlinkRtt(BOARD, timeout=0.1) + + def test_close_reaps_the_server(self): + con = self._console() + proc = con._proc + con.close() + self.assertIsNotNone(proc.poll()) # no zombie, no probe held + + def test_cli_exits_when_server_dies(self): + # --seconds 0 must end on server EOF (rc 1), not hang forever + env = dict(os.environ, PATH=self._path, FAKE_JLINK_MODE='die_after_greet') + r = subprocess.run([sys.executable, str(CLI), + '--backend', 'jlink', '--probe', '000', '--device', 'FAKE', '--seconds', '0'], + env=env, capture_output=True, timeout=20) + self.assertEqual(r.returncode, 1) + self.assertIn(b'hello from target', r.stdout) + self.assertIn(b'server closed', r.stderr) + + def test_peer_reset_latches_eof(self): + # a killed server closes with RST when bytes are unread; the read side must + # LATCH eof (so the harness's `assert not ser.eof` triage fires) and never + # leak ConnectionResetError/ValueError to in_waiting/eof callers + con = self._console(mode='rst') + # rst mode sends only the banner (it RSTs on first input) -- wait for the + # banner tail, not target output that never comes + self._read_until(con, b'Process: JLinkExe') + con.write(b'x') # fake resets the connection on input + end = time.monotonic() + 3 + try: + while not con.eof and time.monotonic() < end: + con.in_waiting # must not raise across the RST + time.sleep(0.05) + except Exception as e: # noqa: BLE001 - the regression this guards + self.fail(f'{type(e).__name__} escaped the latch-only contract: {e}') + self.assertTrue(con.eof) + with self.assertRaises(hil_util.RttError): + con.write(b'y') # dead server refuses writes + + def test_write_timeout_env_rejects_inf(self): + # hil_util's twin rejects inf for the same reason: an unbounded write is what + # this knob exists to bound + import importlib.util as ilu + from pathlib import Path as _P + spec = ilu.spec_from_file_location('rtt_env_probe', _P(CLI)) + mod = ilu.module_from_spec(spec) + old = os.environ.get('HIL_SERIAL_WRITE_TIMEOUT') + os.environ['HIL_SERIAL_WRITE_TIMEOUT'] = 'inf' + self.addCleanup(lambda: os.environ.__setitem__('HIL_SERIAL_WRITE_TIMEOUT', old) + if old is not None else os.environ.pop('HIL_SERIAL_WRITE_TIMEOUT', None)) + spec.loader.exec_module(mod) + self.assertEqual(mod.RTT_WRITE_TIMEOUT, 10) + + def test_cli_rejects_bad_seconds_and_jlink_channel(self): + def run(*a): + return subprocess.run([sys.executable, str(CLI), *a], capture_output=True, timeout=15) + for bad in ('-5', 'nan'): + r = run('--backend', 'jlink', '--probe', '000', '--device', 'FAKE', '--seconds', bad) + self.assertEqual(r.returncode, 2, f'--seconds {bad} was accepted') + # the jlink telnet route serves channel 0 only; asking for another is an error, + # not silence (--dump can read any ring, so it stays allowed there) + r = run('--backend', 'jlink', '--probe', '000', '--device', 'FAKE', '--channel', '1') + self.assertEqual(r.returncode, 2) + self.assertIn(b'channel 0 only', r.stderr) + # a negative index would walk backwards off aUp[] (dump route included) + r = run('--backend', 'jlink', '--probe', '000', '--device', 'FAKE', '--channel', '-1') + self.assertEqual(r.returncode, 2) + self.assertIn(b'>= 0', r.stderr) + + def test_pyserial_surface_contracts(self): + con = self._console() + self._read_until(con, b'hello from target') + con.write(b'abcdef') + self._read_until(con, b'ABC') # echo queued + before = con.in_waiting + self.assertEqual(con.read(0), b'') # pyserial: consumes nothing + self.assertEqual(con.read(-1), b'') # never hand over/destroy bytes + self.assertEqual(con.in_waiting, before) + con.timeout = None # pyserial: block until satisfied + con.write(b'xy') # fresh echo guarantees the read returns + self.assertEqual(len(con.read(2)), 2) + con.timeout = 0.1 + con.close() + with self.assertRaises(hil_util.RttError): + con.in_waiting # closed console reports closed, not healthy + self.assertTrue(con.eof) + + def test_context_manager_closes(self): + self._fake_path() + with hil_util.JlinkRtt(BOARD, timeout=0.1) as con: + proc = con._proc + self.assertIsNotNone(proc.poll()) # __exit__ released the probe + + def test_staging_and_banner_coupling(self): + # tripwires for couplings no import-walk can see: + # (a) hil_ci.sh must stage tools/rtt.py -- hil_util exec_module's it, so an + # unstaged rig tree kills every harness import + hil_ci = (Path(__file__).resolve().parents[1] / 'hil_ci.sh').read_text() + self.assertIn('tools/rtt.py', hil_ci) + # (b) the shared RTT banner filter must drop ALL THREE J-Link banner lines, + # including the middle one, which is the PROBE MODEL string and in + # libjlinkarm carries no 'SEGGER ' prefix (J-Link OH3, J-Trace H9...) + banner_re = hil_util.RTT_BANNER_RE + for line in ('SEGGER J-Link V9.66 - Real time terminal output', + 'SEGGER J-Link LPC-Link 2 V1.0, SN=611000000', + 'J-Link OH3 V1.0, SN=123456789', + 'J-Trace H9 V2.0, SN=123456789002', + 'Process: JLinkExe'): + self.assertTrue(banner_re.match(line), f'banner line not filtered: {line!r}') + for line in ('Hello from TinyUSB', 'USBD init on controller 0', + 'ID 1a86:8010 SN 7FD88F0604B5', 'echo:p'): + self.assertFalse(banner_re.match(line), f'target line wrongly filtered: {line!r}') + + def test_pool_check_dead_rtt_board_is_not_alive(self): + # JLinkExe's banner alone must not score a dead board 'alive': pool_check's + # rtt aliveness judges only target bytes (the bug: unfiltered, the banner + # made `not boardtest_output(data)` true on the first poll) + sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + from helper import hil_pool_check + # a dead board burns the whole poll window; the verdict is the same at 0.5 s + self.addCleanup(setattr, hil_pool_check, 'SERIAL_WAIT', hil_pool_check.SERIAL_WAIT) + hil_pool_check.SERIAL_WAIT = 0.5 + self._fake_path() + os.environ['FAKE_JLINK_MODE'] = 'banner_only' + self.addCleanup(os.environ.pop, 'FAKE_JLINK_MODE', None) + board = dict(BOARD, name='deadboard', logger='rtt') + got = hil_pool_check.check_host_serial(board, do_reset=False, want_hello=True) + self.assertEqual(got, b'') # dead, not "alive on banner" + + def test_cli_arg_contract(self): + # --backend is explicit (no default); vid-pid is openocd-only; the openocd + # backend accepts --addr instead of --elf and --vid-pid instead of --probe + def run(*a, inp=b''): + return subprocess.run([sys.executable, str(CLI), *a], + input=inp, capture_output=True, timeout=15) + r = run('--probe', '000', '--device', 'FAKE') # no --backend + self.assertEqual(r.returncode, 2) + self.assertIn(b'--backend', r.stderr) + r = run('--backend', 'jlink', '--probe', '000', '--device', 'FAKE', '--vid-pid', '0x1 0x2') + self.assertEqual(r.returncode, 2) # vid-pid is openocd-only + r = run('--backend', 'openocd', '--cfg', '-f x.cfg', '--addr', '0x20000000') + self.assertEqual(r.returncode, 2) # needs --probe or --vid-pid + self.assertIn(b'vid-pid', r.stderr) + r = run('--backend', 'openocd', '--probe', '000', '--cfg', '-f x.cfg', '--addr', 'nothex') + self.assertEqual(r.returncode, 2) + self.assertIn(b'hex', r.stderr) + + def test_cli_interactive_echo(self): + env = dict(os.environ, PATH=self._path) + r = subprocess.run([sys.executable, str(CLI), + '--backend', 'jlink', '--probe', '000', '--device', 'FAKE', '--seconds', '2', '-i'], + env=env, input=b'hi', capture_output=True, timeout=20) + self.assertEqual(r.returncode, 0) + self.assertIn(b'HI', r.stdout) # bytes forwarded without needing a newline + self.assertNotIn(b'never forwarded', r.stderr) # forwarding happened: no false alarm + + def test_cli_interactive_input_held_until_output(self): + # input piped at process start must survive the server's control-block hunt + # (the real JLinkExe drops client bytes until the block is found — measured + # on the rig: instant 'ping' lost, delayed 'ping' echoed) + env = dict(os.environ, PATH=self._path, FAKE_JLINK_MODE='late_cb') + r = subprocess.run([sys.executable, str(CLI), + '--backend', 'jlink', '--probe', '000', '--device', 'FAKE', '--seconds', '3', '-i'], + env=env, input=b'hi', capture_output=True, timeout=25) + self.assertEqual(r.returncode, 0) + self.assertIn(b'HI', r.stdout) + + def test_cli_interactive_no_input_diagnostic(self): + # -i with stdin closed immediately: the diagnostic must say stdin was never + # forwarded (true), keyed on actual forwarding -- not on the attach gate, + # which releases after 5 s and forwards anyway on longer runs + env = dict(os.environ, PATH=self._path, FAKE_JLINK_MODE='banner_only') + r = subprocess.run([sys.executable, str(CLI), + '--backend', 'jlink', '--probe', '000', '--device', 'FAKE', '--seconds', '1', '-i'], + env=env, input=b'', capture_output=True, timeout=20) + self.assertEqual(r.returncode, 0) + self.assertIn(b'never forwarded', r.stderr) + self.assertIn(b'no target output', r.stderr) + + def test_cli_downstream_pipe_close(self): + # a real `rtt.py | head`-style consumer: close the read end mid-stream + # and the CLI must exit 0 via its BrokenPipe path, not traceback (this test + # fails if the handler is removed — subprocess.run capture can't cover it) + env = dict(os.environ, PATH=self._path, FAKE_JLINK_MODE='tick') + p = subprocess.Popen([sys.executable, str(CLI), + '--backend', 'jlink', '--probe', '000', '--device', 'FAKE', '--seconds', '8'], + env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + p.stdout.read(10) # let it stream a little + p.stdout.close() # downstream hangs up + rc = p.wait(timeout=20) + err = p.stderr.read() + p.stderr.close() + self.assertEqual(rc, 0, err) + self.assertNotIn(b'Traceback', err) + + def test_cli_feeder_races_shutdown(self): + # a feeder still writing when --seconds expires must not crash the CLI + # (pump thread vs close() race: historically tracebacks and SIGABRT rc 134) + env = dict(os.environ, PATH=self._path) + for _ in range(3): + p = subprocess.Popen([sys.executable, str(CLI), + '--backend', 'jlink', '--probe', '000', '--device', 'FAKE', '--seconds', '1', '-i'], + env=env, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE) + try: + while True: + p.stdin.write(b'hi\n') + p.stdin.flush() + time.sleep(0.01) + except (BrokenPipeError, OSError): + pass + rc = p.wait(timeout=20) + err = p.stderr.read() + p.stderr.close() + with contextlib_suppress(OSError, ValueError): + p.stdin.close() + self.assertEqual(rc, 0, err) + self.assertNotIn(b'Exception in thread', err) + + + +class StripBanner(unittest.TestCase): + # both harness consumers (device_info verdict, pool_check aliveness) judge + # target-aliveness through this ONE filter -- pin its shape here + def test_drops_banner_keeps_target(self): + raw = (b'SEGGER J-Link V9.66 - Real time terminal output\r\n' + b'J-Link OH3 V1.0, SN=123456789\r\nProcess: JLinkExe\r\n' + b'Hello from TinyUSB\r\n') + self.assertEqual(hil_util.strip_banner(raw), b'Hello from TinyUSB') + + def test_complete_only_drops_split_banner_fragment(self): + # a poll loop can catch the banner mid-line at a read boundary; the + # fragment must not defeat the prefix regex and score as target output + frag = b'SEGGER J-Link V9.66 - Real time terminal output\r\nProce' + self.assertEqual(hil_util.strip_banner(frag, complete_only=True), b'') + # the final verdict keeps a genuine unterminated target tail + self.assertEqual(hil_util.strip_banner(b'tud_task\r\nrunn'), b'tud_task\nrunn') + self.assertEqual(hil_util.strip_banner(b'', complete_only=True), b'') + + +# Serves like `openocd ... -c "rtt server start PORT CH"`: parses the port from its +# single shell-quoted command line, greets, echoes uppercased. No banner (matches the +# real openocd rtt server, which sends target data only). +FAKE_OPENOCD = '''#!/usr/bin/env python3 +import os, re, socket, sys, threading, time +if os.environ.get('FAKE_OPENOCD_ARGV'): + open(os.environ['FAKE_OPENOCD_ARGV'], 'w').write(' '.join(sys.argv)) +port = int(re.search(r'rtt server start (\\d+)', ' '.join(sys.argv)).group(1)) +srv = socket.socket(); srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +srv.bind(('127.0.0.1', port)); srv.listen(1) +conn, _ = srv.accept() +conn.sendall(b'hello from target\\r\\n') +while True: + d = conn.recv(4096) + if not d: break + conn.sendall(d.upper()) +''' + + [email protected](os.name == 'nt', 'POSIX PATH/exec semantics') +class OpenocdRttFakeProbe(unittest.TestCase): + """The openocd-backend class shares its whole read/write/eof contract with + JlinkRtt via the base class (covered above); this exercises the parts it owns: + spawn/connect, echo round-trip, teardown.""" + + @classmethod + def setUpClass(cls): + cls._dir = tempfile.TemporaryDirectory() + fake = Path(cls._dir.name) / 'openocd' + fake.write_text(FAKE_OPENOCD) + fake.chmod(0o755) + cls._path = f'{cls._dir.name}{os.pathsep}{os.environ["PATH"]}' + + @classmethod + def tearDownClass(cls): + cls._dir.cleanup() + + def _fake_path(self): + self.addCleanup(os.environ.__setitem__, 'PATH', os.environ['PATH']) + os.environ['PATH'] = self._path + + def test_reset_before_attach_shapes_the_command(self): + # SystemView-style consumers need the server draining WHEN the target boots + # (its Init record is emitted once); the opt-in flag must put `reset run` + # between init and rtt setup, and must not appear otherwise + self._fake_path() + argv_file = os.path.join(self._dir.name, 'argv.txt') + os.environ['FAKE_OPENOCD_ARGV'] = argv_file + self.addCleanup(os.environ.pop, 'FAKE_OPENOCD_ARGV', None) + for flag, want in ((True, True), (False, False)): + con = hil_util.OpenocdRtt('-f fake.cfg', 0x20000000, 1, serial_no='000', + reset_before_attach=flag) + try: + argv = Path(argv_file).read_text() + finally: + con.close() + self.assertEqual('reset run' in argv, want, argv) + if want: # ordering is the whole point: reset, settle, THEN attach + self.assertLess(argv.index('reset run'), argv.index('rtt setup'), argv) + self.assertIn('sleep 2000', argv) + self.assertIn('rtt server start', argv) + self.assertTrue(argv.rstrip().endswith('1'), argv) # channel threaded through + + def test_openocd_route_echo_and_teardown(self): + self._fake_path() + con = hil_util.OpenocdRtt('-f fake.cfg', 0x20000000, 0, + serial_no='000', vid_pid='0x1234 0x5678') + self.addCleanup(con.close) + out = b'' + end = time.monotonic() + 3 + while b'hello from target' not in out and time.monotonic() < end: + out += con.read(con.in_waiting or 1) + self.assertIn(b'hello from target', out) + con.write(b'ping') + end = time.monotonic() + 3 + while b'PING' not in out and time.monotonic() < end: + out += con.read(con.in_waiting or 1) + self.assertIn(b'PING', out) + proc = con._proc + con.close() + self.assertIsNotNone(proc.poll()) # no zombie, no probe held + with self.assertRaises(RuntimeError): + con.write(b'x') # same post-close contract as JlinkRtt + + +if __name__ == '__main__': + unittest.main() diff --git a/test/hil/test/test_hil_util.py b/test/hil/test/test_hil_util.py index c95e20b6d..17abe52aa 100644 --- a/test/hil/test/test_hil_util.py +++ b/test/hil/test/test_hil_util.py @@ -6,11 +6,11 @@ # python3 test/hil/test/test_hil_util.py import io import os -import shutil -import tempfile import sys import time +import threading import unittest +from tempfile import TemporaryDirectory from contextlib import redirect_stdout from pathlib import Path @@ -19,7 +19,6 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from helper import hil_util [email protected](os.name == 'nt', 'POSIX shell commands') class RunCmdModes(unittest.TestCase): def test_default_mode_unchanged(self): r = hil_util.run_cmd('printf out; printf err >&2') @@ -146,9 +145,13 @@ class BottomLayer(unittest.TestCase): # hil_pool_check included: test_hil_util_is_a_single_module_instance imports it # on the bare runner, and its `import serial` is function-local for exactly # this reason -- hoisting it must fail HERE, not on every PR's pre-commit CI + # ../../tools/rtt: hil_util exec_module's it at import (helper/hil_util.py's + # loader block), so a non-stdlib import THERE kills ci_select on the bare + # runner just as surely -- and the spec_from_file_location call is invisible to + # the ast.Import walk below, which is why it must be listed explicitly for mod in ('helper/hil_util', 'hil_flash', '../../tools/ci_select', 'helper/hil_health', 'helper/hil_lock', 'helper/hil_pool_check', - '../../tools/build', '../../tools/build_utils'): + '../../tools/build', '../../tools/build_utils', '../../tools/rtt'): tree = ast.parse((hil_dir / f'{mod}.py').read_text()) # module level only: a deferred import inside a function cannot break # importability (hil_pool_check keeps `import serial` function-local @@ -164,52 +167,6 @@ class BottomLayer(unittest.TestCase): f'{mod}.py imports {root}, not stdlib/local - breaks the bare CI runner') -class BoundedReadBookkeeping(unittest.TestCase): - """Two ways the strand accounting lied, both of which cost a blindness credit -- and - the process goes blind after four.""" - - def test_a_value_that_arrived_at_the_deadline_is_not_a_strand(self): - """join() returns, is_alive() is still True, but the reader HAS deposited its - value. read_sysfs booked a strand from is_alive() alone, so a merely-slow healthy - read was memoised as unreadable forever. bounded_open already gets this right.""" - import threading, time as _t - before = hil_util._sysfs_stuck - self.addCleanup(setattr, hil_util, '_sysfs_stuck', before) - real_thread = threading.Thread - - class Lingering(real_thread): - """Deposits the value, then outlives the join by a hair.""" - def run(self): - super().run() - _t.sleep(0.6) # still alive when join(grace) returns - - self.addCleanup(setattr, threading, 'Thread', real_thread) - threading.Thread = Lingering - with tempfile.NamedTemporaryFile('w', suffix='_attr', delete=False) as fh: - fh.write('cafe\n') - path = fh.name - self.addCleanup(os.unlink, path) - hil_util.read_sysfs(path, grace=0.2) - self.assertEqual(hil_util._sysfs_stuck, before, - 'a value that arrived was still counted as a strand') - - def test_bounded_open_does_not_re_strand_a_known_path(self): - """Same rule read_sysfs has: re-opening a path known to hang costs another thread, - another fd and another blindness credit to learn what we already know. The printer - test re-opens ONE lp node on every retry.""" - d = tempfile.mkdtemp() - self.addCleanup(shutil.rmtree, d, True) - fifo = os.path.join(d, 'lp0') - os.mkfifo(fifo) # open() blocks: no writer, ever - self.addCleanup(setattr, hil_util, '_sysfs_stuck', hil_util._sysfs_stuck) - self.addCleanup(setattr, hil_util, '_sysfs_stranded', dict(hil_util._sysfs_stranded)) - before = hil_util._sysfs_stuck - for _ in range(3): - hil_util.bounded_open(fifo, os.O_WRONLY, 0.3) - self.assertLessEqual(hil_util._sysfs_stuck - before, 1, - 'each retry spent another blindness credit on the same path') - - class RunAlongsideKeepsStderrOffThePayload(unittest.TestCase): """test_device_printer_to_cdc byte-compares run_alongside's stdout against the payload it wrote. Merging stderr into that stream turns any stray child stderr byte -- a @@ -227,5 +184,265 @@ class RunAlongsideKeepsStderrOffThePayload(unittest.TestCase): 'child stderr leaked into the payload stream') +class RunCmdCleanupShape(unittest.TestCase): + """run_cmd's two cleanup paths, asserted structurally. + + Both must kill the process GROUP: start_new_session puts the child in its own group, so + a flasher run through a shell keeps children a p.kill() cannot reach, and on the + BaseException path the child never receives the terminal's SIGINT either. + + Structural rather than behavioural on purpose. Driving a real SIGINT into a blocked + communicate() from a unit test is timing-dependent, and a flaky guard on this block is + worse than none -- while what actually breaks it is an edit that rebinds a branch. Both + times this block has been mis-edited, an `else:` ended up attached to the `try` instead + of the `if` it belonged to, so `p.kill()` ran when killpg had SUCCEEDED and its + ProcessLookupError masked the caller's exception. That is a shape, and shapes are + exactly what an AST can pin. + """ + + def _run_cmd_ast(self): + import ast + src = Path(hil_util.__file__).read_text() + return next(n for n in ast.walk(ast.parse(src)) + if isinstance(n, ast.FunctionDef) and n.name == 'run_cmd') + + def test_no_cleanup_try_has_an_else(self): + import ast + for n in ast.walk(self._run_cmd_ast()): + if isinstance(n, ast.Try) and n.orelse: + self.fail(f'try/else at line {n.lineno}: an else here runs when the kill ' + f'SUCCEEDED, and its ProcessLookupError masks the caller\'s ' + f'exception -- this block has been mis-edited that way twice') + + def test_both_cleanup_paths_kill_the_group(self): + import ast + fn = self._run_cmd_ast() + killers = [getattr(c.func, 'attr', '') for c in ast.walk(fn) + if isinstance(c, ast.Call) and getattr(c.func, 'attr', '') in + ('killpg', 'kill')] + self.assertEqual(killers.count('killpg'), 2, + 'both the timeout and the BaseException path must killpg') + self.assertEqual(killers.count('kill'), 0, + 'p.kill() reaches only the direct child; a flasher run through a ' + 'shell keeps grandchildren it cannot touch') + + def test_the_interrupt_path_reraises(self): + import ast + fn = self._run_cmd_ast() + base = [h for n in ast.walk(fn) if isinstance(n, ast.Try) for h in n.handlers + if isinstance(h.type, ast.Name) and h.type.id == 'BaseException'] + self.assertTrue(base, 'the BaseException cleanup path is gone') + for h in base: + self.assertTrue(any(isinstance(x, ast.Raise) for x in ast.walk(h)), + 'the interrupt path must re-raise, or Ctrl-C is swallowed') + + +class BoundedReadForGuardlessCallers(unittest.TestCase): + """`serial` is served under the device lock a wedged usbfs ioctl holds, so the read is + bounded BY DEFAULT -- not opt-in. usb_scan reads it on every device matching the VID to + find the one it wants, and hil_lock.controller_of does that from controller_permit on + essentially every board, so one wedged DUT would stall every worker rather than one. + hil_pool_check has no guard behind it at all.""" + + def setUp(self): + from helper import hil_util + self.hil_util = hil_util + # the pre-commit hook runs all four suites in ONE interpreter, so capture and + # restore rather than assuming these start (or end) empty + for name in ('_stranded', '_strand_hits'): + self.addCleanup(setattr, hil_util, name, dict(getattr(hil_util, name))) + getattr(hil_util, name).clear() + self.addCleanup(setattr, hil_util, '_ever_stranded', hil_util._ever_stranded) + hil_util._ever_stranded = False + self.td = TemporaryDirectory() + self.addCleanup(self.td.cleanup) + self.fifo = os.path.join(self.td.name, 'serial') + os.mkfifo(self.fifo) # a read that never answers + + def test_a_wedged_attribute_gives_up_instead_of_hanging(self): + t0 = time.monotonic() + self.assertIsNone(self.hil_util.read_sysfs(self.fifo, timeout=0.3)) + self.assertLess(time.monotonic() - t0, 5, 'the bounded read did not give up') + + def test_the_bound_is_the_default_not_an_opt_in(self): + """usb_scan reads `serial` on every device matching the VID to find the one it + wants, and hil_lock's controller_of does that from controller_permit on + essentially every board -- so an opt-in bound that ONE call site forgets lets a + single wedged DUT stall every worker, not one. Three call sites forgot it once.""" + import inspect + for fn in (self.hil_util.read_sysfs, self.hil_util.usb_scan): + default = inspect.signature(fn).parameters['timeout'].default + self.assertEqual(default, self.hil_util.SYSFS_READ_GRACE, + f'{fn.__name__} must be bounded without being asked') + t0 = time.monotonic() + self.assertIsNone(self.hil_util.read_sysfs(self.fifo)) # no timeout= passed + self.assertLess(time.monotonic() - t0, 5, 'the default path did not bound') + + def test_a_node_that_returns_during_the_grace_is_not_memoised_as_wedged(self): + """The inode must be captured BEFORE the reader starts. Stat it afterwards and a + board that came back mid-read has its brand-new HEALTHY inode recorded as the + wedged one -- only a SECOND re-enumeration could ever clear it, and hil_pool_check + would report a successful recovery as still off the bus.""" + def swap(): + time.sleep(0.15) + os.unlink(self.fifo) + Path(self.fifo).write_text('CAFE01\n') + + threading.Thread(target=swap, daemon=True).start() + self.hil_util.read_sysfs(self.fifo, timeout=0.6) + self.assertEqual(self.hil_util.read_sysfs(self.fifo, timeout=1), 'CAFE01', + 'the healthy new inode was recorded as the wedged one') + + def test_concurrent_readers_of_one_path_spend_one_credit(self): + """hil_pool_check polls one bus from four threads. Counting each READER let four + threads on ONE wedged device spend four of the process budget between them -- + latching on the single wedge the tool was run to find.""" + ts = [threading.Thread(target=lambda: self.hil_util.read_sysfs(self.fifo, timeout=0.3)) + for _ in range(4)] + [t.start() for t in ts] + [t.join() for t in ts] + self.assertEqual(len(self.hil_util._stranded), 1) + self.assertEqual(self.hil_util._strand_hits[self.fifo], 1, + 'four readers of one path spent four credits') + + def test_a_flapping_wedged_device_cannot_leak_without_bound(self): + """The inode all-clear re-arms on every re-enumeration, so a device that flaps + while STILL wedged strands again each pass -- a thread and an fd per cycle.""" + for _ in range(self.hil_util._PATH_STRAND_MAX + 4): + self.hil_util.read_sysfs(self.fifo, timeout=0.2) + os.unlink(self.fifo) + os.mkfifo(self.fifo) # back on the same path, still wedged + self.assertEqual(self.hil_util._strand_hits[self.fifo], + self.hil_util._PATH_STRAND_MAX, + 'a flapping device kept stranding past its per-path cap') + + def test_a_value_that_arrived_at_the_deadline_is_not_a_strand(self): + """`out` is checked BEFORE is_alive(): a reader can deposit its value and still be + alive for a moment after join() returns. Counting that as a strand blacklists a + healthy attribute by inode forever AND latches sysfs_stranded for the process.""" + good = Path(self.td.name) / 'idVendor' + good.write_text('cafe\n') + real_thread = threading.Thread + + class Lingering(real_thread): # deposits, then outlives the join + def run(self): + super().run() + time.sleep(2) + + self.hil_util.threading.Thread = Lingering + self.addCleanup(setattr, self.hil_util.threading, 'Thread', real_thread) + self.assertEqual(self.hil_util.read_sysfs(str(good), timeout=0.3), 'cafe') + self.assertNotIn(str(good), self.hil_util._stranded) + self.assertFalse(self.hil_util.sysfs_stranded()) + + def test_path_stranded_answers_per_device_not_per_process(self): + """usbtest decides whether to run lock-taking cleanup on this result; the sticky + process-wide flag would let any peer's wedge answer for our board.""" + other = Path(self.td.name) / 'peer' + other.write_text('PEER\n') + self.hil_util.read_sysfs(self.fifo, timeout=0.3) + self.assertTrue(self.hil_util.path_stranded(self.fifo)) + self.assertFalse(self.hil_util.path_stranded(str(other))) + self.assertTrue(self.hil_util.sysfs_stranded(), 'the process-wide flag is sticky') + + def test_a_refused_read_is_stranded_not_vouched_for(self): + """usbtest fails CLOSED on path_stranded() before running remove_id/unbind, which + take the uninterruptible device_lock. Past _STRAND_MAX read_sysfs answers None + WITHOUT looking -- so answering False there hands that guard a fabricated + all-clear for a device nobody read, and the lock-taking cleanup runs on a wedge.""" + self.hil_util._stranded.update( + {f'/sys/fake/{i}': i for i in range(self.hil_util._STRAND_MAX)}) + self.assertIsNone(self.hil_util.read_sysfs(self.fifo, timeout=0.3)) + self.assertTrue(self.hil_util.path_stranded(self.fifo), + 'a path the reader refused to open was reported readable-and-absent') + + def test_a_stat_that_races_the_reader_still_memoises(self): + """The pre-read stat is the memo KEY, and it can fail while the open that follows + succeeds and blocks -- a node replaced between the two. Without a key the give-up + records nothing, so hil_pool_check's next poll starts another permanent thread and + fd for the same path, and repeats it every pass.""" + real_stat = self.hil_util.os.stat + calls = [] + + def flaky(path, *a, **kw): + calls.append(path) + if len(calls) == 1: # only the pre-read stat loses the race + raise OSError('vanished between stat and open') + return real_stat(path, *a, **kw) + + self.addCleanup(setattr, self.hil_util.os, 'stat', real_stat) + self.hil_util.os.stat = flaky + self.assertIsNone(self.hil_util.read_sysfs(self.fifo, timeout=0.3)) + self.assertIn(self.fifo, self.hil_util._stranded, + 'a lost stat race leaks a fresh reader on every later poll') + + def test_a_successful_read_clears_an_earlier_refusal(self): + """_refused feeds path_stranded(), which usbtest reads to tell "cannot tell" from + a real disconnect. Left sticky, a board that recovered and then genuinely left the + bus is classified as an unrecovered wedge for the rest of the process.""" + good = Path(self.td.name) / 'serial2' + good.write_text('ABC123\n') + self.hil_util._refused.add(str(good)) + self.addCleanup(self.hil_util._refused.discard, str(good)) + self.assertEqual(self.hil_util.read_sysfs(str(good), timeout=0.3), 'ABC123') + self.assertFalse(self.hil_util.path_stranded(str(good)), + 'a path that answered is still reported unreadable') + + def test_a_recovered_device_is_seen_again_on_the_same_busport(self): + """THE recovery flow: hil_pool_check resets or reflashes a wedged board, then + wait_device polls find_device -> scan_usb for the NEW inode. A busport does not + change when the board comes back on the same physical port, so a path-only + blacklist would make that poll look at everything except the device it is waiting + for -- the board recovers physically and the tool reports it gone for the rest of + the run. A re-enumeration destroys the kernfs node, so a changed inode is the + all-clear.""" + self.assertIsNone(self.hil_util.read_sysfs(self.fifo, timeout=0.3)) + # re-enumeration: same path, new node + os.unlink(self.fifo) + Path(self.fifo).write_text('CAFE01\n') + self.assertEqual(self.hil_util.read_sysfs(self.fifo, timeout=1), 'CAFE01', + 'a board that came back on the same busport stayed blacklisted') + + def test_the_caveat_stays_true_after_a_recovery(self): + """Rows collected while the device was unreadable keep whatever they said, so the + footer must still warn even once the memo has cleared.""" + self.hil_util.read_sysfs(self.fifo, timeout=0.3) + os.unlink(self.fifo) + Path(self.fifo).write_text('CAFE01\n') + self.hil_util.read_sysfs(self.fifo, timeout=1) + self.assertTrue(self.hil_util.sysfs_stranded()) + + def test_a_stranded_path_is_never_read_twice(self): + """Each expiry strands a thread and an fd for the life of the process, and + hil_pool_check POLLS -- wait_device re-scans every 0.5s until its budget runs + out. Re-reading would leak one pair per poll.""" + self.hil_util.read_sysfs(self.fifo, timeout=0.3) + t0 = time.monotonic() + for _ in range(5): + self.assertIsNone(self.hil_util.read_sysfs(self.fifo, timeout=0.3)) + self.assertLess(time.monotonic() - t0, 0.3, + 'repeat reads of a known-stranded path paid the grace again') + + def test_the_caller_can_say_the_table_may_be_wrong(self): + self.assertFalse(self.hil_util.sysfs_stranded()) + self.hil_util.read_sysfs(self.fifo, timeout=0.3) + self.assertTrue(self.hil_util.sysfs_stranded(), + 'nothing would tell the operator a missing row may be this tool ' + 'losing sight of healthy hardware') + + def test_a_healthy_attribute_is_not_blacklisted(self): + good = os.path.join(self.td.name, 'idVendor') + Path(good).write_text('cafe\n') + for _ in range(3): + self.assertEqual(self.hil_util.read_sysfs(good, timeout=1), 'cafe') + self.assertFalse(self.hil_util.sysfs_stranded()) + + def test_without_a_timeout_the_read_stays_plain(self): + good = os.path.join(self.td.name, 'busnum') + Path(good).write_text('3\n') + self.assertEqual(self.hil_util.read_sysfs(good), '3') + self.assertIsNone(self.hil_util.read_sysfs(os.path.join(self.td.name, 'nope'))) + + if __name__ == '__main__': unittest.main() diff --git a/test/hil/usbtest.py b/test/hil/usbtest.py index 485e9e0e4..d23217417 100755 --- a/test/hil/usbtest.py +++ b/test/hil/usbtest.py @@ -47,31 +47,65 @@ RECOVER_FLASH_TIMEOUT = 90 # bound on the post-hang reflash; typical flash is 1 RECOVER_RESET_TIMEOUT = 30 # bound on the post-hang probe reset; ResetTarget measures ~130ms -def recovery_steps(flasher_name: str, time_left: float) -> list: - """Ordered (kind, bound) recovery attempts that fit in `time_left`. +RECOVER_SETTLE = 5 # after each step, to let a freed ioctl unwind +# The ladder's UNBOUNDED work, which no step timeout covers: two wedged_pids() /proc walks, +# json.loads of the roster entry, the child's first `import hil_flash`, convoy_safe, the +# BUDGET back-fill and the JSON print. The deleted _time_left() carried this as a bare +# '- 35'. Without it the reserve equals its own worst case exactly, and HIL_CMD_TIMEOUT and +# HIL_USBTEST_BATTERY_BUDGET are both env-overridable -- any of them moving up puts +# run_cmd's killpg back inside the reflash, orphaning the flasher on the probe. +RECOVER_OVERHEAD = 40 - RESET FIRST, reflash second. A probe reset fails the in-flight URB at the source just - as a park-flash does, but it is non-destructive -- the firmware under test survives, so - the wedge can still be autopsied -- writes no flash, and cannot brick SWD the way a bad - park image has on mimxrt1064_evk and max32666fthr (survived a power cycle). Measured - 128-129 ms against a full erase+program, and it works on i.MX RT and on DWC2 alike - (stm32f407disco, 2026-08-16: `r; g` -> USB disconnect, re-enumerated 325 ms later). - The reset also fits budgets a reflash does not: the old gate skipped recovery entirely - when RECOVER_FLASH_TIMEOUT did not fit, which left the holder in place for the next - job. Whether either worked is decided by wedged_pids(), never by the exit code -- a - clean flash only proves the probe wrote the MCU. +def recovery_reserve(flasher: dict | str) -> int: + """Seconds this flasher's post-hang ladder can actually spend. + + Every bounded step can cost its own timeout PLUS run_cmd's post-SIGKILL reap, so the + caller must count REAP_GRACE per step or its outer killpg lands mid-reflash and + ORPHANS the flasher on the probe. Derived rather than pinned: the predecessor was an + independent 250s that could not contain its own ladder, which is why the child used to + re-decide before every step and skipped most of them on a real hang. + + Per FLASHER, not one number for the fleet: the Rescue-DP legs are openocd-only + (hil_flash.rescue_openocd returns False for anything else), and a stub reset is + screened out by reset_primitive -- so an esptool board reserving them would hold a + pool worker and a usbtest permit for 200s it can never spend. """ import hil_flash - steps = [] - reset_fn = getattr(hil_flash, f'reset_{flasher_name.lower()}', None) - if getattr(reset_fn, 'no_op', False): - reset_fn = None # a stub that returns rc 0 without resetting: do not claim it - if reset_fn and time_left >= RECOVER_RESET_TIMEOUT: - steps.append(('reset', RECOVER_RESET_TIMEOUT)) - if time_left >= RECOVER_FLASH_TIMEOUT: - steps.append(('flash', RECOVER_FLASH_TIMEOUT)) - return steps + from helper import hil_util + if isinstance(flasher, str): + flasher = {'name': flasher, 'args': ''} + name = (flasher.get('name') or '').lower() + + def step(bound): + return bound + hil_util.REAP_GRACE + + total = step(RECOVER_FLASH_TIMEOUT) + 2 * RECOVER_SETTLE + RECOVER_OVERHEAD + if reset_primitive(name): + total += step(RECOVER_RESET_TIMEOUT) + # The ARGS, not just the name: rescue_openocd also needs the target cfg to be an RP + # one (RESCUE_CFG), so the five WCH/max32666 openocd boards on this rig can never run + # it. Reserving its two legs for them holds a pool worker and a usbtest permit for + # 200s of dead time -- the same waste the esptool case exists to remove. + if name == 'openocd' and any(cfg in (flasher.get('args') or '') + for cfg in hil_flash.RESCUE_CFG): + total += 2 * step(RECOVER_FLASH_TIMEOUT) # Rescue-DP POR + one retry + return total + + +def reset_primitive(flasher_name: str): + """The flasher's probe-reset callable, or None when there is nothing real to run. + + Two things gate it. A flasher may have no reset_* at all, and reset_esptool / + reset_lm4flash return rc 0 WITHOUT resetting anything -- running those makes the log + say "resetting <board> via <flasher>" for a step that did nothing. wedged_pids() + arbitrates the outcome either way, so behaviour was always right; the RECORD was not. + """ + import hil_flash # deferred: stdlib-only unless recovery actually runs + fn = getattr(hil_flash, f'reset_{flasher_name.lower()}', None) + return None if getattr(fn, 'no_op', False) else fn + + HELPER_TIMEOUT = 30 # default bound for sudo helpers (dmesg/modprobe/setpci/tee) # Battery per tier, in run order: control sanity, simple bulk, queued, unaligned, unlink, @@ -190,18 +224,21 @@ def sysfs_write(path, data, check=True): return r.returncode == 0 -def _read_sysfs_bounded(path, grace=1.0): - """Bounded sysfs attribute read. The value, or None, or hil_util.SYSFS_UNKNOWN. +def _hu(): + """The helper module, imported lazily like every other helper use in this file.""" + from helper import hil_util + return hil_util - Delegates to hil_util.read_sysfs (imported here, like every helper import in this - file) so both properties hold: the strand cap -- find_device re-scans after EVERY - case, so a 30-case battery against a wedged peer would otherwise strand dozens of - threads and fds -- and UNKNOWN kept distinct from None. Folding UNKNOWN into None made - a blinded scan read as "device dropped off the bus", which aborts down a path that - skips the HUNG recovery entirely. - """ + +SERIAL_GRACE = 1.0 # tighter than hil_util's shared default on purpose: find_device + # re-scans every cafe:4010 peer after each of ~30 cases and inside + # the 8s startup poll, so N unreadable peers cost N x this per scan + + +def _read_sysfs(path): + """The attribute's value, or None. See hil_util.read_sysfs for why `serial` can block.""" from helper import hil_util - return hil_util.read_sysfs(str(path), grace) + return hil_util.read_sysfs(str(path), SERIAL_GRACE) _DEV_CACHE: dict = {} # serial -> sysname, see find_device @@ -222,7 +259,7 @@ def _reread(sysname, serial): if ((d / 'idVendor').read_text().strip() != VID or (d / 'idProduct').read_text().strip() != PID): return None - dev_serial = _read_sysfs_bounded(d / 'serial') + dev_serial = _read_sysfs(d / 'serial') if not isinstance(dev_serial, str) or dev_serial.lower() != serial.lower(): return None # gone, mismatched, or unconfirmable -> full scan decides return { @@ -253,19 +290,16 @@ def find_device(serial, first=False): if hit: return hit _DEV_CACHE.pop(serial.lower(), None) - matches, inconclusive = [], [] + matches = [] for dev in SYS_USB.iterdir(): try: if (dev / 'idVendor').read_text().strip() != VID or \ (dev / 'idProduct').read_text().strip() != PID: continue - # BOUNDED: idVendor/idProduct are cached descriptors, but `serial` is served - # under device_lock(), so an unbounded read blocks us in D state on exactly the - # DUT whose hang we are here to report, losing every verdict collected so far. - dev_serial = _read_sysfs_bounded(dev / 'serial') - if dev_serial is not None and not isinstance(dev_serial, str): - inconclusive.append(dev.name) # unknown: NOT proof it is not ours - continue + # idVendor/idProduct are cached descriptors; `serial` is served under + # device_lock(), so on a wedged DUT this read blocks until the wedge clears. + # Contained by the caller's bound, not prevented here -- see hil_util.read_sysfs. + dev_serial = _read_sysfs(dev / 'serial') if dev_serial is None: continue if serial and dev_serial.lower() != serial.lower(): @@ -281,10 +315,7 @@ def find_device(serial, first=False): except (OSError, ValueError): continue if not matches: - # "could not tell" is not "gone". The caller aborts the battery on a falsy return - # and that path skips the HUNG reflash, so a blinded scan would report the wedge - # we exist to recover from as a physical disconnect. - return {'inconclusive': inconclusive} if inconclusive else None + return None if serial and len(matches) == 1: _DEV_CACHE[serial.lower()] = matches[0]['sysname'] if len(matches) > 1 and not first: @@ -533,13 +564,11 @@ def main(): p.add_argument('--recover-board', help='board JSON (name + flasher) for the post-hang ' 'reflash recovery; without it a HUNG case leaves the device wedged') p.add_argument('--recover-fw', help='firmware path reflashed by the post-hang recovery') - p.add_argument('--outer-timeout', type=int, default=0, - help='the caller\'s total bound on this process; a reflash that cannot ' - 'finish before it is skipped rather than orphaned mid-flash') p.add_argument('--budget', type=int, default=0, help='stop starting new cases after this many seconds (0 = no limit). ' - 'Callers that impose their own outer timeout set this to reserve ' - 'the remainder for the post-hang recovery path') + 'Callers that impose their own outer bound set this BELOW it, ' + 'reserving the remainder for the post-hang recovery -- see ' + 'recovery_reserve() for what that ladder costs') args = p.parse_args() t_start = time.monotonic() sys.stdout.reconfigure(line_buffering=True) # per-case results visible when piped/logged @@ -554,23 +583,23 @@ def main(): deadline = time.monotonic() + 8 while True: dev = find_device(args.serial) - # find_device is THREE-valued: a device, {'ambiguous': [...]}, or - # {'inconclusive': [...]} when bounded reads could not rule a device out. Screening - # only for 'ambiguous' let the inconclusive marker through as if it were a device, - # and the next statement subscripts dev['tier'] -> KeyError, no JSON on stdout, and - # hil_test reports "usbtest did not run / 0-30" for a merely-unreadable bus. - if dev and not ({'ambiguous', 'inconclusive'} & dev.keys()): + # find_device returns a device or {'ambiguous': [...]}. Screening for the marker + # matters: without it the next statement subscripts dev['tier'] -> KeyError, no + # JSON on stdout, and hil_test reports "usbtest did not run / 0-30". + if dev and 'ambiguous' not in dev: break if time.monotonic() > deadline: if dev and 'ambiguous' in dev: sys.exit(f"multiple devices with serial {args.serial}: {', '.join(dev['ambiguous'])} " '— stale enumeration from another port? replug or retry') - if dev and 'inconclusive' in dev: - from helper import hil_util as _hu - sys.exit(f"cannot tell whether {VID}:{PID} is present: bounded sysfs reads " - f"did not answer for {', '.join(dev['inconclusive'])}" - f"{_hu.sysfs_blind_note()}") - sys.exit(f'no {VID}:{PID} device' + (f' with serial {args.serial}' if args.serial else '')) + # a bounded `serial` read that gave up looks exactly like a disconnect from + # here, and hil_test relays this line verbatim into the report cell. The + # sticky process-wide flag is the RIGHT question at startup -- nothing but + # this scan has read anything yet -- unlike mid-battery, where a peer that + # stranded at case 2 would answer for our board at case 29. + sys.exit(f'no {VID}:{PID} device' + + (f' with serial {args.serial}' if args.serial else '') + + _hu().strand_note()) time.sleep(0.5) # a stale/foreign device advertising an out-of-range tier must not silently run an @@ -638,31 +667,16 @@ def main(): print('no --recover-board/--recover-fw: the device stays wedged and ' 'cleanup is skipped', file=sys.stderr) break - # The reflash is bounded to RECOVER_FLASH_TIMEOUT and skipped when the - # caller's outer bound cannot contain it: the flasher runs in its own - # session, so an outer killpg mid-flash would ORPHAN it on the probe. Gate - # each step on the time actually LEFT -- reserving for the worst case up - # front skipped recovery for nearly every real hang, since the hang-prone - # cases run late in the tier order. - def _time_left(): - if not args.outer_timeout: - return float('inf') - # what still runs after a step: run_cmd's post-kill reap (10s), - # the settle (5s), the sudo-escalated descendant reap run_case may - # have just paid (up to 7s) and the JSON write - return args.outer_timeout - (time.monotonic() - t_start) - 35 - - if _time_left() < RECOVER_RESET_TIMEOUT: - print('insufficient time before the outer bound for even a bounded ' - 'reset; the device stays wedged and cleanup is skipped', - file=sys.stderr) - break + # Both steps are bounded (RECOVER_RESET_TIMEOUT / RECOVER_FLASH_TIMEOUT) + # and the caller RESERVES room for both -- hil_test derives its + # bound from recovery_reserve(). No re-derivation here: + # the old per-step "does it still fit?" arithmetic carried an unexplained + # 35s fudge for costs paid downstream, and nobody could re-derive it. try: board = json.loads(args.recover_board) bname, fname = board['name'], board['flasher']['name'] import hil_flash # deferred: stdlib-only unless recovery actually runs flash_fn = getattr(hil_flash, f'flash_{fname.lower()}') - reset_fn = getattr(hil_flash, f'reset_{fname.lower()}', None) except Exception as e: # malformed/short json, import failure, unknown flasher print(f'reflash recovery unavailable ({e})', file=sys.stderr) break @@ -680,25 +694,33 @@ def main(): f'openocd flasher to enable recovery for this board.', file=sys.stderr) break - # RESET FIRST (see recovery_steps). Non-destructive, ~130 ms, and it - # clears the wedge by the same mechanism as the reflash. wedged_pids is the - # arbiter: reset_esptool is a stub that returns rc 0 without resetting - # anything, so an exit code here proves nothing. - steps = recovery_steps(fname, _time_left()) - if reset_fn and any(k == 'reset' for k, _ in steps): + # RESET FIRST: a probe reset fails the in-flight URB at the source just + # as a reflash does, but it is non-destructive -- the firmware under test + # survives for autopsy -- writes no flash, and cannot brick SWD the way a + # bad park image has (mimxrt1064_evk, max32666fthr). Measured ~130 ms. + # wedged_pids is the arbiter either way: reset_esptool is a stub that + # returns rc 0 without resetting anything, so an exit code proves nothing. + reset_fn = reset_primitive(fname) + if reset_fn: print(f'auto-recovering: resetting {bname} via {fname} probe ' f'(non-destructive; reflash only if this does not clear it)', file=sys.stderr) + # Inspect the signature rather than catching TypeError around the + # call: a TypeError raised INSIDE the primitive would re-run it with + # no bound (run_cmd's 180s CMD_TIMEOUT, against a 40s reserve), and a + # raise from that retry does not reach the sibling except Exception -- + # it unwinds past the recovery block, so the battery exits on a + # traceback with no JSON and ~29 real verdicts are discarded. + import inspect + kw = ({'timeout': RECOVER_RESET_TIMEOUT} + if 'timeout' in inspect.signature(reset_fn).parameters else {}) try: with redirect_stdout(sys.stderr): - reset_fn(board, timeout=RECOVER_RESET_TIMEOUT) - except TypeError: - with redirect_stdout(sys.stderr): - reset_fn(board) # older primitives take no bound + reset_fn(board, **kw) except Exception as e: print(f'probe reset raised: {e}; falling through to the reflash', file=sys.stderr) - time.sleep(5) # let the freed ioctl unwind + time.sleep(RECOVER_SETTLE) # let the freed ioctl unwind stuck, complete = wedged_pids(dev['node']) if complete and not stuck: print('probe reset cleared the wedge; skipping the reflash ' @@ -706,10 +728,6 @@ def main(): file=sys.stderr) unrecovered_hang = False break - if _time_left() < RECOVER_FLASH_TIMEOUT: - print('reset did not clear it and no budget left for a reflash; ' - 'the device stays wedged', file=sys.stderr) - break print(f'auto-recovering: reflashing {bname} via ' f'{fname} (see .claude/skills/usb-kernel-recover). ' f'Unbudgeted by flash_permit, like the root-cycle it replaced: the ' @@ -734,10 +752,9 @@ def main(): # still fits before the outer kill rescued = False try: - if _time_left() >= 2 * RECOVER_FLASH_TIMEOUT: - with redirect_stdout(sys.stderr): - rescued = hil_flash.rescue_openocd( - board, out_txt, timeout=RECOVER_FLASH_TIMEOUT) + with redirect_stdout(sys.stderr): + rescued = hil_flash.rescue_openocd( + board, out_txt, timeout=RECOVER_FLASH_TIMEOUT) if rescued: print('DAP wedged; rescued via Rescue DP, retrying reflash', file=sys.stderr) @@ -754,7 +771,7 @@ def main(): # settle even on a non-zero exit: the reset may have landed before the # flasher failed, and the freed ioctl needs a moment to unwind before # wedged_pids samples - time.sleep(5) + time.sleep(RECOVER_SETTLE) # Authoritative either way: a clean flash only proves the probe wrote the # MCU, not that the D-state holder let go. stuck, complete = wedged_pids(dev['node']) @@ -788,16 +805,24 @@ def main(): f'({", ".join(live["ambiguous"])}) after case {num}') unrecovered_hang = True break - if live and live.get('inconclusive'): - # bounded reads stopped answering, so we cannot say the device left -- - # treat it as the wedge it probably is, which keeps the HUNG reflash and - # the lock-safe cleanup in play - from helper import hil_util as _hu - abort_reason = ('cannot tell whether the device is still present: bounded ' - 'sysfs reads stopped answering' + _hu.sysfs_blind_note()) - unrecovered_hang = True - break if not live: + # ABSENT vs UNREADABLE: a bounded `serial` read that gave up looks exactly + # like a disconnect from here, and the difference decides whether the + # cleanup below runs. remove_id/unbind take the UNINTERRUPTIBLE + # device_lock (see the driver-registry note above), so performing them + # against a device that is merely unreadable -- i.e. probably wedged -- + # deadlocks the bus rather than tidying up. Fail CLOSED: if anything gave + # up during this scan, treat it as the wedge it probably is, which also + # keeps the recovery and the board_wedged latch in play. + # OUR device's own attribute, not the process-wide sysfs_stranded(): + # that flag is sticky and every DUT here is cafe:4010, so a peer that + # stranded at case 2 would make a genuine disconnect at case 29 report as + # an unrecovered wedge for the rest of the run. + if _hu().path_stranded(str(SYS_USB / dev['sysname'] / 'serial')): + abort_reason = (f'cannot tell whether the device is still present ' + f'after case {num}: its serial read gave up') + unrecovered_hang = True + break # no second entry for `num`: run_case already recorded it, and a duplicate # inflates the denominator (31/30) and reports a PASSing case as failed abort_reason = f'device dropped off the bus after case {num}' @@ -830,12 +855,27 @@ def main(): 'power cycle (a VM reboot is not reliable — hubs latch up across the PCIe reset)', file=sys.stderr) elif not args.keep_binding: - sysfs_write(DRIVER / 'remove_id', f'{VID} {PID}', check=False) - # release every claimed interface: another device sharing the VID:PID - # (stale example firmware) may have been grabbed on probe and would stay - # bound to usbtest until re-plugged, hijacking the next test's device - for intf in DRIVER.glob('*:*'): - sysfs_write(DRIVER / 'unbind', intf.name, check=False) + # PROCESS-WIDE, unlike the per-case verdict above. That one is per-DUT on + # purpose -- a peer that stranded must not make OUR board report wedged. + # This cleanup is GLOBAL: it unbinds every interface under the driver, + # including the peer we could not read, and unbind takes the + # uninterruptible device_lock. Narrowing this gate to path_stranded() + # would add a driver-registry writer to an existing wedge. + # INSIDE keep_binding rather than before it: hil_test always passes that + # flag, so a check further out announced a skip of cleanup that was never + # going to run -- one line of noise ahead of the real cause in every + # stranded row. ONE line for the same reason: the finally runs before + # SystemExit's message reaches stderr. + if _hu().sysfs_stranded(): + print('cleanup skipped: a sysfs read gave up, so unbind could take a ' + 'wedged device lock', file=sys.stderr) + else: + sysfs_write(DRIVER / 'remove_id', f'{VID} {PID}', check=False) + # release every claimed interface: another device sharing the VID:PID + # (stale example firmware) may have been grabbed on probe and would + # stay bound to usbtest until re-plugged, hijacking the next test + for intf in DRIVER.glob('*:*'): + sysfs_write(DRIVER / 'unbind', intf.name, check=False) except SystemExit: pass @@ -849,7 +889,7 @@ def main(): if args.json: # `wedged` is the verdict this process ALREADY computed; without it the caller had # to infer one from 'HUNG' in our stdout, which misses a recovery that ran and - # failed, the inconclusive abort (no case reaches status HUNG), and any battery + # failed, the ambiguous abort (no case reaches status HUNG), and any battery # killed before it printed. print(json.dumps({'serial': dev['serial'], 'speed': dev['speed'], 'tier': tier, 'passed': ran - len(failed) - len(notrun), diff --git a/tools/build.py b/tools/build.py index eeefca22d..aa8868cb8 100755 --- a/tools/build.py +++ b/tools/build.py @@ -34,6 +34,7 @@ ci_skip_boards = { 'adafruit_fruit_jam', 'adafruit_metro_rp2350', 'feather_rp2040_max3421', + 'pico2_etm_trace', 'pico_sdk', 'raspberry_pi_pico_w', ], @@ -356,11 +357,11 @@ def get_family_boards(family, one_random, one_first, examples=None, build_system # the WHOLE preferred list, in order - stopping at entry one would abandon a # curated list for the raw alphabetical order the moment its first board cannot # build the filter, which also moves the board the metrics baseline is keyed on + # the whole preferred list, in order. Unreachable-when-unfiltered: with + # examples is None, buildable() is True and the loop returns on entry one. for b in preferred_list: if buildable(b): return [b] - if preferred_list and examples is None: - return [preferred_list[0]] candidates = [b for b in all_boards if buildable(b)] or all_boards if one_first: return [candidates[0]] diff --git a/tools/ci_select.py b/tools/ci_select.py index 89a0d214c..1526f2064 100755 --- a/tools/ci_select.py +++ b/tools/ci_select.py @@ -13,6 +13,38 @@ JSON: full, boards (name -> 'all' | [tests]), families (bsp families the diff touches, including ones with no rig board - build-only consumers such as /pre-pr sample from these), args (hil_test.py args per config) and args_flasher (the same args split by each board's flasher, for CI legs that split one rig by flasher). + +THE RULE TABLE. First match wins; answers union per family (build) and per board +(HIL). A CARBON COPY of the table in the design spec above - edit both, or +TestRuleTableIsCarbonOfTheSpec fails. `FAM` = the families whose family.cmake +references the changed path (CMake only; make follows it). `DEV`/`HOST`/`DUAL`/ +`TYPEC`/`ALL` are the example role sets. The Build families column is PRE-PRUNE: +_prune_buildable then intersects each family with what it can actually build. + +| # | Changed path | Build families | Build examples | HIL boards → tests | +| 1 | `docs/`, `.claude/`, `*.md`, `*.rst`, `LICENSE` | — | — | — | +| 1b | `.gitignore`, `.clang-format`, `.idea/**`, `test/{fuzz,unit-test}/**`, `test/hil/test/**`, non-build `.github/**`, packaging manifests | — | — | — | +| 2 | `test/hil/**` (not `test/hil/test/**`) | — | — | all boards → all tests | +| 2b | `tools/metrics.py`, `.github/scripts/metrics_*.py` | `ALL` (unchanged — `tinyusb_metrics` runs `metrics.py` as a build target) | `ALL` | — (nothing on the rig runs it) | +| 3 | `src/portable/<port>/dcd_*`, `*_device.[ch]` | `FAM` | `DEV`+`DUAL` | `FAM`'s device-role boards → device+dual tests | +| 4 | `src/portable/<port>/hcd_*`, `*_host.[ch]` | `FAM` | `HOST`+`DUAL` | `FAM`'s host-role boards → host+dual tests | +| 5 | `src/portable/<port>/**` (anything else) | `FAM` | `ALL` | `FAM`'s boards → all their tests | +| 5b | `src/portable/<port>/**` where `FAM` is empty | — | — | — (empty resolves to nothing on BOTH axes) | +| 6 | `hw/bsp/<family>/**` | that family | `ALL` | that family's boards → all tests (a `boards/<board>/` path narrows to that board) | +| 7 | `hw/mcu/<vendor>/**` | `FAM` — empty resolves to nothing (maintainer ruling) | `ALL` | `FAM`'s boards → all tests; empty resolves to nothing (maintainer ruling) ⚠ *see below* | +| 8 | `src/class/<cls>/*_device.[ch]` | `ALL` | examples enabling `CFG_TUD_<CLS>` | device-role boards → HIL tests enabling `CFG_TUD_<CLS>` | +| 9 | `src/class/<cls>/*_host.[ch]` | `ALL` | examples enabling `CFG_TUH_<CLS>` | host-role boards → HIL tests enabling `CFG_TUH_<CLS>` | +| 10 | `src/class/<cls>/**` (shared header) | `ALL` | either, **plus include-edge classes** | both roles → same, plus include-edge classes | +| 11 | `src/device/**` | `ALL` | `DEV`+`DUAL` | device-role boards → device+dual tests | +| 12 | `src/host/**` | `ALL` | `HOST`+`DUAL` | host-role boards → host+dual tests | +| 12b | `src/typec/**` | `ALL` | examples enabling `CFG_TUC_ENABLED` | — (no rig board runs a typec test) | +| 13 | `examples/<role>/<name>/**` | `ALL` | just `<name>` | if `<name>` is a HIL test: all boards → that test; else nothing | +| 14 | `examples/device/board_test/**` | `ALL` | just `board_test` | all boards → all tests (HIL parking firmware) | +| 15 | `examples/build_system/**`, `examples/CMakeLists.txt`, `examples/<role>/CMakeLists.txt` | `ALL` | `ALL` | all boards → all tests | +| 16 | `src/common/`, `src/osal/`, `src/tusb.[ch]`, `src/tusb_option.h`, `tools/{build,build_utils,ci_select}.py`, `tools/cmake/**`, `src/CMakeLists.txt`, `src/tinyusb.mk`, `hw/bsp/{family_support.{cmake,mk},family_rules.mk,zephyr_board_aliases.cmake,board.c,board_api.h,ansi_escape.h}`, `.github/**`, `.circleci/**` | `ALL` | `ALL` | all boards → all tests | +| 16a | `lib/<name>/**` | `ALL` | examples whose own `CMakeLists.txt`/`Makefile` names `lib/<name>` | those examples that are HIL tests, on all boards; empty resolves to nothing | +| 16b | `tools/get_deps.py` | families whose `deps_mandatory`/`deps_optional` entries changed | `ALL` | those families' boards → all tests; a logic change, an `'all'` entry, no base content or a changed token naming no family → full | +| 17 | anything unclassified (no tracked file reaches this — TestNoTrackedFileIsUnclassified) | `ALL` | `ALL` | all boards → all tests (fail-open) | """ import argparse import ast @@ -53,7 +85,10 @@ def _read(path: str) -> str: _NONCODE_RE = re.compile( - r'^(docs/|\.claude/|.*\.(md|rst)$|LICENSE)') + # LICENSE is anchored and LICENSES/ named separately: a bare `LICENSE` alternative + # also swallowed anything merely STARTING with it (a future LICENSE_extra.c), + # which is the silent-under-selection direction + r'^(docs/|\.claude/|.*\.(md|rst)$|LICENSE$|LICENSES/)') # Repo metadata and tooling that no CI build reads. Enumerated rather than left to # rule 17, which widens BOTH axes: a PR touching only .gitignore and a README was # creating 74 cmake legs (each a runner doing checkout + toolchain + get_deps before @@ -73,7 +108,11 @@ _META_RE = re.compile( r'version\.yml$|SConscript$|' r'.*CMakePresets\.json$|hw/bsp/BoardPresets\.json$|examples/west\.yml$|' r'.*/[0-9]+-tinyusb[^/]*\.rules$|tools/usb_drivers/|tools/codespell/|' - r'test/(fuzz|unit-test)/|' + # test/hil/test/ holds the harness's own unit tests, not the harness: nothing on + # the rig runs them (pre-commit does, and build.yml runs test_ci_select.py as the + # gate before trusting a selection), so they cannot change what the rig does. + # The harness itself stays under _FULL_RE's test/hil/ prefix. + r'test/(fuzz|unit-test)/|test/hil/test/|' # .github, minus the build machinery named in _FULL_RE r'\.github/(FUNDING\.yml$|labeler\.yml$|membrowse_pr_message\.j2$|ISSUE_TEMPLATE/|' r'workflows/(cifuzz|claude|claude-code-review|labeler|membrowse-comment|' @@ -93,7 +132,11 @@ _METRICS_RE = re.compile( r'^(tools/metrics[^/]*\.py$|\.github/scripts/metrics_[^/]*\.py$)') _FULL_RE = re.compile( r'^(src/common/|src/osal/|src/tusb\.c$|src/tusb\.h$|src/tusb_option\.h$|' - r'test/hil/|\.github/workflows/build.*\.yml$|\.github/actions/|\.github/scripts/|' + # tools/rtt.py is part of the harness, not a standalone tool: hil_util imports it + # at module load, so a break in it breaks every rig run the same way a test/hil/ + # edit can (the pre-commit hil-test hook runs its unit tests for the same reason) + r'test/hil/|tools/rtt\.py$|' + r'\.github/workflows/build.*\.yml$|\.github/actions/|\.github/scripts/|' # generates the whole CircleCI matrix, same authority as .github/** r'\.circleci/|' # rule 16 says `tools/build*.py`; name the two siblings the glob implies. Both @@ -152,10 +195,20 @@ def board_tests(board: dict) -> list: return [x for x in run if x not in t.get('skip', [])] + +def _rg(repo_root: str, *parts: str) -> str: + """A glob pattern rooted at repo_root, with the ROOT escaped and the parts left as + patterns. The root is a filesystem path, not a pattern: a checkout at + /w/pr[1]/tinyusb (a worktree named after a PR, a CI workspace with brackets) makes + an unescaped '[1]' a character class that matches nothing, and every lookup below + then resolves to zero - families=0 instead of 30, i.e. the selector fails CLOSED + and the whole matrix compiles nothing while reporting green.""" + return os.path.join(glob.escape(repo_root), *parts) + # cached: called per changed file x roster board, and the tree doesn't change mid-run @functools.lru_cache(maxsize=None) def board_family(board_name: str, repo_root: str): - hits = glob.glob(os.path.join(repo_root, 'hw/bsp/*/boards', board_name)) + hits = glob.glob(_rg(repo_root, 'hw/bsp/*/boards', board_name)) return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None @@ -263,10 +316,10 @@ def _family_file_texts(repo_root: str) -> tuple: CMakeLists.txt, read once. path_families is called per distinct directory in the diff and its own cache only helps repeats: a 6,000-file hw/mcu dep bump re-read these 84 files 99,892 times (2.2 s) before this.""" - bsp_root = os.path.join(repo_root, 'hw/bsp') + bsp_root = os.path.join(repo_root, 'hw/bsp') # escaped by _rg below out = [] - for f in sorted(glob.glob(os.path.join(bsp_root, '*/family.cmake')) + - glob.glob(os.path.join(bsp_root, '*/components/*/CMakeLists.txt'))): + for f in sorted(glob.glob(_rg(bsp_root, '*/family.cmake')) + + glob.glob(_rg(bsp_root, '*/components/*/CMakeLists.txt'))): try: out.append((os.path.relpath(f, bsp_root).split(os.sep, 1)[0], _read(f))) except OSError: @@ -386,7 +439,7 @@ def class_include_edges(repo_root: str) -> dict: Derived from the actual #include lines rather than a hand-written table so it cannot rot when a class picks up or drops a cross-class include.""" edges = {} - for f in sorted(glob.glob(os.path.join(repo_root, 'src/class/*/*.[ch]'))): + for f in sorted(glob.glob(_rg(repo_root, 'src/class/*/*.[ch]'))): cls = os.path.basename(os.path.dirname(f)) try: text = _read(f) @@ -470,11 +523,23 @@ def _class_roles(base: str) -> set: return {'device', 'host'} -def _config_enables(cfg_path: str, macros) -> bool: [email protected]_cache(maxsize=None) +def _config_text(cfg_path: str) -> str: + """An example's tusb_config.h, read once. Every class path re-asks the same 46 + configs on both axes, so the reads go up with the diff: 4,240 of the same 46 files + for a diff touching all of src/class (0.48s -> 0.13s), and they cannot change + mid-run. Cached here rather than on _config_enables so the macros argument stays an + ordinary list at every call site.""" try: with open(cfg_path, encoding='utf-8', errors='replace') as f: - text = f.read() + return f.read() except OSError: + return '' + + +def _config_enables(cfg_path: str, macros) -> bool: + text = _config_text(cfg_path) + if not text: return False for m in macros: for value in re.findall(_DEF_VALUE.format(m), text, re.M): @@ -511,10 +576,13 @@ def lib_examples(lib_name: str, repo_root: str) -> set: pat = re.compile(re.escape('lib/' + lib_name) + r'(?=[/\s"\')}]|$)', re.M) out = set() for ex in all_examples(repo_root): - for f in sorted(glob.glob(os.path.join(repo_root, 'examples', ex, '**', '*'), + # the two filenames directly: '**/*' enumerated 489 entries per lib against a + # clean tree to use 107, and grows without bound once `make BOARD=... all` has + # written examples/<role>/<name>/_build/ - which is where /pre-pr runs + for f in sorted(glob.glob(_rg(repo_root, 'examples', ex, '**', 'CMakeLists.txt'), + recursive=True) + + glob.glob(_rg(repo_root, 'examples', ex, '**', 'Makefile'), recursive=True)): - if os.path.basename(f) not in ('CMakeLists.txt', 'Makefile'): - continue try: text = _read(f) except OSError: @@ -580,7 +648,7 @@ def _classify_one(path, repo_root, roster_boards, extras: set, s: _Sel, if _NONCODE_RE.match(path) or _META_RE.match(path): s.reasons.append(f'{path}: non-code, no contribution') return - if _METRICS_RE.match(path): + if _METRICS_RE.match(path): # rule 2b s.reasons.append(f'{path}: build-size metrics tooling, no HIL contribution') return if _FULL_RE.match(path): @@ -700,6 +768,16 @@ def _classify_one(path, repo_root, roster_boards, extras: set, s: _Sel, # only the tests whose example builds the lib, and only those the rig runs tests = {e for e in lib_examples(lib, repo_root) if any(e in pool for pool in ALL_TESTS.values()) or e in extras} + if lib == 'SEGGER_RTT': + # no example names this lib, but a board whose roster entry says + # "logger": "rtt" (variant defines LOGGER=rtt) reads EVERY test's console + # through it -- a break here silently breaks all of that board's rows + rtt_boards = [b['name'] for b in roster_boards if b.get('logger') == 'rtt'] + if rtt_boards: + s.roles.update(('device', 'host')) + s.add(rtt_boards, 'all', + f'{path}: SEGGER_RTT is the rtt console on {rtt_boards} -> all tests') + return if not tests: s.reasons.append(f'{path}: lib {lib} used by no HIL test, no contribution') return @@ -903,7 +981,13 @@ def main(): print(f'ci_select[build]: {r}', file=sys.stderr) for r in s['reasons']: print(f'ci_select: {r}', file=sys.stderr) - print(json.dumps(s)) + # reasons go to stderr ONLY - they are a human diagnostic and no consumer reads them + # back. They are also ~97% of the payload (a whole-tree diff: 453 KB -> 12 KB), which + # build.yml re-parses with ci_set_matrix, hil_ci_set_matrix, an inline python and + # three jq calls. The in-process dicts still carry them, for the log and the tests. + out = {k: v for k, v in s.items() if k != 'reasons'} + out['build'] = {k: v for k, v in s['build'].items() if k != 'reasons'} + print(json.dumps(out)) # ------------------------------------------------------------- @@ -925,7 +1009,7 @@ def all_examples(repo_root: str) -> tuple: """Every examples/<role>/<name> with a CMakeLists.txt, as 'role/name'.""" out = [] for role in _EX_ROLES: - for d in sorted(glob.glob(os.path.join(repo_root, 'examples', role, '*/'))): + for d in sorted(glob.glob(_rg(repo_root, 'examples', role, '*/'))): if os.path.isfile(os.path.join(d, 'CMakeLists.txt')): out.append(f'{role}/{os.path.basename(d.rstrip(os.sep))}') return tuple(out) @@ -979,13 +1063,13 @@ class _BSel: def _classify_build_one(path, repo_root, s: _BSel, get_deps_families=None): base = os.path.basename(path) - if _NONCODE_RE.match(path) or _META_RE.match(path): # rule 1 + if _NONCODE_RE.match(path) or _META_RE.match(path): # rules 1, 1b s.reasons.append(f'{path}: non-code, no build contribution') return if re.match(r'test/hil/', path): # rule 2 s.reasons.append(f'{path}: HIL harness, no build contribution') return - if path == GET_DEPS_PATH: # get_deps rule + if path == GET_DEPS_PATH: # rule 16b if get_deps_families is None: s.force_full(f'{path}: dep changes not resolvable -> full build matrix') return @@ -1002,6 +1086,7 @@ def _classify_build_one(path, repo_root, s: _BSel, get_deps_families=None): roles = _port_roles(base) exs = 'all' if roles == {'device', 'host'} else \ role_examples(repo_root, tuple(roles) + ('dual',)) + # rule 5b: fams empty -> s.add iterates nothing -> no contribution s.add(fams, exs, f'{path}: port {port} -> families {sorted(fams)}') return if re.match(r'hw/bsp/[^/]+/', path): # rule 6 @@ -1064,13 +1149,17 @@ def _classify_build_one(path, repo_root, s: _BSel, get_deps_families=None): s.add(all_bsp_families(repo_root), exs, f'{path}: typec -> {sorted(exs)}') return m = re.match(r'lib/([^/]+)/', path) - if m: # lib rule + if m: # rule 16a lib = m.group(1) exs = lib_examples(lib, repo_root) if not exs: - # empty means empty: no example's build pulls this lib in, so no build - # compiles it (lib/SEGGER_RTT is only reached through LOGGER=rtt, which - # no CI build sets) + # empty means empty: no example's build pulls this lib in, so no MAIN- + # matrix build compiles it. (lib/SEGGER_RTT is reached through LOGGER=rtt, + # which the main matrix never sets; the hil-build legs set it only for + # roster boards whose variant defines carry it, via the HIL SEGGER_RTT rule. + # No committed CI roster has such a board yet, so a SEGGER_RTT edit is + # currently neither built nor HIL-tested by CI -- verify vendor bumps + # manually until a rig board adopts "logger": "rtt".) s.reasons.append(f'{path}: lib {lib} built by no example, no contribution') return s.add(all_bsp_families(repo_root), exs, f'{path}: lib {lib} -> {sorted(exs)}') @@ -1150,11 +1239,25 @@ def _prune_buildable(fams, fam_ex, repo_root): # for anything else spins up CI's most expensive leg to skip every example # it was given. Identical to the unfiltered list on all 81 other families. pool = set(build_py.get_examples(fam)) + + # asked per example instead of materialising the family's whole buildable + # list: skip_example is by far the hottest call in the selector, and every + # question below short-circuits (one cdc_device.c diff: 6,883 calls -> 1,889) + def can_build(ex): + # EITHER build system: this one list gates CircleCI's make legs too, and + # the two answer differently (build_utils.skip_example) + return ex in pool and any( + not build_utils.skip_example(ex, b) or + not build_utils.skip_example(ex, b, (), 'make') for b in boards) + + want = fam_ex.get(fam) try: - buildable = [e for e in allex if e in pool and - any(not build_utils.skip_example(e, b) or - not build_utils.skip_example(e, b, (), 'make') - for b in boards)] + if want is None: + kept = None if any(can_build(e) for e in allex) else [] + else: + kept = [e for e in want if can_build(e)] + if kept and not any(can_build(e) for e in allex if e not in want): + kept = None # already everything the family can build except OSError as e: # a family mid-bring-up (boards/ but no family.cmake/family.mk yet) # reads as unbuildable to the scrape; keep it rather than tracebacking @@ -1162,13 +1265,10 @@ def _prune_buildable(fams, fam_ex, repo_root): reasons.append(f'{fam}: mcu scrape unreadable ({e}), kept unfiltered') out_fams.append(fam) continue - want = fam_ex.get(fam) - have = set(buildable) - kept = buildable if want is None else [e for e in want if e in have] - if not kept: + if kept == []: continue # this diff builds nothing for this family out_fams.append(fam) - if set(kept) != set(buildable): + if kept is not None: out_ex[fam] = kept return out_fams, out_ex, reasons diff --git a/tools/rtt.py b/tools/rtt.py new file mode 100644 index 000000000..e3aef36f2 --- /dev/null +++ b/tools/rtt.py @@ -0,0 +1,727 @@ +#!/usr/bin/env python3 +"""RTT console/capture over a debug probe — importable classes + CLI (the rtt +skill's SKILL.md is the manual). + +Three routes (see the skill's transport matrix for which route a probe gets). +--backend is always explicit: + + J-Link route (console/capture, channel 0 only) + rtt.py --backend jlink --probe <sn> --device <JLINK_DEVICE> [--seconds N] [-i] + OpenOCD route (native probes: ST-Link/CMSIS-DAP; console/capture, any channel) + rtt.py --backend openocd [--probe <sn>] [--vid-pid "0xVVVV 0xPPPP"] \\ + --cfg "-f interface/stlink.cfg -f target/stm32h7x.cfg" \\ + (--elf <flashed.elf> | --addr 0x2000xxxx) [--channel N] [--seconds N] [-i] + [--reset-before-attach] # capture from the target's boot (SystemView) + Post-mortem ring dump (J-Link, no halt — debug-AP reads) + rtt.py --backend jlink --dump <out.bin> --probe <sn> --device <JLINK_DEVICE> \\ + (--elf <flashed.elf> | --addr 0x...) + +The probe is owned for the whole run: flash and reset BEFORE starting this, never +reset the target while it is attached. Pin the probe: rigs and benches run +several (jlink: --probe serial; openocd: --probe and/or --vid-pid). + +The classes (JlinkRtt for J-Link, OpenocdRtt for openocd-driven probes) expose +the slice of pyserial the HIL harness uses — read/in_waiting/write/close/timeout, +reset_input_buffer, context-manager use, plus an `eof` latch — and are imported +by test/hil/helper/hil_util.py, so this file is HARNESS-CRITICAL: a change here +is classified like a test/hil/ harness change (tools/ci_select.py) and runs the +console unit tests (pre-commit hil-test hook, test/hil/test/test_hil_rtt.py). +Stdlib only — hil_util imports this file, never the other way around. +""" +import argparse +import contextlib +import os +import re +import select +import shlex +import signal +import socket +import subprocess +import sys +import tempfile +import threading +import time + + +class RttError(RuntimeError): + """Every way a console can break: stall, closed, dead or reset server. + + A RuntimeError subclass so existing `except RuntimeError` callers keep working, + but named so the harness can tell a console failure from an unrelated + NotImplementedError / 'dictionary changed size during iteration' and stop + reporting harness bugs as board failures.""" + + +def _pos_float_env(name: str, default: float) -> float: + # mirrors hil_util.pos_float_env, including its rejection of inf/nan: an infinite + # write timeout is an unbounded write, the very thing this knob exists to bound + raw = os.environ.get(name) + if raw is None: + return default + try: + v = float(raw) + except ValueError: + print(f'warning: {name} is not a number; using {default}', file=sys.stderr, flush=True) + return default + if not (v > 0 and v < float('inf')): + print(f'warning: {name}={v} is not usable; using {default}', file=sys.stderr, flush=True) + return default + return v + + +# whole-call deadline for write() — same env knob as the harness's serial twin +RTT_WRITE_TIMEOUT = _pos_float_env('HIL_SERIAL_WRITE_TIMEOUT', 10) + +# J-Link Commander's telnet greeting, sent at connect BEFORE (or without) the control +# block being found: never target output. Three lines; the middle one is the PROBE +# MODEL string, which in libjlinkarm carries no 'SEGGER ' prefix (J-Link OH3, +# J-Trace H9, ...) though some builds do prefix it — match both shapes. Consumers +# judging "did the target speak" must strip these lines first. +RTT_BANNER_RE = re.compile(r'^(SEGGER J-|J-Link[ 0-9]|J-Trace[ 0-9]|Process:\s)') + + +def strip_banner(data: bytes, complete_only: bool = False) -> bytes: + """Target bytes only: drop the J-Link server banner lines and blanks. + + Both harness consumers (hil_test's device_info verdict, hil_pool_check's + aliveness score) must judge "did the target speak" through this one filter, + or the same byte stream scores differently per consumer. complete_only=True + additionally drops a trailing unterminated line — for poll loops judging a + growing buffer, where a banner FRAGMENT at a read boundary (b'SEGG', b'Proce') + would defeat the prefix regex and count as target output; the final verdict + after the window should pass complete_only=False to keep a genuine + unterminated tail.""" + lines = data.splitlines(keepends=False) + if complete_only and data and not data.endswith((b'\n', b'\r')) and lines: + lines = lines[:-1] + return b'\n'.join(l for l in lines + if l.strip() and not RTT_BANNER_RE.match(l.decode('utf-8', errors='ignore'))) + + +def free_ports(count: int) -> list: + """Bind ephemeral ports and hand back the numbers. Boards run in parallel, so the + RTT/GDB ports cannot be the SEGGER defaults or two boards collide. + + Known TOCTOU: the port is free when released here, but another process can claim + it before the server binds it. Accepted — the server binds the port itself, so + there is no fd to hand over. The post-connect re-poll catches the common outcome + (our server lost the bind and died); a foreign listener that stays alive is not + detectable here and would need the connected peer to be validated.""" + socks = [] + try: + for _ in range(count): + s = socket.socket() + s.bind(('127.0.0.1', 0)) + socks.append(s) + return [s.getsockname()[1] for s in socks] + finally: + for s in socks: + s.close() + + +def nm_rtt_addr(elf: str, nm: str = None) -> int: + """Control-block address from the FLASHED elf's symbol table. --addr is the way + out when nm cannot read the file (another architecture, no toolchain).""" + nm = nm or os.environ.get('RTT_NM', 'arm-none-eabi-nm') + try: + r = subprocess.run([nm, elf], capture_output=True, text=True, timeout=30) + except FileNotFoundError: + raise SystemExit(f'{nm} not on PATH — set RTT_NM=<your-nm>, or pass --addr') + except subprocess.TimeoutExpired: + raise SystemExit(f'{nm} did not finish reading {elf} in 30 s — pass --addr instead') + if r.returncode != 0: + raise SystemExit(f'{nm} could not read {elf}: {r.stderr.strip()[:200]}\n' + f'(wrong architecture? set RTT_NM=<your-nm>, or pass --addr)') + for line in r.stdout.splitlines(): + # "<addr> <type> _SEGGER_RTT": a defined data symbol only — an undefined one + # (" U _SEGGER_RTT") has no address and would int('U', 16) + m = re.match(r'^([0-9a-fA-F]+)\s+[bBdD]\s+_SEGGER_RTT$', line.strip()) + if m: + return int(m.group(1), 16) + raise SystemExit(f'no defined _SEGGER_RTT symbol in {elf} — was it built with LOGGER=rtt?') + + +class _SocketRtt: + """Shared console core: a TCP socket onto an RTT server owned by self._proc. + + Subclasses build their server argv and call _spawn() + _connect() in __init__. + One failure contract: RttError for every way the console can break (stall, + closed, dead server) — callers are written for exactly it. A dead or resetting + server LATCHES `eof` rather than raising from the read side, so read loops and + the harness's `assert not ser.eof` triage see it without an exception racing + them to a generic handler.""" + + server = 'RTT server' # for error messages + + def __init__(self, timeout: float = 0.1): + self.timeout = timeout + self._buf = b'' + self._eof = False + self._sock = None + self._proc = None + self._log = None + self._lock = threading.Lock() # _buf is touched by the CLI pump thread too + + def _spawn(self, cmd: list, stdin=None) -> None: + # server output spools to a temp file: a PIPE nobody drains blocks a + # single-threaded server once 64 KiB of log accumulates (openocd at + # polling_interval 1 against a resetting target fills that in minutes) and + # the console goes silent with no error; the file also feeds _server_tail + self._log = tempfile.NamedTemporaryFile(prefix='rtt-server-', suffix='.log') + try: + self._proc = subprocess.Popen(cmd, stdin=stdin, stdout=self._log, + stderr=subprocess.STDOUT, start_new_session=True) + except FileNotFoundError as e: + self.close() + raise RttError(f'RTT console: {e.filename or cmd[0]} not on PATH') from e + except BaseException: + # any other spawn failure (PermissionError...) must not leak the log fd + self.close() + raise + + def _connect(self, port: int) -> None: + try: + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + try: + self._sock = socket.create_connection(('127.0.0.1', port), timeout=2) + break + except OSError: + if self._proc.poll() is not None: + break + time.sleep(0.2) + if self._sock is None: + tail = self._server_tail() + self.close() + raise RttError(f'RTT console: {self.server} did not serve port {port}{tail}') + if self._proc.poll() is not None: + # the connect succeeded but our server is dead: a foreign process claimed + # the port in the free_ports window — refuse a console wired to a stranger + self.close() + raise RttError(f'RTT console: {self.server} died after connect (port {port} hijacked?)') + self._sock.setblocking(False) + except (KeyboardInterrupt, SystemExit): + # a signal mid-construction must not orphan the server we just spawned + self.close() + raise + + def _server_tail(self) -> str: + log = getattr(self, '_log', None) + if log: + with contextlib.suppress(OSError, ValueError): + with open(log.name, 'rb') as fh: + tail = fh.read()[-400:].decode(errors='replace') + if tail: + return ' — ' + tail + return '' + + def _drain(self) -> None: + # LATCH, never raise: a peer reset or a socket closed under us ends the + # stream exactly like an orderly EOF. Raising here raced the harness's + # `assert not ser.eof` triage into a generic handler that re-flashes the + # board, and leaked ConnectionResetError/ValueError to in_waiting callers. + # the WHOLE body under the lock, not just the append: the CLI's -i pump thread + # and the read loop drain the same socket concurrently, and recv->append being + # non-atomic let chunks land out of order (measured: transposed 64-byte + # segments in 3/6 stress trials) + try: + with self._lock: + while self._sock and select.select([self._sock], [], [], 0)[0]: + try: + chunk = self._sock.recv(65536) + except (BlockingIOError, InterruptedError): + return + if not chunk: + self._eof = True + return + self._buf += chunk + except (OSError, ValueError, TypeError, AttributeError): + self._eof = True + + @property + def eof(self) -> bool: + """True once the server hung up AND everything it sent has been read out.""" + if self._sock is None: + return True + self._drain() + return self._eof and not self._buf + + @property + def in_waiting(self) -> int: + if self._sock is None: + # pyserial raises on a closed port; answering "N bytes waiting" from a + # closed dead console would let a caller bug look like a healthy board + raise RttError('RTT console is closed') + self._drain() + return len(self._buf) + + def read(self, size: int = 1) -> bytes: + if size is None or size <= 0: + # pyserial's read(0) returns b'' and consumes nothing; a negative size + # must not silently hand over (or destroy) buffered bytes + return b'' + if self._sock is None: + raise RttError('RTT console is closed') + self._drain() + deadline = None if self.timeout is None else time.monotonic() + self.timeout + while (len(self._buf) < size and not self._eof + and (deadline is None or time.monotonic() < deadline)): + time.sleep(0.005) + self._drain() + if self._eof and len(self._buf) < size: + # dead server: pace the empty returns like a serial timeout would, so a + # caller's read loop cannot busy-spin at 100% CPU (416k empty reads/s + # measured unpaced). timeout=None deliberately diverges from pyserial's + # block-forever: the eof latch makes "server is gone" knowable, and an + # eternal block on it helps nobody -- paced empties + .eof is the contract. + pace = self.timeout if self.timeout is not None else 0.1 + remaining = (deadline - time.monotonic()) if deadline is not None else pace + time.sleep(max(0.0, min(remaining, pace))) + with self._lock: + out, self._buf = self._buf[:size], self._buf[size:] + return out + + def reset_input_buffer(self) -> None: + # pyserial surface: the host tests flush pre-reset backlog through this + if self._sock is None: + raise RttError('RTT console is closed') + self._drain() + with self._lock: + self._buf = b'' + + def write(self, data: bytes) -> int: + # select+send, not sendall(): the socket is non-blocking for reads, and sendall() + # on a non-blocking socket raises BlockingIOError as soon as the send buffer is + # full, with no count of what already went out -- a caller cannot resume without + # duplicating bytes. Same reason serial_write_all treats a short write as fatal. + sock = self._sock # snapshot: close() from another thread nulls the attribute + if sock is None: + raise RttError('RTT console is closed') + self._drain() + if self._eof: + # TCP accepts exactly one send after peer death — without this the bytes + # would "succeed" into the void and the read timeout gets blamed on the target + raise RttError(f'RTT console write to a dead server ({self.server} gone)') + sent = 0 + deadline = time.monotonic() + RTT_WRITE_TIMEOUT + while sent < len(data): + if time.monotonic() > deadline: + raise RttError(f'RTT console write stalled after {sent}/{len(data)} bytes') + try: + if not select.select([], [sock], [], 0.1)[1]: + continue + sent += sock.send(data[sent:]) + except (BlockingIOError, InterruptedError): + continue + except (OSError, ValueError, TypeError, AttributeError) as e: + # peer death (BrokenPipe/ConnectionReset) or the socket closed under us + # mid-call: keep the class's one failure contract + raise RttError(f'RTT console write failed after {sent}/{len(data)} bytes: {e}') from e + return sent + + def _gentle_stop(self, proc) -> None: + """Subclass hook: ask the server to exit before the group takedown.""" + + def close(self) -> None: + self._eof = True # latch: post-close eof reads True, like a hung-up server + if getattr(self, '_sock', None): + self._sock.close() + self._sock = None + with self._lock: + self._buf = b'' # pyserial contract: nothing is readable after close + proc = getattr(self, '_proc', None) + if proc: + if proc.poll() is None: + self._gentle_stop(proc) + with contextlib.suppress(subprocess.TimeoutExpired): + proc.wait(timeout=5) + if proc and proc.poll() is None: + # own session (start_new_session), so the group takedown gets the server and + # anything it spawned; leaving one alive would hold the probe for the next test + try: + os.killpg(proc.pid, signal.SIGTERM) + proc.wait(timeout=5) + except (ProcessLookupError, PermissionError): + pass + except subprocess.TimeoutExpired: + with contextlib.suppress(ProcessLookupError, PermissionError): + os.killpg(proc.pid, signal.SIGKILL) + # reap, or the server stays a zombie for the caller's lifetime + with contextlib.suppress(subprocess.TimeoutExpired): + proc.wait(timeout=2) + if proc: + for pipe in (proc.stdin, proc.stdout): + if pipe: + with contextlib.suppress(OSError, ValueError): + pipe.close() + # the server spool file: one fd plus a /tmp file per console, and the server + # grows it while alive -- GC is not a release policy on a rig + log = getattr(self, '_log', None) + if log: + with contextlib.suppress(OSError, ValueError): + log.close() + self._log = None + + # a console dropped without close() must not hold the probe for the process's life + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + def __del__(self): + with contextlib.suppress(Exception): + self.close() + + +class JlinkRtt(_SocketRtt): + """Bidirectional console over SEGGER RTT channel 0, for J-Link probes (the only + console on boards whose probe has no VCOM or whose BSP has no UART). + + J-Link Commander (JLinkExe) owns the probe and serves RTT channel 0 on + -RTTTelnetPort -- what JLinkRTTClient talks to, minus its banner. It keeps + hunting for the control block and streams whatever the buffer already holds, + where JLinkRTTLogger searches once when it attaches and gives up. It also + carries input, which the host tests that drive a menu need. + + The probe is held for as long as this is open, so flashing and resetting the + board must happen before it is created or after close(). Select the probe by + serial: rigs run more than one.""" + + server = 'JLinkExe' + + def __init__(self, board: dict, timeout: float = 0.1): + super().__init__(timeout) + flasher = board['flasher'] + args = shlex.split(flasher.get('args', '')) + if '-device' not in args: + # fail with the real cause now: JLinkExe without a device blocks prompting + # and would surface 15 s later as a misleading port error + raise RttError(f'RTT console: no -device in flasher args: {flasher.get("args")!r}') + port = free_ports(1)[0] + # defaults first, the roster's args after so they can override (-if jtag, + # -JLinkScriptFile, an explicit -speed). NOTE: hil_flash orders it the other + # way (roster args first, its own -if/-speed last, so ITS defaults win) -- + # a roster override honored here is ignored by flash/reset; align them if a + # roster ever carries such args. -ExitOnError makes a failed target connect + # EXIT Commander + # (a clean error with the log tail) instead of leaving a banner-only console + cmd = ['JLinkExe', '-USB', str(flasher['uid']), '-if', 'swd', + '-JTAGConf', '-1,-1', '-speed', 'auto', '-NoGui', '1', + '-ExitOnError', '1', '-AutoConnect', '1', + *args, '-RTTTelnetPort', str(port)] + # stdin stays open: Commander exits when it runs out of input; close() writes + # 'exit' there. + self._spawn(cmd, stdin=subprocess.PIPE) + self._connect(port) + + def _gentle_stop(self, proc) -> None: + with contextlib.suppress(OSError, ValueError): + proc.stdin.write(b'exit\n') + proc.stdin.flush() + # close our pipe end in its own suppress: a BrokenPipe on the write above must + # not skip it (the base close also closes it for the server-already-dead path) + with contextlib.suppress(OSError, ValueError): + proc.stdin.close() + + +class OpenocdRtt(_SocketRtt): + """The console surface over an openocd `rtt server` (native probes: + ST-Link/CMSIS-DAP — never point openocd at ea4088's LPC-Link2, measured to + knock that probe off USB; other J-Link-OB probes untested). + + Exact control-block address (never a full-RAM scan), polling_interval 1 + (default 100 ms polling loses most of a busy stream), attach WITHOUT reset — + flash and reset before starting; `rtt start` needs the block to exist. + reset_before_attach opts into an in-session reset for streams that only + decode from byte 0 (SystemView).""" + + server = 'openocd' + + def __init__(self, cfg: str, addr: int, channel: int, serial_no: str = None, + vid_pid: str = None, timeout: float = 0.1, reset_before_attach: bool = False): + super().__init__(timeout) + port = free_ports(1)[0] + # argv, never a shell string: cfg/serial/vid_pid come from roster JSON and the + # command line, and a '$', backtick or quote in any of them would otherwise be + # substituted by the shell or break out of it + cmd = ['openocd', '-c', 'tcl_port disabled', '-c', 'gdb_port disabled', + '-c', 'telnet_port disabled'] + # probe pin: vid_pid keeps discovery from opening foreign usbfs nodes (a + # wedged one hangs the open), serial disambiguates same-model probes — + # both before the -f scripts, like hil_flash does + if vid_pid: + if not re.fullmatch(r'0x[0-9a-fA-F]{1,4} 0x[0-9a-fA-F]{1,4}', vid_pid.strip()): + # openocd only WARNS and exits 0 on a malformed value, so the pin + # silently does not apply and discovery reopens every usbfs node -- + # the convoy hil_flash.valid_vid_pid exists to stop + raise RttError(f'--vid-pid must be "0xVVVV 0xPPPP", got {vid_pid!r}') + cmd += ['-c', f'adapter usb vid_pid {vid_pid.strip()}'] + if serial_no: + cmd += ['-c', f'adapter serial {serial_no}'] + cmd += shlex.split(cfg) + cmd += ['-c', 'init'] + # opt-in: reset the target INSIDE this session, give it 2 s to boot, THEN + # attach and drain. The order is forced: `rtt start` needs the control block + # to already exist in RAM (the firmware creates it at init), and attaching + # ahead of the reset would latch the PREVIOUS run's stale block. Byte 0 still + # reaches the consumer because NO_BLOCK_SKIP retains the ring's HEAD: a boot + # burst bigger than the ring loses its tail until the drain catches up, never + # its first bytes -- which is the part a boot-anchored decoder needs + # (SystemView's Init record, carrying the timestamp frequency, is emitted once + # at boot; a mid-flight attach yields a stream no decoder can lock onto; size + # BUFFER_SIZE_UP to the boot burst if the tail matters too). Costs the tool's + # usual no-reset invariant, and is unsafe on parts where an in-session reset + # leaves the core held (SAMD5x DSU) or perturbs the target (WCH SDI). + if reset_before_attach: + cmd += ['-c', 'reset run', '-c', 'sleep 2000'] + cmd += ['-c', f'rtt setup 0x{addr:x} 0x800 "SEGGER RTT"', + '-c', 'rtt polling_interval 1', '-c', 'rtt start', + '-c', f'rtt server start {port} {channel}'] + self._spawn(cmd) + self._connect(port) + + def _gentle_stop(self, proc) -> None: + # no stdin channel to ask openocd to exit, and it keeps its listener up after + # the client disconnects: go straight to the group takedown instead of blocking + # the base class's 5 s wait on a process that has no reason to leave + with contextlib.suppress(ProcessLookupError, PermissionError): + os.killpg(proc.pid, signal.SIGTERM) + + +def dump_ring(probe: str, device: str, addr: int, out_path: str, channel: int = 0) -> int: + """Post-mortem: read aUp[channel]'s ring over the debug AP (no halt) via JLinkExe. + NO_BLOCK_SKIP means an undrained ring holds the FIRST KB after boot, not the + tail — interpretation rules in the target-debug skill.""" + if re.search(r'[\s"\']', out_path): + raise SystemExit(f'--dump path must not contain whitespace or quotes: {out_path!r} ' + f'(it is spliced into a JLinkExe script line)') + # a stale file from an earlier run must not satisfy the success check below + with contextlib.suppress(OSError): + os.remove(out_path) + # SEGGER_RTT_CB: acID[16], MaxNumUpBuffers, MaxNumDownBuffers, then aUp[] at 0x18, + # each ring 6 words {sName, pBuffer, SizeOfBuffer, WrOff, RdOff, Flags}. Read the + # counts with the descriptor so an out-of-range channel is rejected instead of + # reading whatever RAM follows the array. + jlink = ['JLinkExe', '-USB', probe, '-device', device, '-if', 'swd', + '-speed', '4000', '-NoGui', '1', '-AutoConnect', '1'] + + def _jlink_run(script: str): + # same clean-exit contract as nm_rtt_addr/_spawn: a missing binary or a wedged + # probe must not reach the CLI as a traceback + try: + return subprocess.run(jlink, input=script, capture_output=True, text=True, timeout=60) + except FileNotFoundError: + raise SystemExit('JLinkExe not on PATH — the --dump route needs J-Link Commander') + except subprocess.TimeoutExpired: + raise SystemExit('JLinkExe did not finish in 60 s — probe wedged or target unreachable?') + + script = f'mem32 {addr + 0x10:#x}, 2\nmem32 {addr + 0x18 + channel * 24:#x}, 6\nexit\n' + r = _jlink_run(script) + words = [] + for line in r.stdout.splitlines(): + # UNANCHORED: when the script arrives on stdin, some JLinkExe versions glue + # the 'J-Link>' prompt onto the result line with no newline between + m = re.search(r'([0-9A-Fa-f]{8}) = ((?:[0-9A-Fa-f]{8} ?)+)$', line.strip()) + if m: + words += [int(w, 16) for w in m.group(2).split()] + if len(words) < 8: + print(r.stdout[-500:], file=sys.stderr) + raise SystemExit(f'could not read the aUp[{channel}] descriptor — wrong control block address?') + max_up = words[0] + if not 0 < max_up <= 32: + raise SystemExit(f'control block at {addr:#x} looks uninitialized ' + f'(MaxNumUpBuffers={max_up}) — the target has not written to RTT yet, ' + f'or the address is wrong') + if channel >= max_up: + raise SystemExit(f'--channel {channel}: this firmware has {max_up} up-buffer(s) (0..{max_up - 1})') + _, pbuf, size, wroff, rdoff, _ = words[2:8] + if not pbuf or not size: + raise SystemExit(f'up-buffer {channel} is not initialized (pBuffer={pbuf:#x} size={size}) — ' + f'the target has not written to it yet') + script = f'savebin {out_path}, {pbuf:#x}, {size:#x}\nexit\n' + _jlink_run(script) + # JLinkExe exits 0 even when a command inside its script fails, so the only proof + # savebin worked is the file itself: it must hold the WHOLE ring, since a read that + # dies partway (probe disconnect, unreadable address) still leaves a short file that + # would otherwise be reported as a complete dump. Removing it also keeps the + # invariant above -- no stale file can satisfy a later run's check. + got = os.path.getsize(out_path) if os.path.exists(out_path) else 0 + if got < size: + with contextlib.suppress(OSError): + os.remove(out_path) + if got == 0: + raise SystemExit(f'savebin produced no data at {out_path} — probe or address problem') + raise SystemExit(f'savebin wrote {got}/{size} B to {out_path} (truncated dump removed) ' + f'— probe or address problem') + print(f'ring: {size} B at {pbuf:#x}, WrOff={wroff:#x} RdOff={rdoff:#x} -> {out_path}\n' + f'valid bytes wrap at WrOff; default NO_BLOCK_SKIP holds the FIRST data after ' + f'boot, not the tail', file=sys.stderr) + return 0 + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument('--backend', choices=['jlink', 'openocd'], required=True, + help='transport route — explicit, no default (skill transport matrix)') + ap.add_argument('--probe', help='probe serial (JLinkExe -USB / openocd "adapter serial")') + ap.add_argument('--vid-pid', help='openocd probe pin by USB IDs, e.g. "0x2e8a 0x000c" ' + '(with or instead of --probe)') + ap.add_argument('--device', help='JLINK_DEVICE from board.cmake/family.cmake (jlink backend)') + ap.add_argument('--cfg', help='openocd -f/-c args, e.g. "-f interface/stlink.cfg -f target/stm32h7x.cfg"') + ap.add_argument('--elf', help='the FLASHED elf: exact _SEGGER_RTT address via nm (openocd/--dump)') + ap.add_argument('--addr', help='SEGGER RTT control block address (hex), instead of --elf') + ap.add_argument('--channel', type=int, default=0, help='up-buffer index (0 console, 1 SysView)') + ap.add_argument('--seconds', type=float, default=0, help='capture duration; 0 = until Ctrl-C/EOF') + ap.add_argument('-i', '--interactive', action='store_true', help='forward stdin to the target') + ap.add_argument('--reset-before-attach', action='store_true', + help='openocd: reset the target inside the capture session so the ' + 'server is draining when it boots (needed for streams that must ' + 'include the boot preamble, e.g. SystemView); unsafe on SAMD5x/WCH') + ap.add_argument('--dump', metavar='OUT.bin', + help='post-mortem ring dump (jlink backend; needs --elf or --addr)') + args = ap.parse_args() + + if args.seconds < 0 or args.seconds != args.seconds: # negative or nan + ap.error(f'--seconds must be >= 0 (0 = until Ctrl-C/EOF), got {args.seconds}') + if args.channel < 0: + # a negative index would walk backwards off aUp[] into the control-block + # header and read garbage as a descriptor + ap.error(f'--channel must be >= 0, got {args.channel}') + + def rtt_addr(): + if args.addr: + try: + return int(args.addr, 16) + except ValueError: + ap.error(f'--addr must be hex, got {args.addr!r}') + if args.elf: + return nm_rtt_addr(args.elf) + ap.error('need --elf (flashed elf, address via nm) or --addr') + + if args.backend == 'jlink': + if args.reset_before_attach: + ap.error('--reset-before-attach is openocd-only (the J-Link route attaches ' + 'to a running target; flash and reset before starting it)') + if args.channel and not args.dump: + # -RTTTelnetPort serves the Terminal buffer only; --dump can read any ring + ap.error('the jlink backend streams channel 0 only (use --backend openocd ' + 'for another channel, or --dump to read one)') + if args.vid_pid: + ap.error('--vid-pid is openocd-only; J-Link probes are selected by serial (--probe)') + if not (args.probe and args.device): + ap.error('the jlink backend needs --probe and --device') + elif not (args.probe or args.vid_pid): + ap.error('the openocd backend needs --probe and/or --vid-pid') + + if args.dump: + if args.backend != 'jlink': + ap.error('--dump uses the jlink backend (debug-AP reads via JLinkExe)') + return dump_ring(args.probe, args.device, rtt_addr(), args.dump, args.channel) + + # install BEFORE the console exists: an external `timeout`/kill during the + # up-to-15 s connect window must still reach the cleanup below, or the openocd + # route leaves a server holding the probe and the port (JLinkExe would exit on + # stdin EOF; openocd has no such channel and its own session shields it) + def _terminate(signum, _frame): + raise KeyboardInterrupt + for _sig in (signal.SIGTERM, signal.SIGHUP): + with contextlib.suppress(ValueError, OSError): + signal.signal(_sig, _terminate) + + try: + if args.backend == 'openocd': + if not args.cfg: + ap.error('--backend openocd needs --cfg') + con = OpenocdRtt(args.cfg, rtt_addr(), args.channel, + serial_no=args.probe, vid_pid=args.vid_pid, + reset_before_attach=args.reset_before_attach) + else: + con = JlinkRtt({'flasher': {'uid': args.probe, 'args': f'-device {args.device}'}}, + timeout=0.1) + except RttError as e: + print(e, file=sys.stderr) + return 1 + except KeyboardInterrupt: + return 130 # constructors clean up after themselves on the way out + + saw_output = threading.Event() + forwarded = threading.Event() + if args.interactive: + def pump_stdin(): + # Hold input until the capture side has seen TARGET output (or 5 s for a + # quiet firmware): the J-Link telnet route silently DROPS client bytes + # until Commander locates the control block, so input forwarded at attach + # vanishes (measured on the rig: instant 'ping' lost, delayed 'ping' + # echoed). The gate must ignore the server's own banner — it arrives at + # connect, BEFORE the block is found. Raw os.read, not sys.stdin.buffer: + # bytes with no newline wait, and no BufferedReader lock — a daemon + # thread blocked holding that lock at interpreter shutdown aborts + # CPython (_enter_buffered_busy). + saw_output.wait(5) + try: + while True: + data = os.read(0, 4096) + if not data: + return + con.write(data) + forwarded.set() + except (RttError, OSError, ValueError): + return # console closed/stalled/dead; capture side reports the state + threading.Thread(target=pump_stdin, daemon=True).start() + + deadline = time.monotonic() + args.seconds if args.seconds else None + rc = 0 + seen = b'' # pre-release accumulator for the banner check only + try: + while deadline is None or time.monotonic() < deadline: + try: + chunk = con.read(con.in_waiting or 1) + except RttError as e: + print(f'rtt: {e}', file=sys.stderr) + rc = 1 + break + if chunk: + if args.interactive and not saw_output.is_set(): + # target data = anything past the J-Link banner's final line + # ('Process: <name>'); the openocd server has no banner + seen = (seen + chunk)[-65536:] + if args.backend != 'jlink': + saw_output.set() + else: + i = seen.find(b'Process: ') + j = seen.find(b'\n', i) if i >= 0 else -1 + if j >= 0 and len(seen) > j + 1: + saw_output.set() + sys.stdout.buffer.write(chunk) + sys.stdout.buffer.flush() + elif con.eof: + print('rtt: server closed the connection', file=sys.stderr) + rc = 1 + break + except KeyboardInterrupt: + pass + except BrokenPipeError: + # downstream consumer (head/grep -m) closed the pipe: a normal way to end a + # capture, not an error. Point stdout at devnull so interpreter shutdown does + # not raise on the final implicit flush. + os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno()) + finally: + if args.interactive and not forwarded.is_set(): + # only claim what is true: the gate releases after 5 s and forwards anyway, + # so "never forwarded" must come from the forwarded flag, not the gate + print('rtt: -i stdin was never forwarded to the target (no input arrived, ' + 'or the console closed first)', file=sys.stderr) + if args.interactive and not saw_output.is_set(): + print('rtt: no target output within the window', file=sys.stderr) + # a late TERM landing during the up-to-12 s teardown must not skip the kill + # escalation and orphan the server -- cleanup is committed at this point + for _sig in (signal.SIGTERM, signal.SIGHUP): + with contextlib.suppress(ValueError, OSError): + signal.signal(_sig, signal.SIG_IGN) + con.close() + return rc + + +if __name__ == '__main__': + sys.exit(main()) |