From bab21a20ba1395dcff4b54236f5d44c0b3359311 Mon Sep 17 00:00:00 2001 From: hathach Date: Thu, 9 Jul 2026 23:38:49 +0700 Subject: test/hil: usbtest.py runner + HIL integration Binds the kernel usbtest driver (gadget-zero profile), runs the tier-based battery, auto-recovers kernel-side hangs, and skips cases the host controller cannot run (MosChip MCS9990 EHCI int-OUT). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HeF2gZ1M7GWkz6Av4BpKPg --- test/hil/tinyusb.json | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) (limited to 'test/hil/tinyusb.json') diff --git a/test/hil/tinyusb.json b/test/hil/tinyusb.json index 59ab57a06..be39c9eb5 100644 --- a/test/hil/tinyusb.json +++ b/test/hil/tinyusb.json @@ -495,9 +495,13 @@ } }, { - "name": "ch582m_evt", - "uid": "D443627B5450", + "name": "ch32v307v_r1_1v0", + "uid": "DE6B3E263B3857CAFFFFFFFF", "toolchain": "riscv-gcc", + "variant": [ + {"name": "ch32v307v_r1_1v0-usbhs", "defines": ["SPEED=high"]}, + {"name": "ch32v307v_r1_1v0-usbfs", "defines": ["SPEED=full"]} + ], "tests": { "device": true, "host": false, @@ -505,12 +509,42 @@ }, "flasher": { "name": "openocd_wch", - "uid": "7FD88F0604B5", + "uid": "BC5DA47360D0", "args": "" } } ], "boards-skip": [ + { + "name": "ch582m_evt", + "uid": "D443627B5450", + "toolchain": "riscv-gcc", + "comment": "unplugged: fixture (board + WCH-Link) failed to re-enumerate after the 2026-07-06 rig reboot; replug to re-enable", + "tests": { + "device": true, + "host": false, + "dual": false + }, + "flasher": { + "name": "openocd_wch", + "uid": "7FD88F0604B5", + "args": "" + } + }, + { + "name": "nrf54lm20dk", + "uid": "899C3DE5B0F4D5CA", + "tests": { + "device": true, + "host": false, + "dual": false + }, + "flasher": { + "name": "jlink", + "uid": "1051856258", + "args": "-device NRF54LM20A_M33" + } + }, { "name": "stm32f769disco", "uid": "21002F000F51363531383437", -- cgit v1.3.1 From e02f93158cc602ba6f20945a187172abf2546718 Mon Sep 17 00:00:00 2001 From: hathach Date: Mon, 13 Jul 2026 17:53:34 +0700 Subject: test/hil: usbtest fleet enablement, shuffled scheduling, unique PIDs Pool/config: - record real uids (ra8m1_ek), enable usbtest for espressif s3/p4, then park ra6m5_ek and ra8m1_ek in boards-skip (ra6m5's usbtest/MSC traffic can kill the uPD720201 host on its ROM firmware; ra8m1 USBHS bring-up pending); max32666/nrf54lm20 stay enabled - their MosChip flakiness never wedges - re-enable device/usbtest on HS boards (mimxrt1064, ch32v307) now that uPD720201 firmware 2.0.2.6 fixes the command-ring death; mimxrt1015 stays skipped - its HS battery killed the controller on both ROM and 2.0.2.6 firmware (board-specific); match the moved host-test bundles (f723 <-> rt1064); skip never-passing tests on the new nrf5340dk/nrf54lm20dk boards and the detached pico host bundle, each documented with a comment Host-controller quirk gating in usbtest.py (auto-skip, self-heals on a healthy xHCI): - MosChip MCS9990 EHCI: case 25 (int-OUT never scheduled, FRINDEX bug) and case 11 (unlinked reads complete short/EREMOTEIO) - Renesas uPD720201 xHCI: firmware-gated. The card must run firmware >= 2.0.2.6 (RAM-uploaded - it reverts to ROM on every power cycle): on older firmware the command ring dies under unlink stress (a Configure Endpoint command stops completing; the hub worker deadlocks holding the device lock; only a host power cycle recovers; three boards reproduced it). usbtest.py reads the FW version register (PCI config 0x6c) and refuses to run at all on older firmware - hil_test surfaces that as a failed test with the reason. On current firmware the full 30-case battery runs (validated FS+HS: metro_m4, f723, f723-DMA all 30/30). Scheduling (hil_test.py): - Shuffle each (board, variant)'s test order with a seeded RNG (HIL_SHUFFLE_SEED to replay) so usbtest batteries and flash churn spread across the timeline instead of convoying on one controller. - Per-controller usbtest + flash semaphores: HIL_USBTEST_PARALLEL (default 4) concurrent usbtest batteries and HIL_FLASH_PARALLEL (default 8) concurrent flashes per host controller. Profiled on uPD720201 firmware 2.0.2.6 across 8/1..12/8: wall time falls 22.2/14.3/12.5/10.8 min at usbtest width 1/2/3/4 and plateaus there; zero controller errors everywhere; first battery case failures (leaf-hub bandwidth stretch) appear at 12/8, and flash width 12 only amplifies flasher-hub contention flakes - so 8/4 is the optimum. A separate battery-window flash throttle was profiled and dropped. - Give every example a unique hardcoded USB PID (0x4001-0x4022, usbtest keeps 0x4010) instead of the PID_MAP interface bitmap: different examples now always re-enumerate back-to-back, even on boards whose CPU reset does not drop D+ (WCH CH58x), so the EXAMPLE_PID table and same-PID adjacency reordering in hil_test.py are gone; only the variant-boundary same-example repeat needs a swap. - Report matrix: stable columns with the metric-bearing tests pinned first (usbtest, cdc_msc_throughput, msc_file_explorer[_freertos]), the rest alphabetical. Fail fast: - enum wait budget 8 s on the first attempt, 4 s on retries; dfu waits are deadline-based so dfu-util's own runtime counts against the budget. A device-absent failure now costs ~3-5x a passing test (20-30 s) instead of 10-30x (47-150 s). - CI runs hil_test with --retry 1 and no in-run second pass: a broken fixture fails the job fast instead of holding the self-hosted runner for hours and blocking other PRs' HIL jobs. hil_test still writes the .skip sidecar, so a manual re-run attempt only retests what failed. Review fixes (multi-agent adversarial review of this commit): - tinyusb_win_usbser.inf: the PID rework moved five CDC examples onto even PIDs the INF's odd-only DeviceList never matched (legacy-Windows usbser binding) - appended 0x4006/4008/400a/4020/4022 to both lists. - usbtest example: USBTEST_TIER is now overridable and the descriptors and pumps are tier-conditional, so a board whose DCD cannot serve a tier lowers it instead of skipping the whole example - RA2A1 (RUSB2 with no isochronous pipe) builds at tier 3 via its BOARD_ define; the host battery follows the tier advertised in bcdDevice. Tier-4 output verified byte-identical after the refactor. - dynamic_configuration's second config derived USB_PID + 11 = 0x4018, colliding with net_lwip_webserver - now USB_PID + 0x0100, outside the per-example space. tools/check_example_pids.py (pre-commit hook) enforces PID uniqueness incl. derived and literal idProduct values. - usbtest.py firmware gate: matched by device ID (uPD720201/720202, both use the 0x6c FW register), and an unreadable version (setpci missing/denied) now refuses with its own message instead of masquerading as "firmware 0x00000000"; noted the gate is necessary but not sufficient (board-specific kills stay per-board skips). - hil_test: deadline waits use time.monotonic(); multiprocessing context pinned to fork (raw semaphores in Pool initargs); flash and usbtest permits unified into one fail-closed, exception-safe ctrl_permit (unknown controller takes every slot and logs a warning instead of silently borrowing slot 0); an all-skipped battery reports as skip, not "0/0" failure; slow-body polls (mtp, printer, disk read) go through a shared deadline-based wait_until so their bodies count against the enum budget; throughput's FS detection compares serials case-insensitively like every other walk; a missing MSC read-speed line now fails the host msc_file_explorer test instead of passing with an empty metric. Hardening: - fail fast (15 s) when a driver-registry sysfs write blocks: a wedged device otherwise turns every subsequent battery into an unkillable D-state writer and silently hangs the whole run - usb-recover skill: a VM reboot is not a reliable cure (MosChip hubs latch up across the PCIe reset); full host power cycle is Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HeF2gZ1M7GWkz6Av4BpKPg --- .claude/skills/usb-recover/SKILL.md | 16 +- .github/workflows/build.yml | 14 +- .pre-commit-config.yaml | 7 + .../audio_4_channel_mic/src/usb_descriptors.c | 11 +- .../src/usb_descriptors.c | 11 +- examples/device/audio_test/src/usb_descriptors.c | 11 +- .../audio_test_freertos/src/usb_descriptors.c | 11 +- .../audio_test_multi_rate/src/usb_descriptors.c | 11 +- .../device/cdc_dual_ports/src/usb_descriptors.c | 11 +- examples/device/cdc_msc/src/usb_descriptors.c | 11 +- .../device/cdc_msc_freertos/src/usb_descriptors.c | 11 +- .../cdc_msc_throughput/src/usb_descriptors.c | 3 +- examples/device/cdc_uac2/src/usb_descriptors.c | 11 +- examples/device/dfu/src/usb_descriptors.c | 10 +- examples/device/dfu_runtime/src/usb_descriptors.c | 10 +- .../dynamic_configuration/src/usb_descriptors.c | 13 +- .../hid_boot_interface/src/usb_descriptors.c | 10 +- .../device/hid_composite/src/usb_descriptors.c | 11 +- .../hid_composite_freertos/src/usb_descriptors.c | 11 +- .../device/hid_generic_inout/src/usb_descriptors.c | 11 +- .../hid_multiple_interface/src/usb_descriptors.c | 11 +- examples/device/midi_test/src/usb_descriptors.c | 11 +- .../midi_test_freertos/src/usb_descriptors.c | 11 +- examples/device/msc_dual_lun/src/usb_descriptors.c | 11 +- examples/device/mtp/src/usb_descriptors.c | 11 +- .../net_lwip_webserver/src/usb_descriptors.c | 12 +- .../device/printer_to_cdc/src/usb_descriptors.c | 2 +- examples/device/uac2_headset/src/usb_descriptors.c | 11 +- .../device/uac2_speaker_fb/src/usb_descriptors.c | 11 +- examples/device/usbtest/src/main.c | 8 + examples/device/usbtest/src/usb_descriptors.c | 24 +- examples/device/usbtest/src/usb_descriptors.h | 11 +- examples/device/usbtmc/src/usb_descriptors.c | 11 +- .../device/video_capture/src/usb_descriptors.c | 11 +- .../device/video_capture_2ch/src/usb_descriptors.c | 11 +- .../device/webusb_serial/src/usb_descriptors.c | 11 +- examples/dual/dynamic_switch/src/usb_descriptors.c | 11 +- .../host_hid_to_device_cdc/src/usb_descriptors.c | 11 +- .../host_info_to_device_cdc/src/usb_descriptors.c | 11 +- test/hil/hil_test.py | 356 ++++++++++++++++----- test/hil/tinyusb.json | 83 ++++- test/hil/usbtest.py | 77 ++++- tools/check_example_pids.py | 55 ++++ tools/usb_drivers/tinyusb_win_usbser.inf | 4 +- 44 files changed, 582 insertions(+), 419 deletions(-) create mode 100644 tools/check_example_pids.py (limited to 'test/hil/tinyusb.json') diff --git a/.claude/skills/usb-recover/SKILL.md b/.claude/skills/usb-recover/SKILL.md index 7f3e86632..3f72b7e21 100644 --- a/.claude/skills/usb-recover/SKILL.md +++ b/.claude/skills/usb-recover/SKILL.md @@ -36,7 +36,9 @@ the ioctl then returns and the convoy unwinds on its own. **Not every controller supports FLR.** The Renesas uPD720201 (`0000:01:00.0`) has no reset method — `pci-reset` fails with `Inappropriate ioctl for device` -(ENOTTY). On those, there is no clean D-state cure short of a **reboot**; do NOT +(ENOTTY). On those, there is no clean software D-state cure — a VM reboot is NOT +reliable (the MosChip downstream hubs latch up across the PCIe reset and need a +physical replug); ask the operator for a full PVE host power cycle instead. Do NOT fall through to `pci-rebind` (see next). **`pci-rebind` can strand the controller driverless.** Its unbind succeeds but, @@ -44,8 +46,8 @@ with a D-state process still holding a URB, the *re-bind* hangs — leaving the PCI device with **no driver** (`/sys/bus/pci/devices//driver` gone) and the whole controller's fixtures offline. A second `pci-rebind` then dies with "no driver bound". Recover with `pci-bind ` (re-attaches the xHCI driver); -if that also hangs because the D-state URB is unkillable, **reboot** is the only -cure. The Renesas binds via `xhci-pci-renesas` (firmware loader), others via +if that also hangs because the D-state URB is unkillable, only a full PVE host +power cycle (operator action) recovers. The Renesas binds via `xhci-pci-renesas` (firmware loader), others via `xhci_hcd` — `pci-bind` auto-tries both, or pass the driver explicitly. **Ordering is critical.** `authorized`/`rebind`/`pci-rebind` all take the @@ -53,7 +55,7 @@ per-device lock the stuck ioctl holds — they block and join the convoy, and soon every libusb tool (uhubctl, JLinkExe) hangs too. Worse, a blocked `pci-rebind` grabs the PCI device lock on its way in, which `pci-reset` also needs: once a rebind has been attempted and is stuck, even FLR deadlocks and -**only a rig reboot recovers**. pci-reset first (if supported), and never +**only a full PVE host power cycle recovers**. pci-reset first (if supported), and never `pci-rebind` a D-state wedge. **If no** (device merely dead or silent), escalate gently: @@ -82,10 +84,10 @@ port power switching — uhubctl reports "No compatible devices" there. - Command produces no output and doesn't return → it is blocked on the device lock: a D-state holder exists; see above. - Trying `pci-rebind` on a D-state hang — its re-bind hangs and strands the - controller **driverless**; recover with `pci-bind `, or reboot if the - D-state URB is unkillable. Use `pci-reset` (if supported) for D-state, never + controller **driverless**; recover with `pci-bind `, or a PVE host power + cycle if the D-state URB is unkillable. Use `pci-reset` (if supported) for D-state, never `pci-rebind`. - Running `pci-reset` on a controller without FLR support (Renesas) → ENOTTY; - no recovery but reboot. + no software recovery — needs a PVE host power cycle. - A J-Link reset (`r; go`) does not disconnect a wedged DUT from the host: the DWC2 soft-connect pullup stays up through a core halt, so stuck URBs stay stuck. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c8c597e50..f24f3ae1f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -333,15 +333,11 @@ jobs: merge-multiple: true - name: Test on actual hardware - run: | - python3 test/hil/hil_test.py ${{ env.HIL_JSON }} $SKIP_BOARDS || \ - (if [ -f "${{ env.HIL_JSON }}.skip" ]; then - SKIP_BOARDS=$(cat "${{ env.HIL_JSON }}.skip") - echo "Re-running with SKIP_BOARDS=$SKIP_BOARDS" - python3 test/hil/hil_test.py ${{ env.HIL_JSON }} $SKIP_BOARDS - else - exit 1 - fi) + # Single attempt per test (--retry 1), no in-run second pass: a broken fixture + # fails fast instead of holding the runner (and other PRs' HIL jobs) for hours. + # hil_test.py still writes ${HIL_JSON}.skip, so a manual re-run attempt only + # retests what failed (see "Get Skip Boards from previous run"). + run: python3 test/hil/hil_test.py --retry 1 ${{ env.HIL_JSON }} $SKIP_BOARDS - name: Upload HIL report if: always() && github.event_name == 'pull_request' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f4a297289..e87b935dd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -33,6 +33,13 @@ repos: - repo: local hooks: + - id: unique-example-pids + name: unique example USB PIDs + files: usb_descriptors\.c$ + entry: python3 tools/check_example_pids.py + pass_filenames: false + language: system + - id: unit-test name: unit-test files: ^(src/|test/unit-test/) diff --git a/examples/device/audio_4_channel_mic/src/usb_descriptors.c b/examples/device/audio_4_channel_mic/src/usb_descriptors.c index 6b9a9bbae..42da9442c 100644 --- a/examples/device/audio_4_channel_mic/src/usb_descriptors.c +++ b/examples/device/audio_4_channel_mic/src/usb_descriptors.c @@ -26,15 +26,8 @@ #include "bsp/board_api.h" #include "tusb.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] AUDIO | MIDI | HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(AUDIO, 4) | PID_MAP(VENDOR, 5) ) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x4001 //--------------------------------------------------------------------+ // Device Descriptors diff --git a/examples/device/audio_4_channel_mic_freertos/src/usb_descriptors.c b/examples/device/audio_4_channel_mic_freertos/src/usb_descriptors.c index 216cd062a..0afb3df0a 100644 --- a/examples/device/audio_4_channel_mic_freertos/src/usb_descriptors.c +++ b/examples/device/audio_4_channel_mic_freertos/src/usb_descriptors.c @@ -26,15 +26,8 @@ #include "bsp/board_api.h" #include "tusb.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] AUDIO | MIDI | HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(AUDIO, 4) | PID_MAP(VENDOR, 5) ) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x4002 //--------------------------------------------------------------------+ // Device Descriptors diff --git a/examples/device/audio_test/src/usb_descriptors.c b/examples/device/audio_test/src/usb_descriptors.c index cea4eb8d1..8c25fc290 100644 --- a/examples/device/audio_test/src/usb_descriptors.c +++ b/examples/device/audio_test/src/usb_descriptors.c @@ -26,15 +26,8 @@ #include "bsp/board_api.h" #include "tusb.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] AUDIO | MIDI | HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(AUDIO, 4) | PID_MAP(VENDOR, 5) ) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x4003 //--------------------------------------------------------------------+ // Device Descriptors diff --git a/examples/device/audio_test_freertos/src/usb_descriptors.c b/examples/device/audio_test_freertos/src/usb_descriptors.c index 37ebf84d3..709425c49 100644 --- a/examples/device/audio_test_freertos/src/usb_descriptors.c +++ b/examples/device/audio_test_freertos/src/usb_descriptors.c @@ -26,15 +26,8 @@ #include "bsp/board_api.h" #include "tusb.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] AUDIO | MIDI | HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(AUDIO, 4) | PID_MAP(VENDOR, 5) ) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x4004 //--------------------------------------------------------------------+ // Device Descriptors diff --git a/examples/device/audio_test_multi_rate/src/usb_descriptors.c b/examples/device/audio_test_multi_rate/src/usb_descriptors.c index b1f60dd10..008c8cd76 100644 --- a/examples/device/audio_test_multi_rate/src/usb_descriptors.c +++ b/examples/device/audio_test_multi_rate/src/usb_descriptors.c @@ -28,15 +28,8 @@ #include "tusb.h" #include "usb_descriptors.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] AUDIO | MIDI | HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(AUDIO, 4) | PID_MAP(VENDOR, 5) ) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x4005 //--------------------------------------------------------------------+ // Device Descriptors diff --git a/examples/device/cdc_dual_ports/src/usb_descriptors.c b/examples/device/cdc_dual_ports/src/usb_descriptors.c index adfd8cf9d..779221c0c 100644 --- a/examples/device/cdc_dual_ports/src/usb_descriptors.c +++ b/examples/device/cdc_dual_ports/src/usb_descriptors.c @@ -26,15 +26,8 @@ #include "bsp/board_api.h" #include "tusb.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(VENDOR, 4) ) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x4006 #define USB_VID 0xCafe #define USB_BCD 0x0200 diff --git a/examples/device/cdc_msc/src/usb_descriptors.c b/examples/device/cdc_msc/src/usb_descriptors.c index 5dc80dee3..140ef2140 100644 --- a/examples/device/cdc_msc/src/usb_descriptors.c +++ b/examples/device/cdc_msc/src/usb_descriptors.c @@ -26,15 +26,8 @@ #include "bsp/board_api.h" #include "tusb.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(VENDOR, 4) ) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x4007 #define USB_VID 0xCafe #define USB_BCD 0x0200 diff --git a/examples/device/cdc_msc_freertos/src/usb_descriptors.c b/examples/device/cdc_msc_freertos/src/usb_descriptors.c index f5b015051..8398f0365 100644 --- a/examples/device/cdc_msc_freertos/src/usb_descriptors.c +++ b/examples/device/cdc_msc_freertos/src/usb_descriptors.c @@ -26,15 +26,8 @@ #include "bsp/board_api.h" #include "tusb.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(VENDOR, 4) ) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x4008 #define USB_VID 0xCafe #define USB_BCD 0x0200 diff --git a/examples/device/cdc_msc_throughput/src/usb_descriptors.c b/examples/device/cdc_msc_throughput/src/usb_descriptors.c index 3b0ff6e17..ba0b0a26f 100644 --- a/examples/device/cdc_msc_throughput/src/usb_descriptors.c +++ b/examples/device/cdc_msc_throughput/src/usb_descriptors.c @@ -26,7 +26,8 @@ #include "bsp/board_api.h" #include "tusb.h" -#define USB_PID (0x4000 | ((CFG_TUD_CDC) ? (1 << 0) : 0) | ((CFG_TUD_MSC) ? (1 << 1) : 0)) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x4009 #define USB_VID 0xCafe #define USB_BCD 0x0200 diff --git a/examples/device/cdc_uac2/src/usb_descriptors.c b/examples/device/cdc_uac2/src/usb_descriptors.c index fdffc761e..9c1bbee47 100644 --- a/examples/device/cdc_uac2/src/usb_descriptors.c +++ b/examples/device/cdc_uac2/src/usb_descriptors.c @@ -29,15 +29,8 @@ #include "tusb.h" #include "usb_descriptors.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] AUDIO | MIDI | HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(AUDIO, 4) | PID_MAP(VENDOR, 5) ) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x400a //--------------------------------------------------------------------+ // Device Descriptors diff --git a/examples/device/dfu/src/usb_descriptors.c b/examples/device/dfu/src/usb_descriptors.c index e4291be2d..5bfb32b2e 100644 --- a/examples/device/dfu/src/usb_descriptors.c +++ b/examples/device/dfu/src/usb_descriptors.c @@ -26,14 +26,8 @@ #include "bsp/board_api.h" #include "tusb.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | PID_MAP(MIDI, 3) | PID_MAP(VENDOR, 4)) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x400b //--------------------------------------------------------------------+ // Device Descriptors diff --git a/examples/device/dfu_runtime/src/usb_descriptors.c b/examples/device/dfu_runtime/src/usb_descriptors.c index 8fa078da2..273566414 100644 --- a/examples/device/dfu_runtime/src/usb_descriptors.c +++ b/examples/device/dfu_runtime/src/usb_descriptors.c @@ -26,14 +26,8 @@ #include "bsp/board_api.h" #include "tusb.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | PID_MAP(MIDI, 3) | PID_MAP(VENDOR, 4)) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x400c //--------------------------------------------------------------------+ // Device Descriptors diff --git a/examples/device/dynamic_configuration/src/usb_descriptors.c b/examples/device/dynamic_configuration/src/usb_descriptors.c index c4049414f..838052ea1 100644 --- a/examples/device/dynamic_configuration/src/usb_descriptors.c +++ b/examples/device/dynamic_configuration/src/usb_descriptors.c @@ -26,15 +26,8 @@ #include "bsp/board_api.h" #include "tusb.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(VENDOR, 4) ) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x400d // Configuration mode // 0 : enumerated as CDC/MIDI. Board button is not pressed when enumerating @@ -79,7 +72,7 @@ tusb_desc_device_t const desc_device_1 = .bMaxPacketSize0 = CFG_TUD_ENDPOINT0_SIZE, .idVendor = 0xCafe, - .idProduct = USB_PID + 11, // should be different PID than desc0 + .idProduct = USB_PID + 0x0100, // must differ from desc0's PID and stay outside the 0x40xx per-example space .bcdDevice = 0x0100, .iManufacturer = 0x01, diff --git a/examples/device/hid_boot_interface/src/usb_descriptors.c b/examples/device/hid_boot_interface/src/usb_descriptors.c index b5c31a94a..4d5caa835 100644 --- a/examples/device/hid_boot_interface/src/usb_descriptors.c +++ b/examples/device/hid_boot_interface/src/usb_descriptors.c @@ -27,14 +27,8 @@ #include "tusb.h" #include "usb_descriptors.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | PID_MAP(MIDI, 3) | PID_MAP(VENDOR, 4)) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x400e //--------------------------------------------------------------------+ // Device Descriptors diff --git a/examples/device/hid_composite/src/usb_descriptors.c b/examples/device/hid_composite/src/usb_descriptors.c index 46e4b63f9..7f5f74c70 100644 --- a/examples/device/hid_composite/src/usb_descriptors.c +++ b/examples/device/hid_composite/src/usb_descriptors.c @@ -27,15 +27,8 @@ #include "tusb.h" #include "usb_descriptors.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(VENDOR, 4) ) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x400f #define USB_VID 0xCafe #define USB_BCD 0x0200 diff --git a/examples/device/hid_composite_freertos/src/usb_descriptors.c b/examples/device/hid_composite_freertos/src/usb_descriptors.c index a745c17b5..a0464afae 100644 --- a/examples/device/hid_composite_freertos/src/usb_descriptors.c +++ b/examples/device/hid_composite_freertos/src/usb_descriptors.c @@ -27,15 +27,8 @@ #include "tusb.h" #include "usb_descriptors.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(VENDOR, 4) ) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x4011 #define USB_VID 0xCafe #define USB_BCD 0x0200 diff --git a/examples/device/hid_generic_inout/src/usb_descriptors.c b/examples/device/hid_generic_inout/src/usb_descriptors.c index 93e718461..f179b74f7 100644 --- a/examples/device/hid_generic_inout/src/usb_descriptors.c +++ b/examples/device/hid_generic_inout/src/usb_descriptors.c @@ -26,15 +26,8 @@ #include "bsp/board_api.h" #include "tusb.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(VENDOR, 4) ) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x4012 //--------------------------------------------------------------------+ // Device Descriptors diff --git a/examples/device/hid_multiple_interface/src/usb_descriptors.c b/examples/device/hid_multiple_interface/src/usb_descriptors.c index cd2d93c44..90aef6dd7 100644 --- a/examples/device/hid_multiple_interface/src/usb_descriptors.c +++ b/examples/device/hid_multiple_interface/src/usb_descriptors.c @@ -26,15 +26,8 @@ #include "bsp/board_api.h" #include "tusb.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(VENDOR, 4) ) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x4013 //--------------------------------------------------------------------+ // Device Descriptors diff --git a/examples/device/midi_test/src/usb_descriptors.c b/examples/device/midi_test/src/usb_descriptors.c index 99c798ce1..fc7228c35 100644 --- a/examples/device/midi_test/src/usb_descriptors.c +++ b/examples/device/midi_test/src/usb_descriptors.c @@ -26,15 +26,8 @@ #include "bsp/board_api.h" #include "tusb.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(VENDOR, 4) ) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x4014 //--------------------------------------------------------------------+ // Device Descriptors diff --git a/examples/device/midi_test_freertos/src/usb_descriptors.c b/examples/device/midi_test_freertos/src/usb_descriptors.c index 99c798ce1..bfdbc555e 100644 --- a/examples/device/midi_test_freertos/src/usb_descriptors.c +++ b/examples/device/midi_test_freertos/src/usb_descriptors.c @@ -26,15 +26,8 @@ #include "bsp/board_api.h" #include "tusb.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(VENDOR, 4) ) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x4015 //--------------------------------------------------------------------+ // Device Descriptors diff --git a/examples/device/msc_dual_lun/src/usb_descriptors.c b/examples/device/msc_dual_lun/src/usb_descriptors.c index b328cf17f..5e036a8c4 100644 --- a/examples/device/msc_dual_lun/src/usb_descriptors.c +++ b/examples/device/msc_dual_lun/src/usb_descriptors.c @@ -26,15 +26,8 @@ #include "bsp/board_api.h" #include "tusb.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(VENDOR, 4) ) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x4016 //--------------------------------------------------------------------+ // Device Descriptors diff --git a/examples/device/mtp/src/usb_descriptors.c b/examples/device/mtp/src/usb_descriptors.c index 4c840560e..fefa8a239 100644 --- a/examples/device/mtp/src/usb_descriptors.c +++ b/examples/device/mtp/src/usb_descriptors.c @@ -26,15 +26,8 @@ #include "bsp/board_api.h" #include "tusb.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] MTP | VENDOR | MIDI | HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(VENDOR, 4) | PID_MAP(MTP, 5)) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x4017 #define USB_VID 0xCafe #define USB_BCD 0x0200 diff --git a/examples/device/net_lwip_webserver/src/usb_descriptors.c b/examples/device/net_lwip_webserver/src/usb_descriptors.c index 09090bb92..85a6a420c 100644 --- a/examples/device/net_lwip_webserver/src/usb_descriptors.c +++ b/examples/device/net_lwip_webserver/src/usb_descriptors.c @@ -27,16 +27,8 @@ #include "class/net/net_device.h" #include "tusb.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] NET | VENDOR | MIDI | HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID \ - (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | PID_MAP(MIDI, 3) | PID_MAP(VENDOR, 4) | \ - PID_MAP(ECM_RNDIS, 5) | PID_MAP(NCM, 5)) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x4018 // String Descriptor Index enum { diff --git a/examples/device/printer_to_cdc/src/usb_descriptors.c b/examples/device/printer_to_cdc/src/usb_descriptors.c index db7bfe97a..b9450c87e 100644 --- a/examples/device/printer_to_cdc/src/usb_descriptors.c +++ b/examples/device/printer_to_cdc/src/usb_descriptors.c @@ -29,7 +29,7 @@ #include "usb_descriptors.h" #define USB_VID 0xCafe -#define USB_PID 0x4005 +#define USB_PID 0x4019 #define USB_BCD 0x0200 //--------------------------------------------------------------------+ diff --git a/examples/device/uac2_headset/src/usb_descriptors.c b/examples/device/uac2_headset/src/usb_descriptors.c index b554e7195..1615b92ec 100644 --- a/examples/device/uac2_headset/src/usb_descriptors.c +++ b/examples/device/uac2_headset/src/usb_descriptors.c @@ -28,15 +28,8 @@ #include "tusb.h" #include "usb_descriptors.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] AUDIO | MIDI | HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(AUDIO, 4) | PID_MAP(VENDOR, 5) ) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x401a //--------------------------------------------------------------------+ // Device Descriptors diff --git a/examples/device/uac2_speaker_fb/src/usb_descriptors.c b/examples/device/uac2_speaker_fb/src/usb_descriptors.c index f0c780e38..40a36cbf4 100644 --- a/examples/device/uac2_speaker_fb/src/usb_descriptors.c +++ b/examples/device/uac2_speaker_fb/src/usb_descriptors.c @@ -28,15 +28,8 @@ #include "usb_descriptors.h" #include "common_types.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] AUDIO | MIDI | HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(AUDIO, 4) | PID_MAP(VENDOR, 5) ) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x401b //--------------------------------------------------------------------+ // Device Descriptors diff --git a/examples/device/usbtest/src/main.c b/examples/device/usbtest/src/main.c index 78575ac9b..e57a90161 100644 --- a/examples/device/usbtest/src/main.c +++ b/examples/device/usbtest/src/main.c @@ -61,7 +61,9 @@ static uint32_t blink_interval_ms = BLINK_NOT_MOUNTED; // unintended short packet or ZLP. static uint8_t const tx_chunk[CFG_TUD_VENDOR_TX_EPSIZE]; static uint8_t const int_tx_chunk[USBTEST_INT_EP_MPS]; +#if USBTEST_TIER >= 4 static uint8_t const iso_tx_chunk[USBTEST_ISO_EP_MPS]; +#endif // Interrupt/iso submit one packet per (micro)frame, sized to the NEGOTIATED speed's mps — a // high-speed build enumerated at full speed must submit the FS length, not the HS-capacity buffer @@ -69,9 +71,11 @@ static uint8_t const iso_tx_chunk[USBTEST_ISO_EP_MPS]; static inline uint16_t usbtest_int_len(void) { return (tud_speed_get() == TUSB_SPEED_HIGH) ? USBTEST_INT_EP_MPS_HS : USBTEST_INT_EP_MPS_FS; } +#if USBTEST_TIER >= 4 static inline uint16_t usbtest_iso_len(void) { return (tud_speed_get() == TUSB_SPEED_HIGH) ? USBTEST_ISO_EP_MPS_HS : USBTEST_ISO_EP_MPS_FS; } +#endif //------------- prototypes -------------// void led_blinking_task(void* param); @@ -125,10 +129,12 @@ static void usbtest_pump(void) { tud_vendor_int_write(int_tx_chunk, usbtest_int_len()); } +#if USBTEST_TIER >= 4 tud_vendor_iso_read_xfer(); // isochronous sink if (tud_vendor_iso_write_available()) { tud_vendor_iso_write(iso_tx_chunk, usbtest_iso_len()); } +#endif } } @@ -175,6 +181,7 @@ void tud_vendor_int_tx_cb(uint8_t idx, uint32_t sent_bytes) { // Isochronous pair: same discard/refill pumps; a completion may be a missed // frame, re-arm regardless +#if USBTEST_TIER >= 4 void tud_vendor_iso_rx_cb(uint8_t idx, const uint8_t* buffer, uint32_t bufsize) { (void) idx; (void) buffer; @@ -187,6 +194,7 @@ void tud_vendor_iso_tx_cb(uint8_t idx, uint32_t sent_bytes) { (void) sent_bytes; tud_vendor_iso_write(iso_tx_chunk, usbtest_iso_len()); } +#endif //--------------------------------------------------------------------+ // Vendor control requests (EP0) diff --git a/examples/device/usbtest/src/usb_descriptors.c b/examples/device/usbtest/src/usb_descriptors.c index 24efef453..b4f46adb8 100644 --- a/examples/device/usbtest/src/usb_descriptors.c +++ b/examples/device/usbtest/src/usb_descriptors.c @@ -67,21 +67,29 @@ enum { // Vendor interface, Gadget-Zero style altsettings: alt 0 carries no endpoints (an // isochronous endpoint must not claim bandwidth in the default altsetting, USB 2.0 -// 5.6.3), alt 1 carries bulk + interrupt + isochronous IN/OUT. The host usbtest -// driver skips altsettings without pipes and selects alt 1 itself. No TUD_ macro -// covers this layout, hand-rolled. -#define USBTEST_DESC_LEN (9 + 9 + 6*7) +// 5.6.3), alt 1 carries bulk + interrupt (+ isochronous IN/OUT at tier 4). The host +// usbtest driver skips altsettings without pipes and selects alt 1 itself. No TUD_ +// macro covers this layout, hand-rolled. +#if USBTEST_TIER >= 4 + #define USBTEST_EP_COUNT 6 + #define USBTEST_ISO_EPS(_isoout, _isoin, _iso_mps, _iso_interval) \ + ,7, TUSB_DESC_ENDPOINT, _isoout, (uint8_t)(TUSB_XFER_ISOCHRONOUS | (uint8_t)(TUSB_ISO_EP_ATT_ASYNCHRONOUS)), U16_TO_U8S_LE(_iso_mps), _iso_interval,\ + 7, TUSB_DESC_ENDPOINT, _isoin, (uint8_t)(TUSB_XFER_ISOCHRONOUS | (uint8_t)(TUSB_ISO_EP_ATT_ASYNCHRONOUS)), U16_TO_U8S_LE(_iso_mps), _iso_interval +#else + #define USBTEST_EP_COUNT 4 + #define USBTEST_ISO_EPS(_isoout, _isoin, _iso_mps, _iso_interval) +#endif +#define USBTEST_DESC_LEN (9 + 9 + USBTEST_EP_COUNT*7) #define USBTEST_DESCRIPTOR(_itfnum, _stridx, _epout, _epin, _bulk_mps, _intout, _intin, _int_mps, _int_interval, _isoout, _isoin, _iso_mps, _iso_interval) \ /* alt 0: zero bandwidth, no endpoints */\ 9, TUSB_DESC_INTERFACE, _itfnum, 0, 0, TUSB_CLASS_VENDOR_SPECIFIC, 0x00, 0x00, _stridx,\ /* alt 1: full source/sink set */\ - 9, TUSB_DESC_INTERFACE, _itfnum, 1, 6, TUSB_CLASS_VENDOR_SPECIFIC, 0x00, 0x00, _stridx,\ + 9, TUSB_DESC_INTERFACE, _itfnum, 1, USBTEST_EP_COUNT, TUSB_CLASS_VENDOR_SPECIFIC, 0x00, 0x00, _stridx,\ 7, TUSB_DESC_ENDPOINT, _epout, TUSB_XFER_BULK, U16_TO_U8S_LE(_bulk_mps), 0,\ 7, TUSB_DESC_ENDPOINT, _epin, TUSB_XFER_BULK, U16_TO_U8S_LE(_bulk_mps), 0,\ 7, TUSB_DESC_ENDPOINT, _intout, TUSB_XFER_INTERRUPT, U16_TO_U8S_LE(_int_mps), _int_interval,\ - 7, TUSB_DESC_ENDPOINT, _intin, TUSB_XFER_INTERRUPT, U16_TO_U8S_LE(_int_mps), _int_interval,\ - 7, TUSB_DESC_ENDPOINT, _isoout, (uint8_t)(TUSB_XFER_ISOCHRONOUS | (uint8_t)(TUSB_ISO_EP_ATT_ASYNCHRONOUS)), U16_TO_U8S_LE(_iso_mps), _iso_interval,\ - 7, TUSB_DESC_ENDPOINT, _isoin, (uint8_t)(TUSB_XFER_ISOCHRONOUS | (uint8_t)(TUSB_ISO_EP_ATT_ASYNCHRONOUS)), U16_TO_U8S_LE(_iso_mps), _iso_interval + 7, TUSB_DESC_ENDPOINT, _intin, TUSB_XFER_INTERRUPT, U16_TO_U8S_LE(_int_mps), _int_interval\ + USBTEST_ISO_EPS(_isoout, _isoin, _iso_mps, _iso_interval) #define CONFIG_TOTAL_LEN (TUD_CONFIG_DESC_LEN + USBTEST_DESC_LEN) diff --git a/examples/device/usbtest/src/usb_descriptors.h b/examples/device/usbtest/src/usb_descriptors.h index 61b931bd0..bcf8b5ec4 100644 --- a/examples/device/usbtest/src/usb_descriptors.h +++ b/examples/device/usbtest/src/usb_descriptors.h @@ -32,7 +32,16 @@ // 2: + vendor control 0x5b/0x5c (ctrl_out) // 3: + interrupt source/sink // 4: + isochronous source/sink -#define USBTEST_TIER 4 +// Default is the full tier 4; a board whose DCD cannot serve a tier lowers it here +// (BOARD_ is defined by both build systems) and the host battery follows. +#ifndef USBTEST_TIER + #if defined(BOARD_RA2A1_EK) + // RA2A1's RUSB2 instance has no isochronous pipe (other RA parts have pipes 1-2) + #define USBTEST_TIER 3 + #else + #define USBTEST_TIER 4 + #endif +#endif // Interrupt/isochronous endpoint max packet sizes, must match the configuration descriptor. // TUD_OPT_HIGH_SPEED is a compile-time capability flag, NOT the live bus speed, so the full-speed diff --git a/examples/device/usbtmc/src/usb_descriptors.c b/examples/device/usbtmc/src/usb_descriptors.c index ecdcef834..5ba5d9367 100644 --- a/examples/device/usbtmc/src/usb_descriptors.c +++ b/examples/device/usbtmc/src/usb_descriptors.c @@ -28,15 +28,8 @@ #include "class/usbtmc/usbtmc.h" #include "class/usbtmc/usbtmc_device.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(VENDOR, 4) ) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x401c #define USB_VID 0xcafe #define USB_BCD 0x0200 diff --git a/examples/device/video_capture/src/usb_descriptors.c b/examples/device/video_capture/src/usb_descriptors.c index b3382c82d..d5d805f0b 100644 --- a/examples/device/video_capture/src/usb_descriptors.c +++ b/examples/device/video_capture/src/usb_descriptors.c @@ -27,15 +27,8 @@ #include "tusb.h" #include "usb_descriptors.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] VIDEO | AUDIO | MIDI | HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(AUDIO, 4) | PID_MAP(VIDEO, 5) | PID_MAP(VENDOR, 6) ) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x401d #define USB_VID 0xCafe #define USB_BCD 0x0200 diff --git a/examples/device/video_capture_2ch/src/usb_descriptors.c b/examples/device/video_capture_2ch/src/usb_descriptors.c index 8dc986da6..ad65cc019 100644 --- a/examples/device/video_capture_2ch/src/usb_descriptors.c +++ b/examples/device/video_capture_2ch/src/usb_descriptors.c @@ -27,15 +27,8 @@ #include "tusb.h" #include "usb_descriptors.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] VIDEO | AUDIO | MIDI | HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(AUDIO, 4) | PID_MAP(VIDEO, 5) | PID_MAP(VENDOR, 6) ) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x401e #define USB_VID 0xCafe #define USB_BCD 0x0200 diff --git a/examples/device/webusb_serial/src/usb_descriptors.c b/examples/device/webusb_serial/src/usb_descriptors.c index 527837161..5986bffdf 100644 --- a/examples/device/webusb_serial/src/usb_descriptors.c +++ b/examples/device/webusb_serial/src/usb_descriptors.c @@ -27,15 +27,8 @@ #include "tusb.h" #include "usb_descriptors.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] MIDI | HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(VENDOR, 4) ) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x401f //--------------------------------------------------------------------+ // Device Descriptors diff --git a/examples/dual/dynamic_switch/src/usb_descriptors.c b/examples/dual/dynamic_switch/src/usb_descriptors.c index ef6d795b7..c6d80e2ab 100644 --- a/examples/dual/dynamic_switch/src/usb_descriptors.c +++ b/examples/dual/dynamic_switch/src/usb_descriptors.c @@ -26,15 +26,8 @@ #include "bsp/board_api.h" #include "tusb.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] AUDIO | MIDI | HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(VENDOR, 4)) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x4020 #define USB_VID 0xCafe #define USB_BCD 0x0200 diff --git a/examples/dual/host_hid_to_device_cdc/src/usb_descriptors.c b/examples/dual/host_hid_to_device_cdc/src/usb_descriptors.c index 3efa30e20..3dc32e3f4 100644 --- a/examples/dual/host_hid_to_device_cdc/src/usb_descriptors.c +++ b/examples/dual/host_hid_to_device_cdc/src/usb_descriptors.c @@ -26,15 +26,8 @@ #include "bsp/board_api.h" #include "tusb.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(VENDOR, 4) ) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x4021 #define USB_VID 0xCafe #define USB_BCD 0x0200 diff --git a/examples/dual/host_info_to_device_cdc/src/usb_descriptors.c b/examples/dual/host_info_to_device_cdc/src/usb_descriptors.c index 3efa30e20..4fa3bcbc7 100644 --- a/examples/dual/host_info_to_device_cdc/src/usb_descriptors.c +++ b/examples/dual/host_info_to_device_cdc/src/usb_descriptors.c @@ -26,15 +26,8 @@ #include "bsp/board_api.h" #include "tusb.h" -/* A combination of interfaces must have a unique product id, since PC will save device driver after the first plug. - * Same VID/PID with different interface e.g MSC (first), then CDC (later) will possibly cause system error on PC. - * - * Auto ProductID layout's Bitmap: - * [MSB] HID | MSC | CDC [LSB] - */ -#define PID_MAP(itf, n) ((CFG_TUD_##itf) ? (1 << (n)) : 0) -#define USB_PID (0x4000 | PID_MAP(CDC, 0) | PID_MAP(MSC, 1) | PID_MAP(HID, 2) | \ - PID_MAP(MIDI, 3) | PID_MAP(VENDOR, 4) ) +// Unique PID per example: guarantees re-enumeration on re-flash and a fresh host driver match. +#define USB_PID 0x4022 #define USB_VID 0xCafe #define USB_BCD 0x0200 diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py index 862d2b5bd..5f4cef7a6 100755 --- a/test/hil/hil_test.py +++ b/test/hil/hil_test.py @@ -53,14 +53,46 @@ import serial import subprocess import json import glob -from multiprocessing import Pool, Lock +import multiprocessing from multiprocessing import TimeoutError as MpTimeoutError + +# Raw Lock/Semaphore objects passed via Pool initargs are inheritable only under the fork +# start method (spawn/forkserver pickle them and fail at Pool creation) — pin it so a +# future interpreter default change cannot break the run at startup. +_mp = multiprocessing.get_context('fork') +Pool, Lock, Semaphore, Manager = _mp.Pool, _mp.Lock, _mp.Semaphore, _mp.Manager import hashlib import ctypes from pymtp import MTP import string -ENUM_TIMEOUT = 15 +# Enumeration wait budget. The first attempt gets ENUM_TIMEOUT; retry attempts get the +# shorter ENUM_TIMEOUT_RETRY - the board was just re-flashed again, and a device that is +# going to enumerate shows up within a few seconds, so a failing test costs ~3-5x a +# passing one instead of 10-30x. Per-attempt value is set by test_example(); each pool +# worker is its own process, so a module global is safe. +ENUM_TIMEOUT = 8 +ENUM_TIMEOUT_RETRY = 4 +_enum_timeout = ENUM_TIMEOUT + + +def enum_timeout_s() -> int: + """Enumeration wait budget for the current test attempt.""" + return _enum_timeout + + +def wait_until(predicate, step: float = 1.0): + """Poll predicate under the per-attempt enum budget. Deadline-based so a slow predicate + body (subprocess, libmtp scan) counts against the budget. Returns the first truthy + predicate value, or None on timeout.""" + deadline = time.monotonic() + enum_timeout_s() + while True: + r = predicate() + if r: + return r + if time.monotonic() >= deadline: + return None + time.sleep(step) STATUS_OK = "\033[32mOK\033[0m" STATUS_FAILED = "\033[31mFailed\033[0m" @@ -85,13 +117,34 @@ board_test = {} build_dir = 'cmake-build' skip_flash = False print_lock = None -usbtest_lock = None # serializes the usbtest batteries across the board worker pool - - -def init_worker(lock, ut_lock): - global print_lock, usbtest_lock +shuffle_seed = None # per-run seed for the per-board test-order shuffle (HIL_SHUFFLE_SEED to replay) + +# Per-host-controller concurrency (see controller_of/ctrl_slot below): a usbtest battery +# saturates its DUT's host controller, so batteries and flashes are budgeted per controller. +# NOTE: a Renesas uPD720201 host card must run its latest firmware (>= 2.0.2.6; RAM-uploaded, +# so it must be re-loaded every power cycle) - its ROM firmware dies under battery + +# flash/re-enumeration churn, and usbtest.py refuses the unlink-stress cases on old firmware. +# Widths profiled 2026-07-13/14 on fw 2.0.2.6 (8/1 through 12/8): wall time falls +# 22.2/14.3/12.5/10.8 min at usbtest width 1/2/3/4 and plateaus there; flash width beyond 8 +# buys nothing and only amplifies flasher-hub contention; the first battery case failures +# (bandwidth stretch on shared leaf-hub uplinks) appear at 12/8. Hence the 8/4 defaults. +FLASH_PARALLEL = max(1, int(os.getenv('HIL_FLASH_PARALLEL', '8'))) +USBTEST_PARALLEL = max(1, int(os.getenv('HIL_USBTEST_PARALLEL', '4'))) +CTRL_SLOTS = 12 # lock slots; controllers are assigned to slots on first sight +usbtest_sems = None # CTRL_SLOTS semaphores: up to USBTEST_PARALLEL batteries per controller +flash_sems = None # CTRL_SLOTS semaphores(FLASH_PARALLEL): flash permits per controller +ctrl_map = None # shared dict: 'pci:' -> slot, 'uid:' -> pci addr cache +ctrl_meta = None # guards slot assignment in ctrl_map + + +def init_worker(lock, seed, b_mutexes, f_sems, cmap, cmeta): + global print_lock, shuffle_seed, usbtest_sems, flash_sems, ctrl_map, ctrl_meta print_lock = lock - usbtest_lock = ut_lock + shuffle_seed = seed + usbtest_sems = b_mutexes + flash_sems = f_sems + ctrl_map = cmap + ctrl_meta = cmeta def log_line(msg: str) -> None: @@ -103,6 +156,95 @@ def log_line(msg: str) -> None: print(msg, file=out, flush=True) +# ------------------------------------------------------------- +# Per-controller scheduling +# ------------------------------------------------------------- +def controller_of(uid: str): + """Resolve a DUT uid to its root host controller's PCI address, or None if the device + is not enumerated (e.g. parked in board_test firmware with USB off). Successful + resolutions are cached — cabling does not change mid-run. Dual-port parts (e.g. + CH32V307 usbhs/usbfs variants) share one uid and one cache entry: budgeting is only + exact when both ports sit on the same controller (true on this rig).""" + if ctrl_map is None: + return None + cached = ctrl_map.get(f'uid:{uid}') + if cached: + return cached + for f in glob.glob('/sys/bus/usb/devices/*/serial'): + d = os.path.dirname(f) + try: + if open(f).read().strip().lower() != uid.lower(): + continue + bus = int(open(os.path.join(d, 'busnum')).read()) + root = os.path.realpath(f'/sys/bus/usb/devices/usb{bus}') + m = re.findall(r'[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]', root) + if m: + ctrl_map[f'uid:{uid}'] = m[-1] + return m[-1] + except (OSError, ValueError): + continue + return None + + +def ctrl_slot(pci: str) -> int: + """Map a controller PCI address to a lock slot (assigned on first sight).""" + key = f'pci:{pci}' + with ctrl_meta: + slot = ctrl_map.get(key) + if slot is None: + slot = ctrl_map.get('nslots', 0) + if slot >= CTRL_SLOTS: + slot = 0 # more controllers than slots: overflow shares slot 0 (safe, over-serialized) + else: + ctrl_map['nslots'] = slot + 1 + ctrl_map[key] = slot + return slot + + +class ctrl_permit: + """Context manager: one permit from `sems` on the board's controller slot. If the + controller is unknown, fail closed: take one permit from EVERY slot, in order, so the + operation respects the budget wherever it might land. `warn_unknown` logs that fallback + (used by usbtest, where the device is expected to be enumerated by the caller).""" + def __init__(self, sems, uid: str, warn_unknown: bool = False): + self.sems = sems + self.slots = None + if sems is None: + return + pci = controller_of(uid) + if pci is None and warn_unknown: + log_line(f'warning: cannot resolve {uid} to a host controller; ' + 'taking a permit on every slot (over-serialized)') + self.slots = [ctrl_slot(pci)] if pci else list(range(CTRL_SLOTS)) + + def __enter__(self): + if self.slots: + taken = [] + try: + for s in self.slots: + self.sems[s].acquire() + taken.append(s) + except BaseException: + for s in reversed(taken): + self.sems[s].release() + raise + return self + + def __exit__(self, *exc): + if self.slots: + for s in reversed(self.slots): + self.sems[s].release() + return False + + +def flash_permit(uid: str) -> ctrl_permit: + return ctrl_permit(flash_sems, uid) + + +def usbtest_permit(uid: str) -> ctrl_permit: + return ctrl_permit(usbtest_sems, uid, warn_unknown=True) + + def compact_output(raw: str) -> str: if not raw: return '' @@ -239,7 +381,7 @@ def get_alsa_capture_dev(id): def open_serial_dev(port: str): - timeout = ENUM_TIMEOUT + timeout = enum_timeout_s() ser = None while timeout > 0: if os.path.exists(port): @@ -274,27 +416,31 @@ def read_disk_file(uid: str, lun: int, fname: str) -> bytes: # Reads a file from a FAT volume on a block device without mounting it. # Requires mtools: `apt install mtools` (no pip dependency). dev = get_disk_dev(uid, 'TinyUSB', lun) - timeout = ENUM_TIMEOUT last_err = None - while timeout > 0: - if os.path.exists(dev): - try: - data = subprocess.check_output( - ['mtype', '-i', dev, f'::/{fname}'], stderr=subprocess.PIPE) - assert data, f'Cannot read file {fname} from {dev}' - return data - except subprocess.CalledProcessError as e: - last_err = e.stderr.decode(errors='replace').strip() - time.sleep(1) - timeout -= 1 - raise AssertionError(f'mtype failed on {dev}: {last_err}' if last_err else f'Storage {dev} not existed') + def try_read(): + nonlocal last_err + if not os.path.exists(dev): + return None + try: + data = subprocess.check_output( + ['mtype', '-i', dev, f'::/{fname}'], stderr=subprocess.PIPE) + assert data, f'Cannot read file {fname} from {dev}' + return data + except subprocess.CalledProcessError as e: + last_err = e.stderr.decode(errors='replace').strip() + return None + + data = wait_until(try_read) + if data is None: + raise AssertionError(f'mtype failed on {dev}: {last_err}' if last_err else f'Storage {dev} not existed') + return data def open_mtp_dev(uid): mtp = MTP() - timeout = ENUM_TIMEOUT - while timeout > 0: + + def try_open(): # unmount gio/gvfs MTP mount which blocks libmtp from accessing the device subprocess.run(f"gio mount -u mtp://TinyUsb_TinyUsb_Device_{uid}/", shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) @@ -305,9 +451,9 @@ def open_mtp_dev(uid): if sn == uid: return mtp mtp.disconnect() - time.sleep(1) - timeout -= 1 - return None + return None + + return wait_until(try_open) def get_printer_dev(id: str, vendor_str, product_str, ifnum: int): @@ -326,14 +472,13 @@ def get_printer_dev(id: str, vendor_str, product_str, ifnum: int): def open_printer_dev(id: str, vendor_str, product_str, ifnum: int) -> str: """Wait for printer device to enumerate and return its path""" - timeout = ENUM_TIMEOUT - while timeout > 0: + def try_find(): lp_dev = get_printer_dev(id, vendor_str, product_str, ifnum) - if lp_dev and os.path.exists(lp_dev): - return lp_dev - time.sleep(1) - timeout -= 1 - assert False, f'Printer device not found for {id} if{ifnum:02d}' + return lp_dev if lp_dev and os.path.exists(lp_dev) else None + + lp_dev = wait_until(try_find) + assert lp_dev, f'Printer device not found for {id} if{ifnum:02d}' + return lp_dev # ------------------------------------------------------------- @@ -559,7 +704,7 @@ def test_dual_host_info_to_device_cdc(board): # read until all expected devices are enumerated data = b'' - timeout = ENUM_TIMEOUT + timeout = enum_timeout_s() while timeout > 0: new_data = ser.read(ser.in_waiting or 1) if new_data: @@ -611,7 +756,7 @@ def test_host_device_info(board): # read until all expected devices are enumerated data = b'' - timeout = ENUM_TIMEOUT + timeout = enum_timeout_s() while timeout > 0: new_data = ser.read(ser.in_waiting or 1) if new_data: @@ -690,7 +835,7 @@ def test_host_cdc_msc_hid(board): # Wait for all expected mount messages data = b'' - timeout = ENUM_TIMEOUT + timeout = enum_timeout_s() wait_cdc = len(cdc_devs) > 0 wait_msc = len(msc_devs) > 0 while timeout > 0: @@ -783,7 +928,7 @@ def test_host_msc_file_explorer(board): # Wait for MSC mount (Disk Size message) data = b'' - timeout = ENUM_TIMEOUT + timeout = enum_timeout_s() while timeout > 0: new_data = ser.read(ser.in_waiting or 1) if new_data: @@ -848,6 +993,7 @@ def test_host_msc_file_explorer(board): break ser.close() + assert speed is not None, 'MSC read produced no speed report (dd stalled or failed)' return speed @@ -947,7 +1093,7 @@ def test_device_cdc_msc_throughput(board): # Wait for MSC disk enumeration dev = get_disk_dev(uid, 'TinyUSB', 0) - timeout = ENUM_TIMEOUT + timeout = enum_timeout_s() while timeout > 0: if os.path.exists(dev): break @@ -956,7 +1102,7 @@ def test_device_cdc_msc_throughput(board): # Wait for CDC tty enumeration tty = get_serial_dev(uid, 'TinyUSB', 'Throughput', 0) - timeout = ENUM_TIMEOUT + timeout = enum_timeout_s() while timeout > 0: if os.path.exists(tty): break @@ -967,7 +1113,7 @@ def test_device_cdc_msc_throughput(board): is_fs = False for f in glob.glob('/sys/bus/usb/devices/*/serial'): try: - if open(f).read().strip() == uid: + if open(f).read().strip().lower() == uid.lower(): is_fs = (open(os.path.join(os.path.dirname(f), 'speed')).read().strip() == '12') break except (OSError, ValueError): @@ -1013,17 +1159,19 @@ def test_device_cdc_msc_throughput(board): def test_device_dfu(board): uid = board['uid'] - # Wait device enum - timeout = ENUM_TIMEOUT - while timeout > 0: + # Wait device enum. Deadline-based: dfu-util -l itself takes ~1 s per call, which a + # per-iteration countdown would not charge against the budget. + deadline = time.monotonic() + enum_timeout_s() + found = False + while time.monotonic() < deadline: ret = run_cmd(f'dfu-util -l') stdout = cmd_stdout_text(ret.stdout) - if f'serial="{uid}"' in stdout and 'Found DFU: [cafe:4000]' in stdout: + if f'serial="{uid}"' in stdout and 'Found DFU: [cafe:400b]' in stdout: + found = True break time.sleep(1) - timeout = timeout - 1 - assert timeout > 0, 'Device not available' + assert found, 'Device not available' f_dfu0 = f'dfu0_{uid}' f_dfu1 = f'dfu1_{uid}' @@ -1053,17 +1201,18 @@ def test_device_dfu(board): def test_device_dfu_runtime(board): uid = board['uid'] - # Wait device enum - timeout = ENUM_TIMEOUT - while timeout > 0: + # Wait device enum (deadline-based, see test_device_dfu) + deadline = time.monotonic() + enum_timeout_s() + found = False + while time.monotonic() < deadline: ret = run_cmd(f'dfu-util -l') stdout = cmd_stdout_text(ret.stdout) - if f'serial="{uid}"' in stdout and 'Found Runtime: [cafe:4000]' in stdout: + if f'serial="{uid}"' in stdout and 'Found Runtime: [cafe:400c]' in stdout: + found = True break time.sleep(1) - timeout = timeout - 1 - assert timeout > 0, 'Device not available' + assert found, 'Device not available' def test_device_hid_boot_interface(board): @@ -1072,7 +1221,7 @@ def test_device_hid_boot_interface(board): mouse1 = get_hid_dev(uid, 'TinyUSB', 'TinyUSB_Device', 'if01-event-mouse') mouse2 = get_hid_dev(uid, 'TinyUSB', 'TinyUSB_Device', 'if01-mouse') # Wait device enum - timeout = ENUM_TIMEOUT + timeout = enum_timeout_s() while timeout > 0: if os.path.exists(kbd) and os.path.exists(mouse1) and os.path.exists(mouse2): break @@ -1270,9 +1419,9 @@ def test_device_net_lwip_webserver(board): # Wait for the host to get an IPv4 address in the device's subnet (DHCP served by the device). # USB enum + DHCP serve can take longer on the CI HIL hardware than on local — give it 30s. iface_timeout = 30 - deadline = time.time() + iface_timeout + deadline = time.monotonic() + iface_timeout host_ip = None - while time.time() < deadline: + while time.monotonic() < deadline: ret = subprocess.run(['ip', '-o', '-4', 'addr', 'show', iface], capture_output=True, text=True, timeout=2) m = re.search(r'inet (192\.168\.7\.\d+)/', ret.stdout) if ret.returncode == 0 else None @@ -1284,9 +1433,9 @@ def test_device_net_lwip_webserver(board): # Poll the iperf TCP port until the device is accepting. The net stack comes up a bit # after DHCP completes; iperf server binding isn't instantaneous after reflash. - deadline = time.time() + ENUM_TIMEOUT + deadline = time.monotonic() + enum_timeout_s() last_err = None - while time.time() < deadline: + while time.monotonic() < deadline: try: with socket.create_connection((device_ip, iperf_port), timeout=1): last_err = None @@ -1294,7 +1443,7 @@ def test_device_net_lwip_webserver(board): except OSError as e: last_err = e time.sleep(0.3) - assert last_err is None, f'iperf TCP {device_ip}:{iperf_port} not accepting within {ENUM_TIMEOUT}s: {last_err}' + assert last_err is None, f'iperf TCP {device_ip}:{iperf_port} not accepting within {enum_timeout_s()}s: {last_err}' # Throughput: 5-second iperf2 TCP test, CSV output for stable parsing. # iperf2 CSV final summary line: timestamp,src_ip,src_port,dst_ip,dst_port,id,interval,bytes,bps @@ -1334,7 +1483,7 @@ def test_device_midi_test(board): uid = board['uid'] # Find MIDI device via /dev/snd/by-id using board UID - timeout = ENUM_TIMEOUT + timeout = enum_timeout_s() midi_port = None while timeout > 0: pattern = f'/dev/snd/by-id/usb-*_{uid}-*' @@ -1356,8 +1505,8 @@ def test_device_midi_test(board): with open(midi_port, 'rb') as f: notes = [] # Read for up to 3 seconds to capture a few notes (286ms interval) - end_time = time.time() + 3 - while time.time() < end_time: + end_time = time.monotonic() + 3 + while time.monotonic() < end_time: ready, _, _ = select.select([f], [], [], 0.5) if ready: data = f.read(64) @@ -1393,7 +1542,7 @@ def test_device_audio_test_freertos(board): return 'skipped' pcm = None - timeout = ENUM_TIMEOUT + timeout = enum_timeout_s() while timeout > 0: pcm = get_alsa_capture_dev(uid) if pcm: @@ -1460,7 +1609,7 @@ def test_device_hid_generic_inout(board): import hid # cython-hidapi (pip: hidapi, apt: python3-hid) # Find HID device by UID (VID=0xCafe) - timeout = ENUM_TIMEOUT + timeout = enum_timeout_s() dev = None while timeout > 0: for d in hid.enumerate(0xCafe): @@ -1511,25 +1660,26 @@ def test_device_usbtest(board): pass return False - end = time.time() + ENUM_TIMEOUT - while time.time() < end and not usbtest_enumerated(): + end = time.monotonic() + enum_timeout_s() + while time.monotonic() < end and not usbtest_enumerated(): time.sleep(0.2) + # fail before usbtest_permit: an absent device would otherwise queue on the battery + # mutex for minutes behind real batteries just to have usbtest.py report "no device" + assert usbtest_enumerated(), f'no cafe:4010 device with serial {uid}' # settle: right after flashing the enumeration can bounce once (and on dual-port parts like # CH32V307 the other port's stale usbtest node — same serial and PID — lingers a moment); # running testusb into that gap sees the device drop mid-case time.sleep(3) - # --keep-binding leaves the usbtest dynamic id registered: the cleanup path unbinds every - # claimed interface, which has wedged the host xHCI (usb_hcd_alloc_bandwidth) on this rig. - # Boards test in a worker pool, but the batteries must run one at a time: each one saturates - # the host controller (bulk perf, iso streams, unlink storms), and several at once have - # hard-frozen the CI rig (fatal PCIe error on its VFIO-passed xHCI). + # --keep-binding is required for concurrent batteries: usbtest.py's cleanup unbinds + # EVERY usbtest-bound interface (releasing stale same-PID grabs), which would kill a + # peer battery mid-run under USBTEST_PARALLEL > 1; the unbind path has also wedged a + # host xHCI (usb_hcd_alloc_bandwidth) on this rig. Leaving bindings is harmless with + # unique example PIDs - the next example re-enumerates under a different PID and binds + # its normal driver. usbtest_permit budgets USBTEST_PARALLEL batteries per controller. script = Path(__file__).resolve().parent / 'usbtest.py' cmd = f'python3 "{script}" --serial "{uid}" --json --keep-binding --timeout 60' - if usbtest_lock is not None: - with usbtest_lock: - r = run_cmd(cmd, timeout=200) - else: + with usbtest_permit(uid): r = run_cmd(cmd, timeout=200) out = cmd_stdout_text(r.stdout) brace = out.find('{') @@ -1541,6 +1691,8 @@ def test_device_usbtest(board): skipped = int(data.get('skipped', 0)) # host-controller limitation (see usbtest.py host_broken_cases) total = passed + failed + if total == 0 and skipped > 0: + return 'skipped' # every case host-skipped: a skip, not a 0/0 failure if failed == 0 and total > 0: return f'{REPORT_CELL["pass"]} {passed}/{total}' + (f' +{skipped}skip' if skipped else '') bad = [c.get('num') for c in data.get('cases', []) if c.get('status') not in ('PASS', 'SKIP')] @@ -1552,12 +1704,11 @@ def test_device_usbtest(board): # Main # ------------------------------------------------------------- # device tests -# note don't test 2 examples with cdc or 2 msc next to each other device_tests = [ - # Order matters: cdc_msc and cdc_msc_throughput share the same VID:PID (cafe:4003), so keep a - # differently-PID'd example (dfu, cafe:4000) between them. Boards whose CPU-reset does not drop - # D+ (e.g. WCH CH58x via openocd) only re-enumerate when the PID changes; back-to-back same-PID - # firmware would otherwise leave the host on the previous example's cached descriptors. + # The per-board run order is shuffled (see test_board). Every example carries a unique + # hardcoded idProduct (see its usb_descriptors.c), so any two different examples always + # re-enumerate back-to-back — even on boards whose CPU-reset does not drop D+ (e.g. WCH + # CH58x via openocd), which only re-enumerate when the PID changes. 'device/cdc_dual_ports', 'device/cdc_msc', 'device/dfu', @@ -1629,15 +1780,18 @@ def test_example(board: Board, variant: str, example: str) -> tuple[int, str, st # flash firmware (unless --skip-flash), then run the test. Both may fail randomly, # retry a few times. + global _enum_timeout start_s = time.time() flash_ok = True last_err = '' last_detail = '' for i in range(max_retry): + _enum_timeout = ENUM_TIMEOUT if i == 0 else ENUM_TIMEOUT_RETRY attempt_out = io.StringIO() with redirect_stdout(attempt_out): if not skip_flash: - ret = globals()[f'flash_{board["flasher"]["name"].lower()}'](board, str(fw_name)) + with flash_permit(board['uid']): + ret = globals()[f'flash_{board["flasher"]["name"].lower()}'](board, str(fw_name)) flash_ok = (ret.returncode == 0) if flash_ok: try: @@ -1771,10 +1925,24 @@ def test_board(board: Board) -> tuple[str, int, list[str], list]: rows = [] # list of (row_label, {example: status}) — one row per build variant variants = board.get('variant') or [{'name': name, 'flags': ''}] + prev_last = None # last test of the previous variant: the variant boundary is an adjacency too for v in variants: vname = v['name'] + # Shuffle each (board, variant)'s run order — de-synchronizes the worker pool so + # usbtest batteries and flash churn spread across the timeline instead of convoying, + # and surfaces order-dependent bugs. Seeded for replay (HIL_SHUFFLE_SEED, logged by + # main). Unique per-example PIDs make any two different examples re-enumerate; only + # the variant boundary can repeat the same example (same PID) — swap it away. + run_list = list(test_list) + if shuffle_seed is not None and len(run_list) > 1: + random.Random(f'{shuffle_seed}:{name}:{vname}').shuffle(run_list) + if run_list[0] == prev_last: + run_list[0], run_list[-1] = run_list[-1], run_list[0] + log_line(f'{vname:40} test order: {", ".join(t.rsplit("/", 1)[-1] for t in run_list)}') + if run_list: + prev_last = run_list[-1] cells = {} - for test in test_list: + for test in run_list: ec, status, metric = test_example(board, vname, test) err_count += ec cells[test] = metric if metric else status @@ -1797,16 +1965,21 @@ REPORT_JSON = 'hil_report.json' def render_matrix(rows_all: list) -> str: """Render rows (list of (row_label, {example: status})) as an aligned markdown matrix: columns = tests (bare names) centered, boards left-aligned.""" - canonical = device_tests + dual_tests + host_test seen = set() for _, cells in rows_all: seen.update(cells) if not seen: return 'No tests were run.' - # columns: canonical order first, then any extras (e.g. from -t) alphabetically - columns = [t for t in canonical if t in seen] - columns += [t for t in sorted(seen) if t not in canonical] + # metric-bearing columns pinned first (usbtest score, throughput, explorer read speed), + # the rest alphabetical by bare test name: stable regardless of the (shuffled) execution order + pinned = ['usbtest', 'cdc_msc_throughput', 'msc_file_explorer', 'msc_file_explorer_freertos'] + + def col_key(t): + name = t.rsplit('/', 1)[-1] + return (pinned.index(name) if name in pinned else len(pinned), name, t) + + columns = sorted(seen, key=col_key) headers = [c.rsplit('/', 1)[-1] for c in columns] # bare example name def cell(cells, col): @@ -1953,7 +2126,16 @@ def main() -> None: for f in (REPORT_JSON, REPORT_MD): (report_dir / f).unlink(missing_ok=True) - with Pool(processes=os.cpu_count() or 1, initializer=init_worker, initargs=(Lock(), Lock())) as pool: + seed = os.getenv('HIL_SHUFFLE_SEED') or str(int(time.time())) + log_line(f'test-order shuffle seed: {seed} (HIL_SHUFFLE_SEED={seed} to replay); ' + f'flash/usbtest parallel per controller: {FLASH_PARALLEL}/{USBTEST_PARALLEL}; ' + f'enum timeout first/retry: {ENUM_TIMEOUT}/{ENUM_TIMEOUT_RETRY}s') + mgr = Manager() + initargs = (Lock(), seed, + [Semaphore(USBTEST_PARALLEL) for _ in range(CTRL_SLOTS)], + [Semaphore(FLASH_PARALLEL) for _ in range(CTRL_SLOTS)], + mgr.dict(), Lock()) + with Pool(processes=os.cpu_count() or 1, initializer=init_worker, initargs=initargs) as pool: async_ret = pool.map_async(test_board, config_boards) try: mret = async_ret.get(timeout=POOL_TIMEOUT) diff --git a/test/hil/tinyusb.json b/test/hil/tinyusb.json index be39c9eb5..8f121b6f8 100644 --- a/test/hil/tinyusb.json +++ b/test/hil/tinyusb.json @@ -22,6 +22,7 @@ { "name": "espressif_p4_function_ev-DMA", "flags": "-DCFG_TUD_DWC2_DMA_ENABLE=1 -DCFG_TUH_DWC2_DMA_ENABLE=1" } ], "tests": { + "comment": "only IDF/FreeRTOS examples are part of the espressif fleet build; device/usbtest builds under IDF but is not built/flashed by the fleet, so it is not listed", "only": [ "device/cdc_msc_freertos", "device/hid_composite_freertos", @@ -152,6 +153,8 @@ "name": "mimxrt1015_evk", "uid": "DC28F865D2111D228D00B0543A70463C", "tests": { + "skip": ["device/usbtest"], + "comment": "this board's HS battery killed the uPD720201 twice (2026-07-11 on ROM fw, 2026-07-13 case 27 on fw 2.0.2.6 - stop-endpoint timeout, HC died); mimxrt1064/ch32v307 batteries pass, so it is board-specific - keep skipped", "device": true, "host": false, "dual": false @@ -166,19 +169,20 @@ "name": "mimxrt1064_evk", "uid": "BAE96FB95AFA6DBB8F00005002001200", "tests": { + "skip": ["host/cdc_msc_hid"], + "comment-cdc-echo": "CH9102+Lexar bundle (moved here from stm32f723disco) mounts fine but echo returns nothing - TX-RX loopback jumper likely lost in the move; re-check wiring then re-enable", "device": true, "host": true, "dual": true, "dev_attached": [ { - "vid_pid": "10c4_ea60", - "serial": "0001", - "is_cdc": true, - "comment": "cp2102" + "vid_pid": "1a86_55d4", + "serial": "52D2003414", + "is_cdc": true }, { "vid_pid": "21c4_0cc7", - "serial": "900058874D871F66", + "serial": "90005893730A1A63", "is_msc": true, "block_size": 512, "block_count": 60620800, @@ -230,6 +234,8 @@ "device": true, "host": true, "dual": true, + "skip": ["host/cdc_msc_hid", "host/device_info", "host/msc_file_explorer", "host/msc_file_explorer_freertos", "dual/host_info_to_device_cdc"], + "comment-skip": "PIO-USB host port enumerates nothing since the board moves (CH340+UDisk bundle unplugged or unpowered) - re-attach the bundle then drop these skips", "dev_attached": [ { "vid_pid": "1a86_7523", @@ -379,13 +385,14 @@ "dual": false, "dev_attached": [ { - "vid_pid": "1a86_55d4", - "serial": "52D2003414", - "is_cdc": true + "vid_pid": "10c4_ea60", + "serial": "0001", + "is_cdc": true, + "comment": "cp2102" }, { "vid_pid": "21c4_0cc7", - "serial": "90005893730A1A63", + "serial": "900058874D871F66", "is_msc": true, "block_size": 512, "block_count": 60620800, @@ -512,14 +519,11 @@ "uid": "BC5DA47360D0", "args": "" } - } - ], - "boards-skip": [ + }, { "name": "ch582m_evt", "uid": "D443627B5450", "toolchain": "riscv-gcc", - "comment": "unplugged: fixture (board + WCH-Link) failed to re-enumerate after the 2026-07-06 rig reboot; replug to re-enable", "tests": { "device": true, "host": false, @@ -531,19 +535,70 @@ "args": "" } }, + { + "name": "nrf5340dk", + "uid": "78E60E166B5F88BE", + "tests": { + "device": true, + "host": false, + "dual": false, + "skip": ["device/cdc_msc_freertos", "device/audio_test_freertos"], + "comment": "board new to HIL: FreeRTOS examples hardfault (UFSR=INVPC) at first task launch on the CM33_NTZ port - pre-existing upstream issue, non-FreeRTOS examples and usbtest pass; fix separately" + }, + "flasher": { + "name": "jlink", + "uid": "001050076405", + "args": "-device NRF5340_XXAA_APP" + } + }, { "name": "nrf54lm20dk", "uid": "899C3DE5B0F4D5CA", "tests": { "device": true, "host": false, - "dual": false + "dual": false, + "skip": ["device/audio_test_freertos"], + "comment": "board new to HIL: audio_test_freertos never reaches dcd_init (FreeRTOS itself runs; cdc_msc_freertos and usbtest pass) - example-level issue on nRF54L, fix separately" }, "flasher": { "name": "jlink", "uid": "1051856258", "args": "-device NRF54LM20A_M33" } + } + ], + "boards-skip": [ + { + "name": "ra6m5_ek", + "uid": "8419032D32363657364EF4622D294B4E", + "tests": { + "device": true, + "host": false, + "dual": false, + "skip": ["device/cdc_msc_throughput", "device/msc_dual_lun"], + "comment": "MSC writes wedge the uPD720201 host (URBs queued, zero wire activity, bus-15 ctrl xfers time out until the device's URBs are killed); reproduced identically with master firmware - device side armed+BUF and exonerated. MSC reads and usbtest bulk (15.8 MB/s) are fine" + }, + "flasher": { + "name": "jlink", + "uid": "000831915224", + "args": "-device R7FA6M5BH" + } + }, + { + "name": "ra8m1_ek", + "uid": "797D142D36345030364E1737922E4B4E", + "comment": "USBHS bring-up pending (HS chirp completes digitally but terminations never switch; FS-forced build passed usbtest 30/30). Parked until fixed", + "tests": { + "device": true, + "host": false, + "dual": false + }, + "flasher": { + "name": "jlink", + "uid": "001083115236", + "args": "-device R7FA8M1AH" + } }, { "name": "stm32f769disco", diff --git a/test/hil/usbtest.py b/test/hil/usbtest.py index 3a174a510..9aec8ac0a 100755 --- a/test/hil/usbtest.py +++ b/test/hil/usbtest.py @@ -104,7 +104,15 @@ def sudo(cmd, **kw): def sysfs_write(path, data, check=True): - r = sudo(['tee', str(path)], input=data) + # A driver-registry write (new_id/remove_id/bind) blocks in D state when a wedged device + # holds its lock (driver_attach walks the bus): fail fast and loud instead of piling up + # unkillable writers and hanging the whole run -- the rig needs USB recovery first. + try: + r = sudo(['tee', str(path)], input=data, timeout=15) + except subprocess.TimeoutExpired: + sys.exit(f'write "{data}" > {path} blocked >15s: USB subsystem is wedged ' + '(a D-state device lock exists). Recover the rig (usb_recover.sh) ' + 'before running batteries.') if check and r.returncode != 0: sys.exit(f'write "{data}" > {path} failed: {r.stderr.strip()}') return r.returncode == 0 @@ -145,7 +153,8 @@ def find_device(serial, first=False): def host_broken_cases(dev): - """Cases the DUT's upstream host controller cannot run: {case: reason}. + """Cases the DUT's upstream host controller cannot run: {case: reason}. Exits the + whole run instead if the host is a uPD720201 on pre-2.0.2.6 firmware (see below). The MosChip MCS9990 (9710:9990) EHCI cannot run interrupt-OUT: its FRINDEX register is buggy silicon (the kernel probes it with "applying MosChip frame-index workaround") and ehci-hcd never keeps the int-OUT QH in the @@ -154,15 +163,58 @@ def host_broken_cases(dev): same board+hub: EHCI FAIL (QH absent from the debugfs periodic schedule the whole hang), OHCI companion PASS, xHCI fine; int-IN unaffected. Skip with a visible SKIP so the battery self-heals once the DUT tree is back on an xHCI.""" - try: - root = Path(f"/sys/bus/usb/devices/usb{int(dev['node'].split('/')[-2])}") - drv = (root / '../driver').resolve().name - pci = (root / '..').resolve() - vid_did = ((pci / 'vendor').read_text().strip(), (pci / 'device').read_text().strip()) - except (OSError, ValueError): - return {} + for attempt in range(3): + try: + root = Path(f"/sys/bus/usb/devices/usb{int(dev['node'].split('/')[-2])}") + drv = (root / '../driver').resolve().name + pci = (root / '..').resolve() + vid_did = ((pci / 'vendor').read_text().strip(), (pci / 'device').read_text().strip()) + break + except (OSError, ValueError): + # transient sysfs error (e.g. racing a re-enumeration): retry so a blip doesn't + # silently run known-broken cases; if the probe truly fails, fail open but say so + if attempt == 2: + print('warning: cannot probe the upstream host controller; ' + 'known-broken-host cases will run instead of being skipped', file=sys.stderr) + return {} + time.sleep(1) if drv.startswith('ehci') and vid_did == ('0x9710', '0x9990'): - return {25: 'host EHCI (MosChip MCS9990) loses interrupt-OUT completions'} + return { + 25: 'host EHCI (MosChip MCS9990) loses interrupt-OUT completions', + # Unlinking an in-progress read intermittently completes it as a short transfer + # (EREMOTEIO) instead of -ECONNRESET; device-side exonerated by TX counters (only + # full-mps loads, no ZLP). Passes on xHCI. Some boards dodge it by timing. + 11: 'host EHCI (MosChip MCS9990) completes unlinked reads as short (EREMOTEIO)', + } + if drv.startswith('xhci') and vid_did in (('0x1912', '0x0014'), ('0x1912', '0x0015')): + # The Renesas uPD720201/uPD720202 must run its latest firmware (>= 2.0.2.6, + # K2026090.mem; RAM-uploaded, so it reverts to ROM on every power cycle unless + # re-loaded). On the ROM firmware its command ring intermittently dies under unlink + # stress: a Configure Endpoint command stops completing, the hub worker deadlocks + # holding the device lock (needs a host power cycle). Three separate boards killed + # it this way (ch32v307 2026-07-10; ra6m5 test 24, mimxrt1015 2026-07-11). Both + # parts expose the FW version register at PCI config offset 0x6c. NOTE this check + # is necessary, not sufficient: board-specific batteries have killed the controller + # on current firmware too (mimxrt1015, stop-endpoint timeout) - those are handled + # by per-board skips in the rig config. + fw = None + try: + r = sudo(['setpci', '-s', pci.name, '0x6c.l'], capture_output=True, text=True) + if r.returncode == 0: + fw = int(r.stdout.strip(), 16) + except (OSError, ValueError): + pass + if fw is None: + sys.exit(f'REFUSING to run: cannot read host xHCI Renesas ({pci.name}) firmware ' + 'version (setpci missing or not permitted) - usbtest requires verified ' + 'firmware >= 0x00202609 (2.0.2.6); on older firmware the command ring ' + 'dies under unlink stress. Install pciutils / fix sudo, or load the ' + 'firmware and re-check.') + if fw < 0x00202609: + sys.exit(f'REFUSING to run: host xHCI Renesas ({pci.name}) firmware 0x{fw:08x} ' + '< 0x00202609 (2.0.2.6) - its command ring dies under usbtest unlink ' + 'stress. Load the latest firmware (K2026090.mem; it is RAM-uploaded and ' + 'reverts to ROM on every power cycle).') return {} @@ -327,13 +379,16 @@ def main(): if not args.json: print(info) + # probe the upstream controller before touching the device: an unsupported host + # (uPD720201 on pre-2.0.2.6 firmware) exits here, before any bind + broken = host_broken_cases(dev) + results = [] unrecovered_hang = False try: bind_usbtest(dev) set_pattern(0) # tier 1 firmware sources zeros; also required by perf cases 27/28 - broken = host_broken_cases(dev) for num in cases: if num in broken: results.append({'num': num, 'name': CASE_NAMES[num], 'status': 'SKIP', diff --git a/tools/check_example_pids.py b/tools/check_example_pids.py new file mode 100644 index 000000000..d8795af34 --- /dev/null +++ b/tools/check_example_pids.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Check that every example enumerates with a unique USB PID. + +Each example's usb_descriptors.c hardcodes its idProduct (0x40xx). Uniqueness is what +guarantees back-to-back re-enumeration on the HIL rig and a fresh host driver match, and +it is easy to break by hand: a new example copying a neighbour's PID, or an arithmetic +PID (dynamic_configuration derives a second one from USB_PID). This collects every +`#define USB_PID 0x....`, every literal `.idProduct = 0x....`, and every `USB_PID + ` +derivation across examples/, and fails on any duplicate value. +""" + +import re +import sys +from pathlib import Path + +EXAMPLES = Path(__file__).resolve().parents[1] / 'examples' + +RE_DEFINE = re.compile(r'#define\s+USB_PID\s+\(?(0x[0-9a-fA-F]+)\)?') +RE_LITERAL = re.compile(r'\.idProduct\s*=\s*(0x[0-9a-fA-F]+)') +RE_DERIVED = re.compile(r'\.idProduct\s*=\s*USB_PID\s*\+\s*(0x[0-9a-fA-F]+|\d+)') + + +def main() -> int: + pids: dict[int, list[str]] = {} + for f in sorted(EXAMPLES.glob('*/*/src/usb_descriptors.c')): + text = f.read_text(errors='replace') + rel = f.relative_to(EXAMPLES.parent) + base = None + m = RE_DEFINE.search(text) + if m: + base = int(m.group(1), 16) + for m in RE_LITERAL.finditer(text): + pids.setdefault(int(m.group(1), 16), []).append(str(rel)) + for m in RE_DERIVED.finditer(text): + if base is None: + print(f'{rel}: derived idProduct but no USB_PID define', file=sys.stderr) + return 1 + pids.setdefault(base + int(m.group(1), 0), []).append(f'{rel} (USB_PID + {m.group(1)})') + # examples whose descriptor uses .idProduct = USB_PID pick up the define itself + if base is not None and re.search(r'\.idProduct\s*=\s*USB_PID\s*[,;]', text): + pids.setdefault(base, []).append(str(rel)) + + dups = {pid: users for pid, users in pids.items() if len(users) > 1} + for pid, users in sorted(dups.items()): + print(f'duplicate USB PID 0x{pid:04x}:', file=sys.stderr) + for u in users: + print(f' {u}', file=sys.stderr) + if dups: + return 1 + print(f'{len(pids)} unique example USB PIDs') + return 0 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/usb_drivers/tinyusb_win_usbser.inf b/tools/usb_drivers/tinyusb_win_usbser.inf index 659f048ae..e3875a478 100644 --- a/tools/usb_drivers/tinyusb_win_usbser.inf +++ b/tools/usb_drivers/tinyusb_win_usbser.inf @@ -88,11 +88,11 @@ ServiceBinary=%12%\%DRIVERFILENAME%.sys [SourceDisksNames] [DeviceList] -%DESCRIPTION%=DriverInstall, USB\VID_CAFE&PID_4001&MI_00, USB\VID_CAFE&PID_4003&MI_00, USB\VID_CAFE&PID_4005&MI_00, USB\VID_CAFE&PID_4007&MI_00, USB\VID_CAFE&PID_4009&MI_00, USB\VID_CAFE&PID_400b&MI_00, USB\VID_CAFE&PID_400d&MI_00, USB\VID_CAFE&PID_400f&MI_00, USB\VID_CAFE&PID_4011&MI_00, USB\VID_CAFE&PID_4013&MI_00, USB\VID_CAFE&PID_4015&MI_00, USB\VID_CAFE&PID_4017&MI_00, USB\VID_CAFE&PID_4019&MI_00, USB\VID_CAFE&PID_401b&MI_00, USB\VID_CAFE&PID_401d&MI_00, USB\VID_CAFE&PID_401f&MI_00, USB\VID_CAFE&PID_4021&MI_00, USB\VID_CAFE&PID_4023&MI_00, USB\VID_CAFE&PID_4025&MI_00, USB\VID_CAFE&PID_4027&MI_00, USB\VID_CAFE&PID_4029&MI_00, USB\VID_CAFE&PID_402b&MI_00, USB\VID_CAFE&PID_402d&MI_00, USB\VID_CAFE&PID_402f&MI_00, USB\VID_CAFE&PID_4031&MI_00, USB\VID_CAFE&PID_4033&MI_00, USB\VID_CAFE&PID_4035&MI_00, USB\VID_CAFE&PID_4037&MI_00, USB\VID_CAFE&PID_4039&MI_00, USB\VID_CAFE&PID_403b&MI_00, USB\VID_CAFE&PID_403d&MI_00, USB\VID_CAFE&PID_403f&MI_00 +%DESCRIPTION%=DriverInstall, USB\VID_CAFE&PID_4001&MI_00, USB\VID_CAFE&PID_4003&MI_00, USB\VID_CAFE&PID_4005&MI_00, USB\VID_CAFE&PID_4007&MI_00, USB\VID_CAFE&PID_4009&MI_00, USB\VID_CAFE&PID_400b&MI_00, USB\VID_CAFE&PID_400d&MI_00, USB\VID_CAFE&PID_400f&MI_00, USB\VID_CAFE&PID_4011&MI_00, USB\VID_CAFE&PID_4013&MI_00, USB\VID_CAFE&PID_4015&MI_00, USB\VID_CAFE&PID_4017&MI_00, USB\VID_CAFE&PID_4019&MI_00, USB\VID_CAFE&PID_401b&MI_00, USB\VID_CAFE&PID_401d&MI_00, USB\VID_CAFE&PID_401f&MI_00, USB\VID_CAFE&PID_4021&MI_00, USB\VID_CAFE&PID_4023&MI_00, USB\VID_CAFE&PID_4025&MI_00, USB\VID_CAFE&PID_4027&MI_00, USB\VID_CAFE&PID_4029&MI_00, USB\VID_CAFE&PID_402b&MI_00, USB\VID_CAFE&PID_402d&MI_00, USB\VID_CAFE&PID_402f&MI_00, USB\VID_CAFE&PID_4031&MI_00, USB\VID_CAFE&PID_4033&MI_00, USB\VID_CAFE&PID_4035&MI_00, USB\VID_CAFE&PID_4037&MI_00, USB\VID_CAFE&PID_4039&MI_00, USB\VID_CAFE&PID_403b&MI_00, USB\VID_CAFE&PID_403d&MI_00, USB\VID_CAFE&PID_403f&MI_00, USB\VID_CAFE&PID_4006&MI_00, USB\VID_CAFE&PID_4008&MI_00, USB\VID_CAFE&PID_400a&MI_00, USB\VID_CAFE&PID_4020&MI_00, USB\VID_CAFE&PID_4022&MI_00 [DeviceList.NTamd64] -%DESCRIPTION%=DriverInstall, USB\VID_CAFE&PID_4001&MI_00, USB\VID_CAFE&PID_4003&MI_00, USB\VID_CAFE&PID_4005&MI_00, USB\VID_CAFE&PID_4007&MI_00, USB\VID_CAFE&PID_4009&MI_00, USB\VID_CAFE&PID_400b&MI_00, USB\VID_CAFE&PID_400d&MI_00, USB\VID_CAFE&PID_400f&MI_00, USB\VID_CAFE&PID_4011&MI_00, USB\VID_CAFE&PID_4013&MI_00, USB\VID_CAFE&PID_4015&MI_00, USB\VID_CAFE&PID_4017&MI_00, USB\VID_CAFE&PID_4019&MI_00, USB\VID_CAFE&PID_401b&MI_00, USB\VID_CAFE&PID_401d&MI_00, USB\VID_CAFE&PID_401f&MI_00, USB\VID_CAFE&PID_4021&MI_00, USB\VID_CAFE&PID_4023&MI_00, USB\VID_CAFE&PID_4025&MI_00, USB\VID_CAFE&PID_4027&MI_00, USB\VID_CAFE&PID_4029&MI_00, USB\VID_CAFE&PID_402b&MI_00, USB\VID_CAFE&PID_402d&MI_00, USB\VID_CAFE&PID_402f&MI_00, USB\VID_CAFE&PID_4031&MI_00, USB\VID_CAFE&PID_4033&MI_00, USB\VID_CAFE&PID_4035&MI_00, USB\VID_CAFE&PID_4037&MI_00, USB\VID_CAFE&PID_4039&MI_00, USB\VID_CAFE&PID_403b&MI_00, USB\VID_CAFE&PID_403d&MI_00, USB\VID_CAFE&PID_403f&MI_00 +%DESCRIPTION%=DriverInstall, USB\VID_CAFE&PID_4001&MI_00, USB\VID_CAFE&PID_4003&MI_00, USB\VID_CAFE&PID_4005&MI_00, USB\VID_CAFE&PID_4007&MI_00, USB\VID_CAFE&PID_4009&MI_00, USB\VID_CAFE&PID_400b&MI_00, USB\VID_CAFE&PID_400d&MI_00, USB\VID_CAFE&PID_400f&MI_00, USB\VID_CAFE&PID_4011&MI_00, USB\VID_CAFE&PID_4013&MI_00, USB\VID_CAFE&PID_4015&MI_00, USB\VID_CAFE&PID_4017&MI_00, USB\VID_CAFE&PID_4019&MI_00, USB\VID_CAFE&PID_401b&MI_00, USB\VID_CAFE&PID_401d&MI_00, USB\VID_CAFE&PID_401f&MI_00, USB\VID_CAFE&PID_4021&MI_00, USB\VID_CAFE&PID_4023&MI_00, USB\VID_CAFE&PID_4025&MI_00, USB\VID_CAFE&PID_4027&MI_00, USB\VID_CAFE&PID_4029&MI_00, USB\VID_CAFE&PID_402b&MI_00, USB\VID_CAFE&PID_402d&MI_00, USB\VID_CAFE&PID_402f&MI_00, USB\VID_CAFE&PID_4031&MI_00, USB\VID_CAFE&PID_4033&MI_00, USB\VID_CAFE&PID_4035&MI_00, USB\VID_CAFE&PID_4037&MI_00, USB\VID_CAFE&PID_4039&MI_00, USB\VID_CAFE&PID_403b&MI_00, USB\VID_CAFE&PID_403d&MI_00, USB\VID_CAFE&PID_403f&MI_00, USB\VID_CAFE&PID_4006&MI_00, USB\VID_CAFE&PID_4008&MI_00, USB\VID_CAFE&PID_400a&MI_00, USB\VID_CAFE&PID_4020&MI_00, USB\VID_CAFE&PID_4022&MI_00 ;------------------------------------------------------------------------------ ; String Definitions -- cgit v1.3.1 From 59f02a1c4c18d7e43a1bd6aaad4b50e71931c9ff Mon Sep 17 00:00:00 2001 From: hathach Date: Wed, 15 Jul 2026 02:31:14 +0700 Subject: dwc2: fix EP0 OUT dcache invalidate range; run usbtest on espressif s3/p4 and mimxrt1015 edpt_schedule_packets() advanced xfer->buffer past each armed EP0 chunk, so the OUT-complete handler invalidated the cache at the ADVANCED pointer: one line past the received data. The CPU then read stale cached bytes instead of the DMA'd packet, and the misplaced invalidate discarded a dirty line of whatever variable follows the buffer - random neighbor corruption on every control-OUT data stage. Found by usbtest ctrl_out (cases 14/21) on espressif_p4_function_ev with DMA enabled, the first DWC2 target combining buffer DMA with a data cache: usbd control state wedged after the first control write (every later request stalled), and one build layout panicked in the usbd memcpy with a wild pointer. Rework the EP0 chunk bookkeeping so xfer->buffer always points at the un-consumed position: the arm no longer advances it; instead the EP0 re-arm paths advance past each completed (full) chunk, invalidating it first on the OUT side. The final OUT completion invalidates exactly the received bytes of its last chunk, taken from DOEPDMA ("incremented on every AHB transaction", databook 7.1.83 - the same semantics the SETUP path relies on) before dma_setup_prepare() re-targets it. EP0 chunking state (ep0_pending) is now also dropped on bus reset and on a new SETUP, so a stale latched completion can no longer re-arm EP0 DMA from dead state. No behavior change for targets without dcache. While root-causing, the FIFO layout was cross-checked against the DWC2 databook/programming guide v4.20a: the existing GDFIFOCFG programming (EPInfoBaseAddr = otg_dfifo_depth - 2*ep_count, one SPRAM word per endpoint direction for buffer DMA) is conformant and needs no change; the P4 HS instance's reset GDFIFOCFG (0x03800400) merely reflects a scatter/gather-sized EP_LOC_CNT of 128 that buffer DMA does not need. With the fix in place, enable the usbtest battery on the espressif fleet: tools/build.py allowlists device/usbtest (a plain IDF component like board_test/video_capture) and both espressif boards' only-lists gain device/usbtest. Also re-enable device/usbtest on mimxrt1015_evk: its skip predated the dcd_ci_hs stale-ACTIVE-overlay fix (already on this branch), which cured the battery that previously killed the uPD720201 host controller twice (2026-07-11 ROM fw, 2026-07-13 case 27 on fw 2.0.2.6); rig-validated 30/30 three consecutive runs. Validated on rig (all 30/30): espressif_p4_function_ev(-DMA) (was 22/30 under DMA), espressif_s3_devkitm(-DMA), stm32f723disco(-DMA), mimxrt1015_evk; p4/s3 slave-mode unaffected (DMA-only code path); compile-checked stm32h743nucleo +TUD DMA, stm32f407disco, stm32l476disco (device ports currently on the dead hub). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_017TQZrFfU3K4Y198aLsUpBC --- src/portable/synopsys/dwc2/dcd_dwc2.c | 34 ++++++++++++++++++++++++++-------- test/hil/tinyusb.json | 6 +++--- tools/build.py | 1 + 3 files changed, 30 insertions(+), 11 deletions(-) (limited to 'test/hil/tinyusb.json') diff --git a/src/portable/synopsys/dwc2/dcd_dwc2.c b/src/portable/synopsys/dwc2/dcd_dwc2.c index 6c88b4f27..86aa54510 100644 --- a/src/portable/synopsys/dwc2/dcd_dwc2.c +++ b/src/portable/synopsys/dwc2/dcd_dwc2.c @@ -393,10 +393,6 @@ static void edpt_schedule_packets(uint8_t rhport, const uint8_t epnum, const uin } dep->diepdma = (uintptr_t) xfer->buffer; dep->diepctl = depctl.value; // enable endpoint - // Advance buffer pointer for EP0 - if (epnum == 0) { - xfer->buffer += total_bytes; - } } else #endif { @@ -732,6 +728,8 @@ static void handle_bus_reset(uint8_t rhport) { tu_memclr(xfer_status, sizeof(xfer_status)); + _dcd_data.ep0_pending[TUSB_DIR_OUT] = 0; + _dcd_data.ep0_pending[TUSB_DIR_IN] = 0; _dcd_data.sof_en = false; _dcd_data.allocated_epin_count = 0; @@ -1009,6 +1007,10 @@ static void handle_epout_dma(uint8_t rhport, uint8_t epnum, dwc2_doepint_t doepi if (edpt_is_enabled(epin0)) { edpt_disable(rhport, 0x80, false); } + // a new SETUP aborts any in-progress control transfer: drop leftover EP0 chunking state so a + // stale latched completion cannot re-arm from it + _dcd_data.ep0_pending[TUSB_DIR_OUT] = 0; + _dcd_data.ep0_pending[TUSB_DIR_IN] = 0; dcd_dcache_invalidate(_dcd_usbbuf.setup_buffer, sizeof(_dcd_usbbuf.setup_buffer)); @@ -1029,24 +1031,37 @@ static void handle_epout_dma(uint8_t rhport, uint8_t epnum, dwc2_doepint_t doepi // only handle data skip if it is setup or status related // Normal OUT transfer complete if (!doepint_bm.status_phase_rx && !doepint_bm.setup_packet_rx) { + xfer_ctl_t* xfer = XFER_CTL_BASE(epnum, TUSB_DIR_OUT); if ((epnum == 0) && _dcd_data.ep0_pending[TUSB_DIR_OUT]) { - // EP0 can only handle one packet Schedule another packet to be received. + // EP0 can only handle one packet: invalidate and advance past the received bytes, then + // schedule the next. + if (xfer->buffer != NULL) { + dcd_dcache_invalidate(xfer->buffer, CFG_TUD_ENDPOINT0_SIZE); + xfer->buffer += CFG_TUD_ENDPOINT0_SIZE; + } edpt_schedule_packets(rhport, epnum, TUSB_DIR_OUT); } else { dwc2_dep_t* epout = &dwc2->epout[epnum]; - xfer_ctl_t* xfer = XFER_CTL_BASE(epnum, TUSB_DIR_OUT); // determine actual received bytes const dwc2_ep_tsize_t tsiz = {.value = epout->tsiz}; const uint16_t remain = tsiz.xfer_size; xfer->total_len -= remain; + // EP0 invalidates only this (final) chunk's DMA-written bytes: DOEPDMA "is incremented on + // every AHB transaction" (databook 7.1.83), i.e. it points past the last word written. + // Read it before dma_setup_prepare() re-targets it at the setup buffer + uint16_t inval_len = xfer->total_len; + if (epnum == 0) { + inval_len = (uint16_t)(epout->doepdma - (uintptr_t)xfer->buffer); + } + // prepare EP0 for next setup if(epnum == 0) { dma_setup_prepare(rhport); } - dcd_dcache_invalidate(xfer->buffer, xfer->total_len); + dcd_dcache_invalidate(xfer->buffer, inval_len); dcd_event_xfer_complete(rhport, epnum, xfer->total_len, XFER_RESULT_SUCCESS, true); } } @@ -1058,7 +1073,10 @@ static void handle_epin_dma(uint8_t rhport, uint8_t epnum, dwc2_diepint_t diepin if (diepint_bm.xfer_complete) { if ((epnum == 0) && _dcd_data.ep0_pending[TUSB_DIR_IN]) { - // EP0 can only handle one packet. Schedule another packet to be transmitted. + // EP0 can only handle one packet: advance past the sent bytes, then schedule the next. + if (xfer->buffer != NULL) { + xfer->buffer += CFG_TUD_ENDPOINT0_SIZE; + } edpt_schedule_packets(rhport, epnum, TUSB_DIR_IN); } else { dcd_event_xfer_complete(rhport, epnum | TUSB_DIR_IN_MASK, xfer->total_len, XFER_RESULT_SUCCESS, true); diff --git a/test/hil/tinyusb.json b/test/hil/tinyusb.json index 8f121b6f8..8ed33c8a2 100644 --- a/test/hil/tinyusb.json +++ b/test/hil/tinyusb.json @@ -22,11 +22,12 @@ { "name": "espressif_p4_function_ev-DMA", "flags": "-DCFG_TUD_DWC2_DMA_ENABLE=1 -DCFG_TUH_DWC2_DMA_ENABLE=1" } ], "tests": { - "comment": "only IDF/FreeRTOS examples are part of the espressif fleet build; device/usbtest builds under IDF but is not built/flashed by the fleet, so it is not listed", + "comment": "espressif fleet build = IDF/FreeRTOS examples plus the IDF-buildable bare-metal-style ones tools/build.py allowlists (board_test, usbtest, video_capture)", "only": [ "device/cdc_msc_freertos", "device/hid_composite_freertos", "device/audio_test_freertos", + "device/usbtest", "host/device_info", "host/msc_file_explorer_freertos" ], @@ -66,6 +67,7 @@ "device/cdc_msc_freertos", "device/hid_composite_freertos", "device/audio_test_freertos", + "device/usbtest", "host/device_info", "host/msc_file_explorer_freertos" ], @@ -153,8 +155,6 @@ "name": "mimxrt1015_evk", "uid": "DC28F865D2111D228D00B0543A70463C", "tests": { - "skip": ["device/usbtest"], - "comment": "this board's HS battery killed the uPD720201 twice (2026-07-11 on ROM fw, 2026-07-13 case 27 on fw 2.0.2.6 - stop-endpoint timeout, HC died); mimxrt1064/ch32v307 batteries pass, so it is board-specific - keep skipped", "device": true, "host": false, "dual": false diff --git a/tools/build.py b/tools/build.py index 5eaaeb513..51d3d0f70 100755 --- a/tools/build.py +++ b/tools/build.py @@ -92,6 +92,7 @@ def get_examples(family): if family == 'espressif': all_examples.append('device/board_test') + all_examples.append('device/usbtest') all_examples.append('device/video_capture') all_examples.append('host/device_info') all_examples.sort() -- cgit v1.3.1 From 9b3259e60f3bd7e2d9637b61dc07265e4a73b362 Mon Sep 17 00:00:00 2001 From: hathach Date: Thu, 16 Jul 2026 20:47:19 +0700 Subject: hil: controller-aware scheduling of flash and usbtest concurrency Full-fleet profiling (HIL_PROFILE=1 instrumentation, included) showed each uPD720201 controller's serialized usbtest battery chain dominates wall time, and a board whose marginal device port bounces during concurrent batteries can wedge or kill the controller ("xHCI host not responding to stop endpoint command"). Every such death traced to mimxrt1015's port (its old "kills the uPD720201" reputation) - it is removed from the config until recabled; mimxrt1064's enum-retry stalls were a loose device cable (re-seated). nrf54lm20dk moves to boards-skip until its failing J-Link probe is replugged. With the hardware fixed both cards run width-4 batteries plus full flash churn clean, so scheduling stays simple: two symmetric knobs, flashes and batteries budgeted per controller. - schedule_boards(): dispatch boards round-robin across host controllers from a persisted hint cache (~/.cache/tinyusb-hil/ctrl_cache.json), learned and merge-on-write refreshed each run (concurrent HIL jobs keep each other's entries). Only the cached PCI address is consumed - dispatch order and first-flash budgeting, never battery serialization (batteries resolve live or fail closed to an all-slot permit). - HIL_FLASH_PARALLEL (8) and HIL_USBTEST_PARALLEL (4) are budgeted per controller via lock slots assigned on first sight. - re-runs: a failed run writes /.failed with the exact re-run spec (--accumulate -b -bt :) instead of the inverted --skip-board list of everything that passed; --skip-board is gone, --flasher/--exclude-flasher scope a config across CI jobs by flasher type (no board names hardcoded in workflows), and -a/--accumulate merges a re-run into the existing report. The spec is stamped with GITHUB_RUN_ID and cleared on fresh runs, so a retry can never consume a spec left behind by a different run's dead or skipped attempt. - CI: esp-idf firmware builds move out of hil-build into hil-build-esp, and the esptool-flashed boards run in their own hil-tinyusb-esp job, so the main hil-tinyusb run starts as soon as the fast toolchains finish instead of waiting on the slow esp-idf build (an esp toolchain flake previously skipped the whole rig run). Artifacts are namespaced per toolchain so the esp job downloads only esp-idf binaries. - HIL_PROFILE=1: timestamped log lines, per-flash durations, permit-wait logging, uid->controller map dump for analysis. - hil_report: per-variant test duration as a dedicated trailing column, recorded only by full runs. Validated on the ci rig (fixed seeds 20260716/777, full fleet at 8/4): 738s/780s walls with only known-flake failures and no controller deaths, vs 1134-1211s serialized-battery baseline. --- .github/workflows/build.yml | 97 ++++++++-- .github/workflows/build_util.yml | 2 +- test/hil/hil_test.py | 379 +++++++++++++++++++++++++++------------ test/hil/tinyusb.json | 85 ++++----- 4 files changed, 386 insertions(+), 177 deletions(-) (limited to 'test/hil/tinyusb.json') diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f24f3ae1f..818e7ba81 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -274,13 +274,25 @@ jobs: toolchain: - 'arm-gcc' - 'riscv-gcc' - - 'esp-idf' with: build-system: 'cmake' toolchain: ${{ matrix.toolchain }} build-args: ${{ toJSON(fromJSON(needs.set-matrix.outputs.hil_json)[matrix.toolchain]) }} upload-artifacts: true + # esp-idf builds are by far the slowest; keep them out of hil-build so the main + # hil-tinyusb run starts as soon as the fast toolchains finish (esp boards get + # their own hil-tinyusb-esp run gated only on this job) + hil-build-esp: + needs: [ check-paths, set-matrix ] + if: needs.check-paths.outputs.code_changed == 'true' && github.repository_owner == 'hathach' + uses: ./.github/workflows/build_util.yml + with: + build-system: 'cmake' + toolchain: 'esp-idf' + build-args: ${{ toJSON(fromJSON(needs.set-matrix.outputs.hil_json)['esp-idf']) }} + upload-artifacts: true + # --------------------------------------- # Hardware in the loop (HIL) # self-hosted on local VM, for attached hardware checkout HIL_JSON @@ -295,9 +307,13 @@ jobs: - display: tinyusb.json runner: [ self-hosted, X64, hathach, hardware-in-the-loop ] hil_json: test/hil/tinyusb.json + # esptool-flashed (espressif) boards run in hil-tinyusb-esp, + # gated on the slow esp-idf build + test_args: '--exclude-flasher esptool' - display: hfp.json runner: [ self-hosted, Linux, X64, hifiphile ] hil_json: test/hil/hfp.json + test_args: '' runs-on: ${{ matrix.runner }} env: HIL_JSON: ${{ matrix.hil_json }} @@ -305,16 +321,19 @@ jobs: - name: Set HIL report dir (sibling of workspace; persists across run attempts) run: echo "HIL_REPORT_DIR=$(dirname "$GITHUB_WORKSPACE")/hil-report" >> "$GITHUB_ENV" - - name: Get Skip Boards from previous run + - name: Get re-run spec from previous attempt if: github.run_attempt != '1' run: | - if [ -f "${{ env.HIL_JSON }}.skip" ]; then - SKIP_BOARDS=$(cat "${{ env.HIL_JSON }}.skip") + # only honor a spec stamped by THIS run: a spec left by another run (attempt 1 + # died or was skipped before hil_test.py could clear it) must not be consumed + SPEC="$HIL_REPORT_DIR/$(basename "${{ env.HIL_JSON }}").failed" + if [ -f "$SPEC" ] && [ "$(cat "$SPEC.run" 2>/dev/null)" = "$GITHUB_RUN_ID" ]; then + RERUN_ARGS=$(cat "$SPEC") else - SKIP_BOARDS="" + RERUN_ARGS="" fi - echo "SKIP_BOARDS=$SKIP_BOARDS" - echo "SKIP_BOARDS=$SKIP_BOARDS" >> $GITHUB_ENV + echo "RERUN_ARGS=$RERUN_ARGS" + echo "RERUN_ARGS=$RERUN_ARGS" >> $GITHUB_ENV - name: Clean workspace run: | @@ -335,9 +354,7 @@ jobs: - name: Test on actual hardware # Single attempt per test (--retry 1), no in-run second pass: a broken fixture # fails fast instead of holding the runner (and other PRs' HIL jobs) for hours. - # hil_test.py still writes ${HIL_JSON}.skip, so a manual re-run attempt only - # retests what failed (see "Get Skip Boards from previous run"). - run: python3 test/hil/hil_test.py --retry 1 ${{ env.HIL_JSON }} $SKIP_BOARDS + run: python3 test/hil/hil_test.py --retry 1 ${{ matrix.test_args }} ${{ env.HIL_JSON }} $RERUN_ARGS - name: Upload HIL report if: always() && github.event_name == 'pull_request' @@ -348,6 +365,66 @@ jobs: if-no-files-found: ignore overwrite: true + # --------------------------------------- + # Hardware in the loop (HIL) - espressif boards only + # Same rig as hil-tinyusb (tinyusb.json) but gated only on the slow esp-idf build, + # so the main run does not wait for it. Per-board flocks arbitrate the shared rig; + # the runner has a single job slot, so the two HIL jobs never overlap - adding a + # second slot would double the per-controller flash/usbtest budgets. + # --------------------------------------- + hil-tinyusb-esp: + needs: hil-build-esp + name: hil-tinyusb (tinyusb-esp.json) + runs-on: [ self-hosted, X64, hathach, hardware-in-the-loop ] + env: + HIL_JSON: test/hil/tinyusb.json + TEST_ARGS: '--flasher esptool' + steps: + - name: Set HIL report dir (sibling of workspace; persists across run attempts) + run: echo "HIL_REPORT_DIR=$(dirname "$GITHUB_WORKSPACE")/hil-report-esp" >> "$GITHUB_ENV" + + - name: Get re-run spec from previous attempt + if: github.run_attempt != '1' + run: | + # only honor a spec stamped by THIS run: a spec left by another run (attempt 1 + # died or was skipped before hil_test.py could clear it) must not be consumed + SPEC="$HIL_REPORT_DIR/$(basename "${{ env.HIL_JSON }}").failed" + if [ -f "$SPEC" ] && [ "$(cat "$SPEC.run" 2>/dev/null)" = "$GITHUB_RUN_ID" ]; then + RERUN_ARGS=$(cat "$SPEC") + else + RERUN_ARGS="" + fi + echo "RERUN_ARGS=$RERUN_ARGS" + echo "RERUN_ARGS=$RERUN_ARGS" >> $GITHUB_ENV + + - name: Clean workspace + run: | + echo "Cleaning up for the first run" + rm -rf "${{ github.workspace }}" + mkdir -p "${{ github.workspace }}" + + - name: Checkout TinyUSB + uses: actions/checkout@v6 + + - name: Download Artifacts + uses: actions/download-artifact@v5 + with: + pattern: binaries-esp-idf-* + path: cmake-build + merge-multiple: true + + - name: Test on actual hardware + run: python3 test/hil/hil_test.py --retry 1 $TEST_ARGS ${{ env.HIL_JSON }} $RERUN_ARGS + + - name: Upload HIL report + if: always() && github.event_name == 'pull_request' + uses: actions/upload-artifact@v7 + with: + name: hil-report-tinyusb-esp.json + path: ${{ env.HIL_REPORT_DIR }}/hil_report.md + if-no-files-found: ignore + overwrite: true + # --------------------------------------- # Hardware in the loop (HIL) # self-hosted by HFP, build with IAR toolchain, for attached hardware checkout test/hil/hfp.json diff --git a/.github/workflows/build_util.yml b/.github/workflows/build_util.yml index 2532caebe..90115862b 100644 --- a/.github/workflows/build_util.yml +++ b/.github/workflows/build_util.yml @@ -99,7 +99,7 @@ jobs: if: inputs.upload-artifacts == true && inputs.code-changed == true uses: actions/upload-artifact@v7 with: - name: binaries-${{ matrix.arg }} + name: binaries-${{ inputs.toolchain }}-${{ matrix.arg }} path: | cmake-build/cmake-build-*/*/*/*.elf cmake-build/cmake-build-*/*/*/*.bin diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py index 40ae80572..448b236fc 100755 --- a/test/hil/hil_test.py +++ b/test/hil/hil_test.py @@ -38,6 +38,7 @@ import argparse import io +import itertools import os import random import re @@ -121,7 +122,7 @@ ENUM_TIMEOUT_RETRY = 4 _enum_timeout = ENUM_TIMEOUT -def enum_timeout_s() -> int: +def enum_timeout() -> int: """Enumeration wait budget for the current test attempt.""" return _enum_timeout @@ -130,7 +131,7 @@ def wait_until(predicate, step: float = 1.0): """Poll predicate under the per-attempt enum budget. Deadline-based so a slow predicate body (subprocess, libmtp scan) counts against the budget. Returns the first truthy predicate value, or None on timeout.""" - deadline = time.monotonic() + enum_timeout_s() + deadline = time.monotonic() + enum_timeout() while True: r = predicate() if r: @@ -157,6 +158,7 @@ class TestFail(AssertionError): verbose = False +PROFILE = os.environ.get('HIL_PROFILE') == '1' # timestamped logs + permit/flash timing + ctrl-map dump test_only = [] board_test = {} build_dir = 'cmake-build' @@ -164,35 +166,41 @@ skip_flash = False print_lock = None shuffle_seed = None # per-run seed for the per-board test-order shuffle (HIL_SHUFFLE_SEED to replay) -# Per-host-controller concurrency (see controller_of/ctrl_slot below): a usbtest battery +# Per-host-controller concurrency (see controller_of/controller_slot below): a usbtest battery # saturates its DUT's host controller, so batteries and flashes are budgeted per controller. -# NOTE: a Renesas uPD720201 host card must run its latest firmware (>= 2.0.2.6; RAM-uploaded, -# so it must be re-loaded every power cycle) - its ROM firmware dies under battery + -# flash/re-enumeration churn, and usbtest.py refuses the unlink-stress cases on old firmware. -# Widths profiled 2026-07-13/14 on fw 2.0.2.6 (8/1 through 12/8): wall time falls -# 22.2/14.3/12.5/10.8 min at usbtest width 1/2/3/4 and plateaus there; flash width beyond 8 -# buys nothing and only amplifies flasher-hub contention; the first battery case failures -# (bandwidth stretch on shared leaf-hub uplinks) appear at 12/8. Hence the 8/4 defaults. -FLASH_PARALLEL = max(1, int(os.getenv('HIL_FLASH_PARALLEL', '8'))) -USBTEST_PARALLEL = max(1, int(os.getenv('HIL_USBTEST_PARALLEL', '4'))) -CTRL_SLOTS = 12 # lock slots; controllers are assigned to slots on first sight -usbtest_sems = None # CTRL_SLOTS semaphores: up to USBTEST_PARALLEL batteries per controller -flash_sems = None # CTRL_SLOTS semaphores(FLASH_PARALLEL): flash permits per controller -ctrl_map = None # shared dict: 'pci:' -> slot, 'uid:' -> pci addr cache -ctrl_meta = None # guards slot assignment in ctrl_map - - -def init_worker(lock, seed, b_mutexes, f_sems, cmap, cmeta): - global print_lock, shuffle_seed, usbtest_sems, flash_sems, ctrl_map, ctrl_meta +# - uPD720201 cards need their latest firmware (>= 2.0.2.6; RAM-uploaded, reloads every +# power cycle): ROM firmware dies under battery + re-enumeration churn, and usbtest.py +# refuses the unlink-stress cases on it. +# - widths (profiled 2026-07-13/14): wall time 22.2/14.3/12.5/10.8 min at usbtest width +# 1/2/3/4, plateau after; flash width beyond 8 only adds flasher-hub contention; +# battery case failures start at 12/8 (bandwidth stretch on shared leaf-hub uplinks). +# - a marginal DUT port bouncing during concurrent batteries can wedge/kill a uPD720201 +# ("xHCI host not responding to stop endpoint command"): fix the port/cable or pull +# the board, don't lower the widths (2026-07-16: every death traced to one board's port). +FLASH_PARALLEL = int(os.getenv('HIL_FLASH_PARALLEL', '8')) +USBTEST_PARALLEL = int(os.getenv('HIL_USBTEST_PARALLEL', '4')) +CONTROLLER_SLOTS = 12 # lock slots; controllers are assigned to slots on first sight +usbtest_sems = None # CONTROLLER_SLOTS semaphores: per-slot usbtest-battery permits +flash_sems = None # CONTROLLER_SLOTS semaphores: per-slot flash permits +controller_map = None # shared dict: 'pci:' -> slot, 'uid:' -> pci addr cache +controller_meta = None # guards slot assignment in controller_map +controller_hints = {} # static uid -> pci from the last run's cache (read-only per worker) + + +def init_worker(lock, seed, b_mutexes, f_sems, cmap, cmeta, hints_by_uid): + global print_lock, shuffle_seed, usbtest_sems, flash_sems, controller_map, controller_meta, controller_hints print_lock = lock shuffle_seed = seed usbtest_sems = b_mutexes flash_sems = f_sems - ctrl_map = cmap - ctrl_meta = cmeta + controller_map = cmap + controller_meta = cmeta + controller_hints = hints_by_uid def log_line(msg: str) -> None: + if PROFILE: + msg = f'{time.time():.3f} {msg}' out = sys.__stdout__ if sys.__stdout__ is not None else sys.stdout if print_lock is not None: with print_lock: @@ -210,9 +218,9 @@ def controller_of(uid: str): resolutions are cached — cabling does not change mid-run. Dual-port parts (e.g. CH32V307 usbhs/usbfs variants) share one uid and one cache entry: budgeting is only exact when both ports sit on the same controller (true on this rig).""" - if ctrl_map is None: + if controller_map is None: return None - cached = ctrl_map.get(f'uid:{uid}') + cached = controller_map.get(f'uid:{uid}') if cached: return cached for f in glob.glob('/sys/bus/usb/devices/*/serial'): @@ -224,29 +232,29 @@ def controller_of(uid: str): root = os.path.realpath(f'/sys/bus/usb/devices/usb{bus}') m = re.findall(r'[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]', root) if m: - ctrl_map[f'uid:{uid}'] = m[-1] + controller_map[f'uid:{uid}'] = m[-1] return m[-1] except (OSError, ValueError): continue return None -def ctrl_slot(pci: str) -> int: +def controller_slot(pci: str) -> int: """Map a controller PCI address to a lock slot (assigned on first sight).""" key = f'pci:{pci}' - with ctrl_meta: - slot = ctrl_map.get(key) + with controller_meta: + slot = controller_map.get(key) if slot is None: - slot = ctrl_map.get('nslots', 0) - if slot >= CTRL_SLOTS: + slot = controller_map.get('nslots', 0) + if slot >= CONTROLLER_SLOTS: slot = 0 # more controllers than slots: overflow shares slot 0 (safe, over-serialized) else: - ctrl_map['nslots'] = slot + 1 - ctrl_map[key] = slot + controller_map['nslots'] = slot + 1 + controller_map[key] = slot return slot -class ctrl_permit: +class controller_permit: """Context manager: one permit from `sems` on the board's controller slot. If the controller is unknown, fail closed: take one permit from EVERY slot, in order, so the operation respects the budget wherever it might land. `warn_unknown` logs that fallback @@ -254,21 +262,34 @@ class ctrl_permit: def __init__(self, sems, uid: str, warn_unknown: bool = False): self.sems = sems self.slots = None + self.uid = uid if sems is None: return pci = controller_of(uid) + if pci is None and not warn_unknown: + # last-run cabling hint, flash budgeting only: a mis-budgeted flash is harmless, + # but a battery must never trust a stale hint (it could stack two batteries on + # one controller). In practice only a board's first flash lands here - batteries + # assert enumeration before taking their permit. + pci = controller_hints.get(uid) if pci is None and warn_unknown: log_line(f'warning: cannot resolve {uid} to a host controller; ' 'taking a permit on every slot (over-serialized)') - self.slots = [ctrl_slot(pci)] if pci else list(range(CTRL_SLOTS)) + self.slots = [controller_slot(pci)] if pci else list(range(CONTROLLER_SLOTS)) def __enter__(self): if self.slots: + t0 = time.monotonic() taken = [] try: for s in self.slots: self.sems[s].acquire() taken.append(s) + # stays inside the try: if this raises (e.g. broken stdout), the permits + # must be released - a failed __enter__ never gets its __exit__ + if PROFILE and time.monotonic() - t0 > 1.0: + log_line(f'[prof] permit wait {time.monotonic() - t0:.1f}s ' + f'(uid {self.uid}, slots {self.slots})') except BaseException: for s in reversed(taken): self.sems[s].release() @@ -282,12 +303,12 @@ class ctrl_permit: return False -def flash_permit(uid: str) -> ctrl_permit: - return ctrl_permit(flash_sems, uid) +def flash_permit(uid: str) -> controller_permit: + return controller_permit(flash_sems, uid) -def usbtest_permit(uid: str) -> ctrl_permit: - return ctrl_permit(usbtest_sems, uid, warn_unknown=True) +def usbtest_permit(uid: str) -> controller_permit: + return controller_permit(usbtest_sems, uid, warn_unknown=True) def compact_output(raw: str) -> str: @@ -426,7 +447,7 @@ def get_alsa_capture_dev(id): def open_serial_dev(port: str): - timeout = enum_timeout_s() + timeout = enum_timeout() ser = None while timeout > 0: if os.path.exists(port): @@ -749,7 +770,7 @@ def test_dual_host_info_to_device_cdc(board): # read until all expected devices are enumerated data = b'' - timeout = enum_timeout_s() + timeout = enum_timeout() while timeout > 0: new_data = ser.read(ser.in_waiting or 1) if new_data: @@ -801,7 +822,7 @@ def test_host_device_info(board): # read until all expected devices are enumerated data = b'' - timeout = enum_timeout_s() + timeout = enum_timeout() while timeout > 0: new_data = ser.read(ser.in_waiting or 1) if new_data: @@ -880,7 +901,7 @@ def test_host_cdc_msc_hid(board): # Wait for all expected mount messages data = b'' - timeout = enum_timeout_s() + timeout = enum_timeout() wait_cdc = len(cdc_devs) > 0 wait_msc = len(msc_devs) > 0 while timeout > 0: @@ -973,7 +994,7 @@ def test_host_msc_file_explorer(board): # Wait for MSC mount (Disk Size message) data = b'' - timeout = enum_timeout_s() + timeout = enum_timeout() while timeout > 0: new_data = ser.read(ser.in_waiting or 1) if new_data: @@ -1032,9 +1053,9 @@ def test_host_msc_file_explorer(board): for line in resp_text.splitlines(): if 'KB/s' in line: print(f'{line.strip()} ', end='') - m = re.search(r'([\d.]+\s*[KMG]B/s)', line) # MSC read speed for the report cell + m = re.search(r'([\d.]+)\s*([KMG]B/s)', line) # MSC read speed for the report cell if m: - speed = 'rd ' + m.group(1).replace(' ', '') + speed = f'{m.group(1)} {m.group(2)}' break ser.close() @@ -1138,7 +1159,7 @@ def test_device_cdc_msc_throughput(board): # Wait for MSC disk enumeration dev = get_disk_dev(uid, 'TinyUSB', 0) - timeout = enum_timeout_s() + timeout = enum_timeout() while timeout > 0: if os.path.exists(dev): break @@ -1147,7 +1168,7 @@ def test_device_cdc_msc_throughput(board): # Wait for CDC tty enumeration tty = get_serial_dev(uid, 'TinyUSB', 'Throughput', 0) - timeout = enum_timeout_s() + timeout = enum_timeout() while timeout > 0: if os.path.exists(tty): break @@ -1196,9 +1217,19 @@ def test_device_cdc_msc_throughput(board): pass print(f' CDC read {cdc_r} write {cdc_w}, MSC read {msc_r} write {msc_w} ', end='') - # compact read/write speed for the report cell, e.g. "✅ CDC 652k/422k MSC 1.1M/783k" - short = lambda s: (s.split()[0].rstrip('0').rstrip('.') + s.split()[-1][0]) if ' ' in s else s - return f'{REPORT_CELL["pass"]} CDC {short(cdc_r)}/{short(cdc_w)} MSC {short(msc_r)}/{short(msc_w)}' + + # compact read/write speeds for the report cell, e.g. "✅ C 652/422k M 1.1M/783k" + # (C=CDC, M=MSC; the unit is shown once when both sides share it) + def short(s): + return (s.split()[0].rstrip('0').rstrip('.') + s.split()[-1][0]) if ' ' in s else s + + def pair(r, w): + r, w = short(r), short(w) + if r[-1:] == w[-1:] and r[-1:].isalpha(): + r = r[:-1] + return f'{r}/{w}' + + return f'{REPORT_CELL["pass"]} C {pair(cdc_r, cdc_w)} M {pair(msc_r, msc_w)}' def test_device_dfu(board): @@ -1206,7 +1237,7 @@ def test_device_dfu(board): # Wait device enum. Deadline-based: dfu-util -l itself takes ~1 s per call, which a # per-iteration countdown would not charge against the budget. - deadline = time.monotonic() + enum_timeout_s() + deadline = time.monotonic() + enum_timeout() found = False while time.monotonic() < deadline: ret = run_cmd(f'dfu-util -l') @@ -1247,7 +1278,7 @@ def test_device_dfu(board): def test_device_dfu_runtime(board): uid = board['uid'] # Wait device enum (deadline-based, see test_device_dfu) - deadline = time.monotonic() + enum_timeout_s() + deadline = time.monotonic() + enum_timeout() found = False while time.monotonic() < deadline: ret = run_cmd(f'dfu-util -l') @@ -1266,7 +1297,7 @@ def test_device_hid_boot_interface(board): mouse1 = get_hid_dev(uid, 'TinyUSB', 'TinyUSB_Device', 'if01-event-mouse') mouse2 = get_hid_dev(uid, 'TinyUSB', 'TinyUSB_Device', 'if01-mouse') # Wait device enum - timeout = enum_timeout_s() + timeout = enum_timeout() while timeout > 0: if os.path.exists(kbd) and os.path.exists(mouse1) and os.path.exists(mouse2): break @@ -1478,7 +1509,7 @@ def test_device_net_lwip_webserver(board): # Poll the iperf TCP port until the device is accepting. The net stack comes up a bit # after DHCP completes; iperf server binding isn't instantaneous after reflash. - deadline = time.monotonic() + enum_timeout_s() + deadline = time.monotonic() + enum_timeout() last_err = None while time.monotonic() < deadline: try: @@ -1488,7 +1519,7 @@ def test_device_net_lwip_webserver(board): except OSError as e: last_err = e time.sleep(0.3) - assert last_err is None, f'iperf TCP {device_ip}:{iperf_port} not accepting within {enum_timeout_s()}s: {last_err}' + assert last_err is None, f'iperf TCP {device_ip}:{iperf_port} not accepting within {enum_timeout()}s: {last_err}' # Throughput: 5-second iperf2 TCP test, CSV output for stable parsing. # iperf2 CSV final summary line: timestamp,src_ip,src_port,dst_ip,dst_port,id,interval,bytes,bps @@ -1528,7 +1559,7 @@ def test_device_midi_test(board): uid = board['uid'] # Find MIDI device via /dev/snd/by-id using board UID - timeout = enum_timeout_s() + timeout = enum_timeout() midi_port = None while timeout > 0: pattern = f'/dev/snd/by-id/usb-*_{uid}-*' @@ -1587,7 +1618,7 @@ def test_device_audio_test_freertos(board): return 'skipped' pcm = None - timeout = enum_timeout_s() + timeout = enum_timeout() while timeout > 0: pcm = get_alsa_capture_dev(uid) if pcm: @@ -1654,7 +1685,7 @@ def test_device_hid_generic_inout(board): import hid # cython-hidapi (pip: hidapi, apt: python3-hid) # Find HID device by UID (VID=0xCafe) - timeout = enum_timeout_s() + timeout = enum_timeout() dev = None while timeout > 0: for d in hid.enumerate(0xCafe): @@ -1705,12 +1736,15 @@ def test_device_usbtest(board): pass return False - end = time.monotonic() + enum_timeout_s() + end = time.monotonic() + enum_timeout() while time.monotonic() < end and not usbtest_enumerated(): time.sleep(0.2) # fail before usbtest_permit: an absent device would otherwise queue on the battery # mutex for minutes behind real batteries just to have usbtest.py report "no device" - assert usbtest_enumerated(), f'no cafe:4010 device with serial {uid}' + if not usbtest_enumerated(): + # 0/30 rather than a bare cell: the battery never ran (30 = standard case count) + raise TestFail(f'no cafe:4010 device with serial {uid}', + metric=f'{REPORT_CELL["fail"]} 0/30') # settle: right after flashing the enumeration can bounce once (and on dual-port parts like # CH32V307 the other port's stale usbtest node — same serial and PID — lingers a moment); # running testusb into that gap sees the device drop mid-case @@ -1732,7 +1766,8 @@ def test_device_usbtest(board): data = json.loads(out[brace:]) passed, failed = int(data['passed']), int(data['failed']) except (ValueError, KeyError, json.JSONDecodeError): - raise AssertionError(f'usbtest did not run: {compact_output(out) or cmd_stdout_text(r.stderr)}') + raise TestFail(f'usbtest did not run: {compact_output(out) or cmd_stdout_text(r.stderr)}', + metric=f'{REPORT_CELL["fail"]} 0/30') total = passed + failed if failed == 0 and total > 0: @@ -1745,12 +1780,12 @@ def test_device_usbtest(board): # ------------------------------------------------------------- # Main # ------------------------------------------------------------- + +# The per-board run order is shuffled (see test_board). +# Every example carries a unique hardcoded idProduct (see its usb_descriptors.c) + # device tests device_tests = [ - # The per-board run order is shuffled (see test_board). Every example carries a unique - # hardcoded idProduct (see its usb_descriptors.c), so any two different examples always - # re-enumerate back-to-back — even on boards whose CPU-reset does not drop D+ (e.g. WCH - # CH58x via openocd), which only re-enumerate when the PID changes. 'device/cdc_dual_ports', 'device/cdc_msc', 'device/dfu', @@ -1833,7 +1868,11 @@ def test_example(board: Board, variant: str, example: str) -> tuple[int, str, st with redirect_stdout(attempt_out): if not skip_flash: with flash_permit(board['uid']): + t_flash = time.monotonic() ret = globals()[f'flash_{board["flasher"]["name"].lower()}'](board, str(fw_name)) + if PROFILE: + log_line(f'[prof] {variant} {example} flash attempt {i + 1}: ' + f'{time.monotonic() - t_flash:.1f}s rc={ret.returncode}') flash_ok = (ret.returncode == 0) if flash_ok: try: @@ -1923,7 +1962,7 @@ def build_board(board: Board) -> tuple[str, int]: return name, failed -def test_board(board: Board) -> tuple[str, int, list[str], list]: +def test_board(board: Board) -> tuple[str, int, list[str], list, float]: name = board['name'] flasher = board['flasher'] @@ -1933,7 +1972,9 @@ def test_board(board: Board) -> tuple[str, int, list[str], list]: log_line(f'{name:25} {STATUS_FAILED}: {e}') # visible report row so the ❌ matches the exit code; failed-tests stays # empty so a re-run repeats the whole board (no bogus -bt test filter) - return name, 1, [], [(name, {'board-locked': 'fail'})] + return name, 1, [], [(name, {'board-locked': 'fail'}, None)], 0.0 + # after the lock: flock wait behind a concurrent run is not board cost + t_board = time.monotonic() try: # default to all tests test_list = [] @@ -1972,7 +2013,10 @@ def test_board(board: Board) -> tuple[str, int, list[str], list]: err_count = 0 failed_tests = [] - rows = [] # list of (row_label, {example: status}) — one row per build variant + rows = [] # list of (row_label, {example: status}, duration) — one row per build variant + # a -t/-bt filtered run times only a subset; report no duration so an accumulate + # re-run keeps the previous full-run value + partial = bool(test_only) or name in board_test variants = board.get('variant') or [{'name': name, 'flags': ''}] prev_last = None # last test of the previous variant: the variant boundary is an adjacency too @@ -1988,9 +2032,9 @@ def test_board(board: Board) -> tuple[str, int, list[str], list]: random.Random(f'{shuffle_seed}:{name}:{vname}').shuffle(run_list) if run_list[0] == prev_last: run_list[0], run_list[-1] = run_list[-1], run_list[0] - log_line(f'{vname:40} test order: {", ".join(t.rsplit("/", 1)[-1] for t in run_list)}') if run_list: prev_last = run_list[-1] + t_variant = time.monotonic() cells = {} for test in run_list: ec, status, metric = test_example(board, vname, test) @@ -1998,14 +2042,19 @@ def test_board(board: Board) -> tuple[str, int, list[str], list]: cells[test] = metric if metric else status if ec > 0: failed_tests.append(test) - rows.append((vname, cells)) + dur = f'{time.monotonic() - t_variant:.0f}s' if run_list and not partial else None + rows.append((vname, cells, dur)) + + # board duration excludes the teardown park-flash below; a partial (filtered) + # run reports 0.0 so it never overwrites a cached full-run duration + t_total = 0.0 if partial else time.monotonic() - t_board # flash board_test last to disable board's usb (skipped when --skip-flash is set); # this is teardown/park, not a test — not recorded in the report if not skip_flash: test_example(board, variants[0]['name'], 'device/board_test') - return name, err_count, sorted(set(failed_tests)), rows + return name, err_count, sorted(set(failed_tests)), rows, t_total finally: if _lock_fh: try: @@ -2021,13 +2070,30 @@ def test_board(board: Board) -> tuple[str, int, list[str], list]: REPORT_MD = 'hil_report.md' REPORT_JSON = 'hil_report.json' +# controller hints learned from previous runs: uid -> {'name', 'pci', 'duration'}. Only +# 'pci' is consumed (dispatch order and first-flash budgeting, never battery +# serialization); name/duration are informational. PCI addresses are boot-stable (bus +# numbers are not), so the cache survives reboots and only goes stale on re-cabling. +CONTROLLER_CACHE = Path.home() / '.cache' / 'tinyusb-hil' / 'controller_cache.json' + + +def schedule_boards(boards: list, pci_of_uid: dict) -> list: + """Dispatch order: round-robin across host controllers so every controller's + serialized usbtest battery chain is fed from t=0 instead of one card's boards + convoying at the head of the queue. Boards without a controller hint form their + own bucket; config order is kept within a bucket.""" + buckets = {} + for b in boards: + buckets.setdefault(pci_of_uid.get(b['uid'], '?'), []).append(b) + return [b for grp in itertools.zip_longest(*buckets.values()) for b in grp if b is not None] def render_matrix(rows_all: list) -> str: - """Render rows (list of (row_label, {example: status})) as an aligned markdown - matrix: columns = tests (bare names) centered, boards left-aligned.""" + """Render rows (list of (row_label, {example: status}, duration)) as an aligned + markdown matrix: columns = tests (bare names) centered, boards left-aligned, + per-row duration as the trailing column.""" seen = set() - for _, cells in rows_all: + for _, cells, _ in rows_all: seen.update(cells) if not seen: return 'No tests were run.' @@ -2041,7 +2107,7 @@ def render_matrix(rows_all: list) -> str: return (pinned.index(name) if name in pinned else len(pinned), name, t) columns = sorted(seen, key=col_key) - headers = [c.rsplit('/', 1)[-1] for c in columns] # bare example name + headers = [c.rsplit('/', 1)[-1] for c in columns] + ['duration'] # bare example names def cell(cells, col): v = cells.get(col) @@ -2049,10 +2115,12 @@ def render_matrix(rows_all: list) -> str: return '' return REPORT_CELL.get(v, v) # status symbol, or a metric string (e.g. speed) verbatim + rows_vals = [(lbl, [cell(cells, c) for c in columns] + [dur or '']) + for lbl, cells, dur in rows_all] board_hdr = 'Board' - board_w = max([len(board_hdr)] + [len(lbl) for lbl, _ in rows_all]) - col_w = [max([len(h)] + [len(cell(cells, c)) for _, cells in rows_all]) - for h, c in zip(headers, columns)] + board_w = max([len(board_hdr)] + [len(lbl) for lbl, _ in rows_vals]) + col_w = [max([len(h)] + [len(vals[i]) for _, vals in rows_vals]) + for i, h in enumerate(headers)] def line(label, values): padded = [label.ljust(board_w)] + [v.center(w) for v, w in zip(values, col_w)] @@ -2060,7 +2128,7 @@ def render_matrix(rows_all: list) -> str: header = line(board_hdr, headers) sep = '| ' + '-' * board_w + ' | ' + ' | '.join(':' + '-' * (w - 2) + ':' for w in col_w) + ' |' - body = [line(lbl, [cell(cells, c) for c in columns]) for lbl, cells in rows_all] + body = [line(lbl, vals) for lbl, vals in rows_vals] # tally run cells (blank/not-run cells are absent from the dicts). A cell is a bare status # ('pass'/'fail'/'skip') or a metric string that carries its own icon (e.g. "❌ 29/30" is a @@ -2071,7 +2139,7 @@ def render_matrix(rows_all: list) -> str: if v == 'skip' or (isinstance(v, str) and v.startswith(REPORT_CELL['skip'])): return 'skip' return 'pass' - kinds = [cell_kind(v) for _, cells in rows_all for v in cells.values()] + kinds = [cell_kind(v) for _, cells, _ in rows_all for v in cells.values()] failed = kinds.count('fail') skipped = kinds.count('skip') passed = kinds.count('pass') @@ -2083,38 +2151,42 @@ def render_matrix(rows_all: list) -> str: def accumulate_report(mret: list, report_dir: Path, fresh: bool) -> str: """Merge this run's results into hil_report.json in report_dir, then (re)write - the markdown matrix to hil_report.md. `fresh` (a full run, no --skip-board/-bt) + the markdown matrix to hil_report.md. `fresh` (a full run, no --accumulate/-bt) starts a new report; otherwise a re-run accumulates so boards/tests that already passed are preserved while re-run cells are updated. Returns the md.""" - acc = {} # ordered {row_label: {example: status}} + acc = {} # ordered {row_label: [cells dict, duration str|None]} jpath = report_dir / REPORT_JSON if not fresh and jpath.is_file(): try: for entry in json.loads(jpath.read_text()).get('rows', []): - acc[entry['board']] = dict(entry['cells']) + acc[entry['board']] = [dict(entry['cells']), entry.get('duration')] except (ValueError, KeyError, TypeError): pass # corrupt/old sidecar: start fresh - # merge this run: current cells override prior for boards/tests that ran - for name, _, _, rows in mret: - if rows and not any('board-locked' in cells for _, cells in rows): + # merge this run: current cells override prior for boards/tests that ran; a filtered + # run reports duration None, keeping the previous full-run value + for name, _, _, rows, _ in mret: + if rows and not any('board-locked' in cells for _, cells, _ in rows): # board ran for real this time: clear a stale lock-failure cell # (its row is keyed by board name; test rows may be variant names) stale = acc.get(name) if stale is not None: - stale.pop('board-locked', None) - if not stale: + stale[0].pop('board-locked', None) + if not stale[0]: # variant-keyed boards never repopulate the board-name row — # drop it or it renders as a blank ghost row del acc[name] - for row_label, cells in rows: - acc.setdefault(row_label, {}).update(cells) + for row_label, cells, dur in rows: + row = acc.setdefault(row_label, [{}, None]) + row[0].update(cells) + if dur is not None: + row[1] = dur report_dir.mkdir(parents=True, exist_ok=True) - jpath.write_text(json.dumps({'rows': [{'board': k, 'cells': v} for k, v in acc.items()]}, - indent=2) + '\n') + jpath.write_text(json.dumps({'rows': [{'board': k, 'cells': c, 'duration': d} + for k, (c, d) in acc.items()]}, indent=2) + '\n') - md = render_matrix(list(acc.items())) + md = render_matrix([(k, c, d) for k, (c, d) in acc.items()]) (report_dir / REPORT_MD).write_text(md + '\n', encoding='utf-8') return md @@ -2135,7 +2207,14 @@ def main() -> None: parser = argparse.ArgumentParser() parser.add_argument('config_file', help='Configuration JSON file') parser.add_argument('-b', '--board', action='append', default=[], help='Boards to test, all if not specified') - parser.add_argument('-s', '--skip-board', action='append', default=[], help='Skip boards from test') + parser.add_argument('--flasher', action='append', default=[], + help='Only boards using these flashers, e.g. esptool ' + '(for splitting one config across CI jobs)') + parser.add_argument('--exclude-flasher', action='append', default=[], + help='Exclude boards using these flashers') + parser.add_argument('-a', '--accumulate', action='store_true', + help='Merge results into the existing report instead of starting fresh ' + '(re-runs; the .failed file starts with this)') parser.add_argument('-sf', '--skip-flash', action='store_true', help='Run tests without flashing firmware (use whatever is already on the board)') parser.add_argument('-t', '--test-only', action='append', default=[], help='Tests to run, all if not specified') parser.add_argument('-bt', '--board-test', action='append', default=[], @@ -2148,7 +2227,6 @@ def main() -> None: config_file = Path(args.config_file) boards = args.board - skip_boards = args.skip_board verbose = args.verbose test_only = args.test_only for entry in args.board_test: @@ -2167,7 +2245,7 @@ def main() -> None: config = cast(HilConfig, json.load(f)) if len(boards) == 0: - config_boards = [e for e in config['boards'] if e['name'] not in skip_boards] + config_boards = list(config['boards']) else: unknown = [b for b in boards if b not in {e['name'] for e in config['boards']}] if unknown: @@ -2175,6 +2253,8 @@ def main() -> None: print(f'ERROR: board(s) not in {config_file.name}: {", ".join(unknown)}') sys.exit(1) config_boards = [e for e in config['boards'] if e['name'] in boards] + config_boards = [e for e in config_boards if e['flasher']['name'] not in args.exclude_flasher + and (not args.flasher or e['flasher']['name'] in args.flasher)] build_err = 0 if args.build: @@ -2191,26 +2271,46 @@ def main() -> None: print(f'Build phase done: {build_err} failed') print('-' * 30) - # HIL report sidecar (hil_report.json/.md). A full run starts fresh; a re-run - # (--skip-board / -bt, i.e. the .skip file) accumulates so already-passed - # boards/tests are preserved. Clear any prior report up front on a fresh run so - # a crash mid-run can't leave stale results to be merged by a retry or posted. + # HIL report sidecar (hil_report.json/.md) and the .failed re-run spec live in + # report_dir (persists across CI run attempts). A full run starts fresh; a re-run + # (--accumulate / -bt, i.e. the .failed file) merges so already-passed boards/tests + # are preserved. Clear prior state up front on a fresh run so a crash mid-run can't + # leave a stale report - or worse, a stale re-run spec from another commit - to be + # consumed by a retry. report_dir = Path(os.environ.get('HIL_REPORT_DIR', '.')) - fresh = not (args.skip_board or args.board_test) + failed_fname = report_dir / (config_file.name + '.failed') + fresh = not (args.accumulate or args.board_test) if fresh: report_dir.mkdir(parents=True, exist_ok=True) for f in (REPORT_JSON, REPORT_MD): (report_dir / f).unlink(missing_ok=True) + failed_fname.unlink(missing_ok=True) + failed_fname.with_suffix(failed_fname.suffix + '.run').unlink(missing_ok=True) seed = os.getenv('HIL_SHUFFLE_SEED') or str(int(time.time())) log_line(f'test-order shuffle seed: {seed} (HIL_SHUFFLE_SEED={seed} to replay); ' f'flash/usbtest parallel per controller: {FLASH_PARALLEL}/{USBTEST_PARALLEL}; ' f'enum timeout first/retry: {ENUM_TIMEOUT}/{ENUM_TIMEOUT_RETRY}s') + + hints = {} + try: + with CONTROLLER_CACHE.open() as f: + loaded = json.load(f) + # tolerate a hand-edited/torn cache: keep only the expected uid -> dict shape + if isinstance(loaded, dict): + hints = {k: v for k, v in loaded.items() if isinstance(v, dict)} + except (OSError, ValueError): + pass + hints_by_uid = {uid: h['pci'] for uid, h in hints.items() if h.get('pci')} + config_boards = schedule_boards(config_boards, hints_by_uid) + log_line('dispatch order: ' + ', '.join(b['name'] for b in config_boards)) + mgr = Manager() + cmap = mgr.dict() initargs = (Lock(), seed, - [Semaphore(USBTEST_PARALLEL) for _ in range(CTRL_SLOTS)], - [Semaphore(FLASH_PARALLEL) for _ in range(CTRL_SLOTS)], - mgr.dict(), Lock()) + [Semaphore(USBTEST_PARALLEL) for _ in range(CONTROLLER_SLOTS)], + [Semaphore(FLASH_PARALLEL) for _ in range(CONTROLLER_SLOTS)], + cmap, Lock(), hints_by_uid) with Pool(processes=os.cpu_count() or 1, initializer=init_worker, initargs=initargs) as pool: async_ret = pool.map_async(test_board, config_boards) try: @@ -2221,17 +2321,66 @@ def main() -> None: raise RuntimeError(f'HIL worker pool timed out after {POOL_TIMEOUT}s') err_count = build_err + sum(e[1] for e in mret) - # generate skip list for next re-run if failed: skip boards that fully passed, - # and emit -bt BOARD:t1,t2 so each failed board only re-runs its own failed tests. - skip_fname = config_file.with_suffix(config_file.suffix + '.skip') - if err_count > 0: - skip_boards += [name for name, err, _, _ in mret if err == 0] - parts = [f'--skip-board {i}' for i in skip_boards] - parts += [f'-bt {name}:{",".join(fts)}' for name, err, fts, _ in mret if err > 0 and fts] - with skip_fname.open('w') as f: + # generate the re-run spec if anything failed: run ONLY the failed boards (-b), + # each restricted to its own failed tests (-bt); a board with failures but no + # test list (e.g. board-locked) re-runs entirely. --accumulate preserves the + # already-passed cells in the report. + parts = ['--accumulate'] + for name, err, fts, _, _ in mret: + if err > 0: + parts.append(f'-b {name}') + if fts: + parts.append(f'-bt {name}:{",".join(fts)}') + stamp_fname = failed_fname.with_suffix(failed_fname.suffix + '.run') + if len(parts) > 1: # build-only failures have no boards to re-run + report_dir.mkdir(parents=True, exist_ok=True) + with failed_fname.open('w') as f: f.write(' '.join(parts)) - elif skip_fname.exists(): - skip_fname.unlink() + # CI stamps the spec with its run id: a later run's retry must not consume a + # spec left by an attempt of a DIFFERENT run (e.g. attempt 1 skipped entirely) + stamp_fname.write_text(os.environ.get('GITHUB_RUN_ID', '')) + else: + failed_fname.unlink(missing_ok=True) + stamp_fname.unlink(missing_ok=True) + + # refresh controller hints: pci resolved this run, plus board durations when the + # full test list ran (a -t/-bt filtered run would understate the board's real cost) + try: + if PROFILE: + # debug snapshot of the run's live uid->PCI / PCI->slot resolutions + report_dir.mkdir(parents=True, exist_ok=True) + with (report_dir / 'hil_profile_ctrl.json').open('w') as f: + json.dump(dict(cmap), f, indent=1, sort_keys=True) + uid_of = {b['name']: b['uid'] for b in config['boards']} + for name, _, _, _, dur in mret: + uid = uid_of.get(name) + if uid is None: + continue + h = dict(hints.get(uid) or {}) + h['name'] = name # informational: cache is keyed by uid + h['pci'] = cmap.get(f'uid:{uid}') or h.get('pci') + if dur > 0: # test_board reports 0.0 for filtered (partial) runs + h['duration'] = round(dur, 1) + hints[uid] = h + # merge-on-write: another HIL job (e.g. the esp split) may have finished since + # our startup read - re-read and overlay only this run's boards so its entries + # survive, then replace atomically so a concurrent reader never sees a torn file + merged = {} + try: + with CONTROLLER_CACHE.open() as f: + cur = json.load(f) + if isinstance(cur, dict): + merged = {k: v for k, v in cur.items() if isinstance(v, dict)} + except (OSError, ValueError): + pass + merged.update({uid_of[n]: hints[uid_of[n]] for n, *_ in mret if n in uid_of}) + CONTROLLER_CACHE.parent.mkdir(parents=True, exist_ok=True) + tmp = CONTROLLER_CACHE.with_suffix('.json.tmp') + with tmp.open('w') as f: + json.dump(merged, f, indent=1, sort_keys=True) + tmp.replace(CONTROLLER_CACHE) + except OSError as e: + print(f'warning: cannot persist controller hints to {CONTROLLER_CACHE}: {e}') # board x test result matrix -> hil_report.md (accumulates across re-runs) + stdout report = accumulate_report(mret, report_dir, fresh) diff --git a/test/hil/tinyusb.json b/test/hil/tinyusb.json index 8ed33c8a2..6871d812f 100644 --- a/test/hil/tinyusb.json +++ b/test/hil/tinyusb.json @@ -22,7 +22,6 @@ { "name": "espressif_p4_function_ev-DMA", "flags": "-DCFG_TUD_DWC2_DMA_ENABLE=1 -DCFG_TUH_DWC2_DMA_ENABLE=1" } ], "tests": { - "comment": "espressif fleet build = IDF/FreeRTOS examples plus the IDF-buildable bare-metal-style ones tools/build.py allowlists (board_test, usbtest, video_capture)", "only": [ "device/cdc_msc_freertos", "device/hid_composite_freertos", @@ -151,51 +150,6 @@ "args": "-device ATSAMD51J19" } }, - { - "name": "mimxrt1015_evk", - "uid": "DC28F865D2111D228D00B0543A70463C", - "tests": { - "device": true, - "host": false, - "dual": false - }, - "flasher": { - "name": "jlink", - "uid": "000726284213", - "args": "-device MIMXRT1015DAF5A" - } - }, - { - "name": "mimxrt1064_evk", - "uid": "BAE96FB95AFA6DBB8F00005002001200", - "tests": { - "skip": ["host/cdc_msc_hid"], - "comment-cdc-echo": "CH9102+Lexar bundle (moved here from stm32f723disco) mounts fine but echo returns nothing - TX-RX loopback jumper likely lost in the move; re-check wiring then re-enable", - "device": true, - "host": true, - "dual": true, - "dev_attached": [ - { - "vid_pid": "1a86_55d4", - "serial": "52D2003414", - "is_cdc": true - }, - { - "vid_pid": "21c4_0cc7", - "serial": "90005893730A1A63", - "is_msc": true, - "block_size": 512, - "block_count": 60620800, - "msc_inquiry": "Lexar USB Flash Drive PMAP" - } - ] - }, - "flasher": { - "name": "jlink", - "uid": "000725299165", - "args": "-device MIMXRT1064xxx6A" - } - }, { "name": "lpcxpresso11u37", "uid": "17121919", @@ -234,8 +188,6 @@ "device": true, "host": true, "dual": true, - "skip": ["host/cdc_msc_hid", "host/device_info", "host/msc_file_explorer", "host/msc_file_explorer_freertos", "dual/host_info_to_device_cdc"], - "comment-skip": "PIO-USB host port enumerates nothing since the board moves (CH340+UDisk bundle unplugged or unpowered) - re-attach the bundle then drop these skips", "dev_attached": [ { "vid_pid": "1a86_7523", @@ -550,10 +502,43 @@ "uid": "001050076405", "args": "-device NRF5340_XXAA_APP" } + } + ], + "boards-skip": [ + { + "name": "mimxrt1064_evk", + "uid": "BAE96FB95AFA6DBB8F00005002001200", + "comment-skip": "device-port cable degraded from enum drops to killing the uPD720201 mid-battery (2026-07-17); replace the cable, verify enum, then move back", + "tests": { + "device": true, + "host": true, + "dual": true, + "dev_attached": [ + { + "vid_pid": "1a86_55d4", + "serial": "52D2003414", + "is_cdc": true + }, + { + "vid_pid": "21c4_0cc7", + "serial": "90005893730A1A63", + "is_msc": true, + "block_size": 512, + "block_count": 60620800, + "msc_inquiry": "Lexar USB Flash Drive PMAP" + } + ] + }, + "flasher": { + "name": "jlink", + "uid": "000725299165", + "args": "-device MIMXRT1064xxx6A" + } }, { "name": "nrf54lm20dk", "uid": "899C3DE5B0F4D5CA", + "comment-skip": "J-Link probe fails most flashes (2026-07-16); replug/repair the probe, then move back", "tests": { "device": true, "host": false, @@ -566,9 +551,7 @@ "uid": "1051856258", "args": "-device NRF54LM20A_M33" } - } - ], - "boards-skip": [ + }, { "name": "ra6m5_ek", "uid": "8419032D32363657364EF4622D294B4E", -- cgit v1.3.1 From d155273ce44f6bcf72494cce396188c90279c6a1 Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 10 Jul 2026 00:17:56 +0700 Subject: hil: add frdm_k64f host test (cdc + msc) to tinyusb.json frdm_k64f as a USB host with a CH9102 CDC (TX-RX loopback) and a Lexar MSC drive behind a hub; flasher = onboard OpenSDA J-Link. host/cdc_msc_hid passes (CDC mount+echo, MSC mount + disk-size check). device_info remains a known device_info/usbh limitation (its synchronous descriptor dump starves a 2nd device's enumeration) and is not ci_fs-specific. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ExGPLP5eU43LR7o6yYLpNi --- test/hil/tinyusb.json | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) (limited to 'test/hil/tinyusb.json') diff --git a/test/hil/tinyusb.json b/test/hil/tinyusb.json index 6871d812f..812eb7571 100644 --- a/test/hil/tinyusb.json +++ b/test/hil/tinyusb.json @@ -1,5 +1,35 @@ { "boards": [ + { + "name": "frdm_k64f", + "uid": "FFFF1A00FFFFFFFF1031454E0F000280", + "tests": { + "device": false, + "host": true, + "dual": false, + "dev_attached": [ + { + "vid_pid": "1a86_55d4", + "serial": "52D2023934", + "is_cdc": true, + "comment": "CH9102 USB-serial (TX-RX loopback)" + }, + { + "vid_pid": "21c4_0cc7", + "serial": "9000588268687E25", + "is_msc": true, + "block_size": 512, + "block_count": 60620800, + "msc_inquiry": "Lexar USB Flash Drive PMAP" + } + ] + }, + "flasher": { + "name": "jlink", + "uid": "000621000000", + "args": "-device MK64FN1M0xxx12" + } + }, { "name": "ek_tm4c123gxl", "uid": "010105186C60A110", -- cgit v1.3.1 From 1d915b6b59cb88f14344521db1f9345d8c7dc9a7 Mon Sep 17 00:00:00 2001 From: Ha Thach Date: Tue, 28 Jul 2026 12:50:28 +0700 Subject: bsp, hil: flash WCH boards with the unified OpenOCD fork (#3791) bsp, hil: flash with the unified OpenOCD fork https://github.com/hathach/openocd (branch tinyusb) is mainline plus every config these boards need: RPi RP2350, ADI max32/max78, the MounRiver WCH configs, and the wlinke adapter on mainline's riscv target. It is a superset of the vendor forks, so one 'openocd' covers all boards; -DOPENOCD=/OPENOCD= still select another, msdk's when MAXIM_PATH is set. Drops family_flash_openocd_wch and the OPENOCD_WCH pair, dedups family_flash_openocd_adi, aligns ch583's work area, and points hil at the flasher's own config instead of generating one per probe. Verified: HIL green on all four WCH boards and max32666fthr. --- .idea/debugServers/wch_riscv.xml | 2 +- hw/bsp/ch32v10x/family.cmake | 2 +- hw/bsp/ch32v10x/family.mk | 2 +- hw/bsp/ch32v20x/family.cmake | 2 +- hw/bsp/ch32v20x/family.mk | 2 +- hw/bsp/ch32v30x/family.cmake | 2 +- hw/bsp/ch32v30x/family.mk | 2 +- hw/bsp/ch583/family.cmake | 2 +- hw/bsp/ch583/family.mk | 2 +- hw/bsp/ch583/wch-riscv.cfg | 2 +- hw/bsp/family_rules.mk | 16 +++++++------- hw/bsp/family_support.cmake | 44 +++++++++++++-------------------------- hw/bsp/rp2040/family.cmake | 2 ++ test/hil/hil_test.py | 45 ++++++++-------------------------------- test/hil/tinyusb.json | 10 ++++----- 15 files changed, 47 insertions(+), 90 deletions(-) (limited to 'test/hil/tinyusb.json') diff --git a/.idea/debugServers/wch_riscv.xml b/.idea/debugServers/wch_riscv.xml index 2e147f1b6..0b2b83b2e 100644 --- a/.idea/debugServers/wch_riscv.xml +++ b/.idea/debugServers/wch_riscv.xml @@ -4,7 +4,7 @@ - + diff --git a/hw/bsp/ch32v10x/family.cmake b/hw/bsp/ch32v10x/family.cmake index fb9ccb3a3..287b8c2ff 100644 --- a/hw/bsp/ch32v10x/family.cmake +++ b/hw/bsp/ch32v10x/family.cmake @@ -93,6 +93,6 @@ function(family_configure_example TARGET RTOS) # Flashing family_add_bin_hex(${TARGET}) - family_flash_openocd_wch(${TARGET}) + family_flash_openocd(${TARGET}) #family_flash_uf2(${TARGET} ${UF2_FAMILY_ID}) endfunction() diff --git a/hw/bsp/ch32v10x/family.mk b/hw/bsp/ch32v10x/family.mk index fb699b0bb..443509699 100644 --- a/hw/bsp/ch32v10x/family.mk +++ b/hw/bsp/ch32v10x/family.mk @@ -49,5 +49,5 @@ INC += \ FREERTOS_PORTABLE_SRC = $(FREERTOS_PORTABLE_PATH)/RISC-V -OPENOCD_WCH_OPTION=-f $(TOP)/$(FAMILY_PATH)/wch-riscv.cfg +OPENOCD_OPTION=-f $(TOP)/$(FAMILY_PATH)/wch-riscv.cfg flash: flash-openocd-wch diff --git a/hw/bsp/ch32v20x/family.cmake b/hw/bsp/ch32v20x/family.cmake index 785f5ee35..a27ff021e 100644 --- a/hw/bsp/ch32v20x/family.cmake +++ b/hw/bsp/ch32v20x/family.cmake @@ -125,7 +125,7 @@ function(family_configure_example TARGET RTOS) # Flashing family_add_bin_hex(${TARGET}) - family_flash_openocd_wch(${TARGET}) + family_flash_openocd(${TARGET}) family_flash_wlink_rs(${TARGET}) #family_flash_uf2(${TARGET} ${UF2_FAMILY_ID}) endfunction() diff --git a/hw/bsp/ch32v20x/family.mk b/hw/bsp/ch32v20x/family.mk index 1d059bcba..1889c4e26 100644 --- a/hw/bsp/ch32v20x/family.mk +++ b/hw/bsp/ch32v20x/family.mk @@ -63,6 +63,6 @@ INC += \ FREERTOS_PORTABLE_SRC = $(FREERTOS_PORTABLE_PATH)/RISC-V -OPENOCD_WCH_OPTION=-f $(TOP)/$(FAMILY_PATH)/wch-riscv.cfg +OPENOCD_OPTION=-f $(TOP)/$(FAMILY_PATH)/wch-riscv.cfg flash: flash-wlink-rs #flash: flash-openocd-wch diff --git a/hw/bsp/ch32v30x/family.cmake b/hw/bsp/ch32v30x/family.cmake index b974bd5e7..e33e4b85d 100644 --- a/hw/bsp/ch32v30x/family.cmake +++ b/hw/bsp/ch32v30x/family.cmake @@ -115,6 +115,6 @@ function(family_configure_example TARGET RTOS) # Flashing family_add_bin_hex(${TARGET}) - family_flash_openocd_wch(${TARGET}) + family_flash_openocd(${TARGET}) family_flash_wlink_rs(${TARGET}) endfunction() diff --git a/hw/bsp/ch32v30x/family.mk b/hw/bsp/ch32v30x/family.mk index 5ccdea8ae..59778ec54 100644 --- a/hw/bsp/ch32v30x/family.mk +++ b/hw/bsp/ch32v30x/family.mk @@ -62,5 +62,5 @@ LD_FILE ?= $(FAMILY_PATH)/linker/ch32v30x.ld # For freeRTOS port source FREERTOS_PORTABLE_SRC = $(FREERTOS_PORTABLE_PATH)/RISC-V -OPENOCD_WCH_OPTION=-f $(TOP)/$(FAMILY_PATH)/wch-riscv.cfg +OPENOCD_OPTION=-f $(TOP)/$(FAMILY_PATH)/wch-riscv.cfg flash: flash-openocd-wch diff --git a/hw/bsp/ch583/family.cmake b/hw/bsp/ch583/family.cmake index a379298e5..f4b874e4e 100644 --- a/hw/bsp/ch583/family.cmake +++ b/hw/bsp/ch583/family.cmake @@ -104,5 +104,5 @@ function(family_configure_example TARGET RTOS) # Flashing family_add_bin_hex(${TARGET}) - family_flash_openocd_wch(${TARGET}) + family_flash_openocd(${TARGET}) endfunction() diff --git a/hw/bsp/ch583/family.mk b/hw/bsp/ch583/family.mk index 98d0f9337..3444c4811 100644 --- a/hw/bsp/ch583/family.mk +++ b/hw/bsp/ch583/family.mk @@ -52,7 +52,7 @@ INC += \ LD_FILE ?= $(FAMILY_PATH)/linker/ch582.ld -OPENOCD_WCH_OPTION=-f $(TOP)/$(FAMILY_PATH)/wch-riscv.cfg +OPENOCD_OPTION=-f $(TOP)/$(FAMILY_PATH)/wch-riscv.cfg flash: flash-openocd-wch # For freeRTOS port source diff --git a/hw/bsp/ch583/wch-riscv.cfg b/hw/bsp/ch583/wch-riscv.cfg index 64d595d8e..aa35aa9c5 100644 --- a/hw/bsp/ch583/wch-riscv.cfg +++ b/hw/bsp/ch583/wch-riscv.cfg @@ -9,7 +9,7 @@ sdi newtap $_CHIPNAME cpu -irlen 5 -expected-id 0x00001 set _TARGETNAME $_CHIPNAME.cpu target create $_TARGETNAME.0 wch_riscv -chain-position $_TARGETNAME -$_TARGETNAME.0 configure -work-area-phys 0x20000000 -work-area-size 0x8000 -work-area-backup 1 +$_TARGETNAME.0 configure -work-area-phys 0x20000000 -work-area-size 10000 -work-area-backup 1 set _FLASHNAME $_CHIPNAME.flash flash bank $_FLASHNAME wch_riscv 0x00000000 0 0 0 $_TARGETNAME.0 diff --git a/hw/bsp/family_rules.mk b/hw/bsp/family_rules.mk index ccf49dd0e..011572888 100644 --- a/hw/bsp/family_rules.mk +++ b/hw/bsp/family_rules.mk @@ -130,20 +130,18 @@ flash-pyocd: $(BUILD)/$(PROJECT).hex #pyocd reset -t $(PYOCD_TARGET) # --------------- openocd ----------------- +# OPENOCD can name another build, e.g. one of the vendor forks, though +# https://github.com/hathach/openocd branch tinyusb covers every board here +OPENOCD ?= openocd OPENOCD_OPTION ?= flash-openocd: $(BUILD)/$(PROJECT).elf - openocd $(OPENOCD_OPTION) -c "program $< verify reset exit" + $(OPENOCD) $(OPENOCD_OPTION) -c "program $< verify reset exit" # --------------- openocd-wch ----------------- -# wch-linke is not supported yet in official openOCD yet. We need to either use -# 1. download openocd as part of mounriver studio http://www.mounriver.com/download or -# 2. compiled from https://github.com/hathach/riscv-openocd-wch or -# https://github.com/dragonlock2/miscboards/blob/main/wch/SDK/riscv-openocd.tar.xz -# with ./configure --disable-werror --enable-wlinke --enable-ch347=no -OPENOCD_WCH ?= /home/${USER}/app/riscv-openocd-wch/src/openocd -OPENOCD_WCH_OPTION ?= +# WCH parts need an openocd built with the wlinke adapter. The image is written +# without verify: WCH code flash is not readable back over the debug bus. flash-openocd-wch: $(BUILD)/$(PROJECT).elf - $(OPENOCD_WCH) $(OPENOCD_WCH_OPTION) -c init -c halt -c "flash write_image $<" -c reset -c exit + $(OPENOCD) $(OPENOCD_OPTION) -c init -c halt -c "flash write_image $<" -c reset -c exit # --------------- wlink-rs ----------------- # flash with https://github.com/ch32-rs/wlink diff --git a/hw/bsp/family_support.cmake b/hw/bsp/family_support.cmake index 1f3952205..33ceb49c2 100644 --- a/hw/bsp/family_support.cmake +++ b/hw/bsp/family_support.cmake @@ -688,7 +688,9 @@ function(family_flash_stflash TARGET) endfunction() -# Add flash openocd target +# Add flash openocd target. +# The default 'openocd' should be https://github.com/hathach/openocd (branch tinyusb): which is mainline plus +# every config the rig needs (RP2350, MAX32/MAX78, WCH) and a drop-in superset of the vendor (downstream) forks function(family_flash_openocd TARGET) if (NOT DEFINED OPENOCD) set(OPENOCD openocd) @@ -715,38 +717,20 @@ function(family_flash_openocd TARGET) #set_property(TARGET ${TARGET}-openocd PROPERTY FOLDER ${TARGET}-group) endfunction() - -# Add flash openocd-wch target -# compiled from https://github.com/hathach/riscv-openocd-wch or https://github.com/dragonlock2/miscboards/blob/main/wch/SDK/riscv-openocd.tar.xz -function(family_flash_openocd_wch TARGET) - if (NOT DEFINED OPENOCD) - set(OPENOCD $ENV{HOME}/app/riscv-openocd-wch/src/openocd) +# Add flash openocd adi (Analog Devices) target using the openocd included +# with msdk (MAXIM_PATH), otherwise the default openocd +function(family_flash_openocd_adi TARGET) + # use openocd from msdk if MAXIM_PATH is set, as cmake variable or in the + # environment. Normalize the latter since msdk can be Windows (MinGW) or Linux + if (NOT DEFINED MAXIM_PATH AND DEFINED ENV{MAXIM_PATH}) + file(TO_CMAKE_PATH "$ENV{MAXIM_PATH}" MAXIM_PATH) endif () - family_flash_openocd(${TARGET}) -endfunction() - - -# Add flash openocd adi (Analog Devices) target -# included with msdk or compiled from release branch of https://github.com/analogdevicesinc/openocd -function(family_flash_openocd_adi TARGET) - if (DEFINED MAXIM_PATH) - # use openocd from msdk with MAXIM_PATH cmake variable first if the user specified it - set(OPENOCD ${MAXIM_PATH}/Tools/OpenOCD/openocd) - set(OPENOCD_OPTION2 "-s ${MAXIM_PATH}/Tools/OpenOCD/scripts") - elseif (DEFINED ENV{MAXIM_PATH}) - # use openocd from msdk with MAXIM_PATH environment variable. Normalize - # since msdk can be Windows (MinGW) or Linux - file(TO_CMAKE_PATH "$ENV{MAXIM_PATH}" MAXIM_PATH_NORM) - set(OPENOCD ${MAXIM_PATH_NORM}/Tools/OpenOCD/openocd) - set(OPENOCD_OPTION2 "-s ${MAXIM_PATH_NORM}/Tools/OpenOCD/scripts") - else() - # compiled from source - if (NOT DEFINED OPENOCD_ADI_PATH) - set(OPENOCD_ADI_PATH $ENV{HOME}/app/openocd_adi) + if (MAXIM_PATH) + if (NOT DEFINED OPENOCD) + set(OPENOCD ${MAXIM_PATH}/Tools/OpenOCD/openocd) endif () - set(OPENOCD ${OPENOCD_ADI_PATH}/src/openocd) - set(OPENOCD_OPTION2 "-s ${OPENOCD_ADI_PATH}/tcl") + set(OPENOCD_OPTION2 "-s ${MAXIM_PATH}/Tools/OpenOCD/scripts") endif () family_flash_openocd(${TARGET}) diff --git a/hw/bsp/rp2040/family.cmake b/hw/bsp/rp2040/family.cmake index aab9a4fae..43b1dc234 100644 --- a/hw/bsp/rp2040/family.cmake +++ b/hw/bsp/rp2040/family.cmake @@ -28,6 +28,8 @@ elseif (PICO_PLATFORM STREQUAL "rp2350-arm-s" OR PICO_PLATFORM STREQUAL "rp2350" set(OPENOCD_TARGET rp2350) elseif (PICO_PLATFORM STREQUAL "rp2350-riscv") set(JLINK_DEVICE rp2350_riscv_0) + # rp2350-riscv.cfg needs the raspberrypi/openocd fork: mainline's riscv + # target does not take the -dap/-ap-num the config uses set(OPENOCD_TARGET rp2350-riscv) endif() diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py index 0efc6826f..80d1e1823 100755 --- a/test/hil/hil_test.py +++ b/test/hil/hil_test.py @@ -24,10 +24,11 @@ # Host setup (required: a missing tool fails its test rather than skipping it): # - System packages: sudo apt install mtools libmtp9 alsa-utils iperf -# mtools - read_disk_file (device/cdc_msc, device/msc_dual_lun) -# libmtp9 - pymtp ctypes load (device/mtp); Debian 13 uses libmtp9t64 -# alsa-utils - arecord (device/audio_test_freertos) -# iperf - throughput tests (device/net_lwip_*) +# mtools read_disk_file (device/cdc_msc, device/msc_dual_lun) +# libmtp9 pymtp ctypes load (device/mtp); Debian 13 uses libmtp9t64 +# alsa-utils arecord (device/audio_test_freertos) +# iperf throughput tests (device/net_lwip_*) +# openocd unified openocd from https://github.com/hathach/openocd (branch tinyusb) for wch, rp2040/rp2350, analog max32 # - device/usbtest: usbtest kernel module + testusb binary (kernel tools/usb/testusb.c) on PATH, # plus sudo for modprobe / sysfs writes # - Python packages: pip install -r requirements.txt @@ -377,25 +378,6 @@ def cmd_stdout_text(out: Any) -> str: return out.decode('utf-8', errors='ignore') return str(out) -WCH_RISCV_CONTENT = """ -adapter driver wlinke -adapter speed 6000 -transport select sdi - -wlink_set_address 0x00000000 -set _CHIPNAME wch_riscv -sdi newtap $_CHIPNAME cpu -irlen 5 -expected-id 0x00001 - -set _TARGETNAME $_CHIPNAME.cpu - -target create $_TARGETNAME.0 wch_riscv -chain-position $_TARGETNAME -$_TARGETNAME.0 configure -work-area-phys 0x20000000 -work-area-size 10000 -work-area-backup 1 -set _FLASHNAME $_CHIPNAME.flash - -flash bank $_FLASHNAME wch_riscv 0x00000000 0 0 0 $_TARGETNAME.0 - -echo "Ready for Remote Connections" -""" MSC_README_TXT = \ b"This is tinyusb's MassStorage Class demo.\r\n\r\n\ @@ -662,24 +644,15 @@ def reset_openocd(board): def flash_openocd_wch(board, firmware): flasher = board['flasher'] - f_wch = f"wch-riscv_{board['uid']}.cfg" - if not os.path.exists(f_wch): - with open(f_wch, 'w') as file: - file.write(WCH_RISCV_CONTENT) - - ret = run_cmd(f'openocd_wch -c "adapter serial {flasher["uid"]}" -f {f_wch} ' - f'-c "program {firmware}.elf reset exit"') + ret = run_cmd(f'openocd -c "tcl_port disabled" -c "gdb_port disabled" -c "telnet_port disabled" ' + f'-c "adapter serial {flasher["uid"]}" {flasher.get("args", "")} -c "program {firmware}.elf reset exit"') return ret def reset_openocd_wch(board): flasher = board['flasher'] - f_wch = f"wch-riscv_{board['uid']}.cfg" - if not os.path.exists(f_wch): - with open(f_wch, 'w') as file: - file.write(WCH_RISCV_CONTENT) - - ret = run_cmd(f'openocd_wch -c "adapter serial {flasher["uid"]}" -f {f_wch} -c "program reset exit"') + ret = run_cmd(f'openocd -c "tcl_port disabled" -c "gdb_port disabled" -c "telnet_port disabled" ' + f'-c "adapter serial {flasher["uid"]}" {flasher.get("args", "")} -c "init; reset run; exit"') return ret diff --git a/test/hil/tinyusb.json b/test/hil/tinyusb.json index 812eb7571..8316dbc33 100644 --- a/test/hil/tinyusb.json +++ b/test/hil/tinyusb.json @@ -147,7 +147,7 @@ "dual": false }, "flasher": { - "name": "openocd_adi", + "name": "openocd", "uid": "E6614C311B597D32", "args": "-f interface/cmsis-dap.cfg -f target/max32665.cfg" } @@ -465,7 +465,7 @@ "flasher": { "name": "openocd_wch", "uid": "EBCA8F0670AF", - "args": "" + "args": "-f target/wch-riscv.cfg" } }, { @@ -480,7 +480,7 @@ "flasher": { "name": "openocd_wch", "uid": "BC4954081051", - "args": "" + "args": "-f target/wch-riscv.cfg" } }, { @@ -499,7 +499,7 @@ "flasher": { "name": "openocd_wch", "uid": "BC5DA47360D0", - "args": "" + "args": "-f target/wch-riscv.cfg" } }, { @@ -514,7 +514,7 @@ "flasher": { "name": "openocd_wch", "uid": "7FD88F0604B5", - "args": "" + "args": "-f target/wch-riscv.cfg" } }, { -- cgit v1.3.1 From e88fc441ddcaaa5abd4f4673ef2bf29499522dc0 Mon Sep 17 00:00:00 2001 From: Ha Thach Date: Wed, 29 Jul 2026 17:29:59 +0700 Subject: hil: split hil_test.py into hil_lock/hil_flash, add pool_check, update rig probes (#3794) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test/hil: add board-pool health check, split hil_test into focused modules (#3794) Add test/hil/hil_pool_check.py: per-board rig health scan — probe presence, light-example flash (dfu_runtime; device_info + serial check for host-only boards), uid re-enumeration, safe recovery (probe authorized-toggle, board reset), verified board_test re-park, USB topology report, and a markdown summary table. Missing firmware is built on the spot (tools/build.py, idf.py for espressif, one get_deps retry); row statuses: ok, flash-failed, failed, locked. Board locks are always respected, never bypassed. Refactor hil_test.py into hil_lock.py (flock protocol, controller permits, hold/release/status CLI; replaces board_lock.py) and hil_flash.py (flashers, find_firmware, run_cmd). Update WCH probe uids and the board roster in tinyusb.json; add the hil-pool-check skill. --- .claude/agents/hil-operator.md | 10 +- .claude/agents/target-debugger.md | 2 +- .claude/skills/etm-trace/SKILL.md | 2 +- .claude/skills/hil-pool-check/SKILL.md | 63 ++ .claude/skills/hil/SKILL.md | 46 +- .claude/skills/pre-pr/SKILL.md | 2 +- .claude/skills/target-debug/SKILL.md | 6 +- .claude/skills/usb-kernel-recover/SKILL.md | 2 +- .idea/codeStyles/Project.xml | 10 + .idea/codeStyles/codeStyleConfig.xml | 5 + .idea/hil-pool-check.iml | 2 + .idea/modules.xml | 8 + .../superpowers/plans/2026-07-28-hil-test-split.md | 355 +++++++ .../specs/2026-07-28-hil-test-refactor-design.md | 141 +++ test/hil/board_lock.py | 254 ----- test/hil/hil_ci.sh | 9 +- test/hil/hil_flash.py | 293 ++++++ test/hil/hil_lock.py | 479 +++++++++ test/hil/hil_pool_check.py | 1013 ++++++++++++++++++++ test/hil/hil_test.py | 530 ++-------- test/hil/tinyusb.json | 28 +- 21 files changed, 2477 insertions(+), 783 deletions(-) create mode 100644 .claude/skills/hil-pool-check/SKILL.md create mode 100644 .idea/codeStyles/Project.xml create mode 100644 .idea/codeStyles/codeStyleConfig.xml create mode 100644 .idea/hil-pool-check.iml create mode 100644 .idea/modules.xml create mode 100644 docs/superpowers/plans/2026-07-28-hil-test-split.md create mode 100644 docs/superpowers/specs/2026-07-28-hil-test-refactor-design.md delete mode 100755 test/hil/board_lock.py create mode 100755 test/hil/hil_flash.py create mode 100755 test/hil/hil_lock.py create mode 100644 test/hil/hil_pool_check.py (limited to 'test/hil/tinyusb.json') diff --git a/.claude/agents/hil-operator.md b/.claude/agents/hil-operator.md index d19eca047..6f04f6dcc 100644 --- a/.claude/agents/hil-operator.md +++ b/.claude/agents/hil-operator.md @@ -7,7 +7,7 @@ model: sonnet You operate physical USB test hardware. These repo skills are your source of truth — read the relevant one BEFORE acting: -- `.claude/skills/hil/SKILL.md` — run `hostname` first (host `ci` = local mode with `test/hil/tinyusb.json`; host `htpc` = local `local.json` or remote via `test/hil/hil_ci.sh`); the board lock protocol; exact `hil_test.py` invocations. +- `.claude/skills/hil/SKILL.md` — run `hostname` first (host `ci` = local mode with `test/hil/tinyusb.json`; host `tusb` = local mode with `test/hil/hfp.json`; any other host (dev PC) = local `local.json` or remote via `test/hil/hil_ci.sh`); the board lock protocol; exact `hil_test.py` invocations. - `.claude/skills/usb-kernel-recover/SKILL.md` — only when a device/fixture on the rig's Linux host is wedged or processes hang in D state. - `.claude/skills/usb-kernel-debug/SKILL.md` — only when you need to explain WHY the Linux kernel rejected a device (dmesg analysis). @@ -18,12 +18,12 @@ The GitHub Actions runner keeps running during your work. Per-board flock locks - `python3 test/hil/hil_test.py ...` runs: do NOT pre-hold those boards — `hil_test.py` self-locks each board for its flash+test and would fail fast with `board locked` against your own hold. - ANY other hardware action (JLinkExe/openocd/GDB, manual flash, usbtest.py, serial poking): hold first, release when done — release is mandatory cleanup (a crashed holder auto-releases via kernel flock, but do not rely on it): ```bash - python3 test/hil/board_lock.py hold --reason "" + python3 test/hil/hil_lock.py hold --reason "" # ... hardware work ... - python3 test/hil/board_lock.py release + python3 test/hil/hil_lock.py release ``` -- Rig-wide operations (uhubctl power cycling, pci-rebind — they renumber buses): `python3 test/hil/board_lock.py hold --all --reason ""` first. -- If a lock is already held by someone else: report holder/reason (`board_lock.py status`) — never force, never kill the holder. If the holder's reason is `hil_test.py`, that is a concurrent CI job mid-test on the board: waiting a few minutes and retrying once is appropriate when your task allows; otherwise return the holder info so the orchestrator can ask the user. +- Rig-wide operations (uhubctl power cycling, pci-rebind — they renumber buses): `python3 test/hil/hil_lock.py hold --all --reason ""` first. +- If a lock is already held by someone else: report holder/reason (`hil_lock.py status`) — never force, never kill the holder. If the holder's reason is `hil_test.py`, that is a concurrent CI job mid-test on the board: waiting a few minutes and retrying once is appropriate when your task allows; otherwise return the holder info so the orchestrator can ask the user. - You cannot ask the user anything. Bypassing a lock (`HIL_NO_BOARD_LOCK=1`, or proceeding with manual hardware work despite a held lock) is allowed ONLY when your prompt explicitly states the user authorized forcing. ## Hard rules diff --git a/.claude/agents/target-debugger.md b/.claude/agents/target-debugger.md index 246ad16fd..655b5f512 100644 --- a/.claude/agents/target-debugger.md +++ b/.claude/agents/target-debugger.md @@ -46,7 +46,7 @@ the next technique you would try. ## Lock discipline -- Hold the board lock for the WHOLE session (`board_lock.py hold +- Hold the board lock for the WHOLE session (`hil_lock.py hold --reason "target debug: "`). Multi-hour holds are fine; never stop the actions-runner. Locks held by others: report holder/reason, never force unless your prompt states the user authorized it. diff --git a/.claude/skills/etm-trace/SKILL.md b/.claude/skills/etm-trace/SKILL.md index a200b14f3..9e2505736 100644 --- a/.claude/skills/etm-trace/SKILL.md +++ b/.claude/skills/etm-trace/SKILL.md @@ -43,7 +43,7 @@ this skill for exact counts, coverage, or instruction-by-instruction history. capture script uses automation port **19201**, never an interactive Ozone's 19200. - Hold the board lock (see the `hil` skill): - `python3 test/hil/board_lock.py hold --reason "etm capture"`. + `python3 test/hil/hil_lock.py hold --reason "etm capture"`. - Committed `hw/bsp/**/ozone/*.jdebug` are the maintainer's interactive projects — automation never opens them (Ozone rewrites project files); the script generates a throwaway project. diff --git a/.claude/skills/hil-pool-check/SKILL.md b/.claude/skills/hil-pool-check/SKILL.md new file mode 100644 index 000000000..49d252f62 --- /dev/null +++ b/.claude/skills/hil-pool-check/SKILL.md @@ -0,0 +1,63 @@ +--- +name: hil-pool-check +description: Use when asked for a pool check or board/probe health scan on a TinyUSB HIL rig, when probes or boards are offline or fail to flash, after rig maintenance, reboot, or re-cabling, or before starting a HIL test campaign. +--- + +# HIL Pool Check (board/probe health) + +Health-scan the HIL board pool with `test/hil/hil_pool_check.py`: per board it checks the flash +probe is on the USB bus, flashes a light example (`device/dfu_runtime`; host-only boards get +`host/device_info`, verified by serial output), waits for the board's uid to re-enumerate, +applies safe per-device recovery (probe authorized-toggle, board reset), re-parks with +`board_test`, and prints a summary table plus a USB topology report. Flags and details: `--help` +and the module docstring. + +**REQUIRED BACKGROUND:** the `hil` skill owns config-by-hostname selection (run `hostname` +first) and the board-lock protocol. Locked boards are reported 🔒 locked and skipped — never +waited on, never bypassed; a needed build peeks the lock first. A CI worker reaching a board the +pool check holds fails it as "board locked" — prefer running between CI runs. + +## A "pool check" means the full check + +A request for a "pool check" means the DEFAULT full check below. Use `--scan-only` only when the +user explicitly asks for a quick look, or when you have VERIFIED a CI sweep is mid-run right now +(`python3 test/hil/hil_lock.py status` shows `hil_test.py` holders) — "CI might be running" is not +that predicate: the full check is already lock-safe (CI-held boards report 🔒 locked and are never +touched), so an unconfirmed suspicion is no reason to downgrade. In either scan case say which +mode ran and why; never silently substitute the scan for the full check. + +```bash +python3 test/hil/hil_pool_check.py # full check: ~10 s + ~1-2 s/board with firmware built; + # first run on an unbuilt tree takes minutes (it builds) +python3 test/hil/hil_pool_check.py --scan-only # USB presence only, <1 s, no locks/flashing/building +python3 test/hil/hil_pool_check.py -b BOARD [-b …] # subset; may name boards-skip (parked) entries + +# from a dev PC, against the ci rig (bash -lc: flashers like STM32_Programmer_CLI live in ~/bin): +ssh ci.lan 'bash -lc "cd ~/code/tinyusb && python3 test/hil/hil_pool_check.py"' +``` + +## Notes + +Missing firmware is **built on the spot** — never skipped (`--no-build` opts out; those boards +then report `flash-failed`). Builds need the family env, exported on the rig in +`~/.profile`/`~/.bashrc`: `PICO_SDK_PATH` for rp2040/rp2350 (`~/code/pico/pico-sdk`), the +ESP-IDF env (`get-idf`) for espressif — which also needs `esptool` on PATH (pip's +`~/.local/bin/esptool`; a non-login shell may lack it — run via `bash -lc`). An explicit `-B` is +searched exclusively for *existing* firmware; builds still land in `cmake-build/` and are noted +`built `. Espressif boards park too when the IDF env is present. A first run on an +unbuilt tree builds for many minutes: use a command timeout ≥ 30 min and NEVER cancel early — a +killed run leaves detached cmake/ninja children still writing to `cmake-build/`. + +Statuses: `ok` (flashed and verified; in `--scan-only` it only means the probe is present), +`flash-failed` (firmware delivery failed: probe missing, build failed, flasher error, silent +no-op, park unverified), `failed` (check ran but did not verify), `locked` (flock held; +untouched). Exit code = `flash-failed` + `failed` (clamped at 125); `locked` and scan-only rows +are *unverified*, not healthy — read the footer, not just `$?`. A `⚠ pid … source says …` note +means stale firmware or a silent flash no-op (J-Link lore); a device off the bus entirely needs +the usb-kernel-recover skill or a physical replug. + +## Reporting + +The user-facing answer to a pool check IS the tool's summary table: paste the complete per-board +table (and footer counts) verbatim — never truncate rows or reduce it to a prose digest like +"27/27 healthy"; at most one line of commentary below it. diff --git a/.claude/skills/hil/SKILL.md b/.claude/skills/hil/SKILL.md index 2aeaa10af..6c4d3a856 100644 --- a/.claude/skills/hil/SKILL.md +++ b/.claude/skills/hil/SKILL.md @@ -1,22 +1,22 @@ --- name: hil -description: Use when running TinyUSB Hardware-in-the-Loop (HIL) tests on physical boards, debugging HIL failures, or copying firmware to the ci.lan test rig. Covers per-host config selection (htpc uses local.json, ci uses tinyusb.json), local execution on either htpc or ci, remote execution over SSH from htpc, and debugging tips. +description: Use when running TinyUSB Hardware-in-the-Loop (HIL) tests on physical boards, debugging HIL failures, or copying firmware to the ci.lan test rig. Covers per-host config selection (infra rigs ci/tusb use tinyusb.json/hfp.json, any dev PC uses local.json), local and remote execution, the board-lock protocol, and debugging tips. For board/probe health scans ("pool check") use the hil-pool-check skill. --- # Hardware-in-the-Loop (HIL) Testing -Run TinyUSB HIL tests on real boards. **Run `hostname` first** — it tells you which host you are on, which determines the default config and whether remote mode is possible. +Run TinyUSB HIL tests on real boards. **Run `hostname` first** — it tells you which host you are on, which determines the default config and whether remote mode is possible. Rule of thumb: only `ci` and `tusb` are infra rigs; **any other hostname is a dev PC** and uses `local.json`. -| Host | Local config | Remote (SSH → ci.lan)? | -|----------------------------|--------------------------------------|------------------------------------------------------| -| `htpc` (dev PC) | `test/hil/local.json` | yes (large pool, `test/hil/tinyusb.json`) | -| `ci` (the rig) | `test/hil/tinyusb.json` (large pool) | no — can't SSH to htpc, and boards are already local | -| `hifiphile` (external rig) | `test/hil/hfp.json` | no outbound SSH to htpc/ci; SSH-reachable FROM both | +| Host | Local config | Remote (SSH → ci.lan)? | +|-----------------------------------|--------------------------------------|--------------------------------------------------------| +| `ci` (the rig) | `test/hil/tinyusb.json` (large pool) | no — boards are already local | +| `tusb` (hifiphile's external rig) | `test/hil/hfp.json` | no outbound SSH to dev PCs/ci; SSH-reachable FROM both | +| anything else (a dev PC) | `test/hil/local.json` | yes (large pool, `test/hil/tinyusb.json`) | -Default to **local**. Use **remote** only when on `htpc` and the user says `remote`/`ci.lan`. Never attempt remote on `ci`. +Default to **local**. Use **remote** only when on a dev PC and the user says `remote`/`ci.lan`. Never attempt remote on `ci`. -`hifiphile` is an external rig (hosted by maintainer hifiphile), exercised by the GitHub CI -`hil-tinyusb (hfp.json)` matrix job — **never run HIL against it unless the user explicitly asks.** +`tusb` (ssh alias `hifiphile`) is an external rig (hosted by maintainer hifiphile), exercised by the +GitHub CI `hil-tinyusb (hfp.json)` matrix job — **never run HIL against it unless the user explicitly asks.** ## Board locks — the CI runner keeps running @@ -26,30 +26,35 @@ The `ci` rig also hosts a GitHub Actions runner that flashes boards and runs HIL - For hardware work outside `hil_test.py` (JLink/GDB, manual flashing, `usbtest.py`, serial poking), hold the lock first: ```bash -python3 test/hil/board_lock.py hold BOARD [BOARD...] --reason "why" +python3 test/hil/hil_lock.py hold BOARD [BOARD...] --reason "why" # ... hardware work ... -python3 test/hil/board_lock.py release BOARD [BOARD...] +python3 test/hil/hil_lock.py release BOARD [BOARD...] ``` - Never pre-hold boards you are about to run `hil_test.py` on — it self-locks and would treat your own hold as a conflict. -- Rig-wide operations (uhubctl power cycling, pci-rebind — bus renumbering) affect every board: `board_lock.py hold --all --reason "..."` first. -- `board_lock.py status` lists holders. Locks auto-release when the holder process dies (kernel flock); `/tmp` clears on reboot. +- Rig-wide operations (uhubctl power cycling, pci-rebind — bus renumbering) affect every board: `hil_lock.py hold --all --reason "..."` first. +- `hil_lock.py status` lists holders. Locks auto-release when the holder process dies (kernel flock); `/tmp` clears on reboot. - Forcing past a lock: `HIL_NO_BOARD_LOCK=1 python3 test/hil/hil_test.py ...` bypasses the guard without killing the holder. Only with the user's explicit go-ahead — they accept the risk of colliding with whatever holds the board. +## Pool check (board/probe health) + +Board/probe health scanning (`test/hil/hil_pool_check.py`) has its own skill: **hil-pool-check**. +Use it before a HIL campaign, after rig maintenance/reboot, or when boards fail to flash. + ## Prerequisites -Examples must be built for the target board(s) — see CLAUDE.md "Build" → "All examples for a board" (produces `examples/cmake-build-/`). `-B examples` points `hil_test.py` at that parent folder. +Examples must be built for the target board(s) — see CLAUDE.md "Build" → "All examples for a board" (produces `examples/cmake-build-/`). `-B examples` points `hil_test.py` at that parent folder. (This applies to `hil_test.py`; `hil_pool_check.py` builds its own missing firmware.) ## Arguments - **Board:** `-b BOARD_NAME` for one board; omit to run all boards in the config. - **Pass-through:** `-v`, `-r N`, etc. forwarded unchanged. -If `local.json` is missing on `htpc`, ask the user to supply one (only fall back to `tinyusb.json` if told to). +If `local.json` is missing on a dev PC, ask the user to supply one (only fall back to `tinyusb.json` if told to). ## Local execution -Set `CONFIG` from `hostname` first (`test/hil/local.json` on htpc, `test/hil/tinyusb.json` on ci): +Set `CONFIG` from `hostname` first (`test/hil/local.json` on a dev PC, `test/hil/tinyusb.json` on ci, `test/hil/hfp.json` on tusb): ```bash CONFIG=test/hil/local.json # on ci use: CONFIG=test/hil/tinyusb.json @@ -61,7 +66,7 @@ python3 test/hil/hil_test.py -B examples "$CONFIG" python3 test/hil/hil_test.py -b stm32f723disco -B examples "$CONFIG" ``` -## Remote execution (htpc → ci.lan only) +## Remote execution (dev PC → ci.lan only) `test/hil/hil_ci.sh` handles dir setup, scp of test scripts, rsync of firmware (`.elf`/`.bin`/`.hex`), and runs `hil_test.py` on `ci.lan` with `tinyusb.json`: @@ -81,4 +86,7 @@ Runs take 2-5 min. Use a timeout ≥ 20 min (1200000 ms). NEVER cancel early. ## Reporting -Show the output, summarize pass/fail per board. On failure, retry with `-v`; if that's not enough, add temporary debug prints to `hil_test.py`. +The user-facing answer to a HIL run IS the tool's summary table: paste the complete per-board +table (and footer counts) verbatim — never truncate rows or reduce it to a prose digest; at most +one line of commentary below it. On failure, retry with `-v`; if that's not enough, add temporary +debug prints to `hil_test.py`. diff --git a/.claude/skills/pre-pr/SKILL.md b/.claude/skills/pre-pr/SKILL.md index 3f062db78..428383424 100644 --- a/.claude/skills/pre-pr/SKILL.md +++ b/.claude/skills/pre-pr/SKILL.md @@ -24,7 +24,7 @@ Run the software + hardware gate for the current branch. The user invoking this ## 3. HIL boards -- `hilBoards` = chosen boards that are on the rig roster. This host must be able to reach the rig (per `.claude/skills/hil/SKILL.md`: host `ci` = local, `htpc` = remote). If none qualify, run software-only. +- `hilBoards` = chosen boards that are on the rig roster. This host must be able to reach the rig (per `.claude/skills/hil/SKILL.md`: host `ci`/`tusb` = local, any other host (dev PC) = remote). If none qualify, run software-only. ## 4. Launch diff --git a/.claude/skills/target-debug/SKILL.md b/.claude/skills/target-debug/SKILL.md index 5165a76b8..28678c309 100644 --- a/.claude/skills/target-debug/SKILL.md +++ b/.claude/skills/target-debug/SKILL.md @@ -30,9 +30,9 @@ Hold the board lock for the WHOLE manual session; never stop the actions-runner (see the `hil` skill for the full lock protocol): ```bash -python3 test/hil/board_lock.py hold --reason "target debug: " +python3 test/hil/hil_lock.py hold --reason "target debug: " # ... instrument / build / flash / capture / GDB ... -python3 test/hil/board_lock.py release +python3 test/hil/hil_lock.py release ``` Board → probe mapping: `test/hil/tinyusb.json` — `flasher.name` is the probe @@ -44,7 +44,7 @@ family, `flasher.uid` the **probe serial** (many identical probes on the rig): `hw/bsp//boards//board.cmake` (or `board.mk`); family via `ls -d hw/bsp/*/boards/`. - Run on the host that owns the probe — config `test/hil/tinyusb.json` on ci, - `local.json` on htpc (`hil` skill). + `test/hil/hfp.json` on tusb, `local.json` on any other host (dev PC) (`hil` skill). - Espressif boards (S3/P4): different toolchain, probe model, and PHY constraints entirely — read `esp-target-debug` first. diff --git a/.claude/skills/usb-kernel-recover/SKILL.md b/.claude/skills/usb-kernel-recover/SKILL.md index 9e456417f..ea5931cc4 100644 --- a/.claude/skills/usb-kernel-recover/SKILL.md +++ b/.claude/skills/usb-kernel-recover/SKILL.md @@ -57,7 +57,7 @@ refuses a busport that now names a different device. It bounces **every fixture under that root port** — on ci that is up to 25 devices. Hold the affected boards' locks first if you can, but note -`board_lock.py` uses `LOCK_EX | LOCK_NB` and so fails immediately when CI already +`hil_lock.py` uses `LOCK_EX | LOCK_NB` and so fails immediately when CI already holds them; there is no wait-for-lock. When CI is mid-run you are choosing between bouncing its fixtures and leaving the bus wedged for everything. The automated path in `usbtest.py` takes no locks at all and accepts that collateral diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml new file mode 100644 index 000000000..35c56fc87 --- /dev/null +++ b/.idea/codeStyles/Project.xml @@ -0,0 +1,10 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml new file mode 100644 index 000000000..79ee123c2 --- /dev/null +++ b/.idea/codeStyles/codeStyleConfig.xml @@ -0,0 +1,5 @@ + + + + \ No newline at end of file diff --git a/.idea/hil-pool-check.iml b/.idea/hil-pool-check.iml new file mode 100644 index 000000000..4c9423543 --- /dev/null +++ b/.idea/hil-pool-check.iml @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 000000000..063c94f51 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/docs/superpowers/plans/2026-07-28-hil-test-split.md b/docs/superpowers/plans/2026-07-28-hil-test-split.md new file mode 100644 index 000000000..6b8528973 --- /dev/null +++ b/docs/superpowers/plans/2026-07-28-hil-test-split.md @@ -0,0 +1,355 @@ +# hil_test.py Split Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Split `test/hil/hil_test.py` (2370 ln) into a test-focused core plus `hil_lock.py` (board locks + controller permits + operator CLI, superseding `board_lock.py`) and `hil_flash.py` (run_cmd + flash backends + firmware/serial lookup), with no behavior change. + +**Architecture:** Pure code motion per `docs/superpowers/specs/2026-07-28-hil-test-refactor-design.md`. Import graph: `hil_test` → {`hil_lock`, `hil_flash`}; helpers import nothing local. Call sites use module-qualified names (`hil_lock.flash_permit(...)`), never wildcard mirroring. + +**Tech Stack:** Python 3.11+ (existing `TypedDict`/`NotRequired` usage), stdlib only in the helpers (fcntl, json, glob, multiprocessing objects passed in). + +## Global Constraints + +- Work in worktree `.claude/worktrees/hil-test-split` (branch `claude/hil-test-split`); never touch the primary checkout. +- Behavior-preserving: `hil_test.py` CLI args, log lines, report format, lock/permit semantics, flash behavior all byte-identical. The ONLY user-visible change is the CLI filename `board_lock.py` → `hil_lock.py`. +- Moved functions are moved **verbatim** — no reformatting, no comment editing, no "improvements". A diff of a moved function's body against its old self must be empty. +- Commit messages: imperative, scoped, no Co-Authored-By/Claude-Session trailers. +- Every commit leaves the tree working: `python3 -m py_compile` clean on all touched modules, and `python3 .claude/skills/hil/pool_check.py --scan-only` exits 0 (safe on the rig: scan-only takes no locks, flashes nothing). +- Hardware steps (Task 4) run on the `ci` rig only, from this worktree, and rely on the tools' own board flocks — never pre-hold boards you are about to run `hil_test.py`/`pool_check.py` on. + +--- + +### Task 1: Create hil_flash.py; repoint hil_test + pool_check flash call sites + +**Files:** +- Create: `test/hil/hil_flash.py` +- Modify: `test/hil/hil_test.py` (delete moved code; add import; qualify call sites) +- Modify: `.claude/skills/hil/pool_check.py` (flash-related imports) +- Modify: `test/hil/hil_ci.sh` (scp list) + +**Interfaces:** +- Produces (used by Tasks 2-4): module `hil_flash` with `CMD_TIMEOUT`, `run_cmd(cmd, cwd=None, timeout=CMD_TIMEOUT)`, `cmd_stdout_text(out)`, `OPENCOD_ADI_PATH`, `TINYUSB_ROOT`, `flash_jlink/reset_jlink`, `flash_stlink/reset_stlink`, `flash_stflash/reset_stflash`, `flash_openocd/reset_openocd`, `flash_openocd_wch/reset_openocd_wch`, `flash_openocd_adi/reset_openocd_adi`, `flash_wlink_rs/reset_wlink_rs`, `flash_esptool/reset_esptool`, `flash_uniflash/reset_uniflash`, `flash_lm4flash/reset_lm4flash`, `find_firmware(variant, example)`, `get_serial_dev(id, vendor_str, product_str, ifnum)`, module globals `build_dir = 'cmake-build'`, `verbose = False`. + +- [ ] **Step 1: Create `test/hil/hil_flash.py`** + +Header (new code), then the moved blocks verbatim: + +```python +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Firmware flashing for the TinyUSB HIL rig: run_cmd, one flash_*/reset_* pair per +# flasher type (dispatched by config name via getattr), find_firmware, and the +# fixture serial-port resolver get_serial_dev (here, not hil_test: flash_esptool +# needs it and helpers must not import hil_test). +# Callers set module globals `build_dir` and `verbose` (hil_test.main from argparse, +# pool_check directly) exactly as they set hil_test's globals today. + +import glob +import json +import os +import signal +import subprocess +import sys +from pathlib import Path + +verbose = False +build_dir = 'cmake-build' +``` + +Then MOVE (cut from `hil_test.py`, paste unchanged, in this order): +1. `CMD_TIMEOUT = int(os.getenv('HIL_CMD_TIMEOUT', '180'))` (from the constants block; leave `POOL_TIMEOUT`/`SERIAL_*_TIMEOUT` in hil_test) +2. `def cmd_stdout_text(out)` +3. `OPENCOD_ADI_PATH = Path.home() / 'app' / 'openocd_adi'` and `TINYUSB_ROOT = Path(__file__).resolve().parents[2]` +4. `def get_serial_dev(id, vendor_str, product_str, ifnum)` +5. `def run_cmd(cmd, cwd=None, timeout=CMD_TIMEOUT)` +6. All ten `flash_*`/`reset_*` pairs listed in Interfaces, in current file order +7. `def find_firmware(variant, example)` + +- [ ] **Step 2: Delete the moved code from `hil_test.py` and qualify call sites** + +In `hil_test.py`: add `import hil_flash` under the existing imports; delete the moved definitions and the `build_dir = 'cmake-build'` global (line ~165) plus `global build_dir` in `main`. Repoint every use, all module-qualified: +- `globals()[f'flash_{...}']` → `getattr(hil_flash, f'flash_{...}')` (1 site, in `test_example`) +- `globals()[f'reset_{...}']` → `getattr(hil_flash, f'reset_{...}')` (3 sites: `test_host_device_info`, `test_host_cdc_msc_hid`, `test_host_msc_file_explorer`) +- bare `run_cmd(` → `hil_flash.run_cmd(` ; `cmd_stdout_text(` → `hil_flash.cmd_stdout_text(` ; `find_firmware(` → `hil_flash.find_firmware(` ; `get_serial_dev(` → `hil_flash.get_serial_dev(` ; `TINYUSB_ROOT` → `hil_flash.TINYUSB_ROOT` (in `build_board`, `CONTROLLER_CACHE` stays hil_test-local) +- In `main()`: `build_dir = args.build_dir` → `hil_flash.build_dir = args.build_dir`; where `verbose` is set, add `hil_flash.verbose = args.verbose` (hil_test keeps its own `verbose` for test-side prints) +- `run_cmd`'s `elif verbose:` branch now reads `hil_flash.verbose` (it moved with the function — verify it references the module-local name, not hil_test's) + +Find every remaining call site mechanically: + +Run: `grep -nE 'run_cmd|cmd_stdout_text|find_firmware|get_serial_dev|flash_[a-z]|reset_[a-z]|TINYUSB_ROOT|OPENCOD' test/hil/hil_test.py | grep -v hil_flash` +Expected: only hits inside comments/strings and the `reset_{flasher}` dispatch f-strings already qualified. + +- [ ] **Step 3: Repoint pool_check's flash imports** + +In `.claude/skills/hil/pool_check.py`: add `import hil_flash` next to `import hil_test`; replace `hil_test.find_firmware` → `hil_flash.find_firmware` (3 sites), `hil_test.cmd_stdout_text` → `hil_flash.cmd_stdout_text`, `hil_test.get_serial_dev` → `hil_flash.get_serial_dev`, `hil_test.TINYUSB_ROOT` → `hil_flash.TINYUSB_ROOT`, `hil_test.build_dir` → `hil_flash.build_dir` (2 sites incl. `main`'s assignment), `hil_test.verbose = args.verbose` → `hil_flash.verbose = args.verbose`, `getattr(hil_test, f'flash_...')`/`getattr(hil_test, f'reset_...')` → `getattr(hil_flash, ...)` (4 sites). Keep `import hil_test` and the pymtp shim for now (locks still live there; removed in Task 2). + +- [ ] **Step 4: Add hil_flash.py to the hil_ci.sh scp list** + +```bash +scp -q "$ROOT_DIR/test/hil/hil_test.py" \ + "$ROOT_DIR/test/hil/hil_flash.py" \ + "$ROOT_DIR/test/hil/pymtp.py" \ + "$CONFIG" \ + "$REMOTE:$REMOTE_DIR/test/hil/" +``` + +- [ ] **Step 5: Verify** + +Run: `python3 -m py_compile test/hil/hil_flash.py test/hil/hil_test.py .claude/skills/hil/pool_check.py && python3 test/hil/hil_test.py --help >/dev/null && python3 .claude/skills/hil/pool_check.py --scan-only` +Expected: compiles; help prints nothing to stderr; scan-only prints the table and exits 0. + +- [ ] **Step 6: Commit** + +```bash +git add test/hil/hil_flash.py test/hil/hil_test.py test/hil/hil_ci.sh .claude/skills/hil/pool_check.py +git commit -m "hil: extract flashing into hil_flash.py" +``` + +--- + +### Task 2: Create hil_lock.py core (flock protocol + controller permits); repoint hil_test + pool_check + +**Files:** +- Create: `test/hil/hil_lock.py` +- Modify: `test/hil/hil_test.py` +- Modify: `.claude/skills/hil/pool_check.py` +- Modify: `test/hil/hil_ci.sh` + +**Interfaces:** +- Produces: module `hil_lock` with `BOARD_LOCK_DIR`, `CI_REASON = 'hil_test.py'`, `lock_path(board)`, `flock_nb(board)`, `write_record(fh, reason)`, `clear_record(fh)`, `read_record(board)`, `acquire_board_lock(board, reason=CI_REASON)`, `FLASH_PARALLEL`, `USBTEST_PARALLEL`, `CONTROLLER_SLOTS`, `controller_of(uid)`, `controller_slot(pci)`, `controller_permit`, `flash_permit(uid)`, `usbtest_permit(uid)`, `init_scheduling(b_sems, f_sems, cmap, cmeta, hints, log_fn=None)`. + +- [ ] **Step 1: Create `test/hil/hil_lock.py` with the flock core** + +New code (the protocol, factored from today's three copies — `board_lock.py` `cmd_hold`/`read_info`, `hil_test.acquire_board_lock`, pool_check `lock_board`; behavior identical to `hil_test.acquire_board_lock` for the acquire path): + +```python +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Board locks + controller permits for the TinyUSB HIL rig. + +Board locks are kernel flocks in BOARD_LOCK_DIR arbitrating hardware access +between dev sessions and CI's hil_test.py (never stop the actions-runner). +Controller permits are in-process semaphores budgeting flashes and usbtest +batteries per host controller; they have no CLI meaning. The CLI below +(hold/release/status) manages board locks only; it supersedes board_lock.py. +""" +import argparse +import fcntl +import glob +import json +import os +import re +import select +import signal +import sys +import time + +BOARD_LOCK_DIR = '/tmp/tinyusb-hil-locks' +CI_REASON = 'hil_test.py' # release-protected holder tag (release refuses to kill it) +PROFILE = os.environ.get('HIL_PROFILE') == '1' + + +def lock_path(board: str) -> str: + return os.path.join(BOARD_LOCK_DIR, f'{board}.lock') + + +def flock_nb(board: str): + """Open-or-create the lock file WITHOUT truncating (a losing racer must not + wipe the winner's record) and take LOCK_EX|LOCK_NB. Returns the open handle; + raises OSError when the flock is held elsewhere (handle already closed).""" + fd = os.open(lock_path(board), os.O_RDWR | os.O_CREAT, 0o666) + fh = os.fdopen(fd, 'r+') + try: + fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + fh.close() + raise + return fh + + +def write_record(fh, reason: str) -> None: + """Best-effort holder record; the flock itself is already held.""" + try: + fh.truncate(0) + fh.seek(0) + json.dump({'pid': os.getpid(), 'reason': reason, + 'since': time.strftime('%Y-%m-%dT%H:%M:%S%z')}, fh) + fh.flush() + except OSError: + pass + + +def clear_record(fh) -> None: + """Clear our record before dropping the flock so records stay truthful.""" + try: + fh.truncate(0) + except OSError: + pass + + +def read_record(board: str): + try: + with open(lock_path(board)) as f: + return json.load(f) + except (OSError, ValueError): + return None +``` + +Then MOVE `acquire_board_lock` from `hil_test.py` verbatim, with exactly two mechanical edits: signature becomes `def acquire_board_lock(board_name, reason=CI_REASON):` and the record-write dict's `'reason': 'hil_test.py'` becomes `'reason': reason`. Do NOT rewrite its body in terms of `flock_nb` — on conflict it reads holder info from the still-open handle before closing, which `flock_nb` (closes on conflict) cannot provide; the fail-open warning text and RuntimeError message must survive character-for-character. + +- [ ] **Step 2: Move the controller-permit block into `hil_lock.py`** + +MOVE verbatim from `hil_test.py`: the scheduling comment block + `FLASH_PARALLEL`, `USBTEST_PARALLEL`, `CONTROLLER_SLOTS`, the five module globals (`usbtest_sems`, `flash_sems`, `controller_map`, `controller_meta`, `controller_hints`), `controller_of`, `controller_slot`, `controller_permit`, `flash_permit`, `usbtest_permit`. Two mechanical adaptations: +- add at module scope `log = print` and a setter, replacing the two `log_line(...)` calls inside `controller_of`/`controller_permit` with `log(...)`: + +```python +log = print # hil_test.init_worker points this at log_line via init_scheduling + + +def init_scheduling(b_sems, f_sems, cmap, cmeta, hints, log_fn=None): + """Install per-worker scheduling state (called from hil_test.init_worker).""" + global usbtest_sems, flash_sems, controller_map, controller_meta, controller_hints, log + usbtest_sems, flash_sems = b_sems, f_sems + controller_map, controller_meta, controller_hints = cmap, cmeta, hints + if log_fn is not None: + log = log_fn +``` + +- `PROFILE` inside `controller_permit` now resolves to hil_lock's own module constant (defined in Step 1). + +- [ ] **Step 3: Repoint `hil_test.py`** + +Add `import hil_lock`. Delete the moved lock + permit code and the five globals. `init_worker` keeps its exact signature and initargs; its body sets the hil_test globals it still owns (`print_lock`, `shuffle_seed`) and forwards the rest: + +```python +def init_worker(lock, seed, b_mutexes, f_sems, cmap, cmeta, hints_by_uid): + global print_lock, shuffle_seed + print_lock = lock + shuffle_seed = seed + hil_lock.init_scheduling(b_mutexes, f_sems, cmap, cmeta, hints_by_uid, log_fn=log_line) +``` + +Qualify remaining uses: `acquire_board_lock(name)` → `hil_lock.acquire_board_lock(name)` (in `test_board`), `flash_permit(` → `hil_lock.flash_permit(`, `usbtest_permit(` → `hil_lock.usbtest_permit(`, and `main()`'s startup log line + Semaphore construction read `hil_lock.FLASH_PARALLEL`/`hil_lock.USBTEST_PARALLEL`/`hil_lock.CONTROLLER_SLOTS`. `controller_map` reads in the hint-persistence block of `main` use the Manager dict it already holds locally (`cmap`) — no hil_lock global access there; verify. + +- [ ] **Step 4: Repoint pool_check to hil_lock and drop its private copies + hil_test import** + +In `pool_check.py`: replace `lock_board`/`unlock_board` bodies with the shared core — + +```python +import hil_lock + +def lock_board(name: str): + try: + fh = hil_lock.flock_nb(name) + except OSError: + info = hil_lock.read_record(name) + return json.dumps(info) if info else 'unknown holder' + hil_lock.write_record(fh, 'pool_check') + return fh + + +def unlock_board(fh) -> None: + hil_lock.clear_record(fh) + fh.close() +``` + +(Behavior note: `lock_board` currently returns the raw record text; JSON-dumping the parsed record is equivalent for display. `hil_lock.BOARD_LOCK_DIR` replaces `hil_test.BOARD_LOCK_DIR`; `os.makedirs(...)` call stays, now on `hil_lock.BOARD_LOCK_DIR`.) Then delete `import hil_test` and the pymtp stub block (`try: import pymtp ... sys.modules['pymtp'] = ...`) — pool_check now imports only `hil_lock` + `hil_flash`. + +Run: `grep -n 'hil_test' .claude/skills/hil/pool_check.py` +Expected: only the docstring mention of the protocol/history, no code references (update the docstring's "imports test/hil/hil_test.py" line to name hil_lock/hil_flash). + +- [ ] **Step 5: Add hil_lock.py to the hil_ci.sh scp list** (same block as Task 1 Step 4, one more line: `"$ROOT_DIR/test/hil/hil_lock.py" \`) + +- [ ] **Step 6: Verify** + +Run: `python3 -m py_compile test/hil/hil_lock.py test/hil/hil_test.py .claude/skills/hil/pool_check.py && python3 test/hil/hil_test.py --help >/dev/null && python3 .claude/skills/hil/pool_check.py --scan-only` +Expected: clean compile, working scan table, exit 0. + +- [ ] **Step 7: Commit** + +```bash +git add test/hil/hil_lock.py test/hil/hil_test.py test/hil/hil_ci.sh .claude/skills/hil/pool_check.py +git commit -m "hil: extract board locks and controller permits into hil_lock.py" +``` + +--- + +### Task 3: Absorb board_lock.py CLI into hil_lock.py; delete board_lock.py; rename in docs + +**Files:** +- Modify: `test/hil/hil_lock.py` (append CLI) +- Delete: `test/hil/board_lock.py` +- Modify: `.claude/skills/hil/SKILL.md`, `.claude/agents/hil-operator.md`, `.claude/agents/target-debugger.md`, `.claude/skills/etm-trace/SKILL.md`, `.claude/skills/usb-kernel-recover/SKILL.md`, `.claude/skills/target-debug/SKILL.md` + +**Interfaces:** +- Produces: `python3 test/hil/hil_lock.py hold|release|status` — identical subcommands, flags, output, and exit codes to today's `board_lock.py`. + +- [ ] **Step 1: Move the CLI from `board_lock.py` into `hil_lock.py`** + +MOVE verbatim to the end of `hil_lock.py`: `boards_from_config`, `is_locked`, `cmd_hold`, `cmd_release`, `cmd_status`, `main()`, and the `if __name__ == '__main__':` guard. Mechanical adaptations only: +- `LOCK_DIR` → `BOARD_LOCK_DIR` (all sites), `lock_path` already exists (delete the duplicate), `read_info` → `read_record` (all sites; delete the duplicate definition) +- `cmd_hold`'s holder loop body (the open/flock/json.dump block) becomes `fh = flock_nb(b)` + `write_record(fh, reason)` inside the existing try/except OSError +- `_bow_out`'s per-handle truncate loop becomes `clear_record(h)` per handle +- `cmd_release`'s probe uses `flock_nb(b)` in a try/except OSError (held → existing record/victim logic, with the literal `'hil_test.py'` comparison becoming `CI_REASON`); the free-path truncate becomes `clear_record(fh)` +- `main()`'s module docstring reference for `--help` text: keep the usage lines, updating the tool name to `hil_lock.py` + +Then delete `test/hil/board_lock.py` (`git rm test/hil/board_lock.py`). + +- [ ] **Step 2: Rename `board_lock.py` → `hil_lock.py` in the six live docs** + +Run: `cd && sed -i 's/board_lock\.py/hil_lock.py/g' .claude/skills/hil/SKILL.md .claude/agents/hil-operator.md .claude/agents/target-debugger.md .claude/skills/etm-trace/SKILL.md .claude/skills/usb-kernel-recover/SKILL.md .claude/skills/target-debug/SKILL.md` +Then: `grep -rn 'board_lock' .claude/ test/ --include='*.md' --include='*.py' --include='*.sh'` +Expected: zero hits outside `docs/superpowers/` history (which stays untouched). + +- [ ] **Step 3: Verify CLI behavior end-to-end** + +```bash +python3 test/hil/hil_lock.py status # expect: no locks (or current holders) +python3 test/hil/hil_lock.py hold stm32f072disco --reason "split test" & +sleep 1 +python3 test/hil/hil_lock.py status # expect: stm32f072disco: {... 'reason': 'split test' ...} +python3 test/hil/hil_lock.py hold stm32f072disco --reason "rival" || echo "conflict OK" # expect: ERROR ... locked + conflict OK +python3 test/hil/hil_lock.py release stm32f072disco # expect: released holder pid NNN +python3 test/hil/hil_lock.py status # expect: no locks +``` + +Also verify CI-holder protection: create a fake record `echo '{"pid": 1, "reason": "hil_test.py"}' > /tmp/tinyusb-hil-locks/faketest.lock` — since pid 1 holds no flock, `release faketest` must clear the stale record without printing the mid-test error; then `rm -f /tmp/tinyusb-hil-locks/faketest.lock`. + +- [ ] **Step 4: Commit** + +```bash +git add -A test/hil .claude +git commit -m "hil: fold board_lock CLI into hil_lock.py, retire board_lock.py" +``` + +--- + +### Task 4: Rig verification + pre-commit + +**Files:** none new (fixes only if verification fails) + +- [ ] **Step 1: pool_check flash path on one board** + +Run: `python3 .claude/skills/hil/pool_check.py -b stm32f407disco` +Expected: `✅ dfu_runtime ✅ cafe:...`, exit 0. + +- [ ] **Step 2: Capture a pre-refactor baseline report** + +Run: `cd /home/hathach/code/tinyusb && python3 test/hil/hil_test.py -b stm32f407disco -B examples test/hil/tinyusb.json && cp hil_report.md /tmp/claude-1000/-home-hathach-code-tinyusb/*/scratchpad/hil_report_master.md` +(Primary checkout = pre-refactor code but same rig/config; its working tree already carries the new probe uids.) + +- [ ] **Step 3: Run the same board from the worktree and diff the report shape** + +Run: `cd .claude/worktrees/hil-test-split && python3 test/hil/hil_test.py -b stm32f407disco -B /home/hathach/code/tinyusb/examples test/hil/tinyusb.json && diff <(sed 's/[0-9.]*s//g;s/[0-9.]* [kMG]B\/s//g' hil_report.md) <(sed 's/[0-9.]*s//g;s/[0-9.]* [kMG]B\/s//g' /tmp/claude-1000/-home-hathach-code-tinyusb/*/scratchpad/hil_report_master.md)` +Expected: empty diff after stripping timings/speeds. Note: `-B` accepts the absolute path so the worktree run reuses the primary checkout's built firmware; `find_firmware` resolves `TINYUSB_ROOT/` and an absolute `-B` overrides relative rooting — if it does not (Path join semantics), instead symlink `ln -s /home/hathach/code/tinyusb/examples/cmake-build-stm32f407disco examples/cmake-build-stm32f407disco` in the worktree and use `-B examples`. + +- [ ] **Step 4: pre-commit + final grep hygiene** + +Run: `pre-commit run --files test/hil/hil_test.py test/hil/hil_lock.py test/hil/hil_flash.py test/hil/hil_ci.sh .claude/skills/hil/pool_check.py $(git diff --name-only HEAD~3 -- '*.md')` +Expected: all hooks pass. + +- [ ] **Step 5: Commit any verification fixes** + +```bash +git add -A && git commit -m "hil: post-split verification fixes" # only if Steps 1-4 required changes +``` diff --git a/docs/superpowers/specs/2026-07-28-hil-test-refactor-design.md b/docs/superpowers/specs/2026-07-28-hil-test-refactor-design.md new file mode 100644 index 000000000..3cd202d95 --- /dev/null +++ b/docs/superpowers/specs/2026-07-28-hil-test-refactor-design.md @@ -0,0 +1,141 @@ +# hil_test.py refactor: test core + infra helpers + +**Date:** 2026-07-28 +**Branch:** `claude/hil-test-split` (based on `claude/hil-pool-check`, which adds `pool_check.py`) + +## Motivation + +`test/hil/hil_test.py` is 2370 lines mixing five concerns: board-lock protocol, per-controller +scheduling permits, flash/reset backends, the actual per-example tests, and orchestration/report/CLI. +The lock protocol additionally exists in three copies (`hil_test.py`, `board_lock.py`, +`.claude/skills/hil/pool_check.py`), which has already produced drift (pool_check's copy lacks +hil_test's fail-open and error guards). Splitting the infrastructure out makes `hil_test.py` +test-focused and gives external tools (pool_check) one canonical import for locks, permits, and +flashing. + +## Goal / non-goals + +**Goal:** behavior-preserving code motion. `hil_test.py`'s CLI, arguments, output, report format, +and runtime behavior stay byte-identical. One deliberate user-visible change: the operator lock CLI +moves from `board_lock.py` to `hil_lock.py` (same subcommands, same behavior); `board_lock.py` is +deleted. + +**Non-goals (explicit follow-ups, not this change):** +- The 15 pool_check findings from the 2026-07-28 code review (exception isolation, park-on-failure, + espressif coverage, probe-recovery criterion, etc.). +- pool_check adopting `flash_permit` controller budgeting (enabled by this split). +- Any change to lock semantics, permit widths, flash behavior, or test logic. + +## Resulting layout (`test/hil/`) + +| File | ~Lines | Role | +|---|---|---| +| `hil_test.py` | 1600 | tests + orchestration + report + CLI (unchanged interface) | +| `hil_lock.py` (new) | 420 | board-lock protocol + controller permits + operator CLI | +| `hil_flash.py` (new) | 250 | `run_cmd` + flash/reset backends + `find_firmware` | +| `board_lock.py` | deleted | superseded by `hil_lock.py` | + +Import graph: `hil_test` → {`hil_lock`, `hil_flash`}; the helpers import nothing local (no cycles). +`pool_check.py` imports all three. + +## hil_lock.py + +Docstring states the scope: board locks + controller flash/battery permits; the CLI manages board +locks only (permits are in-process semaphores with no CLI meaning). + +**Flock core** (protocol defined once; moved from `board_lock.py`/`hil_test.py`): +- `BOARD_LOCK_DIR = '/tmp/tinyusb-hil-locks'`, `lock_path(board)` +- `CI_REASON = 'hil_test.py'` — the release-protected holder tag (release refuses to kill it) +- `flock_nb(board) -> fh` — `os.open(O_RDWR|O_CREAT, 0o666)` **without O_TRUNC** (a losing racer + must not wipe the winner's record), `fdopen('r+')`, `LOCK_EX|LOCK_NB`; raises `OSError` when held +- `write_record(fh, reason)` — truncate+seek+`json.dump({pid, reason, since})`+flush +- `clear_record(fh)` — truncate(0), swallow OSError (records stay truthful on release) +- `read_record(board) -> dict | None` — today's `board_lock.read_info` +- `acquire_board_lock(board, reason=CI_REASON) -> fh | None` — today's `hil_test.acquire_board_lock` + with a `reason` parameter: `HIL_NO_BOARD_LOCK=1` bypass, fail-open with warning on lock-dir + OSError, `RuntimeError` carrying holder info on conflict + +**Controller permits** (moved verbatim from `hil_test.py`): +- `FLASH_PARALLEL`, `USBTEST_PARALLEL`, `CONTROLLER_SLOTS` (env-overridable as today) +- `controller_of(uid)`, `controller_slot(pci)`, `controller_permit`, `flash_permit(uid)`, + `usbtest_permit(uid)` +- Per-worker globals (`usbtest_sems`, `flash_sems`, `controller_map`, `controller_meta`, + `controller_hints`) set by a new `init_scheduling(sems, fsems, cmap, cmeta, hints)` hook that + `hil_test.init_worker` calls from the Pool initializer. `controller_permit`'s PROFILE logging + calls back through a module-level `log = print`-style hook that `hil_test` points at `log_line` + during `init_scheduling` (keeps helpers free of hil_test imports). The `PROFILE` env flag + (`HIL_PROFILE=1`) is read independently in `hil_lock` at import, same derivation as today. + +**Operator CLI** (moved verbatim from `board_lock.py`): `hold`/`release`/`status` subcommands with +the daemon-holder machinery (double-fork, setsid, stdio detach, success pipe, SIGTERM bow-out), +release policy (probe the flock; protect `CI_REASON` holders; SIGTERM other recorded pids), +`is_locked` pid-liveness, `--all`/`--config` roster handling. The hold/release/status internals +switch to the flock-core helpers above; observable behavior unchanged. + +## hil_flash.py + +Moved verbatim from `hil_test.py`: +- `CMD_TIMEOUT` (env-overridable), `run_cmd(cmd, cwd, timeout)`, `cmd_stdout_text(out)` +- `OPENCOD_ADI_PATH`, `TINYUSB_ROOT` +- All backends: `flash_jlink`/`reset_jlink`, `flash_stlink`/`reset_stlink`, + `flash_stflash`/`reset_stflash`, `flash_openocd`/`reset_openocd`, + `flash_openocd_wch`/`reset_openocd_wch`, `flash_openocd_adi`/`reset_openocd_adi`, + `flash_wlink_rs`/`reset_wlink_rs`, `flash_esptool`/`reset_esptool`, + `flash_uniflash`/`reset_uniflash`, `flash_lm4flash`/`reset_lm4flash` +- `find_firmware(variant, example)` +- `get_serial_dev(id, vendor_str, product_str, ifnum)` — moves here (not hil_test) because + `flash_esptool` calls it; keeping it test-side would create a helper→hil_test import cycle. + Tests call `hil_flash.get_serial_dev`. +- Module globals `build_dir = 'cmake-build'` and `verbose = False`, set by callers exactly as the + `hil_test` globals are today (`hil_test.main` sets them from argparse; pool_check sets them + directly). `run_cmd`'s verbose echo reads `hil_flash.verbose`. + +Dispatch in callers stays string-based: `getattr(hil_flash, f'flash_{flasher["name"].lower()}')`. + +## hil_test.py (what remains) + +Config TypedDicts (`Board`, `FlasherCfg`, …), device-node lookup except `get_serial_dev` +(`get_disk_dev`, `get_hid_dev`, `get_alsa_capture_dev`, `open_serial_dev`, `serial_write_all`, +`read_disk_file`, `open_mtp_dev`, `get_printer_dev`/`open_printer_dev`), enum-timeout globals + +`wait_until`, +`log_line`/print-lock, `compact_output`, all `test_*` functions, test lists, `test_example`, +`build_board`, `test_board`, report rendering/accumulation, `main`. Call sites use explicit +module-qualified names (`hil_lock.flash_permit(...)`, `hil_flash.run_cmd(...)`) so provenance is +greppable; no `from … import *`-style mirroring. + +`init_worker` keeps its signature (Pool initargs unchanged) and forwards the scheduling state to +`hil_lock.init_scheduling(...)`. + +## Consumer updates (same commit) + +- **`.claude/skills/hil/pool_check.py`** — drop its private `lock_board`/`unlock_board` in favor of + `hil_lock.flock_nb` + `write_record(fh, 'pool_check')` (+ `clear_record` on release; deliberately NOT `acquire_board_lock`, whose HIL_NO_BOARD_LOCK bypass and fail-open behavior pool_check must not inherit); import + flashers/`find_firmware`/`get_serial_dev`/`cmd_stdout_text`/`TINYUSB_ROOT`/`build_dir` from + `hil_flash`; `BOARD_LOCK_DIR` references move to `hil_lock`. pool_check then imports **only** + `hil_lock` + `hil_flash` (no `hil_test`), so its `pymtp` stub shim is deleted — that shim existed + solely because importing `hil_test` pulls in libmtp. +- **`test/hil/hil_ci.sh`** — the scp list is currently `hil_test.py`, `pymtp.py`, `$CONFIG`; add + `hil_lock.py` and `hil_flash.py` (hil_test cannot even import without them). `board_lock.py` was + never in the list. +- **Docs rename `board_lock.py` → `hil_lock.py`** (live docs only): `.claude/skills/hil/SKILL.md`, + `.claude/agents/hil-operator.md`, `.claude/agents/target-debugger.md`, + `.claude/skills/etm-trace/SKILL.md`, `.claude/skills/usb-kernel-recover/SKILL.md`, + `.claude/skills/target-debug/SKILL.md`. Historical `docs/superpowers/{plans,specs}` stay as + records. +- **CI workflow** — untouched (invokes `hil_test.py` CLI only). + +## Verification + +1. `python3 -m py_compile` on all three modules + pool_check. +2. `hil_lock.py hold/status/release` interplay: hold, conflicting hold, status listing, release, + protection of a `CI_REASON` record, stale-record cleanup. +3. `pool_check.py --scan-only`, then a single flash board (e.g. `-b stm32f407disco`). +4. Full `hil_test.py -b stm32f407disco -B examples tinyusb.json` on the rig; compare the report + row and log shape against a pre-refactor run. +5. `pre-commit run` on all touched files. + +## Sequencing + +Lands on top of `claude/hil-pool-check`. After merge, fix the pool_check review findings as a +separate change on the new module boundaries, and update agent-memory references to +`board_lock.py`. diff --git a/test/hil/board_lock.py b/test/hil/board_lock.py deleted file mode 100755 index c35e13705..000000000 --- a/test/hil/board_lock.py +++ /dev/null @@ -1,254 +0,0 @@ -#!/usr/bin/env python3 -"""Per-board advisory locks for the HIL rig. - -Arbitrates board access between dev sessions and CI's hil_test.py without -stopping the actions-runner. Locks are kernel flocks: the kernel releases -them automatically when the holder process dies, and holders clear their -lock-file record on release so records stay truthful (/tmp also clears on -reboot). - -Usage: - board_lock.py hold BOARD [BOARD...] --reason TEXT - board_lock.py hold --all [--config CONFIG.json] --reason TEXT - board_lock.py release BOARD [BOARD...] | release --all - board_lock.py status - -A holder process holds ALL boards given in one `hold` call; releasing any of -them kills that holder and releases all of its boards. -""" -import argparse -import fcntl -import json -import os -import select -import signal -import sys -import time - -LOCK_DIR = '/tmp/tinyusb-hil-locks' - - -def lock_path(board: str) -> str: - return os.path.join(LOCK_DIR, f'{board}.lock') - - -def boards_from_config(config: str) -> list: - try: - with open(config) as f: - return [b['name'] for b in json.load(f)['boards']] - except (OSError, ValueError, KeyError) as e: - print(f'ERROR: cannot read board roster {config}: {e}', file=sys.stderr) - sys.exit(1) - - -def read_info(board: str): - try: - with open(lock_path(board)) as f: - return json.load(f) - except (OSError, ValueError): - return None - - -def is_locked(board: str) -> bool: - """True if the recorded holder process is still alive. - - Deliberately never touches the flock: even a momentary probe lock would - make a concurrent acquirer's LOCK_NB attempt fail spuriously. The flock - taken by acquirers themselves stays the only authority.""" - info = read_info(board) - pid = info.get('pid') if isinstance(info, dict) else None - if not isinstance(pid, int) or pid <= 0: - return False - try: - os.kill(pid, 0) - except ProcessLookupError: - return False - except PermissionError: - return True # alive but owned by another user (e.g. the CI runner) - return True - - -def cmd_hold(boards, reason): - os.makedirs(LOCK_DIR, exist_ok=True) - # No pre-check: the holder's own LOCK_NB flock is the only authority — a - # recorded pid may be stale or recycled (e.g. a live hil_test.py worker - # that already released this board's flock but not its record). - # The holder signals success through this pipe. A generic is_locked() - # poll would be fooled by a RIVAL invocation's flock — only the holder - # itself knows whether it won every board. - r_fd, w_fd = os.pipe() - pid = os.fork() - if pid > 0: - os.close(w_fd) - os.waitpid(pid, 0) # reap intermediate child - ready, _, _ = select.select([r_fd], [], [], 10) - ok = bool(ready) and os.read(r_fd, 1) == b'1' - os.close(r_fd) - if ok: - print(f'held: {", ".join(boards)}') - return 0 - for b in boards: - info = read_info(b) - if info: - print(f'ERROR: {b} locked: {info}', file=sys.stderr) - print('ERROR: holder failed to acquire locks', file=sys.stderr) - return 1 - # intermediate child: detach, then spawn the actual holder - os.setsid() - if os.fork() > 0: - os._exit(0) - # holder (grandchild): acquire all flocks, signal the parent, sleep until killed - os.close(r_fd) - # Keep the success pipe clear of fds 0-2: invoked with stdio closed, - # os.pipe() can land there and the dup2 loop below would clobber it. - if w_fd <= 2: - w_fd = fcntl.fcntl(w_fd, fcntl.F_DUPFD, 3) - # Detach stdio: a `hold` whose output is captured must see EOF when the - # front-end exits — the immortal holder must not keep that pipe open. - devnull = os.open(os.devnull, os.O_RDWR) - for std_fd in (0, 1, 2): - os.dup2(devnull, std_fd) - if devnull > 2: - os.close(devnull) - try: - handles = [] - for b in boards: - # O_RDWR without O_TRUNC: never truncate before the flock is - # held — a losing racer must not wipe the winner's holder info. - fd = os.open(lock_path(b), os.O_RDWR | os.O_CREAT, 0o666) - fh = os.fdopen(fd, 'r+') - fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) - fh.truncate(0) - fh.seek(0) - json.dump({'pid': os.getpid(), 'reason': reason, - 'since': time.strftime('%Y-%m-%dT%H:%M:%S%z')}, fh) - fh.flush() - handles.append(fh) - except OSError: - try: - os.write(w_fd, b'0') - except OSError: - pass - os._exit(1) # lost a race; parent reports the failure - os.write(w_fd, b'1') - os.close(w_fd) - - def _bow_out(*_): - # clear the records before dying so read_info/status stay truthful - # (the kernel drops the flocks themselves on exit either way) - for h in handles: - try: - h.truncate(0) - except OSError: - pass - os._exit(0) - - signal.signal(signal.SIGTERM, _bow_out) - while True: - signal.pause() - - -def cmd_release(boards): - rc = 0 - victims = set() - for b in boards: - try: - fd = os.open(lock_path(b), os.O_RDWR) - except OSError: - continue # no lock file (or another user's): nothing we can release - fh = os.fdopen(fd, 'r+') - try: - fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError: - # flock genuinely held — never SIGTERM on a mere pid record: the - # pid may be recycled, or a live worker that already moved on. - fh.close() - info = read_info(b) or {} - pid = info.get('pid') - if info.get('reason') == 'hil_test.py': - print(f'ERROR: {b} is mid-test by hil_test.py (pid {pid}) — not killing a ' - 'CI run; wait for it to finish', file=sys.stderr) - rc = 1 - elif isinstance(pid, int) and pid > 0: - victims.add(pid) - else: - print(f'ERROR: {b} is held but its record is unreadable', file=sys.stderr) - rc = 1 - continue - # flock was free: only a stale record remained — clear it - try: - fh.truncate(0) - except OSError: - pass - fh.close() - for holder in sorted(victims): - try: - os.kill(holder, signal.SIGTERM) - print(f'released holder pid {holder}') - except ProcessLookupError: - pass - except PermissionError: - print(f'ERROR: holder pid {holder} belongs to another user — cannot signal it', - file=sys.stderr) - rc = 1 - time.sleep(0.3) - still = [b for b in boards if is_locked(b)] - if still: - print(f'ERROR: still locked: {", ".join(still)}', file=sys.stderr) - return 1 - return rc - - -def cmd_status(): - if not os.path.isdir(LOCK_DIR): - print('no locks') - return 0 - any_locked = False - for fn in sorted(os.listdir(LOCK_DIR)): - if not fn.endswith('.lock'): - continue - b = fn[:-5] - if is_locked(b): - any_locked = True - print(f'{b}: {read_info(b)}') - if not any_locked: - print('no locks') - return 0 - - -def main(): - ap = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - sub = ap.add_subparsers(dest='cmd', required=True) - p_hold = sub.add_parser('hold') - p_hold.add_argument('boards', nargs='*') - p_hold.add_argument('--all', action='store_true') - p_hold.add_argument('--config', - default=os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'tinyusb.json'), - help='board roster JSON (default: tinyusb.json beside this script)') - p_hold.add_argument('--reason', required=True) - p_rel = sub.add_parser('release') - p_rel.add_argument('boards', nargs='*') - p_rel.add_argument('--all', action='store_true') - sub.add_parser('status') - a = ap.parse_args() - if a.cmd == 'hold': - boards = boards_from_config(a.config) if a.all else a.boards - if not boards: - ap.error('no boards given (name boards or use --all)') - sys.exit(cmd_hold(boards, a.reason)) - if a.cmd == 'release': - if a.all: - boards = ([fn[:-5] for fn in os.listdir(LOCK_DIR) if fn.endswith('.lock')] - if os.path.isdir(LOCK_DIR) else []) - else: - boards = a.boards - if not boards: - ap.error('no boards given (name boards or use --all)') - sys.exit(cmd_release(boards)) - sys.exit(cmd_status()) - - -if __name__ == '__main__': - main() diff --git a/test/hil/hil_ci.sh b/test/hil/hil_ci.sh index 3ec907979..3384b4e2e 100644 --- a/test/hil/hil_ci.sh +++ b/test/hil/hil_ci.sh @@ -44,15 +44,22 @@ echo "==> Setting up remote $REMOTE:$REMOTE_DIR" ssh "$REMOTE" bash -s -- "$REMOTE_DIR" <<'REMOTE' set -e rm -rf -- "$1" -mkdir -p -- "$1/test/hil" "$1/examples" +# .claude path: usbtest.py's HUNG recovery resolves usb_recover.sh relative to the +# staged repo root — without it, recovery ENOENTs and the wedge is left in place +mkdir -p -- "$1/test/hil" "$1/examples" "$1/.claude/skills/usb-kernel-recover/scripts" REMOTE # Copy HIL test script and config echo "==> Copying test scripts" scp -q "$ROOT_DIR/test/hil/hil_test.py" \ + "$ROOT_DIR/test/hil/hil_flash.py" \ + "$ROOT_DIR/test/hil/hil_lock.py" \ + "$ROOT_DIR/test/hil/usbtest.py" \ "$ROOT_DIR/test/hil/pymtp.py" \ "$CONFIG" \ "$REMOTE:$REMOTE_DIR/test/hil/" +scp -q "$ROOT_DIR/.claude/skills/usb-kernel-recover/scripts/usb_recover.sh" \ + "$REMOTE:$REMOTE_DIR/.claude/skills/usb-kernel-recover/scripts/" # Copy only firmware binaries (elf/bin/hex) plus esptool metadata # (config.env + flash_args needed by the esptool flasher), preserving structure diff --git a/test/hil/hil_flash.py b/test/hil/hil_flash.py new file mode 100755 index 000000000..814258072 --- /dev/null +++ b/test/hil/hil_flash.py @@ -0,0 +1,293 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Firmware flashing for the TinyUSB HIL rig: run_cmd, one flash_*/reset_* pair per +# flasher type (dispatched by config name via getattr), find_firmware, and the +# fixture serial-port resolver get_serial_dev (here, not hil_test: flash_esptool +# needs it and helpers must not import hil_test). +# Callers set module globals `build_dir` and `verbose` (hil_test.main from argparse, +# pool_check directly) exactly as they set hil_test's globals today. +# +# from __future__ import annotations (below): some moved function signatures use +# type hints (Any, Board) not defined in this module; postponed evaluation (PEP +# 563) keeps those as unevaluated strings so the verbatim-moved defs still load. + +from __future__ import annotations + +import glob +import json +import os +import signal +import subprocess +from pathlib import Path + +verbose = False +build_dir = 'cmake-build' + +CMD_TIMEOUT = int(os.getenv('HIL_CMD_TIMEOUT', '180')) + +# flasher names (dispatch key, board['flasher']['name'].lower()) whose reset_* is a no-op +RESET_NOOP = {'esptool', 'lm4flash', 'stflash', 'uniflash'} + +# extra parents find_firmware ALSO searches after build_dir. Empty by default so +# hil_test's -B stays authoritative (a board missing there must report "Skip (no +# binary)", never silently flash a stale binary from another tree); pool_check +# opts in to cover both standard layouts. +EXTRA_BUILD_DIRS: list = [] + + +def cmd_stdout_text(out: Any) -> str: + if out is None: + return '' + if isinstance(out, bytes): + return out.decode('utf-8', errors='ignore') + return str(out) + + +# ------------------------------------------------------------- +# Path +# ------------------------------------------------------------- +OPENCOD_ADI_PATH = Path.home() / 'app' / 'openocd_adi' +TINYUSB_ROOT = Path(__file__).resolve().parents[2] + +# get usb serial by id +def get_serial_dev(id, vendor_str, product_str, ifnum): + if vendor_str and product_str: + # known vendor and product + vendor_str = vendor_str.replace(' ', '_') + product_str = product_str.replace(' ', '_') + return f'/dev/serial/by-id/usb-{vendor_str}_{product_str}_{id}-if{ifnum:02d}' + else: + # just use id: mostly for cp210x/ftdi flasher + pattern = f'/dev/serial/by-id/usb-*_{id}-if*' + port_list = glob.glob(pattern) + if len(port_list) == 0: + raise RuntimeError(f'No serial device found for {pattern}') + return port_list[0] + + +# ------------------------------------------------------------- +# Flashing firmware +# ------------------------------------------------------------- +def run_cmd(cmd: str, cwd: str | None = None, timeout: int = CMD_TIMEOUT) -> subprocess.CompletedProcess: + popen_kwargs = { + 'cwd': cwd, + 'shell': True, + 'stdout': subprocess.PIPE, + 'stderr': subprocess.STDOUT, + 'text': True, + 'encoding': 'utf-8', + 'errors': 'replace', + } + if os.name != 'nt': + # C-level setsid, same process-group semantics as preexec_fn=os.setsid but + # safe when called from threads (pool_check runs flashes from a thread pool) + popen_kwargs['start_new_session'] = True + + p = subprocess.Popen(cmd, **popen_kwargs) + try: + out, _ = p.communicate(timeout=timeout) + r = subprocess.CompletedProcess(args=cmd, returncode=p.returncode, stdout=out) + except subprocess.TimeoutExpired as ex: + if os.name != 'nt': + try: + os.killpg(p.pid, signal.SIGKILL) + except ProcessLookupError: + pass + else: + p.kill() + try: + out, _ = p.communicate(timeout=10) + except subprocess.TimeoutExpired: # unkillable (e.g. D-state on wedged USB) + out = None + timeout_out = ex.stdout or out or b'' + title = f'COMMAND TIMEOUT ({timeout}s): {cmd}' + print() + if os.getenv('CI'): + print(f"::group::{title}") + print(cmd_stdout_text(timeout_out)) + print(f"::endgroup::") + else: + print(title) + print(cmd_stdout_text(timeout_out)) + return subprocess.CompletedProcess(args=cmd, returncode=124, stdout=timeout_out) + + if r.returncode != 0: + title = f'COMMAND FAILED: {cmd}' + print() + if os.getenv('CI'): + print(f"::group::{title}") + print(cmd_stdout_text(r.stdout)) + print(f"::endgroup::") + else: + print(title) + print(cmd_stdout_text(r.stdout)) + elif verbose: + print(cmd) + print(cmd_stdout_text(r.stdout)) + return r + + +def flash_jlink(board: Board, firmware: str) -> subprocess.CompletedProcess: + flasher = board['flasher'] + script = ['halt', 'r', f'loadfile {firmware}.elf', 'r', 'go', 'exit'] + f_jlink = Path(f'{board["name"]}_{Path(firmware).name}.jlink') + with f_jlink.open('w') as f: + f.writelines(f'{s}\n' for s in script) + ret = run_cmd(f'JLinkExe -USB {flasher["uid"]} {flasher["args"]} -if swd -JTAGConf -1,-1 -speed auto -NoGui 1 -ExitOnError 1 -CommandFile {f_jlink}') + f_jlink.unlink(missing_ok=True) + return ret + + +def reset_jlink(board: Board) -> subprocess.CompletedProcess: + flasher = board['flasher'] + script = ['halt', 'r', 'go', 'exit'] + f_jlink = Path(f'{board["name"]}_reset.jlink') + if not f_jlink.exists(): + with f_jlink.open('w') as f: + f.writelines(f'{s}\n' for s in script) + ret = run_cmd(f'JLinkExe -USB {flasher["uid"]} {flasher["args"]} -if swd -JTAGConf -1,-1 -speed auto -NoGui 1 -ExitOnError 1 -CommandFile {f_jlink}') + return ret + + +def flash_stlink(board, firmware): + flasher = board['flasher'] + return run_cmd(f'STM32_Programmer_CLI --connect port=swd sn={flasher["uid"]} --write {firmware}.elf --go') + + +def reset_stlink(board): + flasher = board['flasher'] + return run_cmd(f'STM32_Programmer_CLI --connect port=swd sn={flasher["uid"]} --rst --go') + +def flash_stflash(board, firmware): + flasher = board['flasher'] + ret = run_cmd(f'st-flash --serial {flasher["uid"]} write {firmware}.bin 0x8000000') + return ret + + +def reset_stflash(board): + flasher = board['flasher'] + return subprocess.CompletedProcess(args=['dummy'], returncode=0) + + +def flash_openocd(board, firmware): + flasher = board['flasher'] + ret = run_cmd(f'openocd -c "tcl_port disabled" -c "gdb_port disabled" -c "adapter serial {flasher["uid"]}" ' + f'{flasher["args"]} -c "init; halt; program {firmware}.elf verify; reset; exit"') + return ret + + +def reset_openocd(board): + flasher = board['flasher'] + ret = run_cmd(f'openocd -c "tcl_port disabled" -c "gdb_port disabled" -c "adapter serial {flasher["uid"]}" ' + f'{flasher["args"]} -c "init; reset run; exit"') + return ret + + +def flash_openocd_wch(board, firmware): + flasher = board['flasher'] + ret = run_cmd(f'openocd -c "tcl_port disabled" -c "gdb_port disabled" -c "telnet_port disabled" ' + f'-c "adapter serial {flasher["uid"]}" {flasher.get("args", "")} -c "program {firmware}.elf reset exit"') + return ret + + +def reset_openocd_wch(board): + flasher = board['flasher'] + ret = run_cmd(f'openocd -c "tcl_port disabled" -c "gdb_port disabled" -c "telnet_port disabled" ' + f'-c "adapter serial {flasher["uid"]}" {flasher.get("args", "")} -c "init; reset run; exit"') + return ret + + +def flash_openocd_adi(board: Board, firmware: str) -> subprocess.CompletedProcess: + flasher = board['flasher'] + openocd = OPENCOD_ADI_PATH / 'src' / 'openocd' + tcl_dir = OPENCOD_ADI_PATH / 'tcl' + ret = run_cmd(f'{openocd} -c "adapter serial {flasher["uid"]}" -s {tcl_dir} ' + f'{flasher["args"]} -c "program {firmware}.elf reset exit"') + return ret + + +def reset_openocd_adi(board: Board) -> subprocess.CompletedProcess: + flasher = board['flasher'] + openocd = OPENCOD_ADI_PATH / 'src' / 'openocd' + tcl_dir = OPENCOD_ADI_PATH / 'tcl' + ret = run_cmd(f'{openocd} -c "adapter serial {flasher["uid"]}" -s {tcl_dir} ' + f'{flasher["args"]} -c "program reset exit"') + return ret + + +def flash_wlink_rs(board, firmware): + flasher = board['flasher'] + # wlink use index for probe selection and lacking usb serial support + ret = run_cmd(f'wlink flash {firmware}.elf') + return ret + + +def reset_wlink_rs(board): + flasher = board['flasher'] + # wlink use index for probe selection and lacking usb serial support + ret = run_cmd(f'wlink reset') + return ret + + +def flash_esptool(board: Board, firmware: str) -> subprocess.CompletedProcess: + flasher = board['flasher'] + port = get_serial_dev(flasher["uid"], None, None, 0) + fw_dir = Path(f'{firmware}.bin').parent + with (fw_dir / 'config.env').open() as f: + idf_target = json.load(f)['IDF_TARGET'] + with (fw_dir / 'flash_args').open() as f: + flash_args = f.read().strip().replace('\n', ' ') + command = (f'esptool --chip {idf_target} -p {port} {flasher["args"]} ' + f'--before=default_reset --after=hard_reset write_flash {flash_args}') + ret = run_cmd(command, cwd=str(fw_dir)) + return ret + + +def reset_esptool(board): + flasher = board['flasher'] + return subprocess.CompletedProcess(args=['dummy'], returncode=0) + + +def flash_uniflash(board, firmware): + flasher = board['flasher'] + ret = run_cmd(f'dslite.sh {flasher["args"]} -f {firmware}.hex') + return ret + + +def reset_uniflash(board): + flasher = board['flasher'] + return subprocess.CompletedProcess(args=['dummy'], returncode=0) + + +def flash_lm4flash(board, firmware): + # TI Tiva-C / Stellaris ICDI: lightweight lm4flash, resets and runs after write + flasher = board['flasher'] + ret = run_cmd(f'lm4flash -s {flasher["uid"]} {flasher["args"]} {firmware}.bin') + return ret + + +def reset_lm4flash(board): + # lm4flash has no reset-only mode; it resets+runs on flash, so reset is a no-op + flasher = board['flasher'] + return subprocess.CompletedProcess(args=['dummy'], returncode=0) + + +def find_firmware(variant: str, example: str, roots: list | None = None): + """Locate a built example's firmware base path (no extension) under + /cmake-build-//, then under EXTRA_BUILD_DIRS + (empty unless the caller opts in — see its comment). `roots` overrides that + search list entirely for one call (e.g. to find a build just produced by + tools/build.py in its fixed cmake-build/ layout without widening the global + policy). Accepts the single-config layout (firmware directly in the example + dir) or Ninja Multi-Config (a per-config subdir like RelWithDebInfo/). + Returns the base Path, or None if not built.""" + base = Path(example).name + for bd in dict.fromkeys(roots if roots is not None else [build_dir, *EXTRA_BUILD_DIRS]): + fw_dir = TINYUSB_ROOT / bd / f'cmake-build-{variant}' / example + if not fw_dir.is_dir(): + continue + for cand in [fw_dir / base, fw_dir / 'RelWithDebInfo' / base, + *(p.with_suffix('') for p in sorted(fw_dir.glob(f'*/{base}.elf')))]: + if cand.with_suffix('.elf').exists() or cand.with_suffix('.bin').exists(): + return cand + return None diff --git a/test/hil/hil_lock.py b/test/hil/hil_lock.py new file mode 100755 index 000000000..e570da16a --- /dev/null +++ b/test/hil/hil_lock.py @@ -0,0 +1,479 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Board locks + controller permits for the TinyUSB HIL rig. + +Board locks are kernel flocks in BOARD_LOCK_DIR arbitrating hardware access +between dev sessions and CI's hil_test.py (never stop the actions-runner). +Controller permits are in-process semaphores budgeting flashes and usbtest +batteries per host controller; they have no CLI meaning. The CLI below +(hold/release/status) manages board locks only. +""" +import argparse +import fcntl +import glob +import json +import os +import re +import select +import signal +import sys +import time + +BOARD_LOCK_DIR = '/tmp/tinyusb-hil-locks' +CI_REASON = 'hil_test.py' # release-protected holder tag (release refuses to kill it) +PROTECTED_REASONS = {CI_REASON, 'pool_check'} # cmd_release refuses to SIGTERM these holders +PROFILE = os.environ.get('HIL_PROFILE') == '1' + + +def lock_path(board: str) -> str: + return os.path.join(BOARD_LOCK_DIR, f'{board}.lock') + + +def flock_nb(board: str): + """Open-or-create the lock file WITHOUT truncating (a losing racer must not + wipe the winner's record) and take LOCK_EX|LOCK_NB. Returns the open handle; + raises OSError when the flock is held elsewhere (handle already closed).""" + fd = os.open(lock_path(board), os.O_RDWR | os.O_CREAT, 0o666) + fh = os.fdopen(fd, 'r+') + try: + fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + fh.close() + raise + return fh + + +def write_record(fh, reason: str) -> bool: + """Holder record; the flock itself is already held. Returns False on a write + failure — acquire_board_lock stays best-effort (the flock is the authority), + but cmd_hold aborts on it like board_lock.py did (a hold whose record is + missing is invisible to status/release).""" + try: + fh.truncate(0) + fh.seek(0) + json.dump({'pid': os.getpid(), 'reason': reason, + 'since': time.strftime('%Y-%m-%dT%H:%M:%S%z')}, fh) + fh.flush() + return True + except OSError: + return False + + +def clear_record(fh) -> None: + """Clear our record before dropping the flock so records stay truthful.""" + try: + fh.truncate(0) + except OSError: + pass + + +def read_record(board: str): + try: + with open(lock_path(board)) as f: + return json.load(f) + except (OSError, ValueError): + return None + + +# --- per-board dev-session locks ------------------------------------------ +def acquire_board_lock(board_name, reason=CI_REASON): + """Take this board's flock for the duration of its flash+test. + Returns an open file handle (keep it referenced; closing releases it), + or None when HIL_NO_BOARD_LOCK=1 or the lock dir is unusable (fail-open: + locking must never break a test run by itself). + Raises RuntimeError only when another session holds the board.""" + import fcntl + if os.environ.get('HIL_NO_BOARD_LOCK') == '1': + return None # user-authorized bypass — see hil skill + try: + os.makedirs(BOARD_LOCK_DIR, exist_ok=True) + fd = os.open(os.path.join(BOARD_LOCK_DIR, f'{board_name}.lock'), + os.O_RDWR | os.O_CREAT, 0o666) + fh = os.fdopen(fd, 'r+') + except OSError as e: + # odd lock dir (perms, path collision): proceed unlocked, but say so — + # a silent fail-open is indistinguishable from the intentional bypass + print(f'warning: board lock unavailable for {board_name} ({e}); proceeding unlocked', + flush=True) + return None + try: + fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + try: + info = fh.read(500).strip() + except (OSError, UnicodeDecodeError): + info = '' + fh.close() + raise RuntimeError(f'board locked: {info or "unknown holder"}') + # announce ourselves so the other side's conflict message is truthful; + # best-effort — the flock itself is already held + try: + fh.truncate(0) + fh.seek(0) + json.dump({'pid': os.getpid(), 'reason': reason, + 'since': time.strftime('%Y-%m-%dT%H:%M:%S%z')}, fh) + fh.flush() + except OSError: + pass + return fh + + +# Per-host-controller concurrency (see controller_of/controller_slot below): a usbtest battery +# saturates its DUT's host controller, so batteries and flashes are budgeted per controller. +# - uPD720201 cards need their latest firmware (>= 2.0.2.6; RAM-uploaded, reloads every +# power cycle): ROM firmware dies under battery + re-enumeration churn, and usbtest.py +# refuses the unlink-stress cases on it. +# - widths (profiled 2026-07-13/14): wall time 22.2/14.3/12.5/10.8 min at usbtest width +# 1/2/3/4, plateau after; flash width beyond 8 only adds flasher-hub contention; +# battery case failures start at 12/8 (bandwidth stretch on shared leaf-hub uplinks). +# - a marginal DUT port bouncing during concurrent batteries can wedge/kill a uPD720201 +# ("xHCI host not responding to stop endpoint command"): fix the port/cable or pull +# the board, don't lower the widths (2026-07-16: every death traced to one board's port). +FLASH_PARALLEL = int(os.getenv('HIL_FLASH_PARALLEL', '8')) +USBTEST_PARALLEL = int(os.getenv('HIL_USBTEST_PARALLEL', '4')) +CONTROLLER_SLOTS = 12 # lock slots; controllers are assigned to slots on first sight +usbtest_sems = None # CONTROLLER_SLOTS semaphores: per-slot usbtest-battery permits +flash_sems = None # CONTROLLER_SLOTS semaphores: per-slot flash permits +controller_map = None # shared dict: 'pci:' -> slot, 'uid:' -> pci addr cache +controller_meta = None # guards slot assignment in controller_map +controller_hints = {} # static uid -> pci from the last run's cache (read-only per worker) + + +log = print # hil_test.init_worker points this at log_line via init_scheduling + + +def init_scheduling(b_sems, f_sems, cmap, cmeta, hints, log_fn=None): + """Install per-worker scheduling state (called from hil_test.init_worker).""" + global usbtest_sems, flash_sems, controller_map, controller_meta, controller_hints, log + usbtest_sems, flash_sems = b_sems, f_sems + controller_map, controller_meta, controller_hints = cmap, cmeta, hints + if log_fn is not None: + log = log_fn + + +# ------------------------------------------------------------- +# Per-controller scheduling +# ------------------------------------------------------------- +def controller_of(uid: str): + """Resolve a DUT uid to its root host controller's PCI address, or None if the device + is not enumerated (e.g. parked in board_test firmware with USB off). Successful + resolutions are cached — cabling does not change mid-run. Dual-port parts (e.g. + CH32V307 usbhs/usbfs variants) share one uid and one cache entry: budgeting is only + exact when both ports sit on the same controller (true on this rig).""" + if controller_map is None: + return None + cached = controller_map.get(f'uid:{uid}') + if cached: + return cached + for f in glob.glob('/sys/bus/usb/devices/*/serial'): + d = os.path.dirname(f) + try: + if open(f).read().strip().lower() != uid.lower(): + continue + bus = int(open(os.path.join(d, 'busnum')).read()) + root = os.path.realpath(f'/sys/bus/usb/devices/usb{bus}') + m = re.findall(r'[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]', root) + if m: + controller_map[f'uid:{uid}'] = m[-1] + return m[-1] + except (OSError, ValueError): + continue + return None + + +def controller_slot(pci: str) -> int: + """Map a controller PCI address to a lock slot (assigned on first sight).""" + key = f'pci:{pci}' + with controller_meta: + slot = controller_map.get(key) + if slot is None: + slot = controller_map.get('nslots', 0) + if slot >= CONTROLLER_SLOTS: + slot = 0 # more controllers than slots: overflow shares slot 0 (safe, over-serialized) + else: + controller_map['nslots'] = slot + 1 + controller_map[key] = slot + return slot + + +class controller_permit: + """Context manager: one permit from `sems` on the board's controller slot. If the + controller is unknown, fail closed: take one permit from EVERY slot, in order, so the + operation respects the budget wherever it might land. `warn_unknown` logs that fallback + (used by usbtest, where the device is expected to be enumerated by the caller).""" + def __init__(self, sems, uid: str, warn_unknown: bool = False): + self.sems = sems + self.slots = None + self.uid = uid + if sems is None: + return + pci = controller_of(uid) + if pci is None and not warn_unknown: + # last-run cabling hint, flash budgeting only: a mis-budgeted flash is harmless, + # but a battery must never trust a stale hint (it could stack two batteries on + # one controller). In practice only a board's first flash lands here - batteries + # assert enumeration before taking their permit. + pci = controller_hints.get(uid) + if pci is None and warn_unknown: + log(f'warning: cannot resolve {uid} to a host controller; ' + 'taking a permit on every slot (over-serialized)') + self.slots = [controller_slot(pci)] if pci else list(range(CONTROLLER_SLOTS)) + + def __enter__(self): + if self.slots: + t0 = time.monotonic() + taken = [] + try: + for s in self.slots: + self.sems[s].acquire() + taken.append(s) + # stays inside the try: if this raises (e.g. broken stdout), the permits + # must be released - a failed __enter__ never gets its __exit__ + if PROFILE and time.monotonic() - t0 > 1.0: + log(f'[prof] permit wait {time.monotonic() - t0:.1f}s ' + f'(uid {self.uid}, slots {self.slots})') + except BaseException: + for s in reversed(taken): + self.sems[s].release() + raise + return self + + def __exit__(self, *exc): + if self.slots: + for s in reversed(self.slots): + self.sems[s].release() + return False + + +def flash_permit(uid: str) -> controller_permit: + return controller_permit(flash_sems, uid) + + +def usbtest_permit(uid: str) -> controller_permit: + return controller_permit(usbtest_sems, uid, warn_unknown=True) + + +# --- operator CLI (hold/release/status) ------------------------------------ +def boards_from_config(config: str) -> list: + """All board names, INCLUDING boards-skip: `hold --all` guards rig-wide + operations, and parked boards can still be touched (pool_check -b names them + explicitly), so a rig-wide hold that skipped them would leave a gap.""" + try: + with open(config) as f: + cfg = json.load(f) + return [b['name'] for b in cfg['boards'] + cfg.get('boards-skip', [])] + except (OSError, ValueError, KeyError) as e: + print(f'ERROR: cannot read board roster {config}: {e}', file=sys.stderr) + sys.exit(1) + + +def is_locked(board: str) -> bool: + """True if the recorded holder process is still alive. + + Deliberately never touches the flock: even a momentary probe lock would + make a concurrent acquirer's LOCK_NB attempt fail spuriously. The flock + taken by acquirers themselves stays the only authority.""" + info = read_record(board) + pid = info.get('pid') if isinstance(info, dict) else None + if not isinstance(pid, int) or pid <= 0: + return False + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True # alive but owned by another user (e.g. the CI runner) + return True + + +def cmd_hold(boards, reason): + os.makedirs(BOARD_LOCK_DIR, exist_ok=True) + # No pre-check: the holder's own LOCK_NB flock is the only authority — a + # recorded pid may be stale or recycled (e.g. a live hil_test.py worker + # that already released this board's flock but not its record). + # The holder signals success through this pipe. A generic is_locked() + # poll would be fooled by a RIVAL invocation's flock — only the holder + # itself knows whether it won every board. + r_fd, w_fd = os.pipe() + pid = os.fork() + if pid > 0: + os.close(w_fd) + os.waitpid(pid, 0) # reap intermediate child + ready, _, _ = select.select([r_fd], [], [], 10) + ok = bool(ready) and os.read(r_fd, 1) == b'1' + os.close(r_fd) + if ok: + print(f'held: {", ".join(boards)}') + return 0 + for b in boards: + info = read_record(b) + if info: + print(f'ERROR: {b} locked: {info}', file=sys.stderr) + print('ERROR: holder failed to acquire locks', file=sys.stderr) + return 1 + # intermediate child: detach, then spawn the actual holder + os.setsid() + if os.fork() > 0: + os._exit(0) + # holder (grandchild): acquire all flocks, signal the parent, sleep until killed + os.close(r_fd) + # Keep the success pipe clear of fds 0-2: invoked with stdio closed, + # os.pipe() can land there and the dup2 loop below would clobber it. + if w_fd <= 2: + w_fd = fcntl.fcntl(w_fd, fcntl.F_DUPFD, 3) + # Detach stdio: a `hold` whose output is captured must see EOF when the + # front-end exits — the immortal holder must not keep that pipe open. + devnull = os.open(os.devnull, os.O_RDWR) + for std_fd in (0, 1, 2): + os.dup2(devnull, std_fd) + if devnull > 2: + os.close(devnull) + try: + handles = [] + for b in boards: + fh = flock_nb(b) + if not write_record(fh, reason): + raise OSError(f'cannot write holder record for {b}') + handles.append(fh) + except OSError: + try: + os.write(w_fd, b'0') + except OSError: + pass + os._exit(1) # lost a race; parent reports the failure + os.write(w_fd, b'1') + os.close(w_fd) + + def _bow_out(*_): + # clear the records before dying so read_record/status stay truthful + # (the kernel drops the flocks themselves on exit either way) + for h in handles: + clear_record(h) + os._exit(0) + + signal.signal(signal.SIGTERM, _bow_out) + while True: + signal.pause() + + +def cmd_release(boards): + rc = 0 + victims = set() + for b in boards: + try: + fd = os.open(lock_path(b), os.O_RDWR) + except OSError: + continue # no lock file (or another user's): nothing we can release + fh = os.fdopen(fd, 'r+') + try: + fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + # flock genuinely held — never SIGTERM on a mere pid record: the + # pid may be recycled, or a live worker that already moved on. + fh.close() + info = read_record(b) or {} + pid = info.get('pid') + reason = info.get('reason') + if reason in PROTECTED_REASONS: + print(f'ERROR: {b} is mid-test by {reason} (pid {pid}) — not killing it; ' + 'wait for it to finish', file=sys.stderr) + rc = 1 + elif isinstance(pid, int) and pid > 0: + victims.add(pid) + else: + print(f'ERROR: {b} is held but its record is unreadable', file=sys.stderr) + rc = 1 + continue + # flock was free: only a stale record remained — clear it + clear_record(fh) + fh.close() + for holder in sorted(victims): + try: + os.kill(holder, signal.SIGTERM) + print(f'released holder pid {holder}') + except ProcessLookupError: + pass + except PermissionError: + print(f'ERROR: holder pid {holder} belongs to another user — cannot signal it', + file=sys.stderr) + rc = 1 + time.sleep(0.3) + still = [b for b in boards if is_locked(b)] + if still: + print(f'ERROR: still locked: {", ".join(still)}', file=sys.stderr) + return 1 + return rc + + +def cmd_status(): + if not os.path.isdir(BOARD_LOCK_DIR): + print('no locks') + return 0 + any_locked = False + for fn in sorted(os.listdir(BOARD_LOCK_DIR)): + if not fn.endswith('.lock'): + continue + b = fn[:-5] + if is_locked(b): + any_locked = True + print(f'{b}: {read_record(b)}') + if not any_locked: + print('no locks') + return 0 + + +_CLI_USAGE = """Per-board advisory locks for the HIL rig. + +Arbitrates board access between dev sessions and CI's hil_test.py without +stopping the actions-runner. Locks are kernel flocks: the kernel releases +them automatically when the holder process dies, and holders clear their +lock-file record on release so records stay truthful (/tmp also clears on +reboot). + +Usage: + hil_lock.py hold BOARD [BOARD...] --reason TEXT + hil_lock.py hold --all [--config CONFIG.json] --reason TEXT + hil_lock.py release BOARD [BOARD...] | release --all + hil_lock.py status + +A holder process holds ALL boards given in one `hold` call; releasing any of +them kills that holder and releases all of its boards. +""" + + +def main(): + ap = argparse.ArgumentParser(description=_CLI_USAGE, + formatter_class=argparse.RawDescriptionHelpFormatter) + sub = ap.add_subparsers(dest='cmd', required=True) + p_hold = sub.add_parser('hold') + p_hold.add_argument('boards', nargs='*') + p_hold.add_argument('--all', action='store_true') + p_hold.add_argument('--config', + default=os.path.join(os.path.dirname(os.path.abspath(__file__)), + 'tinyusb.json'), + help='board roster JSON (default: tinyusb.json beside this script)') + p_hold.add_argument('--reason', required=True) + p_rel = sub.add_parser('release') + p_rel.add_argument('boards', nargs='*') + p_rel.add_argument('--all', action='store_true') + sub.add_parser('status') + a = ap.parse_args() + if a.cmd == 'hold': + boards = boards_from_config(a.config) if a.all else a.boards + if not boards: + ap.error('no boards given (name boards or use --all)') + sys.exit(cmd_hold(boards, a.reason)) + if a.cmd == 'release': + if a.all: + boards = ([fn[:-5] for fn in os.listdir(BOARD_LOCK_DIR) if fn.endswith('.lock')] + if os.path.isdir(BOARD_LOCK_DIR) else []) + else: + boards = a.boards + if not boards: + ap.error('no boards given (name boards or use --all)') + sys.exit(cmd_release(boards)) + sys.exit(cmd_status()) + + +if __name__ == '__main__': + main() diff --git a/test/hil/hil_pool_check.py b/test/hil/hil_pool_check.py new file mode 100644 index 000000000..63284213e --- /dev/null +++ b/test/hil/hil_pool_check.py @@ -0,0 +1,1013 @@ +#!/usr/bin/env python3 +"""Quick HIL pool health check. + +For every board in the rig's HIL config: is the flash probe on the USB bus, does a +light example flash, and does the board's USB device (uid) come back up? Missing +firmware is BUILT on the spot (tools/build.py, idf.py for espressif; one get_deps +retry) — never skipped; --no-build opts out. Applies only per-device-safe recovery +(probe authorized-toggle, board reset/re-flash) and prints a markdown summary +table. Row statuses: ok (flashed and verified; under --scan-only: probe present — +the scan checks presence only), flash-failed (firmware delivery failed: probe +missing, build failed, flasher error, silent flash no-op, park not verified), +failed (the check ran but did not verify: flashed with no enumeration/serial, or +the check itself errored), locked (board flock held by another process — +reported, never waited on or bypassed). + +Config is picked by hostname unless given: ci -> tinyusb.json, tusb (hifiphile +rig) -> hfp.json, anything else is a dev PC -> local.json. + +Lives in test/hil/ beside hil_lock.py and hil_flash.py, which it imports; board +recovery uses the repo's .claude/skills/usb-kernel-recover/scripts/usb_recover.sh. +""" + +import argparse +import io +import json +import glob +import os +import re +import shlex +import shutil +import socket +import subprocess +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(Path(__file__).resolve().parent)) # for import-as-module callers + +import hil_lock +import hil_flash + +USB_RECOVER = REPO_ROOT / '.claude' / 'skills' / 'usb-kernel-recover' / 'scripts' / 'usb_recover.sh' +SEEN_CACHE = Path.home() / '.cache' / 'tinyusb-hil' / 'pool_seen.json' +CONFIG_BY_HOST = {'ci': 'tinyusb.json', 'tusb': 'hfp.json'} # anything else: dev PC -> local.json + +# light-example preference; first built wins +DEVICE_CANDIDATES = ['device/dfu_runtime', 'device/cdc_msc', 'device/cdc_msc_freertos', + 'device/hid_composite_freertos', 'device/cdc_dual_ports'] +HOST_CANDIDATES = ['host/device_info', 'host/cdc_msc_hid', 'host/msc_file_explorer_freertos'] + +ENUM_WAIT = 12 # s, uid wait after flash +ENUM_WAIT_RETRY = 8 # s, uid wait after a recovery reset/re-flash +SERIAL_WAIT = 6 # s, host-board serial-output wait + +print_mutex = threading.Lock() +t0 = time.monotonic() + + +def say(msg: str) -> None: + with print_mutex: + print(f'[{time.monotonic() - t0:6.1f}s] {msg}', file=sys.__stdout__, flush=True) + + +def scan_usb() -> dict: + """busport -> {'serial', 'vidpid', 'ino'} for every enumerated USB device. Only + -[....] dirs match (root hubs, named 'usbN' with no dash, are + excluded: their fabricated PCI-address 'serial' and slow autosuspend-wake read + cost 6-7s/scan on this rig). Keyed by busport, not serial: a serial can be + shared by two different devices (e.g. an Espressif USB-Serial-JTAG bridge and + the cafe TinyUSB device it flashes derive both from the same MAC) — collapsing + them into one dict slot would silently drop whichever lost the race.""" + found = {} + for f in glob.glob('/sys/bus/usb/devices/*-*/serial'): + d = os.path.dirname(f) + busport = os.path.basename(d) + try: + sn = open(f).read().strip().lower() + vidpid = f'{open(d + "/idVendor").read().strip()}:{open(d + "/idProduct").read().strip()}' + found[busport] = {'serial': sn, 'vidpid': vidpid, 'ino': os.stat(d + '/').st_ino} + except OSError: + continue + return found + + +def find_usb(uid: str, devs: dict | None = None): + """Locate a flasher probe by uid, excluding VID cafe (TinyUSB DUT firmware): a + probe's uid can coincidentally equal its DUT's (Espressif USB-Serial-JTAG + bridges derive both from the same MAC), and the DUT is never the probe. + + J-Link zero-pads numeric serials (681295394 -> 000681295394): an all-digit uid + matches an all-digit serial only when that serial equals the uid zero-padded to + the serial's own length (leading zeros only) — never when the zero-stripped uid + is empty, so a placeholder serial (metro_m4_express's probe legitimately reports + '123456') can't be mistaken for an unrelated device.""" + devs = devs if devs is not None else scan_usb() + u = uid.lower() + candidates = [(bp, dev) for bp, dev in devs.items() if not dev['vidpid'].startswith('cafe:')] + for bp, dev in candidates: + if dev['serial'] == u: + return bp, dev['vidpid'], dev['ino'] + stripped = u.lstrip('0') + if u.isdigit() and stripped: + for bp, dev in candidates: + s = dev['serial'] + if s.isdigit() and s == stripped.zfill(len(s)): + return bp, dev['vidpid'], dev['ino'] + return None + + +def find_device(uid: str, pid: str | None): + """Board-online check: TinyUSB device (idVendor cafe) with this uid, optionally + PID-pinned. VID cafe keeps an Espressif USB-Serial-JTAG (303a) sharing the MAC + serial from false-passing.""" + for busport, dev in scan_usb().items(): + if (dev['serial'] == uid.lower() and dev['vidpid'].startswith('cafe:') + and (pid is None or dev['vidpid'].endswith(pid))): + return busport, dev['vidpid'], dev['ino'] + return None + + +def wait_device(uid: str, pid: str | None, old_ino, budget: float): + """Wait for the board's device with a NEW sysfs inode (flash resets the MCU, so a + genuine flash must re-enumerate; the inode is the re-enumeration marker).""" + deadline = time.monotonic() + budget + while time.monotonic() < deadline: + hit = find_device(uid, pid) + if hit and hit[2] != old_ino: + return hit + time.sleep(0.5) + return None + + +def lock_board(name: str): + """Nonblocking flock per hil_lock.py protocol. Returns handle, or a str with + the holder's info when the board is locked elsewhere. Board locks are ALWAYS + respected: a held board is reported as locked and skipped — never waited on, + and there is deliberately no bypass here.""" + os.makedirs(hil_lock.BOARD_LOCK_DIR, exist_ok=True) + try: + fh = hil_lock.flock_nb(name) + except OSError: + # NB: conflates a held flock with open() failures (EACCES/EROFS/ENOSPC) — + # benign while everything on the rig runs as one uid; a cross-uid setup + # would need flock_nb to distinguish the two + info = hil_lock.read_record(name) + return json.dumps(info) if info else 'unknown holder' + if not hil_lock.write_record(fh, 'pool_check'): + # an invisible lock (flock held, no record) is worse than no lock: status + # can't show us and release can't recognize the protected holder — bail out + hil_lock.clear_record(fh) + fh.close() + return 'ERROR: holder record write failed (lock dir unwritable?)' + return fh + + +def unlock_board(fh) -> None: + hil_lock.clear_record(fh) + fh.close() + + +def can_recover() -> bool: + if not USB_RECOVER.is_file(): + return False + try: + r = subprocess.run(['sudo', '-n', 'true'], capture_output=True) + except OSError: # sudo not installed (bare dev PC/container): recovery off, not fatal + return False + return r.returncode == 0 + + +def recover_probe(uid: str, busport: str) -> bool: + """Soft-replug an enumerated-but-wedged probe: deauthorize+reauthorize (no VBUS + cut, touches only this device). Success = the probe re-enumerated (new sysfs + inode), not the helper's exit code (observed to flake while the toggle worked). + J-Links respond with a full disconnect and can stay off the bus for >8 s.""" + pre = find_usb(uid) + try: + # bounded: the sysfs authorized store can block in D state on a wedged + # device, and this runs while the board's (release-protected) flock is held + subprocess.run(['sudo', '-n', str(USB_RECOVER), 'authorized', busport], + capture_output=True, text=True, timeout=30) + except subprocess.TimeoutExpired: + return False + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + post = find_usb(uid) + if post and (pre is None or post[2] != pre[2]): + return True + time.sleep(0.5) + return False + + +def resolve_variant(board: dict, example: str, note: list | None = None) -> str: + """Build-dir variant name for `example`: the first of the board's variants with + already-built firmware, falling back to the board name. Notes the pick when it + differs from the board name (e.g. nanoch32v203's build dir is variant + 'nanoch32v203-fsdev', not the board name).""" + name = board['name'] + for v in board.get('variant') or [{'name': name}]: + vn = v['name'] + if hil_flash.find_firmware(vn, example): + if vn != name and note is not None and f'variant: {vn}' not in note: + note.append(f'variant: {vn}') + return vn + return name + + +def pick_example(board: dict, note: list, build_missing: bool = True): + """(example, kind, variant, fw) with built firmware for this board; kind is + 'device' (uid check) or 'host' (serial-output check); variant is the resolved + build-dir variant that has it (see resolve_variant); fw is the firmware base + path to flash. When nothing is built and build_missing is set (the default — + never skip a board for lack of a build), the preferred candidate is built on + the spot via ensure_fw.""" + tests = board.get('tests', {}) + only = tests.get('only', []) + skip = set(tests.get('skip', [])) # config's known-broken examples: never pick one + is_device = tests.get('device') or any(t.startswith('device/') for t in only) + if is_device: + cand = DEVICE_CANDIDATES + [t for t in only if t.startswith('device/') and t != 'device/usbtest'] + kind = 'device' + else: + cand = HOST_CANDIDATES + [t for t in only if t.startswith('host/')] + kind = 'host' + for ex in dict.fromkeys(cand): + if ex in skip: + continue + variant = resolve_variant(board, ex, note) + fw = hil_flash.find_firmware(variant, ex) + if fw: + return ex, kind, variant, fw + if not build_missing: + return None, kind, None, None + # nothing built anywhere: build the preferred candidate (an only-list board + # must get one of its own examples — dfu_runtime etc. may not even configure) + pref = [c for c in dict.fromkeys(cand) if c not in skip and (not only or c in only)] + if not pref: + return None, kind, None, None + variant = (board.get('variant') or [{'name': board['name']}])[0]['name'] + for ex in pref[:2]: # the second candidate covers a preferred example that fails to build + fw = ensure_fw(board, variant, ex, note) + if fw: + return ex, kind, variant, fw + return None, kind, None, None + + +_pid_cache: dict[str, str | None] = {} + + +def get_expected_pid(example: str) -> str | None: + """USB_PID for `example`'s device descriptor (examples//src/ + usb_descriptors.c, '#define USB_PID 0x....'), lowercased and without the 0x + prefix to match sysfs idProduct. Cached per example; None (also cached) when + the file or define isn't there — host examples have no usb_descriptors.c, and + the caller must stay quiet rather than false-warn.""" + if example not in _pid_cache: + pid = None + try: + text = (REPO_ROOT / 'examples' / example / 'src' / 'usb_descriptors.c').read_text() + # optional parens as in tools/check_example_pids.py's parser + m = re.search(r'#define\s+USB_PID\s+\(?\s*(0x[0-9a-fA-F]+)', text) + if m: + pid = m.group(1)[2:].lower() + except OSError: + pass + _pid_cache[example] = pid + return _pid_cache[example] + + +def call_flasher(fn, *fn_args) -> tuple[int, str]: + """Run a hil_flash flash_*/reset_* backend, normalizing raises to a failure: + several backends raise instead of returning nonzero (get_serial_dev + RuntimeError when a bridge's /dev/serial/by-id node vanishes, config.env + FileNotFoundError, .jlink script OSError) and an exception must not skip the + caller's retry/recovery ladder. Returns (returncode, error line).""" + try: + ret = fn(*fn_args) + if ret.returncode == 0: + return 0, '' + err = flash_error_line(hil_flash.cmd_stdout_text(ret.stdout)) + return ret.returncode, err or f'rc={ret.returncode}' + except Exception as e: + return -1, repr(e)[:90] + + +def flash(board: dict, fw, allow_recovery: bool, probe_port: str, note: list) -> bool: + """Flash the resolved firmware with one retry; on repeated failure soft-replug + the probe and always make one final flash attempt afterward, regardless of + whether the replug is confirmed — some probes (WCH-Link, ST-Link, CP210x, + picoprobe) leave their sysfs kobject intact across an authorized toggle + instead of dropping off the bus. Returns True on success. + + `fw` comes from pick_example: a re-resolve here would use the global search + policy and miss a firmware ensure_fw just built into cmake-build/ under an + exclusive -B.""" + fn = getattr(hil_flash, f'flash_{board["flasher"]["name"].lower()}') + for attempt in range(3): + if attempt == 2: + if not (allow_recovery and probe_port): + return False + cur = find_usb(board['flasher']['uid']) + if cur is None: + # probe gone from the bus: its old busport may now hold an UNRELATED + # device (bus renumbering) and the helper only checks occupancy, so + # toggling would deauthorize an innocent fixture — skip the toggle + note.append('probe vanished before toggle') + else: + say(f'{board["name"]:26} recovery: replugging probe {cur[0]} (authorized toggle)') + if recover_probe(board['flasher']['uid'], cur[0]): + note.append('probe replugged') + time.sleep(2) # udev recreates /dev/serial/by-id symlinks after re-enumeration + else: + note.append('probe toggle unconfirmed') + rc, err = call_flasher(fn, board, str(fw)) + if rc == 0: + return True + if rc == 127: # flasher binary missing: retries/probe recovery can't fix env + note.append(f'flasher tool missing ({err}) — esptool needs the ESP-IDF env (get-idf)' + if board['flasher']['name'].lower() == 'esptool' else + f'flasher tool missing: {err}') + return False + if attempt == 0: + say(f'{board["name"]:26} flash retry: {err}') + else: + note.append(f'flash: {err}') + return False + + +def flash_error_line(out: str) -> str: + """Most informative line of a failed flash's output: last error-looking line, + else the last non-empty one.""" + lines = [l.strip() for l in out.splitlines() if l.strip()] + for l in reversed(lines): + if any(k in l.lower() for k in ('error', 'fail', 'unknown', 'cannot', 'timeout', + 'no valid', 'not found', 'unable')): + return l[:90] + return lines[-1][:90] if lines else '' + + +def check_host_serial(board: dict, do_reset: bool = True, want_hello: bool = False) -> bytes | None: + """Host-only boards never enumerate their uid (their USB port is the host side); + aliveness = output on the flasher's UART bridge after a reset. A probe byte is + written each poll so an echo-only firmware (board_test) also answers. Returns + the first output chunk (b'' when silent, None when the port is absent/drops) so + the caller can also judge WHAT answered — see boardtest_output(). + + do_reset=False listens to the firmware as-is: used right after a flash whose + own reset already started it — a second openocd/JLink session back-to-back on + the same probe can fail transiently and leave the target halted.""" + import serial + try: + port = hil_flash.get_serial_dev(board['flasher']['uid'], None, None, 0) + ser = serial.Serial(port, baudrate=115200, timeout=0.3, write_timeout=1) + except Exception as e: + say(f'{board["name"]:26} no flasher serial port: {e}') + return None + try: + # flush BEFORE issuing the reset: pyserial's open-time flush is long past, + # so this drops the pre-reset CDC backlog (which must not count as life) + # while keeping the board's post-reset boot banner, which prints while the + # reset tool is still tearing down and would be eaten by a post-reset flush + ser.reset_input_buffer() + if do_reset: + getattr(hil_flash, f'reset_{board["flasher"]["name"].lower()}')(board) + # collect the WHOLE window and judge content, not the first chunk: the + # probe's CDC bridge has its own FIFO, so stale pre-flash output (e.g. + # board_test hellos) can arrive after our host-side flush and must not + # decide the verdict alone. Early-exit once non-board_test output proves + # a real example is talking. + data = b'' + deadline = time.monotonic() + SERIAL_WAIT + while time.monotonic() < deadline: + try: + ser.write(b'U') + data += ser.read(256) + except serial.SerialTimeoutException: + pass + except serial.SerialException: + return None # port dropped mid-poll (bridge re-enumerating) + # early-exit on the caller's positive signal: fresh board_test hello + # (park verification) vs any non-board_test output (example liveness); + # stale bridge-FIFO backlog of the OTHER kind must not end the window + if want_hello: + if b'Hello from TinyUSB' in data: + return data + elif data and not boardtest_output(data): + return data + return data + finally: + ser.close() + + +def boardtest_output(data: bytes) -> bool: + """True when (non-empty) serial output is recognizably ONLY board_test's: its + periodic HELLO_STR and echoes of our b'U' pokes, nothing else. Any residue + beyond that (an example banner, log lines) proves other firmware is talking, + however much stale board_test backlog surrounds it. Used as a negative + identity marker — after flashing a host example, board_test-only chatter + means the flash silently didn't take (the host analog of the PID check).""" + residue = data.replace(b'Hello from TinyUSB', b'') + for junk in (b'U', b'\r', b'\n'): + residue = residue.replace(junk, b'') + return len(residue) == 0 + + +def build_example(board: dict, variant: str, example: str) -> int: + """Build one example for this board: tools/build.py (same invocation shape as + hil_test.build_board: -T target, -D per build.args, variant defines/flags, + --build-name), or idf.py directly for espressif (tools/build.py's esp branch + ignores -T and builds everything; variant flags travel as -DCFLAGS_CLI, the + same channel tools/build.py uses). Bounded and process-group-killed via + run_cmd; 600 s: a first configure+build of an SDK-heavy family (pico, nrf, + esp) exceeds the old 300. Builds normally run pre-lock (pick_example / the + pre-park ensure), so a board flock is not held here except on rare recovery + paths. Per-build compile parallelism is capped at cpu/-j so -j concurrent + builds cannot swamp sibling workers' verification windows. Returns the + build's returncode (127 = ESP-IDF env missing).""" + name = board['name'] + variants = board.get('variant') or [{'name': name}] + vcfg = next((v for v in variants if v['name'] == variant), variants[0]) + if board['flasher']['name'].lower() == 'esptool': + if not shutil.which('idf.py'): + return 127 # ESP-IDF env not sourced in this shell + # -B keyed off the VARIANT so ensure_fw's post-build lookup finds it + cmd = ['idf.py', '-C', f'examples/{example}', + '-B', f'cmake-build/cmake-build-{vcfg["name"]}/{example}', + '-G', 'Ninja', f'-DBOARD={name}', 'build'] + for d in board.get('build', {}).get('args', []) + vcfg.get('defines', []): + cmd.insert(-1, f'-D{d}') + if vcfg.get('flags'): + cmd.insert(-1, f'-DCFLAGS_CLI={vcfg["flags"]}') + # the IDF component manager writes examples//dependencies.lock in the + # SOURCE tree (idf.py -B relocates only the build dir), so concurrent esp + # builds of one example for different targets corrupt each other's solve + with _esp_lock, _build_sem: + return hil_flash.run_cmd(shlex.join(cmd), cwd=str(hil_flash.TINYUSB_ROOT), + timeout=600).returncode + cmd = [sys.executable, str(hil_flash.TINYUSB_ROOT / 'tools' / 'build.py'), + '-b', name, '-T', Path(example).name, + '-j', str(max(1, (os.cpu_count() or _jobs) // _jobs))] + for d in board.get('build', {}).get('args', []): + cmd += ['-D', d] + if vcfg['name'] != name: + cmd += ['--build-name', vcfg['name']] + for d in vcfg.get('defines', []): + cmd += ['-D', d] + for tok in vcfg.get('flags', '').split(): + cmd += [f'--cflag={tok}'] + with _build_sem: + return hil_flash.run_cmd(shlex.join(cmd), cwd=str(hil_flash.TINYUSB_ROOT), + timeout=600).returncode + + +_deps_lock = threading.Lock() # one get_deps at a time (it also drains _build_sem) +_esp_lock = threading.Lock() # idf.py mutates source-tree dependencies.lock per example +_no_build = False # --no-build: ensure_fw never invokes a build +_jobs = 4 # mirrors -j; set in main before the pool starts +_build_sem = threading.BoundedSemaphore(4) # build slots; get_deps drains ALL (exclusive) +_builds: dict = {} # (variant, example) -> (fw|None, reason): one attempt per run + + +def ensure_fw(board: dict, variant: str, example: str, note: list): + """Firmware for `example`, building it when absent — never skip a board for + lack of a build (--no-build opts out). One retry with deps fetched and the + CMake caches dropped when the first build fails (fresh checkouts lack the + family deps; a cache configured in a broken env poisons every later attempt). + Returns the firmware path, or None with the failure noted. Call BEFORE + taking the board lock: builds are long. One build attempt per + (variant, example) per run, success or failure — memoized in _builds, so a + repeat call (park, under the held flock) resolves instantly even when an + exclusive -B hides the fresh cmake-build/ artifact from the global search.""" + fw = hil_flash.find_firmware(variant, example) + if fw: + return fw + key, base = (variant, example), Path(example).name + if key in _builds: + return _builds[key][0] + if _no_build: + _builds[key] = (None, 'disabled') + note.append(f'build skipped (--no-build): {base}') + return None + rc = build_example(board, variant, example) + if rc == 127 and board['flasher']['name'].lower() == 'esptool': + _builds[key] = (None, 'no-env') + note.append(f'cannot build {base}: ESP-IDF env missing (get-idf)') + return None + if rc == 124: # hung build: a deps/cache retry cannot cure it, don't double the stall + _builds[key] = (None, 'timeout') + note.append(f'build timeout: {base}') + return None + if rc != 0: + # retry once with deps fetched and the CMake caches dropped (cache only — + # a tree wipe would destroy every other example's firmware). get_deps + # git-resets already-present shared deps (lib/fatfs's ffconf.h dance), so + # it must exclude every in-flight build, not just other get_deps calls: + # it drains ALL build slots before running. + with _deps_lock: + for _ in range(_jobs): + _build_sem.acquire() + try: + r = hil_flash.run_cmd(shlex.join([sys.executable, str(hil_flash.TINYUSB_ROOT / 'tools' / 'get_deps.py'), + '-b', board['name']]), + cwd=str(hil_flash.TINYUSB_ROOT), timeout=600) + finally: + for _ in range(_jobs): + _build_sem.release() + if r.returncode != 0: + note.append('get_deps failed') + bd = hil_flash.TINYUSB_ROOT / 'cmake-build' / f'cmake-build-{variant}' + # esp configures one level deeper (//): wipe both layouts + for d in (bd, bd / example): + shutil.rmtree(d / 'CMakeFiles', ignore_errors=True) + (d / 'CMakeCache.txt').unlink(missing_ok=True) + rc = build_example(board, variant, example) + if rc != 0: + _builds[key] = (None, 'fail') + note.append(f'build failed: {base}') + return None + # tools/build.py and the idf.py invocation above always write to cmake-build/: + # look there too even when an explicit -B narrowed the global search — this is + # OUR fresh build, not a stale-candidate fallback + fw = hil_flash.find_firmware(variant, example, + roots=[hil_flash.build_dir, 'cmake-build']) + _builds[key] = (fw, 'ok' if fw else 'no-fw') + note.append(f'built {base}' if fw else f'build produced no firmware: {base}') + return fw + + +def ensure_board_test(board: dict, variant: str, note: list): + """board_test firmware for parking, building it if absent (via ensure_fw). + Espressif included — tools/build.py builds board_test for that family too; + the build just needs the ESP-IDF env (127 → noted, park is then skipped).""" + fw = hil_flash.find_firmware(variant, 'device/board_test') + if fw: + return fw + variants = board.get('variant') or [{'name': board['name']}] + if not any(v['name'] == variant for v in variants): + variant = variants[0]['name'] + return ensure_fw(board, variant, 'device/board_test', note) + + +def verdict(row: dict, ok: bool) -> str: + """Row status for a verification result, preserving a 'flash-failed' a deeper + layer already recorded (silent flash no-op, board_test delivery failure).""" + return 'ok' if ok else ('flash-failed' if row['status'] == 'flash-failed' else 'failed') + + +def host_alive(board: dict, note: list, row: dict, flashed_example: bool = False) -> bool: + """Serial aliveness with recovery: silent -> (build and) flash board_test (it + hellos every second and echoes) -> recheck. Also cures a silent flash no-op + that left the board crashed. + + With flashed_example=True (a host example was just flashed), board_test-shaped + output FAILS the check: the parked image still talking means the example flash + silently didn't take — the host analog of the device path's PID check. + + Side effect: delivery-class failures (silent no-op, board_test build/flash + failure) set row['status'] = 'flash-failed' so verdict() preserves the cause; + the caller derives the final status from the return value via verdict().""" + data = check_host_serial(board) + if data: + if flashed_example and boardtest_output(data): + note.append('board_test output after example flash: silent flash no-op') + row['status'] = 'flash-failed' + return False + return True + variant = resolve_variant(board, 'device/board_test', note) + fw = ensure_board_test(board, variant, note) + if fw is None: + note.append('serial silent; board_test unavailable') + row['status'] = 'flash-failed' + return False + say(f'{board["name"]:26} recovery: serial silent, flashing board_test') + rc, err = call_flasher(getattr(hil_flash, f'flash_{board["flasher"]["name"].lower()}'), board, str(fw)) + if rc != 0: + note.append(f'serial silent; board_test flash failed: {err}') + row['status'] = 'flash-failed' + return False + if not check_host_serial(board): + return False + if flashed_example: + # board_test talking proves the BOARD is alive, but the just-flashed + # example never produced serial — that verification still fails + note.append('example silent; board alive via board_test reflash') + return False + note.append('recovered via board_test reflash') + return True + + +def device_recover_and_check(board: dict, example: str, variant: str, old_ino, note: list, row: dict, seen: dict) -> bool: + """Wait for the flashed board's uid to re-enumerate; on timeout, try one board + reset (skipped for flashers with no hardware reset — see hil_flash.RESET_NOOP, + it would just burn the wait) and wait again. + + The PID policy is deliberately asymmetric. Pre-reset, the re-enumeration was + caused by the flash itself, so a PID mismatch most likely means the build dir + is stale (the flash DID write what find_firmware found) — warn, don't fail — + UNLESS the firmware was built this very run: then 'stale build' is impossible + and the mismatch can only be a silent flash no-op, which fails. Post-reset, + the re-enumeration proves nothing about the flash (the reset alone explains + it), so a mismatch is treated as a silent flash no-op and fails; an unknown + expected PID scores ok with a 'pid unverified' note in both paths.""" + name = board['name'] + expected_pid = get_expected_pid(example) + built_this_run = _builds.get((variant, example), (None, ''))[1] == 'ok' + + def seen_hit(hit): + seen[board['uid']] = {'name': name, 'busport': hit[0], 'when': time.strftime('%Y-%m-%d %H:%M')} + + hit = wait_device(board['uid'], None, old_ino, ENUM_WAIT) + if hit: + if expected_pid is not None and not hit[1].endswith(expected_pid): + if built_this_run: + row['device'] = f'❌ {hit[1]}' + note.append(f'pid {hit[1]}, this run built {expected_pid}: silent flash no-op') + row['status'] = 'flash-failed' + return False + note.append(f'⚠ pid {hit[1]}, source says {expected_pid}: stale build or silent flash no-op') + elif expected_pid is None: + note.append('pid unverified') + row['device'] = f'✅ {hit[1]}' + seen_hit(hit) + return True + + flasher_name = board['flasher']['name'].lower() + if flasher_name in hil_flash.RESET_NOOP: + note.append(f'no hardware reset available for {flasher_name}') + row['device'] = '❌ not enumerated' + return False + + say(f'{name:26} recovery: uid not up, resetting board') + rc, err = call_flasher(getattr(hil_flash, f'reset_{flasher_name}'), board) + if rc != 0: + note.append(f'reset failed: {err}') + hit = wait_device(board['uid'], None, old_ino, ENUM_WAIT_RETRY) + if not hit: + row['device'] = '❌ not enumerated' + note.append('reset did not help') + return False + if expected_pid is None: + row['device'] = f'✅ {hit[1]}' + note.append('reset recovered (pid unverified)') + seen_hit(hit) + return True + if hit[1].endswith(expected_pid): + row['device'] = f'✅ {hit[1]}' + note.append('reset recovered') + seen_hit(hit) + return True + row['device'] = f'❌ {hit[1]}' + note.append(f'reset recovered wrong pid, expected {expected_pid}: silent flash no-op') + row['status'] = 'flash-failed' + return False + + +def check_board(board: dict, args, allow_recovery: bool, seen: dict) -> dict: + name = board['name'] + row = {'name': name, 'probe': '❌ missing', 'flash': '–', 'device': '–', 'note': [], 'status': 'failed'} + note = row['note'] + + probe = find_usb(board['flasher']['uid']) + if probe: + row['probe'] = f'✅ {probe[0]}' + seen[board['flasher']['uid']] = {'name': f'{name} probe', 'busport': probe[0], + 'when': time.strftime('%Y-%m-%d %H:%M')} + else: + last = seen.get(board['flasher']['uid']) + note.append(f'probe last seen {last["busport"]} {last["when"]}' if last + else 'probe never seen by pool_check') + say(f'{name:26} probe MISSING ({board["flasher"]["name"]} {board["flasher"]["uid"]})') + + # existing firmware only here; a missing build is built on the spot further + # down (after a lock peek), except in scan/no-build modes — and never for a + # missing probe (nothing could be flashed anyway) + example, kind, variant, fw = pick_example(board, note, build_missing=False) + if kind == 'host': + note.append('host-only board') + + if args.scan_only: + hit = find_device(board['uid'], None) + # report the BOARD's usb state, not just the probe's: the enumerated device + # (with busport), off-bus (normal when parked in board_test), or n/a for + # host-only boards whose uid never enumerates + if hit: + row['device'] = f'✅ {hit[1]} @{hit[0]}' + elif kind == 'host': + row['device'] = '– n/a (host-only)' + else: + row['device'] = '⚫ off bus (parked?)' + # scan verifies probe presence only: that check DID run, so probe present + # is ok; a missing probe means no firmware could be delivered → flash-failed + row['status'] = 'ok' if probe else 'flash-failed' + if probe: + say(f'{name:26} probe ✅ {probe[0]}' + (f' device {hit[1]}' if hit else '')) + return row + if not probe: + row['status'] = 'flash-failed' + return row + + bt_variant = resolve_variant(board, 'device/board_test', note) + need_example = example is None and not args.no_build + # board_test is also host_alive's recovery image, so host boards pre-build it + # even under --no-park; --no-build gates EVERY build, board_test included + need_bt = (not args.no_build + and (not args.no_park or kind == 'host') + and hil_flash.find_firmware(bt_variant, 'device/board_test') is None) + if need_example or need_bt: + # builds are long and run BEFORE locking (park must never hold the flock + # through a build); peek the lock first so minutes of building are not + # wasted on — or a rebuilt tree swapped under — a board CI holds right now + peek = lock_board(name) + if isinstance(peek, str): + if peek.startswith('ERROR:'): # environment failure, not a held lock + row['flash'] = '❌ lock' + row['status'] = 'failed' + else: + row['flash'] = '🔒 locked' + row['status'] = 'locked' + note.append(peek) + say(f'{name:26} locked: {peek}') + return row + unlock_board(peek) + if need_example: + example, kind, variant, fw = pick_example(board, note, build_missing=True) + if need_bt and (example is not None or kind == 'host'): + # skip the park-image build when the example build already failed on a + # device board: the row returns before any flash/park could use it + ensure_board_test(board, bt_variant, note) + + if example is None: + if not any(n.startswith(('build failed', 'build timeout', 'build produced', + 'build skipped', 'cannot build')) for n in note): + note.append('no firmware built') + if kind != 'host': + row['status'] = 'flash-failed' + say(f'{name:26} probe ✅ {probe[0]} (no firmware to flash)') + return row + # host-only board: aliveness is still checkable without flashing — reset and + # listen to whatever firmware is on it (the parked board_test echoes and + # prints a periodic hello on the flasher UART) + + lk = lock_board(name) + if isinstance(lk, str): + if lk.startswith('ERROR:'): # environment failure, not a held lock + row['flash'] = '❌ lock' + row['status'] = 'failed' + else: + row['flash'] = '🔒 locked' + row['status'] = 'locked' + note.append(lk) + say(f'{name:26} locked: {lk}') + return row + try: + if example is None: # host-only without firmware: UART-only aliveness check + ok = host_alive(board, note, row) + row['device'] = '✅ serial out' if ok else '❌ no serial out' + row['status'] = verdict(row, ok) + say(f'{name:26} – {row["device"]} (existing firmware)') + return row + + pre = find_device(board['uid'], None) + old_ino = pre[2] if pre else None + + try: + if not flash(board, fw, allow_recovery, probe[0], note): + row['flash'] = f'❌ {Path(example).name}' + row['status'] = 'flash-failed' + say(f'{name:26} flash FAILED ({example})') + return row + row['flash'] = f'✅ {Path(example).name}' + + if kind == 'host': + ok = host_alive(board, note, row, flashed_example=True) + row['device'] = '✅ serial out' if ok else '❌ no serial out' + else: + ok = device_recover_and_check(board, example, variant, old_ino, note, row, seen) + row['status'] = verdict(row, ok) + say(f'{name:26} {row["flash"]} {row["device"]}') + return row + finally: + # teardown for EVERY path that attempted a flash (a failed programmer op + # can still have erased/half-written the target): re-park while the + # board lock is still held + if not args.no_park: + park_board(board, kind, row, note) + finally: + unlock_board(lk) + + +def park_board(board: dict, kind: str, row: dict, note: list) -> None: + """Re-park with board_test, building it if absent (ensure_board_test), and + VERIFY it took: board_test never enumerates USB, so a device board's cafe + device must drop off the bus, and a host board must answer with board_test's + own output — a rc=0 park that changed nothing (silent no-op) must not pass. + A board left unparked marks an ok row flash-failed (never downgrading a + 'failed' verify verdict — that is the more diagnostic signal), with one + exception: an espressif board without the ESP-IDF env cannot build + board_test — noted, not a board fault.""" + # capture BEFORE the park flash: uid-disappearance only verifies the park if + # the device was on the bus to begin with (a fast park drops it immediately) + on_bus_before = kind != 'host' and find_device(board['uid'], None) is not None + variant = resolve_variant(board, 'device/board_test', note) + fw = ensure_board_test(board, variant, note) + if fw is None: + if any(n.startswith('cannot build board_test') for n in note): + note.append('park skipped (no ESP-IDF env)') + else: + # --no-build disables builds, not parking (--no-park is that opt-out): + # a board left running a USB-active image is unparked either way + note.append('unparked: board_test not built (--no-build)' + if any(n.startswith('build skipped (--no-build): board_test') for n in note) + else 'unparked: board_test unavailable (build failed/timed out)') + if row['status'] == 'ok': + row['status'] = 'flash-failed' + return + rc, err = call_flasher(getattr(hil_flash, f'flash_{board["flasher"]["name"].lower()}'), + board, str(fw)) + if rc != 0: + note.append(f'park flash failed: {err}') + if row['status'] == 'ok': + row['status'] = 'flash-failed' + return + if kind == 'host': + # no second reset (the park flash's own reset already started board_test); + # POSITIVE marker: its hello must appear — stale example output may still + # drain from the probe bridge's FIFO alongside it and is not disqualifying + data = check_host_serial(board, do_reset=False, want_hello=True) + if not (data and b'Hello from TinyUSB' in data): + note.append('park unverified: no board_test output') + if row['status'] == 'ok': + row['status'] = 'flash-failed' + return + if not on_bus_before: + # board never enumerated this run: uid-disappearance can't distinguish a + # verified park from a silent no-op — say so instead of passing vacuously + note.append('park unverified (device already off bus)') + return + deadline = time.monotonic() + 6 + while time.monotonic() < deadline: + if find_device(board['uid'], None) is None: + return + time.sleep(0.5) + note.append('park unverified: device still enumerated') + if row['status'] == 'ok': + row['status'] = 'flash-failed' + + +def check_board_safe(board: dict, args, allow_recovery: bool, seen: dict) -> dict: + """Isolate one board's exceptions: a crashing worker must not discard every + other board's row, the table, the topology, and the seen-cache write.""" + try: + return check_board(board, args, allow_recovery, seen) + except Exception as e: + name = board.get('name', '?') + say(f'{name:26} INTERNAL ERROR: {e!r}') + return {'name': name, 'probe': '–', 'flash': '–', 'device': '❌ error', + 'note': [repr(e)[:120]], 'status': 'failed'} + + +def controller_summary() -> list[str]: + """USB topology: controller (PCI addr, vendor) -> bus -> root-port subtree device + counts (hubs included, interfaces/root hubs not). Bus numbers renumber every boot; + PCI addresses and root-port numbers are stable.""" + vendor_names = {'0x1022': 'AMD', '0x1912': 'Renesas', '0x8086': 'Intel', '0x1b21': 'ASMedia'} + ctrl = {} + for root in glob.glob('/sys/bus/usb/devices/usb*'): + bus = int(os.path.basename(root)[3:]) + m = re.findall(r'[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]', os.path.realpath(root)) + pci = m[-1] if m else '?' + c = ctrl.setdefault(pci, {'vendor': '?', 'buses': {}}) + subtrees = {} + for d in glob.glob(f'/sys/bus/usb/devices/{bus}-*'): + b = os.path.basename(d) + if ':' in b: + continue + subtrees[b.split('.')[0]] = subtrees.get(b.split('.')[0], 0) + 1 + c['buses'][bus] = subtrees + try: + vid = open(f'/sys/bus/pci/devices/{pci}/vendor').read().strip() + c['vendor'] = vendor_names.get(vid, vid) + except OSError: + pass + + lines = [] + for pci, c in sorted(ctrl.items()): + lines.append(f'{pci} ({c["vendor"]})') + for bus, subtrees in sorted(c['buses'].items()): + detail = ' '.join(f'{k}: {n} dev' for k, n in + sorted(subtrees.items(), key=lambda i: int(i[0].split('-')[1]))) + lines.append(f' bus {bus}: {sum(subtrees.values())} devices' + + (f' {detail}' if detail else '')) + return lines + + +def main() -> None: + # toolchain/flasher CLIs live in the user bin dirs (arm-none-eabi-gcc + esptool + # in ~/.local/bin, STM32_Programmer_CLI in ~/bin) which non-login shells may + # lack — same PATH shim hil_ci.sh applies on the remote side + for d in (Path.home() / 'bin', Path.home() / '.local' / 'bin'): + if d.is_dir() and str(d) not in os.environ.get('PATH', '').split(os.pathsep): + os.environ['PATH'] = f'{d}{os.pathsep}{os.environ.get("PATH", "")}' + + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument('config', nargs='?', help='HIL config json (default: by hostname)') + parser.add_argument('-b', '--board', action='append', default=[], help='only these boards') + parser.add_argument('-B', '--build-dir', default=None, + help='firmware parent dir, searched EXCLUSIVELY when given ' + '(default: examples, plus cmake-build as fallback)') + parser.add_argument('--scan-only', action='store_true', + help='USB presence scan only: no locks, no flashing') + parser.add_argument('--no-build', action='store_true', + help='do not build missing firmware (default: build the light example on the spot)') + parser.add_argument('--no-park', action='store_true', + help='leave the light example running (default: park with board_test)') + # no cross-process flash budget with a concurrent hil_test.py run yet (would need + # a file-lock budget in hil_lock; hil_test uses in-process semaphores) — keep modest + parser.add_argument('-j', '--jobs', type=int, default=4) + parser.add_argument('-v', '--verbose', action='store_true') + args = parser.parse_args() + global _no_build, _jobs, _build_sem + _no_build = args.no_build + _jobs = max(1, args.jobs) + _build_sem = threading.BoundedSemaphore(_jobs) + + host = socket.gethostname() + cfg_name = args.config or CONFIG_BY_HOST.get(host, 'local.json') + cfg_path = Path(cfg_name) + if not cfg_path.exists(): + cfg_path = REPO_ROOT / 'test' / 'hil' / cfg_name + if not cfg_path.exists(): + sys.exit(f'config not found: {cfg_name} (host {host}; dev PCs need test/hil/local.json)') + with cfg_path.open() as f: + config = json.load(f) + + boards = list(config['boards']) # boards-skip (parked hardware) is not scanned by default + if args.board: + boards += config.get('boards-skip', []) # explicitly named parked boards are fair game + unknown = set(args.board) - {b['name'] for b in boards} + if unknown: + sys.exit(f'board(s) not in {cfg_path.name}: {", ".join(sorted(unknown))}') + boards = [b for b in boards if b['name'] in args.board] + + hil_flash.build_dir = args.build_dir or 'examples' + hil_flash.verbose = args.verbose + if args.build_dir is None: + # default mode: search both standard layouts (cmake-build/ from tools/build.py + # + ESP-IDF, examples/ from manual builds). An EXPLICIT -B is exclusive — the + # caller named an artifact tree, so a miss must report, not silently flash an + # older build from elsewhere. hil_test's -B is likewise untouched by this. + hil_flash.EXTRA_BUILD_DIRS = ['cmake-build', 'examples'] + allow_recovery = not args.scan_only and can_recover() + seen = {} + try: + loaded = json.loads(SEEN_CACHE.read_text()) + if isinstance(loaded, dict): # tolerate a torn/hand-edited cache + seen = {k: v for k, v in loaded.items() if isinstance(v, dict)} + except (OSError, ValueError): + pass + + roots = ' + '.join(dict.fromkeys([hil_flash.build_dir, *hil_flash.EXTRA_BUILD_DIRS])) + say(f'pool check: host {host}, config {cfg_path.name}, {len(boards)} boards, ' + f'{"scan-only" if args.scan_only else f"flash via {{{roots}}}/cmake-build-"}' + f'{"" if allow_recovery or args.scan_only else ", recovery unavailable (no sudo -n / usb_recover.sh)"}') + + if args.verbose: + rows = [check_board_safe(b, args, allow_recovery, seen) for b in boards] + else: + with io.StringIO() as spool, ThreadPoolExecutor(max_workers=args.jobs) as pool: + sys.stdout = spool # silence hil_flash's COMMAND FAILED dumps; say() uses __stdout__ + try: + rows = list(pool.map(lambda b: check_board_safe(b, args, allow_recovery, seen), boards)) + finally: + sys.stdout = sys.__stdout__ + + try: + SEEN_CACHE.parent.mkdir(parents=True, exist_ok=True) + tmp = SEEN_CACHE.with_suffix('.json.tmp') + tmp.write_text(json.dumps(seen, indent=1, sort_keys=True) + '\n') + tmp.replace(SEEN_CACHE) # atomic: a killed run can't tear the cache + except OSError: + pass + + status_mark = {'ok': '✅ ok', 'flash-failed': '❌ flash-failed', 'failed': '❌ failed', + 'locked': '🔒 locked'} + headers = ['Board', 'Probe', 'Flash', 'Device', 'Status', 'Note'] + cells = [[r['name'], r['probe'], r['flash'], r['device'], + status_mark.get(r['status'], r['status']), '; '.join(r['note'])] for r in rows] + widths = [max(len(h), *(len(c[i]) for c in cells)) if cells else len(h) + for i, h in enumerate(headers)] + line = lambda vals: '| ' + ' | '.join(v.ljust(w) for v, w in zip(vals, widths)) + ' |' + print() + print(line(headers)) + print('|' + '|'.join('-' * (w + 2) for w in widths) + '|') + for c in cells: + print(line(c)) + + print('\nUSB topology (controller → root-port subtree):') + for line in controller_summary(): + print(f' {line}') + + counts = {'ok': 0, 'flash-failed': 0, 'failed': 0, 'locked': 0} + for r in rows: + counts[r.get('status', 'failed')] += 1 + print(f'\n{counts["ok"]} ok · {counts["flash-failed"]} flash-failed · {counts["failed"]} failed ' + f'· {counts["locked"]} locked · in {time.monotonic() - t0:.0f}s') + sys.exit(min(counts['flash-failed'] + counts['failed'], 125)) + + +if __name__ == '__main__': + main() diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py index 80d1e1823..71e85f55f 100755 --- a/test/hil/hil_test.py +++ b/test/hil/hil_test.py @@ -46,10 +46,9 @@ import re import select import sys import time -import signal from contextlib import redirect_stdout from pathlib import Path -from typing import Any, TypedDict, NotRequired, cast +from typing import TypedDict, NotRequired, cast import serial import subprocess @@ -58,6 +57,9 @@ import glob import multiprocessing from multiprocessing import TimeoutError as MpTimeoutError +import hil_flash +import hil_lock + # Raw Lock/Semaphore objects passed via Pool initargs are inheritable only under the fork # start method (spawn/forkserver pickle them and fail at Pool creation) — pin it so a # future interpreter default change cannot break the run at startup. @@ -68,51 +70,6 @@ import ctypes from pymtp import MTP import string -# --- per-board dev-session locks (see test/hil/board_lock.py) ------------ -BOARD_LOCK_DIR = '/tmp/tinyusb-hil-locks' - -def acquire_board_lock(board_name): - """Take this board's flock for the duration of its flash+test. - Returns an open file handle (keep it referenced; closing releases it), - or None when HIL_NO_BOARD_LOCK=1 or the lock dir is unusable (fail-open: - locking must never break a test run by itself). - Raises RuntimeError only when another session holds the board.""" - import fcntl - if os.environ.get('HIL_NO_BOARD_LOCK') == '1': - return None # user-authorized bypass — see board_lock.py / hil skill - try: - os.makedirs(BOARD_LOCK_DIR, exist_ok=True) - fd = os.open(os.path.join(BOARD_LOCK_DIR, f'{board_name}.lock'), - os.O_RDWR | os.O_CREAT, 0o666) - fh = os.fdopen(fd, 'r+') - except OSError as e: - # odd lock dir (perms, path collision): proceed unlocked, but say so — - # a silent fail-open is indistinguishable from the intentional bypass - print(f'warning: board lock unavailable for {board_name} ({e}); proceeding unlocked', - flush=True) - return None - try: - fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError: - try: - info = fh.read(500).strip() - except (OSError, UnicodeDecodeError): - info = '' - fh.close() - raise RuntimeError(f'board locked: {info or "unknown holder"}') - # announce ourselves so the other side's conflict message is truthful; - # best-effort — the flock itself is already held - try: - fh.truncate(0) - fh.seek(0) - json.dump({'pid': os.getpid(), 'reason': 'hil_test.py', - 'since': time.strftime('%Y-%m-%dT%H:%M:%S%z')}, fh) - fh.flush() - except OSError: - pass - return fh - - # Enumeration wait budget. The first attempt gets ENUM_TIMEOUT; retry attempts get the # shorter ENUM_TIMEOUT_RETRY - the board was just re-flashed again, and a device that is # going to enumerate shows up within a few seconds, so a failing test costs ~3-5x a @@ -162,41 +119,16 @@ verbose = False PROFILE = os.environ.get('HIL_PROFILE') == '1' # timestamped logs + permit/flash timing + ctrl-map dump test_only = [] board_test = {} -build_dir = 'cmake-build' skip_flash = False print_lock = None shuffle_seed = None # per-run seed for the per-board test-order shuffle (HIL_SHUFFLE_SEED to replay) -# Per-host-controller concurrency (see controller_of/controller_slot below): a usbtest battery -# saturates its DUT's host controller, so batteries and flashes are budgeted per controller. -# - uPD720201 cards need their latest firmware (>= 2.0.2.6; RAM-uploaded, reloads every -# power cycle): ROM firmware dies under battery + re-enumeration churn, and usbtest.py -# refuses the unlink-stress cases on it. -# - widths (profiled 2026-07-13/14): wall time 22.2/14.3/12.5/10.8 min at usbtest width -# 1/2/3/4, plateau after; flash width beyond 8 only adds flasher-hub contention; -# battery case failures start at 12/8 (bandwidth stretch on shared leaf-hub uplinks). -# - a marginal DUT port bouncing during concurrent batteries can wedge/kill a uPD720201 -# ("xHCI host not responding to stop endpoint command"): fix the port/cable or pull -# the board, don't lower the widths (2026-07-16: every death traced to one board's port). -FLASH_PARALLEL = int(os.getenv('HIL_FLASH_PARALLEL', '8')) -USBTEST_PARALLEL = int(os.getenv('HIL_USBTEST_PARALLEL', '4')) -CONTROLLER_SLOTS = 12 # lock slots; controllers are assigned to slots on first sight -usbtest_sems = None # CONTROLLER_SLOTS semaphores: per-slot usbtest-battery permits -flash_sems = None # CONTROLLER_SLOTS semaphores: per-slot flash permits -controller_map = None # shared dict: 'pci:' -> slot, 'uid:' -> pci addr cache -controller_meta = None # guards slot assignment in controller_map -controller_hints = {} # static uid -> pci from the last run's cache (read-only per worker) - def init_worker(lock, seed, b_mutexes, f_sems, cmap, cmeta, hints_by_uid): - global print_lock, shuffle_seed, usbtest_sems, flash_sems, controller_map, controller_meta, controller_hints + global print_lock, shuffle_seed print_lock = lock shuffle_seed = seed - usbtest_sems = b_mutexes - flash_sems = f_sems - controller_map = cmap - controller_meta = cmeta - controller_hints = hints_by_uid + hil_lock.init_scheduling(b_mutexes, f_sems, cmap, cmeta, hints_by_uid, log_fn=log_line) def log_line(msg: str) -> None: @@ -210,108 +142,6 @@ def log_line(msg: str) -> None: print(msg, file=out, flush=True) -# ------------------------------------------------------------- -# Per-controller scheduling -# ------------------------------------------------------------- -def controller_of(uid: str): - """Resolve a DUT uid to its root host controller's PCI address, or None if the device - is not enumerated (e.g. parked in board_test firmware with USB off). Successful - resolutions are cached — cabling does not change mid-run. Dual-port parts (e.g. - CH32V307 usbhs/usbfs variants) share one uid and one cache entry: budgeting is only - exact when both ports sit on the same controller (true on this rig).""" - if controller_map is None: - return None - cached = controller_map.get(f'uid:{uid}') - if cached: - return cached - for f in glob.glob('/sys/bus/usb/devices/*/serial'): - d = os.path.dirname(f) - try: - if open(f).read().strip().lower() != uid.lower(): - continue - bus = int(open(os.path.join(d, 'busnum')).read()) - root = os.path.realpath(f'/sys/bus/usb/devices/usb{bus}') - m = re.findall(r'[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]', root) - if m: - controller_map[f'uid:{uid}'] = m[-1] - return m[-1] - except (OSError, ValueError): - continue - return None - - -def controller_slot(pci: str) -> int: - """Map a controller PCI address to a lock slot (assigned on first sight).""" - key = f'pci:{pci}' - with controller_meta: - slot = controller_map.get(key) - if slot is None: - slot = controller_map.get('nslots', 0) - if slot >= CONTROLLER_SLOTS: - slot = 0 # more controllers than slots: overflow shares slot 0 (safe, over-serialized) - else: - controller_map['nslots'] = slot + 1 - controller_map[key] = slot - return slot - - -class controller_permit: - """Context manager: one permit from `sems` on the board's controller slot. If the - controller is unknown, fail closed: take one permit from EVERY slot, in order, so the - operation respects the budget wherever it might land. `warn_unknown` logs that fallback - (used by usbtest, where the device is expected to be enumerated by the caller).""" - def __init__(self, sems, uid: str, warn_unknown: bool = False): - self.sems = sems - self.slots = None - self.uid = uid - if sems is None: - return - pci = controller_of(uid) - if pci is None and not warn_unknown: - # last-run cabling hint, flash budgeting only: a mis-budgeted flash is harmless, - # but a battery must never trust a stale hint (it could stack two batteries on - # one controller). In practice only a board's first flash lands here - batteries - # assert enumeration before taking their permit. - pci = controller_hints.get(uid) - if pci is None and warn_unknown: - log_line(f'warning: cannot resolve {uid} to a host controller; ' - 'taking a permit on every slot (over-serialized)') - self.slots = [controller_slot(pci)] if pci else list(range(CONTROLLER_SLOTS)) - - def __enter__(self): - if self.slots: - t0 = time.monotonic() - taken = [] - try: - for s in self.slots: - self.sems[s].acquire() - taken.append(s) - # stays inside the try: if this raises (e.g. broken stdout), the permits - # must be released - a failed __enter__ never gets its __exit__ - if PROFILE and time.monotonic() - t0 > 1.0: - log_line(f'[prof] permit wait {time.monotonic() - t0:.1f}s ' - f'(uid {self.uid}, slots {self.slots})') - except BaseException: - for s in reversed(taken): - self.sems[s].release() - raise - return self - - def __exit__(self, *exc): - if self.slots: - for s in reversed(self.slots): - self.sems[s].release() - return False - - -def flash_permit(uid: str) -> controller_permit: - return controller_permit(flash_sems, uid) - - -def usbtest_permit(uid: str) -> controller_permit: - return controller_permit(usbtest_sems, uid, warn_unknown=True) - - def compact_output(raw: str) -> str: if not raw: return '' @@ -365,47 +195,16 @@ class Board(TypedDict): class HilConfig(TypedDict): boards: list[Board] -CMD_TIMEOUT = int(os.getenv('HIL_CMD_TIMEOUT', '180')) POOL_TIMEOUT = int(os.getenv('HIL_POOL_TIMEOUT', '4200')) # usbtest batteries are serialized fleet-wide, lengthening the tail SERIAL_READ_TIMEOUT = float(os.getenv('HIL_SERIAL_READ_TIMEOUT', '5')) SERIAL_WRITE_TIMEOUT = float(os.getenv('HIL_SERIAL_WRITE_TIMEOUT', '10')) -def cmd_stdout_text(out: Any) -> str: - if out is None: - return '' - if isinstance(out, bytes): - return out.decode('utf-8', errors='ignore') - return str(out) - - MSC_README_TXT = \ b"This is tinyusb's MassStorage Class demo.\r\n\r\n\ If you find any bugs or get any questions, feel free to file an\r\n\ issue at github.com/hathach/tinyusb" -# ------------------------------------------------------------- -# Path -# ------------------------------------------------------------- -OPENCOD_ADI_PATH = Path.home() / 'app' / 'openocd_adi' -TINYUSB_ROOT = Path(__file__).resolve().parents[2] - -# get usb serial by id -def get_serial_dev(id, vendor_str, product_str, ifnum): - if vendor_str and product_str: - # known vendor and product - vendor_str = vendor_str.replace(' ', '_') - product_str = product_str.replace(' ', '_') - return f'/dev/serial/by-id/usb-{vendor_str}_{product_str}_{id}-if{ifnum:02d}' - else: - # just use id: mostly for cp210x/ftdi flasher - pattern = f'/dev/serial/by-id/usb-*_{id}-if*' - port_list = glob.glob(pattern) - if len(port_list) == 0: - raise RuntimeError(f'No serial device found for {pattern}') - return port_list[0] - - # get usb disk by id def get_disk_dev(id, vendor_str, lun): return f'/dev/disk/by-id/usb-{vendor_str}_Mass_Storage_{id}-0:{lun}' @@ -529,215 +328,13 @@ def open_printer_dev(id: str, vendor_str, product_str, ifnum: int) -> str: return lp_dev -# ------------------------------------------------------------- -# Flashing firmware -# ------------------------------------------------------------- -def run_cmd(cmd: str, cwd: str | None = None, timeout: int = CMD_TIMEOUT) -> subprocess.CompletedProcess: - popen_kwargs = { - 'cwd': cwd, - 'shell': True, - 'stdout': subprocess.PIPE, - 'stderr': subprocess.STDOUT, - 'text': True, - 'encoding': 'utf-8', - 'errors': 'replace', - } - if os.name != 'nt': - popen_kwargs['preexec_fn'] = os.setsid - - p = subprocess.Popen(cmd, **popen_kwargs) - try: - out, _ = p.communicate(timeout=timeout) - r = subprocess.CompletedProcess(args=cmd, returncode=p.returncode, stdout=out) - except subprocess.TimeoutExpired as ex: - if os.name != 'nt': - try: - os.killpg(p.pid, signal.SIGKILL) - except ProcessLookupError: - pass - else: - p.kill() - out, _ = p.communicate() - timeout_out = ex.stdout or out or b'' - title = f'COMMAND TIMEOUT ({timeout}s): {cmd}' - print() - if os.getenv('CI'): - print(f"::group::{title}") - print(cmd_stdout_text(timeout_out)) - print(f"::endgroup::") - else: - print(title) - print(cmd_stdout_text(timeout_out)) - return subprocess.CompletedProcess(args=cmd, returncode=124, stdout=timeout_out) - - if r.returncode != 0: - title = f'COMMAND FAILED: {cmd}' - print() - if os.getenv('CI'): - print(f"::group::{title}") - print(cmd_stdout_text(r.stdout)) - print(f"::endgroup::") - else: - print(title) - print(cmd_stdout_text(r.stdout)) - elif verbose: - print(cmd) - print(cmd_stdout_text(r.stdout)) - return r - - -def flash_jlink(board: Board, firmware: str) -> subprocess.CompletedProcess: - flasher = board['flasher'] - script = ['halt', 'r', f'loadfile {firmware}.elf', 'r', 'go', 'exit'] - f_jlink = Path(f'{board["name"]}_{Path(firmware).name}.jlink') - with f_jlink.open('w') as f: - f.writelines(f'{s}\n' for s in script) - ret = run_cmd(f'JLinkExe -USB {flasher["uid"]} {flasher["args"]} -if swd -JTAGConf -1,-1 -speed auto -NoGui 1 -ExitOnError 1 -CommandFile {f_jlink}') - f_jlink.unlink(missing_ok=True) - return ret - - -def reset_jlink(board: Board) -> subprocess.CompletedProcess: - flasher = board['flasher'] - script = ['halt', 'r', 'go', 'exit'] - f_jlink = Path(f'{board["name"]}_reset.jlink') - if not f_jlink.exists(): - with f_jlink.open('w') as f: - f.writelines(f'{s}\n' for s in script) - ret = run_cmd(f'JLinkExe -USB {flasher["uid"]} {flasher["args"]} -if swd -JTAGConf -1,-1 -speed auto -NoGui 1 -ExitOnError 1 -CommandFile {f_jlink}') - return ret - - -def flash_stlink(board, firmware): - flasher = board['flasher'] - return run_cmd(f'STM32_Programmer_CLI --connect port=swd sn={flasher["uid"]} --write {firmware}.elf --go') - - -def reset_stlink(board): - flasher = board['flasher'] - return run_cmd(f'STM32_Programmer_CLI --connect port=swd sn={flasher["uid"]} --rst --go') - -def flash_stflash(board, firmware): - flasher = board['flasher'] - ret = run_cmd(f'st-flash --serial {flasher["uid"]} write {firmware}.bin 0x8000000') - return ret - - -def reset_stflash(board): - flasher = board['flasher'] - return subprocess.CompletedProcess(args=['dummy'], returncode=0) - - -def flash_openocd(board, firmware): - flasher = board['flasher'] - ret = run_cmd(f'openocd -c "tcl_port disabled" -c "gdb_port disabled" -c "adapter serial {flasher["uid"]}" ' - f'{flasher["args"]} -c "init; halt; program {firmware}.elf verify; reset; exit"') - return ret - - -def reset_openocd(board): - flasher = board['flasher'] - ret = run_cmd(f'openocd -c "tcl_port disabled" -c "gdb_port disabled" -c "adapter serial {flasher["uid"]}" ' - f'{flasher["args"]} -c "init; reset run; exit"') - return ret - - -def flash_openocd_wch(board, firmware): - flasher = board['flasher'] - ret = run_cmd(f'openocd -c "tcl_port disabled" -c "gdb_port disabled" -c "telnet_port disabled" ' - f'-c "adapter serial {flasher["uid"]}" {flasher.get("args", "")} -c "program {firmware}.elf reset exit"') - return ret - - -def reset_openocd_wch(board): - flasher = board['flasher'] - ret = run_cmd(f'openocd -c "tcl_port disabled" -c "gdb_port disabled" -c "telnet_port disabled" ' - f'-c "adapter serial {flasher["uid"]}" {flasher.get("args", "")} -c "init; reset run; exit"') - return ret - - -def flash_openocd_adi(board: Board, firmware: str) -> subprocess.CompletedProcess: - flasher = board['flasher'] - openocd = OPENCOD_ADI_PATH / 'src' / 'openocd' - tcl_dir = OPENCOD_ADI_PATH / 'tcl' - ret = run_cmd(f'{openocd} -c "adapter serial {flasher["uid"]}" -s {tcl_dir} ' - f'{flasher["args"]} -c "program {firmware}.elf reset exit"') - return ret - - -def reset_openocd_adi(board: Board) -> subprocess.CompletedProcess: - flasher = board['flasher'] - openocd = OPENCOD_ADI_PATH / 'src' / 'openocd' - tcl_dir = OPENCOD_ADI_PATH / 'tcl' - ret = run_cmd(f'{openocd} -c "adapter serial {flasher["uid"]}" -s {tcl_dir} ' - f'{flasher["args"]} -c "program reset exit"') - return ret - - -def flash_wlink_rs(board, firmware): - flasher = board['flasher'] - # wlink use index for probe selection and lacking usb serial support - ret = run_cmd(f'wlink flash {firmware}.elf') - return ret - - -def reset_wlink_rs(board): - flasher = board['flasher'] - # wlink use index for probe selection and lacking usb serial support - ret = run_cmd(f'wlink reset') - return ret - - -def flash_esptool(board: Board, firmware: str) -> subprocess.CompletedProcess: - flasher = board['flasher'] - port = get_serial_dev(flasher["uid"], None, None, 0) - fw_dir = Path(f'{firmware}.bin').parent - with (fw_dir / 'config.env').open() as f: - idf_target = json.load(f)['IDF_TARGET'] - with (fw_dir / 'flash_args').open() as f: - flash_args = f.read().strip().replace('\n', ' ') - command = (f'esptool --chip {idf_target} -p {port} {flasher["args"]} ' - f'--before=default_reset --after=hard_reset write_flash {flash_args}') - ret = run_cmd(command, cwd=str(fw_dir)) - return ret - - -def reset_esptool(board): - flasher = board['flasher'] - return subprocess.CompletedProcess(args=['dummy'], returncode=0) - - -def flash_uniflash(board, firmware): - flasher = board['flasher'] - ret = run_cmd(f'dslite.sh {flasher["args"]} -f {firmware}.hex') - return ret - - -def reset_uniflash(board): - flasher = board['flasher'] - return subprocess.CompletedProcess(args=['dummy'], returncode=0) - - -def flash_lm4flash(board, firmware): - # TI Tiva-C / Stellaris ICDI: lightweight lm4flash, resets and runs after write - flasher = board['flasher'] - ret = run_cmd(f'lm4flash -s {flasher["uid"]} {flasher["args"]} {firmware}.bin') - return ret - - -def reset_lm4flash(board): - # lm4flash has no reset-only mode; it resets+runs on flash, so reset is a no-op - flasher = board['flasher'] - return subprocess.CompletedProcess(args=['dummy'], returncode=0) - - # ------------------------------------------------------------- # Tests: dual # ------------------------------------------------------------- def test_dual_host_info_to_device_cdc(board): uid = board['uid'] declared_devs = [f'{d["vid_pid"]}_{d["serial"]}' for d in board['tests']['dev_attached']] - port = get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0) + port = hil_flash.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0) ser = open_serial_dev(port) ser.timeout = 0.1 @@ -785,12 +382,12 @@ def test_host_device_info(board): flasher = board['flasher'] declared_devs = [f'{d["vid_pid"]}_{d["serial"]}' for d in board['tests']['dev_attached']] - port = get_serial_dev(flasher["uid"], None, None, 0) + port = hil_flash.get_serial_dev(flasher["uid"], None, None, 0) ser = open_serial_dev(port) ser.timeout = 0.1 # reset device since we can miss the first line - ret = globals()[f'reset_{flasher["name"].lower()}'](board) + ret = getattr(hil_flash, f'reset_{flasher["name"].lower()}')(board) assert ret.returncode == 0, 'Failed to reset device' # read until all expected devices are enumerated @@ -864,12 +461,12 @@ def test_host_cdc_msc_hid(board): if not cdc_devs and not msc_devs: return 'skipped' - port = get_serial_dev(flasher["uid"], None, None, 0) + port = hil_flash.get_serial_dev(flasher["uid"], None, None, 0) ser = open_serial_dev(port) ser.timeout = 0.1 # reset device to catch mount messages - ret = globals()[f'reset_{flasher["name"].lower()}'](board) + ret = getattr(hil_flash, f'reset_{flasher["name"].lower()}')(board) assert ret.returncode == 0, 'Failed to reset device' # Wait for all expected mount messages @@ -957,12 +554,12 @@ def test_host_msc_file_explorer(board): if not msc_devs: return 'skipped' - port = get_serial_dev(flasher["uid"], None, None, 0) + port = hil_flash.get_serial_dev(flasher["uid"], None, None, 0) ser = open_serial_dev(port) ser.timeout = 0.1 # reset device to catch mount messages - ret = globals()[f'reset_{flasher["name"].lower()}'](board) + ret = getattr(hil_flash, f'reset_{flasher["name"].lower()}')(board) assert ret.returncode == 0, 'Failed to reset device' # Wait for MSC mount (Disk Size message) @@ -1051,8 +648,8 @@ def test_device_board_test(board): def test_device_cdc_dual_ports(board): uid = board['uid'] port = [ - get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0), - get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 2) + hil_flash.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0), + hil_flash.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 2) ] ser = [open_serial_dev(p) for p in port] @@ -1091,7 +688,7 @@ def test_device_cdc_dual_ports(board): def test_device_cdc_msc(board): uid = board['uid'] # CDC Echo test - port = get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0) + port = hil_flash.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0) ser = open_serial_dev(port) def rand_ascii(length): @@ -1140,7 +737,7 @@ def test_device_cdc_msc_throughput(board): assert timeout > 0, f'Disk {dev} not found' # Wait for CDC tty enumeration - tty = get_serial_dev(uid, 'TinyUSB', 'Throughput', 0) + tty = hil_flash.get_serial_dev(uid, 'TinyUSB', 'Throughput', 0) timeout = enum_timeout() while timeout > 0: if os.path.exists(tty): @@ -1159,8 +756,8 @@ def test_device_cdc_msc_throughput(board): pass # Put tty in raw mode so dd sees pure binary throughput. - rs = run_cmd(f'timeout 30 stty -F {tty} raw -echo') - assert rs.returncode == 0, f'stty failed: {cmd_stdout_text(rs.stdout)}' + rs = hil_flash.run_cmd(f'timeout 30 stty -F {tty} raw -echo') + assert rs.returncode == 0, f'stty failed: {hil_flash.cmd_stdout_text(rs.stdout)}' # Payload aim: ~5 s per direction at FS (~830 kB/s), much less at HS. msc_count = 2 if is_fs else 16 # bs=1M @@ -1168,21 +765,21 @@ def test_device_cdc_msc_throughput(board): tmp_file = f'/tmp/cdc_msc_tp_{uid}.bin' - rw = run_cmd(f'timeout 30 dd if=/dev/zero of={tty} bs=64K count={cdc_count} 2>&1') - assert rw.returncode == 0, f'CDC dd write failed: {cmd_stdout_text(rw.stdout)}' - cdc_w = parse_speed(cmd_stdout_text(rw.stdout)) + rw = hil_flash.run_cmd(f'timeout 30 dd if=/dev/zero of={tty} bs=64K count={cdc_count} 2>&1') + assert rw.returncode == 0, f'CDC dd write failed: {hil_flash.cmd_stdout_text(rw.stdout)}' + cdc_w = parse_speed(hil_flash.cmd_stdout_text(rw.stdout)) - rr = run_cmd(f'timeout 30 dd if={tty} of=/dev/null bs=64K count={cdc_count} iflag=fullblock 2>&1') - assert rr.returncode == 0, f'CDC dd read failed: {cmd_stdout_text(rr.stdout)}' - cdc_r = parse_speed(cmd_stdout_text(rr.stdout)) + rr = hil_flash.run_cmd(f'timeout 30 dd if={tty} of=/dev/null bs=64K count={cdc_count} iflag=fullblock 2>&1') + assert rr.returncode == 0, f'CDC dd read failed: {hil_flash.cmd_stdout_text(rr.stdout)}' + cdc_r = parse_speed(hil_flash.cmd_stdout_text(rr.stdout)) - rmr = run_cmd(f'dd if={dev} of={tmp_file} bs=1M count={msc_count} iflag=direct 2>&1') - assert rmr.returncode == 0, f'MSC dd read failed: {cmd_stdout_text(rmr.stdout)}' - msc_r = parse_speed(cmd_stdout_text(rmr.stdout)) + rmr = hil_flash.run_cmd(f'dd if={dev} of={tmp_file} bs=1M count={msc_count} iflag=direct 2>&1') + assert rmr.returncode == 0, f'MSC dd read failed: {hil_flash.cmd_stdout_text(rmr.stdout)}' + msc_r = parse_speed(hil_flash.cmd_stdout_text(rmr.stdout)) - rmw = run_cmd(f'dd if={tmp_file} of={dev} bs=1M count={msc_count} oflag=direct 2>&1') - assert rmw.returncode == 0, f'MSC dd write failed: {cmd_stdout_text(rmw.stdout)}' - msc_w = parse_speed(cmd_stdout_text(rmw.stdout)) + rmw = hil_flash.run_cmd(f'dd if={tmp_file} of={dev} bs=1M count={msc_count} oflag=direct 2>&1') + assert rmw.returncode == 0, f'MSC dd write failed: {hil_flash.cmd_stdout_text(rmw.stdout)}' + msc_w = parse_speed(hil_flash.cmd_stdout_text(rmw.stdout)) try: os.remove(tmp_file) @@ -1213,8 +810,8 @@ def test_device_dfu(board): deadline = time.monotonic() + enum_timeout() found = False while time.monotonic() < deadline: - ret = run_cmd(f'dfu-util -l') - stdout = cmd_stdout_text(ret.stdout) + ret = hil_flash.run_cmd(f'dfu-util -l') + stdout = hil_flash.cmd_stdout_text(ret.stdout) if f'serial="{uid}"' in stdout and 'Found DFU: [cafe:400b]' in stdout: found = True break @@ -1232,10 +829,10 @@ def test_device_dfu(board): except OSError: pass - ret = run_cmd(f'dfu-util -S {uid} -a 0 -U {f_dfu0}') + ret = hil_flash.run_cmd(f'dfu-util -S {uid} -a 0 -U {f_dfu0}') assert ret.returncode == 0, 'Upload failed' - ret = run_cmd(f'dfu-util -S {uid} -a 1 -U {f_dfu1}') + ret = hil_flash.run_cmd(f'dfu-util -S {uid} -a 1 -U {f_dfu1}') assert ret.returncode == 0, 'Upload failed' with open(f_dfu0) as f: @@ -1254,8 +851,8 @@ def test_device_dfu_runtime(board): deadline = time.monotonic() + enum_timeout() found = False while time.monotonic() < deadline: - ret = run_cmd(f'dfu-util -l') - stdout = cmd_stdout_text(ret.stdout) + ret = hil_flash.run_cmd(f'dfu-util -l') + stdout = hil_flash.cmd_stdout_text(ret.stdout) if f'serial="{uid}"' in stdout and 'Found Runtime: [cafe:400c]' in stdout: found = True break @@ -1291,7 +888,7 @@ def test_device_printer_to_cdc(board): uid = board['uid'] # Wait for CDC port and printer device - cdc_port = get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0) + cdc_port = hil_flash.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0) ser = open_serial_dev(cdc_port) lp_dev = open_printer_dev(uid, 'TinyUSB', 'TinyUSB_Device', 2) @@ -1731,15 +1328,15 @@ def test_device_usbtest(board): # its normal driver. usbtest_permit budgets USBTEST_PARALLEL batteries per controller. script = Path(__file__).resolve().parent / 'usbtest.py' cmd = f'python3 "{script}" --serial "{uid}" --json --keep-binding --timeout 60' - with usbtest_permit(uid): - r = run_cmd(cmd, timeout=200) - out = cmd_stdout_text(r.stdout) + with hil_lock.usbtest_permit(uid): + r = hil_flash.run_cmd(cmd, timeout=200) + out = hil_flash.cmd_stdout_text(r.stdout) brace = out.find('{') try: data = json.loads(out[brace:]) passed, failed = int(data['passed']), int(data['failed']) except (ValueError, KeyError, json.JSONDecodeError): - raise TestFail(f'usbtest did not run: {compact_output(out) or cmd_stdout_text(r.stderr)}', + raise TestFail(f'usbtest did not run: {compact_output(out) or hil_flash.cmd_stdout_text(r.stderr)}', metric=f'{REPORT_CELL["fail"]} 0/30') total = passed + failed @@ -1788,21 +1385,6 @@ host_test = [ ] -def find_firmware(variant: str, example: str): - """Locate a built example's firmware base path (no extension) under - cmake-build-//. Accepts the single-config layout (firmware - directly in the example dir) or Ninja Multi-Config (a per-config subdir like - RelWithDebInfo/). Returns the base Path, or None if not built.""" - fw_dir = TINYUSB_ROOT / build_dir / f'cmake-build-{variant}' / example - base = Path(example).name - if fw_dir.is_dir(): - for cand in [fw_dir / base, fw_dir / 'RelWithDebInfo' / base, - *(p.with_suffix('') for p in sorted(fw_dir.glob(f'*/{base}.elf')))]: - if cand.with_suffix('.elf').exists() or cand.with_suffix('.bin').exists(): - return cand - return None - - def test_example(board: Board, variant: str, example: str) -> tuple[int, str, str | None]: """ Test example firmware @@ -1820,7 +1402,7 @@ def test_example(board: Board, variant: str, example: str) -> tuple[int, str, st test_name = f'{variant:40} {example:30} ...' - fw_name = find_firmware(variant, example) + fw_name = hil_flash.find_firmware(variant, example) if fw_name is None: log_line(f'{test_name} Skip (no binary)') return 0, 'skip', None @@ -1840,9 +1422,9 @@ def test_example(board: Board, variant: str, example: str) -> tuple[int, str, st attempt_out = io.StringIO() with redirect_stdout(attempt_out): if not skip_flash: - with flash_permit(board['uid']): + with hil_lock.flash_permit(board['uid']): t_flash = time.monotonic() - ret = globals()[f'flash_{board["flasher"]["name"].lower()}'](board, str(fw_name)) + ret = getattr(hil_flash, f'flash_{board["flasher"]["name"].lower()}')(board, str(fw_name)) if PROFILE: log_line(f'[prof] {variant} {example} flash attempt {i + 1}: ' f'{time.monotonic() - t_flash:.1f}s rc={ret.returncode}') @@ -1917,7 +1499,7 @@ def build_board(board: Board) -> tuple[str, int]: failed = 0 for v in variants: - cmd = [sys.executable, str(TINYUSB_ROOT / 'tools' / 'build.py'), '-b', name] + cmd = [sys.executable, str(hil_flash.TINYUSB_ROOT / 'tools' / 'build.py'), '-b', name] for d in extra_defs: cmd += ['-D', d] if v['name'] != name: @@ -1929,7 +1511,7 @@ def build_board(board: Board) -> tuple[str, int]: if verbose: cmd.append('-v') print(f' + {" ".join(cmd)}') - r = subprocess.run(cmd, cwd=TINYUSB_ROOT) + r = subprocess.run(cmd, cwd=hil_flash.TINYUSB_ROOT) if r.returncode != 0: failed += 1 return name, failed @@ -1940,7 +1522,7 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: flasher = board['flasher'] try: - _lock_fh = acquire_board_lock(name) + _lock_fh = hil_lock.acquire_board_lock(name) except RuntimeError as e: log_line(f'{name:25} {STATUS_FAILED}: {e}') # visible report row so the ❌ matches the exit code; failed-tests stays @@ -2033,7 +1615,7 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: try: # clear our pid record before dropping the flock: this worker # process lives on (pool reuse), so a stale record would make - # board_lock.py's pid-liveness checks report a freed board as + # hil_lock.py's pid-liveness checks report a freed board as # still locked for the rest of the run _lock_fh.truncate(0) except OSError: @@ -2174,7 +1756,6 @@ def main() -> None: global verbose global test_only global board_test - global build_dir global max_retry global skip_flash @@ -2204,13 +1785,14 @@ def main() -> None: config_file = Path(args.config_file) boards = args.board verbose = args.verbose + hil_flash.verbose = args.verbose test_only = args.test_only for entry in args.board_test: bname, _, tnames = entry.partition(':') if not bname or not tnames: parser.error(f'invalid --board-test value: {entry!r} (expected BOARD:test1,test2)') board_test[bname] = [t for t in tnames.split(',') if t] - build_dir = args.build_dir + hil_flash.build_dir = args.build_dir max_retry = args.retry skip_flash = args.skip_flash @@ -2234,8 +1816,8 @@ def main() -> None: build_err = 0 if args.build: - if build_dir != 'cmake-build': - print(f'warning: --build writes into cmake-build/, but -B is {build_dir!r}; ' + if hil_flash.build_dir != 'cmake-build': + print(f'warning: --build writes into cmake-build/, but -B is {hil_flash.build_dir!r}; ' f'tests will not find the freshly built firmware') print('-' * 30) print(f'Build phase: {len(config_boards)} board(s)') @@ -2264,7 +1846,7 @@ def main() -> None: seed = os.getenv('HIL_SHUFFLE_SEED') or str(int(time.time())) log_line(f'test-order shuffle seed: {seed} (HIL_SHUFFLE_SEED={seed} to replay); ' - f'flash/usbtest parallel per controller: {FLASH_PARALLEL}/{USBTEST_PARALLEL}; ' + f'flash/usbtest parallel per controller: {hil_lock.FLASH_PARALLEL}/{hil_lock.USBTEST_PARALLEL}; ' f'enum timeout first/retry: {ENUM_TIMEOUT}/{ENUM_TIMEOUT_RETRY}s') hints = {} @@ -2283,8 +1865,8 @@ def main() -> None: mgr = Manager() cmap = mgr.dict() initargs = (Lock(), seed, - [Semaphore(USBTEST_PARALLEL) for _ in range(CONTROLLER_SLOTS)], - [Semaphore(FLASH_PARALLEL) for _ in range(CONTROLLER_SLOTS)], + [Semaphore(hil_lock.USBTEST_PARALLEL) for _ in range(hil_lock.CONTROLLER_SLOTS)], + [Semaphore(hil_lock.FLASH_PARALLEL) for _ in range(hil_lock.CONTROLLER_SLOTS)], cmap, Lock(), hints_by_uid) with Pool(processes=os.cpu_count() or 1, initializer=init_worker, initargs=initargs) as pool: async_ret = pool.map_async(test_board, config_boards) diff --git a/test/hil/tinyusb.json b/test/hil/tinyusb.json index 8316dbc33..8f321baef 100644 --- a/test/hil/tinyusb.json +++ b/test/hil/tinyusb.json @@ -464,7 +464,7 @@ }, "flasher": { "name": "openocd_wch", - "uid": "EBCA8F0670AF", + "uid": "A76D8F062C2A", "args": "-f target/wch-riscv.cfg" } }, @@ -513,32 +513,13 @@ }, "flasher": { "name": "openocd_wch", - "uid": "7FD88F0604B5", + "uid": "57468F06DC03", "args": "-f target/wch-riscv.cfg" } }, - { - "name": "nrf5340dk", - "uid": "78E60E166B5F88BE", - "tests": { - "device": true, - "host": false, - "dual": false, - "skip": ["device/cdc_msc_freertos", "device/audio_test_freertos"], - "comment": "board new to HIL: FreeRTOS examples hardfault (UFSR=INVPC) at first task launch on the CM33_NTZ port - pre-existing upstream issue, non-FreeRTOS examples and usbtest pass; fix separately" - }, - "flasher": { - "name": "jlink", - "uid": "001050076405", - "args": "-device NRF5340_XXAA_APP" - } - } - ], - "boards-skip": [ { "name": "mimxrt1064_evk", "uid": "BAE96FB95AFA6DBB8F00005002001200", - "comment-skip": "device-port cable degraded from enum drops to killing the uPD720201 mid-battery (2026-07-17); replace the cable, verify enum, then move back", "tests": { "device": true, "host": true, @@ -568,7 +549,6 @@ { "name": "nrf54lm20dk", "uid": "899C3DE5B0F4D5CA", - "comment-skip": "J-Link probe fails most flashes (2026-07-16); replug/repair the probe, then move back", "tests": { "device": true, "host": false, @@ -597,7 +577,9 @@ "uid": "000831915224", "args": "-device R7FA6M5BH" } - }, + } + ], + "boards-skip": [ { "name": "ra8m1_ek", "uid": "797D142D36345030364E1737922E4B4E", -- cgit v1.3.1 From f3021b337fcea154b898489c417d428c92f88e92 Mon Sep 17 00:00:00 2001 From: Ha Thach Date: Fri, 31 Jul 2026 23:17:36 +0700 Subject: test/hil: fold openocd_wch into openocd, verify per board, resolve firmware by flasher extension (#3804) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit test/hil: one openocd flasher, per-board verify and firmware extension The four WCH boards move to `openocd`, leaving one flasher for all. `verify` is now a per-board opt-out, not dropped fleet-wide: WCH cannot read flash back over the WCH-Link sdi transport; the other seven openocd boards can, and say so explicitly. FLASHER_SUFFIX decides each flasher's extension once — find_firmware returns the full path and the flashers pass it through, so a build with only the wrong artifact is skipped rather than failed mid-flash. --skip-flash bypasses the filter. rescue_openocd() power-on-resets a wedged RP2040/RP2350 via its Rescue DP from the flash retry; the probe has no reset line. Drops unused openocd_adi, stflash, wlink_rs and uniflash, parks the unstable ra6m5_ek, and tests that every roster flasher name dispatches. --- test/hil/hil_flash.py | 172 ++++++++++++++++++++++---------------------- test/hil/hil_pool_check.py | 18 ++--- test/hil/hil_test.py | 16 ++++- test/hil/test_hil_select.py | 39 ++++++++++ test/hil/tinyusb.json | 74 +++++++++++-------- 5 files changed, 193 insertions(+), 126 deletions(-) (limited to 'test/hil/tinyusb.json') diff --git a/test/hil/hil_flash.py b/test/hil/hil_flash.py index 814258072..da81fcc97 100755 --- a/test/hil/hil_flash.py +++ b/test/hil/hil_flash.py @@ -26,7 +26,7 @@ build_dir = 'cmake-build' CMD_TIMEOUT = int(os.getenv('HIL_CMD_TIMEOUT', '180')) # flasher names (dispatch key, board['flasher']['name'].lower()) whose reset_* is a no-op -RESET_NOOP = {'esptool', 'lm4flash', 'stflash', 'uniflash'} +RESET_NOOP = {'esptool', 'lm4flash'} # extra parents find_firmware ALSO searches after build_dir. Empty by default so # hil_test's -B stays authoritative (a board missing there must report "Skip (no @@ -46,7 +46,6 @@ def cmd_stdout_text(out: Any) -> str: # ------------------------------------------------------------- # Path # ------------------------------------------------------------- -OPENCOD_ADI_PATH = Path.home() / 'app' / 'openocd_adi' TINYUSB_ROOT = Path(__file__).resolve().parents[2] # get usb serial by id @@ -129,7 +128,7 @@ def run_cmd(cmd: str, cwd: str | None = None, timeout: int = CMD_TIMEOUT) -> sub def flash_jlink(board: Board, firmware: str) -> subprocess.CompletedProcess: flasher = board['flasher'] - script = ['halt', 'r', f'loadfile {firmware}.elf', 'r', 'go', 'exit'] + script = ['halt', 'r', f'loadfile {firmware}', 'r', 'go', 'exit'] f_jlink = Path(f'{board["name"]}_{Path(firmware).name}.jlink') with f_jlink.open('w') as f: f.writelines(f'{s}\n' for s in script) @@ -151,88 +150,84 @@ def reset_jlink(board: Board) -> subprocess.CompletedProcess: def flash_stlink(board, firmware): flasher = board['flasher'] - return run_cmd(f'STM32_Programmer_CLI --connect port=swd sn={flasher["uid"]} --write {firmware}.elf --go') + return run_cmd(f'STM32_Programmer_CLI --connect port=swd sn={flasher["uid"]} --write {firmware} --go') def reset_stlink(board): flasher = board['flasher'] return run_cmd(f'STM32_Programmer_CLI --connect port=swd sn={flasher["uid"]} --rst --go') -def flash_stflash(board, firmware): - flasher = board['flasher'] - ret = run_cmd(f'st-flash --serial {flasher["uid"]} write {firmware}.bin 0x8000000') - return ret - -def reset_stflash(board): - flasher = board['flasher'] - return subprocess.CompletedProcess(args=['dummy'], returncode=0) +def _openocd_cmd_base(flasher): + return (f'openocd -c "tcl_port disabled" -c "gdb_port disabled" -c "telnet_port disabled" ' + f'-c "adapter serial {flasher["uid"]}" {flasher["args"]}') +# `verify` is on by default and opted out per board with "verify": false in the roster. +# WCH targets must opt out: flash read-back over the WCH-Link sdi transport returns a +# repeated word instead of memory contents, so verification always reports a mismatch and +# fails the flash (measured on ch32v103r and ch32v307v, 2026-07-30). Do NOT drop verify +# fleet-wide to accommodate them — every other openocd board can read back, and without it +# a partial or corrupt write exits 0 and the test phase runs bad firmware. def flash_openocd(board, firmware): flasher = board['flasher'] - ret = run_cmd(f'openocd -c "tcl_port disabled" -c "gdb_port disabled" -c "adapter serial {flasher["uid"]}" ' - f'{flasher["args"]} -c "init; halt; program {firmware}.elf verify; reset; exit"') + verify = ' verify' if flasher.get('verify', True) else '' + ret = run_cmd(f'{_openocd_cmd_base(flasher)} -c "program {firmware}{verify} reset exit"') return ret def reset_openocd(board): flasher = board['flasher'] - ret = run_cmd(f'openocd -c "tcl_port disabled" -c "gdb_port disabled" -c "adapter serial {flasher["uid"]}" ' - f'{flasher["args"]} -c "init; reset run; exit"') - return ret - - -def flash_openocd_wch(board, firmware): - flasher = board['flasher'] - ret = run_cmd(f'openocd -c "tcl_port disabled" -c "gdb_port disabled" -c "telnet_port disabled" ' - f'-c "adapter serial {flasher["uid"]}" {flasher.get("args", "")} -c "program {firmware}.elf reset exit"') - return ret - - -def reset_openocd_wch(board): - flasher = board['flasher'] - ret = run_cmd(f'openocd -c "tcl_port disabled" -c "gdb_port disabled" -c "telnet_port disabled" ' - f'-c "adapter serial {flasher["uid"]}" {flasher.get("args", "")} -c "init; reset run; exit"') - return ret - - -def flash_openocd_adi(board: Board, firmware: str) -> subprocess.CompletedProcess: - flasher = board['flasher'] - openocd = OPENCOD_ADI_PATH / 'src' / 'openocd' - tcl_dir = OPENCOD_ADI_PATH / 'tcl' - ret = run_cmd(f'{openocd} -c "adapter serial {flasher["uid"]}" -s {tcl_dir} ' - f'{flasher["args"]} -c "program {firmware}.elf reset exit"') + ret = run_cmd(f'{_openocd_cmd_base(flasher)} -c "init; reset run; exit"') return ret -def reset_openocd_adi(board: Board) -> subprocess.CompletedProcess: +# OpenOCD's messages for "the target's debug port did not answer". The probe is fine when +# these appear (the log still shows "CMSIS-DAP: Interface ready"); the chip's debug clock +# is gone, which no reset the probe can drive would fix -- the CMSIS-DAP Debug Probe has no +# nRESET line at all. Which message you get depends on the DAP topology, NOT on the board: +# rp2040.cfg creates three multidrop DAPs (cores 0/1 and the Rescue DP at instance 0xf) so +# it fails in swd_multidrop_select, while rp2350.cfg creates a single plain ADIv6 DAP that +# fails earlier in swd_connect. A dead RP2040 can also produce the second one if the very +# first DP read never gets through, so both are accepted for both chips -- it is the target +# cfg in the roster args, below, that picks how to rescue. +DAP_WEDGED = ('Failed to connect multidrop', 'Error connecting DP: cannot read IDR') + +# How each RP target reaches its Rescue DP, keyed by the target cfg named in flasher args. +# (cfg substitution, extra args): rp2040.cfg drives the Rescue DP itself behind a RESCUE +# flag and calls init/shutdown on its own; rp2350 has a separate cfg that pokes the rescue +# bit via an AP register but never shuts down, so it would sit in the server loop until +# CMD_TIMEOUT without an explicit one. +RESCUE_CFG = { + 'target/rp2040.cfg': ('target/rp2040.cfg', '-c "set RESCUE 1" ', ''), + 'target/rp2350.cfg': ('target/rp2350-rescue.cfg', '', ' -c "shutdown"'), +} + + +def rescue_openocd(board, flash_out: str = '') -> bool: + """Power-on-reset a wedged RP2040/RP2350 through its Rescue DP, the one debug port not + gated by the system clock (RP2040 datasheet 2.3.4.2): setting CDBGPWRUPREQ hard-resets + the chip, and the bootrom halts it in a safe state ready to be flashed. This is the + only way back for a target whose cores have stopped answering -- otherwise the board + needs a physical replug, since the probe carries no reset line. + + No-op (returns False) unless this is an openocd RP board AND the flash output shows the + wedge, so a flash that failed for any other reason still just retries. Returns True + when a rescue was attempted; the caller should retry the flash afterwards.""" flasher = board['flasher'] - openocd = OPENCOD_ADI_PATH / 'src' / 'openocd' - tcl_dir = OPENCOD_ADI_PATH / 'tcl' - ret = run_cmd(f'{openocd} -c "adapter serial {flasher["uid"]}" -s {tcl_dir} ' - f'{flasher["args"]} -c "program reset exit"') - return ret - - -def flash_wlink_rs(board, firmware): - flasher = board['flasher'] - # wlink use index for probe selection and lacking usb serial support - ret = run_cmd(f'wlink flash {firmware}.elf') - return ret - - -def reset_wlink_rs(board): - flasher = board['flasher'] - # wlink use index for probe selection and lacking usb serial support - ret = run_cmd(f'wlink reset') - return ret + if flasher['name'].lower() != 'openocd' or not any(m in flash_out for m in DAP_WEDGED): + return False + for cfg, (rescue_cfg, pre, post) in RESCUE_CFG.items(): + if cfg in flasher['args']: + args = flasher['args'].replace(cfg, rescue_cfg) + return run_cmd(f'{_openocd_cmd_base({**flasher, "args": pre + args})}{post}').returncode == 0 + return False def flash_esptool(board: Board, firmware: str) -> subprocess.CompletedProcess: flasher = board['flasher'] port = get_serial_dev(flasher["uid"], None, None, 0) - fw_dir = Path(f'{firmware}.bin').parent + fw_dir = Path(firmware).parent with (fw_dir / 'config.env').open() as f: idf_target = json.load(f)['IDF_TARGET'] with (fw_dir / 'flash_args').open() as f: @@ -248,21 +243,10 @@ def reset_esptool(board): return subprocess.CompletedProcess(args=['dummy'], returncode=0) -def flash_uniflash(board, firmware): - flasher = board['flasher'] - ret = run_cmd(f'dslite.sh {flasher["args"]} -f {firmware}.hex') - return ret - - -def reset_uniflash(board): - flasher = board['flasher'] - return subprocess.CompletedProcess(args=['dummy'], returncode=0) - - def flash_lm4flash(board, firmware): # TI Tiva-C / Stellaris ICDI: lightweight lm4flash, resets and runs after write flasher = board['flasher'] - ret = run_cmd(f'lm4flash -s {flasher["uid"]} {flasher["args"]} {firmware}.bin') + ret = run_cmd(f'lm4flash -s {flasher["uid"]} {flasher["args"]} {firmware}') return ret @@ -272,22 +256,42 @@ def reset_lm4flash(board): return subprocess.CompletedProcess(args=['dummy'], returncode=0) -def find_firmware(variant: str, example: str, roots: list | None = None): - """Locate a built example's firmware base path (no extension) under - /cmake-build-//, then under EXTRA_BUILD_DIRS - (empty unless the caller opts in — see its comment). `roots` overrides that - search list entirely for one call (e.g. to find a build just produced by - tools/build.py in its fixed cmake-build/ layout without widening the global - policy). Accepts the single-config layout (firmware directly in the example - dir) or Ninja Multi-Config (a per-config subdir like RelWithDebInfo/). - Returns the base Path, or None if not built.""" +# The one place a flasher's firmware extension is decided: find_firmware resolves the +# path with it and the flash_* functions pass that path through untouched. A flasher +# added here without an entry falls back to .elf-or-.bin and can be handed the wrong +# file — test_hil_select's TestRosterFlashersDispatch fails if a roster names one. +FLASHER_SUFFIX = { + 'esptool': '.bin', + 'jlink': '.elf', + 'lm4flash': '.bin', + 'openocd': '.elf', + 'stlink': '.elf', +} + + +def find_firmware(variant: str, example: str, roots: list | None = None, flasher: str | None = None): + """Locate a built example's firmware under /cmake-build-//, + then under EXTRA_BUILD_DIRS (empty unless the caller opts in — see its comment). + `roots` overrides that search list entirely for one call (e.g. to find a build just + produced by tools/build.py in its fixed cmake-build/ layout without widening the + global policy). `flasher` is the roster flasher name: it selects which extension + counts (see FLASHER_SUFFIX), so a build that produced only the other one is reported + missing — a clean "Skip (no binary)" — instead of being handed to the flasher, which + would fail opaquely on the absent file and burn every retry plus the board lock. + Accepts the single-config layout (firmware directly in the example dir) or Ninja + Multi-Config (a per-config subdir like RelWithDebInfo/). + Returns the full Path INCLUDING extension, or None if not built.""" base = Path(example).name + suffixes = [FLASHER_SUFFIX.get(flasher.lower())] if flasher else [] + if not suffixes or suffixes == [None]: + suffixes = ['.elf', '.bin'] for bd in dict.fromkeys(roots if roots is not None else [build_dir, *EXTRA_BUILD_DIRS]): fw_dir = TINYUSB_ROOT / bd / f'cmake-build-{variant}' / example if not fw_dir.is_dir(): continue for cand in [fw_dir / base, fw_dir / 'RelWithDebInfo' / base, - *(p.with_suffix('') for p in sorted(fw_dir.glob(f'*/{base}.elf')))]: - if cand.with_suffix('.elf').exists() or cand.with_suffix('.bin').exists(): - return cand + *(p.with_suffix('') for s in suffixes for p in sorted(fw_dir.glob(f'*/{base}{s}')))]: + for s in suffixes: + if cand.with_suffix(s).exists(): + return cand.with_suffix(s) return None diff --git a/test/hil/hil_pool_check.py b/test/hil/hil_pool_check.py index 63284213e..98f24288a 100644 --- a/test/hil/hil_pool_check.py +++ b/test/hil/hil_pool_check.py @@ -201,7 +201,7 @@ def resolve_variant(board: dict, example: str, note: list | None = None) -> str: name = board['name'] for v in board.get('variant') or [{'name': name}]: vn = v['name'] - if hil_flash.find_firmware(vn, example): + if hil_flash.find_firmware(vn, example, flasher=board['flasher']['name']): if vn != name and note is not None and f'variant: {vn}' not in note: note.append(f'variant: {vn}') return vn @@ -211,8 +211,8 @@ def resolve_variant(board: dict, example: str, note: list | None = None) -> str: def pick_example(board: dict, note: list, build_missing: bool = True): """(example, kind, variant, fw) with built firmware for this board; kind is 'device' (uid check) or 'host' (serial-output check); variant is the resolved - build-dir variant that has it (see resolve_variant); fw is the firmware base - path to flash. When nothing is built and build_missing is set (the default — + build-dir variant that has it (see resolve_variant); fw is the firmware path to + flash, extension included. When nothing is built and build_missing is set (the default — never skip a board for lack of a build), the preferred candidate is built on the spot via ensure_fw.""" tests = board.get('tests', {}) @@ -229,7 +229,7 @@ def pick_example(board: dict, note: list, build_missing: bool = True): if ex in skip: continue variant = resolve_variant(board, ex, note) - fw = hil_flash.find_firmware(variant, ex) + fw = hil_flash.find_firmware(variant, ex, flasher=board['flasher']['name']) if fw: return ex, kind, variant, fw if not build_missing: @@ -472,7 +472,7 @@ def ensure_fw(board: dict, variant: str, example: str, note: list): (variant, example) per run, success or failure — memoized in _builds, so a repeat call (park, under the held flock) resolves instantly even when an exclusive -B hides the fresh cmake-build/ artifact from the global search.""" - fw = hil_flash.find_firmware(variant, example) + fw = hil_flash.find_firmware(variant, example, flasher=board['flasher']['name']) if fw: return fw key, base = (variant, example), Path(example).name @@ -523,7 +523,8 @@ def ensure_fw(board: dict, variant: str, example: str, note: list): # look there too even when an explicit -B narrowed the global search — this is # OUR fresh build, not a stale-candidate fallback fw = hil_flash.find_firmware(variant, example, - roots=[hil_flash.build_dir, 'cmake-build']) + roots=[hil_flash.build_dir, 'cmake-build'], + flasher=board['flasher']['name']) _builds[key] = (fw, 'ok' if fw else 'no-fw') note.append(f'built {base}' if fw else f'build produced no firmware: {base}') return fw @@ -533,7 +534,7 @@ def ensure_board_test(board: dict, variant: str, note: list): """board_test firmware for parking, building it if absent (via ensure_fw). Espressif included — tools/build.py builds board_test for that family too; the build just needs the ESP-IDF env (127 → noted, park is then skipped).""" - fw = hil_flash.find_firmware(variant, 'device/board_test') + fw = hil_flash.find_firmware(variant, 'device/board_test', flasher=board['flasher']['name']) if fw: return fw variants = board.get('variant') or [{'name': board['name']}] @@ -706,7 +707,8 @@ def check_board(board: dict, args, allow_recovery: bool, seen: dict) -> dict: # even under --no-park; --no-build gates EVERY build, board_test included need_bt = (not args.no_build and (not args.no_park or kind == 'host') - and hil_flash.find_firmware(bt_variant, 'device/board_test') is None) + and hil_flash.find_firmware(bt_variant, 'device/board_test', + flasher=board['flasher']['name']) is None) if need_example or need_bt: # builds are long and run BEFORE locking (park must never hold the flock # through a build); peek the lock first so minutes of building are not diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py index 96d52e601..e7f82bd7f 100755 --- a/test/hil/hil_test.py +++ b/test/hil/hil_test.py @@ -1370,13 +1370,17 @@ def test_example(board: Board, variant: str, example: str) -> tuple[int, str, st test_name = f'{variant:40} {example:30} ...' - fw_name = hil_flash.find_firmware(variant, example) + # --skip-flash runs whatever is already on the board, so any build counts as present: + # only the flashing path needs the artifact this board's flasher actually consumes. + # Filtering there too would skip the test as "no binary" over an extension it never uses. + fw_name = hil_flash.find_firmware(variant, example, + flasher=None if skip_flash else board['flasher']['name']) if fw_name is None: log_line(f'{test_name} Skip (no binary)') return 0, 'skip', None if verbose: - log_line(f'Flashing {fw_name}.elf') + log_line(f'Firmware {fw_name}') # flash firmware (unless --skip-flash), then run the test. Both may fail randomly, # retry a few times. @@ -1396,7 +1400,13 @@ def test_example(board: Board, variant: str, example: str) -> tuple[int, str, st if PROFILE: log_line(f'[prof] {variant} {example} flash attempt {i + 1}: ' f'{time.monotonic() - t_flash:.1f}s rc={ret.returncode}') - flash_ok = (ret.returncode == 0) + flash_ok = (ret.returncode == 0) + # A wedged RP2040/RP2350 DAP answers nothing and the probe has no reset + # line, so the retry would fail identically; POR it via the Rescue DP + # first. No-op for every other board and every other flash failure. + if not flash_ok and i + 1 < max_retry and \ + hil_flash.rescue_openocd(board, hil_flash.cmd_stdout_text(ret.stdout)): + log_line(f'{variant} {example}: DAP wedged, rescued via Rescue DP') if flash_ok: try: tret = globals()[f'test_{example.replace("/", "_")}'](board) diff --git a/test/hil/test_hil_select.py b/test/hil/test_hil_select.py index 6a2bf6210..5e6b16759 100644 --- a/test/hil/test_hil_select.py +++ b/test/hil/test_hil_select.py @@ -9,6 +9,7 @@ import sys import unittest sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import hil_flash import hil_select from hil_examples import device_tests, dual_tests, host_test @@ -26,6 +27,19 @@ def real_rosters(): return rosters +def roster_flashers(): + """(roster path, board) for every board in the live rosters, `boards-skip` + included: a parked board's flasher name must still dispatch, so that unparking it + is not what discovers the name went stale.""" + for name in ('tinyusb.json', 'hfp.json'): + path = os.path.join(REPO, 'test/hil', name) + with open(path) as f: + cfg = json.load(f) + for key in ('boards', 'boards-skip'): + for b in cfg.get(key, []): + yield f'test/hil/{name}', b + + def on_roster(tc, *names): """The subset of `names` currently in the live rig rosters, skipping the test when none are. Parking/unparking a board is routine rig maintenance and must not @@ -538,5 +552,30 @@ class TestPortWithoutFamilyIsFull(unittest.TestCase): self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons']) +class TestRosterFlashersDispatch(unittest.TestCase): + """hil_test and hil_pool_check resolve a board's flasher with a bare + getattr(hil_flash, f'flash_{name}'), and hil_test does it inside a redirect_stdout — + so a renamed or typo'd roster name raises an AttributeError whose output is swallowed, + with nothing pointing at the roster as the thing to edit. Renaming a flash_*/reset_* + pair without updating every roster must fail here instead.""" + + def test_flash_and_reset_exist_for_every_roster_flasher(self): + for path, board in roster_flashers(): + name = board['flasher']['name'].lower() + for fn in (f'flash_{name}', f'reset_{name}'): + self.assertTrue(callable(getattr(hil_flash, fn, None)), + f'{path}: {board["name"]} uses flasher "{name}" ' + f'but hil_flash.{fn} does not exist') + + def test_firmware_suffix_known_for_every_roster_flasher(self): + """find_firmware falls back to accepting .elf-or-.bin when a flasher is missing + from FLASHER_SUFFIX, silently restoring the mismatch that map exists to catch.""" + for path, board in roster_flashers(): + name = board['flasher']['name'].lower() + self.assertIn(name, hil_flash.FLASHER_SUFFIX, + f'{path}: {board["name"]} uses flasher "{name}" ' + f'with no hil_flash.FLASHER_SUFFIX entry') + + if __name__ == '__main__': unittest.main(verbosity=1) diff --git a/test/hil/tinyusb.json b/test/hil/tinyusb.json index 8f321baef..c9b38992c 100644 --- a/test/hil/tinyusb.json +++ b/test/hil/tinyusb.json @@ -149,7 +149,8 @@ "flasher": { "name": "openocd", "uid": "E6614C311B597D32", - "args": "-f interface/cmsis-dap.cfg -f target/max32665.cfg" + "args": "-f interface/cmsis-dap.cfg -f target/max32665.cfg", + "verify": true } }, { @@ -239,7 +240,8 @@ "flasher": { "name": "openocd", "uid": "E6614103E72C1D2F", - "args": "-f interface/cmsis-dap.cfg -f target/rp2040.cfg -c \"adapter speed 5000\"" + "args": "-f interface/cmsis-dap.cfg -f target/rp2040.cfg -c \"adapter speed 5000\"", + "verify": true } }, { @@ -268,7 +270,8 @@ "flasher": { "name": "openocd", "uid": "E6633861A3819D38", - "args": "-f interface/cmsis-dap.cfg -f target/rp2040.cfg -c \"adapter speed 5000\"" + "args": "-f interface/cmsis-dap.cfg -f target/rp2040.cfg -c \"adapter speed 5000\"", + "verify": true }, "comment": "Test native host" }, @@ -293,7 +296,8 @@ "flasher": { "name": "openocd", "uid": "E6633861A3978538", - "args": "-f interface/cmsis-dap.cfg -f target/rp2350.cfg -c \"adapter speed 5000\"" + "args": "-f interface/cmsis-dap.cfg -f target/rp2350.cfg -c \"adapter speed 5000\"", + "verify": true } }, { @@ -322,7 +326,8 @@ "flasher": { "name": "openocd", "uid": "E663AC91D3359B38", - "args": "-f interface/cmsis-dap.cfg -f target/rp2350.cfg -c \"adapter speed 5000\"" + "args": "-f interface/cmsis-dap.cfg -f target/rp2350.cfg -c \"adapter speed 5000\"", + "verify": true } }, { @@ -404,7 +409,8 @@ "flasher": { "name": "openocd", "uid": "004C00343137510F39383538", - "args": "-f interface/stlink.cfg -f target/stm32h7x.cfg" + "args": "-f interface/stlink.cfg -f target/stm32h7x.cfg", + "verify": true } }, { @@ -418,7 +424,8 @@ "flasher": { "name": "openocd", "uid": "066FFF495087534867063844", - "args": "-f interface/stlink.cfg -f target/stm32g0x.cfg" + "args": "-f interface/stlink.cfg -f target/stm32g0x.cfg", + "verify": true }, "comment": "32-bit scheme, 2KB USB SRAM" }, @@ -463,9 +470,10 @@ "dual": false }, "flasher": { - "name": "openocd_wch", + "name": "openocd", "uid": "A76D8F062C2A", - "args": "-f target/wch-riscv.cfg" + "args": "-f target/wch-riscv.cfg", + "verify": false } }, { @@ -478,9 +486,10 @@ "dual": false }, "flasher": { - "name": "openocd_wch", + "name": "openocd", "uid": "BC4954081051", - "args": "-f target/wch-riscv.cfg" + "args": "-f target/wch-riscv.cfg", + "verify": false } }, { @@ -497,9 +506,10 @@ "dual": false }, "flasher": { - "name": "openocd_wch", + "name": "openocd", "uid": "BC5DA47360D0", - "args": "-f target/wch-riscv.cfg" + "args": "-f target/wch-riscv.cfg", + "verify": false } }, { @@ -512,9 +522,10 @@ "dual": false }, "flasher": { - "name": "openocd_wch", + "name": "openocd", "uid": "57468F06DC03", - "args": "-f target/wch-riscv.cfg" + "args": "-f target/wch-riscv.cfg", + "verify": false } }, { @@ -561,22 +572,6 @@ "uid": "1051856258", "args": "-device NRF54LM20A_M33" } - }, - { - "name": "ra6m5_ek", - "uid": "8419032D32363657364EF4622D294B4E", - "tests": { - "device": true, - "host": false, - "dual": false, - "skip": ["device/cdc_msc_throughput", "device/msc_dual_lun"], - "comment": "MSC writes wedge the uPD720201 host (URBs queued, zero wire activity, bus-15 ctrl xfers time out until the device's URBs are killed); reproduced identically with master firmware - device side armed+BUF and exonerated. MSC reads and usbtest bulk (15.8 MB/s) are fine" - }, - "flasher": { - "name": "jlink", - "uid": "000831915224", - "args": "-device R7FA6M5BH" - } } ], "boards-skip": [ @@ -612,6 +607,23 @@ "uid": "000778170924", "args": "-device stm32f769ni" } + }, + { + "name": "ra6m5_ek", + "uid": "8419032D32363657364EF4622D294B4E", + "comment": "Unstable in CI: intermittent usbtest failures plus cdc_dual_ports/hid_boot_interface/midi_test/mtp/printer_to_cdc flapping. Parked until diagnosed", + "tests": { + "device": true, + "host": false, + "dual": false, + "skip": ["device/cdc_msc_throughput", "device/msc_dual_lun"], + "comment": "MSC writes wedge the uPD720201 host (URBs queued, zero wire activity, bus-15 ctrl xfers time out until the device's URBs are killed); reproduced identically with master firmware - device side armed+BUF and exonerated. MSC reads and usbtest bulk (15.8 MB/s) are fine" + }, + "flasher": { + "name": "jlink", + "uid": "000831915224", + "args": "-device R7FA6M5BH" + } } ] } -- cgit v1.3.1 From 3963a1b70a572132aced1c1a0033e1c8249a0c7e Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 14 Aug 2026 01:08:40 +0700 Subject: test/hil, ci: contain a wedged USB stack instead of stranding the runner A wedged USB device used to take the whole HIL run with it. Every worker that touched the poisoned node blocked uninterruptibly, the pool could not be joined, map_async discarded every board's result, and the job ran to the GitHub ceiling with no report at all -- while the self-hosted runner's single job slot stayed occupied and every queued job waited behind it. Bound the calls a worker makes itself. read_sysfs, bounded_open and run_cmd all answer within a wall clock; read_sysfs distinguishes "absent" from "unknown", because a blocked read is not evidence of absence, and caps stranded readers at four (each costs a thread and an fd for the life of the process) after which the worker declares itself blind. mtype, the gio unmount, the libmtp session and the arecord/iperf reaps go through those bounds; the MTP session runs in a disposable subprocess, since libmtp's ctypes calls block unkillably in D state. Bound the run. A pool guard (HIL_POOL_TIMEOUT, 60 min) fires before any job ceiling and still writes a report. When the pool will not shut down, the sweep kills what the workers spawned -- descendants, not just direct children, since flashers run in their own session -- confirms each kill actually landed, and exits early so the runner is freed. Whatever survived is named in the report. Deliberately shallow past that point. We do not re-scan process groups, prove pid ownership, or escalate through sudo: a root-owned survivor is reported, not force-killed, because signalling a pid we cannot prove is ours is the worse failure, and the job ceiling backstops whatever this misses. A D-state holder was never killable anyway. Recover instead of reporting a wedge. A HUNG usbtest case reflashes its own DUT through its roster flasher, but only where the flasher can reach its probe past a poisoned node -- openocd pinned to a validated vid_pid, or esptool. Where it cannot, the run says so rather than reserving budget for a path that cannot fire. Raise the CI ceilings above the pool guard so the guard fires first and still writes its report, and pin --retry 1 on every HIL leg: the guard is a flat constant and does not scale with max_retry, so argparse's default of 3 would triple the serialized usbtest tail against an unchanged guard. Split the module: execution in hil_test/hil_flash/usbtest, infrastructure in helper/ (locking, health, selection, shared bounded IO), and the two matrix generators into .github/scripts/ -- ci_set_matrix.py sat in workflows/, where GitHub treats every file as a workflow definition. 193 tests cover the bounded paths, the kill ladder, the guard and the selector against synthetic /proc trees and PATH-injected fakes; a real wedge cannot be manufactured on demand. --- .circleci/config.yml | 2 +- .github/scripts/ci_set_matrix.py | 111 ++ .github/scripts/hil_ci_set_matrix.py | 92 ++ .github/workflows/build.yml | 71 +- .github/workflows/ci_set_matrix.py | 111 -- .github/workflows/pre-commit.yml | 1 + .pre-commit-config.yaml | 21 + CLAUDE.md | 3 +- .../2026-07-29-hil-pr-scoped-selection-design.md | 22 +- hw/bsp/mcx/family.cmake | 2 +- test/hil/helper/__init__.py | 4 + test/hil/helper/hil_health.py | 380 +++++ test/hil/helper/hil_lock.py | 525 ++++++ test/hil/helper/hil_pool_check.py | 1019 ++++++++++++ test/hil/helper/hil_select.py | 524 ++++++ test/hil/helper/hil_util.py | 585 +++++++ test/hil/hil_ci.sh | 85 +- test/hil/hil_ci_set_matrix.py | 90 -- test/hil/hil_examples.py | 37 - test/hil/hil_flash.py | 365 +++-- test/hil/hil_lock.py | 479 ------ test/hil/hil_pool_check.py | 1015 ------------ test/hil/hil_select.py | 520 ------ test/hil/hil_test.py | 1604 ++++++++++++------ test/hil/mtp_test.py | 246 +++ test/hil/test/stubs/pymtp.py | 125 ++ test/hil/test/test_hil_bounded.py | 1701 ++++++++++++++++++++ test/hil/test/test_hil_health.py | 636 ++++++++ test/hil/test/test_hil_select.py | 689 ++++++++ test/hil/test/test_hil_util.py | 230 +++ test/hil/test_hil_select.py | 581 ------- test/hil/tinyusb.json | 15 +- test/hil/usbtest.py | 647 ++++++-- tools/metrics_compare_base.py | 4 +- 34 files changed, 8821 insertions(+), 3721 deletions(-) create mode 100755 .github/scripts/ci_set_matrix.py create mode 100644 .github/scripts/hil_ci_set_matrix.py delete mode 100755 .github/workflows/ci_set_matrix.py create mode 100644 test/hil/helper/__init__.py create mode 100644 test/hil/helper/hil_health.py create mode 100755 test/hil/helper/hil_lock.py create mode 100644 test/hil/helper/hil_pool_check.py create mode 100755 test/hil/helper/hil_select.py create mode 100644 test/hil/helper/hil_util.py delete mode 100644 test/hil/hil_ci_set_matrix.py delete mode 100644 test/hil/hil_examples.py delete mode 100755 test/hil/hil_lock.py delete mode 100644 test/hil/hil_pool_check.py delete mode 100755 test/hil/hil_select.py create mode 100644 test/hil/mtp_test.py create mode 100644 test/hil/test/stubs/pymtp.py create mode 100644 test/hil/test/test_hil_bounded.py create mode 100644 test/hil/test/test_hil_health.py create mode 100644 test/hil/test/test_hil_select.py create mode 100644 test/hil/test/test_hil_util.py delete mode 100644 test/hil/test_hil_select.py (limited to 'test/hil/tinyusb.json') diff --git a/.circleci/config.yml b/.circleci/config.yml index 66799910d..48fa87899 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ jobs: - run: name: Set matrix command: | - MATRIX_JSON=$(python .github/workflows/ci_set_matrix.py) + MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py) echo "MATRIX_JSON=$MATRIX_JSON" BUILDSYSTEM_LIST=( diff --git a/.github/scripts/ci_set_matrix.py b/.github/scripts/ci_set_matrix.py new file mode 100755 index 000000000..50ada5964 --- /dev/null +++ b/.github/scripts/ci_set_matrix.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +import json + +# toolchain, url +toolchain_list = [ + "aarch64-gcc", + "arm-clang", + "arm-iar", + "arm-gcc", + "esp-idf", + "ft9xx-gcc", + "msp430-gcc", + "riscv-gcc", + "rx-gcc" +] + +# family: [supported toolchain] +family_list = { + "apm32f0xx": ["arm-gcc"], + "at32f402_405": ["arm-gcc"], + "at32f403a_407": ["arm-gcc"], + "at32f413": ["arm-gcc"], + "at32f415": ["arm-gcc"], + "at32f423": ["arm-gcc"], + "at32f425": ["arm-gcc"], + "at32f435_437": ["arm-gcc"], + "at32f45x": ["arm-gcc"], + "broadcom_32bit": ["arm-gcc"], + "broadcom_64bit": ["aarch64-gcc"], + "ch32f20x": ["arm-gcc"], + "ch32v10x": ["riscv-gcc"], + "ch32v20x": ["riscv-gcc"], + "ch32v30x": ["riscv-gcc"], + "ch583": ["riscv-gcc"], + "da1469x": ["arm-gcc"], + "fomu": ["riscv-gcc"], + "ft9xx": ["ft9xx-gcc"], + "gd32vf103": ["riscv-gcc"], + "hpmicro": ["riscv-gcc"], + "imxrt": ["arm-gcc", "arm-clang"], + "kinetis_k": ["arm-gcc"], + "kinetis_k32l": ["arm-gcc"], + "kinetis_kl": ["arm-gcc"], + "lpc11": ["arm-gcc", "arm-clang"], + "lpc13": ["arm-gcc", "arm-clang"], + "lpc15": ["arm-gcc", "arm-clang"], + "lpc17": ["arm-gcc", "arm-clang"], + "lpc18": ["arm-gcc", "arm-clang"], + "lpc40": ["arm-gcc", "arm-clang"], + "lpc43": ["arm-gcc", "arm-clang"], + "lpc51": ["arm-gcc", "arm-clang"], + "lpc54": ["arm-gcc", "arm-clang"], + "lpc55": ["arm-gcc", "arm-clang"], + "maxim": ["arm-gcc"], + "mcx": ["arm-gcc"], + "mm32": ["arm-gcc"], + "msp430": ["msp430-gcc"], + "msp432e4": ["arm-gcc"], + "nrf": ["arm-gcc", "arm-clang"], + "nuc100_120": ["arm-gcc"], + "nuc121_125": ["arm-gcc"], + "nuc126": ["arm-gcc"], + "nuc505": ["arm-gcc"], + "ra": ["arm-gcc"], + "rp2040": ["arm-gcc"], + "rw61x": ["arm-gcc"], + "rx": ["rx-gcc"], + "samd11": ["arm-gcc", "arm-clang"], + "samd2x_l2x": ["arm-gcc", "arm-clang"], + "samd5x_e5x": ["arm-gcc", "arm-clang"], + "samg": ["arm-gcc", "arm-clang"], + "stm32c0": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32c5": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32f0": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32f1": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32f2": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32f3": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32f4": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32f7": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32g0": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32g4": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32h5": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32h7": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32h7rs": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32l0": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32l4": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32n6": ["arm-gcc"], + "stm32u0": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32u5": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32wb": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32wba": ["arm-gcc", "arm-clang", "arm-iar"], + "tm4c": ["arm-gcc"], + "xmc4000": ["arm-gcc"], + # S3, P4 will be built by hil test + # "-bespressif_s3_devkitm": ["esp-idf"], + # "-bespressif_p4_function_ev": ["esp-idf"], +} + + +def set_matrix_json(): + matrix = {} + for toolchain in toolchain_list: + filtered_families = [family for family, supported_toolchain in family_list.items() if + toolchain in supported_toolchain] + matrix[toolchain] = filtered_families + + print(json.dumps(matrix)) + + +if __name__ == '__main__': + set_matrix_json() diff --git a/.github/scripts/hil_ci_set_matrix.py b/.github/scripts/hil_ci_set_matrix.py new file mode 100644 index 000000000..65f50788e --- /dev/null +++ b/.github/scripts/hil_ci_set_matrix.py @@ -0,0 +1,92 @@ +import argparse +import json +import os + + +def _resolve_config_path(config_file): + if os.path.exists(config_file): + return config_file + + # bare roster names resolve against the repo's test/hil (this script lives in + # .github/scripts); build.yml passes explicit paths, this is for hand-runs + repo_relative = os.path.join(os.path.dirname(__file__), '..', '..', 'test', 'hil', config_file) + if os.path.exists(repo_relative): + return repo_relative + + raise FileNotFoundError(f'Config file not found: {config_file}') + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('config_files', nargs='+', help='Configuration JSON file(s)') + parser.add_argument('--select', help='hil_select.py JSON; scopes boards when full=false') + args = parser.parse_args() + + selected = None + sel = json.loads(args.select) if args.select else None + if sel and not sel.get('full'): + selected = set(sel.get('boards', {})) + + # Toolchain buckets must match the toolchains instantiated by the hil-build + # job in .github/workflows/build.yml. Keep all keys present (even if empty) + # so `fromJSON(hil_json)[toolchain]` always resolves to a list. + matrix = { + 'arm-gcc': [], + 'riscv-gcc': [], + 'esp-idf': [] + } + + seen = {toolchain: set() for toolchain in matrix} + + def append_build_arg(toolchain, build_arg): + if build_arg not in seen[toolchain]: + seen[toolchain].add(build_arg) + matrix[toolchain].append(build_arg) + + for config_file in args.config_files: + with open(_resolve_config_path(config_file)) as f: + config = json.load(f) + + for board in config['boards']: + if selected is not None and board['name'] not in selected: + continue + name = board['name'] + flasher = board['flasher'] + # esptool boards must build under esp-idf; others default to arm-gcc + # but may opt into another bucket via an explicit "toolchain" field + # (e.g. RISC-V boards like ch32v20x need "riscv-gcc"). + if flasher['name'] == 'esptool': + toolchain = 'esp-idf' + else: + toolchain = board.get('toolchain', 'arm-gcc') + if toolchain not in matrix: + # a board in no bucket would never be built, and the bare KeyError + # below would only say so as a traceback from the set-matrix job + raise SystemExit( + f'{name}: toolchain {toolchain!r} is not a build bucket ' + f'({", ".join(matrix)}); add it here and to the hil-build / ' + f'hil-build-esp jobs in .github/workflows/build.yml') + + build_board = f'-b {name}' + if 'build' in board and 'args' in board['build']: + build_board += ' ' + ' '.join(f'-D{a}' for a in board['build']['args']) + + # Each variant builds into cmake-build- with its own cmake + # -D defines and raw CFLAGS. No 'variant' -> a single build named after + # the board. + variants = board.get('variant') or [{'name': name, 'flags': ''}] + for v in variants: + arg = build_board + if v['name'] != name: + arg += f' --build-name {v["name"]}' + for d in v.get('defines', []): + arg += f' -D{d}' + for tok in v.get('flags', '').split(): + arg += f' --cflag={tok}' + append_build_arg(toolchain, arg) + + print(json.dumps(matrix)) + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bdef81553..8f6014f48 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,7 +41,7 @@ jobs: - '.github/actions/**' - '.github/workflows/build.yml' - '.github/workflows/build_util.yml' - - '.github/workflows/ci_set_matrix.py' + - '.github/scripts/**' set-matrix: runs-on: ubuntu-latest @@ -72,10 +72,16 @@ jobs: # so a missing origin/, a shallow-clone hiccup or a selector traceback # must fall back to the FULL matrix (no --select, run=true, no args) instead # of failing the job. Same fail-open shape as pr_comment.yml's `|| true`. + # + # The selector's own unit suite gates it (stdlib-only, seconds): a selector + # whose tests fail can still exit 0 with valid-but-WRONG JSON -- fail-open alone + # never catches that class, and the pre-commit hil-test hook is a separate, + # advisory workflow that nothing here can `needs:`. Test-failing selector => + # full matrix, same as a crashing one. SELECT_JSON='' - if ! python3 test/hil/test_hil_select.py; then - echo "::error::hil_select unit tests failed - falling back to the full HIL matrix" - elif ! SELECT_JSON=$(python3 test/hil/hil_select.py --base "origin/$BASE_REF" test/hil/tinyusb.json test/hil/hfp.json); then + if ! python3 test/hil/test/test_hil_select.py; then + echo "::warning::hil_select unit suite failed - falling back to the full HIL matrix" + elif ! SELECT_JSON=$(python3 test/hil/helper/hil_select.py --base "origin/$BASE_REF" test/hil/tinyusb.json test/hil/hfp.json); then echo "::warning::hil_select failed - falling back to the full HIL matrix" SELECT_JSON='' fi @@ -113,7 +119,7 @@ jobs: SELECT: ${{ steps.hil-select.outputs.select }} run: | # build matrix - MATRIX_JSON=$(python .github/workflows/ci_set_matrix.py) + MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py) echo "matrix=$MATRIX_JSON" echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT @@ -121,13 +127,13 @@ jobs: # Scoping is best-effort too: fall back to the unscoped (full) matrix. HIL_MATRIX_JSON='' if [ -n "$SELECT" ]; then - HIL_MATRIX_JSON=$(python test/hil/hil_ci_set_matrix.py --select "$SELECT" test/hil/tinyusb.json test/hil/hfp.json) || HIL_MATRIX_JSON='' + HIL_MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py --select "$SELECT" test/hil/tinyusb.json test/hil/hfp.json) || HIL_MATRIX_JSON='' if [ -z "$HIL_MATRIX_JSON" ]; then echo "::warning::scoped HIL matrix failed - falling back to the full HIL matrix" fi fi if [ -z "$HIL_MATRIX_JSON" ]; then - HIL_MATRIX_JSON=$(python test/hil/hil_ci_set_matrix.py test/hil/tinyusb.json test/hil/hfp.json) + HIL_MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py test/hil/tinyusb.json test/hil/hfp.json) fi echo "hil_matrix=$HIL_MATRIX_JSON" echo "hil_matrix=$HIL_MATRIX_JSON" >> $GITHUB_OUTPUT @@ -338,7 +344,7 @@ jobs: strategy: fail-fast: false matrix: - # These names are the bucket keys of test/hil/hil_ci_set_matrix.py: every + # These names are the bucket keys of .github/scripts/hil_ci_set_matrix.py: every # non-esptool roster board must land in one of them (esptool boards go to # 'esp-idf', built by hil-build-esp below). hil_ci_set_matrix.py rejects a # board whose "toolchain" is not a bucket, so a new bucket must be added in @@ -379,6 +385,14 @@ jobs: hil-tinyusb: needs: [ hil-build, set-matrix ] name: hil-tinyusb (${{ matrix.display }}) + # Above hil_test.py's pool guard (HIL_POOL_TIMEOUT, 60 min) so the guard fires first + # and still gets to write its report. The 30 min on top is what the job pays OUTSIDE + # the guard clock: workspace cleanup, checkout, the multi-board artifact merge and + # the D-state note before it; kill_worker_children, shutdown_pool's 30 s grace, the + # report write and the upload after it. On a multi-stray convoy that tail alone is + # minutes, and a ceiling below guard+tail cancels the job before hil_report.md exists + # -- the inversion this branch removes. Both legs share the script and the guard. + timeout-minutes: 90 strategy: fail-fast: false matrix: @@ -395,6 +409,9 @@ jobs: test_args: '' runs-on: ${{ matrix.runner }} env: + # HIL_POOL_TIMEOUT deliberately unset: hil_test.py's 60 min default is below every + # ceiling here, so ceiling > guard holds by construction. Pin it to SHORTEN a run + # only -- pinning it above a ceiling re-inverts the two. HIL_JSON: ${{ matrix.hil_json }} steps: - name: Set HIL report dir (per run+job; persists across run attempts) @@ -482,6 +499,11 @@ jobs: needs: [ hil-build-esp, set-matrix ] name: hil-tinyusb (tinyusb-esp.json) runs-on: [ self-hosted, X64, hathach, hardware-in-the-loop ] + # above hil_test.py's pool guard (60 min) with room for the pre-pool checkout + # and the post-guard sweep + report upload, so its own guard still writes a report; + # only a job wedged past that (unkillable D-state worker) hits this ceiling, which + # must exist because the runner has one job slot and holds every queued job hostage + timeout-minutes: 90 env: HIL_JSON: test/hil/tinyusb.json TEST_ARGS: '--flasher esptool' @@ -564,7 +586,13 @@ jobs: github.repository_owner == 'hathach' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == true) runs-on: [ self-hosted, Linux, X64, hifiphile ] - timeout-minutes: 30 + # Unlike the hil-tinyusb jobs, this one BUILDS with IAR in the same job before running + # hil_test.py -- hfp.json's 3 boards, 4 variant entries, "up to 30 minutes" (see the + # comment above the selection step). The ceiling has to cover build + the 60 min pool + # guard + overhead, or GitHub cancels before the guard can write its report -- the + # inversion this branch removes. 30 + 60 = 90; the remaining 30 is the full-history + # checkout, get_deps, the post-guard sweep and the report upload. + timeout-minutes: 120 env: IAR_LMS_BEARER_TOKEN: ${{ secrets.IAR_LMS_BEARER_TOKEN }} PYTHONUNBUFFERED: '1' @@ -601,7 +629,12 @@ jobs: # failures cannot kill hfp coverage - a selector failure here must likewise # fall back to the full hfp matrix (no hil_select.json, no SEL_* vars), never # fail the job. - if ! python3 test/hil/hil_select.py --base "origin/$BASE_REF" test/hil/hfp.json > hil_select.json; then + if ! python3 test/hil/test/test_hil_select.py; then + echo "::warning::hil_select unit suite failed - running the full hfp matrix" + rm -f hil_select.json + exit 0 + fi + if ! python3 test/hil/helper/hil_select.py --base "origin/$BASE_REF" test/hil/hfp.json > hil_select.json; then echo "::warning::hil_select failed - running the full hfp matrix" rm -f hil_select.json exit 0 @@ -629,9 +662,9 @@ jobs: if: env.SEL_RUN != 'false' run: | if [ -f hil_select.json ]; then - MATRIX_JSON=$(python test/hil/hil_ci_set_matrix.py --select "$(cat hil_select.json)" test/hil/hfp.json) + MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py --select "$(cat hil_select.json)" test/hil/hfp.json) else - MATRIX_JSON=$(python test/hil/hil_ci_set_matrix.py test/hil/hfp.json) + MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py test/hil/hfp.json) fi # Each variant carries its own --build-name/--cflag, which are global to a # single build.py invocation — so keep one matrix entry per line and build @@ -648,6 +681,13 @@ jobs: - name: Build if: env.SEL_RUN != 'false' + # Bounded SEPARATELY from the job. This is the only HIL job that builds inline + # (hil-tinyusb downloads artifacts), and the job ceiling went 30 -> 120 to give the + # HIL step room -- which would hand a stalled IAR build the whole two hours on the + # shared self-hosted runner, never reaching hil_test.py or the report upload. That + # is the stranded-runner-with-no-report failure this branch exists to prevent. + # Typical full build here is a few minutes; 30 leaves generous headroom. + timeout-minutes: 30 run: | readarray -t ENTRIES < hil_build_entries.txt for entry in "${ENTRIES[@]}"; do @@ -666,7 +706,12 @@ jobs: fi # empty/absent on a non-PR event or a selector fallback -> full hfp matrix SEL_ARGS=$(cat hil_sel_args.txt 2>/dev/null || true) - python3 test/hil/hil_test.py $SEL_ARGS hfp.json + # --retry 1, like the other two HIL legs. The pool guard is a FLAT 3600s and + # does NOT scale with max_retry, so argparse's default of 3 would multiply the + # serialized usbtest tail (hfp.json runs four batteries) by three against an + # unchanged guard -- on a runner with a single job slot that queues every other + # job behind it. + python3 test/hil/hil_test.py --retry 1 $SEL_ARGS hfp.json - name: Upload HIL report if: always() && github.event_name == 'pull_request' diff --git a/.github/workflows/ci_set_matrix.py b/.github/workflows/ci_set_matrix.py deleted file mode 100755 index 50ada5964..000000000 --- a/.github/workflows/ci_set_matrix.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python3 -import json - -# toolchain, url -toolchain_list = [ - "aarch64-gcc", - "arm-clang", - "arm-iar", - "arm-gcc", - "esp-idf", - "ft9xx-gcc", - "msp430-gcc", - "riscv-gcc", - "rx-gcc" -] - -# family: [supported toolchain] -family_list = { - "apm32f0xx": ["arm-gcc"], - "at32f402_405": ["arm-gcc"], - "at32f403a_407": ["arm-gcc"], - "at32f413": ["arm-gcc"], - "at32f415": ["arm-gcc"], - "at32f423": ["arm-gcc"], - "at32f425": ["arm-gcc"], - "at32f435_437": ["arm-gcc"], - "at32f45x": ["arm-gcc"], - "broadcom_32bit": ["arm-gcc"], - "broadcom_64bit": ["aarch64-gcc"], - "ch32f20x": ["arm-gcc"], - "ch32v10x": ["riscv-gcc"], - "ch32v20x": ["riscv-gcc"], - "ch32v30x": ["riscv-gcc"], - "ch583": ["riscv-gcc"], - "da1469x": ["arm-gcc"], - "fomu": ["riscv-gcc"], - "ft9xx": ["ft9xx-gcc"], - "gd32vf103": ["riscv-gcc"], - "hpmicro": ["riscv-gcc"], - "imxrt": ["arm-gcc", "arm-clang"], - "kinetis_k": ["arm-gcc"], - "kinetis_k32l": ["arm-gcc"], - "kinetis_kl": ["arm-gcc"], - "lpc11": ["arm-gcc", "arm-clang"], - "lpc13": ["arm-gcc", "arm-clang"], - "lpc15": ["arm-gcc", "arm-clang"], - "lpc17": ["arm-gcc", "arm-clang"], - "lpc18": ["arm-gcc", "arm-clang"], - "lpc40": ["arm-gcc", "arm-clang"], - "lpc43": ["arm-gcc", "arm-clang"], - "lpc51": ["arm-gcc", "arm-clang"], - "lpc54": ["arm-gcc", "arm-clang"], - "lpc55": ["arm-gcc", "arm-clang"], - "maxim": ["arm-gcc"], - "mcx": ["arm-gcc"], - "mm32": ["arm-gcc"], - "msp430": ["msp430-gcc"], - "msp432e4": ["arm-gcc"], - "nrf": ["arm-gcc", "arm-clang"], - "nuc100_120": ["arm-gcc"], - "nuc121_125": ["arm-gcc"], - "nuc126": ["arm-gcc"], - "nuc505": ["arm-gcc"], - "ra": ["arm-gcc"], - "rp2040": ["arm-gcc"], - "rw61x": ["arm-gcc"], - "rx": ["rx-gcc"], - "samd11": ["arm-gcc", "arm-clang"], - "samd2x_l2x": ["arm-gcc", "arm-clang"], - "samd5x_e5x": ["arm-gcc", "arm-clang"], - "samg": ["arm-gcc", "arm-clang"], - "stm32c0": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32c5": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32f0": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32f1": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32f2": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32f3": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32f4": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32f7": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32g0": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32g4": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32h5": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32h7": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32h7rs": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32l0": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32l4": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32n6": ["arm-gcc"], - "stm32u0": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32u5": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32wb": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32wba": ["arm-gcc", "arm-clang", "arm-iar"], - "tm4c": ["arm-gcc"], - "xmc4000": ["arm-gcc"], - # S3, P4 will be built by hil test - # "-bespressif_s3_devkitm": ["esp-idf"], - # "-bespressif_p4_function_ev": ["esp-idf"], -} - - -def set_matrix_json(): - matrix = {} - for toolchain in toolchain_list: - filtered_families = [family for family, supported_toolchain in family_list.items() if - toolchain in supported_toolchain] - matrix[toolchain] = filtered_families - - print(json.dumps(matrix)) - - -if __name__ == '__main__': - set_matrix_json() diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 70dd3894d..09f912bd3 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -30,6 +30,7 @@ jobs: #cd test/unit-test #ceedling test:all + # runs --all-files, so the hil-test hook fires here regardless of its `files:` scope - name: Run pre-commit uses: pre-commit/action@v3.0.1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e87b935dd..3d9c8482b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,6 +48,27 @@ repos: types_or: [c, header] language: system + # Two hooks, split by what each suite actually reads. The full discovery run costs + # ~55s (deliberate hang/timeout simulations); only test_hil_select (~0.1s) reads + # hw/bsp (board.cmake), src (portable dirs + class include graph) and examples + # (tusb_config.h per test) -- renaming a board, port dir or example breaks it without + # touching test/hil, and catching that here beats waiting for pre-commit CI. + # No types_or: the rig rosters (*.json) are inputs too. + # examples/device/mtp/src is in scope: test_hil_bounded parses README_TXT_CONTENT + # and md5-checks the logo header from there as its MTP fixtures. + - id: hil-test + name: hil-test + files: ^(test/hil/|examples/device/mtp/src/) + entry: python3 -m unittest discover -s test/hil/test + pass_filenames: false + language: system + - id: hil-select-test + name: hil-select-test + files: ^(hw/bsp/|src/|examples/) + entry: python3 test/hil/test/test_hil_select.py + pass_filenames: false + language: system + # - id: build-fuzzer # name: build-fuzzer # files: ^(src/|test/fuzz/) diff --git a/CLAUDE.md b/CLAUDE.md index 94b8192b7..fd4b9b8e0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -99,7 +99,8 @@ Use the `pvs` skill (`.claude/skills/pvs/SKILL.md`) — it builds the examples w ## Validation After Changes -1. `pre-commit run --all-files` — format, spell, unit tests (10-15 s). +1. `pre-commit run --all-files` — format, spell, unit tests, HIL suites (~55 s; the + HIL hooks deliberately exercise real timeouts and hangs). 2. Build at least one board's full example set (Build → "All examples for a board") for modules you touched. 3. Run relevant unit tests; add fuzz/HIL coverage for parsers or protocol state machines. diff --git a/docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md b/docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md index 8158758bc..898b3c8ab 100644 --- a/docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md +++ b/docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md @@ -1,4 +1,4 @@ -# PR-scoped HIL selection: hil_select.py +# PR-scoped HIL selection: helper/hil_select.py **Date:** 2026-07-29 **Branch:** `claude/hil-select` (based on `claude/hil-pool-check`, which carries the @@ -26,17 +26,17 @@ confident; every uncertainty widens to the full matrix. - Scoping push/master/scheduled runs (always full). - Changing hil_test.py behavior (the selector only *composes* existing `-b`/`-bt` args). -## Component: `test/hil/hil_select.py` +## Component: `test/hil/helper/hil_select.py` Stdlib-only, importable and CLI. Lives beside the harness so `hil_ci.sh` copies are unaffected (it runs on the GitHub runner / dev PC, not on the rig). It must NOT import `hil_test.py` (which drags pyserial/pymtp onto the bare GitHub runner): the three test lists -(`device_tests`, `dual_tests`, `host_test`) move verbatim into a tiny stdlib-only -`test/hil/hil_examples.py` that both `hil_test.py` and `hil_select.py` import (behavior -preserving; `hil_ci.sh` scp list gains the new file). +(`device_tests`, `dual_tests`, `host_test`) move verbatim into the stdlib-only +`test/hil/helper/hil_util.py` that both `hil_test.py` and `hil_select.py` import (behavior +preserving; `hil_ci.sh` copies the whole `helper/` directory). ``` -python3 test/hil/hil_select.py --base [--diff-file ] CONFIG.json [CONFIG.json...] +python3 test/hil/helper/hil_select.py --base [--diff-file ] CONFIG.json [CONFIG.json...] ``` - `--base REF`: changed files = `git diff --name-only $(git merge-base HEAD REF)..HEAD` @@ -118,7 +118,7 @@ is skipped, not widened (running unrelated boards would test nothing relevant). ## CI wiring (`.github/workflows/build.yml`) - `set-matrix` (PR events only): after generating today's matrices, run - `hil_select.py --base origin/${{ github.base_ref }} test/hil/tinyusb.json test/hil/hfp.json` + `helper/hil_select.py --base origin/${{ github.base_ref }} test/hil/tinyusb.json test/hil/hfp.json` (checkout with enough history to reach the merge base: `fetch-depth: 0` on this one job, or an explicit `git fetch origin $BASE_REF`). New job outputs: `hil_select_full`, `hil_args_tinyusb`, `hil_args_hfp`, plus the selected-board list consumed by the matrix @@ -137,15 +137,15 @@ is skipped, not widened (running unrelated boards would test nothing relevant). ## Local use - pre-pr's "Map changes to boards" step delegates to - `python3 test/hil/hil_select.py --base $BASE test/hil/tinyusb.json` and derives its + `python3 test/hil/helper/hil_select.py --base $BASE test/hil/tinyusb.json` and derives its one-board-per-family sample from the selector's board set (its capping/sampling policy is unchanged — the selector provides the affected set, pre-pr samples it). -- Manual: `python3 test/hil/hil_test.py -B examples $(python3 test/hil/hil_select.py --base master test/hil/tinyusb.json | jq -r '.args["tinyusb.json"]') test/hil/tinyusb.json` +- Manual: `python3 test/hil/hil_test.py -B examples $(python3 test/hil/helper/hil_select.py --base master test/hil/tinyusb.json | jq -r '.args["tinyusb.json"]') test/hil/tinyusb.json` — documented in the hil skill. ## Testing -`test/hil/test_hil_select.py` — stdlib `unittest`, no hardware, injected diffs via +`test/hil/test/test_hil_select.py` — stdlib `unittest`, no hardware, injected diffs via `--diff-file`/API. Cases (the acceptance examples): 1. `src/portable/raspberrypi/rp2040/dcd_rp2040.c` → only rp2040-family roster boards, device tests only, host-only boards absent, `full` false. @@ -161,7 +161,7 @@ is skipped, not widened (running unrelated boards would test nothing relevant). 7. `hw/bsp/rp2040/family.cmake` → rp2040-family boards, all their tests. 8. Mixed device+host diff → no pruning (both roles present). The suite runs in `set-matrix` before the selector is used, and locally via -`python3 test/hil/test_hil_select.py`. +`python3 test/hil/test/test_hil_select.py`. ## Safety properties diff --git a/hw/bsp/mcx/family.cmake b/hw/bsp/mcx/family.cmake index 60f43e152..b2b4fd45b 100644 --- a/hw/bsp/mcx/family.cmake +++ b/hw/bsp/mcx/family.cmake @@ -95,7 +95,7 @@ function(family_configure_example TARGET RTOS) endif() # PORT is set per board (board.cmake), so pick the driver at configure time. Spelled out - # rather than $ so the port path stays greppable: test/hil/hil_select.py + # rather than $ so the port path stays greppable: test/hil/helper/hil_select.py # maps a portable-driver change to the families whose build file names that directory. if (PORT) set(PORT_SRC ${TOP}/src/portable/chipidea/ci_hs/dcd_ci_hs.c) diff --git a/test/hil/helper/__init__.py b/test/hil/helper/__init__.py new file mode 100644 index 000000000..a080a2f55 --- /dev/null +++ b/test/hil/helper/__init__.py @@ -0,0 +1,4 @@ +# Marks helper/ as a REGULAR package. Without this it is only a PEP 420 namespace portion, +# and a regular package named `helper` anywhere on sys.path wins over it even though +# test/hil is sys.path[0] -- one transitive pip install would break every HIL entry point +# at import. `helper` is a real distribution name on PyPI. diff --git a/test/hil/helper/hil_health.py b/test/hil/helper/hil_health.py new file mode 100644 index 000000000..92f0accc8 --- /dev/null +++ b/test/hil/helper/hil_health.py @@ -0,0 +1,380 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Shutting a wedged HIL run down: kill what the workers spawned, then report. + +A device whose usbfs node is held by a D-state process cannot be freed -- SIGKILL is not +delivered in uninterruptible sleep -- so the goal is never to fix the rig from here. It is +to free the runner's single job slot and leave a report naming what survived, instead of +letting the job sit until GitHub cancels it with nothing to show. + +Deliberately shallow. We SIGKILL the process groups the workers spawned, wait a grace, +and report whoever is still alive; we do not re-scan groups, prove pid ownership or +escalate through sudo. A root-owned survivor is named in the report for hil_pool_check +and the usb-kernel-recover skill to deal with -- signalling a pid we cannot prove is ours +is the worse failure, and the job ceiling backstops whatever this misses. + +Everything here is stdlib-only and reads /proc unprivileged (dmesg is restricted on the +rig), which keeps it importable -- and testable -- on a bare runner. +""" +import os +import signal +import threading +import time +from pathlib import Path + +PROC = Path('/proc') + + +# How long to let a SIGKILL land before calling a process a survivor. Generous enough to +# cover scheduling delay on a loaded rig, short enough that a fleet-wide sweep stays quick. +CONFIRM_KILL_GRACE = 2.0 + + +def _p(*args, **kwargs) -> None: + # These run on the free-the-runner path, where stdout can already be a dead pipe (a + # dropped ssh session). An unguarded print would raise BrokenPipeError out of + # hil_test's inner finally, skipping shutdown_pool AND the report writing. + try: + print(*args, **kwargs) + except (OSError, ValueError): + # ValueError, not just OSError: printing to a CLOSED stream raises + # "ValueError: I/O operation on closed file", and both hil_pool_check and + # hil_test redirect stdout into a StringIO that can be closed under us. Escaping + # here skips shutdown_pool/kill_pool_children/os._exit -- stranding the runner, + # the exact failure this wrapper exists to prevent. + pass + + +def _state(pid_dir: Path) -> str: + """The state letter from /proc//stat. comm can contain ')', so the field is + located from the right rather than by splitting.""" + # bytes, not read_text(): read_text decodes with the LOCALE encoding, so under LANG=C + # (systemd services, self-hosted runners) a non-ASCII comm raises UnicodeDecodeError + # and the entry silently vanishes from the scan. + stat = (pid_dir / 'stat').read_bytes() + return chr(stat[stat.rindex(b')') + 2]) + + +def _pids(): + """/proc pid entries. Yields nothing rather than raising if /proc is unreadable.""" + try: + entries = list(PROC.iterdir()) + except OSError: + return + for entry in entries: + if entry.name.isdigit(): + yield entry + + +def d_state_note() -> str: + """Pids in uninterruptible sleep, for the report. Never aborts, never blocks. + + A D-state process at start-up is NOT a fault on its own -- a healthy in-flight testusb + looks exactly like this, and the rig supports a dev run alongside CI. It is a hint for + whoever reads a red cell below. Diagnosis proper is hil_pool_check and the + usb-kernel-recover skill; this is one line, not a probe.""" + stuck = [] + for d in PROC.glob('[0-9]*'): + try: + if _state(d) == 'D': + stuck.append(d.name) + except (OSError, ValueError, IndexError): + pass # raced with exit, or /proc is restricted: not our problem here + if not stuck: + return '' + return (f'{len(stuck)} process(es) in D state when this run started: ' + f'{sorted(stuck)[:10]}') + + +def shutdown_pool(pool, grace: float = 30) -> bool: + """terminate() a worker Pool without ever blocking forever. + + multiprocessing joins its workers unbounded (util.py _exit_function terminate()s the + daemonic ones, then calls p.join() -- no timeout -- on every remaining active child, + CPython 3.13.5), and a worker in uninterruptible sleep never + reaps -- so terminate() itself hangs, taking the runner's only job slot with it. False + when the pool refuses to die within `grace` (the caller must then abandon it); a + terminate() that *raises* counts as failure too, the pool being just as alive.""" + outcome = {} + + def _term(): + try: + pool.terminate() + outcome['ok'] = True + except BaseException as e: # noqa: BLE001 - any failure means the pool is still up + # Say what happened: Pool._terminate_pool really can raise (CPython: + # AssertionError 'Cannot have cache with result_handler not alive'), and a + # swallowed one is indistinguishable from an unkillable D-state worker. + outcome['err'] = e + _p(f'warning: Pool.terminate() raised {type(e).__name__}: {e}', flush=True) + + t = threading.Thread(target=_term, daemon=True) + t.start() + t.join(grace) + # Decide on the thread, not the dict: _term may set outcome['ok'] after join(grace) + # expired, reporting a merely-slow terminate as success on one read and abandoned on + # another. Still inside terminate() == not shut down. + if t.is_alive(): + return False + return outcome.get('ok', False) + + +def child_procs(pids) -> dict: + """{ancestor pid in `pids`: [(descendant pid, its pgid), ...]}, from ONE walk of /proc. + + DESCENDANTS, not direct children: a worker's usbtest.py spawns its recovery flasher + through run_cmd (own session), so it is a GRANDCHILD that a direct-child sweep misses + and a kill mid-recovery would orphan on the probe. pgid comes back too because the two + kinds of child need different signals (see kill_pool_children).""" + wanted = set(pids) + by_parent: dict = {} # ppid -> [(pid, pgid), ...] for EVERY process + for entry in _pids(): + try: + stat = (entry / 'stat').read_bytes() + except OSError: + continue # exited between the scan and the read, or not readable + # comm (field 2) is parenthesised and may contain spaces and ')' -- so split only + # what follows the LAST ')': state, ppid, pgrp, ... + try: + fields = stat[stat.rindex(b')') + 2:].split() + ppid, pgid = int(fields[1]), int(fields[2]) + except (ValueError, IndexError): + continue # truncated or unparsable stat line + by_parent.setdefault(ppid, []).append((int(entry.name), pgid)) + out: dict = {} + for root in wanted: + todo = list(by_parent.get(root, [])) + while todo: + pid, pgid = todo.pop() + out.setdefault(root, []).append((pid, pgid)) + todo += by_parent.get(pid, []) + return out + + +def _pool_procs(pool, extra) -> list: + """The pool's worker Process objects, plus each extra's own process. + + Manager() runs in its own child process and inherits the same descriptors as the + workers, so leaving it behind defeats the point: os._exit skips its finalizer.""" + procs = list(getattr(pool, '_pool', []) or []) + for e in extra: + procs.append(getattr(e, '_process', e)) + return procs + + + + +def kill_worker_children(pool, *extra) -> int: + """SIGKILL what the pool's workers spawned; returns how many SURVIVED. + + For the TIMEOUT path only. On the normal path each worker has already run + kill_own_children() and retired (maxtasksperchild=1), so this walks fresh idle workers + and finds nothing -- measured: 4 tasks, zero overlap with the pool at sweep time. + + Call it BEFORE shutdown_pool(): terminate() reaps the (interruptible) worker and its + flasher is reparented to init, erasing the ppid link this matches on. Signalling the + worker's own group instead cannot work -- a forked pool worker inherits OUR group + (CPython 3.13.5 multiprocessing never setsid/setpgid) and run_cmd gives every flasher + a session of its own. + + TWO passes because our SIGKILL can fail an in-flight flash and the worker then retries + in a fresh session, which one /proc snapshot misses. `seen` stops a pid signalled in + pass 1 being confirmed twice. + """ + seen: set = set() + total = 0 + for i in range(2): + if i: + time.sleep(0.5) + procs = _pool_procs(pool, extra) + total += _kill_kids( + child_procs(getattr(p, 'pid', None) for p in procs if p is not None), seen) + return total + + +def kill_own_children() -> int: + """SIGKILL what THIS process spawned. Returns how many survived. + + For the worker to call before it returns. maxtasksperchild=1 retires it the moment the + task ends, reparenting its children to init, so main()'s sweep walks fresh idle workers + and finds nothing (measured over 4 tasks: zero overlap, sweep 0, 4 strays alive). + Inside the worker the ppid link is still live. + """ + return _kill_kids(child_procs([os.getpid()]), set()) + + +def _kill_kids(kids: dict, seen: set) -> int: + """SIGKILL every pid in a ppid-tree snapshot; return how many survived. + + Every pid here is a DESCENDANT of a process we own, so it is ours by construction -- no + argv identity check, because we never signal anything we did not discover through our + own ppid tree. + """ + try: + own = os.getpgid(0) + except OSError: + own = None # cannot tell our own group apart: never killpg, signal pids only + # One list: every pid here is a DESCENDANT of one of our own workers, so it is ours by + # construction -- no argv identity check needed, because we never signal anything we + # did not discover through our own ppid tree. + touched: list = [] + for children in kids.values(): + for cpid, cpgid in children: + if cpid in seen: + continue # a previous pass already signalled it + seen.add(cpid) + try: + if own is not None and cpgid != own: + # A run_cmd child: its own session, so one killpg also reaps what it + # spawned. Recorded because killpg cannot report a partial kill. + os.killpg(cpgid, signal.SIGKILL) + else: + # Shares our group (a plain subprocess.run), so killpg would take + # down the whole run -- it is signalled by pid in _kill_and_confirm. + pass + touched.append(cpid) + except PermissionError: + # NOT "already gone": the signal did not land, so this pid MUST still be + # confirmed, or the one case this handler exists for (an all-root session: + # the sudo wrapper died, its root members did not) is the one case that + # never reaches the report. + touched.append(cpid) + except ProcessLookupError: + pass # already gone + except OSError: + pass + # Both paths need confirming: a killpg'd flasher and a same-group mtype blocked on a + # wedged device are both in D state, and os.kill reported success on either. + denied = _kill_and_confirm(touched) + if denied: + _p(f'warning: could not kill {sorted(denied)}; they still hold whatever they ' + f'had open (probe, usbfs node) into the next job', flush=True) + # SURVIVORS, not the signalled-child count: the caller needs to know the rig is dirty + # for the next job, and a count of what we successfully signalled cannot tell it that. + # (They are different units anyway -- a killpg is counted once per child sharing the + # group -- so the old return was never comparable to anything.) + return len(denied) + + +def _kill_and_confirm(pids) -> list: + """SIGKILL every pid, then return those STILL alive after ONE grace window. + + SIGKILL is QUEUED, not delivered, for a task in uninterruptible sleep -- and testusb + waits in a plain wait_for_completion() with no timeout (v6.12.96 usbtest.c:1404; + usb_sg_wait, message.c:765), so that is the normal state of a healthy in-flight case + too. os.kill returning success proves nothing; only the recheck does. It is also + asynchronous, so probing immediately reports a process we just killed as a survivor + (measured: 11 of 20 plain `sleep`s with no grace). + + Signal all, then poll the set against ONE shared deadline: per-pid windows made this + scale with stray count, minutes on a convoy. A pid we cannot signal is reported, never + sudo-killed. + """ + pending = [] + for pid in pids: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + continue # already gone + except OSError: + pass # EPERM (root-owned): it stays, and the poll below reports it + pending.append(pid) + + deadline = time.monotonic() + CONFIRM_KILL_GRACE + while True: + alive = [] + for pid in pending: + try: + os.kill(pid, 0) + except ProcessLookupError: + continue # ESRCH: genuinely gone + except OSError: + pass # EPERM: it exists; the state check decides + try: + # a ZOMBIE answers kill(pid, 0) too: dead, merely unreaped. Not a survivor. + if _state(PROC / str(pid)) == 'Z': + continue + except (OSError, ValueError, IndexError): + continue # unreadable: assume gone rather than cry wolf + alive.append(pid) + pending = alive + if not pending or time.monotonic() >= deadline: + return pending # outlasted SIGKILL: D state, or not ours to kill + time.sleep(0.02) + + +def kill_pool_children(pool, *extra) -> int: + """SIGKILL the pool's worker processes themselves. Returns how many are STILL ALIVE + after the grace -- not how many were signalled. + + Survivors, not signals: the caller turns this number into "power-cycle the host", so + counting signals would send someone to a hypervisor over workers that all died. + + Call after a shutdown_pool() that returned False, and after kill_worker_children(). + A D-state worker ignores SIGKILL, but every other worker dies and drops the inherited + descriptors -- a survivor holds the runner's stdout pipe open and the runner waits for + EOF even after we exit, so the early exit would not free the job slot.""" + killed_procs: list = [] + for proc in _pool_procs(pool, extra): + try: + # Process.kill(), never a raw pid: multiprocessing's _send_signal re-checks + # `self.returncode is None` first, so once shutdown_pool's thread has reaped a + # worker this is a no-op instead of signalling a pid the OS may have recycled. + # os.pidfd_open(proc.pid) is worse: it skips that guard entirely. + if proc is None or not proc.is_alive(): + continue + proc.kill() + killed_procs.append(proc) + except (OSError, AttributeError, ValueError): + continue # already reaped, never started, or not a real process + # Re-check the Process objects, never the pids collected a moment ago: shutdown_pool's + # thread is STILL join()ing workers, so a pid killed here can be reaped and RECYCLED + # before _kill_and_confirm signals it -- and on EPERM that escalates to `sudo -n kill + # -9 `, killing an unrelated ROOT process as the last act before os._exit. + killed_pids = [] + for proc in killed_procs: + try: + if proc.is_alive() and proc.pid is not None: + killed_pids.append(proc.pid) + except (OSError, AttributeError, ValueError): + continue + # SIGKILL is asynchronous and a D-state task ignores it: only a confirmed survivor + # justifies the caller's power-cycle wording + return len(_kill_and_confirm(killed_pids)) if killed_pids else 0 + + +def write_timeout_report(report_dir: Path, boards, secs: int, md_name: str, + banner: str = '', prefix: str = '') -> None: + """Leave a report behind when the worker pool has to be abandoned. + + map_async is all-or-nothing, so a timeout loses every per-board result and the report + dir would stay empty with no reason for the failure. Any prior attempt's markdown is + kept below the banner.""" + # `prefix` carries the preflight rig-health verdict: the timeout aborts before + # accumulate_report, so without it the report loses the one line saying WHY the pool + # never finished. The '\n' stops Markdown lazy continuation pulling the banner into + # the blockquote. + try: + # Built INSIDE the try: a roster entry without a 'name' key raises KeyError while + # assembling the board list, and outside the try that escaped and stranded the + # runner -- which is exactly what the broad handler below exists to prevent. + head = (prefix + '\n' if prefix else '') + (banner or ( + f'**HIL run abandoned: worker pool timed out after {secs}s.**\n\n' + f'No per-board results could be collected for this attempt, so the ' + f'table below (if any) is from an earlier one. Boards dispatched:\n\n' + + '\n'.join(f'- {b.get("name", "?")}' for b in boards) + '\n')) + report_dir.mkdir(parents=True, exist_ok=True) + md_path = report_dir / md_name + # Its own handler so it cannot take the write down with it: a report torn by an + # attempt killed mid-write raises UnicodeDecodeError (a ValueError, and prior + # reports always contain status emoji), which under a shared try skipped the write + # entirely. Losing the old table is a nicety; losing the banner is the failure. + try: + prior = md_path.read_text(encoding='utf-8') if md_path.is_file() else '' + except (OSError, ValueError): + prior = '' + md_path.write_text(head + (f'\n{prior}' if prior else ''), encoding='utf-8') + except Exception as e: # noqa: BLE001 + # Deliberately broad: this is the first statement of the pool-abandon path, so ANY + # escape skips kill_pool_children and os._exit and strands the runner. + _p(f'warning: cannot write {md_name} to {report_dir}: {e}', flush=True) diff --git a/test/hil/helper/hil_lock.py b/test/hil/helper/hil_lock.py new file mode 100755 index 000000000..7757ef17d --- /dev/null +++ b/test/hil/helper/hil_lock.py @@ -0,0 +1,525 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Board locks + controller permits for the TinyUSB HIL rig. + +Board locks are kernel flocks in BOARD_LOCK_DIR arbitrating hardware access +between dev sessions and CI's hil_test.py (never stop the actions-runner). +Controller permits are in-process semaphores budgeting flashes and usbtest +batteries per host controller; they have no CLI meaning. The CLI below +(hold/release/status) manages board locks only. +""" +import argparse +import fcntl +import json +import os +import re +import select +import signal +import sys +import time + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # helper/ scripts import via the test/hil root +from helper import hil_util + +BOARD_LOCK_DIR = '/tmp/tinyusb-hil-locks' +CI_REASON = 'hil_test.py' # release-protected holder tag (release refuses to kill it) +PROTECTED_REASONS = {CI_REASON, 'pool_check'} # cmd_release refuses to SIGTERM these holders +PROFILE = os.environ.get('HIL_PROFILE') == '1' + + +def lock_path(board: str) -> str: + return os.path.join(BOARD_LOCK_DIR, f'{board}.lock') + + +def flock_nb(board: str): + """Open-or-create the lock file WITHOUT truncating (a losing racer must not + wipe the winner's record) and take LOCK_EX|LOCK_NB. Returns the open handle; + raises OSError when the flock is held elsewhere (handle already closed).""" + fd = os.open(lock_path(board), os.O_RDWR | os.O_CREAT, 0o666) + fh = os.fdopen(fd, 'r+') + try: + fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + fh.close() + raise + return fh + + +def write_record(fh, reason: str) -> bool: + """Holder record; the flock itself is already held. Returns False on a write failure: + acquire_board_lock stays best-effort (the flock is the authority), but cmd_hold aborts + -- a hold whose record is missing is invisible to status/release.""" + try: + fh.truncate(0) + fh.seek(0) + json.dump({'pid': os.getpid(), 'reason': reason, + 'since': time.strftime('%Y-%m-%dT%H:%M:%S%z')}, fh) + fh.flush() + return True + except OSError: + return False + + +def clear_record(fh) -> None: + """Clear our record before dropping the flock so records stay truthful.""" + try: + fh.truncate(0) + except OSError: + pass + + +def read_record(board: str): + try: + with open(lock_path(board)) as f: + return json.load(f) + except (OSError, ValueError): + return None + + +# --- per-board dev-session locks ------------------------------------------ +def acquire_board_lock(board_name, reason=CI_REASON): + """Take this board's flock for the duration of its flash+test. + Returns an open file handle (keep it referenced; closing releases it), + or None when HIL_NO_BOARD_LOCK=1 or the lock dir is unusable (fail-open: + locking must never break a test run by itself). + Raises RuntimeError only when another session holds the board.""" + import fcntl + if os.environ.get('HIL_NO_BOARD_LOCK') == '1': + return None # user-authorized bypass — see hil skill + try: + os.makedirs(BOARD_LOCK_DIR, exist_ok=True) + fd = os.open(os.path.join(BOARD_LOCK_DIR, f'{board_name}.lock'), + os.O_RDWR | os.O_CREAT, 0o666) + fh = os.fdopen(fd, 'r+') + except OSError as e: + # odd lock dir (perms, path collision): proceed unlocked, but say so — + # a silent fail-open is indistinguishable from the intentional bypass + print(f'warning: board lock unavailable for {board_name} ({e}); proceeding unlocked', + flush=True) + return None + try: + fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + try: + info = fh.read(500).strip() + except (OSError, UnicodeDecodeError): + info = '' + fh.close() + raise RuntimeError(f'board locked: {info or "unknown holder"}') + # announce ourselves so the other side's conflict message is truthful; + # best-effort — the flock itself is already held + try: + fh.truncate(0) + fh.seek(0) + json.dump({'pid': os.getpid(), 'reason': reason, + 'since': time.strftime('%Y-%m-%dT%H:%M:%S%z')}, fh) + fh.flush() + except OSError: + pass + return fh + + +# Per-host-controller concurrency (see controller_of/controller_slot below): a usbtest +# battery saturates its DUT's host controller, so batteries and flashes are budgeted per +# controller. The 4/2 defaults trade ~3.5 min on the usbtest leg for bandwidth margin on +# the shared leaf-hub uplinks, where battery case failures were observed from 12/8 +# (profiled 2026-07-13/14: 22.2/14.3/12.5/10.8 min at usbtest width 1/2/3/4, plateau +# after). Raise per run via HIL_FLASH_PARALLEL/HIL_USBTEST_PARALLEL. +# - uPD720201 cards need firmware >= 2.0.2.6 (RAM-uploaded, reloads every power cycle): +# the ROM firmware dies under battery + re-enumeration churn. +# - a marginal DUT port bouncing during concurrent batteries can kill a uPD720201 ("xHCI +# host not responding to stop endpoint command"): fix the port/cable or pull the board +# -- lowering the widths does not fix a bad port (2026-07-16, every death). +FLASH_PARALLEL = hil_util.pos_int_env('HIL_FLASH_PARALLEL', 4) +USBTEST_PARALLEL = hil_util.pos_int_env('HIL_USBTEST_PARALLEL', 2) +CONTROLLER_SLOTS = 12 # lock slots; controllers are assigned to slots on first sight +# Bound on ONE permit wait. Generous: a real queue behind a slow board is normal, +# and this only has to beat the pool guard so a leaked permit cannot consume it. +PERMIT_TIMEOUT = hil_util.pos_int_env('HIL_PERMIT_TIMEOUT', 900) +# CONTROLLER_SLOTS + 1 entries each, built by make_permit_sems: UNKNOWN_SLOT indexes the +# extra one. Sized to CONTROLLER_SLOTS instead, the first unresolved board IndexErrors +# inside a pool worker -- which now surfaces through drain_pool as a worker-raise (the +# finished boards survive), but still loses this board and aborts the run. +usbtest_sems = None # per-slot usbtest-battery permits +flash_sems = None # per-slot flash permits +controller_map = None # shared dict: 'pci:' -> slot, 'uid:' -> pci addr cache +controller_meta = None # guards slot assignment in controller_map +controller_hints = {} # static uid -> pci from the last run's cache (read-only per worker) + + +log = print # hil_test.init_worker points this at log_line via init_scheduling + + +def init_scheduling(b_sems, f_sems, cmap, cmeta, hints, log_fn=None): + """Install per-worker scheduling state (called from hil_test.init_worker).""" + global usbtest_sems, flash_sems, controller_map, controller_meta, controller_hints, log + usbtest_sems, flash_sems = b_sems, f_sems + controller_map, controller_meta, controller_hints = cmap, cmeta, hints + if log_fn is not None: + log = log_fn + + +# ------------------------------------------------------------- +# Per-controller scheduling +# ------------------------------------------------------------- +def controller_of(uid: str): + """Resolve a DUT uid to its root host controller's PCI address, or None when it cannot + be resolved — the device is not enumerated (e.g. parked in board_test firmware with USB + off), or sysfs would not answer. Successful resolutions are cached — cabling does not + change mid-run. Dual-port parts (e.g. CH32V307 usbhs/usbfs variants) share one uid and + one cache entry: budgeting is only exact when both ports sit on the same controller + (true on this rig).""" + if controller_map is None: + return None + cached = controller_map.get(f'uid:{uid}') + if cached: + return cached + # vid='cafe' first: the target is always a TinyUSB DUT, and the VID is a lock-free + # descriptor field. Without it this read every probe's and hub's `serial` -- the + # attribute served under device_lock -- so a HEALTHY peer mid-usbtest would strand a + # reader here and spend one of this worker's four blindness credits. + devs, _ = hil_util.usb_scan(vid='cafe', serial=uid) + for dev in devs: + busnum = hil_util.read_sysfs(os.path.join(dev['dir'], 'busnum')) + if busnum is None or busnum is hil_util.SYSFS_UNKNOWN: + continue + try: + root = os.path.realpath(f'/sys/bus/usb/devices/usb{int(busnum)}') + except ValueError: + continue + m = re.findall(r'[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]', root) + if m: + controller_map[f'uid:{uid}'] = m[-1] + return m[-1] + return None + + +def controller_slot(pci: str) -> int: + """Map a controller PCI address to a lock slot (assigned on first sight).""" + key = f'pci:{pci}' + with controller_meta: + slot = controller_map.get(key) + if slot is None: + slot = controller_map.get('nslots', 0) + if slot >= CONTROLLER_SLOTS: + slot = 0 # more controllers than slots: overflow shares slot 0 (safe, over-serialized) + else: + controller_map['nslots'] = slot + 1 + controller_map[key] = slot + return slot + + +# Unresolved boards budget in a slot of their OWN, one past the real ones, and that slot +# holds exactly ONE permit whatever the per-controller width is. Neither neighbour works: +# a permit on every slot (the old fail-closed rule) serialized the whole fleet the moment +# a worker went blind, while a full private budget let unknown boards run a second +# controller's worth of batteries on top of the resolved ones -- doubling the load on +# whichever physical controller they actually sit on, which is the saturation the +# uPD720201 deaths above are attributed to. Width 1 caps the over-subscription at +1. +UNKNOWN_SLOT = CONTROLLER_SLOTS + + +def make_permit_sems(semaphore, width: int) -> list: + """One semaphore per controller slot at `width`, plus the unknown bucket at 1.""" + return [semaphore(width) for _ in range(CONTROLLER_SLOTS)] + [semaphore(1)] + + +class controller_permit: + """Context manager: one permit from `sems` on the board's controller slot. An + unresolved controller budgets in UNKNOWN_SLOT, which admits one at a time: unresolved + boards serialize against each other, never against the whole rig, and never add a + second full budget to a controller. `warn_unknown` logs that fallback (used by + usbtest, where the device is expected to be enumerated by the caller).""" + def __init__(self, sems, uid: str, warn_unknown: bool = False): + self.sems = sems + self.slots = None + self.uid = uid + # what __enter__ actually ACQUIRED. Not the same as self.slots: a bounded acquire + # that times out is skipped on purpose, and releasing it anyway would add a permit + # that was never taken -- multiprocessing semaphores are unbounded, so the width + # grows for the rest of the run, on the controller throttle that exists to keep + # concurrent batteries from killing the uPD720201 xHCI. + self.taken: list = [] + if sems is None: + return + # Hint FIRST for flash budgeting: a mis-budgeted flash is harmless, and the board + # is usually parked in board_test with USB off at this point, so controller_of + # cannot resolve it anyway -- it just walks the whole bus to say so, once per + # flash permit (~14 examples x ~21 boards a leg), each walk spawning a bounded + # reader per device. usbtest still resolves for real (warn_unknown), and by then + # the DUT is enumerated, so that walk succeeds and caches. + pci = None if warn_unknown else controller_hints.get(uid) + if pci is None: + pci = controller_of(uid) + if pci is None and warn_unknown: + log(f'warning: cannot resolve {uid} to a host controller' + f'{hil_util.sysfs_blind_note()}; budgeting it in the unknown bucket') + self.slots = [controller_slot(pci) if pci else UNKNOWN_SLOT] + + def __enter__(self): + if self.slots: + t0 = time.monotonic() + taken = self.taken = [] + try: + for s in self.slots: + # BOUNDED. multiprocessing semaphores are NOT released when a holder + # dies, and the pool sweep SIGKILLs workers -- so a permit lost that + # way would block every later worker on this controller forever, and + # boards unrelated to the wedge would burn the whole pool guard. On + # expiry proceed over-subscribed and say so: a slower controller is a + # far better failure than a hung run. + if not self.sems[s].acquire(timeout=PERMIT_TIMEOUT): + log(f'warning: waited {PERMIT_TIMEOUT}s for a permit on slot {s} ' + f'(uid {self.uid}); a holder probably died without releasing ' + f'it -- proceeding over-subscribed') + continue + taken.append(s) + # inside the try: a failed __enter__ never gets its __exit__, so a raise + # here (e.g. broken stdout) must still release the permits + if PROFILE and time.monotonic() - t0 > 1.0: + log(f'[prof] permit wait {time.monotonic() - t0:.1f}s ' + f'(uid {self.uid}, slots {self.slots})') + except BaseException: + for s in reversed(taken): + self.sems[s].release() + raise + return self + + def __exit__(self, *exc): + if self.slots: + for s in reversed(self.taken): + self.sems[s].release() + self.taken = [] + return False + + +def flash_permit(uid: str) -> controller_permit: + return controller_permit(flash_sems, uid) + + +def usbtest_permit(uid: str) -> controller_permit: + return controller_permit(usbtest_sems, uid, warn_unknown=True) + + +# --- operator CLI (hold/release/status) ------------------------------------ +def boards_from_config(config: str) -> list: + """All board names, INCLUDING boards-skip: `hold --all` guards rig-wide + operations, and parked boards can still be touched (pool_check -b names them + explicitly), so a rig-wide hold that skipped them would leave a gap.""" + try: + with open(config) as f: + cfg = json.load(f) + return [b['name'] for b in cfg['boards'] + cfg.get('boards-skip', [])] + except (OSError, ValueError, KeyError) as e: + print(f'ERROR: cannot read board roster {config}: {e}', file=sys.stderr) + sys.exit(1) + + +def is_locked(board: str) -> bool: + """True if the recorded holder process is still alive. + + Deliberately never touches the flock: even a momentary probe lock would + make a concurrent acquirer's LOCK_NB attempt fail spuriously. The flock + taken by acquirers themselves stays the only authority.""" + info = read_record(board) + pid = info.get('pid') if isinstance(info, dict) else None + if not isinstance(pid, int) or pid <= 0: + return False + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True # alive but owned by another user (e.g. the CI runner) + return True + + +def cmd_hold(boards, reason): + os.makedirs(BOARD_LOCK_DIR, exist_ok=True) + # No pre-check: the holder's own LOCK_NB flock is the only authority, since a recorded + # pid may be stale or recycled. The holder signals success through this pipe because a + # generic is_locked() poll would be fooled by a RIVAL invocation's flock — only the + # holder knows whether it won every board. + r_fd, w_fd = os.pipe() + pid = os.fork() + if pid > 0: + os.close(w_fd) + os.waitpid(pid, 0) # reap intermediate child + ready, _, _ = select.select([r_fd], [], [], 10) + ok = bool(ready) and os.read(r_fd, 1) == b'1' + os.close(r_fd) + if ok: + print(f'held: {", ".join(boards)}') + return 0 + for b in boards: + info = read_record(b) + if info: + print(f'ERROR: {b} locked: {info}', file=sys.stderr) + print('ERROR: holder failed to acquire locks', file=sys.stderr) + return 1 + # intermediate child: detach, then spawn the actual holder + os.setsid() + if os.fork() > 0: + os._exit(0) + # holder (grandchild): acquire all flocks, signal the parent, sleep until killed + os.close(r_fd) + # Keep the success pipe clear of fds 0-2: invoked with stdio closed, os.pipe() can + # land there and the dup2 loop below would clobber it. + if w_fd <= 2: + w_fd = fcntl.fcntl(w_fd, fcntl.F_DUPFD, 3) + # Detach stdio: a `hold` whose output is captured must see EOF when the front-end + # exits — the immortal holder must not keep that pipe open. + devnull = os.open(os.devnull, os.O_RDWR) + for std_fd in (0, 1, 2): + os.dup2(devnull, std_fd) + if devnull > 2: + os.close(devnull) + try: + handles = [] + for b in boards: + fh = flock_nb(b) + if not write_record(fh, reason): + raise OSError(f'cannot write holder record for {b}') + handles.append(fh) + except OSError: + try: + os.write(w_fd, b'0') + except OSError: + pass + os._exit(1) # lost a race; parent reports the failure + os.write(w_fd, b'1') + os.close(w_fd) + + def _bow_out(*_): + # clear the records before dying so read_record/status stay truthful (the kernel + # drops the flocks themselves on exit either way) + for h in handles: + clear_record(h) + os._exit(0) + + signal.signal(signal.SIGTERM, _bow_out) + while True: + signal.pause() + + +def cmd_release(boards): + rc = 0 + victims = set() + for b in boards: + try: + fd = os.open(lock_path(b), os.O_RDWR) + except OSError: + continue # no lock file (or another user's): nothing we can release + fh = os.fdopen(fd, 'r+') + try: + fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + # flock genuinely held — never SIGTERM on a mere pid record: the pid may be + # recycled, or a live worker that already moved on. + fh.close() + info = read_record(b) or {} + pid = info.get('pid') + reason = info.get('reason') + if reason in PROTECTED_REASONS: + print(f'ERROR: {b} is mid-test by {reason} (pid {pid}) — not killing it; ' + 'wait for it to finish', file=sys.stderr) + rc = 1 + elif isinstance(pid, int) and pid > 0: + victims.add(pid) + else: + print(f'ERROR: {b} is held but its record is unreadable', file=sys.stderr) + rc = 1 + continue + # flock was free: only a stale record remained — clear it + clear_record(fh) + fh.close() + for holder in sorted(victims): + try: + os.kill(holder, signal.SIGTERM) + print(f'released holder pid {holder}') + except ProcessLookupError: + pass + except PermissionError: + print(f'ERROR: holder pid {holder} belongs to another user — cannot signal it', + file=sys.stderr) + rc = 1 + time.sleep(0.3) + still = [b for b in boards if is_locked(b)] + if still: + print(f'ERROR: still locked: {", ".join(still)}', file=sys.stderr) + return 1 + return rc + + +def cmd_status(): + if not os.path.isdir(BOARD_LOCK_DIR): + print('no locks') + return 0 + any_locked = False + for fn in sorted(os.listdir(BOARD_LOCK_DIR)): + if not fn.endswith('.lock'): + continue + b = fn[:-5] + if is_locked(b): + any_locked = True + print(f'{b}: {read_record(b)}') + if not any_locked: + print('no locks') + return 0 + + +_CLI_USAGE = """Per-board advisory locks for the HIL rig. + +Arbitrates board access between dev sessions and CI's hil_test.py without +stopping the actions-runner. Locks are kernel flocks: the kernel releases +them automatically when the holder process dies, and holders clear their +lock-file record on release so records stay truthful (/tmp also clears on +reboot). + +Usage: + hil_lock.py hold BOARD [BOARD...] --reason TEXT + hil_lock.py hold --all [--config CONFIG.json] --reason TEXT + hil_lock.py release BOARD [BOARD...] | release --all + hil_lock.py status + +A holder process holds ALL boards given in one `hold` call; releasing any of +them kills that holder and releases all of its boards. +""" + + +def main(): + ap = argparse.ArgumentParser(description=_CLI_USAGE, + formatter_class=argparse.RawDescriptionHelpFormatter) + sub = ap.add_subparsers(dest='cmd', required=True) + p_hold = sub.add_parser('hold') + p_hold.add_argument('boards', nargs='*') + p_hold.add_argument('--all', action='store_true') + p_hold.add_argument('--config', + default=os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + 'tinyusb.json'), + help='board roster JSON (default: tinyusb.json in test/hil, one level above this script)') + p_hold.add_argument('--reason', required=True) + p_rel = sub.add_parser('release') + p_rel.add_argument('boards', nargs='*') + p_rel.add_argument('--all', action='store_true') + sub.add_parser('status') + a = ap.parse_args() + if a.cmd == 'hold': + boards = boards_from_config(a.config) if a.all else a.boards + if not boards: + ap.error('no boards given (name boards or use --all)') + sys.exit(cmd_hold(boards, a.reason)) + if a.cmd == 'release': + if a.all: + boards = ([fn[:-5] for fn in os.listdir(BOARD_LOCK_DIR) if fn.endswith('.lock')] + if os.path.isdir(BOARD_LOCK_DIR) else []) + else: + boards = a.boards + if not boards: + ap.error('no boards given (name boards or use --all)') + sys.exit(cmd_release(boards)) + sys.exit(cmd_status()) + + +if __name__ == '__main__': + main() diff --git a/test/hil/helper/hil_pool_check.py b/test/hil/helper/hil_pool_check.py new file mode 100644 index 000000000..371aff1e1 --- /dev/null +++ b/test/hil/helper/hil_pool_check.py @@ -0,0 +1,1019 @@ +#!/usr/bin/env python3 +"""Quick HIL pool health check. + +For every board in the rig's HIL config: is the flash probe on the USB bus, does a +light example flash, and does the board's USB device (uid) come back up? Missing +firmware is BUILT on the spot (tools/build.py, idf.py for espressif; one get_deps +retry) — never skipped; --no-build opts out. Applies only per-device-safe recovery +(probe authorized-toggle, board reset/re-flash) and prints a markdown summary +table. Row statuses: ok (flashed and verified; under --scan-only: probe present — +the scan checks presence only), flash-failed (firmware delivery failed: probe +missing, build failed, flasher error, silent flash no-op, park not verified), +failed (the check ran but did not verify: flashed with no enumeration/serial, or +the check itself errored), locked (board flock held by another process — +reported, never waited on or bypassed). + +Config is picked by hostname unless given: ci -> tinyusb.json, tusb (hifiphile +rig) -> hfp.json, anything else is a dev PC -> local.json. + +Lives in test/hil/helper/ beside hil_lock.py; imports it and hil_flash; board +recovery uses the repo's .claude/skills/usb-kernel-recover/scripts/usb_recover.sh. +""" + +import argparse +import io +import json +import glob +import os +import re +import shlex +import shutil +import socket +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # hil_flash + the helper package +import hil_flash +from helper import hil_lock, hil_util + +REPO_ROOT = hil_util.TINYUSB_ROOT +USB_RECOVER = REPO_ROOT / '.claude' / 'skills' / 'usb-kernel-recover' / 'scripts' / 'usb_recover.sh' +SEEN_CACHE = Path.home() / '.cache' / 'tinyusb-hil' / 'pool_seen.json' +CONFIG_BY_HOST = {'ci': 'tinyusb.json', 'tusb': 'hfp.json'} # anything else: dev PC -> local.json + +# light-example preference; first built wins +DEVICE_CANDIDATES = ['device/dfu_runtime', 'device/cdc_msc', 'device/cdc_msc_freertos', + 'device/hid_composite_freertos', 'device/cdc_dual_ports'] +HOST_CANDIDATES = ['host/device_info', 'host/cdc_msc_hid', 'host/msc_file_explorer_freertos'] + +ENUM_WAIT = 12 # s, uid wait after flash +ENUM_WAIT_RETRY = 8 # s, uid wait after a recovery reset/re-flash +SERIAL_WAIT = 6 # s, host-board serial-output wait + +print_mutex = threading.Lock() +_UNKNOWN_WARNED = False # scan_usb's caveat: once per process, not once per poll +t0 = time.monotonic() + + +def say(msg: str) -> None: + with print_mutex: + print(f'[{time.monotonic() - t0:6.1f}s] {msg}', file=sys.__stdout__, flush=True) + + +def scan_usb() -> dict: + """busport -> {'serial', 'vidpid', 'ino'} for every enumerated USB device. Only + -[....] dirs match; root hubs ('usbN', no dash) are excluded because + their 'serial' is a fabricated PCI address, and including them measured 6-7s/scan slower + (an observation; NOT an autosuspend wake -- that read is cached and does no I/O). + Keyed by busport, not serial: two devices can share a serial (an Espressif + USB-Serial-JTAG bridge and the cafe device it flashes both derive it from the same + MAC), and one dict slot would silently drop whichever lost the race.""" + found = {} + # `unknown` matters BEFORE the blindness latch trips: one wedged device is the normal + # reason this tool is run, and its serial read stranding makes it absent from `devs`. + # Reported as fact, that is "probe MISSING" for hardware that is physically present. + devs, unknown = hil_util.usb_scan() + # ONCE per process: this is called from 0.5s poll loops across 4 worker threads and + # ~26 boards, so warning per call buried the table it exists to qualify under 600+ + # identical lines. The memo in read_sysfs makes the condition sticky, so one line is + # as true as six hundred. + global _UNKNOWN_WARNED + if unknown and not _UNKNOWN_WARNED: + _UNKNOWN_WARNED = True + say('WARNING: at least one device did not answer a bounded read; rows below that ' + 'say a probe or board is missing may be this scan losing sight of healthy ' + 'hardware. Find the wedged device (usb-kernel-recover) and re-run.') + for dev in devs: + try: + found[dev['busport']] = { + 'serial': dev['serial'].lower(), + 'vidpid': f"{dev['vid']}:{dev['pid']}", + 'ino': os.stat(dev['dir'] + '/').st_ino} + except OSError: + continue + return found + + +def find_usb(uid: str, devs: dict | None = None): + """Locate a flasher probe by uid, excluding VID cafe (TinyUSB DUT firmware): a + probe's uid can coincidentally equal its DUT's (Espressif USB-Serial-JTAG + bridges derive both from the same MAC), and the DUT is never the probe. + + J-Link zero-pads numeric serials (681295394 -> 000681295394): an all-digit uid + matches an all-digit serial only when that serial equals the uid zero-padded to + the serial's own length (leading zeros only) — never when the zero-stripped uid + is empty, so a placeholder serial (metro_m4_express's probe legitimately reports + '123456') can't be mistaken for an unrelated device.""" + devs = devs if devs is not None else scan_usb() + u = uid.lower() + candidates = [(bp, dev) for bp, dev in devs.items() if not dev['vidpid'].startswith('cafe:')] + for bp, dev in candidates: + if dev['serial'] == u: + return bp, dev['vidpid'], dev['ino'] + stripped = u.lstrip('0') + if u.isdigit() and stripped: + for bp, dev in candidates: + s = dev['serial'] + if s.isdigit() and s == stripped.zfill(len(s)): + return bp, dev['vidpid'], dev['ino'] + return None + + +def find_device(uid: str, pid: str | None): + """Board-online check: TinyUSB device (idVendor cafe) with this uid, optionally + PID-pinned. VID cafe keeps an Espressif USB-Serial-JTAG (303a) that shares the MAC + serial from false-passing.""" + for busport, dev in scan_usb().items(): + if (dev['serial'] == uid.lower() and dev['vidpid'].startswith('cafe:') + and (pid is None or dev['vidpid'].endswith(pid))): + return busport, dev['vidpid'], dev['ino'] + return None + + +def wait_device(uid: str, pid: str | None, old_ino, budget: float): + """Wait for the board's device with a NEW sysfs inode (flash resets the MCU, so a + genuine flash must re-enumerate; the inode is the re-enumeration marker).""" + deadline = time.monotonic() + budget + while time.monotonic() < deadline: + hit = find_device(uid, pid) + if hit and hit[2] != old_ino: + return hit + time.sleep(0.5) + return None + + +def lock_board(name: str): + """Nonblocking flock per hil_lock.py protocol. Returns the handle, or a str with the + holder's info when the board is locked elsewhere. Board locks are ALWAYS respected: a + held board is reported and skipped, never waited on, and there is no bypass here.""" + os.makedirs(hil_lock.BOARD_LOCK_DIR, exist_ok=True) + try: + fh = hil_lock.flock_nb(name) + except OSError: + # NB: conflates a held flock with open() failures (EACCES/EROFS/ENOSPC) — benign + # while everything on the rig runs as one uid + info = hil_lock.read_record(name) + return json.dumps(info) if info else 'unknown holder' + if not hil_lock.write_record(fh, 'pool_check'): + # an invisible lock (flock held, no record) is worse than no lock: status cannot + # show us and release cannot recognize the protected holder + hil_lock.clear_record(fh) + fh.close() + return 'ERROR: holder record write failed (lock dir unwritable?)' + return fh + + +def unlock_board(fh) -> None: + hil_lock.clear_record(fh) + fh.close() + + +def can_recover() -> bool: + if not USB_RECOVER.is_file(): + return False + try: + # run_cmd, not subprocess.run: run's post-timeout reap is an UNBOUNDED wait(), and + # our kill bounces off a setuid-root sudo with EPERM, leaving communicate() on a + # pipe that never closes. run_cmd killpgs, escalates through sudo, reaps bounded. + r = hil_util.run_cmd('sudo -n true', timeout=10, quiet=True) + except OSError: # sudo not installed + return False + return r.returncode == 0 + + +def recover_probe(uid: str, busport: str) -> bool: + """Soft-replug an enumerated-but-wedged probe: deauthorize+reauthorize (no VBUS cut, + touches only this device). Success = the probe re-enumerated (new sysfs inode), not the + helper's exit code, which flakes while the toggle works. J-Links respond with a full + disconnect and can stay off the bus for >8 s.""" + pre = find_usb(uid) + # Bounded through run_cmd (same reason as can_recover): the sysfs authorized store can + # block in D state on a wedged device, and this runs while the board's release- + # PROTECTED flock is held -- a hang here would lock the board until the host reboots. + cmd = ' '.join(shlex.quote(a) for a in + ['sudo', '-n', str(USB_RECOVER), 'authorized', busport]) + if hil_util.run_cmd(cmd, timeout=30, quiet=True).returncode == 124: + return False + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + post = find_usb(uid) + if post and (pre is None or post[2] != pre[2]): + return True + time.sleep(0.5) + return False + + +def resolve_variant(board: dict, example: str, note: list | None = None) -> str: + """Build-dir variant name for `example`: the first of the board's variants with + already-built firmware, falling back to the board name. Notes the pick when it + differs from the board name (e.g. nanoch32v203's build dir is variant + 'nanoch32v203-fsdev', not the board name).""" + name = board['name'] + for v in board.get('variant') or [{'name': name}]: + vn = v['name'] + if hil_flash.find_firmware(vn, example, flasher=board['flasher']['name']): + if vn != name and note is not None and f'variant: {vn}' not in note: + note.append(f'variant: {vn}') + return vn + return name + + +def pick_example(board: dict, note: list, build_missing: bool = True): + """(example, kind, variant, fw) with built firmware for this board; kind is + 'device' (uid check) or 'host' (serial-output check); variant is the resolved + build-dir variant that has it (see resolve_variant); fw is the firmware path to + flash, extension included. When nothing is built and build_missing is set (the default — + never skip a board for lack of a build), the preferred candidate is built on + the spot via ensure_fw.""" + tests = board.get('tests', {}) + only = tests.get('only', []) + skip = set(tests.get('skip', [])) # config's known-broken examples: never pick one + is_device = tests.get('device') or any(t.startswith('device/') for t in only) + if is_device: + cand = DEVICE_CANDIDATES + [t for t in only if t.startswith('device/') and t != 'device/usbtest'] + kind = 'device' + else: + cand = HOST_CANDIDATES + [t for t in only if t.startswith('host/')] + kind = 'host' + for ex in dict.fromkeys(cand): + if ex in skip: + continue + variant = resolve_variant(board, ex, note) + fw = hil_flash.find_firmware(variant, ex, flasher=board['flasher']['name']) + if fw: + return ex, kind, variant, fw + if not build_missing: + return None, kind, None, None + # nothing built anywhere: build the preferred candidate (an only-list board + # must get one of its own examples — dfu_runtime etc. may not even configure) + pref = [c for c in dict.fromkeys(cand) if c not in skip and (not only or c in only)] + if not pref: + return None, kind, None, None + variant = (board.get('variant') or [{'name': board['name']}])[0]['name'] + for ex in pref[:2]: # the second candidate covers a preferred example that fails to build + fw = ensure_fw(board, variant, ex, note) + if fw: + return ex, kind, variant, fw + return None, kind, None, None + + +_pid_cache: dict[str, str | None] = {} + + +def get_expected_pid(example: str) -> str | None: + """USB_PID for `example`'s device descriptor (examples//src/ + usb_descriptors.c, '#define USB_PID 0x....'), lowercased and without the 0x + prefix to match sysfs idProduct. Cached per example; None (also cached) when + the file or define isn't there — host examples have no usb_descriptors.c, and + the caller must stay quiet rather than false-warn.""" + if example not in _pid_cache: + pid = None + try: + text = (REPO_ROOT / 'examples' / example / 'src' / 'usb_descriptors.c').read_text() + # optional parens as in tools/check_example_pids.py's parser + m = re.search(r'#define\s+USB_PID\s+\(?\s*(0x[0-9a-fA-F]+)', text) + if m: + pid = m.group(1)[2:].lower() + except OSError: + pass + _pid_cache[example] = pid + return _pid_cache[example] + + +def call_flasher(fn, *fn_args) -> tuple[int, str]: + """Run a hil_flash flash_*/reset_* backend, normalizing raises to a failure: several + backends raise instead of returning nonzero (get_serial_dev when a bridge's + /dev/serial/by-id node vanishes, a missing config.env, a .jlink script OSError), and an + exception must not skip the caller's retry/recovery ladder. Returns (rc, error line).""" + try: + ret = fn(*fn_args) + if ret.returncode == 0: + return 0, '' + err = flash_error_line(hil_util.cmd_stdout_text(ret.stdout)) + return ret.returncode, err or f'rc={ret.returncode}' + except Exception as e: + return -1, repr(e)[:90] + + +def flash(board: dict, fw, allow_recovery: bool, probe_port: str, note: list) -> bool: + """Flash the resolved firmware with one retry; on repeated failure soft-replug the + probe and always make one final attempt afterward, confirmed replug or not — some + probes (WCH-Link, ST-Link, CP210x, picoprobe) keep their sysfs kobject across an + authorized toggle instead of dropping off the bus. Returns True on success. + + `fw` comes from pick_example: a re-resolve here would use the global search policy and + miss a firmware ensure_fw just built into cmake-build/ under an exclusive -B.""" + fn = getattr(hil_flash, f'flash_{board["flasher"]["name"].lower()}') + for attempt in range(3): + if attempt == 2: + if not (allow_recovery and probe_port): + return False + cur = find_usb(board['flasher']['uid']) + if cur is None: + # probe gone from the bus: its old busport may now hold an UNRELATED device + # (bus renumbering) and the helper only checks occupancy, so toggling would + # deauthorize an innocent fixture + note.append('probe vanished before toggle') + else: + say(f'{board["name"]:26} recovery: replugging probe {cur[0]} (authorized toggle)') + if recover_probe(board['flasher']['uid'], cur[0]): + note.append('probe replugged') + time.sleep(2) # udev recreates /dev/serial/by-id symlinks after re-enumeration + else: + note.append('probe toggle unconfirmed') + rc, err = call_flasher(fn, board, str(fw)) + if rc == 0: + return True + if rc == 127: # flasher binary missing: retries/probe recovery can't fix env + note.append(f'flasher tool missing ({err}) — esptool needs the ESP-IDF env (get-idf)' + if board['flasher']['name'].lower() == 'esptool' else + f'flasher tool missing: {err}') + return False + if attempt == 0: + say(f'{board["name"]:26} flash retry: {err}') + else: + note.append(f'flash: {err}') + return False + + +def flash_error_line(out: str) -> str: + """Most informative line of a failed flash's output: last error-looking line, + else the last non-empty one.""" + lines = [l.strip() for l in out.splitlines() if l.strip()] + for l in reversed(lines): + if any(k in l.lower() for k in ('error', 'fail', 'unknown', 'cannot', 'timeout', + 'no valid', 'not found', 'unable')): + return l[:90] + return lines[-1][:90] if lines else '' + + +def check_host_serial(board: dict, do_reset: bool = True, want_hello: bool = False) -> bytes | None: + """Host-only boards never enumerate their uid (their USB port is the host side); + aliveness = output on the flasher's UART bridge after a reset. A probe byte is + written each poll so an echo-only firmware (board_test) also answers. Returns + the first output chunk (b'' when silent, None when the port is absent/drops) so + the caller can also judge WHAT answered — see boardtest_output(). + + do_reset=False listens to the firmware as-is: used right after a flash whose + own reset already started it — a second openocd/JLink session back-to-back on + the same probe can fail transiently and leave the target halted.""" + import serial + try: + port = hil_util.get_serial_dev(board['flasher']['uid'], None, None, 0) + ser = serial.Serial(port, baudrate=115200, timeout=0.3, write_timeout=1) + except Exception as e: + say(f'{board["name"]:26} no flasher serial port: {e}') + return None + try: + # flush BEFORE the reset: this drops the pre-reset CDC backlog (which must not + # count as life) while keeping the post-reset boot banner, which prints while the + # reset tool is still tearing down and a post-reset flush would eat + ser.reset_input_buffer() + if do_reset: + getattr(hil_flash, f'reset_{board["flasher"]["name"].lower()}')(board) + # judge the WHOLE window, not the first chunk: the probe's CDC bridge has its own + # FIFO, so stale pre-flash output (e.g. board_test hellos) can arrive after our + # host-side flush and must not decide the verdict alone. + data = b'' + deadline = time.monotonic() + SERIAL_WAIT + while time.monotonic() < deadline: + try: + ser.write(b'U') + data += ser.read(256) + except serial.SerialTimeoutException: + pass + except serial.SerialException: + return None # port dropped mid-poll (bridge re-enumerating) + # early-exit on the caller's positive signal (board_test hello for park + # verification, any non-board_test output for example liveness): stale + # bridge-FIFO backlog of the OTHER kind must not end the window + if want_hello: + if b'Hello from TinyUSB' in data: + return data + elif data and not boardtest_output(data): + return data + return data + finally: + ser.close() + + +def boardtest_output(data: bytes) -> bool: + """True when (non-empty) serial output is recognizably ONLY board_test's: its + periodic HELLO_STR and echoes of our b'U' pokes, nothing else. Any residue + beyond that (an example banner, log lines) proves other firmware is talking, + however much stale board_test backlog surrounds it. Used as a negative + identity marker — after flashing a host example, board_test-only chatter + means the flash silently didn't take (the host analog of the PID check).""" + residue = data.replace(b'Hello from TinyUSB', b'') + for junk in (b'U', b'\r', b'\n'): + residue = residue.replace(junk, b'') + return len(residue) == 0 + + +def build_example(board: dict, variant: str, example: str) -> int: + """Build one example for this board: tools/build.py (same invocation shape as + hil_test.build_board), or idf.py directly for espressif (tools/build.py's esp branch + ignores -T and builds everything; variant flags travel as -DCFLAGS_CLI, the channel + tools/build.py uses). Bounded and process-group-killed via run_cmd; 600 s covers a + first configure+build of an SDK-heavy family (pico, nrf, esp). Builds normally run + pre-lock, so a board flock is not held here except on rare recovery paths. Per-build + compile parallelism is capped at cpu/-j so -j concurrent builds cannot swamp sibling + workers' verification windows. Returns the returncode (127 = ESP-IDF env missing).""" + name = board['name'] + variants = board.get('variant') or [{'name': name}] + vcfg = next((v for v in variants if v['name'] == variant), variants[0]) + if board['flasher']['name'].lower() == 'esptool': + if not shutil.which('idf.py'): + return 127 # ESP-IDF env not sourced in this shell + # -B keyed off the VARIANT so ensure_fw's post-build lookup finds it + cmd = ['idf.py', '-C', f'examples/{example}', + '-B', f'cmake-build/cmake-build-{vcfg["name"]}/{example}', + '-G', 'Ninja', f'-DBOARD={name}', 'build'] + for d in board.get('build', {}).get('args', []) + vcfg.get('defines', []): + cmd.insert(-1, f'-D{d}') + if vcfg.get('flags'): + cmd.insert(-1, f'-DCFLAGS_CLI={vcfg["flags"]}') + # the IDF component manager writes examples//dependencies.lock in the + # SOURCE tree (idf.py -B relocates only the build dir), so concurrent esp + # builds of one example for different targets corrupt each other's solve + with _esp_lock, _build_sem: + return hil_util.run_cmd(shlex.join(cmd), cwd=str(hil_util.TINYUSB_ROOT), + timeout=600).returncode + cmd = [sys.executable, str(hil_util.TINYUSB_ROOT / 'tools' / 'build.py'), + '-b', name, '-T', Path(example).name, + '-j', str(max(1, (os.cpu_count() or _jobs) // _jobs))] + for d in board.get('build', {}).get('args', []): + cmd += ['-D', d] + if vcfg['name'] != name: + cmd += ['--build-name', vcfg['name']] + for d in vcfg.get('defines', []): + cmd += ['-D', d] + for tok in vcfg.get('flags', '').split(): + cmd += [f'--cflag={tok}'] + with _build_sem: + return hil_util.run_cmd(shlex.join(cmd), cwd=str(hil_util.TINYUSB_ROOT), + timeout=600).returncode + + +_deps_lock = threading.Lock() # one get_deps at a time (it also drains _build_sem) +_esp_lock = threading.Lock() # idf.py mutates source-tree dependencies.lock per example +_no_build = False # --no-build: ensure_fw never invokes a build +_jobs = 4 # mirrors -j; set in main before the pool starts +_build_sem = threading.BoundedSemaphore(4) # build slots; get_deps drains ALL (exclusive) +_builds: dict = {} # (variant, example) -> (fw|None, reason): one attempt per run + + +def ensure_fw(board: dict, variant: str, example: str, note: list): + """Firmware for `example`, building it when absent — never skip a board for lack of a + build (--no-build opts out). One retry with deps fetched and the CMake caches dropped + when the first build fails (fresh checkouts lack the family deps; a cache configured + in a broken env poisons every later attempt). Returns the firmware path, or None with + the failure noted. Call BEFORE taking the board lock: builds are long. One attempt per + (variant, example) per run, memoized in _builds, so a repeat call (park, under the + held flock) resolves instantly even when an exclusive -B hides the fresh artifact.""" + fw = hil_flash.find_firmware(variant, example, flasher=board['flasher']['name']) + if fw: + return fw + key, base = (variant, example), Path(example).name + if key in _builds: + return _builds[key][0] + if _no_build: + _builds[key] = (None, 'disabled') + note.append(f'build skipped (--no-build): {base}') + return None + rc = build_example(board, variant, example) + if rc == 127 and board['flasher']['name'].lower() == 'esptool': + _builds[key] = (None, 'no-env') + note.append(f'cannot build {base}: ESP-IDF env missing (get-idf)') + return None + if rc == 124: # hung build: a deps/cache retry cannot cure it, don't double the stall + _builds[key] = (None, 'timeout') + note.append(f'build timeout: {base}') + return None + if rc != 0: + # retry once with deps fetched and the CMake caches dropped (cache only — a tree + # wipe would destroy every other example's firmware). get_deps git-resets shared + # deps that are already present, so it drains ALL build slots first. + with _deps_lock: + for _ in range(_jobs): + _build_sem.acquire() + try: + r = hil_util.run_cmd(shlex.join([sys.executable, str(hil_util.TINYUSB_ROOT / 'tools' / 'get_deps.py'), + '-b', board['name']]), + cwd=str(hil_util.TINYUSB_ROOT), timeout=600) + finally: + for _ in range(_jobs): + _build_sem.release() + if r.returncode != 0: + note.append('get_deps failed') + bd = hil_util.TINYUSB_ROOT / 'cmake-build' / f'cmake-build-{variant}' + # esp configures one level deeper (//): wipe both layouts + for d in (bd, bd / example): + shutil.rmtree(d / 'CMakeFiles', ignore_errors=True) + (d / 'CMakeCache.txt').unlink(missing_ok=True) + rc = build_example(board, variant, example) + if rc != 0: + _builds[key] = (None, 'fail') + note.append(f'build failed: {base}') + return None + # both build paths write to cmake-build/, so look there even when an explicit -B + # narrowed the global search — this is OUR fresh build, not a stale fallback + fw = hil_flash.find_firmware(variant, example, + roots=[hil_flash.build_dir, 'cmake-build'], + flasher=board['flasher']['name']) + _builds[key] = (fw, 'ok' if fw else 'no-fw') + note.append(f'built {base}' if fw else f'build produced no firmware: {base}') + return fw + + +def ensure_board_test(board: dict, variant: str, note: list): + """board_test firmware for parking, building it if absent (via ensure_fw). + Espressif included — tools/build.py builds board_test for that family too; + the build just needs the ESP-IDF env (127 → noted, park is then skipped).""" + fw = hil_flash.find_firmware(variant, 'device/board_test', flasher=board['flasher']['name']) + if fw: + return fw + variants = board.get('variant') or [{'name': board['name']}] + if not any(v['name'] == variant for v in variants): + variant = variants[0]['name'] + return ensure_fw(board, variant, 'device/board_test', note) + + +def verdict(row: dict, ok: bool) -> str: + """Row status for a verification result, preserving a 'flash-failed' a deeper + layer already recorded (silent flash no-op, board_test delivery failure).""" + return 'ok' if ok else ('flash-failed' if row['status'] == 'flash-failed' else 'failed') + + +def host_alive(board: dict, note: list, row: dict, flashed_example: bool = False) -> bool: + """Serial aliveness with recovery: silent -> (build and) flash board_test (it + hellos every second and echoes) -> recheck. Also cures a silent flash no-op + that left the board crashed. + + With flashed_example=True (a host example was just flashed), board_test-shaped + output FAILS the check: the parked image still talking means the example flash + silently didn't take — the host analog of the device path's PID check. + + Side effect: delivery-class failures (silent no-op, board_test build/flash + failure) set row['status'] = 'flash-failed' so verdict() preserves the cause; + the caller derives the final status from the return value via verdict().""" + data = check_host_serial(board) + if data: + if flashed_example and boardtest_output(data): + note.append('board_test output after example flash: silent flash no-op') + row['status'] = 'flash-failed' + return False + return True + variant = resolve_variant(board, 'device/board_test', note) + fw = ensure_board_test(board, variant, note) + if fw is None: + note.append('serial silent; board_test unavailable') + row['status'] = 'flash-failed' + return False + say(f'{board["name"]:26} recovery: serial silent, flashing board_test') + rc, err = call_flasher(getattr(hil_flash, f'flash_{board["flasher"]["name"].lower()}'), board, str(fw)) + if rc != 0: + note.append(f'serial silent; board_test flash failed: {err}') + row['status'] = 'flash-failed' + return False + if not check_host_serial(board): + return False + if flashed_example: + # board_test talking proves the BOARD is alive, but the just-flashed + # example never produced serial — that verification still fails + note.append('example silent; board alive via board_test reflash') + return False + note.append('recovered via board_test reflash') + return True + + +def device_recover_and_check(board: dict, example: str, variant: str, old_ino, note: list, row: dict, seen: dict) -> bool: + """Wait for the flashed board's uid to re-enumerate; on timeout, try one board + reset (skipped for flashers with no hardware reset — see hil_flash.RESET_NOOP, + it would just burn the wait) and wait again. + + The PID policy is deliberately asymmetric. Pre-reset, the re-enumeration was + caused by the flash itself, so a PID mismatch most likely means the build dir + is stale (the flash DID write what find_firmware found) — warn, don't fail — + UNLESS the firmware was built this very run: then 'stale build' is impossible + and the mismatch can only be a silent flash no-op, which fails. Post-reset, + the re-enumeration proves nothing about the flash (the reset alone explains + it), so a mismatch is treated as a silent flash no-op and fails; an unknown + expected PID scores ok with a 'pid unverified' note in both paths.""" + name = board['name'] + expected_pid = get_expected_pid(example) + built_this_run = _builds.get((variant, example), (None, ''))[1] == 'ok' + + def seen_hit(hit): + seen[board['uid']] = {'name': name, 'busport': hit[0], 'when': time.strftime('%Y-%m-%d %H:%M')} + + hit = wait_device(board['uid'], None, old_ino, ENUM_WAIT) + if hit: + if expected_pid is not None and not hit[1].endswith(expected_pid): + if built_this_run: + row['device'] = f'❌ {hit[1]}' + note.append(f'pid {hit[1]}, this run built {expected_pid}: silent flash no-op') + row['status'] = 'flash-failed' + return False + note.append(f'⚠ pid {hit[1]}, source says {expected_pid}: stale build or silent flash no-op') + elif expected_pid is None: + note.append('pid unverified') + row['device'] = f'✅ {hit[1]}' + seen_hit(hit) + return True + + flasher_name = board['flasher']['name'].lower() + if flasher_name in hil_flash.RESET_NOOP: + note.append(f'no hardware reset available for {flasher_name}') + row['device'] = '❌ not enumerated' + return False + + say(f'{name:26} recovery: uid not up, resetting board') + rc, err = call_flasher(getattr(hil_flash, f'reset_{flasher_name}'), board) + if rc != 0: + note.append(f'reset failed: {err}') + hit = wait_device(board['uid'], None, old_ino, ENUM_WAIT_RETRY) + if not hit: + row['device'] = '❌ not enumerated' + note.append('reset did not help') + return False + if expected_pid is None: + row['device'] = f'✅ {hit[1]}' + note.append('reset recovered (pid unverified)') + seen_hit(hit) + return True + if hit[1].endswith(expected_pid): + row['device'] = f'✅ {hit[1]}' + note.append('reset recovered') + seen_hit(hit) + return True + row['device'] = f'❌ {hit[1]}' + note.append(f'reset recovered wrong pid, expected {expected_pid}: silent flash no-op') + row['status'] = 'flash-failed' + return False + + +def check_board(board: dict, args, allow_recovery: bool, seen: dict) -> dict: + name = board['name'] + row = {'name': name, 'probe': '❌ missing', 'flash': '–', 'device': '–', 'note': [], 'status': 'failed'} + note = row['note'] + + probe = find_usb(board['flasher']['uid']) + if probe: + row['probe'] = f'✅ {probe[0]}' + seen[board['flasher']['uid']] = {'name': f'{name} probe', 'busport': probe[0], + 'when': time.strftime('%Y-%m-%d %H:%M')} + else: + last = seen.get(board['flasher']['uid']) + note.append(f'probe last seen {last["busport"]} {last["when"]}' if last + else 'probe never seen by pool_check') + say(f'{name:26} probe MISSING ({board["flasher"]["name"]} {board["flasher"]["uid"]})') + + # existing firmware only; a missing build is built further down (after a lock peek), + # except in scan/no-build modes and never for a missing probe + example, kind, variant, fw = pick_example(board, note, build_missing=False) + if kind == 'host': + note.append('host-only board') + + if args.scan_only: + hit = find_device(board['uid'], None) + # report the BOARD's usb state too: enumerated (with busport), off-bus (normal + # when parked in board_test), or n/a for host-only boards + if hit: + row['device'] = f'✅ {hit[1]} @{hit[0]}' + elif kind == 'host': + row['device'] = '– n/a (host-only)' + else: + row['device'] = '⚫ off bus (parked?)' + # scan verifies probe presence only, so probe present is ok; a missing probe means + # no firmware could be delivered → flash-failed + row['status'] = 'ok' if probe else 'flash-failed' + if probe: + say(f'{name:26} probe ✅ {probe[0]}' + (f' device {hit[1]}' if hit else '')) + return row + if not probe: + row['status'] = 'flash-failed' + return row + + bt_variant = resolve_variant(board, 'device/board_test', note) + need_example = example is None and not args.no_build + # board_test is also host_alive's recovery image, so host boards pre-build it + # even under --no-park; --no-build gates EVERY build, board_test included + need_bt = (not args.no_build + and (not args.no_park or kind == 'host') + and hil_flash.find_firmware(bt_variant, 'device/board_test', + flasher=board['flasher']['name']) is None) + if need_example or need_bt: + # builds are long and run BEFORE locking (park must never hold the flock through + # one); peek first so minutes of building are not wasted on — or a rebuilt tree + # swapped under — a board CI holds right now + peek = lock_board(name) + if isinstance(peek, str): + if peek.startswith('ERROR:'): # environment failure, not a held lock + row['flash'] = '❌ lock' + row['status'] = 'failed' + else: + row['flash'] = '🔒 locked' + row['status'] = 'locked' + note.append(peek) + say(f'{name:26} locked: {peek}') + return row + unlock_board(peek) + if need_example: + example, kind, variant, fw = pick_example(board, note, build_missing=True) + if need_bt and (example is not None or kind == 'host'): + # skip the park build when the example build already failed on a device board: + # the row returns before any flash/park could use it + ensure_board_test(board, bt_variant, note) + + if example is None: + if not any(n.startswith(('build failed', 'build timeout', 'build produced', + 'build skipped', 'cannot build')) for n in note): + note.append('no firmware built') + if kind != 'host': + row['status'] = 'flash-failed' + say(f'{name:26} probe ✅ {probe[0]} (no firmware to flash)') + return row + # host-only board: aliveness is still checkable without flashing — reset and listen + # to whatever is on it (parked board_test echoes and hellos on the flasher UART) + + lk = lock_board(name) + if isinstance(lk, str): + if lk.startswith('ERROR:'): # environment failure, not a held lock + row['flash'] = '❌ lock' + row['status'] = 'failed' + else: + row['flash'] = '🔒 locked' + row['status'] = 'locked' + note.append(lk) + say(f'{name:26} locked: {lk}') + return row + try: + if example is None: # host-only without firmware: UART-only aliveness check + ok = host_alive(board, note, row) + row['device'] = '✅ serial out' if ok else '❌ no serial out' + row['status'] = verdict(row, ok) + say(f'{name:26} – {row["device"]} (existing firmware)') + return row + + pre = find_device(board['uid'], None) + old_ino = pre[2] if pre else None + + try: + if not flash(board, fw, allow_recovery, probe[0], note): + row['flash'] = f'❌ {Path(example).name}' + row['status'] = 'flash-failed' + say(f'{name:26} flash FAILED ({example})') + return row + row['flash'] = f'✅ {Path(example).name}' + + if kind == 'host': + ok = host_alive(board, note, row, flashed_example=True) + row['device'] = '✅ serial out' if ok else '❌ no serial out' + else: + ok = device_recover_and_check(board, example, variant, old_ino, note, row, seen) + row['status'] = verdict(row, ok) + say(f'{name:26} {row["flash"]} {row["device"]}') + return row + finally: + # teardown for EVERY path that attempted a flash (a failed programmer op can + # still have erased/half-written the target), while the lock is still held + if not args.no_park: + park_board(board, kind, row, note) + finally: + unlock_board(lk) + + +def park_board(board: dict, kind: str, row: dict, note: list) -> None: + """Re-park with board_test, building it if absent (ensure_board_test), and + VERIFY it took: board_test never enumerates USB, so a device board's cafe + device must drop off the bus, and a host board must answer with board_test's + own output — a rc=0 park that changed nothing (silent no-op) must not pass. + A board left unparked marks an ok row flash-failed (never downgrading a + 'failed' verify verdict — that is the more diagnostic signal), with one + exception: an espressif board without the ESP-IDF env cannot build + board_test — noted, not a board fault.""" + # capture BEFORE the park flash: uid-disappearance only verifies the park if the + # device was on the bus to begin with + on_bus_before = kind != 'host' and find_device(board['uid'], None) is not None + variant = resolve_variant(board, 'device/board_test', note) + fw = ensure_board_test(board, variant, note) + if fw is None: + if any(n.startswith('cannot build board_test') for n in note): + note.append('park skipped (no ESP-IDF env)') + else: + # --no-build disables builds, not parking (--no-park is that opt-out): + # a board left running a USB-active image is unparked either way + note.append('unparked: board_test not built (--no-build)' + if any(n.startswith('build skipped (--no-build): board_test') for n in note) + else 'unparked: board_test unavailable (build failed/timed out)') + if row['status'] == 'ok': + row['status'] = 'flash-failed' + return + rc, err = call_flasher(getattr(hil_flash, f'flash_{board["flasher"]["name"].lower()}'), + board, str(fw)) + if rc != 0: + note.append(f'park flash failed: {err}') + if row['status'] == 'ok': + row['status'] = 'flash-failed' + return + if kind == 'host': + # no second reset (the park flash's own reset started board_test); POSITIVE + # marker: its hello must appear, and stale bridge-FIFO output alongside it is not + # disqualifying + data = check_host_serial(board, do_reset=False, want_hello=True) + if not (data and b'Hello from TinyUSB' in data): + note.append('park unverified: no board_test output') + if row['status'] == 'ok': + row['status'] = 'flash-failed' + return + if not on_bus_before: + # never enumerated this run: uid-disappearance cannot tell a verified park from a + # silent no-op — say so instead of passing vacuously + note.append('park unverified (device already off bus)') + return + deadline = time.monotonic() + 6 + while time.monotonic() < deadline: + if find_device(board['uid'], None) is None: + return + time.sleep(0.5) + note.append('park unverified: device still enumerated') + if row['status'] == 'ok': + row['status'] = 'flash-failed' + + +def check_board_safe(board: dict, args, allow_recovery: bool, seen: dict) -> dict: + """Isolate one board's exceptions: a crashing worker must not discard every + other board's row, the table, the topology, and the seen-cache write.""" + try: + return check_board(board, args, allow_recovery, seen) + except Exception as e: + name = board.get('name', '?') + say(f'{name:26} INTERNAL ERROR: {e!r}') + return {'name': name, 'probe': '–', 'flash': '–', 'device': '❌ error', + 'note': [repr(e)[:120]], 'status': 'failed'} + + +def controller_summary() -> list[str]: + """USB topology: controller (PCI addr, vendor) -> bus -> root-port subtree device + counts (hubs included, interfaces/root hubs not). Bus numbers renumber every boot; + PCI addresses and root-port numbers are stable.""" + vendor_names = {'0x1022': 'AMD', '0x1912': 'Renesas', '0x8086': 'Intel', '0x1b21': 'ASMedia'} + ctrl = {} + for root in glob.glob('/sys/bus/usb/devices/usb*'): + bus = int(os.path.basename(root)[3:]) + m = re.findall(r'[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]', os.path.realpath(root)) + pci = m[-1] if m else '?' + c = ctrl.setdefault(pci, {'vendor': '?', 'buses': {}}) + subtrees = {} + for d in glob.glob(f'/sys/bus/usb/devices/{bus}-*'): + b = os.path.basename(d) + if ':' in b: + continue + subtrees[b.split('.')[0]] = subtrees.get(b.split('.')[0], 0) + 1 + c['buses'][bus] = subtrees + try: + vid = open(f'/sys/bus/pci/devices/{pci}/vendor').read().strip() + c['vendor'] = vendor_names.get(vid, vid) + except OSError: + pass + + lines = [] + for pci, c in sorted(ctrl.items()): + lines.append(f'{pci} ({c["vendor"]})') + for bus, subtrees in sorted(c['buses'].items()): + detail = ' '.join(f'{k}: {n} dev' for k, n in + sorted(subtrees.items(), key=lambda i: int(i[0].split('-')[1]))) + lines.append(f' bus {bus}: {sum(subtrees.values())} devices' + + (f' {detail}' if detail else '')) + return lines + + +def main() -> None: + # toolchain/flasher CLIs live in the user bin dirs, which non-login shells may lack -- + # the same PATH shim hil_ci.sh applies on the remote side + for d in (Path.home() / 'bin', Path.home() / '.local' / 'bin'): + if d.is_dir() and str(d) not in os.environ.get('PATH', '').split(os.pathsep): + os.environ['PATH'] = f'{d}{os.pathsep}{os.environ.get("PATH", "")}' + + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument('config', nargs='?', help='HIL config json (default: by hostname)') + parser.add_argument('-b', '--board', action='append', default=[], help='only these boards') + parser.add_argument('-B', '--build-dir', default=None, + help='firmware parent dir, searched EXCLUSIVELY when given ' + '(default: examples, plus cmake-build as fallback)') + parser.add_argument('--scan-only', action='store_true', + help='USB presence scan only: no locks, no flashing') + parser.add_argument('--no-build', action='store_true', + help='do not build missing firmware (default: build the light example on the spot)') + parser.add_argument('--no-park', action='store_true', + help='leave the light example running (default: park with board_test)') + # no cross-process flash budget against a concurrent hil_test.py run (its semaphores + # are in-process), so keep this modest + parser.add_argument('-j', '--jobs', type=int, default=4) + parser.add_argument('-v', '--verbose', action='store_true') + args = parser.parse_args() + global _no_build, _jobs, _build_sem + _no_build = args.no_build + _jobs = max(1, args.jobs) + _build_sem = threading.BoundedSemaphore(_jobs) + + host = socket.gethostname() + cfg_name = args.config or CONFIG_BY_HOST.get(host, 'local.json') + cfg_path = Path(cfg_name) + if not cfg_path.exists(): + cfg_path = REPO_ROOT / 'test' / 'hil' / cfg_name + if not cfg_path.exists(): + sys.exit(f'config not found: {cfg_name} (host {host}; dev PCs need test/hil/local.json)') + with cfg_path.open() as f: + config = json.load(f) + + boards = list(config['boards']) # boards-skip (parked hardware) is not scanned by default + if args.board: + boards += config.get('boards-skip', []) # explicitly named parked boards are fair game + unknown = set(args.board) - {b['name'] for b in boards} + if unknown: + sys.exit(f'board(s) not in {cfg_path.name}: {", ".join(sorted(unknown))}') + boards = [b for b in boards if b['name'] in args.board] + + hil_flash.build_dir = args.build_dir or 'examples' + hil_util.verbose = args.verbose + if args.build_dir is None: + # default mode: search both standard layouts (cmake-build/ from tools/build.py and + # ESP-IDF, examples/ from manual builds). An EXPLICIT -B stays exclusive: the caller + # named an artifact tree, so a miss must report rather than flash an older build. + hil_flash.EXTRA_BUILD_DIRS = ['cmake-build', 'examples'] + allow_recovery = not args.scan_only and can_recover() + seen = {} + try: + loaded = json.loads(SEEN_CACHE.read_text()) + if isinstance(loaded, dict): # tolerate a torn/hand-edited cache + seen = {k: v for k, v in loaded.items() if isinstance(v, dict)} + except (OSError, ValueError): + pass + + roots = ' + '.join(dict.fromkeys([hil_flash.build_dir, *hil_flash.EXTRA_BUILD_DIRS])) + say(f'pool check: host {host}, config {cfg_path.name}, {len(boards)} boards, ' + f'{"scan-only" if args.scan_only else f"flash via {{{roots}}}/cmake-build-"}' + f'{"" if allow_recovery or args.scan_only else ", recovery unavailable (no sudo -n / usb_recover.sh)"}') + + if args.verbose: + rows = [check_board_safe(b, args, allow_recovery, seen) for b in boards] + else: + with io.StringIO() as spool, ThreadPoolExecutor(max_workers=args.jobs) as pool: + sys.stdout = spool # silence hil_util.run_cmd's COMMAND FAILED dumps; say() uses __stdout__ + try: + rows = list(pool.map(lambda b: check_board_safe(b, args, allow_recovery, seen), boards)) + finally: + sys.stdout = sys.__stdout__ + + try: + SEEN_CACHE.parent.mkdir(parents=True, exist_ok=True) + tmp = SEEN_CACHE.with_suffix('.json.tmp') + tmp.write_text(json.dumps(seen, indent=1, sort_keys=True) + '\n') + tmp.replace(SEEN_CACHE) # atomic: a killed run can't tear the cache + except OSError: + pass + + status_mark = {'ok': '✅ ok', 'flash-failed': '❌ flash-failed', 'failed': '❌ failed', + 'locked': '🔒 locked'} + headers = ['Board', 'Probe', 'Flash', 'Device', 'Status', 'Note'] + cells = [[r['name'], r['probe'], r['flash'], r['device'], + status_mark.get(r['status'], r['status']), '; '.join(r['note'])] for r in rows] + widths = [max(len(h), *(len(c[i]) for c in cells)) if cells else len(h) + for i, h in enumerate(headers)] + line = lambda vals: '| ' + ' | '.join(v.ljust(w) for v, w in zip(vals, widths)) + ' |' + print() + print(line(headers)) + print('|' + '|'.join('-' * (w + 2) for w in widths) + '|') + for c in cells: + print(line(c)) + + print('\nUSB topology (controller → root-port subtree):') + for line in controller_summary(): + print(f' {line}') + + counts = {'ok': 0, 'flash-failed': 0, 'failed': 0, 'locked': 0} + for r in rows: + counts[r.get('status', 'failed')] += 1 + print(f'\n{counts["ok"]} ok · {counts["flash-failed"]} flash-failed · {counts["failed"]} failed ' + f'· {counts["locked"]} locked · in {time.monotonic() - t0:.0f}s') + if hil_util.sysfs_blind(): + # Without this the table is the worst kind of wrong: once the process latches + # blind, every read answers SYSFS_UNKNOWN, scan_usb() returns {}, and EVERY board + # prints "probe MISSING"/"off bus" -- a clean-looking report declaring the whole + # fleet dead, produced during exactly the incident this tool is run to diagnose, + # and it sends the operator to power-cycle a rig where one device is wedged. + print('WARNING: this scan lost sight of the bus' + f'{hil_util.sysfs_blind_note()}. Rows above that say a probe or board is ' + f'missing may be this tool losing sight of healthy hardware, not absent ' + f'hardware. Find the wedged device (see the usb-kernel-recover skill) and ' + f're-run before acting on the table.') + sys.exit(min(counts['flash-failed'] + counts['failed'], 125)) + + +if __name__ == '__main__': + main() diff --git a/test/hil/helper/hil_select.py b/test/hil/helper/hil_select.py new file mode 100755 index 000000000..f0d4f0b9f --- /dev/null +++ b/test/hil/helper/hil_select.py @@ -0,0 +1,524 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""PR-diff -> HIL selection: which rig boards and which tests a change can affect. + +Stdlib-only (runs on bare CI runners; imports hil_util for the example rosters, +never hil_test/pyserial — test_hil_util.BottomLayer enforces the stdlib closure). +Fail-open: any file no rule classifies forces the full matrix. See +docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md. + +JSON: full, boards (name -> 'all' | [tests]), families (bsp families the diff +touches, including ones with no rig board - build-only consumers such as /pre-pr +sample from these), args (hil_test.py args per config) and args_flasher (the same +args split by each board's flasher, for CI legs that split one rig by flasher). +""" +import argparse +import functools +import glob +import json +import os +import re +import subprocess +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # helper/ scripts import via the test/hil root +from helper.hil_util import device_tests, dual_tests, host_test + +ALL_TESTS = {'device': device_tests, 'dual': dual_tests, 'host': host_test} + +# class dir -> config macro suffix exceptions (rule 3); dfu is per-file, handled inline +NET_MACROS = ('ECM_RNDIS', 'NCM') + +_NONCODE_RE = re.compile( + r'^(docs/|\.claude/|.*\.(md|rst)$|LICENSE)') +_FULL_RE = re.compile( + r'^(src/common/|src/osal/|src/tusb\.c$|src/tusb\.h$|src/tusb_option\.h$|' + r'test/hil/|\.github/workflows/build.*\.yml$|\.github/actions/|\.github/scripts/|' + r'tools/build\.py$|tools/get_deps\.py$|tools/cmake/|hw/mcu/|lib/|' + r'hw/bsp/(family_support\.cmake|board_api\.h|board\.c|ansi_escape\.h)$|' + r'examples/build_system/|examples/CMakeLists\.txt$|' + # board_test is HIL infrastructure, not a test: hil_test.py flashes it to park + # every board (variant boundary + end-of-board teardown), so every board depends on it + r'examples/device/board_test/)') + +# --no-renames: with rename detection git reports only a rename's destination, so code +# moved out of an HIL-relevant path would be classified by its new path alone +GIT_DIFF_ARGV = ['git', 'diff', '--no-renames', '--name-only'] + + +def test_role(test: str) -> str: + return test.split('/', 1)[0] # 'device' | 'dual' | 'host' + + +def board_roles(board: dict) -> set: + t = board.get('tests', {}) + roles = set() + if t.get('device'): + roles.add('device') + if t.get('host'): + roles.add('host') + if t.get('dual'): + roles.update(('device', 'host')) + for only in t.get('only', []): + r = test_role(only) + roles.update(('device', 'host') if r == 'dual' else (r,)) + return roles + + +def board_tests(board: dict) -> list: + """Every test this board would run today (mirrors hil_test.test_board's default).""" + t = board.get('tests', {}) + if 'only' in t: + run = list(t['only']) + else: + run = [] + if t.get('device'): + run += device_tests + if t.get('dual'): + run += dual_tests + if t.get('host'): + run += host_test + return [x for x in run if x not in t.get('skip', [])] + + +# cached: called per changed file x roster board, and the tree doesn't change mid-run +@functools.lru_cache(maxsize=None) +def board_family(board_name: str, repo_root: str): + hits = glob.glob(os.path.join(repo_root, 'hw/bsp/*/boards', board_name)) + return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None + + +# `if (OPTION STREQUAL "1")` guards in family_support.cmake, and the option tokens +# a roster entry passes to the build (NAME=VALUE / -DNAME=VALUE) +_CM_IF_RE = re.compile(r'if\s*\(') +_CM_ELSE_RE = re.compile(r'else(if)?\s*\(') +_CM_ENDIF_RE = re.compile(r'endif\s*\(') +_CM_OPT_RE = re.compile(r'if\s*\(\s*\$?\{?([A-Za-z_]\w*)\}?\s+STREQUAL\s+"?1"?\s*\)') +_CM_PORT_RE = re.compile(r'src/portable/((?:[^/\s]+/)?[^/\s]+)/') +_FALSY = ('', '0', 'off', 'false', 'no') + + +@functools.lru_cache(maxsize=None) +def port_option_gates(repo_root: str) -> dict: + """port dir -> build options that compile it regardless of the board's family + file, e.g. {'analog/max3421': {'MAX3421_HOST'}} from family_support.cmake.""" + gates = {} + try: + text = open(os.path.join(repo_root, 'hw/bsp/family_support.cmake')).read() + except OSError: + return gates + stack = [] # one entry per open if(): its option, or None + for line in text.splitlines(): + line = line.strip() + if _CM_IF_RE.match(line): + m = _CM_OPT_RE.match(line) + stack.append(m.group(1) if m else None) + elif _CM_ELSE_RE.match(line): + if stack: + stack[-1] = None # the guard doesn't hold in this branch + elif _CM_ENDIF_RE.match(line): + if stack: + stack.pop() + opts = {o for o in stack if o} + m = _CM_PORT_RE.search(line) + if opts and m: + gates.setdefault(m.group(1), set()).update(opts) + return gates + + +_CM_SET_RE = re.compile(r'set\s*\(\s*([A-Za-z_]\w*)\s+([^)\s]+)\s*\)') + + +# cached: called per changed portable file x roster board +@functools.lru_cache(maxsize=None) +def bsp_board_options(board_name: str, repo_root: str) -> frozenset: + """Build options a board turns on in its own BSP: `set( )` in + hw/bsp//boards//board.cmake, e.g. MAX3421_HOST on the espressif + and rp2040 max3421 boards. CMake only - HIL CI builds nothing with Make, so a + board.mk-only option (e.g. nrf5340dk's MAX3421_HOST) compiles no port here.""" + fam = board_family(board_name, repo_root) + if not fam: + return frozenset() + path = os.path.join(repo_root, 'hw/bsp', fam, 'boards', board_name, 'board.cmake') + try: + text = open(path).read() + except OSError: + return frozenset() + out = set() + for line in text.splitlines(): + line = line.strip() + if line.startswith('#'): + continue + m = _CM_SET_RE.match(line) + if m and m.group(2).strip('"').lower() not in _FALSY: + out.add(m.group(1)) + return frozenset(out) + + +def board_options(board: dict, repo_root: str) -> set: + """Build options a board has truthy: the roster entry's build.args plus each + variant's defines (NAME=VALUE) and raw CFLAGS (-DNAME=VALUE), plus whatever its + own board.cmake sets (a board can enable a gated port without the roster saying so).""" + toks = list(board.get('build', {}).get('args', [])) + for v in board.get('variant', []): + toks += list(v.get('defines', [])) + toks += v.get('flags', '').split() + out = set(bsp_board_options(board['name'], repo_root)) + for t in toks: + name, _, val = (t[2:] if t.startswith('-D') else t).partition('=') + if name and val.strip().strip('"').lower() not in _FALSY: + out.add(name.strip()) + return out + + +@functools.lru_cache(maxsize=None) +def port_families(port_dir: str, repo_root: str) -> set: + """Board families that compile this src/portable dir. CMake only: HIL CI builds + every board with CMake, so a port wired up in family.mk alone is compiled for no + HIL board and must not select one. family.cmake lists portable sources directly + for most families; espressif instead references them from a nested component + CMakeLists.txt (hw/bsp/espressif/components/tinyusb_src/CMakeLists.txt).""" + fams = set() + bsp_root = os.path.join(repo_root, 'hw/bsp') + # trailing '/' so a port dir is not a prefix of a sibling: bare 'microchip/pic' + # would otherwise match '.../microchip/pic32mz/...' and inherit its families + needle = port_dir + '/' + for f in glob.glob(os.path.join(bsp_root, '*/family.cmake')) + \ + glob.glob(os.path.join(bsp_root, '*/components/*/CMakeLists.txt')): + try: + if needle in open(f).read(): + fam = os.path.relpath(f, bsp_root).split(os.sep, 1)[0] + fams.add(fam) + except OSError: + pass + return fams + + +_CLS_INC_RE = re.compile(r'#\s*include\s*[<"]class/([^/"<>]+)/([^"<>]+)[">]') + + +@functools.lru_cache(maxsize=None) +def class_include_edges(repo_root: str) -> dict: + """'/
' -> the other class dirs that include it. A class header + pulled in by a second class ships in every firmware enabling that second class: + src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, and + net_device.h includes class/cdc/cdc.h. The class rule derives macros from the + directory name alone, so without this edge a change to the included header + selects only its own class's examples - and on a board that skips those (e.g. + metro_m4_express skips audio_test_freertos), nothing at all. + + Derived from the actual #include lines rather than a hand-written table so it + cannot rot when a class picks up or drops a cross-class include.""" + edges = {} + for f in sorted(glob.glob(os.path.join(repo_root, 'src/class/*/*.[ch]'))): + cls = os.path.basename(os.path.dirname(f)) + try: + text = open(f).read() + except OSError: + continue + for inc_cls, inc_hdr in _CLS_INC_RE.findall(text): + if inc_cls != cls: + edges.setdefault(f'{inc_cls}/{inc_hdr}', set()).add(cls) + return edges + + +def class_macros(cls: str, base: str, prefix: str) -> list: + """Config macros that compile a class dir's code, for role prefix TUD/TUH. + `base` refines dfu only (it splits DFU from DFU_RUNTIME per file); pass '' for + a class reached through an include edge, where the widest set is correct.""" + if cls == 'net': + return [f'CFG_{prefix}_{m}' for m in NET_MACROS] + if cls == 'dfu': + if base.startswith('dfu_rt'): + return [f'CFG_{prefix}_DFU_RUNTIME'] + if base.startswith('dfu_device') or base.startswith('dfu_host'): + return [f'CFG_{prefix}_DFU'] + return [f'CFG_{prefix}_DFU', f'CFG_{prefix}_DFU_RUNTIME'] + return [f'CFG_{prefix}_{cls.upper()}'] + + +def _config_enables(cfg_path: str, macros) -> bool: + try: + text = open(cfg_path).read() + except OSError: + return False + return any(re.search(rf'#define\s+{m}\s+\(?\s*0*[1-9]', text) for m in macros) + + +def roster_only_tests(all_boards) -> set: + """Test paths that only appear in a roster board's tests.only list (e.g. + espressif boards), not in the shared device/dual/host_test lists.""" + out = set() + for b in all_boards: + out.update(b.get('tests', {}).get('only', [])) + return out + + +def class_examples(macros, role: str, repo_root: str, extra_tests: set) -> set: + """Tests (from role's + dual lists, plus roster-only-list tests of that role) + whose example config enables any macro.""" + pool = role_tests({role}, extra_tests) + out = set() + for test in pool: + cfg = os.path.join(repo_root, 'examples', test, 'src', 'tusb_config.h') + if _config_enables(cfg, macros): + out.add(test) + return out + + +def role_tests(roles: set, extras: set) -> set: + """Every test for the given role(s): each role's own list + dual tests, + plus roster-only-list tests (extras) matching those roles or 'dual'.""" + pool = set(dual_tests) + for r in roles: + pool |= set(ALL_TESTS[r]) + pool |= {t for t in extras if test_role(t) in roles or test_role(t) == 'dual'} + return pool + + +class _Sel: + """Accumulates contributions. board->set(tests) plus 'all-board' markers.""" + def __init__(self): + self.full = False + self.by_board = {} # name -> set of tests, or 'all' + self.roles = set() # roles touched by any contribution + self.families = set() # bsp families touched (incl. off-rig ones: build-only consumers) + self.reasons = [] + + def add(self, boards, tests, reason): + """tests: 'all' or iterable of test paths.""" + self.reasons.append(reason) + for b in boards: + cur = self.by_board.get(b) + if tests == 'all' or cur == 'all': + self.by_board[b] = 'all' + else: + self.by_board[b] = (cur or set()) | set(tests) + + def force_full(self, reason): + self.full = True + self.reasons.append(reason) + + +def _classify_one(path, repo_root, roster_boards, extras: set, s: _Sel): + base = os.path.basename(path) + if _NONCODE_RE.match(path): + s.reasons.append(f'{path}: non-code, no contribution') + return + if _FULL_RE.match(path): + s.force_full(f'{path}: core/infra -> full matrix') + return + + m = re.match(r'src/portable/((?:[^/]+/)?[^/]+)/', path) + if m: + port = m.group(1) + if re.match(r'(dcd_|.*_device)', base): + roles = {'device'} + elif re.match(r'(hcd_|.*_host)', base): + roles = {'host'} + else: + roles = {'device', 'host'} + fams = port_families(port, repo_root) + if not fams: + # no family references this port: either a new/renamed port dir or a + # family.cmake layout the scan misses - widen instead of contributing nothing + s.force_full(f'{path}: port {port} maps to no board family -> full matrix') + return + s.families.update(fams) + # a board can also pull the port in through a build option (e.g. MAX3421_HOST=1 + # from the roster on metro_m4_express, or from its own board.cmake), which its + # family file never names + gates = port_option_gates(repo_root).get(port, set()) + boards = [b['name'] for b in roster_boards + if (board_family(b['name'], repo_root) in fams or + (gates and board_options(b, repo_root) & gates)) and (board_roles(b) & roles)] + tests = role_tests(roles, extras) + s.roles.update(roles) + why = f'{path}: port {port} -> families {sorted(fams)}' + if gates: + why += f' + option {sorted(gates)}' + s.add(boards, tests, f'{why} -> boards {boards} ({"/".join(sorted(roles))})') + return + + m = re.match(r'src/class/([^/]+)/', path) + if m: + cls = m.group(1) + if re.search(r'_device\.[ch]$', base): + roles = {'device'} + elif re.search(r'_host\.[ch]$', base): + roles = {'host'} + else: + roles = {'device', 'host'} + # this file's own class, plus any class whose headers include it + via = sorted(class_include_edges(repo_root).get(f'{cls}/{base}', ())) + + def macros(prefix): + return (class_macros(cls, base, prefix) + + [m2 for c in via for m2 in class_macros(c, '', prefix)]) + tests = set() + if 'device' in roles: + tests |= class_examples(macros('TUD'), 'device', repo_root, extras) + if 'host' in roles: + tests |= class_examples(macros('TUH'), 'host', repo_root, extras) + boards = [b['name'] for b in roster_boards if board_roles(b) & roles] + s.roles.update(roles) + why = f'{path}: class {cls}' + (f' (+ included by {via})' if via else '') + s.add(boards, tests, f'{why} -> {sorted(tests)} ({"/".join(sorted(roles))})') + return + + m = re.match(r'src/(device|host)/', path) + if m: + role = m.group(1) + boards = [b['name'] for b in roster_boards if role in board_roles(b)] + s.roles.add(role) + s.add(boards, role_tests({role}, extras), f'{path}: core {role} stack -> all {role} tests') + return + + m = re.match(r'hw/bsp/([^/]+)/(?:boards/([^/]+)/)?', path) + if m: + fam, brd = m.group(1), m.group(2) + s.families.add(fam) + if brd: + boards = [b['name'] for b in roster_boards if b['name'] == brd] + why = f'{path}: bsp board {brd}' + else: + boards = [b['name'] for b in roster_boards + if board_family(b['name'], repo_root) == fam] + why = f'{path}: bsp family {fam}' + s.roles.update(('device', 'host')) + s.add(boards, 'all', f'{why} -> boards {boards}') + return + + m = re.match(r'examples/(device|host|dual)/([^/]+)/', path) + if m: + test = f'{m.group(1)}/{m.group(2)}' + known = any(test in pool for pool in ALL_TESTS.values()) or test in extras + if known: + boards = [b['name'] for b in roster_boards] + role = test_role(test) + s.roles.update(('device', 'host') if role == 'dual' else (role,)) + s.add(boards, [test], f'{path}: example -> {test} on all boards') + else: + s.reasons.append(f'{path}: example not in HIL lists, no contribution') + return + + s.force_full(f'{path}: unclassified -> full matrix') + + +def classify(changed_files, repo_root, rosters): + all_boards = [] + seen = set() + for _, boards in rosters: + for b in boards: + if b['name'] not in seen: + seen.add(b['name']) + all_boards.append(b) + + extras = roster_only_tests(all_boards) + s = _Sel() + # no early exit once full: keep classifying so `families` still reports every + # family the diff touches (build-only consumers need it). Nothing after the first + # force_full can change full/boards/args - the full branch below ignores by_board. + for path in changed_files: + _classify_one(path, repo_root, all_boards, extras, s) + + if s.full: + return {'full': True, 'boards': {b['name']: 'all' for b in all_boards}, + 'families': sorted(s.families), 'reasons': s.reasons} + + # role pruning: single-role selections drop the other role's tests and boards + by_name = {b['name']: b for b in all_boards} + out = {} + for name, tests in s.by_board.items(): + allowed = board_tests(by_name[name]) + if tests == 'all': + kept = list(allowed) + else: + kept = [t for t in allowed if t in tests] + if s.roles and s.roles != {'device', 'host'}: + role = next(iter(s.roles)) + kept = [t for t in kept if test_role(t) in (role, 'dual')] + if kept: + out[name] = 'all' if set(kept) == set(allowed) else sorted(kept) + return {'full': False, 'boards': out, 'families': sorted(s.families), + 'reasons': s.reasons} + + +def _board_args(name, chosen) -> list: + parts = [f'-b {name}'] + if chosen != 'all': + parts.append(f'-bt {name}:{",".join(chosen)}') + return parts + + +def selection_args(sel, rosters): + """hil_test.py args per config. Empty means either 'full matrix' or 'nothing + selected' - callers must read sel['full'] to tell them apart.""" + args = {} + for cfg_path, boards in rosters: + parts = [] + if not sel['full']: + for b in boards: + chosen = sel['boards'].get(b['name']) + if chosen is not None: + parts += _board_args(b['name'], chosen) + args[os.path.basename(cfg_path)] = ' '.join(parts) + return args + + +def selection_args_by_flasher(sel, rosters): + """{config: {flasher name: args}}. CI runs one rig as several jobs split by + flasher (esptool vs the rest); each must gate on its own subset, otherwise the + other leg runs a filter matching zero boards and reports a vacuous green.""" + out = {} + for cfg_path, boards in rosters: + per = {} + if not sel['full']: + for b in boards: + chosen = sel['boards'].get(b['name']) + if chosen is None: + continue + per.setdefault(b.get('flasher', {}).get('name', ''), []).extend( + _board_args(b['name'], chosen)) + out[os.path.basename(cfg_path)] = {f: ' '.join(p) for f, p in per.items()} + return out + + +def changed_files_from_git(base, repo_root): + mb = subprocess.run(['git', 'merge-base', 'HEAD', base], cwd=repo_root, + capture_output=True, text=True, check=True).stdout.strip() + diff = subprocess.run(GIT_DIFF_ARGV + [f'{mb}..HEAD'], cwd=repo_root, + capture_output=True, text=True, check=True).stdout + return [l for l in diff.splitlines() if l.strip()] + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + g = ap.add_mutually_exclusive_group(required=True) + g.add_argument('--base', help='git ref to diff against (merge-base..HEAD)') + g.add_argument('--diff-file', help='newline-separated changed-file list') + ap.add_argument('configs', nargs='+', help='rig roster JSON file(s)') + a = ap.parse_args() + + # test/hil/helper/ -> repo root is FOUR levels up; three left this at /test + # after the helper/ move and every repo-relative glob silently matched nothing + repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + rosters = [] + for c in a.configs: + with open(c) as f: + rosters.append((c, json.load(f)['boards'])) + + files = (open(a.diff_file).read().splitlines() if a.diff_file + else changed_files_from_git(a.base, repo_root)) + files = [f for f in files if f.strip()] + + s = classify(files, repo_root, rosters) + s['args'] = selection_args(s, rosters) + s['args_flasher'] = selection_args_by_flasher(s, rosters) + for r in s['reasons']: + print(f'hil_select: {r}', file=sys.stderr) + print(json.dumps(s)) + + +if __name__ == '__main__': + main() diff --git a/test/hil/helper/hil_util.py b/test/hil/helper/hil_util.py new file mode 100644 index 000000000..54984d20f --- /dev/null +++ b/test/hil/helper/hil_util.py @@ -0,0 +1,585 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Bottom layer of the HIL harness: the bounded command runner plus the shared helpers and +# data every other module needs. Stays stdlib-only and imports nothing local -- everything +# else imports this, including the unit tests on GitHub's bare runner; never import them +# from here. Callers set the module global `verbose`. + +from __future__ import annotations + +import glob +import os +import signal +import subprocess +import threading +import sys +from pathlib import Path +from typing import Any + + +# ------------------------------------------------------------- +# HIL example test lists, shared by hil_test.py (runner) and hil_select.py (PR-diff +# selector). Run order is shuffled per board (see test_board); every example carries a +# unique hardcoded idProduct (see its usb_descriptors.c). +# ------------------------------------------------------------- + +# device tests +device_tests = [ + 'device/cdc_dual_ports', + 'device/cdc_msc', + 'device/dfu', + 'device/cdc_msc_throughput', + 'device/audio_test_freertos', + 'device/dfu_runtime', + 'device/cdc_msc_freertos', + 'device/hid_boot_interface', + 'device/msc_dual_lun', + 'device/hid_generic_inout', + 'device/printer_to_cdc', + 'device/midi_test', + 'device/mtp', + 'device/usbtest', # cafe:4010, unique PID; runs the Linux testusb tier-4 battery via usbtest.py + # 'device/net_lwip_webserver', # disabled for PR #3605: USB net iface enum is flaky on the CI HIL host +] + +dual_tests = [ + 'dual/host_info_to_device_cdc', +] + +host_test = [ + 'host/cdc_msc_hid', + 'host/msc_file_explorer', + 'host/msc_file_explorer_freertos', + 'host/device_info', +] + +verbose = False + +def pos_int_env(name: str, default: int) -> int: + # One parsing policy for every HIL_* knob: a bare int() crashes every run at import + # on a malformed value, and 0/negative silently removes the bound the knob enforces. + try: + v = int(os.getenv(name, str(default))) + except ValueError: + print(f'warning: {name} is not an integer; using {default}', + file=sys.stderr, flush=True) + return default + if v <= 0: + print(f'warning: {name}={v} is not usable; using {default}', + file=sys.stderr, flush=True) + return default + return v + + +def pos_float_env(name: str, default: float) -> float: + try: + v = float(os.getenv(name, str(default))) + except ValueError: + print(f'warning: {name} is not a number; using {default}', + file=sys.stderr, flush=True) + return default + # float() accepts 'inf'/'nan': an infinite serial timeout is an unbounded read, the + # very thing these knobs exist to prevent, and nan fails every comparison silently + if not (v > 0 and v < float('inf')): + print(f'warning: {name}={v} is not usable; using {default}', + file=sys.stderr, flush=True) + return default + return v + + +CMD_TIMEOUT = pos_int_env('HIL_CMD_TIMEOUT', 180) + +TINYUSB_ROOT = Path(__file__).resolve().parents[3] # test/hil/helper/ -> repo root + + +def cmd_stdout_text(out: Any) -> str: + if out is None: + return '' + if isinstance(out, bytes): + return out.decode('utf-8', errors='ignore') + return str(out) + + +def _banner_body(out: Any, err: Any) -> str: + # split_stderr callers keep the diagnostic in stderr — a banner of stdout alone + # would be blank exactly when something went wrong + body = cmd_stdout_text(out) + err_text = cmd_stdout_text(err) + if err_text: + body = f'{body}\n{err_text}' if body else err_text + return body + + +# Shared with compact_output's stripper in hil_test: duplicated literals let the two +# layers drift and reintroduce literal marker noise mid-row in the GitHub log. +GROUP_MARK, ENDGROUP_MARK = '::group::', '::endgroup::' + + +def strip_workflow_markers(line: str) -> str: + # run_cmd only ever emits markers at line start; mid-line is not a real case. + return line.removeprefix(GROUP_MARK).removeprefix(ENDGROUP_MARK) + + +def _ci_log_groups() -> bool: + # GitHub folds ::group::/::endgroup:: only at line start of the JOB's real stdout; a + # pool worker's capture is compacted into one row line, where they render literally. + return bool(os.getenv('CI')) and sys.stdout is sys.__stdout__ + + +def _print_banner(title: str, out: Any, err: Any) -> None: + print() + if _ci_log_groups(): + print(f'{GROUP_MARK}{title}') + print(_banner_body(out, err)) + print(ENDGROUP_MARK) + else: + print(title) + print(_banner_body(out, err)) + + +SYSFS_READ_GRACE = 2.0 # bound on one attribute read of a possibly-wedged device +SYSFS_STUCK_MAX = 4 # stranded readers tolerated before read_sysfs goes blind +_sysfs_stuck = 0 # each costs a thread + an fd for the life of the process +_sysfs_stuck_lock = threading.Lock() +_sysfs_blind_logged = False + + +class _SysfsUnknown: + """Sentinel: the read did not answer. NOT "the attribute is absent" -- reading it as + absence turns a healthy board into a firmware regression in the report.""" + __slots__ = () + + def __bool__(self) -> bool: + return False + + def __repr__(self) -> str: + return 'SYSFS_UNKNOWN' + + +SYSFS_UNKNOWN = _SysfsUnknown() + + +def sysfs_blind() -> bool: + """True once this process has stranded SYSFS_STUCK_MAX readers: every later read + answers SYSFS_UNKNOWN, so nothing it reports about a device is a fact any more.""" + return _sysfs_stuck >= SYSFS_STUCK_MAX + + +def sysfs_blind_note() -> str: + """Suffix for a failure message, so a blind worker's verdict never reads as hardware.""" + return (f' (this worker is blind: {SYSFS_STUCK_MAX} sysfs reads stranded on a wedged ' + f'device, so the check could not see the bus)') if sysfs_blind() else '' + + +def read_sysfs(path: str, grace: float = SYSFS_READ_GRACE) -> str | None | _SysfsUnknown: + """Read a sysfs attribute with a WALL-CLOCK bound. + + The value, None when the attribute is genuinely unreadable (OSError), or SYSFS_UNKNOWN + when the read did not answer -- it timed out, or this process is already blind. Callers + MUST keep those apart: absence is a fact, unknown is not. + + usb_string_attr (serial/product/manufacturer) is served under the device lock a wedged + usbfs ioctl holds, so a plain open().read() blocks for as long as the wedge lasts, on + exactly the board an incident is about. The reader sleeps INTERRUPTIBLY (every read + takes usb_lock_device_interruptible, v6.12.96 sysfs.c:124-139 -- uninterruptible is the + ioctl holder, not us), so it dies with a SIGKILLed worker; what it costs meanwhile is a + thread and an fd for this process's life, because on sysfs the open() SUCCEEDS and only + the read blocks. Measured: 20 blocking reads leave 20 live threads. + + Hence the cap: callers rescan (hil_lock's controller_of re-reads every unresolved + device on EVERY permit), and hitting RLIMIT_NOFILE or the thread ceiling raises inside + the worker and loses every board's result -- worse than the hang this prevents. + """ + if sysfs_blind(): + return SYSFS_UNKNOWN + # Known-stranded? Re-reading costs another permanent thread+fd and a blindness credit + # to learn what we already know. Lives HERE, not at the call sites: a call-site memo + # has to be remembered by every new scanner, and twice it was not. + was = _sysfs_stranded.get(path, _STRAND_MISS) + if was is not _STRAND_MISS: + if was is None: + return SYSFS_UNKNOWN # stranded, inode unknown: never re-read it + try: + if os.stat(path).st_ino == was: + return SYSFS_UNKNOWN # same node, still wedged + except OSError: + pass # gone: fall through, the read reports it + _sysfs_stranded.pop(path, None) # replaced or gone -> re-read it + out: dict = {} + + def _read(): + try: + with open(path) as f: + out['v'] = f.read().strip() + except (OSError, ValueError): + pass # no such attribute, or not text: unreadable, and that IS a fact + + t = threading.Thread(target=_read, daemon=True) + t.start() + t.join(grace) + # `out` FIRST, not is_alive() alone: a reader can deposit its value and still be alive + # for a moment afterwards, and counting that as a strand memoises a healthy attribute as + # unreadable and spends one of four blindness credits. bounded_open has always checked + # its box for the same reason. + if t.is_alive() and 'v' not in out: + # Count the PATH once, not once per reader. hil_pool_check runs -j4 by default, + # which equals SYSFS_STUCK_MAX, so four threads hitting ONE wedged device used to + # spend the entire blindness budget between them -- latching blind on the single + # wedge the tool was run to find. The strand is real for each thread, but the + # DEVICE is what the cap is about. + # Under the SAME lock as the counter: check-then-act here is a race, and + # hil_pool_check runs a ThreadPoolExecutor of exactly SYSFS_STUCK_MAX workers in + # ONE process, so four threads on one wedged path could each see `first` before any + # of them recorded it -- spending the whole blindness budget on a single device, + # which is what this memo exists to prevent. note_sysfs_strand takes the lock + # itself, so call it after releasing. + with _sysfs_stuck_lock: + first = path not in _sysfs_stranded + if first: + try: + # stat, never the thread's own open(): stat does not call ->show(), so + # it cannot block on the device lock the reader is stuck behind + _sysfs_stranded[path] = os.stat(path).st_ino + except OSError: + _sysfs_stranded[path] = None # unstattable, but still known-stranded + if first: + note_sysfs_strand() + return SYSFS_UNKNOWN + return out.get('v') + + +def note_sysfs_strand() -> None: + """Record ONE stranded sysfs reader. Shared by read_sysfs and bounded_open so both + account against a single counter -- the report caveat keys off it.""" + global _sysfs_stuck, _sysfs_blind_logged + with _sysfs_stuck_lock: + _sysfs_stuck += 1 + announce = sysfs_blind() and not _sysfs_blind_logged + _sysfs_blind_logged = _sysfs_blind_logged or announce + if announce: + # once per process, on stderr: a worker's stdout is compacted into one report + # row, where this would be lost among the test output + print(f'warning: {SYSFS_STUCK_MAX} sysfs reads stranded on a wedged device; ' + f'this process is now blind and answers SYSFS_UNKNOWN for every ' + f'attribute -- its verdicts about device presence are not evidence', + file=sys.stderr, flush=True) + + +# path -> the inode it had when its read stranded. A stranded attribute stays +# stranded until the DEVICE is replaced, and a re-enumeration destroys the kernfs +# node and makes a new one -- so a changed inode is the all-clear. Keyed by path +# alone it would outlive the wedge: a busport does not change when a board comes +# back on the same port, so the HUNG reflash this branch performs would recover a +# board the harness could then never see again. +_sysfs_stranded: dict = {} +# A stranded path whose inode could not be read is stored as None, so a plain .get() cannot +# tell 'known stranded, inode unknown' from 'never seen' -- and treating the first as the +# second re-reads it, stranding another permanent thread and fd every call. Distinct miss +# sentinel, so None keeps its own meaning. +_STRAND_MISS = object() + + +def usb_scan(vid_pid=None, serial=None, vid=None) -> tuple[list, bool]: + """Enumerated USB devices matching the filters, and whether anything is unknown. + + Returns ([{busport, dir, vid, pid, serial}], unknown). `unknown` True means a bounded + read did not answer, so absence is NOT proven -- the same contract as read_sysfs. + + Three rules, one implementation for every caller: + + * Root hubs excluded (glob `*-*`): no DUT is one, and scans including them measured + seconds slower (observation, no mechanism -- the "autosuspend wake" explanation was + wrong; usb_string_attr reads a cached string, sysfs.c:124-139). + * idVendor/idProduct first: lock-free `sysfs_emit` from udev->descriptor + (sysfs.c:688-705), so they rule out nearly every device for free. + * `serial` last and bounded: it is served under the lock a wedged ioctl holds, and a + path that already stranded is never re-read (each strand costs a thread and an fd + for this process's life). + """ + out = [] + unknown = False + for d in glob.glob('/sys/bus/usb/devices/*-*'): + # Interfaces are ':.' (e.g. 2-4:1.0) -- they CONTAIN the + # colon, they do not end with it, so the original endswith() never fired and every + # scan opened idVendor/idProduct on all of them (measured: 31 of 44 matches). + if ':' in os.path.basename(d): + continue + try: + with open(os.path.join(d, 'idVendor')) as f: + dev_vid = f.read().strip() + with open(os.path.join(d, 'idProduct')) as f: + dev_pid = f.read().strip() + except OSError: + continue # vanished mid-walk, or not a device dir: a fact, not unknown + if vid_pid is not None and (dev_vid, dev_pid) != tuple(vid_pid): + continue # ruled out for free, without touching the locked attribute + if vid is not None and dev_vid != vid: + continue # same, for callers that know the VID but not the PID + sn = read_sysfs(os.path.join(d, 'serial')) + if sn is SYSFS_UNKNOWN: + unknown = True # read_sysfs memoises it; a repeat scan costs nothing + continue + if sn is None: + continue # no serial attribute: a fact + if serial is not None and sn.lower() != serial.lower(): + continue + out.append({'busport': os.path.basename(d), 'dir': d, + 'vid': dev_vid, 'pid': dev_pid, 'serial': sn}) + return out, unknown + + +def bounded_open(path: str, flags: int, timeout: float = SYSFS_READ_GRACE): + """os.open() with a wall-clock bound. + + The fd, None when the open genuinely FAILED (OSError: EBUSY, ENOENT, EACCES), or + SYSFS_UNKNOWN when it did not answer -- the same three-valued contract as read_sysfs, + and for the same reason: folding a fact into an unknown made an ordinary EBUSY read as + a wedged device and sent the operator hunting hardware that is healthy. + + An open CAN block on a wedged device -- not on O_NONBLOCK, which usblp_open never + consults, but on usb_autopm_get_interface(), a runtime-PM resume that does I/O + (v6.12.96 drivers/usb/class/usblp.c). It holds usblp_mutex while it waits, and that + mutex is driver-GLOBAL, so one wedged printer blocks opens of every usblp node. + + Unlike read_sysfs the stranded thread cleans up after itself: if we have given up it + closes the fd it eventually got, so only the thread leaks. Both sides take `handoff` + -- "store or close" and "abandon and drain" are a check-then-act pair that can + interleave into an fd stored after the box was drained, which would leak it into a + node that allows a SINGLE opener (usblp_open returns -EBUSY when usblp->used). + """ + # Same short-circuit as read_sysfs: once blind, another stranded thread buys nothing + # and the cap exists precisely to stop them accumulating. + if sysfs_blind(): + return SYSFS_UNKNOWN + # Known-stranded? Re-opening costs another thread, another fd and another blindness + # credit to learn what we already know -- and the printer test re-opens ONE lp node on + # every retry. Same memo and same inode check as read_sysfs. + was = _sysfs_stranded.get(path, _STRAND_MISS) + if was is not _STRAND_MISS: + if was is None: + return SYSFS_UNKNOWN # stranded, inode unknown: never re-read it + try: + if os.stat(path).st_ino == was: + return SYSFS_UNKNOWN + except OSError: + pass + _sysfs_stranded.pop(path, None) + box: dict = {} + done, abandoned = threading.Event(), threading.Event() + handoff = threading.Lock() + + def _open(): + try: + fd = os.open(path, flags) + except OSError: + done.set() + return + with handoff: + stored = not abandoned.is_set() + if stored: + box['fd'] = fd + if not stored: + try: + os.close(fd) + except OSError: + pass + done.set() + + threading.Thread(target=_open, daemon=True).start() + if not done.wait(timeout): + with handoff: + abandoned.set() + fd = box.pop('fd', None) # completed in the gap between timeout and flag + if fd is not None: + # It DID open, just after our deadline -- the thread finished, so nothing is + # stranded. Report unknown (we already gave up on it) but do not spend a + # blindness credit, and do not call a merely-slow node wedged. + try: + os.close(fd) + except OSError: + pass + return SYSFS_UNKNOWN + # counted like a stranded read_sysfs: the thread and (eventually) its fd are gone + # for the life of the process, and the cap exists to stop that reaching the + # thread/fd ceiling -- an exception there escapes the worker and loses every board. + # Memoised by inode so a retry of the same node does not pay again. + # same lock as read_sysfs, same reason + with _sysfs_stuck_lock: + first = path not in _sysfs_stranded + if first: + try: + _sysfs_stranded[path] = os.stat(path).st_ino + except OSError: + _sysfs_stranded[path] = None + if first: + note_sysfs_strand() + return SYSFS_UNKNOWN + return box.get('fd') + + +def _close_pipes(p: subprocess.Popen) -> None: + """Close OUR ends of an abandoned child's pipes. Never raises.""" + for pipe in (p.stdout, p.stderr, p.stdin): + try: + if pipe is not None: + pipe.close() + except OSError: + pass + + +def run_alongside(argv: list, work, timeout: int) -> subprocess.CompletedProcess: + """Run `argv` alongside `work()`, which runs in THIS thread, then reap it -- bounded. + + The read-while-we-write shape run_cmd cannot express: the caller needs the child + RUNNING while it does something else. Everything else about the contract is run_cmd's + -- own session, killpg, bounded reap, our pipe ends closed, rc 124 on the kill. + + A PROCESS, not a thread: an abandoned thread keeps the fd, and usblp_open returns + -EBUSY while usblp->used (v6.12.96 usblp.c), so every later open in this long-lived + worker would read as a wedged device. A killed process takes its fd with it. + + stdout is captured as BYTES and kept CLEAN -- a caller byte-compares it against the + payload it sent, so a single stderr byte (a PYTHONWARNINGS chirp, a sitecustomize + print, a .pth deprecation from a venv) would read as USB data corruption. stderr gets + its own pipe; communicate() drains both, so the split cannot deadlock. + `work` runs even if the child dies immediately -- the caller's own asserts decide. + """ + p = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + start_new_session=True) + + def _reap() -> subprocess.CompletedProcess: + try: + out, err = p.communicate(timeout=timeout) + return subprocess.CompletedProcess(argv, p.returncode, out, err) + except subprocess.TimeoutExpired: + try: + os.killpg(p.pid, signal.SIGKILL) + except OSError: + p.kill() + try: + out, err = p.communicate(timeout=5) + except subprocess.TimeoutExpired: + # Outlasted SIGKILL: uninterruptible, still holding whatever it opened. + # Abandoned like any other stray -- but as a real child in its own + # session, so the containment sweep FINDS it (child_procs walks the ppid + # tree) and the report names it. That is the whole difference from a + # blocked thread, which no sweep can see and no signal can reach. + out, err = b'', b'' + _close_pipes(p) # our own fds must not leak either + return subprocess.CompletedProcess(argv, 124, out, err) + + try: + work() + except BaseException: + # Reap first so the child never outlives us, then let the caller's error through. + # A `return` inside a `finally` would SWALLOW it -- an assert in `work` would + # vanish and the caller would compare data it never finished sending. + _reap() + raise + return _reap() + + +def run_cmd(cmd: str, cwd: str | None = None, timeout: int | None = None, + binary: bool = False, split_stderr: bool = False, + quiet: bool = False) -> subprocess.CompletedProcess: + if timeout is None: + timeout = CMD_TIMEOUT + # binary: raw bytes (text mode's errors='replace' mangles non-UTF-8 file content). + # split_stderr: keep stderr out of stdout, for callers that parse stdout. quiet: no + # COMMAND FAILED banner, for retry loops that report failures themselves (timeouts + # still print: a killed child is always noteworthy). + popen_kwargs = { + 'cwd': cwd, + 'shell': True, + 'stdout': subprocess.PIPE, + 'stderr': subprocess.PIPE if split_stderr else subprocess.STDOUT, + } + if not binary: + popen_kwargs.update({'text': True, 'encoding': 'utf-8', 'errors': 'replace'}) + if os.name != 'nt': + # C-level setsid, same process-group semantics as preexec_fn=os.setsid but + # safe when called from threads (pool_check runs flashes from a thread pool) + popen_kwargs['start_new_session'] = True + + p = subprocess.Popen(cmd, **popen_kwargs) + try: + out, err = p.communicate(timeout=timeout) + r = subprocess.CompletedProcess(args=cmd, returncode=p.returncode, stdout=out, stderr=err) + except subprocess.TimeoutExpired as ex: + if os.name != 'nt': + try: + os.killpg(p.pid, signal.SIGKILL) + except OSError: + # ProcessLookupError: already gone. PermissionError: an all-root group + # refuses the group kill -- letting either escape would skip the bounded + # reap, the pipe close and the rc-124 return this handler exists for. + pass + else: + p.kill() + try: + out, err = p.communicate(timeout=10) + except subprocess.TimeoutExpired: + # Something in the group outlived SIGKILL: D state (truly unkillable), or + # root-owned because sudo FORKS rather than execs, so the wrapper dies and its + # root child does not. Abandon it and let the report name it; the harness never + # sudo-kills its way out. Our ends of its pipes must not leak, though: a pool + # worker lives for the whole run, so every wedged command would cost it two fds. + out, err = None, None + _close_pipes(p) + # prefer the post-kill buffers (supersets of the exception's), falling back to ex.* + # when the child was unkillable. TimeoutExpired carries BYTES even for a text-mode + # Popen, so the fallbacks must be decoded or a text-mode caller gets bytes exactly + # when the child wedged in D state. + def _typed(v): + if not binary and isinstance(v, bytes): + return v.decode('utf-8', errors='replace') + return v + + timeout_out = _typed(out or ex.stdout) or (b'' if binary else '') + # ...and never None: with split_stderr the SUCCESS path always yields a str/bytes, + # so a caller that does `r.stderr.strip()` works everywhere except the timeout -- + # the one path it was written for. Without split_stderr stderr stays None, as on + # the success path (it was merged into stdout). + timeout_err = _typed(err if err is not None else ex.stderr) + if split_stderr and timeout_err is None: + timeout_err = b'' if binary else '' + _print_banner(f'COMMAND TIMEOUT ({timeout}s): {cmd}', timeout_out, timeout_err) + return subprocess.CompletedProcess(args=cmd, returncode=124, stdout=timeout_out, stderr=timeout_err) + except BaseException: + # BaseException, not Exception (as in CPython's own subprocess.run): + # KeyboardInterrupt is the case that matters, and start_new_session put the child in + # its OWN group, so it never got the terminal's SIGINT -- without this, Ctrl-C + # leaves the flasher or testusb holding the probe and its usbfs node. Kill and + # close, never wait: this path must not add a hang of its own. + if os.name != 'nt': + try: + os.killpg(p.pid, signal.SIGKILL) + except OSError: + pass + else: + p.kill() + _close_pipes(p) + raise + + if r.returncode != 0 and not quiet: + _print_banner(f'COMMAND FAILED: {cmd}', r.stdout, r.stderr) + elif verbose: + print(cmd) + print(cmd_stdout_text(r.stdout)) + return r + + +# get usb serial by id +def get_serial_dev(id, vendor_str, product_str, ifnum): + if vendor_str and product_str: + # known vendor and product + vendor_str = vendor_str.replace(' ', '_') + product_str = product_str.replace(' ', '_') + return f'/dev/serial/by-id/usb-{vendor_str}_{product_str}_{id}-if{ifnum:02d}' + else: + # just use id: mostly for cp210x/ftdi flasher + pattern = f'/dev/serial/by-id/usb-*_{id}-if*' + port_list = glob.glob(pattern) + if len(port_list) == 0: + raise RuntimeError(f'No serial device found for {pattern}') + return port_list[0] diff --git a/test/hil/hil_ci.sh b/test/hil/hil_ci.sh index ef93bcb49..c7dfa95df 100644 --- a/test/hil/hil_ci.sh +++ b/test/hil/hil_ci.sh @@ -20,6 +20,30 @@ CONFIG=${CONFIG:-$ROOT_DIR/test/hil/tinyusb.json} exit 1 } +# REMOTE_DIR reaches the rig as `rm -rf` input, an scp remote path and an rsync remote +# path -- the remote shell re-splits and expands all three, so no amount of LOCAL quoting +# protects them (and %q would escape the ~ that REMOTE_DIR=~/dir needs). Screen it once. +# The tilde is the whole hazard: the REMOTE shell expands it, so `~/` alone -- one typo +# away from the documented ~/dir override -- means `rm -rf` on that account's HOME. Hence +# `/` or `~/` followed by at least one named component, ending in a name character. +[[ $REMOTE_DIR =~ ^(/|~/)[A-Za-z0-9_.~/-]*[A-Za-z0-9_-]$ && $REMOTE_DIR != *..* + && $REMOTE_DIR != *//* ]] || { + echo "error: REMOTE_DIR must be /path or ~/path of [A-Za-z0-9_.~/-], no '..', no" \ + "trailing slash -- it is an rm -rf target on $REMOTE: $REMOTE_DIR" >&2 + exit 1 +} + +# --build would run tools/build.py ON THE RIG, and this script stages binaries, not the +# build tree -- it is not copied, so the run dies there with a confusing missing-file +# error. Building is the local half of this workflow by design. +for a in "$@"; do + [ "$a" = "--build" ] || continue + echo "error: --build builds on the REMOTE, but this script copies prebuilt binaries" >&2 + echo " (tools/build.py is not staged). Build locally first, then re-run:" >&2 + echo " cd examples && cmake --preset && cmake --build --preset " >&2 + exit 1 +done + # Parse -b BOARD from arguments to know which build to copy BOARD="" ARGS=() @@ -38,29 +62,35 @@ while [[ $# -gt 0 ]]; do esac done -# Setup remote directory. Use `bash -s` + heredoc so REMOTE_DIR (user-overridable) -# is passed as a positional parameter and never reinterpreted by the remote shell. +# Setup remote directory. `bash -s` + heredoc so REMOTE_DIR arrives as a positional +# parameter, keeping the `rm -rf` target out of the command string the heredoc runs. echo "==> Setting up remote $REMOTE:$REMOTE_DIR" ssh "$REMOTE" bash -s -- "$REMOTE_DIR" <<'REMOTE' set -e +# Second gate, on the side that knows what ~ expanded to: only here is $HOME a value +# rather than a guess, and this is the line that actually runs rm -rf. +case "$1" in + ''|/|"$HOME"|"$HOME"/) echo "refusing to rm -rf '$1'" >&2; exit 1 ;; +esac rm -rf -- "$1" -# .claude path: usbtest.py's HUNG recovery resolves usb_recover.sh relative to the -# staged repo root — without it, recovery ENOENTs and the wedge is left in place -mkdir -p -- "$1/test/hil" "$1/examples" "$1/.claude/skills/usb-kernel-recover/scripts" +mkdir -p -- "$1/test/hil/helper" "$1/examples" REMOTE # Copy HIL test script and config echo "==> Copying test scripts" scp -q "$ROOT_DIR/test/hil/hil_test.py" \ "$ROOT_DIR/test/hil/hil_flash.py" \ - "$ROOT_DIR/test/hil/hil_lock.py" \ "$ROOT_DIR/test/hil/usbtest.py" \ - "$ROOT_DIR/test/hil/hil_examples.py" \ "$ROOT_DIR/test/hil/pymtp.py" \ + "$ROOT_DIR/test/hil/mtp_test.py" \ "$CONFIG" \ "$REMOTE:$REMOTE_DIR/test/hil/" -scp -q "$ROOT_DIR/.claude/skills/usb-kernel-recover/scripts/usb_recover.sh" \ - "$REMOTE:$REMOTE_DIR/.claude/skills/usb-kernel-recover/scripts/" +scp -q "$ROOT_DIR/test/hil/helper/__init__.py" \ + "$ROOT_DIR/test/hil/helper/hil_util.py" \ + "$ROOT_DIR/test/hil/helper/hil_health.py" \ + "$ROOT_DIR/test/hil/helper/hil_lock.py" \ + "$ROOT_DIR/test/hil/helper/hil_select.py" \ + "$REMOTE:$REMOTE_DIR/test/hil/helper/" # Copy only firmware binaries (elf/bin/hex) plus esptool metadata # (config.env + flash_args needed by the esptool flasher), preserving structure @@ -90,16 +120,25 @@ if [ -n "$BOARD" ]; then add_build_dir "$d" done shopt -u nullglob - while IFS= read -r v; do - add_build_dir "$ROOT_DIR/examples/cmake-build-$v" - done < <(python3 -c ' + # to a file, not a process substitution: `set -e`/pipefail cannot see the exit + # status of the latter, so a malformed roster silently yielded zero variant dirs + VARIANTS_FILE=$(mktemp) + python3 -c ' import json, sys cfg = json.load(open(sys.argv[1])) for b in cfg.get("boards", []): if b["name"] == sys.argv[2]: for v in b.get("variant") or []: print(v["name"]) -' "$CONFIG" "$BOARD") +' "$CONFIG" "$BOARD" > "$VARIANTS_FILE" || { + echo "Error: could not read variants for $BOARD from $CONFIG" + rm -f "$VARIANTS_FILE" + exit 1 + } + while IFS= read -r v; do + add_build_dir "$ROOT_DIR/examples/cmake-build-$v" + done < "$VARIANTS_FILE" + rm -f "$VARIANTS_FILE" if [ ${#BUILD_DIRS[@]} -eq 0 ]; then echo "Error: no build directory found for $BOARD under $ROOT_DIR/examples/" echo "Build first with: cd examples && cmake --preset $BOARD && cmake --build --preset $BOARD" @@ -119,12 +158,24 @@ else done fi -# Run test. Use `bash -s` so REMOTE_DIR + ARGS reach the remote shell as positional -# parameters; quoting and metacharacters in args are preserved. -CONFIG_BASENAME="$(basename "$CONFIG")" +# Run test via `bash -s`, so REMOTE_DIR and the args arrive as positional parameters. +# %q the ARGS -- ssh joins its argv into ONE string that the remote shell re-splits, so +# `-t 'host/cdc msc'` would arrive as two arguments and hil_test.py would see a stray +# word where it expects the config path. REMOTE_DIR is deliberately NOT quoted here: it +# is screened above precisely so it can keep its ~ expansion. +ARGS_Q=() +for a in ${ARGS[@]+"${ARGS[@]}"}; do ARGS_Q+=("$(printf '%q' "$a")"); done +# same re-split, same fix: CONFIG is a user-supplied path and its basename lands in the +# command string too +CONFIG_Q="$(printf '%q' "test/hil/$(basename "$CONFIG")")" echo "==> Running HIL test on $REMOTE" rc=0 -ssh "$REMOTE" bash -s -- "$REMOTE_DIR" "${ARGS[@]}" "test/hil/$CONFIG_BASENAME" <<'REMOTE' || rc=$? +# --retry 1 FIRST, before the user's args: this targets the same shared rig CI uses, and +# the pool guard is a flat constant that does not scale with max_retry -- argparse's +# default of 3 lets a few flaky boards re-pay 510s each until the 3600s guard fires, +# abandoning the pool and holding board flocks against concurrent CI. Placed first, not +# appended, so argparse's last-wins means `hil_ci.sh -r 3` still gets 3. +ssh "$REMOTE" bash -s -- "$REMOTE_DIR" --retry 1 ${ARGS_Q[@]+"${ARGS_Q[@]}"} "$CONFIG_Q" <<'REMOTE' || rc=$? cd -- "$1" shift # Flasher CLIs live in the user bin dirs on ci.lan (esptool/idf in ~/.local/bin, diff --git a/test/hil/hil_ci_set_matrix.py b/test/hil/hil_ci_set_matrix.py deleted file mode 100644 index bca989bd1..000000000 --- a/test/hil/hil_ci_set_matrix.py +++ /dev/null @@ -1,90 +0,0 @@ -import argparse -import json -import os - - -def _resolve_config_path(config_file): - if os.path.exists(config_file): - return config_file - - script_relative = os.path.join(os.path.dirname(__file__), config_file) - if os.path.exists(script_relative): - return script_relative - - raise FileNotFoundError(f'Config file not found: {config_file}') - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('config_files', nargs='+', help='Configuration JSON file(s)') - parser.add_argument('--select', help='hil_select.py JSON; scopes boards when full=false') - args = parser.parse_args() - - selected = None - sel = json.loads(args.select) if args.select else None - if sel and not sel.get('full'): - selected = set(sel.get('boards', {})) - - # Toolchain buckets must match the toolchains instantiated by the hil-build - # job in .github/workflows/build.yml. Keep all keys present (even if empty) - # so `fromJSON(hil_json)[toolchain]` always resolves to a list. - matrix = { - 'arm-gcc': [], - 'riscv-gcc': [], - 'esp-idf': [] - } - - seen = {toolchain: set() for toolchain in matrix} - - def append_build_arg(toolchain, build_arg): - if build_arg not in seen[toolchain]: - seen[toolchain].add(build_arg) - matrix[toolchain].append(build_arg) - - for config_file in args.config_files: - with open(_resolve_config_path(config_file)) as f: - config = json.load(f) - - for board in config['boards']: - if selected is not None and board['name'] not in selected: - continue - name = board['name'] - flasher = board['flasher'] - # esptool boards must build under esp-idf; others default to arm-gcc - # but may opt into another bucket via an explicit "toolchain" field - # (e.g. RISC-V boards like ch32v20x need "riscv-gcc"). - if flasher['name'] == 'esptool': - toolchain = 'esp-idf' - else: - toolchain = board.get('toolchain', 'arm-gcc') - if toolchain not in matrix: - # a board in no bucket would never be built, and the bare KeyError - # below would only say so as a traceback from the set-matrix job - raise SystemExit( - f'{name}: toolchain {toolchain!r} is not a build bucket ' - f'({", ".join(matrix)}); add it here and to the hil-build / ' - f'hil-build-esp jobs in .github/workflows/build.yml') - - build_board = f'-b {name}' - if 'build' in board and 'args' in board['build']: - build_board += ' ' + ' '.join(f'-D{a}' for a in board['build']['args']) - - # Each variant builds into cmake-build- with its own cmake - # -D defines and raw CFLAGS. No 'variant' -> a single build named after - # the board. - variants = board.get('variant') or [{'name': name, 'flags': ''}] - for v in variants: - arg = build_board - if v['name'] != name: - arg += f' --build-name {v["name"]}' - for d in v.get('defines', []): - arg += f' -D{d}' - for tok in v.get('flags', '').split(): - arg += f' --cflag={tok}' - append_build_arg(toolchain, arg) - - print(json.dumps(matrix)) - - -if __name__ == '__main__': - main() diff --git a/test/hil/hil_examples.py b/test/hil/hil_examples.py deleted file mode 100644 index 4c8b6918b..000000000 --- a/test/hil/hil_examples.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MIT -# HIL example test lists, shared by hil_test.py (runner) and hil_select.py -# (PR-diff selector). Stdlib-only: hil_select runs on bare CI runners. - -# The per-board run order is shuffled (see test_board). -# Every example carries a unique hardcoded idProduct (see its usb_descriptors.c) - -# device tests -device_tests = [ - 'device/cdc_dual_ports', - 'device/cdc_msc', - 'device/dfu', - 'device/cdc_msc_throughput', - 'device/audio_test_freertos', - 'device/dfu_runtime', - 'device/cdc_msc_freertos', - 'device/hid_boot_interface', - 'device/msc_dual_lun', - 'device/hid_generic_inout', - 'device/printer_to_cdc', - 'device/midi_test', - 'device/mtp', - 'device/usbtest', # cafe:4010, unique PID; runs the Linux testusb tier-4 battery via usbtest.py - # 'device/net_lwip_webserver', # disabled for PR #3605: USB net iface enum is flaky on the CI HIL host -] - -dual_tests = [ - 'dual/host_info_to_device_cdc', -] - -host_test = [ - 'host/cdc_msc_hid', - 'host/msc_file_explorer', - 'host/msc_file_explorer_freertos', - 'host/device_info', -] diff --git a/test/hil/hil_flash.py b/test/hil/hil_flash.py index da81fcc97..f4bed45a6 100755 --- a/test/hil/hil_flash.py +++ b/test/hil/hil_flash.py @@ -1,138 +1,48 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: MIT -# Firmware flashing for the TinyUSB HIL rig: run_cmd, one flash_*/reset_* pair per -# flasher type (dispatched by config name via getattr), find_firmware, and the -# fixture serial-port resolver get_serial_dev (here, not hil_test: flash_esptool -# needs it and helpers must not import hil_test). -# Callers set module globals `build_dir` and `verbose` (hil_test.main from argparse, -# pool_check directly) exactly as they set hil_test's globals today. -# -# from __future__ import annotations (below): some moved function signatures use -# type hints (Any, Board) not defined in this module; postponed evaluation (PEP -# 563) keeps those as unevaluated strings so the verbatim-moved defs still load. +# Firmware flashing for the TinyUSB HIL rig: one flash_*/reset_* pair per flasher type +# (dispatched by config name via getattr) plus find_firmware. The bounded runner run_cmd +# lives in hil_util (never import hil_test here). Callers set the module global +# `build_dir`. `from __future__ import annotations` keeps the Board hints below +# unevaluated: the type is not defined in this module. from __future__ import annotations -import glob import json -import os -import signal +import re import subprocess from pathlib import Path -verbose = False -build_dir = 'cmake-build' +import os +import sys -CMD_TIMEOUT = int(os.getenv('HIL_CMD_TIMEOUT', '180')) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # PYTHONSAFEPATH drops it +from helper import hil_util + +build_dir = 'cmake-build' # flasher names (dispatch key, board['flasher']['name'].lower()) whose reset_* is a no-op RESET_NOOP = {'esptool', 'lm4flash'} # extra parents find_firmware ALSO searches after build_dir. Empty by default so -# hil_test's -B stays authoritative (a board missing there must report "Skip (no -# binary)", never silently flash a stale binary from another tree); pool_check -# opts in to cover both standard layouts. +# hil_test's -B stays authoritative: a board missing there must report "Skip (no +# binary)", never silently flash a stale binary from another tree. EXTRA_BUILD_DIRS: list = [] - -def cmd_stdout_text(out: Any) -> str: - if out is None: - return '' - if isinstance(out, bytes): - return out.decode('utf-8', errors='ignore') - return str(out) - - -# ------------------------------------------------------------- -# Path -# ------------------------------------------------------------- -TINYUSB_ROOT = Path(__file__).resolve().parents[2] - -# get usb serial by id -def get_serial_dev(id, vendor_str, product_str, ifnum): - if vendor_str and product_str: - # known vendor and product - vendor_str = vendor_str.replace(' ', '_') - product_str = product_str.replace(' ', '_') - return f'/dev/serial/by-id/usb-{vendor_str}_{product_str}_{id}-if{ifnum:02d}' - else: - # just use id: mostly for cp210x/ftdi flasher - pattern = f'/dev/serial/by-id/usb-*_{id}-if*' - port_list = glob.glob(pattern) - if len(port_list) == 0: - raise RuntimeError(f'No serial device found for {pattern}') - return port_list[0] +_VID_PID_WARNED: set = set() # one warning per probe, not per command # ------------------------------------------------------------- # Flashing firmware # ------------------------------------------------------------- -def run_cmd(cmd: str, cwd: str | None = None, timeout: int = CMD_TIMEOUT) -> subprocess.CompletedProcess: - popen_kwargs = { - 'cwd': cwd, - 'shell': True, - 'stdout': subprocess.PIPE, - 'stderr': subprocess.STDOUT, - 'text': True, - 'encoding': 'utf-8', - 'errors': 'replace', - } - if os.name != 'nt': - # C-level setsid, same process-group semantics as preexec_fn=os.setsid but - # safe when called from threads (pool_check runs flashes from a thread pool) - popen_kwargs['start_new_session'] = True - - p = subprocess.Popen(cmd, **popen_kwargs) - try: - out, _ = p.communicate(timeout=timeout) - r = subprocess.CompletedProcess(args=cmd, returncode=p.returncode, stdout=out) - except subprocess.TimeoutExpired as ex: - if os.name != 'nt': - try: - os.killpg(p.pid, signal.SIGKILL) - except ProcessLookupError: - pass - else: - p.kill() - try: - out, _ = p.communicate(timeout=10) - except subprocess.TimeoutExpired: # unkillable (e.g. D-state on wedged USB) - out = None - timeout_out = ex.stdout or out or b'' - title = f'COMMAND TIMEOUT ({timeout}s): {cmd}' - print() - if os.getenv('CI'): - print(f"::group::{title}") - print(cmd_stdout_text(timeout_out)) - print(f"::endgroup::") - else: - print(title) - print(cmd_stdout_text(timeout_out)) - return subprocess.CompletedProcess(args=cmd, returncode=124, stdout=timeout_out) - - if r.returncode != 0: - title = f'COMMAND FAILED: {cmd}' - print() - if os.getenv('CI'): - print(f"::group::{title}") - print(cmd_stdout_text(r.stdout)) - print(f"::endgroup::") - else: - print(title) - print(cmd_stdout_text(r.stdout)) - elif verbose: - print(cmd) - print(cmd_stdout_text(r.stdout)) - return r - - -def flash_jlink(board: Board, firmware: str) -> subprocess.CompletedProcess: +def flash_jlink(board: Board, firmware: str, timeout=None) -> subprocess.CompletedProcess: flasher = board['flasher'] script = ['halt', 'r', f'loadfile {firmware}', 'r', 'go', 'exit'] f_jlink = Path(f'{board["name"]}_{Path(firmware).name}.jlink') with f_jlink.open('w') as f: f.writelines(f'{s}\n' for s in script) - ret = run_cmd(f'JLinkExe -USB {flasher["uid"]} {flasher["args"]} -if swd -JTAGConf -1,-1 -speed auto -NoGui 1 -ExitOnError 1 -CommandFile {f_jlink}') + ret = hil_util.run_cmd(f'JLinkExe -USB {flasher["uid"]} {flasher["args"]} -if swd -JTAGConf -1,-1 -speed auto -NoGui 1 -ExitOnError 1 -CommandFile {f_jlink}', + timeout=timeout) f_jlink.unlink(missing_ok=True) return ret @@ -144,89 +54,208 @@ def reset_jlink(board: Board) -> subprocess.CompletedProcess: if not f_jlink.exists(): with f_jlink.open('w') as f: f.writelines(f'{s}\n' for s in script) - ret = run_cmd(f'JLinkExe -USB {flasher["uid"]} {flasher["args"]} -if swd -JTAGConf -1,-1 -speed auto -NoGui 1 -ExitOnError 1 -CommandFile {f_jlink}') + ret = hil_util.run_cmd(f'JLinkExe -USB {flasher["uid"]} {flasher["args"]} -if swd -JTAGConf -1,-1 -speed auto -NoGui 1 -ExitOnError 1 -CommandFile {f_jlink}') return ret -def flash_stlink(board, firmware): +def flash_stlink(board, firmware, timeout=None): + # --verify catches the partial/corrupt write that exits 0 and sends the test phase + # off to exercise bad firmware. Opt-IN here ("verify": true), unlike flash_openocd's + # opt-out: a default-on read-back silently changes every roster entry that lacks the + # key, including boards on rigs this was never validated against. flasher = board['flasher'] - return run_cmd(f'STM32_Programmer_CLI --connect port=swd sn={flasher["uid"]} --write {firmware} --go') + verify = ' --verify' if flasher.get('verify', False) else '' + return hil_util.run_cmd(f'STM32_Programmer_CLI --connect port=swd sn={flasher["uid"]} --write {firmware}{verify} --go', + timeout=timeout) def reset_stlink(board): flasher = board['flasher'] - return run_cmd(f'STM32_Programmer_CLI --connect port=swd sn={flasher["uid"]} --rst --go') + return hil_util.run_cmd(f'STM32_Programmer_CLI --connect port=swd sn={flasher["uid"]} --rst --go') def _openocd_cmd_base(flasher): + # Optional roster field vid_pid, openocd-verbatim (e.g. "0x1a86 0x8010"), pins probe + # discovery to the probe's IDs so openocd never opens foreign usbfs nodes to read + # strings -- a wedged node makes that open hang unkillably (the 2026-08-10 convoy). + # BEFORE args, because the rescue cfgs run `init` internally and reject (or never see) + # a config command that follows it. + vid_pid = '' + if 'vid_pid' in flasher: + # Validated HERE too, not just in convoy_safe: openocd only warns ("incomplete + # vid_pid configuration directive") and exits 0 on a malformed value, so the pin + # silently does not apply and discovery goes back to opening every usbfs node -- + # the convoy this field exists to stop. The same key name carries a DIFFERENT + # syntax under tests.dev_attached ('1a86_55d4'), so the typo is one copy away. + if valid_vid_pid(flasher['vid_pid']): + vid_pid = f'-c "adapter usb vid_pid {flasher["vid_pid"]}" ' + else: + # stderr + once-per-probe, like the missing-pin branch below: stdout here is + # captured by test_example's redirect_stdout (shown only when the test FAILS) + # and by hil_pool_check's StringIO spool, so on a PASSING run the operator + # would never learn the pin was silently dropped. + uid = flasher.get('uid', '?') + if uid not in _VID_PID_WARNED: + _VID_PID_WARNED.add(uid) + print(f'warning: {uid} has a malformed vid_pid {flasher["vid_pid"]!r} ' + f'(want "0xVVVV 0xPPPP"); probe pin DROPPED, so discovery will open ' + f'foreign usbfs nodes', file=sys.stderr, flush=True) + elif flasher.get('uid') not in _VID_PID_WARNED: + # stderr, once per probe: test_example captures stdout, so a passing run would + # swallow this and the operator would never learn discovery still opens every + # usbfs node + _VID_PID_WARNED.add(flasher.get('uid')) + print(f'warning: openocd flasher {flasher.get("uid", "?")} has no vid_pid pin; ' + f'probe discovery will open every usbfs node (hangs on a wedged one)', + file=sys.stderr, flush=True) return (f'openocd -c "tcl_port disabled" -c "gdb_port disabled" -c "telnet_port disabled" ' - f'-c "adapter serial {flasher["uid"]}" {flasher["args"]}') + f'-c "adapter serial {flasher["uid"]}" {vid_pid}{flasher["args"]}') -# `verify` is on by default and opted out per board with "verify": false in the roster. -# WCH targets must opt out: flash read-back over the WCH-Link sdi transport returns a -# repeated word instead of memory contents, so verification always reports a mismatch and -# fails the flash (measured on ch32v103r and ch32v307v, 2026-07-30). Do NOT drop verify -# fleet-wide to accommodate them — every other openocd board can read back, and without it -# a partial or corrupt write exits 0 and the test phase runs bad firmware. -def flash_openocd(board, firmware): +# `verify` is on by default, opted out per board with "verify": false. WCH targets must +# opt out: read-back over the WCH-Link sdi transport returns a repeated word instead of +# memory contents, so verification always mismatches (measured on ch32v103r and ch32v307v, +# 2026-07-30). Do NOT drop verify fleet-wide for them — every other openocd board reads +# back, and without it a partial or corrupt write exits 0 and the tests run bad firmware. +def flash_openocd(board, firmware, timeout=None): flasher = board['flasher'] verify = ' verify' if flasher.get('verify', True) else '' - ret = run_cmd(f'{_openocd_cmd_base(flasher)} -c "program {firmware}{verify} reset exit"') + ret = hil_util.run_cmd(f'{_openocd_cmd_base(flasher)} -c "program {firmware}{verify} reset exit"', + timeout=timeout) return ret -def reset_openocd(board): +def reset_openocd(board, timeout=None): + # timeout: usbtest's post-hang recovery bounds this (RECOVER_RESET_TIMEOUT); an + # unbounded reset there would outlive the caller's outer kill and orphan openocd on + # the probe, which is the stray the recovery exists to avoid. flasher = board['flasher'] - ret = run_cmd(f'{_openocd_cmd_base(flasher)} -c "init; reset run; exit"') + ret = hil_util.run_cmd(f'{_openocd_cmd_base(flasher)} -c "init; reset run; exit"', + timeout=timeout) return ret # OpenOCD's messages for "the target's debug port did not answer". The probe is fine when -# these appear (the log still shows "CMSIS-DAP: Interface ready"); the chip's debug clock -# is gone, which no reset the probe can drive would fix -- the CMSIS-DAP Debug Probe has no -# nRESET line at all. Which message you get depends on the DAP topology, NOT on the board: -# rp2040.cfg creates three multidrop DAPs (cores 0/1 and the Rescue DP at instance 0xf) so -# it fails in swd_multidrop_select, while rp2350.cfg creates a single plain ADIv6 DAP that -# fails earlier in swd_connect. A dead RP2040 can also produce the second one if the very -# first DP read never gets through, so both are accepted for both chips -- it is the target -# cfg in the roster args, below, that picks how to rescue. +# these appear ("CMSIS-DAP: Interface ready" is still logged); the chip's debug clock is +# gone, which no probe-driven reset fixes -- the CMSIS-DAP probe has no nRESET line. Which +# message appears depends on DAP topology, not the board, so both are accepted for both +# chips; RESCUE_CFG below picks the rescue. DAP_WEDGED = ('Failed to connect multidrop', 'Error connecting DP: cannot read IDR') -# How each RP target reaches its Rescue DP, keyed by the target cfg named in flasher args. -# (cfg substitution, extra args): rp2040.cfg drives the Rescue DP itself behind a RESCUE -# flag and calls init/shutdown on its own; rp2350 has a separate cfg that pokes the rescue -# bit via an AP register but never shuts down, so it would sit in the server loop until -# CMD_TIMEOUT without an explicit one. +# How each RP target reaches its Rescue DP, keyed by the target cfg named in flasher args: +# (cfg substitution, pre args, post args). rp2040.cfg drives the Rescue DP behind a RESCUE +# flag and init/shutdowns itself; rp2350-rescue.cfg never shuts down, so it needs an +# explicit one or it sits in the server loop until CMD_TIMEOUT. RESCUE_CFG = { 'target/rp2040.cfg': ('target/rp2040.cfg', '-c "set RESCUE 1" ', ''), 'target/rp2350.cfg': ('target/rp2350-rescue.cfg', '', ' -c "shutdown"'), } -def rescue_openocd(board, flash_out: str = '') -> bool: +def rescue_openocd(board, flash_out: str = '', timeout=None) -> bool: """Power-on-reset a wedged RP2040/RP2350 through its Rescue DP, the one debug port not - gated by the system clock (RP2040 datasheet 2.3.4.2): setting CDBGPWRUPREQ hard-resets - the chip, and the bootrom halts it in a safe state ready to be flashed. This is the - only way back for a target whose cores have stopped answering -- otherwise the board - needs a physical replug, since the probe carries no reset line. - - No-op (returns False) unless this is an openocd RP board AND the flash output shows the - wedge, so a flash that failed for any other reason still just retries. Returns True - when a rescue was attempted; the caller should retry the flash afterwards.""" + gated by the system clock (RP2040 datasheet 2.3.4.2): CDBGPWRUPREQ hard-resets the + chip and the bootrom halts it ready to be flashed. Without it the board needs a + physical replug -- the probe carries no reset line. + + No-op (False) unless this is an openocd RP board AND the flash output shows the wedge, + so a flash that failed for any other reason still just retries. True when a rescue was + attempted; the caller should retry the flash afterwards.""" flasher = board['flasher'] if flasher['name'].lower() != 'openocd' or not any(m in flash_out for m in DAP_WEDGED): return False for cfg, (rescue_cfg, pre, post) in RESCUE_CFG.items(): if cfg in flasher['args']: args = flasher['args'].replace(cfg, rescue_cfg) - return run_cmd(f'{_openocd_cmd_base({**flasher, "args": pre + args})}{post}').returncode == 0 + return hil_util.run_cmd(f'{_openocd_cmd_base({**flasher, "args": pre + args})}{post}', + timeout=timeout).returncode == 0 return False -def flash_esptool(board: Board, firmware: str) -> subprocess.CompletedProcess: +# openocd's own syntax: one or more "0xVVVV 0xPPPP" pairs. Validated rather than merely +# tested for truthiness -- `vid_pid` is a hand-edited roster field whose NAME is also used, +# with a different syntax, by tests.dev_attached, and convoy_safe reads a non-empty value +# as PROOF the flasher can deliver a recovery past a poisoned node. A typo there silently +# promised a recovery that openocd would reject at startup. +_VID_PID_RE = re.compile(r'^0x[0-9a-fA-F]{4}(\s+0x[0-9a-fA-F]{4})+$') + + +def valid_vid_pid(value) -> bool: + return isinstance(value, str) and bool(_VID_PID_RE.match(value.strip())) + + +def recover_flasher(board: dict) -> dict: + """The flasher that delivers RECOVERY for this board. + + Optional roster key `flasher_recover`, else the primary. It exists because delivery and + normal flashing have different requirements: a board flashed by jlink/stlink/lm4flash + cannot reach its probe past a poisoned usbfs node, but the same probe driven by openocd + often can (see convoy_safe). Keeping it a separate key rather than a list means the + primary's shape never changes, so nothing that reads board['flasher'] has to care. + """ + return board.get('flasher_recover') or board['flasher'] + + +def convoy_safe(flasher: dict) -> bool: + """Can this flasher DELIVER a recovery while a usbfs node on the rig is poisoned? + + A post-HUNG reflash only helps if the flasher reaches its probe without opening the + wedged node. Two shapes qualify: + + * openocd pinned with the roster's `vid_pid` -- the match is made from the cached + descriptor and the loop `continue`s BEFORE libusb_open, so a foreign node is never + opened. On 2026-08-12 it was the only flasher that still reached its probe. + * esptool -- delivery is `-p `, a named port; it never enumerates usbfs. + + Everything else enumerates by OPENING nodes, would block in D state on the poisoned + one, survive SIGKILL and become a second stray. JLinkExe cannot be pinned: selection + is serial-only (-USB/-SelectEmuBySN) and reading a serial requires the open (J-Link + Commander V9.66 exposes no VID/PID filter), so those boards can only become + convoy-safe by moving to openocd. + + Verified against openocd 0ce743125 (the rig's build), because the INVERSE is what + bites: cmsis_dap_usb_bulk.c:107 skips on `id_filter && !id_match`, and `id_filter` is + only `vids[0] || pids[0]` -- so without the pin nothing is skipped and every device on + the bus is opened, which the code itself expects to mostly fail. Enumeration cannot + block: libusb reads the `descriptors` sysfs attribute, and descriptors_read (v6.12.101 + drivers/usb/core/sysfs.c) is a memcpy from udev->rawdescriptors under no lock. + + The pin gates the BULK backend, which is the one that runs: `auto` tries usb_bulk -> + hid -> tcp (cmsis_dap.c:62) and stops at the first that opens, so a CMSIS-DAP v2 probe + never reaches the rest. It does NOT cover the HID fallback that a v1 probe or a failed + bulk open takes -- cmsis_dap_usb_hid.c:91 calls hid_enumerate(0x0, 0x0), pin ignored, + and filters afterwards, while hidapi's hidraw backend reads `manufacturer` and + `product` for every HID device it lists (linux/hid.c:744), both usb_string_attr and so + served under the device lock. A wedged DUT running hid_generic_inout, + hid_boot_interface or hid_composite_freertos is a HID device and would stall that walk + -- interruptibly, so it hangs rather than joining the D-state convoy and run_cmd's + timeout ends it, but "never opens a foreign node" is true of the bulk path, not of + every path openocd can take. + """ + name = (flasher.get('name') or '').lower() + if name == 'esptool': + return True + # EXACT, not startswith: rescue_openocd and usbtest's + # getattr(hil_flash, f'flash_{name}') both require the exact name, so an + # 'openocd_wch'-style entry would pass this gate, reserve USBTEST_RECOVERY_BUDGET, + # and then find no recovery path at all -- paying for a path that cannot fire, which + # is the precise cost this gate exists to avoid. + if name != 'openocd': + return False + if valid_vid_pid(flasher.get('vid_pid')): + return True + # openocd over the JLINK driver is safe WITHOUT a pin, and cannot use one: jlink.c + # never reads adapter_usb_get_vids/pids (selection is adapter serial / usb address / + # usb location), but libjaylink's discovery returns early unless idVendor == 0x1366 and + # the PID is in its table, and only THEN calls libusb_open (discovery_usb.c). So it + # never opens a foreign node -- which is exactly what JLinkExe, SEGGER's own tool, + # does do. Verified against openocd 0ce743125 and libjaylink master. + return 'interface/jlink.cfg' in (flasher.get('args') or '') + + +def flash_esptool(board: Board, firmware: str, timeout=None) -> subprocess.CompletedProcess: flasher = board['flasher'] - port = get_serial_dev(flasher["uid"], None, None, 0) + port = hil_util.get_serial_dev(flasher["uid"], None, None, 0) fw_dir = Path(firmware).parent with (fw_dir / 'config.env').open() as f: idf_target = json.load(f)['IDF_TARGET'] @@ -234,32 +263,39 @@ def flash_esptool(board: Board, firmware: str) -> subprocess.CompletedProcess: flash_args = f.read().strip().replace('\n', ' ') command = (f'esptool --chip {idf_target} -p {port} {flasher["args"]} ' f'--before=default_reset --after=hard_reset write_flash {flash_args}') - ret = run_cmd(command, cwd=str(fw_dir)) + ret = hil_util.run_cmd(command, cwd=str(fw_dir), timeout=timeout) return ret def reset_esptool(board): - flasher = board['flasher'] + # NO-OP, and marked as one: esptool's reset would be `--after hard_reset`, which is not + # wired here. Returning rc 0 without resetting is why callers must never read the exit + # code as proof -- recovery_steps skips a primitive carrying `no_op`. return subprocess.CompletedProcess(args=['dummy'], returncode=0) -def flash_lm4flash(board, firmware): +reset_esptool.no_op = True + + +def flash_lm4flash(board, firmware, timeout=None): # TI Tiva-C / Stellaris ICDI: lightweight lm4flash, resets and runs after write flasher = board['flasher'] - ret = run_cmd(f'lm4flash -s {flasher["uid"]} {flasher["args"]} {firmware}') + ret = hil_util.run_cmd(f'lm4flash -s {flasher["uid"]} {flasher["args"]} {firmware}', + timeout=timeout) return ret def reset_lm4flash(board): # lm4flash has no reset-only mode; it resets+runs on flash, so reset is a no-op - flasher = board['flasher'] return subprocess.CompletedProcess(args=['dummy'], returncode=0) -# The one place a flasher's firmware extension is decided: find_firmware resolves the -# path with it and the flash_* functions pass that path through untouched. A flasher -# added here without an entry falls back to .elf-or-.bin and can be handed the wrong -# file — test_hil_select's TestRosterFlashersDispatch fails if a roster names one. +reset_lm4flash.no_op = True + + +# The one place a flasher's firmware extension is decided. A flasher with no entry falls +# back to .elf-or-.bin and can be handed the wrong file — test_hil_select's +# TestRosterFlashersDispatch fails if a roster names one. FLASHER_SUFFIX = { 'esptool': '.bin', 'jlink': '.elf', @@ -271,13 +307,12 @@ FLASHER_SUFFIX = { def find_firmware(variant: str, example: str, roots: list | None = None, flasher: str | None = None): """Locate a built example's firmware under /cmake-build-//, - then under EXTRA_BUILD_DIRS (empty unless the caller opts in — see its comment). - `roots` overrides that search list entirely for one call (e.g. to find a build just - produced by tools/build.py in its fixed cmake-build/ layout without widening the - global policy). `flasher` is the roster flasher name: it selects which extension - counts (see FLASHER_SUFFIX), so a build that produced only the other one is reported - missing — a clean "Skip (no binary)" — instead of being handed to the flasher, which - would fail opaquely on the absent file and burn every retry plus the board lock. + then under EXTRA_BUILD_DIRS. `roots` overrides that search list entirely for one call + (e.g. a build just produced by tools/build.py in its fixed cmake-build/ layout) + without widening the global policy. `flasher` is the roster flasher name and selects + which extension counts (FLASHER_SUFFIX), so a build that produced only the other one + is reported missing — a clean "Skip (no binary)" — instead of being handed to the + flasher, which would fail opaquely and burn every retry plus the board lock. Accepts the single-config layout (firmware directly in the example dir) or Ninja Multi-Config (a per-config subdir like RelWithDebInfo/). Returns the full Path INCLUDING extension, or None if not built.""" @@ -286,7 +321,7 @@ def find_firmware(variant: str, example: str, roots: list | None = None, flasher if not suffixes or suffixes == [None]: suffixes = ['.elf', '.bin'] for bd in dict.fromkeys(roots if roots is not None else [build_dir, *EXTRA_BUILD_DIRS]): - fw_dir = TINYUSB_ROOT / bd / f'cmake-build-{variant}' / example + fw_dir = hil_util.TINYUSB_ROOT / bd / f'cmake-build-{variant}' / example if not fw_dir.is_dir(): continue for cand in [fw_dir / base, fw_dir / 'RelWithDebInfo' / base, diff --git a/test/hil/hil_lock.py b/test/hil/hil_lock.py deleted file mode 100755 index e570da16a..000000000 --- a/test/hil/hil_lock.py +++ /dev/null @@ -1,479 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MIT -"""Board locks + controller permits for the TinyUSB HIL rig. - -Board locks are kernel flocks in BOARD_LOCK_DIR arbitrating hardware access -between dev sessions and CI's hil_test.py (never stop the actions-runner). -Controller permits are in-process semaphores budgeting flashes and usbtest -batteries per host controller; they have no CLI meaning. The CLI below -(hold/release/status) manages board locks only. -""" -import argparse -import fcntl -import glob -import json -import os -import re -import select -import signal -import sys -import time - -BOARD_LOCK_DIR = '/tmp/tinyusb-hil-locks' -CI_REASON = 'hil_test.py' # release-protected holder tag (release refuses to kill it) -PROTECTED_REASONS = {CI_REASON, 'pool_check'} # cmd_release refuses to SIGTERM these holders -PROFILE = os.environ.get('HIL_PROFILE') == '1' - - -def lock_path(board: str) -> str: - return os.path.join(BOARD_LOCK_DIR, f'{board}.lock') - - -def flock_nb(board: str): - """Open-or-create the lock file WITHOUT truncating (a losing racer must not - wipe the winner's record) and take LOCK_EX|LOCK_NB. Returns the open handle; - raises OSError when the flock is held elsewhere (handle already closed).""" - fd = os.open(lock_path(board), os.O_RDWR | os.O_CREAT, 0o666) - fh = os.fdopen(fd, 'r+') - try: - fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError: - fh.close() - raise - return fh - - -def write_record(fh, reason: str) -> bool: - """Holder record; the flock itself is already held. Returns False on a write - failure — acquire_board_lock stays best-effort (the flock is the authority), - but cmd_hold aborts on it like board_lock.py did (a hold whose record is - missing is invisible to status/release).""" - try: - fh.truncate(0) - fh.seek(0) - json.dump({'pid': os.getpid(), 'reason': reason, - 'since': time.strftime('%Y-%m-%dT%H:%M:%S%z')}, fh) - fh.flush() - return True - except OSError: - return False - - -def clear_record(fh) -> None: - """Clear our record before dropping the flock so records stay truthful.""" - try: - fh.truncate(0) - except OSError: - pass - - -def read_record(board: str): - try: - with open(lock_path(board)) as f: - return json.load(f) - except (OSError, ValueError): - return None - - -# --- per-board dev-session locks ------------------------------------------ -def acquire_board_lock(board_name, reason=CI_REASON): - """Take this board's flock for the duration of its flash+test. - Returns an open file handle (keep it referenced; closing releases it), - or None when HIL_NO_BOARD_LOCK=1 or the lock dir is unusable (fail-open: - locking must never break a test run by itself). - Raises RuntimeError only when another session holds the board.""" - import fcntl - if os.environ.get('HIL_NO_BOARD_LOCK') == '1': - return None # user-authorized bypass — see hil skill - try: - os.makedirs(BOARD_LOCK_DIR, exist_ok=True) - fd = os.open(os.path.join(BOARD_LOCK_DIR, f'{board_name}.lock'), - os.O_RDWR | os.O_CREAT, 0o666) - fh = os.fdopen(fd, 'r+') - except OSError as e: - # odd lock dir (perms, path collision): proceed unlocked, but say so — - # a silent fail-open is indistinguishable from the intentional bypass - print(f'warning: board lock unavailable for {board_name} ({e}); proceeding unlocked', - flush=True) - return None - try: - fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError: - try: - info = fh.read(500).strip() - except (OSError, UnicodeDecodeError): - info = '' - fh.close() - raise RuntimeError(f'board locked: {info or "unknown holder"}') - # announce ourselves so the other side's conflict message is truthful; - # best-effort — the flock itself is already held - try: - fh.truncate(0) - fh.seek(0) - json.dump({'pid': os.getpid(), 'reason': reason, - 'since': time.strftime('%Y-%m-%dT%H:%M:%S%z')}, fh) - fh.flush() - except OSError: - pass - return fh - - -# Per-host-controller concurrency (see controller_of/controller_slot below): a usbtest battery -# saturates its DUT's host controller, so batteries and flashes are budgeted per controller. -# - uPD720201 cards need their latest firmware (>= 2.0.2.6; RAM-uploaded, reloads every -# power cycle): ROM firmware dies under battery + re-enumeration churn, and usbtest.py -# refuses the unlink-stress cases on it. -# - widths (profiled 2026-07-13/14): wall time 22.2/14.3/12.5/10.8 min at usbtest width -# 1/2/3/4, plateau after; flash width beyond 8 only adds flasher-hub contention; -# battery case failures start at 12/8 (bandwidth stretch on shared leaf-hub uplinks). -# - a marginal DUT port bouncing during concurrent batteries can wedge/kill a uPD720201 -# ("xHCI host not responding to stop endpoint command"): fix the port/cable or pull -# the board, don't lower the widths (2026-07-16: every death traced to one board's port). -FLASH_PARALLEL = int(os.getenv('HIL_FLASH_PARALLEL', '8')) -USBTEST_PARALLEL = int(os.getenv('HIL_USBTEST_PARALLEL', '4')) -CONTROLLER_SLOTS = 12 # lock slots; controllers are assigned to slots on first sight -usbtest_sems = None # CONTROLLER_SLOTS semaphores: per-slot usbtest-battery permits -flash_sems = None # CONTROLLER_SLOTS semaphores: per-slot flash permits -controller_map = None # shared dict: 'pci:' -> slot, 'uid:' -> pci addr cache -controller_meta = None # guards slot assignment in controller_map -controller_hints = {} # static uid -> pci from the last run's cache (read-only per worker) - - -log = print # hil_test.init_worker points this at log_line via init_scheduling - - -def init_scheduling(b_sems, f_sems, cmap, cmeta, hints, log_fn=None): - """Install per-worker scheduling state (called from hil_test.init_worker).""" - global usbtest_sems, flash_sems, controller_map, controller_meta, controller_hints, log - usbtest_sems, flash_sems = b_sems, f_sems - controller_map, controller_meta, controller_hints = cmap, cmeta, hints - if log_fn is not None: - log = log_fn - - -# ------------------------------------------------------------- -# Per-controller scheduling -# ------------------------------------------------------------- -def controller_of(uid: str): - """Resolve a DUT uid to its root host controller's PCI address, or None if the device - is not enumerated (e.g. parked in board_test firmware with USB off). Successful - resolutions are cached — cabling does not change mid-run. Dual-port parts (e.g. - CH32V307 usbhs/usbfs variants) share one uid and one cache entry: budgeting is only - exact when both ports sit on the same controller (true on this rig).""" - if controller_map is None: - return None - cached = controller_map.get(f'uid:{uid}') - if cached: - return cached - for f in glob.glob('/sys/bus/usb/devices/*/serial'): - d = os.path.dirname(f) - try: - if open(f).read().strip().lower() != uid.lower(): - continue - bus = int(open(os.path.join(d, 'busnum')).read()) - root = os.path.realpath(f'/sys/bus/usb/devices/usb{bus}') - m = re.findall(r'[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]', root) - if m: - controller_map[f'uid:{uid}'] = m[-1] - return m[-1] - except (OSError, ValueError): - continue - return None - - -def controller_slot(pci: str) -> int: - """Map a controller PCI address to a lock slot (assigned on first sight).""" - key = f'pci:{pci}' - with controller_meta: - slot = controller_map.get(key) - if slot is None: - slot = controller_map.get('nslots', 0) - if slot >= CONTROLLER_SLOTS: - slot = 0 # more controllers than slots: overflow shares slot 0 (safe, over-serialized) - else: - controller_map['nslots'] = slot + 1 - controller_map[key] = slot - return slot - - -class controller_permit: - """Context manager: one permit from `sems` on the board's controller slot. If the - controller is unknown, fail closed: take one permit from EVERY slot, in order, so the - operation respects the budget wherever it might land. `warn_unknown` logs that fallback - (used by usbtest, where the device is expected to be enumerated by the caller).""" - def __init__(self, sems, uid: str, warn_unknown: bool = False): - self.sems = sems - self.slots = None - self.uid = uid - if sems is None: - return - pci = controller_of(uid) - if pci is None and not warn_unknown: - # last-run cabling hint, flash budgeting only: a mis-budgeted flash is harmless, - # but a battery must never trust a stale hint (it could stack two batteries on - # one controller). In practice only a board's first flash lands here - batteries - # assert enumeration before taking their permit. - pci = controller_hints.get(uid) - if pci is None and warn_unknown: - log(f'warning: cannot resolve {uid} to a host controller; ' - 'taking a permit on every slot (over-serialized)') - self.slots = [controller_slot(pci)] if pci else list(range(CONTROLLER_SLOTS)) - - def __enter__(self): - if self.slots: - t0 = time.monotonic() - taken = [] - try: - for s in self.slots: - self.sems[s].acquire() - taken.append(s) - # stays inside the try: if this raises (e.g. broken stdout), the permits - # must be released - a failed __enter__ never gets its __exit__ - if PROFILE and time.monotonic() - t0 > 1.0: - log(f'[prof] permit wait {time.monotonic() - t0:.1f}s ' - f'(uid {self.uid}, slots {self.slots})') - except BaseException: - for s in reversed(taken): - self.sems[s].release() - raise - return self - - def __exit__(self, *exc): - if self.slots: - for s in reversed(self.slots): - self.sems[s].release() - return False - - -def flash_permit(uid: str) -> controller_permit: - return controller_permit(flash_sems, uid) - - -def usbtest_permit(uid: str) -> controller_permit: - return controller_permit(usbtest_sems, uid, warn_unknown=True) - - -# --- operator CLI (hold/release/status) ------------------------------------ -def boards_from_config(config: str) -> list: - """All board names, INCLUDING boards-skip: `hold --all` guards rig-wide - operations, and parked boards can still be touched (pool_check -b names them - explicitly), so a rig-wide hold that skipped them would leave a gap.""" - try: - with open(config) as f: - cfg = json.load(f) - return [b['name'] for b in cfg['boards'] + cfg.get('boards-skip', [])] - except (OSError, ValueError, KeyError) as e: - print(f'ERROR: cannot read board roster {config}: {e}', file=sys.stderr) - sys.exit(1) - - -def is_locked(board: str) -> bool: - """True if the recorded holder process is still alive. - - Deliberately never touches the flock: even a momentary probe lock would - make a concurrent acquirer's LOCK_NB attempt fail spuriously. The flock - taken by acquirers themselves stays the only authority.""" - info = read_record(board) - pid = info.get('pid') if isinstance(info, dict) else None - if not isinstance(pid, int) or pid <= 0: - return False - try: - os.kill(pid, 0) - except ProcessLookupError: - return False - except PermissionError: - return True # alive but owned by another user (e.g. the CI runner) - return True - - -def cmd_hold(boards, reason): - os.makedirs(BOARD_LOCK_DIR, exist_ok=True) - # No pre-check: the holder's own LOCK_NB flock is the only authority — a - # recorded pid may be stale or recycled (e.g. a live hil_test.py worker - # that already released this board's flock but not its record). - # The holder signals success through this pipe. A generic is_locked() - # poll would be fooled by a RIVAL invocation's flock — only the holder - # itself knows whether it won every board. - r_fd, w_fd = os.pipe() - pid = os.fork() - if pid > 0: - os.close(w_fd) - os.waitpid(pid, 0) # reap intermediate child - ready, _, _ = select.select([r_fd], [], [], 10) - ok = bool(ready) and os.read(r_fd, 1) == b'1' - os.close(r_fd) - if ok: - print(f'held: {", ".join(boards)}') - return 0 - for b in boards: - info = read_record(b) - if info: - print(f'ERROR: {b} locked: {info}', file=sys.stderr) - print('ERROR: holder failed to acquire locks', file=sys.stderr) - return 1 - # intermediate child: detach, then spawn the actual holder - os.setsid() - if os.fork() > 0: - os._exit(0) - # holder (grandchild): acquire all flocks, signal the parent, sleep until killed - os.close(r_fd) - # Keep the success pipe clear of fds 0-2: invoked with stdio closed, - # os.pipe() can land there and the dup2 loop below would clobber it. - if w_fd <= 2: - w_fd = fcntl.fcntl(w_fd, fcntl.F_DUPFD, 3) - # Detach stdio: a `hold` whose output is captured must see EOF when the - # front-end exits — the immortal holder must not keep that pipe open. - devnull = os.open(os.devnull, os.O_RDWR) - for std_fd in (0, 1, 2): - os.dup2(devnull, std_fd) - if devnull > 2: - os.close(devnull) - try: - handles = [] - for b in boards: - fh = flock_nb(b) - if not write_record(fh, reason): - raise OSError(f'cannot write holder record for {b}') - handles.append(fh) - except OSError: - try: - os.write(w_fd, b'0') - except OSError: - pass - os._exit(1) # lost a race; parent reports the failure - os.write(w_fd, b'1') - os.close(w_fd) - - def _bow_out(*_): - # clear the records before dying so read_record/status stay truthful - # (the kernel drops the flocks themselves on exit either way) - for h in handles: - clear_record(h) - os._exit(0) - - signal.signal(signal.SIGTERM, _bow_out) - while True: - signal.pause() - - -def cmd_release(boards): - rc = 0 - victims = set() - for b in boards: - try: - fd = os.open(lock_path(b), os.O_RDWR) - except OSError: - continue # no lock file (or another user's): nothing we can release - fh = os.fdopen(fd, 'r+') - try: - fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError: - # flock genuinely held — never SIGTERM on a mere pid record: the - # pid may be recycled, or a live worker that already moved on. - fh.close() - info = read_record(b) or {} - pid = info.get('pid') - reason = info.get('reason') - if reason in PROTECTED_REASONS: - print(f'ERROR: {b} is mid-test by {reason} (pid {pid}) — not killing it; ' - 'wait for it to finish', file=sys.stderr) - rc = 1 - elif isinstance(pid, int) and pid > 0: - victims.add(pid) - else: - print(f'ERROR: {b} is held but its record is unreadable', file=sys.stderr) - rc = 1 - continue - # flock was free: only a stale record remained — clear it - clear_record(fh) - fh.close() - for holder in sorted(victims): - try: - os.kill(holder, signal.SIGTERM) - print(f'released holder pid {holder}') - except ProcessLookupError: - pass - except PermissionError: - print(f'ERROR: holder pid {holder} belongs to another user — cannot signal it', - file=sys.stderr) - rc = 1 - time.sleep(0.3) - still = [b for b in boards if is_locked(b)] - if still: - print(f'ERROR: still locked: {", ".join(still)}', file=sys.stderr) - return 1 - return rc - - -def cmd_status(): - if not os.path.isdir(BOARD_LOCK_DIR): - print('no locks') - return 0 - any_locked = False - for fn in sorted(os.listdir(BOARD_LOCK_DIR)): - if not fn.endswith('.lock'): - continue - b = fn[:-5] - if is_locked(b): - any_locked = True - print(f'{b}: {read_record(b)}') - if not any_locked: - print('no locks') - return 0 - - -_CLI_USAGE = """Per-board advisory locks for the HIL rig. - -Arbitrates board access between dev sessions and CI's hil_test.py without -stopping the actions-runner. Locks are kernel flocks: the kernel releases -them automatically when the holder process dies, and holders clear their -lock-file record on release so records stay truthful (/tmp also clears on -reboot). - -Usage: - hil_lock.py hold BOARD [BOARD...] --reason TEXT - hil_lock.py hold --all [--config CONFIG.json] --reason TEXT - hil_lock.py release BOARD [BOARD...] | release --all - hil_lock.py status - -A holder process holds ALL boards given in one `hold` call; releasing any of -them kills that holder and releases all of its boards. -""" - - -def main(): - ap = argparse.ArgumentParser(description=_CLI_USAGE, - formatter_class=argparse.RawDescriptionHelpFormatter) - sub = ap.add_subparsers(dest='cmd', required=True) - p_hold = sub.add_parser('hold') - p_hold.add_argument('boards', nargs='*') - p_hold.add_argument('--all', action='store_true') - p_hold.add_argument('--config', - default=os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'tinyusb.json'), - help='board roster JSON (default: tinyusb.json beside this script)') - p_hold.add_argument('--reason', required=True) - p_rel = sub.add_parser('release') - p_rel.add_argument('boards', nargs='*') - p_rel.add_argument('--all', action='store_true') - sub.add_parser('status') - a = ap.parse_args() - if a.cmd == 'hold': - boards = boards_from_config(a.config) if a.all else a.boards - if not boards: - ap.error('no boards given (name boards or use --all)') - sys.exit(cmd_hold(boards, a.reason)) - if a.cmd == 'release': - if a.all: - boards = ([fn[:-5] for fn in os.listdir(BOARD_LOCK_DIR) if fn.endswith('.lock')] - if os.path.isdir(BOARD_LOCK_DIR) else []) - else: - boards = a.boards - if not boards: - ap.error('no boards given (name boards or use --all)') - sys.exit(cmd_release(boards)) - sys.exit(cmd_status()) - - -if __name__ == '__main__': - main() diff --git a/test/hil/hil_pool_check.py b/test/hil/hil_pool_check.py deleted file mode 100644 index 98f24288a..000000000 --- a/test/hil/hil_pool_check.py +++ /dev/null @@ -1,1015 +0,0 @@ -#!/usr/bin/env python3 -"""Quick HIL pool health check. - -For every board in the rig's HIL config: is the flash probe on the USB bus, does a -light example flash, and does the board's USB device (uid) come back up? Missing -firmware is BUILT on the spot (tools/build.py, idf.py for espressif; one get_deps -retry) — never skipped; --no-build opts out. Applies only per-device-safe recovery -(probe authorized-toggle, board reset/re-flash) and prints a markdown summary -table. Row statuses: ok (flashed and verified; under --scan-only: probe present — -the scan checks presence only), flash-failed (firmware delivery failed: probe -missing, build failed, flasher error, silent flash no-op, park not verified), -failed (the check ran but did not verify: flashed with no enumeration/serial, or -the check itself errored), locked (board flock held by another process — -reported, never waited on or bypassed). - -Config is picked by hostname unless given: ci -> tinyusb.json, tusb (hifiphile -rig) -> hfp.json, anything else is a dev PC -> local.json. - -Lives in test/hil/ beside hil_lock.py and hil_flash.py, which it imports; board -recovery uses the repo's .claude/skills/usb-kernel-recover/scripts/usb_recover.sh. -""" - -import argparse -import io -import json -import glob -import os -import re -import shlex -import shutil -import socket -import subprocess -import sys -import threading -import time -from concurrent.futures import ThreadPoolExecutor -from pathlib import Path - -REPO_ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(Path(__file__).resolve().parent)) # for import-as-module callers - -import hil_lock -import hil_flash - -USB_RECOVER = REPO_ROOT / '.claude' / 'skills' / 'usb-kernel-recover' / 'scripts' / 'usb_recover.sh' -SEEN_CACHE = Path.home() / '.cache' / 'tinyusb-hil' / 'pool_seen.json' -CONFIG_BY_HOST = {'ci': 'tinyusb.json', 'tusb': 'hfp.json'} # anything else: dev PC -> local.json - -# light-example preference; first built wins -DEVICE_CANDIDATES = ['device/dfu_runtime', 'device/cdc_msc', 'device/cdc_msc_freertos', - 'device/hid_composite_freertos', 'device/cdc_dual_ports'] -HOST_CANDIDATES = ['host/device_info', 'host/cdc_msc_hid', 'host/msc_file_explorer_freertos'] - -ENUM_WAIT = 12 # s, uid wait after flash -ENUM_WAIT_RETRY = 8 # s, uid wait after a recovery reset/re-flash -SERIAL_WAIT = 6 # s, host-board serial-output wait - -print_mutex = threading.Lock() -t0 = time.monotonic() - - -def say(msg: str) -> None: - with print_mutex: - print(f'[{time.monotonic() - t0:6.1f}s] {msg}', file=sys.__stdout__, flush=True) - - -def scan_usb() -> dict: - """busport -> {'serial', 'vidpid', 'ino'} for every enumerated USB device. Only - -[....] dirs match (root hubs, named 'usbN' with no dash, are - excluded: their fabricated PCI-address 'serial' and slow autosuspend-wake read - cost 6-7s/scan on this rig). Keyed by busport, not serial: a serial can be - shared by two different devices (e.g. an Espressif USB-Serial-JTAG bridge and - the cafe TinyUSB device it flashes derive both from the same MAC) — collapsing - them into one dict slot would silently drop whichever lost the race.""" - found = {} - for f in glob.glob('/sys/bus/usb/devices/*-*/serial'): - d = os.path.dirname(f) - busport = os.path.basename(d) - try: - sn = open(f).read().strip().lower() - vidpid = f'{open(d + "/idVendor").read().strip()}:{open(d + "/idProduct").read().strip()}' - found[busport] = {'serial': sn, 'vidpid': vidpid, 'ino': os.stat(d + '/').st_ino} - except OSError: - continue - return found - - -def find_usb(uid: str, devs: dict | None = None): - """Locate a flasher probe by uid, excluding VID cafe (TinyUSB DUT firmware): a - probe's uid can coincidentally equal its DUT's (Espressif USB-Serial-JTAG - bridges derive both from the same MAC), and the DUT is never the probe. - - J-Link zero-pads numeric serials (681295394 -> 000681295394): an all-digit uid - matches an all-digit serial only when that serial equals the uid zero-padded to - the serial's own length (leading zeros only) — never when the zero-stripped uid - is empty, so a placeholder serial (metro_m4_express's probe legitimately reports - '123456') can't be mistaken for an unrelated device.""" - devs = devs if devs is not None else scan_usb() - u = uid.lower() - candidates = [(bp, dev) for bp, dev in devs.items() if not dev['vidpid'].startswith('cafe:')] - for bp, dev in candidates: - if dev['serial'] == u: - return bp, dev['vidpid'], dev['ino'] - stripped = u.lstrip('0') - if u.isdigit() and stripped: - for bp, dev in candidates: - s = dev['serial'] - if s.isdigit() and s == stripped.zfill(len(s)): - return bp, dev['vidpid'], dev['ino'] - return None - - -def find_device(uid: str, pid: str | None): - """Board-online check: TinyUSB device (idVendor cafe) with this uid, optionally - PID-pinned. VID cafe keeps an Espressif USB-Serial-JTAG (303a) sharing the MAC - serial from false-passing.""" - for busport, dev in scan_usb().items(): - if (dev['serial'] == uid.lower() and dev['vidpid'].startswith('cafe:') - and (pid is None or dev['vidpid'].endswith(pid))): - return busport, dev['vidpid'], dev['ino'] - return None - - -def wait_device(uid: str, pid: str | None, old_ino, budget: float): - """Wait for the board's device with a NEW sysfs inode (flash resets the MCU, so a - genuine flash must re-enumerate; the inode is the re-enumeration marker).""" - deadline = time.monotonic() + budget - while time.monotonic() < deadline: - hit = find_device(uid, pid) - if hit and hit[2] != old_ino: - return hit - time.sleep(0.5) - return None - - -def lock_board(name: str): - """Nonblocking flock per hil_lock.py protocol. Returns handle, or a str with - the holder's info when the board is locked elsewhere. Board locks are ALWAYS - respected: a held board is reported as locked and skipped — never waited on, - and there is deliberately no bypass here.""" - os.makedirs(hil_lock.BOARD_LOCK_DIR, exist_ok=True) - try: - fh = hil_lock.flock_nb(name) - except OSError: - # NB: conflates a held flock with open() failures (EACCES/EROFS/ENOSPC) — - # benign while everything on the rig runs as one uid; a cross-uid setup - # would need flock_nb to distinguish the two - info = hil_lock.read_record(name) - return json.dumps(info) if info else 'unknown holder' - if not hil_lock.write_record(fh, 'pool_check'): - # an invisible lock (flock held, no record) is worse than no lock: status - # can't show us and release can't recognize the protected holder — bail out - hil_lock.clear_record(fh) - fh.close() - return 'ERROR: holder record write failed (lock dir unwritable?)' - return fh - - -def unlock_board(fh) -> None: - hil_lock.clear_record(fh) - fh.close() - - -def can_recover() -> bool: - if not USB_RECOVER.is_file(): - return False - try: - r = subprocess.run(['sudo', '-n', 'true'], capture_output=True) - except OSError: # sudo not installed (bare dev PC/container): recovery off, not fatal - return False - return r.returncode == 0 - - -def recover_probe(uid: str, busport: str) -> bool: - """Soft-replug an enumerated-but-wedged probe: deauthorize+reauthorize (no VBUS - cut, touches only this device). Success = the probe re-enumerated (new sysfs - inode), not the helper's exit code (observed to flake while the toggle worked). - J-Links respond with a full disconnect and can stay off the bus for >8 s.""" - pre = find_usb(uid) - try: - # bounded: the sysfs authorized store can block in D state on a wedged - # device, and this runs while the board's (release-protected) flock is held - subprocess.run(['sudo', '-n', str(USB_RECOVER), 'authorized', busport], - capture_output=True, text=True, timeout=30) - except subprocess.TimeoutExpired: - return False - deadline = time.monotonic() + 20 - while time.monotonic() < deadline: - post = find_usb(uid) - if post and (pre is None or post[2] != pre[2]): - return True - time.sleep(0.5) - return False - - -def resolve_variant(board: dict, example: str, note: list | None = None) -> str: - """Build-dir variant name for `example`: the first of the board's variants with - already-built firmware, falling back to the board name. Notes the pick when it - differs from the board name (e.g. nanoch32v203's build dir is variant - 'nanoch32v203-fsdev', not the board name).""" - name = board['name'] - for v in board.get('variant') or [{'name': name}]: - vn = v['name'] - if hil_flash.find_firmware(vn, example, flasher=board['flasher']['name']): - if vn != name and note is not None and f'variant: {vn}' not in note: - note.append(f'variant: {vn}') - return vn - return name - - -def pick_example(board: dict, note: list, build_missing: bool = True): - """(example, kind, variant, fw) with built firmware for this board; kind is - 'device' (uid check) or 'host' (serial-output check); variant is the resolved - build-dir variant that has it (see resolve_variant); fw is the firmware path to - flash, extension included. When nothing is built and build_missing is set (the default — - never skip a board for lack of a build), the preferred candidate is built on - the spot via ensure_fw.""" - tests = board.get('tests', {}) - only = tests.get('only', []) - skip = set(tests.get('skip', [])) # config's known-broken examples: never pick one - is_device = tests.get('device') or any(t.startswith('device/') for t in only) - if is_device: - cand = DEVICE_CANDIDATES + [t for t in only if t.startswith('device/') and t != 'device/usbtest'] - kind = 'device' - else: - cand = HOST_CANDIDATES + [t for t in only if t.startswith('host/')] - kind = 'host' - for ex in dict.fromkeys(cand): - if ex in skip: - continue - variant = resolve_variant(board, ex, note) - fw = hil_flash.find_firmware(variant, ex, flasher=board['flasher']['name']) - if fw: - return ex, kind, variant, fw - if not build_missing: - return None, kind, None, None - # nothing built anywhere: build the preferred candidate (an only-list board - # must get one of its own examples — dfu_runtime etc. may not even configure) - pref = [c for c in dict.fromkeys(cand) if c not in skip and (not only or c in only)] - if not pref: - return None, kind, None, None - variant = (board.get('variant') or [{'name': board['name']}])[0]['name'] - for ex in pref[:2]: # the second candidate covers a preferred example that fails to build - fw = ensure_fw(board, variant, ex, note) - if fw: - return ex, kind, variant, fw - return None, kind, None, None - - -_pid_cache: dict[str, str | None] = {} - - -def get_expected_pid(example: str) -> str | None: - """USB_PID for `example`'s device descriptor (examples//src/ - usb_descriptors.c, '#define USB_PID 0x....'), lowercased and without the 0x - prefix to match sysfs idProduct. Cached per example; None (also cached) when - the file or define isn't there — host examples have no usb_descriptors.c, and - the caller must stay quiet rather than false-warn.""" - if example not in _pid_cache: - pid = None - try: - text = (REPO_ROOT / 'examples' / example / 'src' / 'usb_descriptors.c').read_text() - # optional parens as in tools/check_example_pids.py's parser - m = re.search(r'#define\s+USB_PID\s+\(?\s*(0x[0-9a-fA-F]+)', text) - if m: - pid = m.group(1)[2:].lower() - except OSError: - pass - _pid_cache[example] = pid - return _pid_cache[example] - - -def call_flasher(fn, *fn_args) -> tuple[int, str]: - """Run a hil_flash flash_*/reset_* backend, normalizing raises to a failure: - several backends raise instead of returning nonzero (get_serial_dev - RuntimeError when a bridge's /dev/serial/by-id node vanishes, config.env - FileNotFoundError, .jlink script OSError) and an exception must not skip the - caller's retry/recovery ladder. Returns (returncode, error line).""" - try: - ret = fn(*fn_args) - if ret.returncode == 0: - return 0, '' - err = flash_error_line(hil_flash.cmd_stdout_text(ret.stdout)) - return ret.returncode, err or f'rc={ret.returncode}' - except Exception as e: - return -1, repr(e)[:90] - - -def flash(board: dict, fw, allow_recovery: bool, probe_port: str, note: list) -> bool: - """Flash the resolved firmware with one retry; on repeated failure soft-replug - the probe and always make one final flash attempt afterward, regardless of - whether the replug is confirmed — some probes (WCH-Link, ST-Link, CP210x, - picoprobe) leave their sysfs kobject intact across an authorized toggle - instead of dropping off the bus. Returns True on success. - - `fw` comes from pick_example: a re-resolve here would use the global search - policy and miss a firmware ensure_fw just built into cmake-build/ under an - exclusive -B.""" - fn = getattr(hil_flash, f'flash_{board["flasher"]["name"].lower()}') - for attempt in range(3): - if attempt == 2: - if not (allow_recovery and probe_port): - return False - cur = find_usb(board['flasher']['uid']) - if cur is None: - # probe gone from the bus: its old busport may now hold an UNRELATED - # device (bus renumbering) and the helper only checks occupancy, so - # toggling would deauthorize an innocent fixture — skip the toggle - note.append('probe vanished before toggle') - else: - say(f'{board["name"]:26} recovery: replugging probe {cur[0]} (authorized toggle)') - if recover_probe(board['flasher']['uid'], cur[0]): - note.append('probe replugged') - time.sleep(2) # udev recreates /dev/serial/by-id symlinks after re-enumeration - else: - note.append('probe toggle unconfirmed') - rc, err = call_flasher(fn, board, str(fw)) - if rc == 0: - return True - if rc == 127: # flasher binary missing: retries/probe recovery can't fix env - note.append(f'flasher tool missing ({err}) — esptool needs the ESP-IDF env (get-idf)' - if board['flasher']['name'].lower() == 'esptool' else - f'flasher tool missing: {err}') - return False - if attempt == 0: - say(f'{board["name"]:26} flash retry: {err}') - else: - note.append(f'flash: {err}') - return False - - -def flash_error_line(out: str) -> str: - """Most informative line of a failed flash's output: last error-looking line, - else the last non-empty one.""" - lines = [l.strip() for l in out.splitlines() if l.strip()] - for l in reversed(lines): - if any(k in l.lower() for k in ('error', 'fail', 'unknown', 'cannot', 'timeout', - 'no valid', 'not found', 'unable')): - return l[:90] - return lines[-1][:90] if lines else '' - - -def check_host_serial(board: dict, do_reset: bool = True, want_hello: bool = False) -> bytes | None: - """Host-only boards never enumerate their uid (their USB port is the host side); - aliveness = output on the flasher's UART bridge after a reset. A probe byte is - written each poll so an echo-only firmware (board_test) also answers. Returns - the first output chunk (b'' when silent, None when the port is absent/drops) so - the caller can also judge WHAT answered — see boardtest_output(). - - do_reset=False listens to the firmware as-is: used right after a flash whose - own reset already started it — a second openocd/JLink session back-to-back on - the same probe can fail transiently and leave the target halted.""" - import serial - try: - port = hil_flash.get_serial_dev(board['flasher']['uid'], None, None, 0) - ser = serial.Serial(port, baudrate=115200, timeout=0.3, write_timeout=1) - except Exception as e: - say(f'{board["name"]:26} no flasher serial port: {e}') - return None - try: - # flush BEFORE issuing the reset: pyserial's open-time flush is long past, - # so this drops the pre-reset CDC backlog (which must not count as life) - # while keeping the board's post-reset boot banner, which prints while the - # reset tool is still tearing down and would be eaten by a post-reset flush - ser.reset_input_buffer() - if do_reset: - getattr(hil_flash, f'reset_{board["flasher"]["name"].lower()}')(board) - # collect the WHOLE window and judge content, not the first chunk: the - # probe's CDC bridge has its own FIFO, so stale pre-flash output (e.g. - # board_test hellos) can arrive after our host-side flush and must not - # decide the verdict alone. Early-exit once non-board_test output proves - # a real example is talking. - data = b'' - deadline = time.monotonic() + SERIAL_WAIT - while time.monotonic() < deadline: - try: - ser.write(b'U') - data += ser.read(256) - except serial.SerialTimeoutException: - pass - except serial.SerialException: - return None # port dropped mid-poll (bridge re-enumerating) - # early-exit on the caller's positive signal: fresh board_test hello - # (park verification) vs any non-board_test output (example liveness); - # stale bridge-FIFO backlog of the OTHER kind must not end the window - if want_hello: - if b'Hello from TinyUSB' in data: - return data - elif data and not boardtest_output(data): - return data - return data - finally: - ser.close() - - -def boardtest_output(data: bytes) -> bool: - """True when (non-empty) serial output is recognizably ONLY board_test's: its - periodic HELLO_STR and echoes of our b'U' pokes, nothing else. Any residue - beyond that (an example banner, log lines) proves other firmware is talking, - however much stale board_test backlog surrounds it. Used as a negative - identity marker — after flashing a host example, board_test-only chatter - means the flash silently didn't take (the host analog of the PID check).""" - residue = data.replace(b'Hello from TinyUSB', b'') - for junk in (b'U', b'\r', b'\n'): - residue = residue.replace(junk, b'') - return len(residue) == 0 - - -def build_example(board: dict, variant: str, example: str) -> int: - """Build one example for this board: tools/build.py (same invocation shape as - hil_test.build_board: -T target, -D per build.args, variant defines/flags, - --build-name), or idf.py directly for espressif (tools/build.py's esp branch - ignores -T and builds everything; variant flags travel as -DCFLAGS_CLI, the - same channel tools/build.py uses). Bounded and process-group-killed via - run_cmd; 600 s: a first configure+build of an SDK-heavy family (pico, nrf, - esp) exceeds the old 300. Builds normally run pre-lock (pick_example / the - pre-park ensure), so a board flock is not held here except on rare recovery - paths. Per-build compile parallelism is capped at cpu/-j so -j concurrent - builds cannot swamp sibling workers' verification windows. Returns the - build's returncode (127 = ESP-IDF env missing).""" - name = board['name'] - variants = board.get('variant') or [{'name': name}] - vcfg = next((v for v in variants if v['name'] == variant), variants[0]) - if board['flasher']['name'].lower() == 'esptool': - if not shutil.which('idf.py'): - return 127 # ESP-IDF env not sourced in this shell - # -B keyed off the VARIANT so ensure_fw's post-build lookup finds it - cmd = ['idf.py', '-C', f'examples/{example}', - '-B', f'cmake-build/cmake-build-{vcfg["name"]}/{example}', - '-G', 'Ninja', f'-DBOARD={name}', 'build'] - for d in board.get('build', {}).get('args', []) + vcfg.get('defines', []): - cmd.insert(-1, f'-D{d}') - if vcfg.get('flags'): - cmd.insert(-1, f'-DCFLAGS_CLI={vcfg["flags"]}') - # the IDF component manager writes examples//dependencies.lock in the - # SOURCE tree (idf.py -B relocates only the build dir), so concurrent esp - # builds of one example for different targets corrupt each other's solve - with _esp_lock, _build_sem: - return hil_flash.run_cmd(shlex.join(cmd), cwd=str(hil_flash.TINYUSB_ROOT), - timeout=600).returncode - cmd = [sys.executable, str(hil_flash.TINYUSB_ROOT / 'tools' / 'build.py'), - '-b', name, '-T', Path(example).name, - '-j', str(max(1, (os.cpu_count() or _jobs) // _jobs))] - for d in board.get('build', {}).get('args', []): - cmd += ['-D', d] - if vcfg['name'] != name: - cmd += ['--build-name', vcfg['name']] - for d in vcfg.get('defines', []): - cmd += ['-D', d] - for tok in vcfg.get('flags', '').split(): - cmd += [f'--cflag={tok}'] - with _build_sem: - return hil_flash.run_cmd(shlex.join(cmd), cwd=str(hil_flash.TINYUSB_ROOT), - timeout=600).returncode - - -_deps_lock = threading.Lock() # one get_deps at a time (it also drains _build_sem) -_esp_lock = threading.Lock() # idf.py mutates source-tree dependencies.lock per example -_no_build = False # --no-build: ensure_fw never invokes a build -_jobs = 4 # mirrors -j; set in main before the pool starts -_build_sem = threading.BoundedSemaphore(4) # build slots; get_deps drains ALL (exclusive) -_builds: dict = {} # (variant, example) -> (fw|None, reason): one attempt per run - - -def ensure_fw(board: dict, variant: str, example: str, note: list): - """Firmware for `example`, building it when absent — never skip a board for - lack of a build (--no-build opts out). One retry with deps fetched and the - CMake caches dropped when the first build fails (fresh checkouts lack the - family deps; a cache configured in a broken env poisons every later attempt). - Returns the firmware path, or None with the failure noted. Call BEFORE - taking the board lock: builds are long. One build attempt per - (variant, example) per run, success or failure — memoized in _builds, so a - repeat call (park, under the held flock) resolves instantly even when an - exclusive -B hides the fresh cmake-build/ artifact from the global search.""" - fw = hil_flash.find_firmware(variant, example, flasher=board['flasher']['name']) - if fw: - return fw - key, base = (variant, example), Path(example).name - if key in _builds: - return _builds[key][0] - if _no_build: - _builds[key] = (None, 'disabled') - note.append(f'build skipped (--no-build): {base}') - return None - rc = build_example(board, variant, example) - if rc == 127 and board['flasher']['name'].lower() == 'esptool': - _builds[key] = (None, 'no-env') - note.append(f'cannot build {base}: ESP-IDF env missing (get-idf)') - return None - if rc == 124: # hung build: a deps/cache retry cannot cure it, don't double the stall - _builds[key] = (None, 'timeout') - note.append(f'build timeout: {base}') - return None - if rc != 0: - # retry once with deps fetched and the CMake caches dropped (cache only — - # a tree wipe would destroy every other example's firmware). get_deps - # git-resets already-present shared deps (lib/fatfs's ffconf.h dance), so - # it must exclude every in-flight build, not just other get_deps calls: - # it drains ALL build slots before running. - with _deps_lock: - for _ in range(_jobs): - _build_sem.acquire() - try: - r = hil_flash.run_cmd(shlex.join([sys.executable, str(hil_flash.TINYUSB_ROOT / 'tools' / 'get_deps.py'), - '-b', board['name']]), - cwd=str(hil_flash.TINYUSB_ROOT), timeout=600) - finally: - for _ in range(_jobs): - _build_sem.release() - if r.returncode != 0: - note.append('get_deps failed') - bd = hil_flash.TINYUSB_ROOT / 'cmake-build' / f'cmake-build-{variant}' - # esp configures one level deeper (//): wipe both layouts - for d in (bd, bd / example): - shutil.rmtree(d / 'CMakeFiles', ignore_errors=True) - (d / 'CMakeCache.txt').unlink(missing_ok=True) - rc = build_example(board, variant, example) - if rc != 0: - _builds[key] = (None, 'fail') - note.append(f'build failed: {base}') - return None - # tools/build.py and the idf.py invocation above always write to cmake-build/: - # look there too even when an explicit -B narrowed the global search — this is - # OUR fresh build, not a stale-candidate fallback - fw = hil_flash.find_firmware(variant, example, - roots=[hil_flash.build_dir, 'cmake-build'], - flasher=board['flasher']['name']) - _builds[key] = (fw, 'ok' if fw else 'no-fw') - note.append(f'built {base}' if fw else f'build produced no firmware: {base}') - return fw - - -def ensure_board_test(board: dict, variant: str, note: list): - """board_test firmware for parking, building it if absent (via ensure_fw). - Espressif included — tools/build.py builds board_test for that family too; - the build just needs the ESP-IDF env (127 → noted, park is then skipped).""" - fw = hil_flash.find_firmware(variant, 'device/board_test', flasher=board['flasher']['name']) - if fw: - return fw - variants = board.get('variant') or [{'name': board['name']}] - if not any(v['name'] == variant for v in variants): - variant = variants[0]['name'] - return ensure_fw(board, variant, 'device/board_test', note) - - -def verdict(row: dict, ok: bool) -> str: - """Row status for a verification result, preserving a 'flash-failed' a deeper - layer already recorded (silent flash no-op, board_test delivery failure).""" - return 'ok' if ok else ('flash-failed' if row['status'] == 'flash-failed' else 'failed') - - -def host_alive(board: dict, note: list, row: dict, flashed_example: bool = False) -> bool: - """Serial aliveness with recovery: silent -> (build and) flash board_test (it - hellos every second and echoes) -> recheck. Also cures a silent flash no-op - that left the board crashed. - - With flashed_example=True (a host example was just flashed), board_test-shaped - output FAILS the check: the parked image still talking means the example flash - silently didn't take — the host analog of the device path's PID check. - - Side effect: delivery-class failures (silent no-op, board_test build/flash - failure) set row['status'] = 'flash-failed' so verdict() preserves the cause; - the caller derives the final status from the return value via verdict().""" - data = check_host_serial(board) - if data: - if flashed_example and boardtest_output(data): - note.append('board_test output after example flash: silent flash no-op') - row['status'] = 'flash-failed' - return False - return True - variant = resolve_variant(board, 'device/board_test', note) - fw = ensure_board_test(board, variant, note) - if fw is None: - note.append('serial silent; board_test unavailable') - row['status'] = 'flash-failed' - return False - say(f'{board["name"]:26} recovery: serial silent, flashing board_test') - rc, err = call_flasher(getattr(hil_flash, f'flash_{board["flasher"]["name"].lower()}'), board, str(fw)) - if rc != 0: - note.append(f'serial silent; board_test flash failed: {err}') - row['status'] = 'flash-failed' - return False - if not check_host_serial(board): - return False - if flashed_example: - # board_test talking proves the BOARD is alive, but the just-flashed - # example never produced serial — that verification still fails - note.append('example silent; board alive via board_test reflash') - return False - note.append('recovered via board_test reflash') - return True - - -def device_recover_and_check(board: dict, example: str, variant: str, old_ino, note: list, row: dict, seen: dict) -> bool: - """Wait for the flashed board's uid to re-enumerate; on timeout, try one board - reset (skipped for flashers with no hardware reset — see hil_flash.RESET_NOOP, - it would just burn the wait) and wait again. - - The PID policy is deliberately asymmetric. Pre-reset, the re-enumeration was - caused by the flash itself, so a PID mismatch most likely means the build dir - is stale (the flash DID write what find_firmware found) — warn, don't fail — - UNLESS the firmware was built this very run: then 'stale build' is impossible - and the mismatch can only be a silent flash no-op, which fails. Post-reset, - the re-enumeration proves nothing about the flash (the reset alone explains - it), so a mismatch is treated as a silent flash no-op and fails; an unknown - expected PID scores ok with a 'pid unverified' note in both paths.""" - name = board['name'] - expected_pid = get_expected_pid(example) - built_this_run = _builds.get((variant, example), (None, ''))[1] == 'ok' - - def seen_hit(hit): - seen[board['uid']] = {'name': name, 'busport': hit[0], 'when': time.strftime('%Y-%m-%d %H:%M')} - - hit = wait_device(board['uid'], None, old_ino, ENUM_WAIT) - if hit: - if expected_pid is not None and not hit[1].endswith(expected_pid): - if built_this_run: - row['device'] = f'❌ {hit[1]}' - note.append(f'pid {hit[1]}, this run built {expected_pid}: silent flash no-op') - row['status'] = 'flash-failed' - return False - note.append(f'⚠ pid {hit[1]}, source says {expected_pid}: stale build or silent flash no-op') - elif expected_pid is None: - note.append('pid unverified') - row['device'] = f'✅ {hit[1]}' - seen_hit(hit) - return True - - flasher_name = board['flasher']['name'].lower() - if flasher_name in hil_flash.RESET_NOOP: - note.append(f'no hardware reset available for {flasher_name}') - row['device'] = '❌ not enumerated' - return False - - say(f'{name:26} recovery: uid not up, resetting board') - rc, err = call_flasher(getattr(hil_flash, f'reset_{flasher_name}'), board) - if rc != 0: - note.append(f'reset failed: {err}') - hit = wait_device(board['uid'], None, old_ino, ENUM_WAIT_RETRY) - if not hit: - row['device'] = '❌ not enumerated' - note.append('reset did not help') - return False - if expected_pid is None: - row['device'] = f'✅ {hit[1]}' - note.append('reset recovered (pid unverified)') - seen_hit(hit) - return True - if hit[1].endswith(expected_pid): - row['device'] = f'✅ {hit[1]}' - note.append('reset recovered') - seen_hit(hit) - return True - row['device'] = f'❌ {hit[1]}' - note.append(f'reset recovered wrong pid, expected {expected_pid}: silent flash no-op') - row['status'] = 'flash-failed' - return False - - -def check_board(board: dict, args, allow_recovery: bool, seen: dict) -> dict: - name = board['name'] - row = {'name': name, 'probe': '❌ missing', 'flash': '–', 'device': '–', 'note': [], 'status': 'failed'} - note = row['note'] - - probe = find_usb(board['flasher']['uid']) - if probe: - row['probe'] = f'✅ {probe[0]}' - seen[board['flasher']['uid']] = {'name': f'{name} probe', 'busport': probe[0], - 'when': time.strftime('%Y-%m-%d %H:%M')} - else: - last = seen.get(board['flasher']['uid']) - note.append(f'probe last seen {last["busport"]} {last["when"]}' if last - else 'probe never seen by pool_check') - say(f'{name:26} probe MISSING ({board["flasher"]["name"]} {board["flasher"]["uid"]})') - - # existing firmware only here; a missing build is built on the spot further - # down (after a lock peek), except in scan/no-build modes — and never for a - # missing probe (nothing could be flashed anyway) - example, kind, variant, fw = pick_example(board, note, build_missing=False) - if kind == 'host': - note.append('host-only board') - - if args.scan_only: - hit = find_device(board['uid'], None) - # report the BOARD's usb state, not just the probe's: the enumerated device - # (with busport), off-bus (normal when parked in board_test), or n/a for - # host-only boards whose uid never enumerates - if hit: - row['device'] = f'✅ {hit[1]} @{hit[0]}' - elif kind == 'host': - row['device'] = '– n/a (host-only)' - else: - row['device'] = '⚫ off bus (parked?)' - # scan verifies probe presence only: that check DID run, so probe present - # is ok; a missing probe means no firmware could be delivered → flash-failed - row['status'] = 'ok' if probe else 'flash-failed' - if probe: - say(f'{name:26} probe ✅ {probe[0]}' + (f' device {hit[1]}' if hit else '')) - return row - if not probe: - row['status'] = 'flash-failed' - return row - - bt_variant = resolve_variant(board, 'device/board_test', note) - need_example = example is None and not args.no_build - # board_test is also host_alive's recovery image, so host boards pre-build it - # even under --no-park; --no-build gates EVERY build, board_test included - need_bt = (not args.no_build - and (not args.no_park or kind == 'host') - and hil_flash.find_firmware(bt_variant, 'device/board_test', - flasher=board['flasher']['name']) is None) - if need_example or need_bt: - # builds are long and run BEFORE locking (park must never hold the flock - # through a build); peek the lock first so minutes of building are not - # wasted on — or a rebuilt tree swapped under — a board CI holds right now - peek = lock_board(name) - if isinstance(peek, str): - if peek.startswith('ERROR:'): # environment failure, not a held lock - row['flash'] = '❌ lock' - row['status'] = 'failed' - else: - row['flash'] = '🔒 locked' - row['status'] = 'locked' - note.append(peek) - say(f'{name:26} locked: {peek}') - return row - unlock_board(peek) - if need_example: - example, kind, variant, fw = pick_example(board, note, build_missing=True) - if need_bt and (example is not None or kind == 'host'): - # skip the park-image build when the example build already failed on a - # device board: the row returns before any flash/park could use it - ensure_board_test(board, bt_variant, note) - - if example is None: - if not any(n.startswith(('build failed', 'build timeout', 'build produced', - 'build skipped', 'cannot build')) for n in note): - note.append('no firmware built') - if kind != 'host': - row['status'] = 'flash-failed' - say(f'{name:26} probe ✅ {probe[0]} (no firmware to flash)') - return row - # host-only board: aliveness is still checkable without flashing — reset and - # listen to whatever firmware is on it (the parked board_test echoes and - # prints a periodic hello on the flasher UART) - - lk = lock_board(name) - if isinstance(lk, str): - if lk.startswith('ERROR:'): # environment failure, not a held lock - row['flash'] = '❌ lock' - row['status'] = 'failed' - else: - row['flash'] = '🔒 locked' - row['status'] = 'locked' - note.append(lk) - say(f'{name:26} locked: {lk}') - return row - try: - if example is None: # host-only without firmware: UART-only aliveness check - ok = host_alive(board, note, row) - row['device'] = '✅ serial out' if ok else '❌ no serial out' - row['status'] = verdict(row, ok) - say(f'{name:26} – {row["device"]} (existing firmware)') - return row - - pre = find_device(board['uid'], None) - old_ino = pre[2] if pre else None - - try: - if not flash(board, fw, allow_recovery, probe[0], note): - row['flash'] = f'❌ {Path(example).name}' - row['status'] = 'flash-failed' - say(f'{name:26} flash FAILED ({example})') - return row - row['flash'] = f'✅ {Path(example).name}' - - if kind == 'host': - ok = host_alive(board, note, row, flashed_example=True) - row['device'] = '✅ serial out' if ok else '❌ no serial out' - else: - ok = device_recover_and_check(board, example, variant, old_ino, note, row, seen) - row['status'] = verdict(row, ok) - say(f'{name:26} {row["flash"]} {row["device"]}') - return row - finally: - # teardown for EVERY path that attempted a flash (a failed programmer op - # can still have erased/half-written the target): re-park while the - # board lock is still held - if not args.no_park: - park_board(board, kind, row, note) - finally: - unlock_board(lk) - - -def park_board(board: dict, kind: str, row: dict, note: list) -> None: - """Re-park with board_test, building it if absent (ensure_board_test), and - VERIFY it took: board_test never enumerates USB, so a device board's cafe - device must drop off the bus, and a host board must answer with board_test's - own output — a rc=0 park that changed nothing (silent no-op) must not pass. - A board left unparked marks an ok row flash-failed (never downgrading a - 'failed' verify verdict — that is the more diagnostic signal), with one - exception: an espressif board without the ESP-IDF env cannot build - board_test — noted, not a board fault.""" - # capture BEFORE the park flash: uid-disappearance only verifies the park if - # the device was on the bus to begin with (a fast park drops it immediately) - on_bus_before = kind != 'host' and find_device(board['uid'], None) is not None - variant = resolve_variant(board, 'device/board_test', note) - fw = ensure_board_test(board, variant, note) - if fw is None: - if any(n.startswith('cannot build board_test') for n in note): - note.append('park skipped (no ESP-IDF env)') - else: - # --no-build disables builds, not parking (--no-park is that opt-out): - # a board left running a USB-active image is unparked either way - note.append('unparked: board_test not built (--no-build)' - if any(n.startswith('build skipped (--no-build): board_test') for n in note) - else 'unparked: board_test unavailable (build failed/timed out)') - if row['status'] == 'ok': - row['status'] = 'flash-failed' - return - rc, err = call_flasher(getattr(hil_flash, f'flash_{board["flasher"]["name"].lower()}'), - board, str(fw)) - if rc != 0: - note.append(f'park flash failed: {err}') - if row['status'] == 'ok': - row['status'] = 'flash-failed' - return - if kind == 'host': - # no second reset (the park flash's own reset already started board_test); - # POSITIVE marker: its hello must appear — stale example output may still - # drain from the probe bridge's FIFO alongside it and is not disqualifying - data = check_host_serial(board, do_reset=False, want_hello=True) - if not (data and b'Hello from TinyUSB' in data): - note.append('park unverified: no board_test output') - if row['status'] == 'ok': - row['status'] = 'flash-failed' - return - if not on_bus_before: - # board never enumerated this run: uid-disappearance can't distinguish a - # verified park from a silent no-op — say so instead of passing vacuously - note.append('park unverified (device already off bus)') - return - deadline = time.monotonic() + 6 - while time.monotonic() < deadline: - if find_device(board['uid'], None) is None: - return - time.sleep(0.5) - note.append('park unverified: device still enumerated') - if row['status'] == 'ok': - row['status'] = 'flash-failed' - - -def check_board_safe(board: dict, args, allow_recovery: bool, seen: dict) -> dict: - """Isolate one board's exceptions: a crashing worker must not discard every - other board's row, the table, the topology, and the seen-cache write.""" - try: - return check_board(board, args, allow_recovery, seen) - except Exception as e: - name = board.get('name', '?') - say(f'{name:26} INTERNAL ERROR: {e!r}') - return {'name': name, 'probe': '–', 'flash': '–', 'device': '❌ error', - 'note': [repr(e)[:120]], 'status': 'failed'} - - -def controller_summary() -> list[str]: - """USB topology: controller (PCI addr, vendor) -> bus -> root-port subtree device - counts (hubs included, interfaces/root hubs not). Bus numbers renumber every boot; - PCI addresses and root-port numbers are stable.""" - vendor_names = {'0x1022': 'AMD', '0x1912': 'Renesas', '0x8086': 'Intel', '0x1b21': 'ASMedia'} - ctrl = {} - for root in glob.glob('/sys/bus/usb/devices/usb*'): - bus = int(os.path.basename(root)[3:]) - m = re.findall(r'[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]', os.path.realpath(root)) - pci = m[-1] if m else '?' - c = ctrl.setdefault(pci, {'vendor': '?', 'buses': {}}) - subtrees = {} - for d in glob.glob(f'/sys/bus/usb/devices/{bus}-*'): - b = os.path.basename(d) - if ':' in b: - continue - subtrees[b.split('.')[0]] = subtrees.get(b.split('.')[0], 0) + 1 - c['buses'][bus] = subtrees - try: - vid = open(f'/sys/bus/pci/devices/{pci}/vendor').read().strip() - c['vendor'] = vendor_names.get(vid, vid) - except OSError: - pass - - lines = [] - for pci, c in sorted(ctrl.items()): - lines.append(f'{pci} ({c["vendor"]})') - for bus, subtrees in sorted(c['buses'].items()): - detail = ' '.join(f'{k}: {n} dev' for k, n in - sorted(subtrees.items(), key=lambda i: int(i[0].split('-')[1]))) - lines.append(f' bus {bus}: {sum(subtrees.values())} devices' - + (f' {detail}' if detail else '')) - return lines - - -def main() -> None: - # toolchain/flasher CLIs live in the user bin dirs (arm-none-eabi-gcc + esptool - # in ~/.local/bin, STM32_Programmer_CLI in ~/bin) which non-login shells may - # lack — same PATH shim hil_ci.sh applies on the remote side - for d in (Path.home() / 'bin', Path.home() / '.local' / 'bin'): - if d.is_dir() and str(d) not in os.environ.get('PATH', '').split(os.pathsep): - os.environ['PATH'] = f'{d}{os.pathsep}{os.environ.get("PATH", "")}' - - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument('config', nargs='?', help='HIL config json (default: by hostname)') - parser.add_argument('-b', '--board', action='append', default=[], help='only these boards') - parser.add_argument('-B', '--build-dir', default=None, - help='firmware parent dir, searched EXCLUSIVELY when given ' - '(default: examples, plus cmake-build as fallback)') - parser.add_argument('--scan-only', action='store_true', - help='USB presence scan only: no locks, no flashing') - parser.add_argument('--no-build', action='store_true', - help='do not build missing firmware (default: build the light example on the spot)') - parser.add_argument('--no-park', action='store_true', - help='leave the light example running (default: park with board_test)') - # no cross-process flash budget with a concurrent hil_test.py run yet (would need - # a file-lock budget in hil_lock; hil_test uses in-process semaphores) — keep modest - parser.add_argument('-j', '--jobs', type=int, default=4) - parser.add_argument('-v', '--verbose', action='store_true') - args = parser.parse_args() - global _no_build, _jobs, _build_sem - _no_build = args.no_build - _jobs = max(1, args.jobs) - _build_sem = threading.BoundedSemaphore(_jobs) - - host = socket.gethostname() - cfg_name = args.config or CONFIG_BY_HOST.get(host, 'local.json') - cfg_path = Path(cfg_name) - if not cfg_path.exists(): - cfg_path = REPO_ROOT / 'test' / 'hil' / cfg_name - if not cfg_path.exists(): - sys.exit(f'config not found: {cfg_name} (host {host}; dev PCs need test/hil/local.json)') - with cfg_path.open() as f: - config = json.load(f) - - boards = list(config['boards']) # boards-skip (parked hardware) is not scanned by default - if args.board: - boards += config.get('boards-skip', []) # explicitly named parked boards are fair game - unknown = set(args.board) - {b['name'] for b in boards} - if unknown: - sys.exit(f'board(s) not in {cfg_path.name}: {", ".join(sorted(unknown))}') - boards = [b for b in boards if b['name'] in args.board] - - hil_flash.build_dir = args.build_dir or 'examples' - hil_flash.verbose = args.verbose - if args.build_dir is None: - # default mode: search both standard layouts (cmake-build/ from tools/build.py - # + ESP-IDF, examples/ from manual builds). An EXPLICIT -B is exclusive — the - # caller named an artifact tree, so a miss must report, not silently flash an - # older build from elsewhere. hil_test's -B is likewise untouched by this. - hil_flash.EXTRA_BUILD_DIRS = ['cmake-build', 'examples'] - allow_recovery = not args.scan_only and can_recover() - seen = {} - try: - loaded = json.loads(SEEN_CACHE.read_text()) - if isinstance(loaded, dict): # tolerate a torn/hand-edited cache - seen = {k: v for k, v in loaded.items() if isinstance(v, dict)} - except (OSError, ValueError): - pass - - roots = ' + '.join(dict.fromkeys([hil_flash.build_dir, *hil_flash.EXTRA_BUILD_DIRS])) - say(f'pool check: host {host}, config {cfg_path.name}, {len(boards)} boards, ' - f'{"scan-only" if args.scan_only else f"flash via {{{roots}}}/cmake-build-"}' - f'{"" if allow_recovery or args.scan_only else ", recovery unavailable (no sudo -n / usb_recover.sh)"}') - - if args.verbose: - rows = [check_board_safe(b, args, allow_recovery, seen) for b in boards] - else: - with io.StringIO() as spool, ThreadPoolExecutor(max_workers=args.jobs) as pool: - sys.stdout = spool # silence hil_flash's COMMAND FAILED dumps; say() uses __stdout__ - try: - rows = list(pool.map(lambda b: check_board_safe(b, args, allow_recovery, seen), boards)) - finally: - sys.stdout = sys.__stdout__ - - try: - SEEN_CACHE.parent.mkdir(parents=True, exist_ok=True) - tmp = SEEN_CACHE.with_suffix('.json.tmp') - tmp.write_text(json.dumps(seen, indent=1, sort_keys=True) + '\n') - tmp.replace(SEEN_CACHE) # atomic: a killed run can't tear the cache - except OSError: - pass - - status_mark = {'ok': '✅ ok', 'flash-failed': '❌ flash-failed', 'failed': '❌ failed', - 'locked': '🔒 locked'} - headers = ['Board', 'Probe', 'Flash', 'Device', 'Status', 'Note'] - cells = [[r['name'], r['probe'], r['flash'], r['device'], - status_mark.get(r['status'], r['status']), '; '.join(r['note'])] for r in rows] - widths = [max(len(h), *(len(c[i]) for c in cells)) if cells else len(h) - for i, h in enumerate(headers)] - line = lambda vals: '| ' + ' | '.join(v.ljust(w) for v, w in zip(vals, widths)) + ' |' - print() - print(line(headers)) - print('|' + '|'.join('-' * (w + 2) for w in widths) + '|') - for c in cells: - print(line(c)) - - print('\nUSB topology (controller → root-port subtree):') - for line in controller_summary(): - print(f' {line}') - - counts = {'ok': 0, 'flash-failed': 0, 'failed': 0, 'locked': 0} - for r in rows: - counts[r.get('status', 'failed')] += 1 - print(f'\n{counts["ok"]} ok · {counts["flash-failed"]} flash-failed · {counts["failed"]} failed ' - f'· {counts["locked"]} locked · in {time.monotonic() - t0:.0f}s') - sys.exit(min(counts['flash-failed'] + counts['failed'], 125)) - - -if __name__ == '__main__': - main() diff --git a/test/hil/hil_select.py b/test/hil/hil_select.py deleted file mode 100755 index 3ac3f1fdb..000000000 --- a/test/hil/hil_select.py +++ /dev/null @@ -1,520 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MIT -"""PR-diff -> HIL selection: which rig boards and which tests a change can affect. - -Stdlib-only (runs on bare CI runners; never imports hil_test/hil_flash/hil_lock). -Fail-open: any file no rule classifies forces the full matrix. See -docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md. - -JSON: full, boards (name -> 'all' | [tests]), families (bsp families the diff -touches, including ones with no rig board - build-only consumers such as /pre-pr -sample from these), args (hil_test.py args per config) and args_flasher (the same -args split by each board's flasher, for CI legs that split one rig by flasher). -""" -import argparse -import functools -import glob -import json -import os -import re -import subprocess -import sys - -from hil_examples import device_tests, dual_tests, host_test - -ALL_TESTS = {'device': device_tests, 'dual': dual_tests, 'host': host_test} - -# class dir -> config macro suffix exceptions (rule 3); dfu is per-file, handled inline -NET_MACROS = ('ECM_RNDIS', 'NCM') - -_NONCODE_RE = re.compile( - r'^(docs/|\.claude/|.*\.(md|rst)$|LICENSE)') -_FULL_RE = re.compile( - r'^(src/common/|src/osal/|src/tusb\.c$|src/tusb\.h$|src/tusb_option\.h$|' - r'test/hil/|\.github/workflows/build.*\.yml$|\.github/actions/|' - r'tools/build\.py$|tools/get_deps\.py$|tools/cmake/|hw/mcu/|lib/|' - r'hw/bsp/(family_support\.cmake|board_api\.h|board\.c|ansi_escape\.h)$|' - r'examples/build_system/|examples/CMakeLists\.txt$|' - # board_test is HIL infrastructure, not a test: hil_test.py flashes it to park - # every board (variant boundary + end-of-board teardown), so every board depends on it - r'examples/device/board_test/)') - -# --no-renames: with rename detection git reports only a rename's destination, so code -# moved out of an HIL-relevant path would be classified by its new path alone -GIT_DIFF_ARGV = ['git', 'diff', '--no-renames', '--name-only'] - - -def test_role(test: str) -> str: - return test.split('/', 1)[0] # 'device' | 'dual' | 'host' - - -def board_roles(board: dict) -> set: - t = board.get('tests', {}) - roles = set() - if t.get('device'): - roles.add('device') - if t.get('host'): - roles.add('host') - if t.get('dual'): - roles.update(('device', 'host')) - for only in t.get('only', []): - r = test_role(only) - roles.update(('device', 'host') if r == 'dual' else (r,)) - return roles - - -def board_tests(board: dict) -> list: - """Every test this board would run today (mirrors hil_test.test_board's default).""" - t = board.get('tests', {}) - if 'only' in t: - run = list(t['only']) - else: - run = [] - if t.get('device'): - run += device_tests - if t.get('dual'): - run += dual_tests - if t.get('host'): - run += host_test - return [x for x in run if x not in t.get('skip', [])] - - -# cached: called per changed file x roster board, and the tree doesn't change mid-run -@functools.lru_cache(maxsize=None) -def board_family(board_name: str, repo_root: str): - hits = glob.glob(os.path.join(repo_root, 'hw/bsp/*/boards', board_name)) - return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None - - -# `if (OPTION STREQUAL "1")` guards in family_support.cmake, and the option tokens -# a roster entry passes to the build (NAME=VALUE / -DNAME=VALUE) -_CM_IF_RE = re.compile(r'if\s*\(') -_CM_ELSE_RE = re.compile(r'else(if)?\s*\(') -_CM_ENDIF_RE = re.compile(r'endif\s*\(') -_CM_OPT_RE = re.compile(r'if\s*\(\s*\$?\{?([A-Za-z_]\w*)\}?\s+STREQUAL\s+"?1"?\s*\)') -_CM_PORT_RE = re.compile(r'src/portable/((?:[^/\s]+/)?[^/\s]+)/') -_FALSY = ('', '0', 'off', 'false', 'no') - - -@functools.lru_cache(maxsize=None) -def port_option_gates(repo_root: str) -> dict: - """port dir -> build options that compile it regardless of the board's family - file, e.g. {'analog/max3421': {'MAX3421_HOST'}} from family_support.cmake.""" - gates = {} - try: - text = open(os.path.join(repo_root, 'hw/bsp/family_support.cmake')).read() - except OSError: - return gates - stack = [] # one entry per open if(): its option, or None - for line in text.splitlines(): - line = line.strip() - if _CM_IF_RE.match(line): - m = _CM_OPT_RE.match(line) - stack.append(m.group(1) if m else None) - elif _CM_ELSE_RE.match(line): - if stack: - stack[-1] = None # the guard doesn't hold in this branch - elif _CM_ENDIF_RE.match(line): - if stack: - stack.pop() - opts = {o for o in stack if o} - m = _CM_PORT_RE.search(line) - if opts and m: - gates.setdefault(m.group(1), set()).update(opts) - return gates - - -_CM_SET_RE = re.compile(r'set\s*\(\s*([A-Za-z_]\w*)\s+([^)\s]+)\s*\)') - - -# cached: called per changed portable file x roster board -@functools.lru_cache(maxsize=None) -def bsp_board_options(board_name: str, repo_root: str) -> frozenset: - """Build options a board turns on in its own BSP: `set( )` in - hw/bsp//boards//board.cmake, e.g. MAX3421_HOST on the espressif - and rp2040 max3421 boards. CMake only - HIL CI builds nothing with Make, so a - board.mk-only option (e.g. nrf5340dk's MAX3421_HOST) compiles no port here.""" - fam = board_family(board_name, repo_root) - if not fam: - return frozenset() - path = os.path.join(repo_root, 'hw/bsp', fam, 'boards', board_name, 'board.cmake') - try: - text = open(path).read() - except OSError: - return frozenset() - out = set() - for line in text.splitlines(): - line = line.strip() - if line.startswith('#'): - continue - m = _CM_SET_RE.match(line) - if m and m.group(2).strip('"').lower() not in _FALSY: - out.add(m.group(1)) - return frozenset(out) - - -def board_options(board: dict, repo_root: str) -> set: - """Build options a board has truthy: the roster entry's build.args plus each - variant's defines (NAME=VALUE) and raw CFLAGS (-DNAME=VALUE), plus whatever its - own board.cmake sets (a board can enable a gated port without the roster saying so).""" - toks = list(board.get('build', {}).get('args', [])) - for v in board.get('variant', []): - toks += list(v.get('defines', [])) - toks += v.get('flags', '').split() - out = set(bsp_board_options(board['name'], repo_root)) - for t in toks: - name, _, val = (t[2:] if t.startswith('-D') else t).partition('=') - if name and val.strip().strip('"').lower() not in _FALSY: - out.add(name.strip()) - return out - - -@functools.lru_cache(maxsize=None) -def port_families(port_dir: str, repo_root: str) -> set: - """Board families that compile this src/portable dir. CMake only: HIL CI builds - every board with CMake, so a port wired up in family.mk alone is compiled for no - HIL board and must not select one. family.cmake lists portable sources directly - for most families; espressif instead references them from a nested component - CMakeLists.txt (hw/bsp/espressif/components/tinyusb_src/CMakeLists.txt).""" - fams = set() - bsp_root = os.path.join(repo_root, 'hw/bsp') - # trailing '/' so a port dir is not a prefix of a sibling: bare 'microchip/pic' - # would otherwise match '.../microchip/pic32mz/...' and inherit its families - needle = port_dir + '/' - for f in glob.glob(os.path.join(bsp_root, '*/family.cmake')) + \ - glob.glob(os.path.join(bsp_root, '*/components/*/CMakeLists.txt')): - try: - if needle in open(f).read(): - fam = os.path.relpath(f, bsp_root).split(os.sep, 1)[0] - fams.add(fam) - except OSError: - pass - return fams - - -_CLS_INC_RE = re.compile(r'#\s*include\s*[<"]class/([^/"<>]+)/([^"<>]+)[">]') - - -@functools.lru_cache(maxsize=None) -def class_include_edges(repo_root: str) -> dict: - """'/
' -> the other class dirs that include it. A class header - pulled in by a second class ships in every firmware enabling that second class: - src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, and - net_device.h includes class/cdc/cdc.h. The class rule derives macros from the - directory name alone, so without this edge a change to the included header - selects only its own class's examples - and on a board that skips those (e.g. - metro_m4_express skips audio_test_freertos), nothing at all. - - Derived from the actual #include lines rather than a hand-written table so it - cannot rot when a class picks up or drops a cross-class include.""" - edges = {} - for f in sorted(glob.glob(os.path.join(repo_root, 'src/class/*/*.[ch]'))): - cls = os.path.basename(os.path.dirname(f)) - try: - text = open(f).read() - except OSError: - continue - for inc_cls, inc_hdr in _CLS_INC_RE.findall(text): - if inc_cls != cls: - edges.setdefault(f'{inc_cls}/{inc_hdr}', set()).add(cls) - return edges - - -def class_macros(cls: str, base: str, prefix: str) -> list: - """Config macros that compile a class dir's code, for role prefix TUD/TUH. - `base` refines dfu only (it splits DFU from DFU_RUNTIME per file); pass '' for - a class reached through an include edge, where the widest set is correct.""" - if cls == 'net': - return [f'CFG_{prefix}_{m}' for m in NET_MACROS] - if cls == 'dfu': - if base.startswith('dfu_rt'): - return [f'CFG_{prefix}_DFU_RUNTIME'] - if base.startswith('dfu_device') or base.startswith('dfu_host'): - return [f'CFG_{prefix}_DFU'] - return [f'CFG_{prefix}_DFU', f'CFG_{prefix}_DFU_RUNTIME'] - return [f'CFG_{prefix}_{cls.upper()}'] - - -def _config_enables(cfg_path: str, macros) -> bool: - try: - text = open(cfg_path).read() - except OSError: - return False - return any(re.search(rf'#define\s+{m}\s+\(?\s*0*[1-9]', text) for m in macros) - - -def roster_only_tests(all_boards) -> set: - """Test paths that only appear in a roster board's tests.only list (e.g. - espressif boards), not in the shared device/dual/host_test lists.""" - out = set() - for b in all_boards: - out.update(b.get('tests', {}).get('only', [])) - return out - - -def class_examples(macros, role: str, repo_root: str, extra_tests: set) -> set: - """Tests (from role's + dual lists, plus roster-only-list tests of that role) - whose example config enables any macro.""" - pool = role_tests({role}, extra_tests) - out = set() - for test in pool: - cfg = os.path.join(repo_root, 'examples', test, 'src', 'tusb_config.h') - if _config_enables(cfg, macros): - out.add(test) - return out - - -def role_tests(roles: set, extras: set) -> set: - """Every test for the given role(s): each role's own list + dual tests, - plus roster-only-list tests (extras) matching those roles or 'dual'.""" - pool = set(dual_tests) - for r in roles: - pool |= set(ALL_TESTS[r]) - pool |= {t for t in extras if test_role(t) in roles or test_role(t) == 'dual'} - return pool - - -class _Sel: - """Accumulates contributions. board->set(tests) plus 'all-board' markers.""" - def __init__(self): - self.full = False - self.by_board = {} # name -> set of tests, or 'all' - self.roles = set() # roles touched by any contribution - self.families = set() # bsp families touched (incl. off-rig ones: build-only consumers) - self.reasons = [] - - def add(self, boards, tests, reason): - """tests: 'all' or iterable of test paths.""" - self.reasons.append(reason) - for b in boards: - cur = self.by_board.get(b) - if tests == 'all' or cur == 'all': - self.by_board[b] = 'all' - else: - self.by_board[b] = (cur or set()) | set(tests) - - def force_full(self, reason): - self.full = True - self.reasons.append(reason) - - -def _classify_one(path, repo_root, roster_boards, extras: set, s: _Sel): - base = os.path.basename(path) - if _NONCODE_RE.match(path): - s.reasons.append(f'{path}: non-code, no contribution') - return - if _FULL_RE.match(path): - s.force_full(f'{path}: core/infra -> full matrix') - return - - m = re.match(r'src/portable/((?:[^/]+/)?[^/]+)/', path) - if m: - port = m.group(1) - if re.match(r'(dcd_|.*_device)', base): - roles = {'device'} - elif re.match(r'(hcd_|.*_host)', base): - roles = {'host'} - else: - roles = {'device', 'host'} - fams = port_families(port, repo_root) - if not fams: - # no family references this port: either a new/renamed port dir or a - # family.cmake layout the scan misses - widen instead of contributing nothing - s.force_full(f'{path}: port {port} maps to no board family -> full matrix') - return - s.families.update(fams) - # a board can also pull the port in through a build option (e.g. MAX3421_HOST=1 - # from the roster on metro_m4_express, or from its own board.cmake), which its - # family file never names - gates = port_option_gates(repo_root).get(port, set()) - boards = [b['name'] for b in roster_boards - if (board_family(b['name'], repo_root) in fams or - (gates and board_options(b, repo_root) & gates)) and (board_roles(b) & roles)] - tests = role_tests(roles, extras) - s.roles.update(roles) - why = f'{path}: port {port} -> families {sorted(fams)}' - if gates: - why += f' + option {sorted(gates)}' - s.add(boards, tests, f'{why} -> boards {boards} ({"/".join(sorted(roles))})') - return - - m = re.match(r'src/class/([^/]+)/', path) - if m: - cls = m.group(1) - if re.search(r'_device\.[ch]$', base): - roles = {'device'} - elif re.search(r'_host\.[ch]$', base): - roles = {'host'} - else: - roles = {'device', 'host'} - # this file's own class, plus any class whose headers include it - via = sorted(class_include_edges(repo_root).get(f'{cls}/{base}', ())) - - def macros(prefix): - return (class_macros(cls, base, prefix) + - [m2 for c in via for m2 in class_macros(c, '', prefix)]) - tests = set() - if 'device' in roles: - tests |= class_examples(macros('TUD'), 'device', repo_root, extras) - if 'host' in roles: - tests |= class_examples(macros('TUH'), 'host', repo_root, extras) - boards = [b['name'] for b in roster_boards if board_roles(b) & roles] - s.roles.update(roles) - why = f'{path}: class {cls}' + (f' (+ included by {via})' if via else '') - s.add(boards, tests, f'{why} -> {sorted(tests)} ({"/".join(sorted(roles))})') - return - - m = re.match(r'src/(device|host)/', path) - if m: - role = m.group(1) - boards = [b['name'] for b in roster_boards if role in board_roles(b)] - s.roles.add(role) - s.add(boards, role_tests({role}, extras), f'{path}: core {role} stack -> all {role} tests') - return - - m = re.match(r'hw/bsp/([^/]+)/(?:boards/([^/]+)/)?', path) - if m: - fam, brd = m.group(1), m.group(2) - s.families.add(fam) - if brd: - boards = [b['name'] for b in roster_boards if b['name'] == brd] - why = f'{path}: bsp board {brd}' - else: - boards = [b['name'] for b in roster_boards - if board_family(b['name'], repo_root) == fam] - why = f'{path}: bsp family {fam}' - s.roles.update(('device', 'host')) - s.add(boards, 'all', f'{why} -> boards {boards}') - return - - m = re.match(r'examples/(device|host|dual)/([^/]+)/', path) - if m: - test = f'{m.group(1)}/{m.group(2)}' - known = any(test in pool for pool in ALL_TESTS.values()) or test in extras - if known: - boards = [b['name'] for b in roster_boards] - role = test_role(test) - s.roles.update(('device', 'host') if role == 'dual' else (role,)) - s.add(boards, [test], f'{path}: example -> {test} on all boards') - else: - s.reasons.append(f'{path}: example not in HIL lists, no contribution') - return - - s.force_full(f'{path}: unclassified -> full matrix') - - -def classify(changed_files, repo_root, rosters): - all_boards = [] - seen = set() - for _, boards in rosters: - for b in boards: - if b['name'] not in seen: - seen.add(b['name']) - all_boards.append(b) - - extras = roster_only_tests(all_boards) - s = _Sel() - # no early exit once full: keep classifying so `families` still reports every - # family the diff touches (build-only consumers need it). Nothing after the first - # force_full can change full/boards/args - the full branch below ignores by_board. - for path in changed_files: - _classify_one(path, repo_root, all_boards, extras, s) - - if s.full: - return {'full': True, 'boards': {b['name']: 'all' for b in all_boards}, - 'families': sorted(s.families), 'reasons': s.reasons} - - # role pruning: single-role selections drop the other role's tests and boards - by_name = {b['name']: b for b in all_boards} - out = {} - for name, tests in s.by_board.items(): - allowed = board_tests(by_name[name]) - if tests == 'all': - kept = list(allowed) - else: - kept = [t for t in allowed if t in tests] - if s.roles and s.roles != {'device', 'host'}: - role = next(iter(s.roles)) - kept = [t for t in kept if test_role(t) in (role, 'dual')] - if kept: - out[name] = 'all' if set(kept) == set(allowed) else sorted(kept) - return {'full': False, 'boards': out, 'families': sorted(s.families), - 'reasons': s.reasons} - - -def _board_args(name, chosen) -> list: - parts = [f'-b {name}'] - if chosen != 'all': - parts.append(f'-bt {name}:{",".join(chosen)}') - return parts - - -def selection_args(sel, rosters): - """hil_test.py args per config. Empty means either 'full matrix' or 'nothing - selected' - callers must read sel['full'] to tell them apart.""" - args = {} - for cfg_path, boards in rosters: - parts = [] - if not sel['full']: - for b in boards: - chosen = sel['boards'].get(b['name']) - if chosen is not None: - parts += _board_args(b['name'], chosen) - args[os.path.basename(cfg_path)] = ' '.join(parts) - return args - - -def selection_args_by_flasher(sel, rosters): - """{config: {flasher name: args}}. CI runs one rig as several jobs split by - flasher (esptool vs the rest); each must gate on its own subset, otherwise the - other leg runs a filter matching zero boards and reports a vacuous green.""" - out = {} - for cfg_path, boards in rosters: - per = {} - if not sel['full']: - for b in boards: - chosen = sel['boards'].get(b['name']) - if chosen is None: - continue - per.setdefault(b.get('flasher', {}).get('name', ''), []).extend( - _board_args(b['name'], chosen)) - out[os.path.basename(cfg_path)] = {f: ' '.join(p) for f, p in per.items()} - return out - - -def changed_files_from_git(base, repo_root): - mb = subprocess.run(['git', 'merge-base', 'HEAD', base], cwd=repo_root, - capture_output=True, text=True, check=True).stdout.strip() - diff = subprocess.run(GIT_DIFF_ARGV + [f'{mb}..HEAD'], cwd=repo_root, - capture_output=True, text=True, check=True).stdout - return [l for l in diff.splitlines() if l.strip()] - - -def main(): - ap = argparse.ArgumentParser(description=__doc__) - g = ap.add_mutually_exclusive_group(required=True) - g.add_argument('--base', help='git ref to diff against (merge-base..HEAD)') - g.add_argument('--diff-file', help='newline-separated changed-file list') - ap.add_argument('configs', nargs='+', help='rig roster JSON file(s)') - a = ap.parse_args() - - repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - rosters = [] - for c in a.configs: - with open(c) as f: - rosters.append((c, json.load(f)['boards'])) - - files = (open(a.diff_file).read().splitlines() if a.diff_file - else changed_files_from_git(a.base, repo_root)) - files = [f for f in files if f.strip()] - - s = classify(files, repo_root, rosters) - s['args'] = selection_args(s, rosters) - s['args_flasher'] = selection_args_by_flasher(s, rosters) - for r in s['reasons']: - print(f'hil_select: {r}', file=sys.stderr) - print(json.dumps(s)) - - -if __name__ == '__main__': - main() diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py index 5d9407883..174251343 100755 --- a/test/hil/hil_test.py +++ b/test/hil/hil_test.py @@ -45,7 +45,10 @@ import os import random import re import select +import signal +import shlex import sys +import tempfile import time from contextlib import redirect_stdout from pathlib import Path @@ -53,30 +56,29 @@ from typing import TypedDict, NotRequired, cast import serial import subprocess +import traceback import json import glob import multiprocessing from multiprocessing import TimeoutError as MpTimeoutError +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # PYTHONSAFEPATH drops it import hil_flash -import hil_lock -from hil_examples import device_tests, dual_tests, host_test +from helper import hil_health, hil_lock, hil_util +from helper.hil_util import device_tests, dual_tests, host_test -# Raw Lock/Semaphore objects passed via Pool initargs are inheritable only under the fork -# start method (spawn/forkserver pickle them and fail at Pool creation) — pin it so a -# future interpreter default change cannot break the run at startup. -_mp = multiprocessing.get_context('fork') +# Raw Lock/Semaphore objects in Pool initargs are inheritable only under fork +# (spawn/forkserver pickle them and fail at Pool creation), so pin it against an +# interpreter default change. Windows has no fork: fall back so it still IMPORTS there. + +_mp = multiprocessing.get_context('fork') if os.name != 'nt' else multiprocessing.get_context() Pool, Lock, Semaphore, Manager = _mp.Pool, _mp.Lock, _mp.Semaphore, _mp.Manager -import hashlib -import ctypes -from pymtp import LIBMTP_DeviceEntry, LIBMTP_RawDevice, MTP import string -# Enumeration wait budget. The first attempt gets ENUM_TIMEOUT; retry attempts get the -# shorter ENUM_TIMEOUT_RETRY - the board was just re-flashed again, and a device that is -# going to enumerate shows up within a few seconds, so a failing test costs ~3-5x a -# passing one instead of 10-30x. Per-attempt value is set by test_example(); each pool -# worker is its own process, so a module global is safe. +# Enumeration wait budget: first attempt ENUM_TIMEOUT, retries the shorter +# ENUM_TIMEOUT_RETRY -- a device that will enumerate shows up within seconds, so a failing +# test costs ~3-5x a passing one instead of 10-30x. Set per attempt by test_example(); a +# module global is safe because each pool worker is its own process. ENUM_TIMEOUT = 8 ENUM_TIMEOUT_RETRY = 4 _enum_timeout = ENUM_TIMEOUT @@ -104,26 +106,36 @@ STATUS_OK = "\033[32mOK\033[0m" STATUS_FAILED = "\033[31mFailed\033[0m" STATUS_SKIPPED = "\033[33mSkipped\033[0m" -# Plain (non-ANSI) cell symbols for the markdown matrix report (hil_report.md). -# A missing binary is reported as skipped too. +# Plain (non-ANSI) cell symbols for hil_report.md; a missing binary counts as skipped. REPORT_CELL = {'pass': '✅', 'fail': '❌', 'skip': '⚪'} class TestFail(AssertionError): """Fail a test but still surface a metric string in its report cell (e.g. usbtest's '❌ 29/30' instead of a bare ❌). The cell metric is icon-prefixed so render/tally treat it as a failure.""" - def __init__(self, msg: str, metric: str | None = None): + def __init__(self, msg: str, metric: str | None = None, parsed: bool = False): super().__init__(msg) self.metric = metric + # parsed=True: a real per-case verdict, so a retry would only re-observe it + # (test_example skips the rest). A failure to RUN the tool stays retryable. + self.parsed = parsed verbose = False +# Set when a HUNG usbtest case could not be recovered: the DUT's usbfs node still has a +# D-state holder, so every later flash on that board enumerates into it, blocks, survives +# SIGKILL and becomes another stray. maxtasksperchild=1 gives each board its own worker, +# so this global is board-scoped; test_board resets it anyway. +board_wedged = '' +max_retry = 1 # mirrors argparse's -r default (see main); defined HERE too so + # test_example is callable (and testable) without going through main() PROFILE = os.environ.get('HIL_PROFILE') == '1' # timestamped logs + permit/flash timing + ctrl-map dump test_only = [] board_test = {} skip_flash = False print_lock = None shuffle_seed = None # per-run seed for the per-board test-order shuffle (HIL_SHUFFLE_SEED to replay) +_current_fw = None # firmware test_example resolved for the RUNNING test (set before each test fn) def init_worker(lock, seed, b_mutexes, f_sems, cmap, cmeta, hints_by_uid): @@ -147,13 +159,21 @@ def log_line(msg: str) -> None: def compact_output(raw: str) -> str: if not raw: return '' - lines = [ln.strip() for ln in raw.replace('\r', '\n').split('\n') if ln.strip()] + # Defense in depth (the emitter already suppresses them, see _ci_log_groups): markers + # piped into this capture land mid-row, where GitHub renders them literally. + lines = [] + for ln in raw.replace('\r', '\n').split('\n'): + ln = hil_util.strip_workflow_markers(ln.strip()).strip() + if ln: + lines.append(ln) return ' | '.join(lines) class FlasherCfg(TypedDict): name: str uid: str - args: str + args: NotRequired[str] # stlink entries carry no args + vid_pid: NotRequired[str] # openocd probe pin, verbatim (e.g. "0x2e8a 0x000c") + verify: NotRequired[bool] # openocd read-back verify opt-out (WCH) class AttachedDevCfg(TypedDict, total=False): @@ -197,9 +217,34 @@ class Board(TypedDict): class HilConfig(TypedDict): boards: list[Board] -POOL_TIMEOUT = int(os.getenv('HIL_POOL_TIMEOUT', '4200')) # usbtest batteries are serialized fleet-wide, lengthening the tail -SERIAL_READ_TIMEOUT = float(os.getenv('HIL_SERIAL_READ_TIMEOUT', '5')) -SERIAL_WRITE_TIMEOUT = float(os.getenv('HIL_SERIAL_WRITE_TIMEOUT', '10')) +# Below the CI job ceilings so THIS guard fires first and still writes a report, well +# above a healthy fleet run (~14 min measured), and deliberately generous: firing early +# abandons boards that were still in flight (30 min fired on 5 of the last 8 HIL jobs), +# while firing late costs minutes on an already-wedged run. The drain keeps whatever had +# already finished either way. +POOL_TIMEOUT = hil_util.pos_int_env('HIL_POOL_TIMEOUT', 3600) + + +# Headroom on top of a battery's own budget so ONE HUNG recovery (case timeout, SIGKILL +# wait, bounded reflash, settle) can finish. Only spent when cases actually time out. +USBTEST_RECOVERY_BUDGET = hil_util.pos_int_env('HIL_USBTEST_RECOVERY_BUDGET', 250) +# How long usbtest.py may keep starting new cases (--budget). The outer run_cmd timeout is +# always this PLUS the recovery headroom, never a separate literal, or lowering one eats +# the reserve the recovery needs. 0 is refused (usbtest.py reads it as "no limit"); the +# margin over a healthy battery (~200s) keeps contention from becoming BUDGET entries. +USBTEST_BATTERY_BUDGET = hil_util.pos_int_env('HIL_USBTEST_BATTERY_BUDGET', 260) + +# The battery checks its budget BEFORE dispatching a case, so it can overshoot by one +# already-started case. Our outer kill must sit ABOVE that or we SIGKILL the battery just +# as it goes to print its JSON, turning ~29 real per-case verdicts into "usbtest did not +# run" and re-paying the whole battery on retry. +# Worst case, from usbtest.py: --timeout 60 (the case) + 5s post-SIGKILL reap + +# dmesg_tail(), which is bounded by HELPER_TIMEOUT=30 and runs on BOTH the FAIL and HUNG +# timeout paths = 95s. 120 leaves a margin; 75 (my first estimate, taken before checking +# dmesg_tail) was 20s SHORT and would have killed the battery mid-print. +USBTEST_OVERSHOOT = 120 +SERIAL_READ_TIMEOUT = hil_util.pos_float_env('HIL_SERIAL_READ_TIMEOUT', 5) +SERIAL_WRITE_TIMEOUT = hil_util.pos_float_env('HIL_SERIAL_WRITE_TIMEOUT', 10) MSC_README_TXT = \ @@ -207,7 +252,6 @@ b"This is tinyusb's MassStorage Class demo.\r\n\r\n\ If you find any bugs or get any questions, feel free to file an\r\n\ issue at github.com/hathach/tinyusb" -# get usb disk by id def get_disk_dev(id, vendor_str, lun): return f'/dev/disk/by-id/usb-{vendor_str}_Mass_Storage_{id}-0:{lun}' @@ -235,8 +279,7 @@ def open_serial_dev(port: str): while timeout > 0: if os.path.exists(port): try: - # write_timeout: a wedged device otherwise blocks ser.write() forever, - # hanging the worker until the pool/job timeout kills the whole run + # write_timeout: see serial_write_all ser = serial.Serial(port, baudrate=115200, timeout=SERIAL_READ_TIMEOUT, write_timeout=SERIAL_WRITE_TIMEOUT) break @@ -252,18 +295,43 @@ def open_serial_dev(port: str): def serial_write_all(ser: serial.Serial, data: bytes): - # write_timeout is a total deadline for the whole call (pyserial keeps partial progress - # internally). A timeout means the device stopped draining — treat it as fatal: pyserial - # loses the partial-write count on raise, so retrying would duplicate bytes on the wire. + # write_timeout is a deadline for the whole call. A timeout means the device stopped + # draining, and it is fatal: pyserial loses the partial-write count on raise, so + # retrying would duplicate bytes on the wire. try: ser.write(data) except serial.SerialTimeoutException: raise AssertionError(f'Serial write timeout after {SERIAL_WRITE_TIMEOUT:.1f}s') +LP_OPEN_TIMEOUT = 5 # bound on opening the printer lp node; see test_device_printer_to_cdc +# Runs under hil_util.run_alongside as `python3 -c`. Inline rather than a file so hil_ci.sh's +# staging list does not need another entry to keep the rig working. +LP_READER = ( + 'import os, sys\n' + 'fd = os.open(sys.argv[1], os.O_RDONLY)\n' + # readiness marker: the parent must not send a byte before the node is open, or the + # bytes are lost. A blind sleep raced CPython start-up on a loaded rig. + 'open(sys.argv[3], "w").close()\n' + 'want = int(sys.argv[2])\n' + 'buf = b""\n' + 'while len(buf) < want:\n' + ' chunk = os.read(fd, min(64, want - len(buf)))\n' + ' if not chunk:\n' + ' break\n' + ' buf += chunk\n' + 'sys.stdout.buffer.write(buf)\n' +) +MTYPE_TIMEOUT = 30 # a README-sized read is <1 s; bounds a D-state hang on a wedged device + + def read_disk_file(uid: str, lun: int, fname: str) -> bytes: - # Reads a file from a FAT volume on a block device without mounting it. - # Requires mtools: `apt install mtools` (no pip dependency). + # Reads a file from an unmounted FAT volume; needs mtools. run_cmd everywhere in this + # file rather than subprocess.run/check_output: its post-timeout reap is an unbounded + # communicate() with no killpg (CPython 3.13.5 subprocess.py:558-565 -- kill(), then + # communicate() with NO timeout), which never returns on a device wedged in D state, + # where the kill is queued and never delivered. binary + # keeps the bytes exact, split_stderr keeps mtype warnings out of them. dev = get_disk_dev(uid, 'TinyUSB', lun) last_err = None @@ -271,101 +339,27 @@ def read_disk_file(uid: str, lun: int, fname: str) -> bytes: nonlocal last_err if not os.path.exists(dev): return None - try: - data = subprocess.check_output( - ['mtype', '-i', dev, f'::/{fname}'], stderr=subprocess.PIPE) - assert data, f'Cannot read file {fname} from {dev}' - return data - except subprocess.CalledProcessError as e: - last_err = e.stderr.decode(errors='replace').strip() - return None + r = hil_util.run_cmd(f"mtype -i {shlex.quote(dev)} ::/{shlex.quote(fname)}", + timeout=MTYPE_TIMEOUT, binary=True, split_stderr=True, quiet=True) + if r.returncode == 0: + if r.stdout: + return r.stdout + # rc 0 with no data is an answer (empty file, zeroed sectors), not "not + # ready" — fail now instead of spinning the budget + raise AssertionError(f'Cannot read file {fname} from {dev}: mtype returned no data') + last_err = (r.stderr or b'').decode(errors='replace').strip() or f'mtype rc {r.returncode}' + return None data = wait_until(try_read) if data is None: - raise AssertionError(f'mtype failed on {dev}: {last_err}' if last_err else f'Storage {dev} not existed') + raise AssertionError(f'Cannot read file {fname} from {dev}: {last_err}' if last_err + else f'Storage {dev} not existed') return data -def open_mtp_dev(uid: str): - mtp = MTP() - last_detail = None - deadline = time.monotonic() + 2 * enum_timeout() - - def find_ready_mtp(): - nonlocal last_detail - for marker_name in glob.glob('/dev/libmtp-*'): - marker = Path(marker_name) - serial = '' - try: - # libmtp-runtime publishes libmtp-%k only after its synchronous - # mtp-probe has accepted the device. Starting from that small, ready-only - # set avoids a broad sysfs scan racing unrelated parallel re-enumerations. - sysname = marker.name[len('libmtp-'):] - dev_path = Path('/sys/bus/usb/devices') / sysname - serial = (dev_path / 'serial').read_text().strip() - if (serial.lower() != uid.lower() - or (dev_path / 'idVendor').read_text().strip() != 'cafe' - or (dev_path / 'idProduct').read_text().strip() != '4017'): - continue - - busnum = int((dev_path / 'busnum').read_text()) - devnum = int((dev_path / 'devnum').read_text()) - usb_node = Path('/dev/bus/usb') / f'{busnum:03d}' / f'{devnum:03d}' - if marker.resolve(strict=True) != usb_node or not os.access( - usb_node, os.R_OK | os.W_OK): - last_detail = f'{marker} did not resolve to an accessible {usb_node}' - continue - return busnum, devnum - except (OSError, ValueError) as e: - # A marker can disappear while another board flashes. Only retain - # diagnostics for this board's marker, not unrelated MTP devices. - if serial.lower() == uid.lower(): - last_detail = f'{marker}: {e}' - return None - - def remaining() -> float: - return max(0.0, deadline - time.monotonic()) - - target = wait_until(find_ready_mtp, step=0.05, timeout=remaining()) - if target is None: - detail = f': {last_detail}' if last_detail else '; install libmtp-runtime' - raise AssertionError(f'MTP udev device not ready for {uid}{detail}') - - # A desktop GVFS session may claim MTP after udev probing. This is a no-op on - # headless runners, but preserves support for rigs where the mount exists. - try: - subprocess.run(['gio', 'mount', '-u', f'mtp://TinyUsb_TinyUsb_Device_{uid}/'], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=2) - except (FileNotFoundError, subprocess.TimeoutExpired): - pass - - # GIO can race a disconnect/re-enumeration. Resolve the completed marker again - # rather than opening a stale bus/device tuple. - target = wait_until(find_ready_mtp, step=0.05, timeout=remaining()) - if target is None: - raise AssertionError(f'MTP udev device disappeared for {uid}') - busnum, devnum = target - - # TinyUSB needs no libmtp device quirks. Construct its raw entry directly so - # this test never probes another MTP board that is still being initialized. - entry = LIBMTP_DeviceEntry(None, 0xcafe, None, 0x4017, 0) - raw = LIBMTP_RawDevice(entry, busnum, devnum) - mtp.device = mtp.mtp.LIBMTP_Open_Raw_Device(ctypes.byref(raw)) - if not mtp.device: - raise AssertionError(f'libmtp could not open MTP {uid} at {busnum:03d}/{devnum:03d}') - - try: - serial_raw = mtp.get_serialnumber() - serial = serial_raw.decode('utf-8') if serial_raw else '' - if serial.lower() != uid.lower(): - raise AssertionError(f'MTP serial mismatch at {busnum:03d}/{devnum:03d}: {serial}') - except Exception: - try: - mtp.disconnect() - except Exception: - pass - raise - return mtp +# ~5 KB of transfers plus libmtp setup takes seconds, not minutes; a larger value makes a +# wedged MTP board cost that much on every retry, all charged to the pool guard. +MTP_SESSION_MARGIN = 30 # transfer budget after enumeration; past it the session is killed def get_printer_dev(id: str, vendor_str, product_str, ifnum: int): @@ -374,10 +368,16 @@ def get_printer_dev(id: str, vendor_str, product_str, ifnum: int): product_str = product_str.replace(' ', '_') if product_str else '' for lp in glob.glob('/sys/class/usbmisc/lp*'): try: - sn = open(f'{lp}/device/../serial').read().strip() + # bounded: same device_lock() exposure as the sibling reads (see read_sysfs) + sn = hil_util.read_sysfs(f'{lp}/device/../serial') + # UNKNOWN is not None: the sentinel has no __eq__, so an unanswered read + # would fall through both tests and read as 'not this board' -- the exact + # absence/unknown conflation read_sysfs exists to prevent. + if sn is None or sn is hil_util.SYSFS_UNKNOWN: + continue if sn == id: return f'/dev/usb/{os.path.basename(lp)}' - except (FileNotFoundError, PermissionError, ValueError): + except OSError: # read_sysfs swallows its own OSError/ValueError; glob can race pass return None @@ -389,7 +389,8 @@ def open_printer_dev(id: str, vendor_str, product_str, ifnum: int) -> str: return lp_dev if lp_dev and os.path.exists(lp_dev) else None lp_dev = wait_until(try_find) - assert lp_dev, f'Printer device not found for {id} if{ifnum:02d}' + assert lp_dev, (f'Printer device not found for {id} if{ifnum:02d}' + + hil_util.sysfs_blind_note()) return lp_dev @@ -399,18 +400,16 @@ def open_printer_dev(id: str, vendor_str, product_str, ifnum: int) -> str: def test_dual_host_info_to_device_cdc(board): uid = board['uid'] declared_devs = [f'{d["vid_pid"]}_{d["serial"]}' for d in board['tests']['dev_attached']] - port = hil_flash.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0) + port = hil_util.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0) ser = open_serial_dev(port) ser.timeout = 0.1 - # read until all expected devices are enumerated data = b'' timeout = enum_timeout() while timeout > 0: new_data = ser.read(ser.in_waiting or 1) if new_data: data += new_data - # check if all devices found enum_dev_sn = [] for l in data.decode('utf-8', errors='ignore').splitlines(): vid_pid_sn = re.search(r'ID ([0-9a-fA-F]+):([0-9a-fA-F]+) SN (\w+)', l) @@ -447,7 +446,7 @@ def test_host_device_info(board): flasher = board['flasher'] declared_devs = [f'{d["vid_pid"]}_{d["serial"]}' for d in board['tests']['dev_attached']] - port = hil_flash.get_serial_dev(flasher["uid"], None, None, 0) + port = hil_util.get_serial_dev(flasher["uid"], None, None, 0) ser = open_serial_dev(port) ser.timeout = 0.1 @@ -455,14 +454,12 @@ def test_host_device_info(board): ret = getattr(hil_flash, f'reset_{flasher["name"].lower()}')(board) assert ret.returncode == 0, 'Failed to reset device' - # read until all expected devices are enumerated data = b'' timeout = enum_timeout() while timeout > 0: new_data = ser.read(ser.in_waiting or 1) if new_data: data += new_data - # check if all devices found enum_dev_sn = [] for l in data.decode('utf-8', errors='ignore').splitlines(): vid_pid_sn = re.search(r'ID ([0-9a-fA-F]+):([0-9a-fA-F]+) SN (\w+)', l) @@ -526,7 +523,7 @@ def test_host_cdc_msc_hid(board): if not cdc_devs and not msc_devs: return 'skipped' - port = hil_flash.get_serial_dev(flasher["uid"], None, None, 0) + port = hil_util.get_serial_dev(flasher["uid"], None, None, 0) ser = open_serial_dev(port) ser.timeout = 0.1 @@ -534,7 +531,6 @@ def test_host_cdc_msc_hid(board): ret = getattr(hil_flash, f'reset_{flasher["name"].lower()}')(board) assert ret.returncode == 0, 'Failed to reset device' - # Wait for all expected mount messages data = b'' timeout = enum_timeout() wait_cdc = len(cdc_devs) > 0 @@ -550,7 +546,6 @@ def test_host_cdc_msc_hid(board): time.sleep(0.1) timeout -= 0.1 - # Lookup serial chip name from vid_pid vid_pid_name = { '0403_6001': 'FTDI', '0403_6010': 'FTDI', '0403_6011': 'FTDI', '0403_6014': 'FTDI', '10c4_ea60': 'CP210x', '10c4_ea70': 'CP210x', @@ -561,7 +556,6 @@ def test_host_cdc_msc_hid(board): lines = data.decode('utf-8', errors='ignore').splitlines() - # Verify and print CDC mount if cdc_devs: assert b'CDC Interface is mounted' in data, 'CDC device not mounted on host' dev = cdc_devs[0] @@ -570,7 +564,6 @@ def test_host_cdc_msc_hid(board): if 'CDC Interface is mounted' in l: print(f'\r\n {chip_name}: {l} ', end='') - # Verify and print MSC mount (inquiry + disk size) if msc_devs: assert b'MassStorage device is mounted' in data, 'MSC device not mounted on host' assert b'Disk Size' in data, 'MSC Disk Size not reported' @@ -590,7 +583,6 @@ def test_host_cdc_msc_hid(board): packet_size = 64 - # Echo test: write random 1-packet_size chunks, wait for echo before sending next echo_len = 1024 echo_data = rand_ascii(echo_len) ser.reset_input_buffer() @@ -598,7 +590,6 @@ def test_host_cdc_msc_hid(board): while offset < echo_len: chunk_size = min(random.randint(1, packet_size), echo_len - offset) serial_write_all(ser, echo_data[offset:offset + chunk_size]) - # wait until this chunk is echoed back echo = b'' t_end = time.monotonic() + 1.0 while time.monotonic() < t_end and len(echo) < chunk_size: @@ -619,7 +610,7 @@ def test_host_msc_file_explorer(board): if not msc_devs: return 'skipped' - port = hil_flash.get_serial_dev(flasher["uid"], None, None, 0) + port = hil_util.get_serial_dev(flasher["uid"], None, None, 0) ser = open_serial_dev(port) ser.timeout = 0.1 @@ -627,7 +618,6 @@ def test_host_msc_file_explorer(board): ret = getattr(hil_flash, f'reset_{flasher["name"].lower()}')(board) assert ret.returncode == 0, 'Failed to reset device' - # Wait for MSC mount (Disk Size message) data = b'' timeout = enum_timeout() while timeout > 0: @@ -664,14 +654,12 @@ def test_host_msc_file_explorer(board): if MSC_README_TXT.decode() in resp_text: print('README.TXT matched ', end='') - # MSC throughput test: send dd command to read sectors time.sleep(0.5) ser.reset_input_buffer() for ch in 'dd 1024\r': serial_write_all(ser, ch.encode()) time.sleep(0.002) - # Read dd output until prompt resp = b'' t = 30.0 while t > 0: @@ -706,15 +694,14 @@ def test_host_msc_file_explorer_freertos(board): # Tests: device # ------------------------------------------------------------- def test_device_board_test(board): - # Dummy test pass def test_device_cdc_dual_ports(board): uid = board['uid'] port = [ - hil_flash.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0), - hil_flash.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 2) + hil_util.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0), + hil_util.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 2) ] ser = [open_serial_dev(p) for p in port] @@ -753,7 +740,7 @@ def test_device_cdc_dual_ports(board): def test_device_cdc_msc(board): uid = board['uid'] # CDC Echo test - port = hil_flash.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0) + port = hil_util.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0) ser = open_serial_dev(port) def rand_ascii(length): @@ -782,6 +769,20 @@ def test_device_cdc_msc_freertos(board): test_device_cdc_msc(board) +def link_is_fs(speed) -> bool: + """Payload scaling from a `speed` attribute. Anything not positively read as high speed + counts as FS -- including None and SYSFS_UNKNOWN: the FS payload merely tests an HS + board less, while the HS payload hard-fails a healthy FS board.""" + return speed not in ('480', '5000', '10000') + + +def dd_timeout(mib: float) -> int: + """Bound one dd by what was ASKED for: 2.5 s/MiB is the slowest rate this test has + measured (FS CDC, ~420 kB/s), over a 30 s floor. A flat bound fails a healthy board as + soon as the payload grows or the leaf-hub uplink is shared.""" + return int(30 + 2.5 * mib) + + def test_device_cdc_msc_throughput(board): uid = board['uid'] @@ -792,7 +793,6 @@ def test_device_cdc_msc_throughput(board): return f'{float(m.group(1)):.1f} {m.group(2)}ps' return '?' - # Wait for MSC disk enumeration dev = get_disk_dev(uid, 'TinyUSB', 0) timeout = enum_timeout() while timeout > 0: @@ -801,8 +801,7 @@ def test_device_cdc_msc_throughput(board): time.sleep(0.1); timeout -= 0.1 assert timeout > 0, f'Disk {dev} not found' - # Wait for CDC tty enumeration - tty = hil_flash.get_serial_dev(uid, 'TinyUSB', 'Throughput', 0) + tty = hil_util.get_serial_dev(uid, 'TinyUSB', 'Throughput', 0) timeout = enum_timeout() while timeout > 0: if os.path.exists(tty): @@ -810,41 +809,48 @@ def test_device_cdc_msc_throughput(board): time.sleep(0.1); timeout -= 0.1 assert timeout > 0, f'CDC tty {tty} not found' - # Detect speed (12 Mbps FS / 480 Mbps HS) for payload scaling - is_fs = False - for f in glob.glob('/sys/bus/usb/devices/*/serial'): - try: - if open(f).read().strip().lower() == uid.lower(): - is_fs = (open(os.path.join(os.path.dirname(f), 'speed')).read().strip() == '12') - break - except (OSError, ValueError): - pass + # Detect speed (12 Mbps FS / 480 Mbps HS) for payload scaling; a device we never find + # keeps the FS payload (see link_is_fs) + # usb_scan, not a private glob: it skips root hubs and remembers paths that already + # stranded, so one wedged peer cannot spend this worker's blindness budget four reads + # at a time. + is_fs = True + speed_known = False + devs, _ = hil_util.usb_scan(vid='cafe', serial=uid) + if devs: + speed = hil_util.read_sysfs(os.path.join(devs[0]['dir'], 'speed')) + is_fs = link_is_fs(speed) + speed_known = speed not in (None, hil_util.SYSFS_UNKNOWN) # Put tty in raw mode so dd sees pure binary throughput. - rs = hil_flash.run_cmd(f'timeout 30 stty -F {tty} raw -echo') - assert rs.returncode == 0, f'stty failed: {hil_flash.cmd_stdout_text(rs.stdout)}' + rs = hil_util.run_cmd(f'timeout 30 stty -F {tty} raw -echo') + assert rs.returncode == 0, f'stty failed: {hil_util.cmd_stdout_text(rs.stdout)}' # Payload aim: ~5 s per direction at FS (~830 kB/s), much less at HS. msc_count = 2 if is_fs else 16 # bs=1M cdc_count = 16 if is_fs else 128 # bs=64K tmp_file = f'/tmp/cdc_msc_tp_{uid}.bin' + t_cdc, t_msc = dd_timeout(cdc_count / 16), dd_timeout(msc_count) - rw = hil_flash.run_cmd(f'timeout 30 dd if=/dev/zero of={tty} bs=64K count={cdc_count} 2>&1') - assert rw.returncode == 0, f'CDC dd write failed: {hil_flash.cmd_stdout_text(rw.stdout)}' - cdc_w = parse_speed(hil_flash.cmd_stdout_text(rw.stdout)) + rw = hil_util.run_cmd(f'timeout {t_cdc} dd if=/dev/zero of={tty} bs=64K count={cdc_count} 2>&1') + assert rw.returncode == 0, f'CDC dd write failed: {hil_util.cmd_stdout_text(rw.stdout)}' + cdc_w = parse_speed(hil_util.cmd_stdout_text(rw.stdout)) - rr = hil_flash.run_cmd(f'timeout 30 dd if={tty} of=/dev/null bs=64K count={cdc_count} iflag=fullblock 2>&1') - assert rr.returncode == 0, f'CDC dd read failed: {hil_flash.cmd_stdout_text(rr.stdout)}' - cdc_r = parse_speed(hil_flash.cmd_stdout_text(rr.stdout)) + rr = hil_util.run_cmd(f'timeout {t_cdc} dd if={tty} of=/dev/null bs=64K count={cdc_count} iflag=fullblock 2>&1') + assert rr.returncode == 0, f'CDC dd read failed: {hil_util.cmd_stdout_text(rr.stdout)}' + cdc_r = parse_speed(hil_util.cmd_stdout_text(rr.stdout)) - rmr = hil_flash.run_cmd(f'dd if={dev} of={tmp_file} bs=1M count={msc_count} iflag=direct 2>&1') - assert rmr.returncode == 0, f'MSC dd read failed: {hil_flash.cmd_stdout_text(rmr.stdout)}' - msc_r = parse_speed(hil_flash.cmd_stdout_text(rmr.stdout)) + # inner bound, like the CDC pair above: run_cmd's SIGKILL is merely QUEUED against a + # dd blocked in the block layer on a half-dead device, so without one the call rides + # CMD_TIMEOUT and is abandoned holding the disk and usbfs nodes. + rmr = hil_util.run_cmd(f'timeout {t_msc} dd if={dev} of={tmp_file} bs=1M count={msc_count} iflag=direct 2>&1') + assert rmr.returncode == 0, f'MSC dd read failed: {hil_util.cmd_stdout_text(rmr.stdout)}' + msc_r = parse_speed(hil_util.cmd_stdout_text(rmr.stdout)) - rmw = hil_flash.run_cmd(f'dd if={tmp_file} of={dev} bs=1M count={msc_count} oflag=direct 2>&1') - assert rmw.returncode == 0, f'MSC dd write failed: {hil_flash.cmd_stdout_text(rmw.stdout)}' - msc_w = parse_speed(hil_flash.cmd_stdout_text(rmw.stdout)) + rmw = hil_util.run_cmd(f'timeout {t_msc} dd if={tmp_file} of={dev} bs=1M count={msc_count} oflag=direct 2>&1') + assert rmw.returncode == 0, f'MSC dd write failed: {hil_util.cmd_stdout_text(rmw.stdout)}' + msc_w = parse_speed(hil_util.cmd_stdout_text(rmw.stdout)) try: os.remove(tmp_file) @@ -853,8 +859,7 @@ def test_device_cdc_msc_throughput(board): print(f' CDC read {cdc_r} write {cdc_w}, MSC read {msc_r} write {msc_w} ', end='') - # compact read/write speeds for the report cell, e.g. "✅ C 652/422k M 1.1M/783k" - # (C=CDC, M=MSC; the unit is shown once when both sides share it) + # report cell, e.g. "✅ C 652/422k M 1.1M/783k" (C=CDC, M=MSC; shared unit shown once) def short(s): return (s.split()[0].rstrip('0').rstrip('.') + s.split()[-1][0]) if ' ' in s else s @@ -864,20 +869,29 @@ def test_device_cdc_msc_throughput(board): r = r[:-1] return f'{r}/{w}' - return f'{REPORT_CELL["pass"]} C {pair(cdc_r, cdc_w)} M {pair(msc_r, msc_w)}' + # 'FS?' when the speed could not be read: the numbers below were produced against the FS + # payload, so an HS board reads as suspiciously slow. Say so rather than publish a green + # cell whose scale is a guess. + scale = '' if speed_known else ' FS?' + return f'{REPORT_CELL["pass"]} C {pair(cdc_r, cdc_w)} M {pair(msc_r, msc_w)}{scale}' def test_device_dfu(board): uid = board['uid'] - - # Wait device enum. Deadline-based: dfu-util -l itself takes ~1 s per call, which a - # per-iteration countdown would not charge against the budget. + vid_pid = 'cafe:400b' + + # Deadline-based: dfu-util takes ~1 s per call, which a countdown would not charge + # against the budget. -d pins enumeration to THIS example's ids: a bare `-l` opens every + # DFU-capable node, and one wedged node blocks that open in D state. The pair is doubled + # because dfu-util matches run-time and DFU-mode devices against SEPARATE id pairs + # (parse_vendprod: an omitted DFU-mode pair matches ANY DFU-mode device). The deadline + # is only tested BETWEEN calls, so the per-call bound is what caps a blocked open. deadline = time.monotonic() + enum_timeout() found = False while time.monotonic() < deadline: - ret = hil_flash.run_cmd(f'dfu-util -l') - stdout = hil_flash.cmd_stdout_text(ret.stdout) - if f'serial="{uid}"' in stdout and 'Found DFU: [cafe:400b]' in stdout: + ret = hil_util.run_cmd(f'dfu-util -d {vid_pid},{vid_pid} -l', timeout=15) + stdout = hil_util.cmd_stdout_text(ret.stdout) + if f'serial="{uid}"' in stdout and f'Found DFU: [{vid_pid}]' in stdout: found = True break time.sleep(1) @@ -887,17 +901,23 @@ def test_device_dfu(board): f_dfu0 = f'dfu0_{uid}' f_dfu1 = f'dfu1_{uid}' - # Test upload try: os.remove(f_dfu0) os.remove(f_dfu1) except OSError: pass - ret = hil_flash.run_cmd(f'dfu-util -S {uid} -a 0 -U {f_dfu0}') + # -d as well as -S: dfu-util matches the SERIAL only after libusb_open() (dfu_util.c + # probes the descriptor for iSerialNumber), so -S alone still opens every DFU-capable + # node. The id filter runs BEFORE the open; -S then picks our board (see the poll). + # Each partition is one short string, so a healthy upload is ~1 s; the bound is there + # for a node that stops answering mid-transfer. + ret = hil_util.run_cmd(f'dfu-util -d {vid_pid},{vid_pid} -S {uid} -a 0 -U {f_dfu0}', + timeout=30) assert ret.returncode == 0, 'Upload failed' - ret = hil_flash.run_cmd(f'dfu-util -S {uid} -a 1 -U {f_dfu1}') + ret = hil_util.run_cmd(f'dfu-util -d {vid_pid},{vid_pid} -S {uid} -a 1 -U {f_dfu1}', + timeout=30) assert ret.returncode == 0, 'Upload failed' with open(f_dfu0) as f: @@ -912,13 +932,14 @@ def test_device_dfu(board): def test_device_dfu_runtime(board): uid = board['uid'] - # Wait device enum (deadline-based, see test_device_dfu) + vid_pid = 'cafe:400c' + # enumeration pinned to this example's ids, same per-call bound (see test_device_dfu) deadline = time.monotonic() + enum_timeout() found = False while time.monotonic() < deadline: - ret = hil_flash.run_cmd(f'dfu-util -l') - stdout = hil_flash.cmd_stdout_text(ret.stdout) - if f'serial="{uid}"' in stdout and 'Found Runtime: [cafe:400c]' in stdout: + ret = hil_util.run_cmd(f'dfu-util -d {vid_pid},{vid_pid} -l', timeout=15) + stdout = hil_util.cmd_stdout_text(ret.stdout) + if f'serial="{uid}"' in stdout and f'Found Runtime: [{vid_pid}]' in stdout: found = True break time.sleep(1) @@ -931,7 +952,6 @@ def test_device_hid_boot_interface(board): kbd = get_hid_dev(uid, 'TinyUSB', 'TinyUSB_Device', 'event-kbd') mouse1 = get_hid_dev(uid, 'TinyUSB', 'TinyUSB_Device', 'if01-event-mouse') mouse2 = get_hid_dev(uid, 'TinyUSB', 'TinyUSB_Device', 'if01-mouse') - # Wait device enum timeout = enum_timeout() while timeout > 0: if os.path.exists(kbd) and os.path.exists(mouse1) and os.path.exists(mouse2): @@ -948,12 +968,9 @@ def test_device_hid_composite_freertos(id): def test_device_printer_to_cdc(board): - import threading - uid = board['uid'] - # Wait for CDC port and printer device - cdc_port = hil_flash.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0) + cdc_port = hil_util.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0) ser = open_serial_dev(cdc_port) lp_dev = open_printer_dev(uid, 'TinyUSB', 'TinyUSB_Device', 2) @@ -973,7 +990,6 @@ def test_device_printer_to_cdc(board): sizes = [32, 64, 128, 256, 512, random.randint(2000, 5000)] - # flush any stale data ser.reset_input_buffer() # Test 1: Printer -> CDC with multiple sizes, write in random 1-64 byte chunks @@ -983,7 +999,17 @@ def test_device_printer_to_cdc(board): ser.reset_input_buffer() rd = b'' offset = 0 - lp_fd = os.open(lp_dev, os.O_WRONLY | os.O_NONBLOCK) + # bounded: O_NONBLOCK does NOT save us -- usblp_open() takes the device mutex + # first -- and this open runs on the worker itself, with no thread to abandon + lp_fd = hil_util.bounded_open(lp_dev, os.O_WRONLY | os.O_NONBLOCK, 5) + # Three-valued on purpose: an OSError here is a FACT about the node (EBUSY from + # usblp's single-opener rule, ENOENT from a re-enumeration race, EACCES from a + # udev gap) and must not be reported as a wedge -- that sends the operator to + # usb-kernel-recover for hardware that is fine. + assert lp_fd is not hil_util.SYSFS_UNKNOWN, ( + f'printer: opening {lp_dev} for write blocked (device wedged)' + f'{hil_util.sysfs_blind_note()}') + assert lp_fd is not None, f'printer: {lp_dev} could not be opened for write' try: while offset < size: chunk_size = min(random.randint(1, 64), size - offset) @@ -1007,128 +1033,88 @@ def test_device_printer_to_cdc(board): assert rd == test_data, (f'Printer->CDC wrong data ({size} bytes):\n' f' expected: {test_data[:64]}\n received: {rd[:64]}') - # Test 2: CDC -> Printer with multiple sizes, write in random 1-64 byte chunks - # Use a thread to read from printer since /dev/usb/lp read blocks + # Test 2: CDC -> Printer with multiple sizes, write in random 1-64 byte chunks. + # The lp read runs in a PROCESS, not a thread: /dev/usb/lp* blocks on read, usblp + # allows a SINGLE opener, and a blocked thread cannot be abandoned without keeping + # that fd -- which poisoned the node for every later test this worker ran. A killed + # process takes its fd with it. ser.reset_input_buffer() time.sleep(0.5) for size in sizes: test_data = rand_ascii(size) - rd_result = [b'', None] # [data, error] - reader_ready = threading.Event() - - def lp_reader(): - try: - rd = b'' - fd = os.open(lp_dev, os.O_RDONLY) - reader_ready.set() - try: - while len(rd) < size: - chunk = os.read(fd, min(64, size - len(rd))) - if not chunk: - break - rd += chunk - finally: - os.close(fd) - rd_result[0] = rd - except Exception as e: - rd_result[1] = e - reader_ready.set() - reader = threading.Thread(target=lp_reader, daemon=True) - reader.start() - # wait for reader to open lp device before writing - reader_ready.wait(timeout=5) - time.sleep(0.1) - - # Write to CDC in small chunks with flush to avoid overflowing device FIFO - offset = 0 - while offset < size: - chunk_size = min(random.randint(1, 64), size - offset) - serial_write_all(ser, test_data[offset:offset + chunk_size]) - time.sleep(0.01) - offset += chunk_size + ready = Path(tempfile.gettempdir()) / f'hil-lp-ready-{os.getpid()}-{size}' + ready.unlink(missing_ok=True) + + def write_cdc(): + # WAIT for the reader to have the node open. The child has to fork, exec and + # boot a CPython interpreter; on a loaded rig that routinely exceeds the 0.3s + # this used to sleep, and every byte sent early is lost -- surfacing as a + # spurious data mismatch rather than a timeout. + deadline = time.monotonic() + LP_OPEN_TIMEOUT + 5 + while not ready.exists(): + if time.monotonic() > deadline: + return # reader never opened; the rc/compare below reports it + time.sleep(0.02) + offset = 0 + while offset < size: + chunk_size = min(random.randint(1, 64), size - offset) + serial_write_all(ser, test_data[offset:offset + chunk_size]) + time.sleep(0.01) + offset += chunk_size - reader.join(timeout=10) - assert not reader.is_alive(), f'CDC->Printer timeout ({size} bytes)' - assert rd_result[1] is None, f'CDC->Printer read error: {rd_result[1]}' - assert rd_result[0] == test_data, (f'CDC->Printer wrong data ({size} bytes):\n' - f' expected: {test_data[:64]}\n received: {rd_result[0][:64]}') + try: + r = hil_util.run_alongside( + [sys.executable, '-c', LP_READER, lp_dev, str(size), str(ready)], + write_cdc, LP_OPEN_TIMEOUT + 12) + finally: + ready.unlink(missing_ok=True) + # stderr, not stdout: run_alongside keeps the payload stream clean, so a traceback + # from the reader now arrives on its own pipe + assert r.returncode == 0, (f'CDC->Printer reader failed ({size} bytes, rc ' + f'{r.returncode}): {hil_util.cmd_stdout_text(r.stderr)[:200]}') + assert r.stdout == test_data, (f'CDC->Printer wrong data ({size} bytes):\n' + f' expected: {test_data[:64]}\n received: {r.stdout[:64]}') time.sleep(0.2) ser.close() def test_device_mtp(board): + # The whole session lives in mtp_test.py under run_cmd: libmtp calls are synchronous + # ctypes that block unkillably (D state) on a wedged device, so a disposable process is + # the only thing the harness can walk away from. uid = board['uid'] - - # --- BEFORE: mute C-level stderr for libmtp vid/pid warnings --- - fd = sys.stderr.fileno() - _saved = os.dup(fd) - _null = os.open(os.devnull, os.O_WRONLY) - os.dup2(_null, fd) - - try: - mtp = open_mtp_dev(uid) - finally: - # --- AFTER: restore stderr --- - os.dup2(_saved, fd) - os.close(_null) - os.close(_saved) - - try: - assert b"TinyUSB" == mtp.get_manufacturer(), 'MTP wrong manufacturer' - assert b"MTP Example" == mtp.get_modelname(), 'MTP wrong model' - assert b'1.0' == mtp.get_deviceversion(), 'MTP wrong version' - assert b'TinyUSB MTP' == mtp.get_devicename(), 'MTP wrong device name' - - # read and compare readme.txt and logo.png - f1_expect = b'TinyUSB MTP Filesystem example' - f2_md5_expect = '40ef23fc2891018d41a05d4a0d5f822f' # md5sum of logo.png - f1 = uid.encode("utf-8") + b'_file1' - f2 = uid.encode("utf-8") + b'_file2' - f3 = uid.encode("utf-8") + b'_file3' - mtp.get_file_to_file(1, f1) - with open(f1, 'rb') as file: - f1_data = file.read() - os.remove(f1) - assert f1_data == f1_expect, 'MTP file1 wrong data' - mtp.get_file_to_file(2, f2) - with open(f2, 'rb') as file: - f2_data = file.read() - os.remove(f2) - assert f2_md5_expect == hashlib.md5(f2_data).hexdigest(), 'MTP file2 wrong data' - # test send file - with open(f3, "wb") as file: - # 1524-byte payload + 12-byte MTP header = 3 full 512-byte buffers. - # This exercises delivery of the final OUT payload before its ZLP. - f3_data = bytes((i % 251) + 1 for i in range(1524)) - file.write(f3_data) - file.close() - fid = mtp.send_file_from_file(f3, b'file3') - f3_readback = f3 + b'_readback' - mtp.get_file_to_file(fid, f3_readback) - with open(f3_readback, 'rb') as f: - f3_rb_data = f.read() - os.remove(f3_readback) - assert f3_rb_data == f3_data, 'MTP file3 wrong data' - os.remove(f3) - mtp.delete_object(fid) - finally: - mtp.disconnect() + script = Path(__file__).resolve().parent / 'mtp_test.py' + # 2x, as master's in-process open_mtp_dev used: libmtp-runtime publishes + # /dev/libmtp-* only after its SYNCHRONOUS mtp-probe finishes, seconds on a freshly + # flashed FS board, and the gio unmount eats part of what is left before the first + # probe. Extracting the session into a subprocess halved this by accident (8s/4s), + # which fails healthy hardware on the retry. + t = 2 * enum_timeout() + r = hil_util.run_cmd( + f'{shlex.quote(sys.executable)} {shlex.quote(str(script))} --uid {shlex.quote(uid)} --timeout {t}', + timeout=t + MTP_SESSION_MARGIN) + if r.returncode == 124: + # "abandoned", not "killed": a session blocked in a usbfs ioctl (D state) never + # receives the SIGKILL -- it lingers until its device path clears, by design + raise AssertionError(f'MTP session wedged (abandoned after {t + MTP_SESSION_MARGIN}s; ' + f'the session process may linger unkillable in D state)') + assert r.returncode == 0, f'MTP session failed (rc {r.returncode}):\n{r.stdout}' def test_device_net_lwip_webserver(board): # MAC hard-coded in examples/device/net_lwip_webserver/src/main.c; Linux names the - # USB network interface enx. Device IP is 192.168.7.1 and - # the example runs an iperf2 TCP server on port 5001 (INCLUDE_IPERF). + # iface enx. Device IP 192.168.7.1, iperf2 TCP server on 5001 + # (INCLUDE_IPERF). import socket mac_no_colons = '0202846a9600' iface = 'enx' + mac_no_colons device_ip = '192.168.7.1' iperf_port = 5001 - # Wait for the host to get an IPv4 address in the device's subnet (DHCP served by the device). - # USB enum + DHCP serve can take longer on the CI HIL hardware than on local — give it 30s. + # Wait for an IPv4 address in the device's subnet (it serves DHCP); 30s because USB + # enum + DHCP serve is slower on the CI HIL hardware than locally. iface_timeout = 30 deadline = time.monotonic() + iface_timeout host_ip = None @@ -1142,8 +1128,7 @@ def test_device_net_lwip_webserver(board): time.sleep(0.5) assert host_ip, f'USB net iface {iface} did not come up with 192.168.7.x within {iface_timeout}s' - # Poll the iperf TCP port until the device is accepting. The net stack comes up a bit - # after DHCP completes; iperf server binding isn't instantaneous after reflash. + # Poll until the device accepts: the net stack and the iperf bind come up after DHCP. deadline = time.monotonic() + enum_timeout() last_err = None while time.monotonic() < deadline: @@ -1156,12 +1141,12 @@ def test_device_net_lwip_webserver(board): time.sleep(0.3) assert last_err is None, f'iperf TCP {device_ip}:{iperf_port} not accepting within {enum_timeout()}s: {last_err}' - # Throughput: 5-second iperf2 TCP test, CSV output for stable parsing. - # iperf2 CSV final summary line: timestamp,src_ip,src_port,dst_ip,dst_port,id,interval,bytes,bps - ret = subprocess.run(['iperf', '-c', device_ip, '-t', '5', '-y', 'C'], - capture_output=True, text=True, timeout=30) - stderr = ret.stderr.strip() - stdout = ret.stdout.strip() + # 5-second iperf2 TCP test; -y C for stable parsing (final summary line is + # timestamp,src_ip,src_port,dst_ip,dst_port,id,interval,bytes,bps). + ret = hil_util.run_cmd(f'iperf -c {device_ip} -t 5 -y C', + timeout=30, split_stderr=True, quiet=True) + stderr = (ret.stderr or '').strip() + stdout = (ret.stdout or '').strip() assert ret.returncode == 0, f'iperf rc={ret.returncode}: stderr={stderr!r} stdout={stdout!r}' lines = [l for l in stdout.splitlines() if l] assert lines, f'iperf produced no output (rc={ret.returncode}, stderr={stderr!r})' @@ -1172,19 +1157,16 @@ def test_device_net_lwip_webserver(board): mbps = bps / 1e6 print(f' iperf {mbps:5.1f} Mbps', end='') - # Reject implausibly low throughput - a working USB-net link should clear this easily. assert mbps >= 1.0, f'iperf throughput too low: {mbps:.2f} Mbps' def test_device_msc_dual_lun(board): uid = board['uid'] - # Read README from LUN 0 data0 = read_disk_file(uid, 0, 'README0.TXT') readme0 = b"LUN0: " + MSC_README_TXT assert data0 == readme0, f'MSC LUN0 wrong data in README0.TXT\n expected: {readme0}\n received: {data0}' - # Read README from LUN 1 data1 = read_disk_file(uid, 1, 'README1.TXT') readme1 = b"LUN1: " + MSC_README_TXT assert data1 == readme1, f'MSC LUN1 wrong data in README1.TXT\n expected: {readme1}\n received: {data1}' @@ -1193,7 +1175,6 @@ def test_device_msc_dual_lun(board): def test_device_midi_test(board): uid = board['uid'] - # Find MIDI device via /dev/snd/by-id using board UID timeout = enum_timeout() midi_port = None while timeout > 0: @@ -1211,7 +1192,6 @@ def test_device_midi_test(board): timeout -= 1 assert midi_port is not None, f'MIDI device not found for {uid}' - # Read MIDI messages and verify note on/off import select midi_fd = os.open(midi_port, os.O_RDONLY | os.O_NONBLOCK) try: @@ -1246,7 +1226,6 @@ def test_device_midi_test(board): i += 1 assert len(notes) >= 2, f'Expected at least 2 MIDI notes, got {len(notes)}' - # Verify notes are from the expected sequence note_sequence = [ 74, 78, 81, 86, 90, 93, 98, 102, 57, 61, 66, 69, 73, 78, 81, 85, 88, 92, 97, 100, 97, 92, 88, 85, 81, 78, 74, 69, 66, 62, 57, 62, @@ -1287,8 +1266,11 @@ def test_device_audio_test_freertos(board): raw_path, ] - ret = subprocess.run(cmd, capture_output=True, text=True, timeout=20) - assert ret.returncode == 0, f'arecord failed: {ret.stderr.strip() or ret.stdout.strip()}' + # run_cmd: ALSA capture from a wedged device blocks in D state (see read_disk_file) + ret = hil_util.run_cmd(' '.join(shlex.quote(c) for c in cmd), + timeout=20, split_stderr=True, quiet=True) + assert ret.returncode == 0, \ + f'arecord failed: {(ret.stderr or "").strip() or (ret.stdout or "").strip()}' try: with open(raw_path, 'rb') as f: @@ -1322,7 +1304,6 @@ def test_device_hid_generic_inout(board): uid = board['uid'] import hid # cython-hidapi (pip: hidapi, apt: python3-hid) - # Find HID device by UID (VID=0xCafe) timeout = enum_timeout() dev = None while timeout > 0: @@ -1339,7 +1320,6 @@ def test_device_hid_generic_inout(board): h = hid.device() h.open(dev['vendor_id'], dev['product_id'], uid) try: - # Echo test: send random data and verify echo for size in [8, 32, 63]: # Report ID (0) + payload, padded to 64 bytes payload = bytes([random.randint(1, 255) for _ in range(size)]) @@ -1356,63 +1336,177 @@ def test_device_hid_generic_inout(board): def test_device_usbtest(board): - # Run the Linux testusb tier-4 battery (test/hil/usbtest.py) against the enumerated cafe:4010 - # device; surface the pass count in the report cell ("✅ 30/30", or "❌ 29/30" on a partial). + global board_wedged + # Runs test/hil/usbtest.py against the cafe:4010 device; the pass count goes in the + # report cell ("✅ 30/30", or "❌ 29/30" on a partial). uid = board['uid'] def usbtest_enumerated(): - # match VID:PID too, not just the serial: right after flashing, the previous example's - # enumeration (same serial, different PID) can linger and would fail usbtest.py's lookup - for f in glob.glob('/sys/bus/usb/devices/*/serial'): - d = os.path.dirname(f) - try: - if (open(f).read().strip().lower() == uid.lower() - and open(os.path.join(d, 'idVendor')).read().strip() == 'cafe' - and open(os.path.join(d, 'idProduct')).read().strip() == '4010'): - return True - except OSError: - pass - return False + """True, False, or None when a bounded read did not answer -- absence unproven.""" + # vid_pid FIRST: right after flashing, the previous example's enumeration (same + # serial, different PID) can linger and would fail usbtest.py's lookup -- and + # filtering on the two lock-free descriptor fields rules out every other device + # on the bus before the one read that can block. usb_scan memoises paths that + # already stranded, so one wedged peer cannot spend the blindness budget here. + devs, unknown = hil_util.usb_scan(vid_pid=('cafe', '4010'), serial=uid) + if devs: + return True + return None if unknown else False end = time.monotonic() + enum_timeout() - while time.monotonic() < end and not usbtest_enumerated(): + seen = usbtest_enumerated() + while time.monotonic() < end and seen is not True: time.sleep(0.2) + seen = usbtest_enumerated() # fail before usbtest_permit: an absent device would otherwise queue on the battery # mutex for minutes behind real batteries just to have usbtest.py report "no device" - if not usbtest_enumerated(): + if seen is not True: # 0/30 rather than a bare cell: the battery never ran (30 = standard case count) - raise TestFail(f'no cafe:4010 device with serial {uid}', - metric=f'{REPORT_CELL["fail"]} 0/30') - # settle: right after flashing the enumeration can bounce once (and on dual-port parts like - # CH32V307 the other port's stale usbtest node — same serial and PID — lingers a moment); - # running testusb into that gap sees the device drop mid-case + raise TestFail( + f'no cafe:4010 device with serial {uid}' if seen is False else + f'cannot tell whether cafe:4010 {uid} is present: the bounded sysfs reads did ' + f'not answer{hil_util.sysfs_blind_note()}', + metric=f'{REPORT_CELL["fail"]} 0/30') + # settle: right after flashing the enumeration can bounce once (and on dual-port parts + # the other port's stale node — same serial and PID — lingers), and testusb run into + # that gap sees the device drop mid-case time.sleep(3) # --keep-binding is required for concurrent batteries: usbtest.py's cleanup unbinds - # EVERY usbtest-bound interface (releasing stale same-PID grabs), which would kill a - # peer battery mid-run under USBTEST_PARALLEL > 1; the unbind path has also wedged a - # host xHCI (usb_hcd_alloc_bandwidth) on this rig. Leaving bindings is harmless with - # unique example PIDs - the next example re-enumerates under a different PID and binds - # its normal driver. usbtest_permit budgets USBTEST_PARALLEL batteries per controller. + # EVERY usbtest-bound interface, killing a peer battery under USBTEST_PARALLEL > 1, and + # that unbind path has also wedged a host xHCI (usb_hcd_alloc_bandwidth) here. Harmless + # to leave: the next example enumerates under a different PID. script = Path(__file__).resolve().parent / 'usbtest.py' - cmd = f'python3 "{script}" --serial "{uid}" --json --keep-binding --timeout 60' + # --budget makes the battery a real bound: repeated case timeouts (a FAIL, not a HUNG, + # so the battery keeps going) can otherwise spend the whole outer timeout inside the + # case loop, leaving the recovery below nothing. + cmd = (f'{shlex.quote(sys.executable)} {shlex.quote(str(script))} ' + f'--serial {shlex.quote(uid)} --json --keep-binding ' + f'--timeout 60 --budget {USBTEST_BATTERY_BUDGET}') + # Post-hang recovery reflashes the DUT through its own probe, NEVER a root-port cycle + # (one board reached instead of every fixture under the port; see usb-kernel-recover). + # _current_fw is the artifact test_example flashed for THIS test: re-deriving it from + # board['name'] reflashes the wrong build on variant-only boards. --outer-timeout lets + # usbtest skip a reflash it cannot finish before our run_cmd kill, which would orphan + # the flasher (own session) on the probe. Never under --skip-flash -- and say so: a + # HUNG case then holds the DUT's usbfs lock for the rest of the run, and a probe reset + # is no substitute (the DWC2 pullup survives a core halt). + # ...and only when this flasher can DELIVER that reflash past a poisoned node + # (hil_flash.convoy_safe). Otherwise the flags cost twice: the delivery adds a SECOND + # stray, and the board reserves recovery budget for a path that cannot fire. + # The RECOVERY flasher, which may be the roster's optional `flasher_recover` rather + # than the primary -- a jlink/stlink board can name an openocd entry that reaches the + # same probe convoy-safely without changing how the board is normally flashed. + _rec_flasher = hil_flash.recover_flasher(board) + recovery = bool(_current_fw and not skip_flash and hil_flash.convoy_safe(_rec_flasher)) + # ONE bound, computed here and used for BOTH the child's --outer-timeout and our own + # run_cmd kill below. Three separate expressions disagreed: --skip-flash appended no + # --outer-timeout at all (usbtest reads 0 as "no limit"), and the no-recovery branch + # narrowed only the CHILD's view while run_cmd still waited the full reserve -- so a + # board that cannot recover held a pool worker AND its battery permit idle for + # USBTEST_RECOVERY_BUDGET it had no way to spend, under a usbtest width of 2. + outer = USBTEST_BATTERY_BUDGET + (USBTEST_RECOVERY_BUDGET if recovery + else USBTEST_OVERSHOOT) + if _current_fw and skip_flash: + print('note: --skip-flash disables usbtest hang recovery; a HUNG case will leave ' + 'the device wedged until it is reflashed', flush=True) + elif _current_fw and not recovery: + print(f'note: {_rec_flasher["name"]} cannot deliver a reflash past a poisoned ' + f'usbfs node, so usbtest hang recovery is disabled for {board["name"]}; a ' + f'HUNG case will leave it wedged for the rest of the run', flush=True) + if recovery: + # ship the RECOVERY flasher as `flasher`: usbtest.py, recovery_steps and + # convoy_safe all read board['flasher'], so substituting here keeps the entire + # child side unaware that a second roster entry exists + rb = json.dumps({'name': board['name'], 'flasher': _rec_flasher}) + cmd += f' --recover-board {shlex.quote(rb)} --recover-fw {shlex.quote(_current_fw)}' + cmd += f' --outer-timeout {outer}' + # The reserve above USBTEST_BATTERY_BUDGET exists because the battery can overrun by + # one already-started case, and a hang there needs room for the recovery (whose reflash + # is bounded by usbtest.RECOVER_FLASH_TIMEOUT, not HIL_CMD_TIMEOUT). Without it run_cmd + # SIGKILLs usbtest.py mid-recovery, losing the JSON and the diagnosis. with hil_lock.usbtest_permit(uid): - r = hil_flash.run_cmd(cmd, timeout=200) - out = hil_flash.cmd_stdout_text(r.stdout) + # split_stderr: the battery's final JSON is parsed from stdout, and stderr is the + # only detail left when the outer timeout kills the battery before it prints + r = hil_util.run_cmd(cmd, timeout=outer, split_stderr=True) + out = hil_util.cmd_stdout_text(r.stdout) brace = out.find('{') try: + # brace < 0 would slice from the END ('...rc 0' -> '0' -> int 0, whose subscript + # raises TypeError outside the tuple below and loses the diagnosis) + if brace < 0: + raise ValueError('no JSON object on stdout') data = json.loads(out[brace:]) passed, failed = int(data['passed']), int(data['failed']) - except (ValueError, KeyError, json.JSONDecodeError): - raise TestFail(f'usbtest did not run: {compact_output(out) or hil_flash.cmd_stdout_text(r.stderr)}', + except (ValueError, KeyError, TypeError, json.JSONDecodeError): + # compact BOTH, never `or`: a battery SIGKILLed mid-print leaves a truthy JSON + # fragment on stdout, so an `or` drops the stderr that explains the failure + parts = [compact_output(hil_util.cmd_stdout_text(r.stderr)), compact_output(out)] + detail = ' | '.join(p for p in parts if p) + # Retryable even on rc 124 (run_cmd's outer kill), though the retry re-pays the + # whole budget: 124 only says the timer expired, which a healthy battery can hit + # under load, and test_example REFLASHES before each attempt. Where usbtest's + # in-band recovery is off (--skip-flash, a flasher failing convoy_safe, a terminal + # wedge) that reflash is the only thing left to unpoison the DUT for the boards + # that share its controller. + # No JSON to read the verdict from, so fall back to the text: a battery SIGKILLed + # mid-hang still says HUNG on stdout, and this raise happens BEFORE the latch below + # -- which is why the outer-timeout case, the likeliest real wedge, never latched. + if 'HUNG' in out: + board_wedged = (f'{board["name"]}: usbtest reported a hang and was killed ' + f'before it could report a verdict') + raise TestFail(f'usbtest did not run: {detail}', metric=f'{REPORT_CELL["fail"]} 0/30') - total = passed + failed - if failed == 0 and total > 0: + # A HUNG case that recovery could not clear leaves a D-state holder on this board's + # usbfs node. Latch it: the remaining examples would each flash THROUGH that node, + # block, survive SIGKILL and add another stray -- turning one wedge into one stray per + # remaining example, which is the convoy this branch exists to contain. + # The battery's OWN verdict first: `recovery` only says the flags were passed, not that + # the reflash worked, so a convoy-safe board whose recovery failed used to come back + # unlatched and flash every remaining example through the poisoned node. + if data.get('wedged') or (not recovery and 'HUNG' in out): + # _rec_flasher, NOT board['flasher']: recovery was decided against recover_flasher() + # at the top of this function, and the two diverge as soon as a roster carries the + # optional `flasher_recover` key -- naming the wrong one sends the operator to the + # wrong probe. The wording stays on what usbtest actually reported ("still wedged"), + # because unrecovered_hang is also set by the ambiguous/inconclusive aborts, where + # nothing hung and the old text was false on both clauses. + board_wedged = (f'{board["name"]}: usbtest reports the device still wedged ' + + (f'after a recovery reflash via {_rec_flasher["name"]}' if recovery + else f'and {_rec_flasher["name"]} cannot deliver a recovery reflash')) + + # notrun counts toward the denominator but is NOT a failure: listing cases that never + # ran as failures sends a maintainer bisecting one of them. + notrun = int(data.get('notrun', 0)) + total = passed + failed + notrun + if board_wedged and failed == 0 and notrun == 0: + # Every case passed and the device STILL wedged -- usbtest's inconclusive/ambiguous + # abort fires after the last case, so nothing back-fills a BUDGET entry. Reporting + # the pass would exit 0 with a D-state holder on the rig and the board absent from + # the re-run spec. parsed=True: a retry re-pays the whole battery to re-observe a + # wedge, and flashes through the poisoned node to do it. + raise TestFail(f'usbtest {passed}/{total} but the device wedged ({board_wedged})', + metric=f'{REPORT_CELL["fail"]} {passed}/{total}', parsed=True) + if failed == 0 and notrun == 0 and total > 0: return f'{REPORT_CELL["pass"]} {passed}/{total}' - bad = [c.get('num') for c in data.get('cases', []) if c.get('status') != 'PASS'] - raise TestFail(f'usbtest {passed}/{total} (cases failed: {bad})', - metric=f'{REPORT_CELL["fail"]} {passed}/{total}') + bad = [c.get('num') for c in data.get('cases', []) + if c.get('status') not in ('PASS', 'BUDGET')] + why = f'usbtest {passed}/{total}' + if bad: + why += f' (cases failed: {bad})' + if notrun: + # the reason is per BUDGET entry: a hang or a device drop also aborts the battery, + # and blaming the budget points the maintainer at the wrong thing + reasons = {c.get('detail', '') for c in data.get('cases', []) + if c.get('status') == 'BUDGET'} + reason = (reasons.pop().replace('not run: ', '') if len(reasons) == 1 + else 'the battery stopped early') + why += f'; {notrun} case(s) never ran ({reason}), so this says nothing about them' + # parsed ONLY when every case ran: an aborted battery (budget expiry, kernel hang, bus + # drop) leaves BUDGET entries, and those are exactly what a reflash retry can fix. + raise TestFail(why, metric=f'{REPORT_CELL["fail"]} {passed}/{total}', + parsed=(notrun == 0)) # ------------------------------------------------------------- @@ -1437,42 +1531,68 @@ def test_example(board: Board, variant: str, example: str) -> tuple[int, str, st test_name = f'{variant:40} {example:30} ...' - # --skip-flash runs whatever is already on the board, so any build counts as present: - # only the flashing path needs the artifact this board's flasher actually consumes. - # Filtering there too would skip the test as "no binary" over an extension it never uses. + # --skip-flash runs whatever is already on the board, so any build counts as present; + # filtering by flasher there would skip the test over an extension it never uses. fw_name = hil_flash.find_firmware(variant, example, flasher=None if skip_flash else board['flasher']['name']) if fw_name is None: log_line(f'{test_name} Skip (no binary)') return 0, 'skip', None + # usbtest's hang recovery reflashes the exact artifact under test; re-deriving it from + # board['name'] breaks on variant-only boards + global _current_fw + _current_fw = str(fw_name) if verbose: log_line(f'Firmware {fw_name}') - # flash firmware (unless --skip-flash), then run the test. Both may fail randomly, - # retry a few times. global _enum_timeout start_s = time.time() flash_ok = True last_err = '' last_detail = '' + wedge_break = False for i in range(max_retry): + if board_wedged and i: + # The latch is set MID-attempt (a HUNG usbtest whose flasher cannot recover), + # so test_board's check between tests is too late for THIS test's own retries: + # every further attempt re-flashes into the D-state-held node, blocks, survives + # SIGKILL and leaves another stray. The wedge is not something a retry can fix. + log_line(f'{test_name} not retrying: {board_wedged}') + # COUNT it. Breaking out here skips the i == max_retry - 1 branch that would + # have incremented err_count, so the board rendered a red cell, contributed 0 + # to the exit status and was omitted from the re-run spec -- a rig left with a + # D-state holder published under sys.exit(0). Latent at CI's --retry 1, live + # for every local run and for the workflows that pass no -r. + wedge_break = True + break _enum_timeout = ENUM_TIMEOUT if i == 0 else ENUM_TIMEOUT_RETRY attempt_out = io.StringIO() with redirect_stdout(attempt_out): if not skip_flash: with hil_lock.flash_permit(board['uid']): t_flash = time.monotonic() - ret = getattr(hil_flash, f'flash_{board["flasher"]["name"].lower()}')(board, str(fw_name)) + try: + ret = getattr(hil_flash, + f'flash_{board["flasher"]["name"].lower()}')(board, str(fw_name)) + except Exception as e: + # A flasher that RAISES (esptool's get_serial_dev when the adapter + # drops off the bus, a missing config.env, an unwritable CWD) would + # propagate out of the worker and abort the whole drain, costing + # every board still in flight. + print(f'flash raised: {type(e).__name__}: {e}', flush=True) + ret = subprocess.CompletedProcess(args='flash', returncode=1, + stdout=f'{type(e).__name__}: {e}') if PROFILE: log_line(f'[prof] {variant} {example} flash attempt {i + 1}: ' f'{time.monotonic() - t_flash:.1f}s rc={ret.returncode}') flash_ok = (ret.returncode == 0) - # A wedged RP2040/RP2350 DAP answers nothing and the probe has no reset - # line, so the retry would fail identically; POR it via the Rescue DP - # first. No-op for every other board and every other flash failure. - if not flash_ok and i + 1 < max_retry and \ - hil_flash.rescue_openocd(board, hil_flash.cmd_stdout_text(ret.stdout)): + # A wedged RP2040/RP2350 DAP answers nothing and the probe has no + # reset line, so the retry fails identically; POR it via the Rescue DP + # first (no-op otherwise). NOT gated on a remaining attempt: CI HIL jobs + # run --retry 1, and this leaves the DAP POR'd for the jobs that follow. + if not flash_ok and \ + hil_flash.rescue_openocd(board, hil_util.cmd_stdout_text(ret.stdout)): log_line(f'{variant} {example}: DAP wedged, rescued via Rescue DP') if flash_ok: try: @@ -1484,7 +1604,6 @@ def test_example(board: Board, variant: str, example: str) -> tuple[int, str, st else: status = STATUS_OK result_status = 'pass' - # a test may return a string to show in its report cell (e.g. speed) metric = tret if isinstance(tret, str) else None msg = f'{test_name} {status}' if last_detail: @@ -1495,9 +1614,20 @@ def test_example(board: Board, variant: str, example: str) -> tuple[int, str, st except Exception as e: last_err = str(e) last_detail = compact_output(attempt_out.getvalue()) + if getattr(e, 'parsed', False): + # a PARSED per-case result (usbtest's "29/30"): retrying re-pays + # the whole battery, inside the fleet's usbtest permit, to + # re-observe a number the JSON already reported. Only that case. + err_count += 1 + metric = getattr(e, 'metric', None) + msg = f'{test_name} {STATUS_FAILED}: {e}' + if last_detail: + msg += f' {last_detail}' + msg += f' in {time.time() - start_s:.1f}s' + log_line(msg) + break if i == max_retry - 1: err_count += 1 - # a failing test may still carry a metric to show in its cell (e.g. "❌ 29/30") metric = getattr(e, 'metric', None) msg = f'{test_name} {STATUS_FAILED}: {e}' if last_detail: @@ -1530,13 +1660,21 @@ def test_example(board: Board, variant: str, example: str) -> tuple[int, str, st msg += f' in {time.time() - start_s:.1f}s' log_line(msg) + if wedge_break and not err_count: + # ONE error for the test, never two: a board that also failed to flash has already + # been counted just above. Without this the test returns 0 -- red cell, clean exit + # status, absent from the re-run spec. + err_count += 1 return err_count, result_status, metric def build_board(board: Board) -> tuple[str, int]: """Build firmware for this board via tools/build.py. Honors board config's variant list and build.args defines. - Output goes to cmake-build/cmake-build-/ (tools/build.py layout).""" + Output goes to cmake-build/cmake-build-/ (tools/build.py layout). + + Unbounded on purpose: --build is a local convenience (no CI workflow passes it), so + the developer watching the build is the timeout.""" name = board['name'] bcfg = cast(BuildCfg, board.get('build', {})) extra_defs = bcfg.get('args', []) @@ -1544,7 +1682,7 @@ def build_board(board: Board) -> tuple[str, int]: failed = 0 for v in variants: - cmd = [sys.executable, str(hil_flash.TINYUSB_ROOT / 'tools' / 'build.py'), '-b', name] + cmd = [sys.executable, str(hil_util.TINYUSB_ROOT / 'tools' / 'build.py'), '-b', name] for d in extra_defs: cmd += ['-D', d] if v['name'] != name: @@ -1556,8 +1694,19 @@ def build_board(board: Board) -> tuple[str, int]: if verbose: cmd.append('-v') print(f' + {" ".join(cmd)}') - r = subprocess.run(cmd, cwd=hil_flash.TINYUSB_ROOT) - if r.returncode != 0: + # stdio is inherited so the build STREAMS: a silent buffer is + # indistinguishable from a stall. + proc = subprocess.Popen(cmd, cwd=hil_util.TINYUSB_ROOT, start_new_session=True) + try: + rc = proc.wait() + except KeyboardInterrupt: + # start_new_session means the build never saw the terminal's SIGINT + try: + os.killpg(proc.pid, signal.SIGKILL) + except OSError: + proc.kill() + raise + if rc != 0: failed += 1 return name, failed @@ -1567,28 +1716,29 @@ BOUNDARY_CELL = 'same-PID boundary' def test_board(board: Board) -> tuple[str, int, list[str], list, float]: + swept = False name = board['name'] flasher = board['flasher'] + global board_wedged + board_wedged = '' try: _lock_fh = hil_lock.acquire_board_lock(name) except RuntimeError as e: log_line(f'{name:25} {STATUS_FAILED}: {e}') - # visible report row so the ❌ matches the exit code; failed-tests stays - # empty so a re-run repeats the whole board (no bogus -bt test filter) + # visible report row so the ❌ matches the exit code; failed-tests stays empty so a + # re-run repeats the whole board (no bogus -bt filter) return name, 1, [], [(name, {'board-locked': 'fail'}, None)], 0.0 # after the lock: flock wait behind a concurrent run is not board cost t_board = time.monotonic() try: - # default to all tests test_list = [] if name in board_test: test_list = board_test[name] elif len(test_only) > 0: - # Explicit -t: filter against the board's capabilities so a device-only - # board doesn't try to run host/dual tests (the test functions need a - # `dev_attached` entry in the board config that won't exist). + # Explicit -t: filter against the board's capabilities, or a device-only board + # runs host/dual tests whose `dev_attached` config entry does not exist. board_tests = board.get('tests', {}) if 'only' in board_tests: allowed = set(board_tests['only']) @@ -1618,7 +1768,7 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: err_count = 0 failed_tests = [] board_wide_fail = False # re-run the whole board, not a subset of its tests - rows = [] # list of (row_label, {example: status}, duration) — one row per build variant + rows = [] # list of (row_label, {example: status}, duration) — one per build variant # a -t/-bt filtered run times only a subset; report no duration so an accumulate # re-run keeps the previous full-run value partial = bool(test_only) or name in board_test @@ -1627,11 +1777,10 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: prev_last = None # last test of the previous variant: the variant boundary is an adjacency too for v in variants: vname = v['name'] - # Shuffle each (board, variant)'s run order — de-synchronizes the worker pool so - # usbtest batteries and flash churn spread across the timeline instead of convoying, - # and surfaces order-dependent bugs. Seeded for replay (HIL_SHUFFLE_SEED, logged by - # main). Unique per-example PIDs make any two different examples re-enumerate; only - # the variant boundary can repeat the same example (same PID) — swap it away. + # Shuffle each (board, variant)'s run order: spreads batteries and flash churn + # across the timeline instead of convoying, and surfaces order-dependent bugs. + # Seeded for replay (HIL_SHUFFLE_SEED). Unique per-example PIDs re-enumerate + # between examples; only the variant boundary can repeat one. run_list = list(test_list) if shuffle_seed is not None and len(run_list) > 1: random.Random(f'{shuffle_seed}:{name}:{vname}').shuffle(run_list) @@ -1639,23 +1788,33 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: run_list[0], run_list[-1] = run_list[-1], run_list[0] cells = {} if run_list and run_list[0] == prev_last and not skip_flash: - # Same example (same PID) still repeats across the boundary: a one-test - # list (the common case for a -bt scoped run) leaves nothing to swap - # with. Park on board_test first - it disables the board's USB, so the - # PID goes away and the next flash must re-enumerate to be seen. + # Same example (same PID) still repeats across the boundary (a one-test + # -bt run has nothing to swap with). Park on board_test first: it disables + # the board's USB, so the next flash must re-enumerate to be seen. t_park = time.monotonic() - park_ec, park_status, _ = test_example(board, vname, 'device/board_test') + # _should_park, same as the teardown park: this is attempt 0, so + # test_example's retry guard does not stop it flashing into a poisoned node + park_ec, park_status, _ = ( + test_example(board, vname, 'device/board_test') if _should_park(skip_flash) + else (0, 'skip', None)) if park_ec or park_status == 'skip': - # Boundary not cleared: the previous variant's device may still be - # enumerated under the same PID, so this variant's tests could pass - # against its firmware. Skip them - a false green proves nothing and - # is worse than a gap - and record the boundary itself as the failure - # (a visible ❌ cell, mirroring the board-lock row above) so the report - # matches the exit code instead of rendering all-green. - why = 'no board_test binary' if park_status == 'skip' else 'park flash failed' + # Boundary not cleared: the previous variant may still be enumerated + # under the same PID, so this variant's tests could pass against ITS + # firmware. Skip them and record the boundary as the failure, so the + # report matches the exit code instead of rendering all-green. + # A 'skip' here has two very different causes: no board_test build, or + # _should_park refusing to flash a WEDGED board. Reporting the latter as + # a missing binary sends the operator hunting a build that exists. + wedge_skip = park_status == 'skip' and bool(board_wedged) + why = ('the board is wedged' if wedge_skip else + 'no board_test binary' if park_status == 'skip' else + 'park flash failed') log_line(f'{vname:40} {"same-PID boundary":30} {STATUS_FAILED}: not cleared ({why}); ' f'skipping {len(run_list)} test(s) on this variant') - err_count += 1 + # the wedge already charged its own error through test_device_usbtest; + # charging again would double-count one incident in the exit code + if not wedge_skip: + err_count += 1 cells[BOUNDARY_CELL] = 'fail' # blaming run_list[0] would re-run an innocent test that then passes, # leaving the boundary unretested; re-run the whole board instead @@ -1668,31 +1827,72 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: prev_last = run_list[-1] t_variant = time.monotonic() for test in run_list: + if board_wedged: + # Do NOT flash through a poisoned node: each attempt enumerates into + # it, blocks uninterruptibly and leaves another stray behind. Report + # the skip so the cell is not mistaken for a pass. + cells[test] = f'{REPORT_CELL["skip"]} board wedged' + # ...and re-run the WHOLE board, like the boundary-failure path above: + # these tests never executed, so naming them individually in the .failed + # spec is not enough -- an --accumulate re-run that fixes only the wedged + # test would merge a green cell over it and leave these skips standing + # from the earlier attempt, forever, under a green job. + board_wide_fail = True + continue ec, status, metric = test_example(board, vname, test) err_count += ec cells[test] = metric if metric else status if ec > 0: failed_tests.append(test) + if board_wedged: + log_line(f'{vname:40} SKIPPING the rest of this board: {board_wedged}; ' + f'flashing through the poisoned node would add a stray per test') dur = f'{time.monotonic() - t_variant:.0f}s' if run_list and not partial else None rows.append((vname, cells, dur)) - # board duration excludes the teardown park-flash below; a partial (filtered) - # run reports 0.0 so it never overwrites a cached full-run duration + # excludes the teardown park-flash below; a partial (filtered) run reports 0.0 so + # it never overwrites a cached full-run duration t_total = 0.0 if partial else time.monotonic() - t_board - # flash board_test last to disable board's usb (skipped when --skip-flash is set); - # this is teardown/park, not a test — not recorded in the report - if not skip_flash: + # park: flash board_test last to disable the board's usb; teardown, not a test, + # so it is not recorded in the report. + # + # NOT on a wedged board: the latch has just skipped every remaining test precisely + # because flashing through a D-state-held node blocks, survives SIGKILL and leaves + # a stray -- and this park is a flash like any other. test_example's own guard does + # not stop it (that one only suppresses RETRIES, and this is attempt 0), so the + # containment path would add the very stray it exists to prevent. + if _should_park(skip_flash): test_example(board, variants[0]['name'], 'device/board_test') - return name, err_count, [] if board_wide_fail else sorted(set(failed_tests)), rows, t_total + # Sweep HERE, not in main()'s finally: maxtasksperchild=1 retires this process as + # soon as it returns, reparenting anything it spawned to init and off the pool's + # ppid tree, so the main-side sweep walks fresh idle workers and finds nothing. + # Measured: 4 tasks, zero overlap, sweep 0, all 4 strays alive. + stray = hil_health.kill_own_children() + swept = True + + # LAST fields: whether this worker ran out of bounded-read budget, and what it could + # not kill. Only the worker can answer either -- the blindness latch is + # process-global and this is a separate process -- and the result tuple already + # crosses back, so no Manager round-trip. + return (name, err_count, [] if board_wide_fail else sorted(set(failed_tests)), + rows, t_total, hil_util.sysfs_blind(), stray) finally: + # A raise skips the sweep above, and maxtasksperchild=1 retires this process + # immediately afterwards -- reparenting its flasher to init and erasing the ppid + # link, so main's sweep cannot see it either. The count cannot reach the report on + # this path (there is no result tuple), but the KILL still frees the probe. + if not swept: + try: + hil_health.kill_own_children() + except Exception as se: # noqa: BLE001 - never mask the original failure + print(f'warning: stray sweep failed: {type(se).__name__}: {se}', flush=True) if _lock_fh: try: - # clear our pid record before dropping the flock: this worker - # process lives on (pool reuse), so a stale record would make - # hil_lock.py's pid-liveness checks report a freed board as - # still locked for the rest of the run + # clear our pid record before dropping the flock: this worker process + # lives on (pool reuse), so a stale record would make hil_lock's + # pid-liveness checks report a freed board as locked for the rest of the run _lock_fh.truncate(0) except OSError: pass @@ -1701,10 +1901,9 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: REPORT_MD = 'hil_report.md' REPORT_JSON = 'hil_report.json' -# controller hints learned from previous runs: uid -> {'name', 'pci', 'duration'}. Only -# 'pci' is consumed (dispatch order and first-flash budgeting, never battery -# serialization); name/duration are informational. PCI addresses are boot-stable (bus -# numbers are not), so the cache survives reboots and only goes stale on re-cabling. +# controller hints from previous runs: uid -> {'name', 'pci', 'duration'}. Only 'pci' is +# consumed (dispatch order and first-flash budgeting, never battery serialization). PCI +# addresses are boot-stable, so the cache survives reboots and goes stale on re-cabling. CONTROLLER_CACHE = Path.home() / '.cache' / 'tinyusb-hil' / 'controller_cache.json' @@ -1729,8 +1928,8 @@ def render_matrix(rows_all: list) -> str: if not seen: return 'No tests were run.' - # metric-bearing columns pinned first (usbtest score, throughput, explorer read speed), - # the rest alphabetical by bare test name: stable regardless of the (shuffled) execution order + # metric-bearing columns pinned first, the rest alphabetical: stable regardless of the + # shuffled execution order pinned = ['usbtest', 'cdc_msc_throughput', 'msc_file_explorer', 'msc_file_explorer_freertos'] def col_key(t): @@ -1761,9 +1960,8 @@ def render_matrix(rows_all: list) -> str: sep = '| ' + '-' * board_w + ' | ' + ' | '.join(':' + '-' * (w - 2) + ':' for w in col_w) + ' |' body = [line(lbl, vals) for lbl, vals in rows_vals] - # tally run cells (blank/not-run cells are absent from the dicts). A cell is a bare status - # ('pass'/'fail'/'skip') or a metric string that carries its own icon (e.g. "❌ 29/30" is a - # fail, "✅ 30/30" / "✅ CDC …" a pass), so classify by the leading icon. + # tally run cells (not-run cells are absent from the dicts). A cell is a bare status or + # a metric string carrying its own icon ("❌ 29/30"), so classify by the leading icon. def cell_kind(v): if v == 'fail' or (isinstance(v, str) and v.startswith(REPORT_CELL['fail'])): return 'fail' @@ -1780,7 +1978,119 @@ def render_matrix(rows_all: list) -> str: return summary + '\n\n' + '\n'.join([header, sep] + body) -def accumulate_report(mret: list, report_dir: Path, fresh: bool, scope: str = '') -> str: +def _write_failed_spec(failed_fname: Path, report_dir: Path, mret: list) -> None: + """Re-run spec: only the failed boards (-b), each restricted to its own failed tests + (-bt); a board with failures but no test list re-runs entirely. + + Shared with the pool-guard path, which feeds it the boards that never reported. That + path used to leave this unwritten -- and a fresh run has already unlinked it -- so + build.yml's "Get re-run spec" step found nothing and the GitHub re-run repeated the + whole fleet to find the one board that wedged.""" + parts = ['--accumulate'] + for name, err, fts, *_ in mret: + if err > 0: + parts.append(f'-b {name}') + if fts: + parts.append(f'-bt {name}:{",".join(fts)}') + if len(parts) > 1: # build-only failures have no boards to re-run + report_dir.mkdir(parents=True, exist_ok=True) + with failed_fname.open('w') as f: + f.write(' '.join(parts)) + else: + failed_fname.unlink(missing_ok=True) + + +class PoolDrainTimeout(MpTimeoutError): + """Guard expiry, carrying the rows that DID finish. + + They ride on the exception because the raise is the containment path: losing them here + is what map_async did, and what the drain exists to stop. + """ + + def __init__(self, finished: list): + super().__init__() + self.finished = finished + + +def drain_pool(it, boards: list, deadline: float, out: list | None = None) -> list: + """Collect imap_unordered results against ONE deadline. Returns the finished rows. + + Raises PoolDrainTimeout (carrying those same rows) when the deadline passes with boards + still in flight -- the caller keeps them, names only what is missing, and writes a + re-run spec covering just those. + + A function, not an inline loop, so the tests can call THIS instead of a copy of it: the + loop's previous test built its own ThreadPool and its own drain and asserted on those, + so deleting the real one outright kept the suite green. + """ + # `out` is the CALLER's list: a worker that raises something other than a timeout + # (get_serial_dev on a dropped adapter, a Manager EOFError) propagates bare, and a + # local accumulator would take every finished board with it -- the exact loss the + # drain replaced map_async to prevent. + mret: list = out if out is not None else [] + for _ in boards: + left = deadline - time.monotonic() + if left <= 0: + raise PoolDrainTimeout(mret) + try: + mret.append(it.next(timeout=left)) + except MpTimeoutError: + raise PoolDrainTimeout(mret) from None + return mret + + +def _should_park(skip_flash: bool) -> bool: + """Flash the teardown park (device/board_test, to switch the DUT's USB off)? + + Not on a wedged board. The latch has just skipped every remaining test precisely + because flashing through a D-state-held node blocks, survives SIGKILL and leaves a + stray -- and the park is a flash like any other. test_example's own guard does not stop + it either: that one only suppresses RETRIES, and the park is always attempt 0. So the + containment path would end by adding the very stray it exists to prevent. + """ + return not skip_flash and not board_wedged + + +def _stray_note(mret: list) -> str: + """Name the strays the workers could not kill, for the report banner. + + Summed from the result tuples rather than computed in main()'s finally: that finally + runs AFTER accumulate_report on both abort paths, so a banner appended there was + written to a variable nobody read again. + """ + dirty = [(r[0], r[6]) for r in mret if len(r) > 6 and r[6]] + if not dirty: + return '' + total = sum(n for _, n in dirty) + return (f'> **Rig dirty.** {total} process(es) survived SIGKILL and still hold a probe ' + f'or usbfs node into the next job: ' + f'{", ".join(f"{b} ({n})" for b, n in dirty)}.\n') + + +def _blind_note(mret: list) -> str: + """Name the boards whose worker went blind, for the report banner. + + A blind worker answers SYSFS_UNKNOWN for every attribute, so its "device not found" is + "could not tell". That already reaches the log and the per-cell failure text, but the + TABLE is what gets quoted -- and a red cell there is read as a broken board. Seen live + (run 31794359407): four workers blind, several cells red because of it, and a report + that said nothing. + + Per-board, not global: maxtasksperchild=1 gives every board a fresh worker, so a board + that ran on a healthy one is not smeared by a neighbour's wedge. Rows synthesised by + the timeout path are 5 fields wide and have nothing to report. + """ + blind = [r[0] for r in mret if len(r) > 5 and r[5]] + if not blind: + return '' + return (f'> **Not all verdicts are evidence.** {len(blind)} board(s) ran on a worker ' + f'that went blind on sysfs -- too many bounded reads stranded on a wedged ' + f'device -- so "not found" from them means "could not tell": ' + f'{", ".join(blind)}. See the usb-kernel-recover skill.\n') + + +def accumulate_report(mret: list, report_dir: Path, fresh: bool, scope: str = '', + banner: str = '') -> str: """Merge this run's results into hil_report.json in report_dir, then (re)write the markdown matrix to hil_report.md. `fresh` (a first run, no --accumulate) starts a new report; otherwise a re-run accumulates so boards/tests that @@ -1788,29 +2098,34 @@ def accumulate_report(mret: list, report_dir: Path, fresh: bool, scope: str = '' board filter, if any, so a scoped table is not mistaken for a full one. Returns the md.""" acc = {} # ordered {row_label: [cells dict, duration str|None]} + prior_banner = '' jpath = report_dir / REPORT_JSON if not fresh and jpath.is_file(): try: saved = json.loads(jpath.read_text()) - # CI keys the report dir by run id, so the sidecar can only have been - # written by an earlier attempt of the same run + # CI keys the report dir by run id, so the sidecar is from an earlier attempt for entry in saved.get('rows', []): acc[entry['board']] = [dict(entry['cells']), entry.get('duration')] + # ... and so is the caveat those cells were collected under. A rerun on a rig + # that has since recovered contributes no banner, and the .failed spec reruns + # only FAILURES -- so the earlier attempt's passes are never re-earned and + # would be published as clean results of a rig that was not. + prior_banner = saved.get('banner', '') except (ValueError, KeyError, TypeError): pass # corrupt/old sidecar: start fresh - # merge this run: current cells override prior for boards/tests that ran; a filtered - # run reports duration None, keeping the previous full-run value - for name, _, _, rows, _ in mret: + # current cells override prior for boards/tests that ran; a filtered run reports + # duration None, keeping the previous full-run value + for name, _, _, rows, *_ in mret: if rows and not any('board-locked' in cells for _, cells, _ in rows): - # board ran for real this time: clear a stale lock-failure cell - # (its row is keyed by board name; test rows may be variant names) + # board ran for real: clear a stale lock-failure cell (its row is keyed by + # board name; test rows may be variant names) stale = acc.get(name) if stale is not None: stale[0].pop('board-locked', None) if not stale[0]: - # variant-keyed boards never repopulate the board-name row — - # drop it or it renders as a blank ghost row + # variant-keyed boards never repopulate the board-name row, so drop it + # or it renders as a blank ghost row del acc[name] for row_label, cells, dur in rows: row = acc.setdefault(row_label, [{}, None]) @@ -1823,18 +2138,100 @@ def accumulate_report(mret: list, report_dir: Path, fresh: bool, scope: str = '' row[1] = dur report_dir.mkdir(parents=True, exist_ok=True) + # by LINE, deduped: attempts repeat the same caveat far more often than they add a new + # one, and three copies of the D-state note reads as three incidents + seen, merged = set(), [] + for line in (prior_banner + banner).splitlines(): + if line.strip() and line not in seen: + seen.add(line) + merged.append(line) + banner = '\n'.join(merged) + '\n' if merged else '' jpath.write_text(json.dumps({'rows': [{'board': k, 'cells': c, 'duration': d} - for k, (c, d) in acc.items()]}, indent=2) + '\n') + for k, (c, d) in acc.items()], + 'banner': banner}, indent=2) + '\n') md = render_matrix([(k, c, d) for k, (c, d) in acc.items()]) if scope: - # a scoped run's small table is otherwise indistinguishable from a full one, - # and it replaces the previous full table in the sticky PR comment + # a scoped run's small table is otherwise indistinguishable from a full one, and + # it replaces the previous full table in the sticky PR comment md = f'_Scoped run: {scope}. Boards/tests not listed were not run._\n\n' + md + # LAST, so it is outermost: a rig-health caveat outranks the table AND the scope note, + # and the top of the report is where hil/SKILL.md tells the agent to look for it. + if banner: + md = banner + '\n' + md (report_dir / REPORT_MD).write_text(md + '\n', encoding='utf-8') return md +# containment paths print through hil_health._p: stdout may already be a dead pipe (a +# dropped ssh session), and a BrokenPipeError there would skip os._exit +_p = hil_health._p + + +def _abandon_exit(pool, mgr, abandoned: bool, err_count: int, + report: Path | None = None) -> None: + """Free the runner when the pool could not be shut down. Returns only if not abandoned. + + Must run even while an exception is propagating: multiprocessing's atexit handler + SIGTERMs its daemon workers (ignored in uninterruptible sleep) and then join()s them + with NO timeout, so an abandoned pool plus any raise between the pool's finally and + here hangs the interpreter until the job ceiling kills it. Reproduced: rc=124 at 25s + with SIGTERM-ignoring workers standing in for D state.""" + if not abandoned: + return + try: + if sys.exc_info()[0] is not None: + # os._exit below discards the traceback, and this is often the only place the + # real failure would ever be printed + traceback.print_exc() + except OSError: + pass + # Word this on evidence: shutdown_pool also returns False when terminate() RAISES, and + # a live worker after terminate() is what distinguishes a wedge from a harness bug. + # Count WORKERS only -- _pool_procs appends the Manager, our own healthy child, so + # including it made n >= 1 always and the harness-error branch unreachable. It is killed + # separately: os._exit skips its finalizer, and orphaned it holds the runner's stdout. + n = hil_health.kill_pool_children(pool) + hil_health.kill_pool_children(None, mgr) + if n: + _p(f'HIL worker pool would not terminate ({n} worker(s) still live, ' + f'uninterruptible); SIGKILLed them and abandoned the rest to free the ' + f'runner. Boards held by any leaked worker stay locked until the host is ' + f'power-cycled.', flush=True) + else: + _p('HIL worker pool shutdown failed but left no live worker behind, so this is ' + 'a harness error rather than a wedged rig -- see the Pool.terminate() ' + 'warning above. Exiting early anyway to free the runner; no board should ' + 'stay locked.', flush=True) + # A report already written by accumulate_report says nothing about the abandon, and a + # green table under a red job is how an agent ends up pasting it as this run's result. + # Prepend the caveat; best-effort, never at the cost of exiting. + if report is not None: + try: + if report.exists(): + # utf-8 explicitly (the cells are ✅/❌/⚪) and catch ValueError too: a torn + # report or a LANG=C locale raises UnicodeDecodeError -- NOT an OSError -- + # straight past os._exit, stranding the runner. + body = report.read_text(encoding='utf-8', errors='replace') + # Only when no banner is there yet, searched anywhere in the head rather + # than at char 0: write_timeout_report's banner must stay FIRST (its table + # is a PREVIOUS attempt's) and it puts the rig-health quote above itself. + if '**HIL run ab' not in body[:2000]: + report.write_text( + '**HIL run abandoned: the worker pool would not shut down.** The ' + 'table below was collected before the abandon; treat board ' + 'results as unverified.\n\n' + body, encoding='utf-8') + except (OSError, ValueError): + pass + try: + sys.stdout.flush() + except OSError: + pass + # Clamped: os._exit takes a status byte, so err_count == 256 would truncate to 0 and + # report a failing, abandoned run as green. + os._exit(min(err_count, 125) if err_count else 1) + + def main() -> None: """ Hardware test on specified boards @@ -1864,14 +2261,21 @@ def main() -> None: help='Per-board test list as BOARD:test1,test2 (overrides -t for that board); repeat for multiple boards') parser.add_argument('-B', '--build-dir', default='cmake-build', help='Build folder name (default: cmake-build)') parser.add_argument('--build', action='store_true', help='Build firmware for selected boards with cmake before running tests') - parser.add_argument('-r', '--retry', type=int, default=3, help='Retry count for failed tests (default: 3)') + # default 1, not 3: the pool guard is a FLAT 3600s that does not scale with max_retry, + # and one usbtest test at default 3 can burn 1530s of it (510s outer x3) for a single + # board. Every CI caller already pins --retry 1; the bare invocations in the hil skill + # and hil-validate.js run against the same one-slot rig and used to inherit 3. + parser.add_argument('-r', '--retry', type=int, default=1, help='Retry count for failed tests (default: 1)') parser.add_argument('-v', '--verbose', action='store_true', help='Verbose output') args = parser.parse_args() + if args.retry < 1: + # 0 would make every test loop body never run: all-red cells, exit 0 + parser.error('--retry must be >= 1') config_file = Path(args.config_file) boards = args.board verbose = args.verbose - hil_flash.verbose = args.verbose + hil_util.verbose = args.verbose test_only = args.test_only for entry in args.board_test: bname, _, tnames = entry.partition(':') @@ -1899,6 +2303,32 @@ def main() -> None: config_boards = [e for e in config['boards'] if e['name'] in boards] config_boards = [e for e in config_boards if e['flasher']['name'] not in args.exclude_flasher and (not args.flasher or e['flasher']['name'] in args.flasher)] + if not config_boards: + # same reason the unknown -b board exits 1: 'No tests were run.' with rc 0 reads as + # a green HIL leg, so a roster edit emptying a leg's filter stops testing silently + msg = (f'No boards left after the flasher filter (--flasher ' + f'{args.flasher or "-"}, --exclude-flasher {args.exclude_flasher or "-"})') + print(msg, flush=True) + # loud AND leaving evidence: exiting with no report at all lets the PR comment + # keep the previous push's stale table under a red job + try: + rd = Path(os.environ.get('HIL_REPORT_DIR', '.')) + rd.mkdir(parents=True, exist_ok=True) + (rd / REPORT_MD).write_text(f'**HIL run selected no boards.** {msg}\n', + encoding='utf-8') + except OSError: + pass + sys.exit(1) + + + # Before the build: the probe needs nothing from it, and the annotation is more useful + # early than after a multi-board cmake build has been paid for. + # One line, not a probe: a D-state pid at start-up is a hint for whoever reads a red + # cell, never a reason to refuse the run. hil_pool_check does diagnosis. + note = hil_health.d_state_note() + if note: + log_line(f'rig note: {note}') + health_banner = f'> **Rig note.** {note}. Not a fault on its own -- a healthy testusb sits in D state for most of every case.\n' if note else '' build_err = 0 if args.build: @@ -1915,26 +2345,24 @@ def main() -> None: print(f'Build phase done: {build_err} failed') print('-' * 30) - # HIL report sidecar (hil_report.json/.md) and the .failed re-run spec live in - # report_dir (CI keys it by run id, so it persists across run attempts but is - # private to one run). A full run starts fresh; a re-run (--accumulate, which - # the generated .failed spec always starts with) merges so already-passed - # boards/tests are preserved. Clear prior state up front on a fresh run so a - # crash mid-run can't leave a stale report or re-run spec for a retry. - # -bt alone is not a re-run marker: PR-scoped first attempts pass -bt too. + # The report sidecar and the .failed re-run spec live in report_dir (CI keys it by run + # id: persistent across attempts, private to one run). A full run starts fresh; a re-run + # (--accumulate, which .failed always starts with) merges so already-passed boards + # survive. -bt alone is not a re-run marker. report_dir = Path(os.environ.get('HIL_REPORT_DIR', '.')) failed_fname = report_dir / (config_file.name + '.failed') fresh = not args.accumulate - if fresh: - report_dir.mkdir(parents=True, exist_ok=True) - for f in (REPORT_JSON, REPORT_MD): - (report_dir / f).unlink(missing_ok=True) - failed_fname.unlink(missing_ok=True) + # The unlink is DEFERRED to inside the pool try/except below: wiping here leaves + # Manager() and Pool() running with the old report gone and no report-writing path + # armed, so an EAGAIN/ENOMEM on fork gives CI an EMPTY report dir with no reason. seed = os.getenv('HIL_SHUFFLE_SEED') or str(int(time.time())) log_line(f'test-order shuffle seed: {seed} (HIL_SHUFFLE_SEED={seed} to replay); ' f'flash/usbtest parallel per controller: {hil_lock.FLASH_PARALLEL}/{hil_lock.USBTEST_PARALLEL}; ' - f'enum timeout first/retry: {ENUM_TIMEOUT}/{ENUM_TIMEOUT_RETRY}s') + f'enum timeout first/retry: {ENUM_TIMEOUT}/{ENUM_TIMEOUT_RETRY}s; ' + # all three are env-tunable, so a run that dies on the guard is otherwise + # unattributable from the log alone + f'pool guard: {POOL_TIMEOUT}s') hints = {} try: @@ -1949,94 +2377,232 @@ def main() -> None: config_boards = schedule_boards(config_boards, hints_by_uid) log_line('dispatch order: ' + ', '.join(b['name'] for b in config_boards)) - mgr = Manager() - cmap = mgr.dict() - initargs = (Lock(), seed, - [Semaphore(hil_lock.USBTEST_PARALLEL) for _ in range(hil_lock.CONTROLLER_SLOTS)], - [Semaphore(hil_lock.FLASH_PARALLEL) for _ in range(hil_lock.CONTROLLER_SLOTS)], - cmap, Lock(), hints_by_uid) - with Pool(processes=os.cpu_count() or 1, initializer=init_worker, initargs=initargs) as pool: - async_ret = pool.map_async(test_board, config_boards) + # Bound BEFORE the try so the finally can name them whatever failed: Pool() forks, and + # the EAGAIN/ENOMEM the wipe comment below worries about is most likely to come from + # that fork -- after a convoy, where every stranded read holds a thread and an fd. Left + # outside, an OSError there escaped with mgr LIVE and `pool` unbound, so no report was + # written and the interpreter unwound into multiprocessing's unbounded atexit join. + pool = mgr = cmap = None + # Defined before the pool so _abandon_exit always has a value: a raise before + # `err_count = build_err + ...` would turn the containment path into a NameError. + err_count = build_err + # Fail CLOSED: only a shutdown_pool() that actually returned True clears this, and the + # assignment sits at the END of the inner finally, so anything raising before it + # (kill_worker_children, a BrokenPipeError from its print) leaves _abandon_exit armed. + pool_abandoned = True + # BEFORE Manager()/Pool(), not inside the try: hil_ci.sh reuses a persistent REMOTE_DIR + # and scp's the report back unconditionally, so if a fork failure (OSError/EAGAIN right + # after a convoy -- the case this whole block guards) skipped the wipe, the finally's + # _abandon_exit would prepend "HIL run abandoned" to the PREVIOUS run's table and + # publish last night's board results as this run's. Nothing is live yet here, so an + # OSError from the wipe itself just exits with its traceback -- it cannot strand the + # interpreter in multiprocessing's unbounded atexit join, which is what deferring it + # was protecting against. + if fresh: + report_dir.mkdir(parents=True, exist_ok=True) + for f in (REPORT_JSON, REPORT_MD): + (report_dir / f).unlink(missing_ok=True) + failed_fname.unlink(missing_ok=True) + try: + mgr = Manager() + cmap = mgr.dict() + initargs = (Lock(), seed, + hil_lock.make_permit_sems(Semaphore, hil_lock.USBTEST_PARALLEL), + hil_lock.make_permit_sems(Semaphore, hil_lock.FLASH_PARALLEL), + cmap, Lock(), hints_by_uid) + # maxtasksperchild=1: the sysfs blindness latch is process-global and permanent + # (no decrement anywhere -- see hil_util.SYSFS_STUCK_MAX), so a worker that goes + # blind on ONE wedged board would report 0/30 and "probe missing" for the 2-3 + # healthy boards it picked up afterwards. A fresh worker per board confines the + # damage to the board that caused it; the extra fork is noise against a + # flash+test cycle. + pool = Pool(processes=os.cpu_count() or 1, initializer=init_worker, + initargs=initargs, maxtasksperchild=1) + # OUTER: encloses the pool block too, not just the reporting below. An exception + # escaping async_ret.get() (a worker exception, a Ctrl-C) runs the pool finally and + # then propagates straight out of main(); with _abandon_exit in a sibling try it + # was never reached. try: - mret = async_ret.get(timeout=POOL_TIMEOUT) - except MpTimeoutError: - pool.terminate() - pool.join() - raise RuntimeError(f'HIL worker pool timed out after {POOL_TIMEOUT}s') - - err_count = build_err + sum(e[1] for e in mret) - # generate the re-run spec if anything failed: run ONLY the failed boards (-b), - # each restricted to its own failed tests (-bt); a board with failures but no - # test list (e.g. board-locked) re-runs entirely. --accumulate preserves the - # already-passed cells in the report. - parts = ['--accumulate'] - for name, err, fts, _, _ in mret: - if err > 0: - parts.append(f'-b {name}') - if fts: - parts.append(f'-bt {name}:{",".join(fts)}') - if len(parts) > 1: # build-only failures have no boards to re-run - report_dir.mkdir(parents=True, exist_ok=True) - with failed_fname.open('w') as f: - f.write(' '.join(parts)) - else: - failed_fname.unlink(missing_ok=True) + # imap_unordered, NOT map_async: map_async is all-or-nothing, so a guard expiry + # threw away every board that had already finished -- up to a worker-width of + # completed rig time -- and left the re-run spec unwritten, so CI re-tested all + # ~26 boards to find the one that wedged. Draining as results arrive keeps what + # finished and names only what was still in flight. + it = pool.imap_unordered(test_board, config_boards) + mret = [] + deadline = time.monotonic() + POOL_TIMEOUT + try: + mret = drain_pool(it, config_boards, deadline, out=mret) + except MpTimeoutError as te: + mret = te.finished + stuck = [b['name'] for b in config_boards + if b['name'] not in {r[0] for r in mret}] + # The re-run spec FIRST and before the raise: a fresh run already unlinked + # it, so leaving it unwritten is what made the GitHub re-run repeat the + # whole fleet. Only the boards that never reported go in it. + _write_failed_spec(failed_fname, report_dir, + [(n, 1, [], None, 0) for n in stuck] + + [r for r in mret if r[1] > 0]) + # Then the report, with the rows that DID finish, before anything that can + # block. Then RAISE into the ONE containment path: the inner finally runs + # the ordered sweep (kill_worker_children BEFORE terminate, or a reaped + # worker's flasher reparents out of reach), the outer one os._exit's. + banner = (f'**HIL run abandoned: worker pool timed out after ' + f'{POOL_TIMEOUT}s.** {len(mret)} board(s) below finished and ' + f'are this run\'s; {len(stuck)} never reported and are NOT in ' + f'the table: {", ".join(stuck)}. Re-run covers those.\n') + try: + accumulate_report(mret, report_dir, fresh, '', + health_banner + _blind_note(mret) + + _stray_note(mret) + banner) + except Exception as rerr: # noqa: BLE001 - the raise below must still happen + # FALL BACK, do not just warn: accumulate_report can raise on an + # unwritable/root-owned report dir or a torn JSON, and _abandon_exit + # only PREPENDS to a report that exists. Without this the artifact + # upload finds nothing (if-no-files-found: ignore) and the sticky PR + # comment keeps the previous push's green table under a red job. + print(f'warning: partial report failed: {type(rerr).__name__}: {rerr}; ' + f'falling back to the board list', flush=True) + try: + hil_health.write_timeout_report( + report_dir, [b for b in config_boards + if b['name'] in stuck], POOL_TIMEOUT, REPORT_MD, + prefix=health_banner) + except Exception as re2: # noqa: BLE001 + print(f'warning: fallback report failed too: ' + f'{type(re2).__name__}: {re2}', flush=True) + _p(f'HIL worker pool timed out after {POOL_TIMEOUT}s; sweeping and ' + f'shutting it down (abandoning it if a worker is unkillable)', + flush=True) + raise RuntimeError(f'HIL worker pool timed out after {POOL_TIMEOUT}s') + except Exception as e: + # A worker RAISED -- e.g. a flasher adapter dropping off the bus makes + # get_serial_dev raise in the worker's flash section, which no per-test + # handler guards. Same treatment as the timeout path: the drain means + # `mret` already holds every board that finished, so keep those rows and + # name only the ones still in flight. (Under map_async they were all lost, + # which is what the old banner here claimed.) + done = {r[0] for r in mret} + stuck = [b['name'] for b in config_boards if b['name'] not in done] + _write_failed_spec(failed_fname, report_dir, + [(n, 1, [], None, 0) for n in stuck] + + [r for r in mret if r[1] > 0]) + banner = (f'**HIL run aborted: a worker raised {type(e).__name__}: {e}.** ' + f'{len(mret)} board(s) below finished and are this run\'s; ' + f'{len(stuck)} did not report: {", ".join(stuck)}.\n') + try: + accumulate_report(mret, report_dir, fresh, '', + health_banner + _blind_note(mret) + + _stray_note(mret) + banner) + except Exception as re2: # noqa: BLE001 - the raise below must still happen + print(f'warning: partial report failed: {type(re2).__name__}: {re2}', + flush=True) + raise + + err_count = build_err + sum(e[1] for e in mret) + _write_failed_spec(failed_fname, report_dir, mret) + finally: + # Not `with Pool(...)`: its __exit__ joins the workers unbounded, hanging on + # any worker in uninterruptible sleep. shutdown_pool bounds the same terminate() + # by a grace period, so the pool is NOT cleanly closed/joined when it returns + # False. Record the outcome but never exit here: the report below is the only + # record of a run that otherwise passed. + # + # Same ordering as the timeout path: what the workers spawned must be + # snapshotted and killed while its parent is alive, or terminate() reparents it + # out of reach. + # + # Both calls must stay guarded: a raise here skips accumulate_report(), so a run + # whose boards ALL passed publishes an empty report dir -- and both can raise + # for reasons unrelated to the results. pool_abandoned stays fail-CLOSED, so + # _abandon_exit still arms. + try: + # Still worth running for the TIMEOUT path, where the workers are + # genuinely stuck mid-task and their children are still reachable through + # the pool's ppid tree. On the normal path every worker has already swept + # its own (kill_own_children) and retired, so this finds nothing. + # + # No banner from here: this finally runs AFTER accumulate_report on both + # abort paths, so anything appended to health_banner now is written to a + # variable nobody reads again. The report gets its count from the result + # tuples instead, via _stray_note. + hil_health.kill_worker_children(pool, mgr) + except Exception as e: + print(f'warning: worker-child sweep failed: {type(e).__name__}: {e}', + flush=True) + try: + pool_abandoned = not hil_health.shutdown_pool(pool) + except Exception as e: + print(f'warning: pool shutdown failed: {type(e).__name__}: {e}', flush=True) - # refresh controller hints: pci resolved this run, plus board durations when the - # full test list ran (a -t/-bt filtered run would understate the board's real cost) - try: - if PROFILE: - # debug snapshot of the run's live uid->PCI / PCI->slot resolutions - report_dir.mkdir(parents=True, exist_ok=True) - with (report_dir / 'hil_profile_ctrl.json').open('w') as f: - json.dump(dict(cmap), f, indent=1, sort_keys=True) - uid_of = {b['name']: b['uid'] for b in config['boards']} - for name, _, _, _, dur in mret: - uid = uid_of.get(name) - if uid is None: - continue - h = dict(hints.get(uid) or {}) - h['name'] = name # informational: cache is keyed by uid - h['pci'] = cmap.get(f'uid:{uid}') or h.get('pci') - if dur > 0: # test_board reports 0.0 for filtered (partial) runs - h['duration'] = round(dur, 1) - hints[uid] = h - # merge-on-write: another HIL job (e.g. the esp split) may have finished since - # our startup read - re-read and overlay only this run's boards so its entries - # survive, then replace atomically so a concurrent reader never sees a torn file - merged = {} + # refresh controller hints: pci resolved this run, plus durations from full runs + # only (a filtered run would understate the board's real cost) try: - with CONTROLLER_CACHE.open() as f: - cur = json.load(f) - if isinstance(cur, dict): - merged = {k: v for k, v in cur.items() if isinstance(v, dict)} - except (OSError, ValueError): - pass - merged.update({uid_of[n]: hints[uid_of[n]] for n, *_ in mret if n in uid_of}) - CONTROLLER_CACHE.parent.mkdir(parents=True, exist_ok=True) - tmp = CONTROLLER_CACHE.with_suffix('.json.tmp') - with tmp.open('w') as f: - json.dump(merged, f, indent=1, sort_keys=True) - tmp.replace(CONTROLLER_CACHE) - except OSError as e: - print(f'warning: cannot persist controller hints to {CONTROLLER_CACHE}: {e}') - - # board x test result matrix -> hil_report.md (accumulates across re-runs) + stdout - # -b/-bt in play means a filtered run (PR selection or a re-run spec): say so in the - # report, which otherwise looks exactly like a full run that happened to be small - scoped = sorted(set(args.board) | set(board_test)) - scope = f'{len(scoped)} board(s) — {", ".join(scoped)}' if scoped else '' - report = accumulate_report(mret, report_dir, fresh, scope) - print() - print(report) - print(f'\nReport written to {(report_dir / REPORT_MD).resolve()}') - - duration = time.time() - duration - print() - print("-" * 30) - print(f'Total failed: {err_count} in {duration:.1f}s') - print("-" * 30) - sys.exit(err_count) + if PROFILE: + # debug snapshot of the run's live uid->PCI / PCI->slot resolutions + report_dir.mkdir(parents=True, exist_ok=True) + with (report_dir / 'hil_profile_ctrl.json').open('w') as f: + json.dump(dict(cmap), f, indent=1, sort_keys=True) + uid_of = {b['name']: b['uid'] for b in config['boards']} + for name, _, _, _, dur, *_ in mret: + uid = uid_of.get(name) + if uid is None: + continue + h = dict(hints.get(uid) or {}) + h['name'] = name # informational: cache is keyed by uid + h['pci'] = cmap.get(f'uid:{uid}') or h.get('pci') + if dur > 0: # test_board reports 0.0 for filtered (partial) runs + h['duration'] = round(dur, 1) + hints[uid] = h + # merge-on-write: another HIL job (e.g. the esp split) may have finished since + # our startup read, so overlay only this run's boards and replace atomically + merged = {} + try: + with CONTROLLER_CACHE.open() as f: + cur = json.load(f) + if isinstance(cur, dict): + merged = {k: v for k, v in cur.items() if isinstance(v, dict)} + except (OSError, ValueError): + pass + merged.update({uid_of[n]: hints[uid_of[n]] for n, *_ in mret if n in uid_of}) + CONTROLLER_CACHE.parent.mkdir(parents=True, exist_ok=True) + tmp = CONTROLLER_CACHE.with_suffix('.json.tmp') + with tmp.open('w') as f: + json.dump(merged, f, indent=1, sort_keys=True) + tmp.replace(CONTROLLER_CACHE) + except Exception as e: + # Deliberately broad, and it must stay that way: this best-effort refresh makes + # Manager proxy RPCs that raise EOFError / BrokenPipeError / RemoteError when + # the Manager child has died, none of them OSErrors -- an OSError-only guard let + # those skip accumulate_report(). Nothing here is worth the report. + print(f'warning: cannot persist controller hints to {CONTROLLER_CACHE}: ' + f'{type(e).__name__}: {e}') + + + # board x test result matrix -> hil_report.md (accumulates across re-runs) + stdout. + # -b/-bt means a filtered run (PR selection or a re-run spec): say so, or the report + # looks exactly like a full run that happened to be small + scoped = sorted(set(args.board) | set(board_test)) + scope = f'{len(scoped)} board(s) — {", ".join(scoped)}' if scoped else '' + report = accumulate_report(mret, report_dir, fresh, scope, + health_banner + _blind_note(mret) + + _stray_note(mret)) + print() + print(report) + print(f'\nReport written to {(report_dir / REPORT_MD).resolve()}') + + duration = time.time() - duration + print() + print("-" * 30) + print(f'Total failed: {err_count} in {duration:.1f}s') + print("-" * 30) + finally: + # In the finally, not after: any raise above (accumulate_report sits outside the + # OSError handler) would skip the abandon path and unwind into multiprocessing's + # unbounded atexit join, hanging the runner. + _abandon_exit(pool, mgr, pool_abandoned, err_count, report_dir / REPORT_MD) + # Same clamp: exit status is a byte either way, so 256 failures would report green. + sys.exit(min(err_count, 125)) if __name__ == '__main__': diff --git a/test/hil/mtp_test.py b/test/hil/mtp_test.py new file mode 100644 index 000000000..92d54bdbe --- /dev/null +++ b/test/hil/mtp_test.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# One MTP test session for one board, in a disposable process. Every libmtp call is +# synchronous ctypes in our own address space and blocks in a usbfs ioctl in D state on +# a wedged device, where not even SIGKILL is delivered — so the session must be +# something the harness can abandon: hil_test.test_device_mtp runs it under +# hil_util.run_cmd (killpg + bounded reap, rc 124 on timeout). Imports stay stdlib + +# pymtp: nothing here may pull in the harness. +# +# Exit 0 on a fully passing session; 1 with the failure on stdout/stderr otherwise. +import argparse +import ctypes +import glob +import hashlib +import os +import signal +import subprocess +import sys +import threading +import time + +sys.path.append(os.path.dirname(os.path.abspath(__file__))) # PYTHONSAFEPATH drops it +# -- APPEND so PYTHONPATH still wins (the tests steer a fake pymtp that way) + +from pathlib import Path +from pymtp import LIBMTP_DeviceEntry, LIBMTP_RawDevice, MTP + +FILE1_EXPECT = b'TinyUSB MTP Filesystem example' +FILE2_MD5_EXPECT = '40ef23fc2891018d41a05d4a0d5f822f' # md5sum of logo.png + + +# Real paths by default; the offline tests point these at a fixture tree, the same way +# they steer the pymtp fake through FAKE_PYMTP_*. +# The one test seam: '' in production, a tmpdir in the offline tests, which mirror the +# real layout beneath it. This runs as a SUBPROCESS (a libmtp call blocked in a usbfs +# ioctl hangs its thread forever, so the session must be somewhere killable), and neither +# monkeypatching nor import shadowing crosses that boundary -- unlike the fake pymtp, +# which the tests inject through PYTHONPATH alone. +_ROOT = os.environ.get('HIL_MTP_FAKE_ROOT', '') +_MARKER_GLOB = f'{_ROOT}/dev/libmtp-*' +_SYS_USB = Path(f'{_ROOT}/sys/bus/usb/devices') +_USB_DEV = Path(f'{_ROOT}/dev/bus/usb') + + +def _bounded_read(path, grace: float = 2.0): + """Read a sysfs attribute with a wall-clock bound, or return None. + + `serial` is served under the device lock a wedged usbfs ioctl holds, and EVERY MTP DUT + is cafe:4017 -- so the vid/pid filter below cannot rule out a wedged NEIGHBOUR, and an + unbounded read of its serial would burn this session's whole budget and report a + healthy board as wedged. Stdlib only by design (this file never imports the harness), + so this is a small local twin of hil_util.read_sysfs. + """ + out = {} + + def _read(): + try: + out['v'] = path.read_text().strip() + except OSError: + pass + + t = threading.Thread(target=_read, daemon=True) + t.start() + t.join(grace) + return out.get('v') + + +def _ready_marker(uid: str): + """(busnum, devnum) of the udev-ready MTP device with this serial, or None. + + /dev/libmtp- is published by libmtp-runtime AFTER its synchronous mtp-probe + accepts the device, so this set is both small and ready -- unlike a sysfs-wide scan, + which races re-enumerations from other boards' jobs. Requires the libmtp-runtime + package. + """ + for marker_name in glob.glob(_MARKER_GLOB): + marker = Path(marker_name) + try: + dev = _SYS_USB / marker.name[len('libmtp-'):] + # vid/pid first: lock-free descriptor fields, so they rule out every other + # device before the `serial` read, which the kernel serves under the device + # lock a wedged usbfs ioctl would hold + if ((dev / 'idVendor').read_text().strip() != 'cafe' + or (dev / 'idProduct').read_text().strip() != '4017'): + continue + # bounded: this one CAN block, and a wedged neighbour shares the vid/pid above + serial = _bounded_read(dev / 'serial') + if serial is None or serial.lower() != uid.lower(): + continue + busnum = int((dev / 'busnum').read_text()) + devnum = int((dev / 'devnum').read_text()) + node = _USB_DEV / f'{busnum:03d}' / f'{devnum:03d}' + if marker.resolve(strict=True) != node or not os.access(node, os.R_OK | os.W_OK): + continue + return busnum, devnum + except (OSError, ValueError): + # a marker can vanish while another board flashes: not our device's problem + continue + return None + + +def _gvfs_unmount(uid: str, deadline: float) -> None: + """Drop any gvfs claim on this device, immediately before opening it. + + Called only once the udev marker exists. gvfs claims an MTP device AFTER udev + probing, so before the marker there is nothing to unmount: an earlier call is a + guaranteed no-op that still forks a process, and it leaves the gap between the + unmount and the open unprotected -- the hang this exists to prevent. Per-iteration + calls also forked one gio per second of the enumeration budget. + """ + # Popen, not run(timeout=): run's post-timeout reap is an unbounded wait(), and a gio + # blocked in D state on a wedged usbfs node does not die on SIGKILL, so run(timeout=2) + # can hang for good. Bounded by at most HALF of what is LEFT of our own budget, never + # a fixed sub-bound: the parent gives us --timeout 8 (4 on a retry), so anything larger + # collapsed the poll loop to one attempt and made a slow gio look like a wedged session. + gio_bound = max(0.5, min(3.0, (deadline - time.monotonic()) / 2)) + try: + # argv, not shell=True: uid comes from a hand-edited roster and is board firmware + # output, so a space or $(...) would unmount the wrong URI (leaving the gvfs mount + # held) or run as us. + gio = subprocess.Popen(['gio', 'mount', '-u', + f'mtp://TinyUsb_TinyUsb_Device_{uid}/'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, start_new_session=True) + except OSError: + # glib2.0-bin absent (ci.lan has no gio at all): nothing holds a gvfs mount + # either, so go straight on to the open. + return + try: + gio.wait(timeout=gio_bound) + except subprocess.TimeoutExpired: + try: + os.killpg(gio.pid, signal.SIGKILL) + except OSError: + gio.kill() + try: + gio.wait(timeout=2) # reap it: an abandoned gio leaves a zombie + except subprocess.TimeoutExpired: + pass + print('gio unmount timed out; continuing', file=sys.stderr) + + +def open_mtp_dev(uid: str, timeout: float): + mtp = MTP() + deadline = time.monotonic() + timeout + while True: + try: + # pymtp raises USB_LAYER/PTP_LAYER/GENERAL/AlreadyConnected on a board still + # settling right after a flash; an unguarded raise would skip the rest of the + # enumeration budget (and the disconnect) instead of retrying. + # + # Never detect_devices(): that PROBES every MTP device on the rig, so a board + # still initialising in a parallel job answers our scan (the race #3790 fixed). + # libmtp-runtime publishes /dev/libmtp- only after its own mtp-probe + # has accepted a device, so start from that small, ready-only set and open OUR + # device directly by bus/dev address. + target = _ready_marker(uid) + if target: + # ready first, THEN unmount, then open -- see _gvfs_unmount + _gvfs_unmount(uid, deadline) + busnum, devnum = target + # TinyUSB needs no libmtp quirks, so the raw entry can be built here + entry = LIBMTP_DeviceEntry(None, 0xcafe, None, 0x4017, 0) + raw = LIBMTP_RawDevice(entry, busnum, devnum) + mtp.device = mtp.mtp.LIBMTP_Open_Raw_Device(ctypes.byref(raw)) + if mtp.device: + serial = mtp.get_serialnumber() + if (serial.decode('utf-8') if serial else '').lower() == uid.lower(): + return mtp + mtp.disconnect() + except Exception as e: + print(f'mtp poll: {type(e).__name__}: {e}', file=sys.stderr) + # only when a device was actually opened: pymtp's `self.device == None` + # guard does NOT catch a ctypes NULL pointer (falsy, but != None), so + # disconnecting blindly calls LIBMTP_Release_Device(NULL) + if getattr(mtp, 'device', None): + try: + mtp.disconnect() + except Exception: + pass + mtp.device = None + if time.monotonic() >= deadline: + return None + time.sleep(1) + + +def run_session(uid: str, timeout: float) -> int: + mtp = open_mtp_dev(uid, timeout) + if mtp is None or mtp.device is None: + print('MTP device not found') + return 1 + + try: + assert b"TinyUSB" == mtp.get_manufacturer(), 'MTP wrong manufacturer' + assert b"MTP Example" == mtp.get_modelname(), 'MTP wrong model' + assert b'1.0' == mtp.get_deviceversion(), 'MTP wrong version' + assert b'TinyUSB MTP' == mtp.get_devicename(), 'MTP wrong device name' + + f1 = uid.encode("utf-8") + b'_file1' + f2 = uid.encode("utf-8") + b'_file2' + f3 = uid.encode("utf-8") + b'_file3' + mtp.get_file_to_file(1, f1) + with open(f1, 'rb') as file: + f1_data = file.read() + os.remove(f1) + assert f1_data == FILE1_EXPECT, 'MTP file1 wrong data' + mtp.get_file_to_file(2, f2) + with open(f2, 'rb') as file: + f2_data = file.read() + os.remove(f2) + assert FILE2_MD5_EXPECT == hashlib.md5(f2_data).hexdigest(), 'MTP file2 wrong data' + with open(f3, "wb") as file: + # 1524-byte payload + 12-byte MTP header = 3 full 512-byte buffers, so this + # exercises delivery of the final OUT payload before its ZLP. Deliberate and + # FIXED: a random size hits that boundary in ~0.2% of runs, which is not a test + # of it. Deterministic content so a mismatch is reproducible. + f3_data = bytes((i % 251) + 1 for i in range(1524)) + file.write(f3_data) + file.close() + fid = mtp.send_file_from_file(f3, b'file3') + f3_readback = f3 + b'_readback' + mtp.get_file_to_file(fid, f3_readback) + with open(f3_readback, 'rb') as f: + f3_rb_data = f.read() + os.remove(f3_readback) + assert f3_rb_data == f3_data, 'MTP file3 wrong data' + os.remove(f3) + mtp.delete_object(fid) + except AssertionError as e: + print(e) + return 1 + finally: + mtp.disconnect() + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument('--uid', required=True, help='board_get_unique_id serial to match') + parser.add_argument('--timeout', type=float, default=30, help='enumeration wait budget (s)') + args = parser.parse_args() + return run_session(args.uid, args.timeout) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/test/hil/test/stubs/pymtp.py b/test/hil/test/stubs/pymtp.py new file mode 100644 index 000000000..2720321f6 --- /dev/null +++ b/test/hil/test/stubs/pymtp.py @@ -0,0 +1,125 @@ +# SPDX-License-Identifier: MIT +# Fake pymtp for the hil unit tests — stands in both for the import (GitHub's bare +# pre-commit runner has no libmtp/pymtp) and for a scripted MTP device. Behavior is +# driven by env vars so subprocesses (mtp_test.py under run_cmd) can be steered: +# FAKE_PYMTP_MODE absent (default) | ok | hang +# FAKE_PYMTP_UID serial number the fake device reports +# FAKE_PYMTP_FILE1 text served as file id 1 (README.TXT) +# FAKE_PYMTP_LOGO path to the logo bytes served as file id 2 +# File contents come from env, not constants: the test extracts them from the example's +# own sources, so this stub cannot drift out of sync with the firmware. +# 'hang' blocks forever inside detect_devices — the in-process libmtp equivalent of a +# D-state usbfs ioctl on a wedged device. +import ctypes +import os +import time + + +class NotConnected(Exception): + pass + + +class LIBMTP_DeviceEntry(ctypes.Structure): + """Real pymtp exposes this; mtp_test builds one to open a KNOWN device instead of + probing every MTP device on the bus.""" + _fields_ = [('vendor', ctypes.c_char_p), ('vendor_id', ctypes.c_uint16), + ('product', ctypes.c_char_p), ('product_id', ctypes.c_uint16), + ('device_flags', ctypes.c_uint32)] + + +class LIBMTP_RawDevice(ctypes.Structure): + _fields_ = [('device_entry', LIBMTP_DeviceEntry), ('bus_location', ctypes.c_uint32), + ('devnum', ctypes.c_uint8)] + + +class _LibShim: + @staticmethod + def LIBMTP_Open_Raw_Device(_ref): + # mtp_test no longer calls detect_devices() (it probed every MTP device on the + # rig), so the scripted modes have to act here -- this is the only libmtp entry + # point the marker-based open goes through. + mode = os.environ.get('FAKE_PYMTP_MODE', 'absent') + if mode == 'hang': + time.sleep(10000) + if mode == 'error': + raise RuntimeError('CommandFailed: LIBMTP_ERROR_USB_LAYER') + if mode == 'error_then_ok': + flag = os.environ.get('FAKE_PYMTP_ERRED_MARKER', '/tmp/.fake_pymtp_erred') + if not os.path.exists(flag): + open(flag, 'w').close() + raise RuntimeError('CommandFailed: LIBMTP_ERROR_PTP_LAYER') + if mode == 'absent': + return ctypes.POINTER(ctypes.c_int)() # NULL: nothing to open + # the real one has restype POINTER(LIBMTP_MTPDevice): a failed open returns a + # NULL pointer, which is FALSY but compares unequal to None -- the distinction + # mtp_test's `if mtp.device:` guards depend on + if os.environ.get('FAKE_PYMTP_OPEN') == 'null': + return ctypes.POINTER(ctypes.c_int)() + return 1 + + +class MTP: + def __init__(self): + self.mtp = _LibShim() + self.device = None + self._sent = {} + self._next_id = 3 + + def detect_devices(self): + mode = os.environ.get('FAKE_PYMTP_MODE', 'absent') + if mode == 'hang': + time.sleep(10000) + if mode == 'error': + # real pymtp raises for USB_LAYER/PTP_LAYER/GENERAL/AlreadyConnected; + # the first poll after a flash routinely hits one + raise RuntimeError('CommandFailed: LIBMTP_ERROR_USB_LAYER') + if mode == 'error_then_ok': + if not getattr(self, '_erred', False): + self._erred = True + raise RuntimeError('CommandFailed: LIBMTP_ERROR_USB_LAYER') + return [ctypes.c_int(1)] + if mode != 'ok': + return [] + return [ctypes.c_int(1)] + + def get_serialnumber(self): + return os.environ.get('FAKE_PYMTP_UID', '').encode() + + def get_manufacturer(self): + return b'TinyUSB' + + def get_modelname(self): + return b'MTP Example' + + def get_deviceversion(self): + return b'1.0' + + def get_devicename(self): + return b'TinyUSB MTP' + + def get_file_to_file(self, fid, path): + if fid == 1: + data = os.environ['FAKE_PYMTP_FILE1'].encode() + elif fid == 2: + with open(os.environ['FAKE_PYMTP_LOGO'], 'rb') as f: + data = f.read() + else: + data = self._sent[fid] + with open(path, 'wb') as f: + f.write(data) + + def send_file_from_file(self, path, _name): + with open(path, 'rb') as f: + self._sent[self._next_id] = f.read() + self._next_id += 1 + return self._next_id - 1 + + def delete_object(self, fid): + del self._sent[fid] + + def disconnect(self): + # vendored pymtp raises when nothing is connected; a stub that silently accepts + # it hides a LIBMTP_Release_Device(NULL) call on real hardware + if self.device is None: + raise NotConnected('no device connected') + self.device = None diff --git a/test/hil/test/test_hil_bounded.py b/test/hil/test/test_hil_bounded.py new file mode 100644 index 000000000..908a142d5 --- /dev/null +++ b/test/hil/test/test_hil_bounded.py @@ -0,0 +1,1701 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests proving hil_test's storage and MTP helpers cannot hang the worker: a +# wedged device blocks the call in D state forever (child process or in-process ioctl), +# so these paths go through a bounded runner. Fakes stand in for the wedge (a real one +# cannot be manufactured on demand): a PATH-injected `mtype` script and a +# PYTHONPATH-injected `pymtp` module, each with a mode that blocks forever. +# Scope: mtype, the gio unmount, the libmtp session, the arecord/iperf reaps, and the +# printer read (a process now, via run_alongside, so a killed reader takes its fd with +# it -- usblp allows ONE opener, and a blocked thread kept the node for the worker's life). +# Known residue (unbounded, backstopped only by the pool guard): hid open/write and +# midi's read(64). +# +# hil_test imports pyserial, which GitHub's bare pre-commit runner does not have — so +# an inert serial module is stubbed into sys.modules BEFORE the import (nothing here +# exercises serial paths). MTP traffic never touches hil_test: it all goes through the +# mtp_test.py subprocess, which gets the fake pymtp via PYTHONPATH. +# Run directly: +# python3 test/hil/test/test_hil_bounded.py +import os +import stat +import sys +import threading +from multiprocessing import TimeoutError as MpTimeoutError +import time +import types +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory + +TEST_DIR = os.path.dirname(os.path.abspath(__file__)) +# the modules under test live in the parent dir (test/hil), not here +sys.path.insert(0, os.path.dirname(TEST_DIR)) + +serial_stub = types.ModuleType('serial') +serial_stub.Serial = type('Serial', (), {}) +serial_stub.SerialException = type('SerialException', (Exception,), {}) +serial_stub.SerialTimeoutException = type('SerialTimeoutException', (Exception,), {}) +sys.modules.setdefault('serial', serial_stub) +import hil_flash +import hil_test + + +def write_script(path: Path, body: str) -> None: + path.write_text('#!/bin/sh\n' + body + '\n') + path.chmod(path.stat().st_mode | stat.S_IEXEC) + + +def run_bounded(fn, timeout: float): + """Run fn in a daemon thread; return (finished, exception). A still-running thread is + the hang under test — leave it to die with the interpreter.""" + exc = [] + + def wrapper(): + try: + fn() + except BaseException as e: # noqa: BLE001 - tests inspect the exception + exc.append(e) + + t = threading.Thread(target=wrapper, daemon=True) + t.start() + t.join(timeout) + return not t.is_alive(), exc[0] if exc else None + + +@unittest.skipIf(os.name == 'nt', 'POSIX shell fakes') +class ReadDiskFile(unittest.TestCase): + def setUp(self): + self.tmp = TemporaryDirectory() + tmp = Path(self.tmp.name) + self.addCleanup(self.tmp.cleanup) + # fake block device node: get_disk_dev is patched to this existing path + self.dev = tmp / 'fakedev' + self.dev.write_bytes(b'') + # addCleanup, not tearDown: tearDown does NOT run when setUp raises, and a leaked + # PATH entry points at a temp bin dir this class already deleted. + for name in ('get_disk_dev', '_enum_timeout', 'MTYPE_TIMEOUT'): + self.addCleanup(setattr, hil_test, name, getattr(hil_test, name)) + hil_test.get_disk_dev = lambda uid, vendor, lun: str(self.dev) + hil_test._enum_timeout = 2 + self.bin = tmp / 'bin' + self.bin.mkdir() + self.addCleanup(os.environ.__setitem__, 'PATH', os.environ['PATH']) + os.environ['PATH'] = f'{self.bin}:{os.environ["PATH"]}' + self.pidfile = tmp / 'mtype.pid' + self.addCleanup(self._reap_mtype) + + def _reap_mtype(self): + if self.pidfile.exists(): # reap a leaked hang-mode mtype + try: + os.kill(int(self.pidfile.read_text()), 9) + except (OSError, ValueError): + pass + + def test_returns_exact_bytes_despite_stderr_noise(self): + # \377 is invalid UTF-8 and stderr noise must not leak into the data + write_script(self.bin / 'mtype', r"printf 'R\377EADME-DATA'; printf 'vfat warning' >&2") + data = hil_test.read_disk_file('uid0', 0, 'README.TXT') + self.assertEqual(data, b'R\xffEADME-DATA') + + def test_failure_message_carries_mtype_stderr_and_fname(self): + write_script(self.bin / 'mtype', "printf 'mtype: cannot read' >&2; exit 1") + with self.assertRaises(AssertionError) as cm: + hil_test.read_disk_file('uid0', 0, 'README.TXT') + self.assertIn('cannot read', str(cm.exception)) + self.assertIn('README.TXT', str(cm.exception)) + + def test_empty_read_fails_immediately_with_fname(self): + # rc 0 with no data is a real answer (bad sectors, empty file), not "not ready": + # fail at once like the old assert did, naming the file — don't spin the budget + write_script(self.bin / 'mtype', 'exit 0') + t0 = time.monotonic() + with self.assertRaises(AssertionError) as cm: + hil_test.read_disk_file('uid0', 0, 'README.TXT') + self.assertLess(time.monotonic() - t0, 1.5) + self.assertIn('README.TXT', str(cm.exception)) + + def test_hung_mtype_cannot_hang_the_worker(self): + # a D-state child never exits; the bounded runner must give up without it + write_script(self.bin / 'mtype', f'echo $$ > {self.pidfile}; exec sleep 1000') + hil_test.MTYPE_TIMEOUT = 2 + finished, exc = run_bounded(lambda: hil_test.read_disk_file('uid0', 0, 'README.TXT'), 20) + self.assertTrue(finished, 'read_disk_file hung on a stuck mtype') + self.assertIsInstance(exc, AssertionError) + + +class CompactOutput(unittest.TestCase): + def test_strips_workflow_command_markers(self): + """Defense-in-depth: the historical marker source was worker-side run_cmd + (now suppressed at the emitter); anything future that pipes markers into a + captured stdout would land them mid-row where GitHub renders them literally.""" + raw = '::group::COMMAND TIMEOUT (1s): x\nboom\n::endgroup::\ntail' + self.assertEqual(hil_test.compact_output(raw), 'COMMAND TIMEOUT (1s): x | boom | tail') + + +class UsbtestRecovery(unittest.TestCase): + def test_recovery_flags_and_flash_bound_fit_the_reserve(self): + """The post-hang reflash plumbing: the CLI flags exist, and the bounded reflash + plus the fixed recovery costs (60s case timeout + 5s kill wait + 5s settle) + fits inside USBTEST_RECOVERY_BUDGET -- otherwise the outer run_cmd kill lands + mid-flash and orphans the flasher (own session) on the probe.""" + import subprocess + hil_dir = Path(TEST_DIR).parents[0] + r = subprocess.run([sys.executable, str(hil_dir / 'usbtest.py'), '--help'], + capture_output=True, text=True, timeout=30) + self.assertEqual(r.returncode, 0, r.stderr) + for flag in ('--recover-board', '--recover-fw', '--outer-timeout'): + self.assertIn(flag, r.stdout) + + def test_the_bounded_reflash_actually_fits_the_reserve(self): + """The arithmetic the docstring above claims but never checked -- the two + constants never met in any test, so bumping either silently broke the promise. + Overrun means run_cmd's outer kill lands MID-FLASH and orphans the flasher + (start_new_session, so killpg misses it) holding the probe.""" + import re + import usbtest + hil_dir = Path(TEST_DIR).parents[0] + # read the case timeout hil_test actually passes, so this cannot drift silently + src = (hil_dir / 'hil_test.py').read_text() + m = re.search(r'--timeout (\d+) --budget', src) + self.assertIsNotNone(m, 'usbtest invocation changed shape; re-derive this bound') + case_timeout = int(m.group(1)) + kill_wait, settle, time_left_reserve = 5, 5, 35 # usbtest.py's fixed costs + worst = (case_timeout + kill_wait + usbtest.RECOVER_FLASH_TIMEOUT + + settle + time_left_reserve) + self.assertLessEqual( + worst, hil_test.USBTEST_RECOVERY_BUDGET, + f'a HUNG case needs {worst}s to recover but only ' + f'{hil_test.USBTEST_RECOVERY_BUDGET}s is reserved') + + +class UsbtestRunHelper(unittest.TestCase): + """usbtest.run() is the bounded replacement for subprocess.run: sysfs_write feeds it + input=, and every battery calls that before case 1.""" + + def setUp(self): + import usbtest + self.usbtest = usbtest + + def test_input_kwarg_is_honoured(self): + r = self.usbtest.run(['cat'], input='payload', timeout=10) + self.assertEqual(r.returncode, 0) + self.assertEqual(r.stdout, 'payload') + + def test_capture_output_kwarg_is_accepted(self): + r = self.usbtest.run(['printf', 'x'], capture_output=True, timeout=10) + self.assertEqual(r.stdout, 'x') + + def test_timeout_is_bounded_and_raises(self): + import subprocess + t0 = time.monotonic() + with self.assertRaises(subprocess.TimeoutExpired): + self.usbtest.run(['sleep', '30'], timeout=1) + self.assertLess(time.monotonic() - t0, 15) + + +class BuildBoardContract(unittest.TestCase): + def test_every_return_path_is_a_pair(self): + """main() unpacks `_, nfail = build_board(board)`; a bare int on any path + (the timeout path did) raises TypeError before the pool exists.""" + import ast + src = (Path(TEST_DIR).parents[0] / 'hil_test.py').read_text() + fn = next(n for n in ast.walk(ast.parse(src)) + if isinstance(n, ast.FunctionDef) and n.name == 'build_board') + for node in ast.walk(fn): + if isinstance(node, ast.Return) and node.value is not None: + self.assertIsInstance(node.value, ast.Tuple, + f'build_board returns a non-tuple at line {node.lineno}') + + +class RemoteStaging(unittest.TestCase): + def test_import_closure_is_staged_to_the_rig(self): + # hil_ci.sh stages an explicit scp whitelist; a module that is not on it exists + # locally and in CI checkouts but silently never reaches the remote rig (how + # mtp_test.py was first missed). Walk the local-import closure of everything + # the rig executes and require each file's exact scp entry — a bare-substring + # match would be satisfied by a mention in a comment or the run line. + import ast + hil_dir = Path(TEST_DIR).parents[0] + staged = (hil_dir / 'hil_ci.sh').read_text() + + def imported_paths(pyfile): + # ast, not regex: an earlier regex walker went silently vacuous on a + # multi-line import. ast also sees function-local deferred imports + # (usbtest.py's `import hil_flash` inside the recovery branch). + for node in ast.walk(ast.parse(pyfile.read_text())): + if isinstance(node, ast.Import): + for a in node.names: + yield a.name.replace('.', '/') + '.py' + elif isinstance(node, ast.ImportFrom) and node.module: + if node.module == 'helper': + for a in node.names: + yield f'helper/{a.name}.py' + else: + yield node.module.replace('.', '/') + '.py' + + seeds = ['hil_test.py', 'usbtest.py', 'mtp_test.py'] # CLI + spawned helpers + for f in seeds: # a renamed seed must fail loudly, not fall out of the walk + self.assertTrue((hil_dir / f).exists(), f'stale RemoteStaging seed: {f}') + todo, seen = list(seeds), set() + while todo: + f = todo.pop() + if f in seen or not (hil_dir / f).exists(): + continue # stdlib/site-packages imports have no test/hil file + seen.add(f) + todo += list(imported_paths(hil_dir / f)) + for f in sorted(seen): + self.assertIn(f'"$ROOT_DIR/test/hil/{f}"', staged, + f'{f} runs on the rig but hil_ci.sh does not scp it') + + +class _MtpFakeRig: + """The fake rig shared by the MTP cases: a udev-marker tree under one tmp root and + the scripted pymtp on PYTHONPATH. A plain mixin, NOT a TestCase -- subclassing a + TestCase to reuse a fixture re-runs every inherited test in each subclass.""" + + @classmethod + def setUpClass(cls): + # both file fixtures come from the example's sources, so drift there fails here: + # file id 1 is README.TXT (C define), file id 2 is logo.png (C byte array) + import hashlib + import re + src = Path(TEST_DIR).parents[2] / 'examples/device/mtp/src' + m = re.search(r'#define README_TXT_CONTENT "([^"]+)"', (src / 'mtp_fs_example.c').read_text()) + assert m, 'README_TXT_CONTENT define not found in mtp_fs_example.c' + cls.readme = m.group(1) + data = bytes(int(x, 16) for x in + re.findall(r'0x([0-9a-fA-F]{2})', (src / 'tinyusb_logo_png.h').read_text())) + assert hashlib.md5(data).hexdigest() == '40ef23fc2891018d41a05d4a0d5f822f' + cls.logo = data + + def setUp(self): + self.tmp = TemporaryDirectory() + tmp = Path(self.tmp.name) + self.addCleanup(self.tmp.cleanup) + logo = tmp / 'logo.bin' + logo.write_bytes(self.logo) + self.board = {'uid': 'CAFE01', 'name': 'fakeboard'} + # addCleanup, not tearDown: tearDown does NOT run when setUp raises, and a leaked + # chdir into a deleted temp dir breaks every test after it. + self.saved_env = {k: os.environ.get(k) for k in + ('FAKE_PYMTP_MODE', 'FAKE_PYMTP_UID', 'FAKE_PYMTP_LOGO', + 'FAKE_PYMTP_FILE1', 'PYTHONPATH', 'PYTHONSAFEPATH', + 'HIL_MTP_FAKE_ROOT', 'FAKE_PYMTP_ERRED_MARKER')} + self.addCleanup(self._restore_env) + # A udev-ready marker tree: libmtp-runtime publishes /dev/libmtp- only + # after mtp-probe accepts a device, and mtp_test opens THAT device directly rather + # than probing every MTP device on the rig (the parallel-probe race #3790 fixed). + # mirrors the real layout under one root, so /sys/bus/usb/devices/1-1 reads + # as the stand-in for /sys/bus/usb/devices/1-1 that it is + dev = tmp / 'sys/bus/usb/devices/1-1' + usbdev = tmp / 'dev/bus/usb/001' + markers = tmp / 'dev' # created by usbdev's parents=True + dev.mkdir(parents=True); usbdev.mkdir(parents=True) + (dev / 'idVendor').write_text('cafe\n') + (dev / 'idProduct').write_text('4017\n') + (dev / 'serial').write_text(self.board['uid'] + '\n') + (dev / 'busnum').write_text('1\n') + (dev / 'devnum').write_text('2\n') + node = usbdev / '002' + node.write_bytes(b'') + (markers / 'libmtp-1-1').symlink_to(node) + os.environ['HIL_MTP_FAKE_ROOT'] = str(tmp) + os.environ['FAKE_PYMTP_ERRED_MARKER'] = str(tmp / 'erred') + os.environ['FAKE_PYMTP_UID'] = self.board['uid'] + os.environ['FAKE_PYMTP_LOGO'] = str(logo) + os.environ['FAKE_PYMTP_FILE1'] = self.readme + stubs = os.path.join(TEST_DIR, 'stubs') + pp = self.saved_env['PYTHONPATH'] + os.environ['PYTHONPATH'] = stubs if not pp else f'{stubs}:{pp}' + # pymtp is vendored next to mtp_test.py, and a script's own dir (sys.path[0]) + # outranks PYTHONPATH — safe-path mode (3.11+) drops it so the fake wins there + os.environ['PYTHONSAFEPATH'] = '1' + for name in ('_enum_timeout', 'MTP_SESSION_MARGIN'): + self.addCleanup(setattr, hil_test, name, getattr(hil_test, name)) + hil_test._enum_timeout = 2 + # the session scratch files land in cwd + self.addCleanup(os.chdir, os.getcwd()) + os.chdir(tmp) + + def _restore_env(self): + for k, v in self.saved_env.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + + +@unittest.skipIf(os.name == 'nt', 'POSIX shell fakes') +@unittest.skipIf(sys.version_info < (3, 11), 'fake-pymtp steering needs PYTHONSAFEPATH') +class DeviceMtp(_MtpFakeRig, unittest.TestCase): + """test_device_mtp end to end: the real mtp_test.py subprocess under run_cmd, + with the scripted pymtp fake steered in via PYTHONPATH.""" + + def test_mtp_session_passes_against_scripted_device(self): + os.environ['FAKE_PYMTP_MODE'] = 'ok' + hil_test.test_device_mtp(self.board) # no exception + + def test_absent_device_fails_cleanly(self): + os.environ['FAKE_PYMTP_MODE'] = 'absent' + finished, exc = run_bounded(lambda: hil_test.test_device_mtp(self.board), 30) + self.assertTrue(finished) + self.assertIsInstance(exc, AssertionError) + self.assertIn('MTP device not found', str(exc)) + + def test_libmtp_error_on_one_poll_retries_instead_of_dying(self): + """pymtp raises for USB_LAYER/PTP_LAYER errors -- routine on the first poll + after a flash. An unguarded raise skipped the whole enumeration budget.""" + os.environ['FAKE_PYMTP_MODE'] = 'error_then_ok' + hil_test.test_device_mtp(self.board) # retries past the error, then passes + + def test_libmtp_error_every_poll_fails_cleanly(self): + os.environ['FAKE_PYMTP_MODE'] = 'error' + finished, exc = run_bounded(lambda: hil_test.test_device_mtp(self.board), 30) + self.assertTrue(finished) + self.assertIsInstance(exc, AssertionError) + + def test_hung_mtp_stack_cannot_hang_the_worker(self): + # in-process libmtp blocking in a usbfs ioctl (D state) hangs whatever thread + # made the call, forever — the session must be somewhere disposable + os.environ['FAKE_PYMTP_MODE'] = 'hang' + hil_test.MTP_SESSION_MARGIN = 3 + finished, exc = run_bounded(lambda: hil_test.test_device_mtp(self.board), 25) + self.assertTrue(finished, 'test_device_mtp hung on a wedged MTP stack') + self.assertIsInstance(exc, AssertionError) + + +class ConvoySafeFlasher(unittest.TestCase): + """hil_flash.convoy_safe decides whether a board gets post-HUNG recovery at all. + + It must be true ONLY for flashers that can reach their probe without opening the + poisoned usbfs node: openocd pinned with a roster vid_pid (filters on kernel-cached + sysfs descriptors) and esptool (delivers to a named tty, never enumerates usbfs). + Anything else enumerates by opening nodes, would block in D state on the wedged one + and become a second stray -- JLinkExe included, whose selection is serial-only and + so cannot be pinned at all.""" + + def setUp(self): + import hil_flash + self.f = hil_flash.convoy_safe + + def test_pinned_openocd_is_safe(self): + self.assertTrue(self.f({'name': 'openocd', 'vid_pid': '0x2e8a 0x000c'})) + + def test_unpinned_openocd_is_not(self): + self.assertFalse(self.f({'name': 'openocd'})) + self.assertFalse(self.f({'name': 'openocd', 'vid_pid': ''})) + + def test_esptool_is_safe_without_a_pin(self): + """Delivery is `-p `; there is no usbfs walk to poison.""" + self.assertTrue(self.f({'name': 'esptool'})) + + def test_enumerating_flashers_are_not(self): + for name in ('jlink', 'stlink', 'lm4flash', 'dfu-util'): + self.assertFalse(self.f({'name': name, 'vid_pid': '0x1366 0x1024'}), + f'{name} must not be treated as convoy-safe') + + def test_missing_or_odd_name_is_not_safe(self): + for flasher in ({}, {'name': None}, {'name': ''}): + self.assertFalse(self.f(flasher)) + + +class BoundedOpen(unittest.TestCase): + """hil_util.bounded_open must return rather than block, and must not leak the fd if + the open completes after we gave up (usblp_open takes the device mutex before it + consults O_NONBLOCK, so a wedged node blocks the open uninterruptibly).""" + + def setUp(self): + from helper import hil_util + self.hil_util = hil_util + self.tmp = TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + # bounded_open counts its stranded threads now, and the counter is process-global + # with no decrement: three wedged-FIFO tests here reach SYSFS_STUCK_MAX and every + # later test in this file reads SYSFS_UNKNOWN for perfectly good attributes + self.addCleanup(setattr, hil_util, '_sysfs_stuck', hil_util._sysfs_stuck) + + def test_opens_a_normal_file(self): + f = Path(self.tmp.name) / 'plain' + f.write_text('x') + fd = self.hil_util.bounded_open(str(f), os.O_RDONLY, 5) + self.assertIsNotNone(fd) + os.close(fd) + + def test_missing_path_returns_none_without_raising(self): + self.assertIsNone(self.hil_util.bounded_open( + str(Path(self.tmp.name) / 'nope'), os.O_RDONLY, 5)) + + @unittest.skipIf(os.name == 'nt', 'POSIX fifo') + def test_blocking_open_gives_up_and_does_not_leak_fds(self): + """A reader-less FIFO blocks open(O_WRONLY) forever -- the closest portable + stand-in for a wedged usblp node.""" + fifo = Path(self.tmp.name) / 'fifo' + os.mkfifo(fifo) + before = len(os.listdir('/proc/self/fd')) + t0 = time.monotonic() + for _ in range(5): + self.assertIs(self.hil_util.bounded_open(str(fifo), os.O_WRONLY, 0.2), + self.hil_util.SYSFS_UNKNOWN) + self.assertLess(time.monotonic() - t0, 10, 'bounded_open did not bound') + self.assertLessEqual(len(os.listdir('/proc/self/fd')) - before, 1, + 'bounded_open leaked fds on the blocking path') + + @unittest.skipIf(os.name == 'nt', 'POSIX fifo') + def test_open_completing_during_the_abandon_does_not_leak(self): + """The window the handoff lock exists for: the worker is at its store-or-close + decision when the caller gives up and drains the box. + + The `abandoned` Event is instrumented to park the worker there, because timing + alone never reaches that window -- 1500 tries against the unlocked version leaked + nothing, so a test that merely completes the open late proves nothing. An empty + `hit` means the instrumentation no longer bites and the window is untested.""" + hil_util = self.hil_util + fifo = Path(self.tmp.name) / 'fifo' + os.mkfifo(fifo) + caller = threading.current_thread() + drained, hit = threading.Event(), [] + + class RacingEvent(threading.Event): + def is_set(self): + v = super().is_set() + if not v and not hit and threading.current_thread() is not caller: + hit.append(True) + # bounded: the fixed bounded_open holds the lock across this call, so + # the caller cannot reach its abandon (and set drained) until we return + drained.wait(0.3) + return v + + shim = types.ModuleType('threading_shim') + shim.__dict__.update(threading.__dict__) + shim.Event = RacingEvent + hil_util.threading = shim + self.addCleanup(setattr, hil_util, 'threading', threading) + + before = len(os.listdir('/proc/self/fd')) + rd = os.open(fifo, os.O_RDONLY | os.O_NONBLOCK) # the O_WRONLY open completes at once + try: + self.assertIs(hil_util.bounded_open(str(fifo), os.O_WRONLY, 0.05), + hil_util.SYSFS_UNKNOWN) + drained.set() + time.sleep(0.1) # let an abandoned worker act on what it saw + self.assertTrue(hit, 'the abandon window was never entered') + self.assertLessEqual(len(os.listdir('/proc/self/fd')) - before, 1, + 'bounded_open stored the fd after the caller drained the box') + finally: + drained.set() + os.close(rd) + + +class SysfsUnknownIsNotAbsent(unittest.TestCase): + """read_sysfs must tell "no such attribute" (a fact) from "the read did not answer" + (not a fact). Every caller that concluded absence from the latter reported a healthy + board as a firmware regression.""" + + def setUp(self): + from helper import hil_util + self.hil_util = hil_util + self.saved = (hil_util._sysfs_stuck, hil_util._sysfs_blind_logged) + self.tmp = TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + + def tearDown(self): + # a blocked read strands a counted daemon thread; leaving the count raised would + # blind every later test in this process + self.hil_util._sysfs_stuck, self.hil_util._sysfs_blind_logged = self.saved + + def test_readable_attribute_returns_its_value(self): + p = Path(self.tmp.name) / 'serial' + p.write_text('CAFE01\n') + self.assertEqual(self.hil_util.read_sysfs(str(p)), 'CAFE01') + + def test_missing_attribute_is_none(self): + self.assertIsNone(self.hil_util.read_sysfs(str(Path(self.tmp.name) / 'nope'))) + + @unittest.skipIf(os.name == 'nt', 'POSIX fifo') + def test_blocking_read_is_unknown_not_absent(self): + """A reader-less FIFO stands in for the wedged device whose sysfs read never + returns; None here would read as "the board is gone".""" + fifo = Path(self.tmp.name) / 'fifo' + os.mkfifo(fifo) + t0 = time.monotonic() + v = self.hil_util.read_sysfs(str(fifo), grace=0.3) + self.assertLess(time.monotonic() - t0, 10, 'read_sysfs did not bound') + self.assertIs(v, self.hil_util.SYSFS_UNKNOWN) + self.assertIsNotNone(v) + + def test_blind_process_answers_unknown_for_a_readable_attribute(self): + p = Path(self.tmp.name) / 'serial' + p.write_text('CAFE01') + self.hil_util._sysfs_stuck = self.hil_util.SYSFS_STUCK_MAX + self.assertTrue(self.hil_util.sysfs_blind()) + self.assertIs(self.hil_util.read_sysfs(str(p)), self.hil_util.SYSFS_UNKNOWN) + self.assertIn('blind', self.hil_util.sysfs_blind_note()) + + def test_unknown_is_falsy_but_not_none(self): + # call sites use `(v or '')` idioms; the sentinel must keep working there while + # still being distinguishable from a real absence + self.assertFalse(self.hil_util.SYSFS_UNKNOWN) + self.assertIsNotNone(self.hil_util.SYSFS_UNKNOWN) + + +class UsbtestEnumerationVerdict(unittest.TestCase): + """test_device_usbtest must not report a healthy board as "no cafe:4010 device" just + because its own sysfs reads stopped answering.""" + + def setUp(self): + from helper import hil_util + self.hil_util = hil_util + self.td = TemporaryDirectory() + self.addCleanup(self.td.cleanup) + # a real device dir: usb_scan reads idVendor/idProduct with a plain open (they are + # lock-free descriptor fields), and only `serial` through the bounded reader + dev = Path(self.td.name) / '1-2' + dev.mkdir() + (dev / 'idVendor').write_text('cafe\n') + (dev / 'idProduct').write_text('4010\n') + (dev / 'serial').write_text('CAFE01\n') + for obj, name, val in ((hil_util, 'read_sysfs', hil_util.read_sysfs), + (hil_util, 'glob', hil_util.glob), + (hil_util, '_sysfs_stranded', {}), + (hil_test, '_enum_timeout', 1)): + self.addCleanup(setattr, obj, name, getattr(obj, name)) + setattr(obj, name, val) + hil_util.glob = types.SimpleNamespace(glob=lambda pat: [str(dev)]) + + def _fail(self, reader): + self.hil_util.read_sysfs = reader + with self.assertRaises(hil_test.TestFail) as cm: + hil_test.test_device_usbtest({'uid': 'CAFE01', 'name': 'fake', 'flasher': {}}) + return str(cm.exception) + + def test_unknown_reads_do_not_claim_the_device_is_absent(self): + msg = self._fail(lambda p, *a, **kw: self.hil_util.SYSFS_UNKNOWN) + self.assertNotIn('no cafe:4010 device', msg) + self.assertIn('did not answer', msg) + + def test_a_readable_bus_without_the_device_still_says_absent(self): + msg = self._fail(lambda p, *a, **kw: 'OTHERUID') + self.assertIn('no cafe:4010 device', msg) + + +class UnresolvedControllerBucket(unittest.TestCase): + """An unresolved controller must budget in ONE bucket. Taking a permit on every slot + serialized the whole fleet the moment a worker went blind.""" + + def setUp(self): + import threading + from helper import hil_lock + self.hil_lock = hil_lock + self.saved = (hil_lock.controller_map, hil_lock.controller_meta, + hil_lock.controller_hints, hil_lock.log) + hil_lock.controller_map, hil_lock.controller_meta = {}, threading.Lock() + hil_lock.controller_hints, hil_lock.log = {}, lambda *a, **k: None + + def tearDown(self): + (self.hil_lock.controller_map, self.hil_lock.controller_meta, + self.hil_lock.controller_hints, self.hil_lock.log) = self.saved + + def _slots(self, uid, warn): + import threading + sems = self.hil_lock.make_permit_sems(threading.Semaphore, 2) + return self.hil_lock.controller_permit(sems, uid, warn_unknown=warn).slots + + def test_unresolved_boards_share_one_slot(self): + for warn in (False, True): + slots = self._slots('NOSUCHUID', warn) + self.assertEqual(len(slots), 1, 'unresolved uid took more than one slot') + self.assertEqual(slots, self._slots('OTHERUID', warn), + 'unresolved boards must share the bucket, not spread over it') + + def test_the_semaphore_array_is_long_enough_for_the_unknown_slot(self): + """UNKNOWN_SLOT indexes one PAST the real slots. An array sized to + CONTROLLER_SLOTS IndexErrors on the first unresolved board, inside a pool worker, + which map_async turns into a total loss of every board's results.""" + import threading + sems = self.hil_lock.make_permit_sems(threading.Semaphore, 2) + self.assertGreater(len(sems), self.hil_lock.UNKNOWN_SLOT) + + def test_the_unknown_bucket_never_lends_a_controller_a_second_budget(self): + """A private FULL budget let 2 unknown batteries join 2 resolved ones on the same + physical controller -- 4 where the width is 2. One at a time caps that at +1.""" + import threading + sems = self.hil_lock.make_permit_sems(threading.Semaphore, 2) + first = self.hil_lock.controller_permit(sems, 'NOSUCHUID') + first.__enter__() + self.addCleanup(first.__exit__) + second = self.hil_lock.controller_permit(sems, 'OTHERUID') + self.assertFalse(sems[second.slots[0]].acquire(blocking=False), + 'a second unresolved board got in alongside the first') + + def test_every_real_slot_keeps_the_full_width(self): + import threading + sems = self.hil_lock.make_permit_sems(threading.Semaphore, 2) + for s in sems[:self.hil_lock.CONTROLLER_SLOTS]: + self.assertTrue(s.acquire(blocking=False) and s.acquire(blocking=False)) + self.assertFalse(s.acquire(blocking=False)) + + +class ThroughputPayloadBound(unittest.TestCase): + """An unknown link speed must pick the FS payload, and each dd must be bounded by the + payload actually requested.""" + + def test_only_a_read_high_speed_gets_the_big_payload(self): + from helper import hil_util + for speed in (None, hil_util.SYSFS_UNKNOWN, '12', '1.5'): + self.assertTrue(hil_test.link_is_fs(speed), f'{speed!r} must scale as FS') + for speed in ('480', '5000', '10000'): + self.assertFalse(hil_test.link_is_fs(speed)) + + def test_dd_bound_scales_with_the_payload_and_stays_bounded(self): + self.assertGreater(hil_test.dd_timeout(16), hil_test.dd_timeout(1)) + self.assertGreaterEqual(hil_test.dd_timeout(1), 30) # setup + flush floor + # still an INNER bound: run_cmd's own timeout must stay the outer one + self.assertLess(hil_test.dd_timeout(16), hil_test.hil_util.CMD_TIMEOUT) + + +class FindDeviceCache(unittest.TestCase): + """usbtest.find_device's cache is keyed by sysname, a bus-topology path: after a + renumber it can name a different cafe:4010 board, and idVendor/idProduct are identical + on every one of them. Only `serial` tells them apart.""" + + def setUp(self): + import usbtest + self.usbtest = usbtest + self.tmp = TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.saved_sys_usb = usbtest.SYS_USB + usbtest.SYS_USB = Path(self.tmp.name) + usbtest._DEV_CACHE.clear() + self._dev('1-2', 'AAAA', devnum=2) + self._dev('1-3', 'BBBB', devnum=3) + + def tearDown(self): + self.usbtest.SYS_USB = self.saved_sys_usb + self.usbtest._DEV_CACHE.clear() + + def _dev(self, sysname, serial, devnum): + d = Path(self.tmp.name) / sysname + d.mkdir() + for name, val in (('idVendor', self.usbtest.VID), ('idProduct', self.usbtest.PID), + ('serial', serial), ('busnum', '1'), ('devnum', str(devnum)), + ('speed', '480'), ('bcdDevice', '0104')): + (d / name).write_text(val + '\n') + + def test_cached_sysname_with_another_boards_serial_is_rejected(self): + self.usbtest._DEV_CACHE['bbbb'] = '1-2' # renumbered: 1-2 is board AAAA now + dev = self.usbtest.find_device('BBBB') + self.assertEqual(dev['sysname'], '1-3') + self.assertEqual(dev['serial'], 'BBBB') + self.assertEqual(self.usbtest._DEV_CACHE['bbbb'], '1-3') + + def test_cached_sysname_with_the_right_serial_is_kept(self): + self.usbtest._DEV_CACHE['bbbb'] = '1-3' + dev = self.usbtest.find_device('BBBB') + self.assertEqual((dev['sysname'], dev['serial']), ('1-3', 'BBBB')) + + def test_a_cached_device_that_vanished_falls_back_to_the_scan(self): + self.usbtest._DEV_CACHE['bbbb'] = '1-9' # gone from sysfs + self.assertEqual(self.usbtest.find_device('BBBB')['sysname'], '1-3') + + +class EnumPollDoesNotReReadAWedgedPath(unittest.TestCase): + """usbtest_enumerated re-globs every device each 0.2 s pass. One wedged peer therefore + strands a fresh bounded reader thread per pass, and SYSFS_STUCK_MAX=4 of those blind + the WHOLE worker for the rest of the run -- measured at 8 s of polling. A path that + already stranded is known-unknown; reading it again buys nothing and costs the + blindness budget.""" + + def test_a_stranded_path_is_read_at_most_once(self): + from contextlib import contextmanager + from helper import hil_lock, hil_util + + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + # A REAL device dir: usb_scan reads idVendor/idProduct with a plain open and + # `continue`s on OSError, so a bare FIFO is skipped before the bounded read is ever + # reached -- this test passed identically with the memo deleted until the ids were + # added. The FIFO must be the `serial` of a device that survives the cheap filter. + devdir = Path(td.name) / '1-2' + devdir.mkdir() + (devdir / 'idVendor').write_text('cafe\n') + (devdir / 'idProduct').write_text('4010\n') + wedged = devdir / 'serial' + os.mkfifo(wedged) # open() blocks forever: no writer, ever + + def patch(obj, name, value): + self.addCleanup(setattr, obj, name, getattr(obj, name)) + setattr(obj, name, value) + + def _permit(uid): + yield + + from helper import hil_util as _hu2 + patch(_hu2, 'glob', types.SimpleNamespace(glob=lambda p: [str(devdir)])) + patch(_hu2, '_sysfs_stranded', {}) + patch(hil_lock, 'usbtest_permit', contextmanager(_permit)) + # Long enough for several 2 s reads, but under the blindness cap -- past the cap + # sysfs_blind() short-circuits reads on its own and would mask the memo entirely. + patch(hil_test, '_enum_timeout', 8) + # the blindness counter is process-global and never decrements: restore it or this + # test blinds every test that runs after it + patch(hil_util, '_sysfs_stuck', hil_util._sysfs_stuck) + + # Count LEAKED THREADS, not _sysfs_stuck: a strand is booked only the first time a + # path is seen, so the counter is deduped by the memo's own bookkeeping and stays 1 + # even when the memo is broken. Each re-read blocks a fresh thread on the FIFO + # forever and leaks its fd -- which is the cost the memo exists to avoid, and the + # only thing here that actually moves when it regresses. + before = threading.active_count() + with self.assertRaises(hil_test.TestFail): # never enumerates, by construction + hil_test.test_device_usbtest({'name': 'b', 'uid': 'UID1', + 'flasher': {'name': 'openocd'}}) + self.assertLessEqual(threading.active_count() - before, 1, + 'the poll re-read a path it already knew was stranded') + + +class ReRunSpecNamesOnlyWhatFailed(unittest.TestCase): + """The pool-guard path used to leave this unwritten -- and a fresh run has already + unlinked it -- so build.yml's re-run step found nothing and GitHub re-tested all ~26 + boards to find the one that wedged.""" + + def test_only_failed_boards_and_their_failed_tests(self): + with TemporaryDirectory() as td: + d = Path(td) + spec = d / 'cfg.failed' + hil_test._write_failed_spec(spec, d, [ + ('good', 0, [], None, 1.0), + ('bad', 2, ['device/cdc_msc'], None, 1.0), + ('wedged', 1, [], None, 0.0), # never reported: no test list + ]) + got = spec.read_text() + self.assertIn('-b bad', got) + self.assertIn('-bt bad:device/cdc_msc', got) + self.assertIn('-b wedged', got) + self.assertNotIn('good', got) + + def test_an_all_green_run_removes_a_stale_spec(self): + with TemporaryDirectory() as td: + d = Path(td) + spec = d / 'cfg.failed' + spec.write_text('--accumulate -b stale') + hil_test._write_failed_spec(spec, d, [('good', 0, [], None, 1.0)]) + self.assertFalse(spec.exists(), 'a stale spec would re-run last time\'s boards') + + +class WedgedPidsFailsClosed(unittest.TestCase): + """A scan that could not SEE the holder must not report "no holder". The holder is + root-owned (run_case uses sudo -n when the node is not writable) and that is exactly + what a hidepid/ProtectProc mount hides — so an unreadable /proc reading as clear + clears unrecovered_hang and lets cleanup unbind a device whose usbfs lock is still + held, which deadlocks the bus rather than one board.""" + + def test_returns_a_completeness_flag_not_just_pids(self): + import usbtest + got = usbtest.wedged_pids('/dev/bus/usb/999/999') + self.assertIsInstance(got, tuple) + self.assertEqual(len(got), 2, 'the caller needs (pids, complete)') + + def test_a_restricted_proc_is_reported_incomplete(self): + import usbtest + self.addCleanup(setattr, usbtest.os, 'geteuid', usbtest.os.geteuid) + self.addCleanup(setattr, usbtest.os, 'access', usbtest.os.access) + usbtest.os.geteuid = lambda: 1000 # not root + usbtest.os.access = lambda p, m: False # /proc/1/cmdline unreadable + _, complete = usbtest.wedged_pids('/dev/bus/usb/999/999') + self.assertFalse(complete, 'a hidden holder was reported as absent') + + +@unittest.skipIf(os.name == 'nt', 'POSIX shell fakes') +@unittest.skipIf(sys.version_info < (3, 11), 'fake-pymtp steering needs PYTHONSAFEPATH') +class StrandMemoRemembersUnstattablePaths(unittest.TestCase): + """A stranded path whose inode could not be read is stored as None -- which dict.get() + also returns for a MISS. Testing `is not None` therefore treats 'known stranded' as + 'never seen', and every later call strands ANOTHER permanent thread and fd on a path we + already know is wedged. That is the exact unbounded growth SYSFS_STUCK_MAX exists to + stop, and it is invisible: `first = path not in _sysfs_stranded` is False, so the + blindness counter does not advance either.""" + + def setUp(self): + from helper import hil_util + self.hil_util = hil_util + self.addCleanup(hil_util._sysfs_stranded.clear) + hil_util._sysfs_stranded.clear() + self.addCleanup(setattr, hil_util, '_sysfs_stuck', hil_util._sysfs_stuck) + hil_util._sysfs_stuck = 0 + self.td = TemporaryDirectory(); self.addCleanup(self.td.cleanup) + self.fifo = os.path.join(self.td.name, 'serial') + os.mkfifo(self.fifo) # open() succeeds, read() never returns + + def test_an_unstattable_strand_is_not_re_read(self): + self.hil_util._sysfs_stranded[self.fifo] = None # as the record path stores it + before = threading.active_count() + self.assertIs(self.hil_util.read_sysfs(self.fifo, grace=0.5), + self.hil_util.SYSFS_UNKNOWN) + self.assertEqual(threading.active_count(), before, + 'a known-stranded path was re-read, stranding another thread') + + def test_a_live_strand_is_still_re_read_when_the_node_is_replaced(self): + """The memo must not become permanent blindness: a NEW inode at the same path is a + different device and has to be read.""" + self.hil_util._sysfs_stranded[self.fifo] = 999999999 # inode that is not this one + with open(os.path.join(self.td.name, 'other'), 'w') as f: + f.write('ok\n') + os.replace(os.path.join(self.td.name, 'other'), self.fifo) + self.assertEqual(self.hil_util.read_sysfs(self.fifo, grace=0.5), 'ok') + + +class MtpGioOrdering(_MtpFakeRig, unittest.TestCase): + """gio must not run until the device is READY. + + gvfs claims an MTP device only AFTER udev probing, so the mount this unmounts cannot + exist before /dev/libmtp- is published -- an unmount issued earlier is a + guaranteed no-op that still forks a process, and it leaves the window between the + unmount and the open unprotected, which is the hang it exists to prevent. Running it + per poll iteration also forks one gio per second of the enumeration budget.""" + + def setUp(self): + super().setUp() + tmp = Path(self.tmp.name) + self.gio_log = tmp / 'gio.log' + binn = tmp / 'bin'; binn.mkdir() + (binn / 'gio').write_text('#!/bin/sh\necho "$@" >> "$GIO_LOG"\n') + (binn / 'gio').chmod(0o755) + for k in ('PATH', 'GIO_LOG'): + old = os.environ.get(k) + self.addCleanup(lambda k=k, v=old: os.environ.__setitem__(k, v) + if v is not None else os.environ.pop(k, None)) + os.environ['GIO_LOG'] = str(self.gio_log) + os.environ['PATH'] = f'{binn}:{os.environ["PATH"]}' + + def _gio_calls(self): + return self.gio_log.read_text().splitlines() if self.gio_log.exists() else [] + + def test_gio_does_not_run_before_the_device_is_ready(self): + (Path(self.tmp.name) / 'dev' / 'libmtp-1-1').unlink() # never becomes ready + os.environ['FAKE_PYMTP_MODE'] = 'absent' + run_bounded(lambda: hil_test.test_device_mtp(self.board), 30) + calls = self._gio_calls() + self.assertEqual(calls, [], f'gio ran {len(calls)}x with no device ready: {calls}') + + def test_gio_still_runs_once_the_device_is_ready(self): + """The guard must delay the unmount, not delete it.""" + os.environ['FAKE_PYMTP_MODE'] = 'ok' + hil_test.test_device_mtp(self.board) + self.assertTrue(self._gio_calls(), 'gio never ran for a ready device') + + +class MtpGioFallthrough(unittest.TestCase): + """The missing-gio path must fall THROUGH to detection. `continue` there skips the + deadline check and the sleep as well, spinning at 100% CPU until the caller's outer + kill — reported as a wedged DUT for a missing apt package.""" + + def test_a_missing_gio_still_bounds_the_session(self): + import subprocess + with TemporaryDirectory() as td: + env = {**os.environ, 'PATH': td, # no gio, no anything + 'PYTHONPATH': os.path.join(TEST_DIR, 'stubs'), + 'FAKE_PYMTP_MODE': 'none', 'PYTHONSAFEPATH': '1'} + t0 = time.monotonic() + r = subprocess.run([sys.executable, + str(Path(TEST_DIR).parents[0] / 'mtp_test.py'), + '--uid', 'CAFE01', '--timeout', '3'], + capture_output=True, text=True, timeout=60, env=env) + elapsed = time.monotonic() - t0 + self.assertLess(elapsed, 30, f'did not honour --timeout 3 ({elapsed:.1f}s)') + self.assertNotEqual(r.returncode, 0) + # The assertions above are satisfied by an immediate CRASH, which is exactly what + # shipped through this test once: `pass` left gio unbound and the next line + # dereferenced it. Assert the behaviour the docstring names -- it POLLED for the + # device (so it spent its budget) and did not die on a traceback. + self.assertGreater(elapsed, 2.0, + f'exited without polling ({elapsed:.1f}s) -- it crashed') + self.assertNotIn('Traceback', r.stderr) + self.assertIn('MTP device not found', r.stdout + r.stderr) + + +class RunWhileContract(unittest.TestCase): + """The read-while-we-write runner. Its child can still outlast SIGKILL -- but unlike + the thread it replaced, an abandoned child is a real process in its own session, so + the containment sweep finds it and the report names it.""" + + def setUp(self): + from helper import hil_util + self.hil_util = hil_util + + def test_an_error_in_work_is_not_swallowed(self): + """A `return` inside the reap's `finally` discarded it: an assert in the CDC + write half vanished and the caller went on to compare data it never sent.""" + def boom(): + raise AssertionError('the write failed') + with self.assertRaises(AssertionError): + self.hil_util.run_alongside(['sh', '-c', 'printf X'], boom, 5) + + def test_the_child_is_reaped_even_when_work_raises(self): + seen = {} + + def boom(): + raise AssertionError('x') + with self.assertRaises(AssertionError): + self.hil_util.run_alongside(['sleep', '20'], boom, 1) + # nothing of ours is left running: the reap ran on the error path too + import subprocess + out = subprocess.run(['pgrep', '-f', '^sleep 20'], capture_output=True, text=True) + seen['strays'] = [p for p in out.stdout.split() if p] + self.assertEqual(seen['strays'], [], 'work() raising leaked the child') + + def test_an_abandoned_child_is_in_its_own_session(self): + """killpg on it reaps whatever it spawned, and it cannot take our group with it.""" + import subprocess + pgids = {} + + def check(): + time.sleep(0.2) + pgids['child'] = os.getpgid(self._proc_pid) + + real_popen = subprocess.Popen + + def spy(argv, **kw): + p = real_popen(argv, **kw) + self._proc_pid = p.pid + return p + self.addCleanup(setattr, subprocess, 'Popen', real_popen) + subprocess.Popen = spy + self.hil_util.run_alongside(['sleep', '0.5'], check, 5) + subprocess.Popen = real_popen + self.assertNotEqual(pgids['child'], os.getpgid(0)) + + +class StrandedPathMemoInvalidates(unittest.TestCase): + """The memo lives in read_sysfs, so every bounded reader gets it -- call-site memos + meant each new scanner had to remember (get_printer_dev and the throughput probe did + not). And it MUST expire on re-enumeration: the key is a bus path, which does not + change when a device comes back on the same port, so a memo that never invalidates + makes a board the branch's own HUNG reflash just recovered permanently invisible.""" + + def setUp(self): + from helper import hil_util + self.hil_util = hil_util + self.td = TemporaryDirectory() + self.addCleanup(self.td.cleanup) + for name in ('_sysfs_stranded', '_sysfs_stuck'): + self.addCleanup(setattr, hil_util, name, getattr(hil_util, name)) + hil_util._sysfs_stranded = {} + hil_util._sysfs_stuck = 0 + + def test_a_stranded_path_is_not_re_read(self): + f = Path(self.td.name) / 'serial' + os.mkfifo(f) # never answers + self.assertIs(self.hil_util.read_sysfs(str(f), 0.3), self.hil_util.SYSFS_UNKNOWN) + after_first = self.hil_util._sysfs_stuck + t0 = time.monotonic() + for _ in range(3): + self.assertIs(self.hil_util.read_sysfs(str(f), 0.3), + self.hil_util.SYSFS_UNKNOWN) + self.assertLess(time.monotonic() - t0, 0.3, 'the memo did not short-circuit') + self.assertEqual(self.hil_util._sysfs_stuck, after_first, + 'repeat reads spent more of the blindness budget') + + def test_re_enumeration_clears_it(self): + """A new device on the same busport gets a fresh sysfs node, hence a fresh inode. + Without this the memo outlives the wedge it recorded.""" + f = Path(self.td.name) / 'serial' + os.mkfifo(f) + self.assertIs(self.hil_util.read_sysfs(str(f), 0.3), self.hil_util.SYSFS_UNKNOWN) + f.unlink() + f.write_text('CAFE01\n') # same path, new inode = re-enumerated + self.assertEqual(self.hil_util.read_sysfs(str(f), 0.3), 'CAFE01', + 'a recovered device stayed invisible') + + def test_a_vanished_path_is_not_remembered_as_stranded(self): + f = Path(self.td.name) / 'serial' + os.mkfifo(f) + self.assertIs(self.hil_util.read_sysfs(str(f), 0.3), self.hil_util.SYSFS_UNKNOWN) + f.unlink() + self.assertIsNone(self.hil_util.read_sysfs(str(f), 0.3)) + + +class UsbScanIsTheOneWalk(unittest.TestCase): + """Three call sites each had a different subset of the three things this must get + right; none had all three. The expensive read is `serial` -- served under the device + lock a wedged usbfs ioctl holds -- so it must come LAST, only for devices the free + descriptor fields could not rule out, and never twice for a path that stranded.""" + + def setUp(self): + from helper import hil_util + self.hil_util = hil_util + self.td = TemporaryDirectory() + self.addCleanup(self.td.cleanup) + self.root = Path(self.td.name) + self.reads = [] + real = hil_util.read_sysfs + + def counting(path, *a, **k): + self.reads.append(path) + return real(path, *a, **k) + self.addCleanup(setattr, hil_util, 'read_sysfs', real) + hil_util.read_sysfs = counting + self.addCleanup(setattr, hil_util, '_sysfs_stranded', + dict(hil_util._sysfs_stranded)) + self.addCleanup(setattr, hil_util, '_sysfs_stuck', hil_util._sysfs_stuck) + + def _dev(self, name, vid, pid, serial='S1', fifo=False): + d = self.root / name + d.mkdir() + (d / 'idVendor').write_text(vid + '\n') + (d / 'idProduct').write_text(pid + '\n') + if fifo: + os.mkfifo(d / 'serial') # a read that never answers + else: + (d / 'serial').write_text(serial + '\n') + return d + + def _scan(self, **kw): + import glob as _g + real_glob = _g.glob + self.addCleanup(setattr, self.hil_util.glob, 'glob', real_glob) + self.hil_util.glob.glob = lambda pat: [str(p) for p in self.root.iterdir()] + return self.hil_util.usb_scan(**kw) + + def test_a_mismatched_vid_pid_costs_no_serial_read(self): + self._dev('1-1', '1234', '5678') + self._dev('1-2', 'cafe', '4010', serial='UID1') + devs, unknown = self._scan(vid_pid=('cafe', '4010')) + self.assertEqual([d['serial'] for d in devs], ['UID1']) + self.assertFalse(unknown) + # the ruled-out device's locked attribute was never touched + self.assertNotIn(str(self.root / '1-1' / 'serial'), self.reads) + + def test_a_wedged_device_stays_unproven_on_every_scan(self): + """The memo lives in read_sysfs now, so usb_scan still CALLS it each pass -- what + must not repeat is the cost. StrandedPathMemoInvalidates covers the short-circuit; + here the invariant is that the device stays out of the results and absence stays + unproven, however many times we look.""" + from helper import hil_util + self._dev('1-1', 'cafe', '4010', fifo=True) + first = None + t0 = time.monotonic() + for _ in range(3): + devs, unknown = self._scan() + self.assertTrue(unknown, 'a stranded read must leave absence unproven') + self.assertEqual(devs, []) + if first is None: + first = hil_util._sysfs_stuck + self.assertEqual(hil_util._sysfs_stuck, first, + 'repeat scans spent more of the blindness budget') + self.assertLess(time.monotonic() - t0, 3.0, 'repeat scans re-paid the grace') + + +class BoundedOpenTellsAbsentFromUnknown(unittest.TestCase): + """Same contract as read_sysfs, in the sibling function of the same file: a real + OSError is a FACT (EBUSY, ENOENT, EACCES), a blocked open is UNKNOWN. Folding both + into None made an ordinary EBUSY report as a USB wedge, sending the operator to + usb-kernel-recover for healthy hardware -- and left the stranded thread uncounted, + so the cap that exists to stop the fd/thread ceiling never saw it.""" + + def setUp(self): + from helper import hil_util + self.hil_util = hil_util + self.td = TemporaryDirectory() + self.addCleanup(self.td.cleanup) + + def test_a_real_oserror_is_a_fact(self): + missing = str(Path(self.td.name) / 'nope') + self.assertIsNone(self.hil_util.bounded_open(missing, os.O_RDONLY, 1)) + + def test_a_blocked_open_is_unknown_and_counted(self): + fifo = Path(self.td.name) / 'fifo' + os.mkfifo(fifo) # no reader: O_WRONLY blocks forever + self.addCleanup(setattr, self.hil_util, '_sysfs_stuck', + self.hil_util._sysfs_stuck) + before = self.hil_util._sysfs_stuck + got = self.hil_util.bounded_open(str(fifo), os.O_WRONLY, 0.3) + self.assertIs(got, self.hil_util.SYSFS_UNKNOWN) + self.assertEqual(self.hil_util._sysfs_stuck, before + 1, + 'a stranded open is invisible to the blindness budget') + + +class UsbtestSysfsReadIsCapped(unittest.TestCase): + """find_device re-scans every cafe:4010 peer after EVERY case, so the local twin -- + which had no SYSFS_STUCK_MAX -- stranded a thread and an fd per wedged peer per case. + Delegating to hil_util gets the cap, and the deferred import keeps usbtest.py + importable standalone.""" + + def test_a_stranded_read_counts_against_the_shared_cap(self): + import usbtest + from helper import hil_util + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + wedged = Path(td.name) / 'serial' + os.mkfifo(wedged) # no writer: open() never returns + self.addCleanup(setattr, hil_util, '_sysfs_stuck', hil_util._sysfs_stuck) + before = hil_util._sysfs_stuck + # UNKNOWN, not None: folding them made a blinded scan read as "device dropped + # off the bus", which aborts past the HUNG reflash + self.assertIs(usbtest._read_sysfs_bounded(wedged, grace=0.5), + hil_util.SYSFS_UNKNOWN) + self.assertEqual(hil_util._sysfs_stuck, before + 1, + 'usbtest reads are invisible to the blindness budget') + + +class AbandonExitSurvivesAFailedFork(unittest.TestCase): + """Pool() forks, and after a convoy -- every stranded read holding a thread and an fd -- + that fork is what hits EAGAIN/ENOMEM. It now runs inside the try, so the finally can + reach _abandon_exit with pool and mgr still None.""" + + def test_none_pool_and_manager_still_write_the_banner(self): + # a subprocess, because _abandon_exit ends in os._exit: in-process it would take + # the test runner with it, before any assertion could run + import subprocess + with TemporaryDirectory() as td: + report = Path(td) / 'hil_report.md' + report.write_text('| board | test |\n|---|---|\n', encoding='utf-8') + src = ( + 'import sys, types\n' + f'sys.path.insert(0, {str(Path(TEST_DIR).parents[0])!r})\n' + 'st = types.ModuleType("serial")\n' + 'st.Serial = type("Serial", (), {})\n' + 'st.SerialException = type("SerialException", (Exception,), {})\n' + 'st.SerialTimeoutException = type("E2", (Exception,), {})\n' + 'sys.modules.setdefault("serial", st)\n' + 'import hil_test\n' + f'hil_test._abandon_exit(None, None, True, 1, __import__("pathlib")' + f'.Path({str(report)!r}))\n') + r = subprocess.run([sys.executable, '-c', src], capture_output=True, + text=True, timeout=120) + self.assertEqual(r.returncode, 1, r.stderr) + self.assertTrue(report.read_text().startswith('**HIL run abandoned'), + 'the abandon banner never reached the report') + + def test_kill_pool_children_tolerates_a_pool_that_never_existed(self): + from helper import hil_health + self.assertEqual(hil_health.kill_pool_children(None), 0) + self.assertEqual(hil_health.kill_pool_children(None, None), 0) + + +class UsbtestOuterBoundIsOneValue(unittest.TestCase): + """The bound usbtest is TOLD and the bound run_cmd ENFORCES must be the same number. + Three separate expressions disagreed: --skip-flash appended no --outer-timeout at all + (usbtest reads 0 as no limit), and the no-recovery branch narrowed only the CHILD's + view while run_cmd still waited for a recovery reserve nothing on that path can + spend -- a pool worker and its battery permit idle for the difference.""" + + def _invoke(self, flasher, skip_flash=False): + from contextlib import contextmanager + from helper import hil_lock, hil_util + + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + dev = Path(td.name) / 'dev1' + dev.mkdir() + for attr, val in (('serial', 'UID1'), ('idVendor', 'cafe'), ('idProduct', '4010')): + (dev / attr).write_text(val + '\n') + + def patch(obj, name, value): + self.addCleanup(setattr, obj, name, getattr(obj, name)) + setattr(obj, name, value) + + def _permit(uid): + yield + + seen = {} + + def fake_run(cmd, **kw): + import subprocess + seen['cmd'], seen['timeout'] = cmd, kw.get('timeout') + return subprocess.CompletedProcess(cmd, 1, stdout=b'', stderr=b'stub') + + from helper import hil_util as _hu + patch(_hu, 'glob', types.SimpleNamespace(glob=lambda p: [str(dev)])) + # the blindness latch and the stranded memo are process-global: another class's + # wedged-FIFO test would otherwise make every read here answer SYSFS_UNKNOWN + patch(_hu, '_sysfs_stuck', 0) + patch(_hu, '_sysfs_stranded', {}) + patch(hil_lock, 'usbtest_permit', contextmanager(_permit)) + patch(hil_test, 'skip_flash', skip_flash) + patch(hil_test, '_current_fw', '/tmp/fw.elf') + patch(hil_util, 'run_cmd', fake_run) + with self.assertRaises(hil_test.TestFail): + hil_test.test_device_usbtest({'name': 'b', 'uid': 'UID1', 'flasher': flasher}) + return seen + + def _outer_flag(self, cmd): + toks = cmd.split() + self.assertIn('--outer-timeout', toks, 'usbtest reads a missing bound as UNLIMITED') + return int(toks[toks.index('--outer-timeout') + 1]) + + def test_a_recoverable_board_reserves_the_recovery_budget(self): + seen = self._invoke({'name': 'openocd', 'vid_pid': '0x1366 0x1024'}) + want = hil_test.USBTEST_BATTERY_BUDGET + hil_test.USBTEST_RECOVERY_BUDGET + self.assertEqual(self._outer_flag(seen['cmd']), want) + self.assertEqual(seen['timeout'], want) + + def test_a_board_with_no_recovery_does_not_pay_for_one(self): + seen = self._invoke({'name': 'stlink', 'uid': 'X'}) # never convoy_safe + outer = self._outer_flag(seen['cmd']) + self.assertEqual(seen['timeout'], outer, 'the two bounds disagree') + # It does not carry the RECOVERY reserve it cannot spend... + self.assertLess(outer, hil_test.USBTEST_BATTERY_BUDGET + + hil_test.USBTEST_RECOVERY_BUDGET) + # ...but it MUST still exceed the child's own --budget. The battery checks the + # budget before dispatching, so it can overshoot by one already-started case; an + # equal bound SIGKILLs it just as it goes to print, turning ~29 real per-case + # verdicts into "usbtest did not run" and re-paying the whole battery on retry. + toks = seen['cmd'].split() + budget = int(toks[toks.index('--budget') + 1]) + case_timeout = int(toks[toks.index('--timeout') + 1]) + self.assertGreaterEqual(outer - budget, case_timeout, + 'the outer kill can land mid-case, before the JSON') + + def test_skip_flash_still_bounds_the_child(self): + seen = self._invoke({'name': 'openocd', 'vid_pid': '0x1366 0x1024'}, skip_flash=True) + self.assertEqual(self._outer_flag(seen['cmd']), seen['timeout']) + + +class UsbtestRetryPolicy(unittest.TestCase): + """The pool guard bounds ONE battery; the retry loop multiplies it by max_retry. + So the loop must retry only what a retry can fix.""" + + def _patch(self, obj, name, value): + # addCleanup, not a finally: a failing assert must not leave the real module + # patched for whatever test runs next (max_retry only exists once main() ran, + # so restoring it means DELETING it again) + if hasattr(obj, name): + self.addCleanup(setattr, obj, name, getattr(obj, name)) + else: + self.addCleanup(delattr, obj, name) + setattr(obj, name, value) + + def _attempts(self, exc): + """How many times test_example runs the test fn before giving up.""" + import hil_flash + calls = [] + + def fake_test(board): + calls.append(1) + raise exc + + self._patch(hil_flash, 'find_firmware', lambda *a, **k: Path('/nonexistent/fw.elf')) + self._patch(hil_test, 'skip_flash', True) # no probe, no hardware + self._patch(hil_test, 'max_retry', 3) + self._patch(hil_test, 'log_line', lambda *a, **k: None) + hil_test.test_fake_example = fake_test + self.addCleanup(delattr, hil_test, 'test_fake_example') + hil_test.test_example({'name': 'b', 'uid': 'u', 'flasher': {'name': 'openocd'}}, + 'v', 'fake/example') + return len(calls) + + def test_a_per_case_verdict_is_not_retried(self): + # re-running the battery only re-observes a number the JSON already reported + self.assertEqual(self._attempts(hil_test.TestFail('29/30', parsed=True)), 1) + + def test_a_transient_failure_is_retried(self): + self.assertEqual(self._attempts(hil_test.TestFail('usbtest did not run')), 3) + + +class UsbtestOuterKillStaysRetryable(unittest.TestCase): + """rc 124 is run_cmd's timer expiring, NOT proof the DUT is wedged -- a healthy + battery can hit it under load. Suppressing the retry to save the budget also + suppresses the reflash test_example does before each attempt, which is the only + thing left to unpoison the DUT where usbtest's in-band recovery is off.""" + + def setUp(self): + from contextlib import contextmanager + from helper import hil_lock + self.td = TemporaryDirectory() + self.addCleanup(self.td.cleanup) + dev = Path(self.td.name) / 'dev1' + dev.mkdir() + # a real (readable) fake sysfs node, so the bounded reads run unmodified + for attr, val in (('serial', 'UID1'), ('idVendor', 'cafe'), ('idProduct', '4010')): + (dev / attr).write_text(val + '\n') + self.dev = dev + + def patch(obj, name, value): + saved = getattr(obj, name) + self.addCleanup(setattr, obj, name, saved) + setattr(obj, name, value) + + from helper import hil_util as _hu + patch(_hu, 'glob', types.SimpleNamespace(glob=lambda p: [str(dev)])) + def _permit(uid): # a real generator: a lambda returning an iterator has + yield # no .throw(), so any raise inside the `with` would + # surface as an AttributeError from contextlib instead + patch(hil_lock, 'usbtest_permit', contextmanager(_permit)) + patch(hil_test, 'skip_flash', True) + + def test_rc_124_stays_retryable(self): + import subprocess + from helper import hil_util + saved = hil_util.run_cmd + self.addCleanup(setattr, hil_util, 'run_cmd', saved) + hil_util.run_cmd = lambda *a, **k: subprocess.CompletedProcess( + 'usbtest', 124, stdout=b'', stderr=b'killed on the outer bound') + with self.assertRaises(hil_test.TestFail) as cm: + hil_test.test_device_usbtest({'name': 'b', 'uid': 'UID1', + 'flasher': {'name': 'openocd'}}) + self.assertFalse(cm.exception.parsed, + 'the retry is the last reflash a poisoned DUT gets') + + def test_a_crashed_tool_stays_retryable(self): + import subprocess + from helper import hil_util + saved = hil_util.run_cmd + self.addCleanup(setattr, hil_util, 'run_cmd', saved) + hil_util.run_cmd = lambda *a, **k: subprocess.CompletedProcess( + 'usbtest', 1, stdout=b'', stderr=b'ImportError: no module named usbtest') + with self.assertRaises(hil_test.TestFail) as cm: + hil_test.test_device_usbtest({'name': 'b', 'uid': 'UID1', + 'flasher': {'name': 'openocd'}}) + self.assertFalse(cm.exception.parsed) + + +class RemoteDirIsScreened(unittest.TestCase): + """REMOTE_DIR reaches the rig through `rm -rf`, an scp remote path and an rsync + remote path -- all re-split and expanded by the REMOTE shell, none of them + protectable by quoting the local variable. So the script screens the value once + instead: it must survive that re-split unchanged, and `~` must keep working.""" + + def _run(self, remote_dir, *args, keep_going=False): + import subprocess + with TemporaryDirectory() as td: + # real ssh/scp/rsync would reach the rig; these just record the argv. Exit 77 + # unless the caller needs the script to run on to the second ssh. + rc = 0 if keep_going else 77 + for tool in ('ssh', 'scp', 'rsync'): + write_script(Path(td) / tool, f'echo "stub-{tool} $*" >&2; exit {rc}') + env = {**os.environ, 'REMOTE_DIR': remote_dir, 'REMOTE': 'stub', + 'PATH': td + os.pathsep + os.environ['PATH']} + return subprocess.run( + ['bash', str(Path(TEST_DIR).parents[0] / 'hil_ci.sh'), *args], + capture_output=True, text=True, timeout=60, env=env) + + def test_whitespace_is_refused(self): + # unscreened, the remote `rm -rf -- "$1"` gets a TRUNCATED path and deletes + # the wrong tree + r = self._run('/tmp/hil dir') + self.assertNotEqual(r.returncode, 0) + self.assertIn('REMOTE_DIR', r.stderr) + + def test_command_substitution_is_refused(self): + r = self._run('/tmp/$(touch pwned)') + self.assertNotEqual(r.returncode, 0) + self.assertIn('REMOTE_DIR', r.stderr) + + def test_bare_root_is_refused(self): + r = self._run('/') + self.assertNotEqual(r.returncode, 0) + self.assertIn('REMOTE_DIR', r.stderr) + + def test_a_tilde_path_is_accepted(self): + """The one override %q broke: `~` must reach the remote shell UNESCAPED or it + creates a literal '~' directory in the login dir.""" + r = self._run('~/tinyusb-hil') + self.assertIn('~/tinyusb-hil', r.stderr) # got as far as the first ssh + self.assertNotIn('\\~', r.stderr) # %q escapes it; the remote shell won't + + def test_paths_that_would_rm_rf_something_huge_are_refused(self): + """Passing the tilde through UNESCAPED is what makes this dangerous: the remote + shell expands `~/` to the login dir, so `rm -rf -- "$1"` takes out $HOME -- one + typo away from the documented REMOTE_DIR=~/dir override. A bare root, a + no-component path and a foreign ~user are the same class.""" + for bad in ('~/', '~root/x', '~-', '//', '/.', '/tmp/hil/'): + with self.subTest(remote_dir=bad): + r = self._run(bad) + self.assertNotEqual(r.returncode, 0, f'{bad!r} was accepted') + self.assertIn('REMOTE_DIR', r.stderr) + + def test_an_arg_containing_a_space_survives_the_remote_resplit(self): + """ssh joins its argv into ONE string the remote shell re-splits, so an unquoted + `-t 'host/cdc msc'` arrives as two arguments and hil_test.py sees a stray word + where it expects the config path.""" + r = self._run('/tmp/tinyusb-hil', '-t', 'host/cdc msc', keep_going=True) + run_line = [l for l in r.stderr.splitlines() if 'bash -s --' in l][-1] + self.assertIn(r'host/cdc\ msc', run_line) + + +class CaveatSurvivesAccumulate(unittest.TestCase): + """CI reruns with --accumulate: the sidecar keeps every earlier attempt's cells, but the + banner was recomputed per attempt. A first attempt on a degraded rig and a clean rerun + therefore published the degraded attempt's PASSES with no caveat on them -- and the + generated .failed spec reruns only failures, so those cells are never re-earned.""" + + def _rows(self, board, cell): + return [(board, 0, 0, [(board, {cell: 'OK'}, '1s')], 0)] + + def test_an_earlier_attempts_caveat_is_still_on_the_report(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + banner = '> **Rig note.** 2 process(es) in D state at start.\n' + + hil_test.accumulate_report(self._rows('boardA', 'cdc_msc'), rd, True, '', banner) + self.assertIn('Rig note', (rd / hil_test.REPORT_MD).read_text()) + + # the rerun: clean rig, so this attempt contributes no banner of its own + md = hil_test.accumulate_report(self._rows('boardB', 'cdc_msc'), rd, False, '', '') + self.assertIn('boardA', md) # the earlier cells are kept ... + self.assertIn('Rig note', md, + 'the caveat the earlier cells were collected under was dropped') + + def test_the_same_caveat_twice_is_not_stacked(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + banner = '> **Rig note.** 2 process(es) in D state at start.\n' + hil_test.accumulate_report(self._rows('boardA', 'cdc_msc'), rd, True, '', banner) + md = hil_test.accumulate_report(self._rows('boardB', 'cdc_msc'), rd, False, '', banner) + self.assertEqual(md.count('Rig note'), 1) + + +class BlindWorkerReachesTheReport(unittest.TestCase): + """A worker that exhausts its bounded-read budget answers SYSFS_UNKNOWN for every + attribute, so its "device not found" means "could not tell". That reached the log and + the per-cell failure text but NOT the table -- and the table is what gets pasted into + the PR. Seen live: run 31794359407 went blind in 4 workers and published 26 red cells + with no mention of it, several of them caused by the blindness rather than the board.""" + + def test_no_note_when_every_worker_could_see(self): + mret = [('boardA', 0, [], [], 1.0, False), ('boardB', 0, [], [], 1.0, False)] + self.assertEqual(hil_test._blind_note(mret), '') + + def test_the_note_names_the_boards_whose_verdicts_are_not_evidence(self): + mret = [('boardA', 0, [], [], 1.0, True), ('boardB', 0, [], [], 1.0, False), + ('boardC', 1, [], [], 1.0, True)] + note = hil_test._blind_note(mret) + self.assertIn('boardA', note) + self.assertIn('boardC', note) + self.assertNotIn('boardB', note) # it could see; do not smear its result + self.assertTrue(note.endswith('\n'), 'banners are line-oriented') + + def test_both_row_widths_survive_the_report_writers(self): + """The blindness flag widened the worker's result tuple to 6, but the pool-timeout + path still synthesises 5-field rows for boards that never reported and feeds them + to the same two writers. A fixed-width unpack in either one raises INSIDE the + containment path, which is where a raise costs every board's results.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + wide = ('boardA', 1, ['device/cdc_msc'], [('boardA', {'cdc_msc': '❌'}, '2s')], 2.0, True) + narrow = ('stuck', 1, [], None, 0) # what the timeout path builds + hil_test._write_failed_spec(rd / 'x.failed', rd, [wide, narrow]) + md = hil_test.accumulate_report([wide], rd, True, '', hil_test._blind_note([wide])) + self.assertIn('boardA', md) + self.assertIn('not all verdicts are evidence', md.lower()) + + def test_the_stray_note_names_the_board_and_survives_narrow_rows(self): + """Survivors ride back on the result tuple because main()'s own sweep runs after + the report is written on both abort paths -- the banner appended there was + computed and discarded.""" + wide = ('boardA', 0, [], [], 1.0, False, 2) + clean = ('boardB', 0, [], [], 1.0, False, 0) + note = hil_test._stray_note([wide, clean]) + self.assertIn('boardA', note) + self.assertNotIn('boardB', note) + self.assertIn('2', note) + self.assertEqual(hil_test._stray_note([clean]), '') + self.assertEqual(hil_test._stray_note([('stuck', 1, [], None, 0)]), '') + + def test_the_timeout_paths_synthetic_rows_do_not_crash_it(self): + """The pool-timeout path builds (name, 1, [], None, 0) for boards that never + reported -- five fields, no blindness to report -- and hands those around.""" + self.assertEqual(hil_test._blind_note([('stuck', 1, [], None, 0)]), '') + + +class PoolGuardKeepsWhatFinished(unittest.TestCase): + """The guard's 30-minute predecessor fired on 5 of the last 8 HIL jobs, so this is the + common failure, not an edge case: map_async discarded every board that had finished and + left the re-run spec unwritten, so CI re-tested all ~26 to find the one that wedged. + + Calls hil_test.drain_pool -- the loop main() actually runs. The predecessor of this test + built its own ThreadPool and its own drain loop and asserted on those, so deleting the + production drain outright left it green.""" + + class _It: + """Stands in for imap_unordered: yields, then blocks past any deadline.""" + + def __init__(self, ready): + self.ready, self.i = ready, 0 + + def next(self, timeout=None): + if self.i < len(self.ready): + self.i += 1 + return self.ready[self.i - 1] + raise MpTimeoutError + + def test_finished_rows_survive_a_guard_expiry(self): + boards = [{'name': 'fast1'}, {'name': 'fast2'}, {'name': 'wedged'}] + rows = [('fast1', 0, [], [], 1.0, False), ('fast2', 0, [], [], 1.0, False)] + with self.assertRaises(hil_test.PoolDrainTimeout) as cm: + hil_test.drain_pool(self._It(rows), boards, time.monotonic() + 5) + self.assertEqual([r[0] for r in cm.exception.finished], ['fast1', 'fast2']) + + def test_an_expired_deadline_stops_before_asking_for_more(self): + """Left <= 0 must not be handed to it.next() as a zero/negative timeout.""" + boards = [{'name': 'a'}, {'name': 'b'}] + it = self._It([('a', 0, [], [], 1.0, False)]) + with self.assertRaises(hil_test.PoolDrainTimeout) as cm: + hil_test.drain_pool(it, boards, time.monotonic() - 1) # already past + self.assertEqual(cm.exception.finished, []) + self.assertEqual(it.i, 0, 'asked the pool for a result after the deadline') + + def test_rows_collected_before_the_deadline_expires_are_kept_too(self): + """The OTHER raise site: boards finish, then the clock runs out between results. + Both sites must carry the rows -- a bare raise here loses a worker-width of rig + time just as map_async did, and the it.next() path alone does not prove it.""" + class Slow(self._It): + def next(self, timeout=None): + time.sleep(0.2) # each result eats into the deadline + return super().next(timeout) + + boards = [{'name': n} for n in ('a', 'b', 'c', 'd')] + rows = [(n, 0, [], [], 1.0, False) for n in ('a', 'b', 'c', 'd')] + with self.assertRaises(hil_test.PoolDrainTimeout) as cm: + hil_test.drain_pool(Slow(rows), boards, time.monotonic() + 0.3) + self.assertTrue(cm.exception.finished, 'rows collected before the expiry were lost') + + def test_every_board_finishing_returns_them_all(self): + boards = [{'name': 'a'}, {'name': 'b'}] + rows = [('a', 0, [], [], 1.0, False), ('b', 1, [], [], 2.0, False)] + got = hil_test.drain_pool(self._It(rows), boards, time.monotonic() + 5) + self.assertEqual(got, rows) + + +class WedgedBoardCosts(unittest.TestCase): + """Two decisions the containment latch makes, tested as decisions rather than through + test_board's loop -- the loop-level predecessor of these tests reimplemented that loop + and asserted on its own copy, which is how both defects survived it.""" + + def setUp(self): + self.addCleanup(setattr, hil_test, 'board_wedged', hil_test.board_wedged) + + def test_a_board_that_wedged_still_counts_as_an_error(self): + """It rendered a red cell but returned err_count 0, so main()'s sys.exit(err_count) + reported success and _write_failed_spec (`if err > 0`) left the board out of the + re-run entirely: a rig holding a D-state process published as a clean pass.""" + hil_test.board_wedged = 'usbtest HUNG' + # no real flasher: skip_flash isolates the accounting from hil_flash + self.addCleanup(setattr, hil_test, 'skip_flash', hil_test.skip_flash) + hil_test.skip_flash = True + # a firmware path must resolve or test_example returns 'skip (no binary)' before + # ever reaching the retry loop this is about + self.addCleanup(setattr, hil_flash, 'find_firmware', hil_flash.find_firmware) + hil_flash.find_firmware = lambda *a, **k: Path('fw.elf') + + def boom(*a, **k): + raise hil_test.TestFail('usbtest did not run') # unparsed: retryable + + self.addCleanup(setattr, hil_test, 'test_device_usbtest', hil_test.test_device_usbtest) + hil_test.test_device_usbtest = boom + board = {'name': 'b', 'uid': 'U', 'flasher': {'name': 'openocd'}, 'tests': []} + err, _status, _metric = hil_test.test_example(board, 'b', 'device/usbtest') + self.assertEqual(err, 1, 'a wedged board contributed nothing to the exit status') + + def test_the_teardown_park_does_not_flash_a_wedged_board(self): + """The park is a flash like any other: on a D-state-held node it blocks, survives + SIGKILL and leaves a stray -- added by the path that just declared the board wedged + and skipped every test for exactly that reason.""" + hil_test.board_wedged = '' + self.assertTrue(hil_test._should_park(False), 'a healthy board must still park') + hil_test.board_wedged = 'usbtest HUNG' + self.assertFalse(hil_test._should_park(False), + 'the teardown park would flash through the poisoned node') + self.assertFalse(hil_test._should_park(True), '--skip-flash must still suppress it') + + +class WedgeVerdictReachesTheLatch(unittest.TestCase): + """usbtest computes `unrecovered_hang` but never reported it, so hil_test inferred the + latch from `not recovery and 'HUNG' in out` and missed three cases: recovery ran and + FAILED (convoy-safe boards -- max32666fthr HUNG in the 08-14 run), the `inconclusive` + abort (which sets the flag but leaves no case at status HUNG), and an unparsable JSON, + which is the outer-timeout kill and the case where a wedge is most likely.""" + + def setUp(self): + self.addCleanup(setattr, hil_test, 'board_wedged', hil_test.board_wedged) + hil_test.board_wedged = '' + + def _run(self, stdout, rc=0): + from helper import hil_lock, hil_util + class R: + returncode = rc + stderr = b'' + R.stdout = stdout.encode() + self.addCleanup(setattr, hil_util, 'run_cmd', hil_util.run_cmd) + hil_util.run_cmd = lambda *a, **k: R() + # usbtest_enumerated is nested in test_device_usbtest, so stub what it calls + self.addCleanup(setattr, hil_util, 'usb_scan', hil_util.usb_scan) + hil_util.usb_scan = lambda **k: ([{'busport': '1-1', 'dir': '/x', 'vid': 'cafe', + 'pid': '4010', 'serial': 'U'}], False) + self.addCleanup(setattr, hil_lock, 'usbtest_permit', hil_lock.usbtest_permit) + from contextlib import contextmanager + hil_lock.usbtest_permit = contextmanager(lambda uid: iter([None])) + board = {'name': 'b', 'uid': 'U', 'flasher': {'name': 'openocd', 'vid_pid': '0x1 0x2'}} + try: + hil_test.test_device_usbtest(board) + except Exception: + pass + return hil_test.board_wedged + + def test_a_reported_wedge_latches_even_when_recovery_ran(self): + """`recovery` True means the flags were PASSED, not that they worked.""" + js = '{"serial":"U","speed":"480","tier":1,"passed":1,"failed":1,"notrun":0,' '"wedged":true,"cases":[{"num":1,"status":"FAIL"}]}' + self.assertTrue(self._run(js), 'a reported wedge did not latch') + + def test_no_wedge_reported_does_not_latch(self): + js = '{"serial":"U","speed":"480","tier":1,"passed":2,"failed":0,"notrun":0,' '"wedged":false,"cases":[]}' + self.assertFalse(self._run(js)) + + def test_an_unparseable_battery_that_mentions_HUNG_still_latches(self): + """rc 124 mid-print: no JSON to read, and this is the likeliest real wedge.""" + self.assertTrue(self._run('TEST 10 HUNG: device wedged mid-transfer', rc=124)) + + +class WedgedBoardCannotReportAPass(unittest.TestCase): + """The latch alone is not enough: it is set BEFORE the pass return, so an all-green + battery that still wedged returned `PASS 30/30`. That board then contributes 0 to + err_count, is omitted from the .failed re-run spec (which keys on err > 0), and the job + exits 0 with a D-state holder on the rig -- the exact silence this branch exists to end. + usbtest's `inconclusive` and `ambiguous` aborts fire AFTER the last case, so nothing + back-fills a BUDGET entry to make failed/notrun non-zero.""" + + def setUp(self): + self.addCleanup(setattr, hil_test, 'board_wedged', hil_test.board_wedged) + hil_test.board_wedged = '' + + def _cell(self, js): + """Returns ('pass', cell) or ('fail', message).""" + from helper import hil_lock, hil_util + class R: + returncode = 0 + stderr = b'' + R.stdout = js.encode() + self.addCleanup(setattr, hil_util, 'run_cmd', hil_util.run_cmd) + hil_util.run_cmd = lambda *a, **k: R() + self.addCleanup(setattr, hil_util, 'usb_scan', hil_util.usb_scan) + hil_util.usb_scan = lambda **k: ([{'busport': '1-1', 'dir': '/x', 'vid': 'cafe', + 'pid': '4010', 'serial': 'U'}], False) + self.addCleanup(setattr, hil_lock, 'usbtest_permit', hil_lock.usbtest_permit) + from contextlib import contextmanager + hil_lock.usbtest_permit = contextmanager(lambda uid: iter([None])) + board = {'name': 'b', 'uid': 'U', 'flasher': {'name': 'openocd', 'vid_pid': '0x1 0x2'}} + try: + return ('pass', hil_test.test_device_usbtest(board)) + except hil_test.TestFail as e: + return ('fail', str(e)) + + def test_an_all_pass_battery_that_wedged_is_not_a_pass(self): + kind, detail = self._cell('{"serial":"U","speed":"480","tier":1,"passed":30,' + '"failed":0,"notrun":0,"wedged":true,"cases":[]}') + self.assertEqual(kind, 'fail', f'a wedged board reported a green cell: {detail}') + self.assertIn('wedged', detail) + + def test_an_all_pass_battery_that_did_not_wedge_is_still_a_pass(self): + """The guard must key on the latch, not merely on having parsed a battery.""" + kind, cell = self._cell('{"serial":"U","speed":"480","tier":1,"passed":30,' + '"failed":0,"notrun":0,"wedged":false,"cases":[]}') + self.assertEqual(kind, 'pass', f'a healthy board was failed: {cell}') + self.assertIn('30/30', cell) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/hil/test/test_hil_health.py b/test/hil/test/test_hil_health.py new file mode 100644 index 000000000..5695cad6d --- /dev/null +++ b/test/hil/test/test_hil_health.py @@ -0,0 +1,636 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests for hil_health.py — pure logic against a synthetic /proc, no hardware. A real +# wedge cannot be manufactured on demand, so the detectors are exercised against fabricated +# inputs. hil_health is stdlib-only on purpose, so all of this runs on a bare CI runner +# with nothing skipped. Run directly: +# python3 test/hil/test/test_hil_health.py +import os +import signal +import sys +import threading +import time +import subprocess +import unittest +from multiprocessing import Pool +from pathlib import Path +from tempfile import TemporaryDirectory + +# the module under test lives in the parent dir (test/hil), not here +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from helper import hil_health + +REAL_PROC = hil_health.PROC + + +def make_proc(root: Path, procs: dict, with_pid1: bool = True) -> None: + """Build a synthetic /proc. `procs` maps pid -> (comm, state, cmdline); a None comm or + cmdline omits that file. `with_pid1=False` simulates a restricted /proc (hidepid=2), + where an empty scan must not be read as an all-clear.""" + for pid, (comm, state, cmdline) in procs.items(): + d = root / str(pid) + d.mkdir() + if comm is not None: + (d / 'comm').write_text(comm + '\n') + if cmdline is not None: + (d / 'cmdline').write_bytes(cmdline) + # field 2 is comm in parens; the state letter follows it. Deliberately use a comm + # containing ')' so a naive split() would pick the wrong field. + (d / 'stat').write_text(f'{pid} (we)ird) {state} 1 1 0 0 -1 0 0\n') + if with_pid1 and 1 not in procs: + d = root / '1' + d.mkdir() + (d / 'comm').write_text('systemd\n') + (d / 'cmdline').write_bytes(b'/sbin/init\0') + (d / 'stat').write_text('1 (systemd) S 0 1 0 0 -1 0 0\n') + (root / 'not-a-pid').mkdir() + + +class PatchCase(unittest.TestCase): + """For classes that patch PROCESS-GLOBAL state (os.kill, time.sleep, subprocess.Popen). + + addCleanup, never tearDown: tearDown does NOT run when setUp raises, so a no-op + os.kill or time.sleep would survive into every later test in this blocking pre-commit + suite -- turning one setUp failure into a cascade of nonsense results.""" + + def patch(self, obj, name, value): + self.addCleanup(setattr, obj, name, getattr(obj, name)) + setattr(obj, name, value) + + def restore(self, obj, name): + """Same guarantee for state a TEST BODY assigns directly: register the restore + from setUp so it holds even when the assert between fails.""" + self.addCleanup(setattr, obj, name, getattr(obj, name)) + + +class ProcCase(unittest.TestCase): + """Every subclass repoints hil_health.PROC at a temp tree; restore it so a later test + cannot silently keep scanning a deleted directory.""" + + def tearDown(self): + hil_health.PROC = REAL_PROC + + +class ShutdownPool(unittest.TestCase): + def test_returns_true_when_the_pool_terminates(self): + pool = Pool(processes=1) + try: + self.assertTrue(hil_health.shutdown_pool(pool, grace=30)) + finally: + pool.terminate() + + def test_returns_false_instead_of_blocking_forever(self): + """The real failure is a worker in uninterruptible sleep, which cannot be created + from userspace. What matters is that shutdown_pool gives up on the deadline rather + than hanging, because the caller must then abandon the pool to free the job slot.""" + # Cancellable, not time.sleep(3600): shutdown_pool returns while its daemon thread + # is still inside terminate(), and an uninterruptible sleep there outlives the test. + # The next test alphabetically forks a real Pool, so the leaked thread made it + # fork-from-multithreaded ('DeprecationWarning: ... may lead to deadlocks in the + # child') and its result order-dependent. addCleanup releases it either way. + release = threading.Event() + self.addCleanup(release.set) + + class NeverDies: + def terminate(self): + release.wait(3600) + + start = time.monotonic() + self.assertFalse(hil_health.shutdown_pool(NeverDies(), grace=0.5)) + self.assertLess(time.monotonic() - start, 10) + + def test_a_raising_terminate_counts_as_failure(self): + """The thread dies on the exception, so is_alive() goes False -- which would report + success for a pool that is just as alive as if terminate() had hung.""" + class Explodes: + def terminate(self): + raise RuntimeError('boom') + + self.assertFalse(hil_health.shutdown_pool(Explodes(), grace=5)) + + +class ChildProcs(ProcCase): + """A pool worker's own group is OUR group (multiprocessing never setpgid's), so its + children can only be found by walking ppid -> pgrp in /proc.""" + + def test_grandchildren_are_swept_too(self): + """usbtest.py (child, own session) spawns its recovery reflash via run_cmd (own + session again): the flasher is a GRANDCHILD no direct-child walk covers, and a + pool-guard kill mid-recovery would orphan it on the probe.""" + got = self.scan([100], { + 100: ('worker', 1, 4242), + 200: ('usbtest.py', 100, 200), # child, own session + 300: ('openocd', 200, 300), # grandchild flasher, own session + 999: ('unrelated', 1, 999), + }) + self.assertEqual(sorted(got.get(100, [])), [(200, 200), (300, 300)]) + + def scan(self, pids, procs): + """`procs` maps pid -> (comm, ppid, pgrp); a None comm omits the stat file.""" + with TemporaryDirectory() as td: + root = Path(td) + for p, (comm, ppid, pgrp) in procs.items(): + d = root / str(p) + d.mkdir() + if comm is not None: + (d / 'stat').write_text(f'{p} ({comm}) S {ppid} {pgrp} 0 0 -1 0 0\n') + (root / 'not-a-pid').mkdir() + hil_health.PROC = root + return hil_health.child_procs(pids) + + def test_finds_direct_children_only(self): + got = self.scan([100], { + 100: ('python3', 1, 4242), # the worker itself + 201: ('openocd', 100, 201), # its detached flasher + 202: ('usbtest.py', 100, 202), # a second detached session + 303: ('unrelated', 7, 303), # someone else's child + }) + self.assertEqual({100: [(201, 201), (202, 202)]}, + {k: sorted(v) for k, v in got.items()}) + + def test_covers_every_parent_in_one_walk(self): + """One pass for all workers, not one pass each: this runs on the free-the-runner + path, and per-parent walks would also see different snapshots.""" + got = self.scan([100, 101], { + 201: ('openocd', 100, 201), + 202: ('JLinkExe', 101, 202), + }) + self.assertEqual(got, {100: [(201, 201)], 101: [(202, 202)]}) + + def test_parses_a_comm_containing_spaces_and_parens(self): + """A naive split() on the whole line would read the wrong fields.""" + got = self.scan([100], {500: ('we ) ird', 100, 500)}) + self.assertEqual(got, {100: [(500, 500)]}) + + def test_reports_a_child_that_shares_our_group(self): + """subprocess.run children (arecord, iperf) get no new session, so they land in + our group. They must still be REPORTED -- kill_pool_children signals them by pid, + since killpg on that group would take down the run itself.""" + got = self.scan([100], {201: ('arecord', 100, 4242)}) + self.assertEqual(got, {100: [(201, 4242)]}) + + def test_tolerates_unreadable_and_truncated_entries(self): + got = self.scan([100], { + 201: (None, 0, 0), # stat missing (exited mid-scan) + 202: ('openocd', 100, 202), # still found + }) + self.assertEqual(got, {100: [(202, 202)]}) + + def test_returns_empty_when_proc_is_unreadable(self): + hil_health.PROC = Path('/nonexistent-proc-for-test') + self.assertEqual(hil_health.child_procs([100]), {}) + + +class FakeProc: + """Stands in for a multiprocessing worker: kill_pool_children goes through + is_alive() and Process.kill(), whose internal returncode guard is what protects + against signalling a recycled pid.""" + + def __init__(self, pid, alive=True, wedged=False): + self.pid = pid + self._alive = alive + self._wedged = wedged # D state: ignores SIGKILL, so is_alive() stays True + self.killed = False + + def is_alive(self): + return self._alive + + def kill(self): + self.killed = True + # A signalled worker DIES unless it is wedged. Modelling every worker as an + # unkillable survivor sent all of them down the confirm/sudo ladder, which is + # what let literal pids reach the real os.kill. + if not self._wedged: + self._alive = False + + +class KillWorkerChildren(PatchCase): + # os.getpgid/killpg are stubbed for the whole class: FakeProc pids are literals like + # 101, which are live pids on a real machine, so an unstubbed killpg SIGKILLs a real + # process GROUP. That happened while writing this and killed the test run itself. + """What the workers spawned, killed while their parents are still alive. + + Verified premise: Pool.terminate() reaps a worker that is merely waiting in + communicate() on a wedged flasher, reparenting that flasher to init -- so this must + run BEFORE shutdown_pool(), or the ppid link is gone and a successful terminate() + skips the cleanup entirely.""" + + OWN_PGID = 4242 + + def setUp(self): + # _kill_and_confirm's grace poll must not touch the real /proc: fake pid + # 900 can be a live process on the host, which stalls the poll for the full grace + # and prints a false survivor warning into the blocking pre-commit hook. + self.proc_tmp = TemporaryDirectory() + self.addCleanup(self.proc_tmp.cleanup) + self.patch(hil_health, 'PROC', Path(self.proc_tmp.name)) # empty: unreadable -> gone + self.patch(hil_health, 'CONFIRM_KILL_GRACE', 0.05) + # the sweep runs two passes with a real gap; the fakes never respawn, so + # stub the wait rather than pay it in every test + self.patch(hil_health.time, 'sleep', lambda _s: None) + self.groups, self.pids = [], [] + self.children = {} # worker pid -> [(pid, pgid), ...] + self.patch(hil_health, 'child_procs', lambda pids: self.children) + self.patch(os, 'killpg', lambda pgid, sig: self.groups.append((pgid, sig))) + self.patch(os, 'kill', lambda pid, sig: self.pids.append((pid, sig))) + self.patch(os, 'getpgid', lambda pid: self.OWN_PGID) + # overwritten directly by some test bodies below (eperm/boom fakes) + self.restore(hil_health, '_kill_and_confirm') + + def test_kills_a_detached_child_by_group(self): + """Flashers are spawned with start_new_session=True, so one killpg also reaps + whatever they spawned; a plain kill would leave them holding the probe with no + timeout enforcer left alive.""" + w = FakeProc(101) + self.children = {101: [(900, 900)]} + + class FakePool: + _pool = [w] + self.assertEqual(hil_health.kill_worker_children(FakePool()), 0) # none survived + # by GROUP, so whatever the flasher spawned dies with it + self.assertEqual(self.groups, [(900, signal.SIGKILL)]) + # and then confirmed by pid: killpg reports success when it reached ANY member, + # so the group kill alone is not evidence this one died + self.assertIn((900, 0), self.pids) + self.assertFalse(w.killed) # the WORKER is not this one's job + + def test_a_root_owned_group_is_still_confirmed_and_reported(self): + """killpg on an all-root session raises EPERM: the sudo wrapper died and only its + root members remain. That is the one case this handler exists for, so it must + still reach the confirm step -- otherwise the holder that strands the NEXT job is + the one holder the report never names.""" + w = FakeProc(101) + self.children = {101: [(900, 900)]} + + def eperm(pgid, sig): + raise PermissionError + os.killpg = eperm + + class FakePool: + _pool = [w] + hil_health.kill_worker_children(FakePool()) + # confirmed by pid: a liveness probe on the member killpg could not touch + self.assertIn((900, 0), self.pids) + + def test_kills_a_same_group_child_by_pid(self): + """arecord/iperf/gio go through plain subprocess.run and stay in OUR group, where + killpg would take down the run itself -- but they must still die, or a blocked + arecord keeps holding the wedged device.""" + w = FakeProc(101) + self.children = {101: [(900, self.OWN_PGID)]} + + class FakePool: + _pool = [w] + hil_health.kill_worker_children(FakePool()) + self.assertEqual(self.groups, []) # never our own group + # SIGKILL, then a (pid, 0) probe: signalling is not dying, so the kill is always + # confirmed -- see _kill_and_confirm. + self.assertIn((900, signal.SIGKILL), self.pids) + self.assertIn((900, 0), self.pids) + + + def test_signals_pids_only_when_our_group_is_unknown(self): + """If getpgid(0) fails we cannot tell our group from a detached one, so killpg is + never safe -- fall back to per-pid signals rather than guessing.""" + def boom(pid): + raise OSError('no pgid') + os.getpgid = boom + w = FakeProc(101) + self.children = {101: [(900, 900)]} + + class FakePool: + _pool = [w] + hil_health.kill_worker_children(FakePool()) + self.assertEqual(self.groups, []) + self.assertIn((900, signal.SIGKILL), self.pids) + + def test_covers_a_dead_workers_orphans(self): + """A worker reaped between the snapshot and now leaves its flasher running. The + children are keyed off the snapshot, not off is_alive(), so they still die.""" + w = FakeProc(101, alive=False) + self.children = {101: [(900, 900)]} + + class FakePool: + _pool = [w] + self.assertEqual(hil_health.kill_worker_children(FakePool()), 0) # none survived + self.assertEqual(self.groups, [(900, signal.SIGKILL)]) + + def test_a_survivor_is_returned_so_the_report_can_say_the_rig_is_dirty(self): + """A stray that ignores SIGKILL is in D state on a usbfs node or holds a probe, and + it persists into the NEXT job. The count used to be discarded by the caller (the + return was the signalled-child count, which nothing read), so the only trace was a + line in the log -- and the run still published a table that looks clean.""" + w = FakeProc(101) + # TWO strays, only ONE unkillable: signalled=2, survivors=1, so this cannot pass + # by accident on the old return value + self.children = {101: [(900, 900), (901, 901)]} + self.patch(hil_health, '_kill_and_confirm', lambda pids: [p for p in pids if p == 901]) + + class FakePool: + _pool = [w] + self.assertEqual(hil_health.kill_worker_children(FakePool()), 1) + + def test_no_signal_when_the_workers_spawned_nothing(self): + w = FakeProc(101) # no self.children entry + + class FakePool: + _pool = [w] + self.assertEqual(hil_health.kill_worker_children(FakePool()), 0) + self.assertEqual((self.groups, self.pids), ([], [])) + + def test_includes_the_managers_children(self): + mgr_proc = FakeProc(402) + self.children = {402: [(900, 900)]} + + class FakePool: + _pool = [] + + class FakeManager: + _process = mgr_proc + self.assertEqual(hil_health.kill_worker_children(FakePool(), FakeManager()), 0) + + +class ConfirmTailIsOneGrace(PatchCase): + """The grace is ONE window for the whole set, not one per pid. Paid serially it + scaled with stray count: 30 strays x 16 workers spent ~154s inside the path whose + only job is to free the runner's single job slot -- which is exactly the + 'multi-stray convoy tail is minutes' the CI ceilings budget +30 min for.""" + + def setUp(self): + self.proc_tmp = TemporaryDirectory() + self.addCleanup(self.proc_tmp.cleanup) + root = Path(self.proc_tmp.name) + # every fake pid is alive and NOT a zombie, so all of them outlast the grace + make_proc(root, {900 + i: ('flasher', 'D', b'openocd\x00') for i in range(20)}) + self.patch(hil_health, 'PROC', root) + self.patch(hil_health, 'CONFIRM_KILL_GRACE', 0.3) + self.patch(os, 'kill', lambda pid, sig: None) # never signal a real pid + + def test_twenty_survivors_cost_one_grace_not_twenty(self): + pids = [900 + i for i in range(20)] + t0 = time.monotonic() + still = hil_health._kill_and_confirm(pids) + elapsed = time.monotonic() - t0 + self.assertEqual(sorted(still), pids) # all reported, none lost + self.assertLess(elapsed, 0.3 * 4, + f'the grace is paid per pid ({elapsed:.2f}s for 20)') + + +class KillPoolChildren(PatchCase): + """The worker processes themselves. + + Verified premise: an orphaned pool worker keeps the CI runner's stdout pipe open, so a + reader never sees EOF even after the parent exits. + + Fakes throughout: FakeProc.kill() only sets a flag, so nothing here can signal a real + process. That matters historically -- an earlier revision drove this through + os.pidfd_open with literal pids (101, 102), which exist on a real machine, so the suite + was asking the kernel to signal unrelated system processes and was saved only by EPERM. + Keep the fake in charge of kill(); never let a test reach os.kill/os.killpg with a + live pid. FakeProc.kill() alone is NOT enough for that: it leaves is_alive() True, so + the pid reaches the confirm/sudo ladder, which signals for real. Stub that too.""" + + def setUp(self): + # Pids 101/102/201 are ordinary user processes on a container or a fresh runner -- + # and pre-commit.yml runs this suite on GitHub's. Unstubbed, the ladder ran + # os.kill(101, SIGKILL) and forked `sudo -n kill -9 101` on an account with + # passwordless sudo, and the assertions passed only because those pids happen to + # be unkillable kernel threads here. + self.signals = [] + self.patch(os, 'kill', lambda pid, sig: self.signals.append((pid, sig))) + self.patch(os, 'killpg', lambda pgid, sig: self.signals.append((pgid, sig))) + self.proc_tmp = TemporaryDirectory() + self.addCleanup(self.proc_tmp.cleanup) + self.patch(hil_health, 'PROC', Path(self.proc_tmp.name)) # empty: unreadable -> gone + self.patch(hil_health, 'CONFIRM_KILL_GRACE', 0.05) + + def test_a_healthy_worker_never_reaches_the_signalling_ladder(self): + """The premise every assertion below rests on. Process.kill() is the fake's job; + only a worker that SURVIVES it goes on to raw os.kill/sudo, and these pids are + literals that belong to somebody else.""" + a, b = FakeProc(101), FakeProc(102) + + class FakePool: + _pool = [a, b] + hil_health.kill_pool_children(FakePool()) + self.assertEqual(self.signals, [], 'a literal pid reached the raw-signal ladder') + + def test_signals_every_live_worker(self): + a, b = FakeProc(101), FakeProc(102) + + class FakePool: + _pool = [a, b] + # 0, not 2: the RETURN is confirmed survivors, and workers that die to SIGKILL are + # not survivors. The operator verdict ("power-cycle the host") hangs off this. + self.assertEqual(hil_health.kill_pool_children(FakePool()), 0) + self.assertTrue(a.killed and b.killed) + + def test_a_wedged_worker_is_reported_as_a_survivor(self): + """The number the power-cycle verdict is worded on.""" + make_proc(Path(self.proc_tmp.name), {301: ('python3', 'D', b'python3 hil_test.py\x00')}) + wedged = FakeProc(301, wedged=True) + + class FakePool: + _pool = [wedged] + self.assertEqual(hil_health.kill_pool_children(FakePool()), 1) + + def test_skips_a_reaped_worker(self): + """Process.kill() re-checks returncode internally, but skipping a dead child keeps + the harness from signalling a pid the OS may have recycled.""" + live, dead = FakeProc(201), FakeProc(202, alive=False) + + class FakePool: + _pool = [live, dead] + self.assertEqual(hil_health.kill_pool_children(FakePool()), 0) + self.assertTrue(live.killed) + self.assertFalse(dead.killed) + + def test_also_kills_the_manager(self): + """Manager() is a separate child holding the same descriptors, and os._exit skips + its finalizer, so leaving it behind defeats the whole purpose. The RETURN is the + confirmed-survivor count (the caller words a power-cycle verdict on it), so a + clean kill of both reports 0.""" + worker, mgr_proc = FakeProc(401), FakeProc(402) + + class FakePool: + _pool = [worker] + + class FakeManager: + _process = mgr_proc + self.assertEqual(hil_health.kill_pool_children(FakePool(), FakeManager()), 0) + self.assertTrue(worker.killed and mgr_proc.killed) + self.assertTrue(mgr_proc.killed) + + def test_tolerates_a_pool_without_workers(self): + class NoPool: + _pool = None + self.assertEqual(hil_health.kill_pool_children(NoPool()), 0) + + +class WriteTimeoutReport(unittest.TestCase): + def test_prefix_carries_the_preflight_diagnosis(self): + """The timeout aborts before accumulate_report, so without the prefix the artifact + and the PR comment lose the one line saying WHY the pool never finished.""" + with TemporaryDirectory() as td: + d = Path(td) + hil_health.write_timeout_report(d, [{'name': 'b1'}], 4200, 'r.md', + prefix='> **wedged usb_hub_wq worker.**\n') + out = (d / 'r.md').read_text() + self.assertTrue(out.startswith('> **wedged usb_hub_wq worker.**')) + self.assertIn('timed out after 4200s', out) + self.assertIn('- b1', out) + + def test_writes_a_report_where_there_would_be_none(self): + with TemporaryDirectory() as td: + hil_health.write_timeout_report(Path(td), [{'name': 'ra6m5_ek'}], 4200, + 'hil_report.md') + md = (Path(td) / 'hil_report.md').read_text() + self.assertIn('4200s', md) + self.assertIn('ra6m5_ek', md) + + def test_keeps_a_previous_attempts_table(self): + with TemporaryDirectory() as td: + path = Path(td) / 'hil_report.md' + path.write_text('| board | cdc_msc |\n') + hil_health.write_timeout_report(Path(td), [{'name': 'b1'}], 4200, 'hil_report.md') + md = path.read_text() + self.assertIn('abandoned', md) + self.assertIn('| board | cdc_msc |', md) + self.assertLess(md.index('abandoned'), md.index('| board |')) + + def test_custom_banner_is_used(self): + with TemporaryDirectory() as td: + hil_health.write_timeout_report(Path(td), [], 0, 'hil_report.md', + banner='**refused to start.**\n') + self.assertIn('refused to start', (Path(td) / 'hil_report.md').read_text()) + + def test_unwritable_dir_does_not_raise(self): + """The caller may be about to os._exit; losing the report must not also lose the + exit path.""" + hil_health.write_timeout_report(Path('/proc/nonexistent/nope'), [], 0, 'x.md') + + +class WorkerSweepsItsOwnChildren(unittest.TestCase): + """maxtasksperchild=1 makes a worker exit the moment its task returns, so by the time + main()'s finally sweeps, the strays have been reparented to init and are off the pool's + ppid tree entirely. Measured over 4 tasks: pool._pool held two FRESH workers with zero + overlap with the four that ran, child_procs() returned {}, the sweep reported 0, and all + four strays were alive. Inside the worker the ppid link is still there.""" + + def test_a_detached_child_is_killed_and_confirmed(self): + kid = subprocess.Popen(['sleep', '120'], start_new_session=True) + self.addCleanup(lambda: kid.poll() is None and kid.kill()) + time.sleep(0.3) # let it appear in /proc + + stray = hil_health.kill_own_children() + + self.assertEqual(stray, 0, 'a killable stray was reported as a survivor') + kid.wait(timeout=5) # TimeoutExpired here means it outlived us + self.assertIsNotNone(kid.poll()) + + def test_no_children_is_not_an_error(self): + self.assertEqual(hil_health.kill_own_children(), 0) + + +class PermitReleasesOnlyWhatItTook(unittest.TestCase): + """The bounded acquire skips a slot it could not get ('proceeding over-subscribed') and + deliberately leaves it out of `taken`, but __exit__ released every slot in self.slots. + multiprocessing.Semaphore is unbounded, so each timeout permanently widened that + controller's permit -- the throttle this branch NARROWED (FLASH_PARALLEL 8->4, + USBTEST_PARALLEL 4->2) for xHCI bandwidth margin.""" + + def test_a_timed_out_slot_is_not_released_on_exit(self): + from helper import hil_lock + import multiprocessing + + sems = [multiprocessing.Semaphore(1)] + sems[0].acquire() # width 1, already held: the next wait times out + self.addCleanup(setattr, hil_lock, 'PERMIT_TIMEOUT', hil_lock.PERMIT_TIMEOUT) + hil_lock.PERMIT_TIMEOUT = 0.1 + + permit = hil_lock.controller_permit(sems, 'UID') + permit.slots = [0] + with permit: + pass + + # one holder still holds it, so a correct exit leaves it unavailable + self.assertFalse(sems[0].acquire(timeout=0.1), + 'the permit released a slot it never acquired: width grew') + + +class RecoveryPrefersResetOverReflash(unittest.TestCase): + """Probe reset is the preferred cure: non-destructive (the wedged firmware survives for + autopsy), no flash wear, no risk of a bad park image (a wfe/wfi park has bricked SWD on + mimxrt1064_evk and max32666fthr through a power cycle), and measured at 128-129 ms + against a full erase+program. It also fits in budgets a reflash does not.""" + + def setUp(self): + import usbtest # test/hil is already on sys.path (see top of file) + self.u = usbtest + + def test_reset_is_attempted_before_the_reflash(self): + steps = self.u.recovery_steps('openocd', time_left=600) + self.assertEqual([s[0] for s in steps], ['reset', 'flash']) + + def test_a_budget_too_small_to_reflash_still_gets_the_reset(self): + """The old gate skipped recovery whole when a reflash did not fit, leaving the + holder in place; a reset needs a fraction of the budget.""" + steps = self.u.recovery_steps('openocd', time_left=self.u.RECOVER_FLASH_TIMEOUT - 1) + self.assertEqual([s[0] for s in steps], ['reset']) + + def test_no_budget_at_all_yields_nothing(self): + self.assertEqual(self.u.recovery_steps('openocd', time_left=1), []) + + def test_a_flasher_with_no_reset_primitive_goes_straight_to_reflash(self): + steps = self.u.recovery_steps('nosuchflasher', time_left=600) + self.assertEqual([s[0] for s in steps], ['flash']) + + +class RecoveryDoesNotClaimAResetItDidNotDo(unittest.TestCase): + """reset_esptool and reset_lm4flash return rc 0 without resetting anything, so a plan + that includes them makes the log say "resetting via " for a step that + did nothing. wedged_pids() arbitrates, so behaviour was already right -- the record was + not, and a false record is what this branch keeps having to unpick.""" + + def setUp(self): + import usbtest + self.u = usbtest + + def test_a_no_op_reset_primitive_is_not_scheduled(self): + self.assertEqual([k for k, _ in self.u.recovery_steps('esptool', 600)], ['flash']) + self.assertEqual([k for k, _ in self.u.recovery_steps('lm4flash', 600)], ['flash']) + + def test_a_real_reset_primitive_still_is(self): + self.assertEqual([k for k, _ in self.u.recovery_steps('openocd', 600)], + ['reset', 'flash']) + + +class SudoSoftNeverRaises(unittest.TestCase): + """Two of its four call sites are inside run_case's timeout handler, where ANY raise + costs the HUNG verdict, the recovery and the JSON report -- and sudo() sys.exit()s on + 'a password is required', which is a raise like any other.""" + + def setUp(self): + import usbtest + self.u = usbtest + self.addCleanup(setattr, usbtest, 'sudo', usbtest.sudo) + + def _check(self, exc): + def boom(*a, **k): + raise exc + self.u.sudo = boom + r = self.u._sudo_soft(['dmesg']) # must not propagate + self.assertEqual(r.returncode, 1) + + def test_systemexit_from_a_password_prompt_is_contained(self): + self._check(SystemExit('sudo needs a password')) + + def test_oserror_is_contained(self): + self._check(OSError('no such binary')) + + def test_subprocess_error_is_contained(self): + self._check(subprocess.SubprocessError('timed out')) + + +if __name__ == '__main__': + unittest.main(verbosity=1) diff --git a/test/hil/test/test_hil_select.py b/test/hil/test/test_hil_select.py new file mode 100644 index 000000000..9a1261878 --- /dev/null +++ b/test/hil/test/test_hil_select.py @@ -0,0 +1,689 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests for hil_select.py — pure logic, no hardware, no git. Run directly: +# python3 test/hil/test/test_hil_select.py +# +# Imports stay stdlib + hil_select/hil_util/hil_flash ONLY: the pre-commit hil-test +# hook runs this suite, on GitHub's bare runner in the pre-commit workflow as well as +# locally, and that runner has no pyserial/pymtp. hil_flash is admissible because it +# is stdlib + hil_util only (test_hil_util.BottomLayer enforces the stdlib closure of +# both) and the roster-dispatch tests need its flash_* table; never import hil_test, +# which pulls pyserial. +import glob +import json +import os +import sys +import unittest + +# the modules under test live in the parent dir (test/hil), not here +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import hil_flash +from helper import hil_select +from helper.hil_util import device_tests, dual_tests + +REPO = os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.abspath(__file__))))) + + +def real_rosters(): + """The actual rig rosters, for regression tests that need real-world data + (a specific board/family/only-list) rather than the synthetic ROSTER above.""" + rosters = [] + for name in ('tinyusb.json', 'hfp.json'): + path = os.path.join(REPO, 'test/hil', name) + with open(path) as f: + rosters.append((f'test/hil/{name}', json.load(f)['boards'])) + return rosters + + +def roster_flashers(): + """(roster path, board) for every board in the live rosters, `boards-skip` + included: a parked board's flasher name must still dispatch, so that unparking it + is not what discovers the name went stale.""" + for name in ('tinyusb.json', 'hfp.json'): + path = os.path.join(REPO, 'test/hil', name) + with open(path) as f: + cfg = json.load(f) + for key in ('boards', 'boards-skip'): + for b in cfg.get(key, []): + yield f'test/hil/{name}', b + + +def on_roster(tc, *names): + """The subset of `names` currently in the live rig rosters, skipping the test + when none are, because parking/unparking a board is routine rig maintenance. + + That skip now matters MORE than it used to, not less: this suite is a blocking + pre-commit hook AND build.yml's selector steps gate on it (a failing suite falls + open to the full matrix), so an assertion that depends on a specific board being + present goes red on every PR -- including src/-only ones that never touched the + rig -- until someone fixes the roster. Keep roster-dependent assertions behind + on_roster.""" + have = {b['name'] for _, boards in real_rosters() for b in boards} + got = [n for n in names if n in have] + if not got: + tc.skipTest(f'not in the rig roster: {", ".join(names)}') + return got + + +ROSTER = [ + # device-only, rp2040 family + {'name': 'raspberry_pi_pico', 'uid': 'u1', 'flasher': {'name': 'openocd'}, + 'tests': {'device': True, 'host': True, 'dual': True}}, + # device-only, stm32f4 family + {'name': 'stm32f407disco', 'uid': 'u2', 'flasher': {'name': 'jlink'}, + 'tests': {'device': True, 'host': False, 'dual': False}}, + # host-only board + {'name': 'raspberry_pi_pico2', 'uid': 'u3', 'flasher': {'name': 'openocd'}, + 'tests': {'device': False, 'host': True, 'dual': False}}, + # only-list board (espressif-style), flashed by the CI leg that splits on esptool + {'name': 'espressif_s3_devkitm', 'uid': 'u4', 'flasher': {'name': 'esptool'}, + 'tests': {'only': ['device/cdc_msc_freertos', 'host/device_info']}}, +] +ROSTERS = [('test/hil/tinyusb.json', ROSTER)] + + +def sel(files): + return hil_select.classify(files, REPO, ROSTERS) + + +class TestPortRule(unittest.TestCase): + def test_dcd_rp2040_selects_pico_family_only(self): + s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico', s['boards']) + self.assertNotIn('stm32f407disco', s['boards']) + self.assertNotIn('espressif_s3_devkitm', s['boards']) + # device role: no host tests in pico's list + self.assertTrue(all(not t.startswith('host/') for t in s['boards']['raspberry_pi_pico'])) + # host-only boards drop out entirely on a device-role change + self.assertNotIn('raspberry_pi_pico2', s['boards']) + + def test_shared_port_file_is_both_roles(self): + s = sel(['src/portable/synopsys/dwc2/dwc2_common.c']) + self.assertFalse(s['full']) + self.assertNotIn('raspberry_pi_pico', s['boards']) # rp2040 is not a dwc2 family + self.assertIn('stm32f407disco', s['boards']) # stm32f4 is + + +class TestCoreRoleRule(unittest.TestCase): + def test_usbd_selects_all_device_tests_everywhere(self): + s = sel(['src/device/usbd.c']) + self.assertFalse(s['full']) + self.assertNotIn('raspberry_pi_pico2', s['boards']) # host-only board dropped + pico = s['boards']['raspberry_pi_pico'] + self.assertTrue(set(device_tests).issubset(set(pico))) + self.assertTrue(set(dual_tests).issubset(set(pico))) # dual survives device role + self.assertTrue(all(not t.startswith('host/') for t in pico)) + # only-list board: selection intersects its only-list + esp = s['boards']['espressif_s3_devkitm'] + self.assertEqual(esp, ['device/cdc_msc_freertos']) + + def test_host_change_drops_device(self): + s = sel(['src/host/usbh.c']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico2', s['boards']) + self.assertNotIn('stm32f407disco', s['boards']) # device-only board dropped + + +class TestClassRule(unittest.TestCase): + def test_cdc_device_selects_cdc_examples_only(self): + s = sel(['src/class/cdc/cdc_device.c']) + self.assertFalse(s['full']) + pico = s['boards']['raspberry_pi_pico'] + self.assertIn('device/cdc_msc', pico) + self.assertIn('device/cdc_dual_ports', pico) + self.assertNotIn('device/msc_dual_lun', pico) # CFG_TUD_CDC 0 there + self.assertNotIn('device/usbtest', pico) # CFG_TUD_CDC 0 there + self.assertTrue(all(not t.startswith('host/') for t in pico)) + + def test_msc_host_selects_host_side(self): + s = sel(['src/class/msc/msc_host.c']) + self.assertFalse(s['full']) + self.assertNotIn('stm32f407disco', s['boards']) # device-only board + pico2 = s['boards']['raspberry_pi_pico2'] + self.assertIn('host/msc_file_explorer', pico2) + self.assertTrue(all(not t.startswith('device/') for t in pico2)) + + +class TestClassIncludeEdges(unittest.TestCase): + """A class header another class includes reaches that class's examples too. + src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, so + midi_test's firmware contains audio.h - but the class rule derives macros from + the directory name alone, so an audio.h change used to select only + device/audio_test_freertos. On boards that skip that example the per-board + intersection emptied and an audio.h-only PR ran ZERO HIL on them.""" + def test_edges_derived_from_includes(self): + edges = hil_select.class_include_edges(REPO) + self.assertEqual(edges.get('audio/audio.h'), {'midi'}) + self.assertEqual(edges.get('cdc/cdc.h'), {'net'}) + + def test_audio_header_selects_midi_example(self): + s = hil_select.classify(['src/class/audio/audio.h'], REPO, real_rosters()) + self.assertFalse(s['full']) + # every board that runs device/midi_test at all must run it here (boards with + # a tests.only list, e.g. espressif, run the freertos examples instead) + by_name = {b['name']: b for _, bs in real_rosters() for b in bs} + checked = 0 + for name, tests in s['boards'].items(): + if 'device/midi_test' in hil_select.board_tests(by_name[name]): + self.assertIn('device/midi_test', tests, name) + checked += 1 + self.assertTrue(checked) + + def test_audio_header_reaches_boards_that_skip_audio(self): + # both skip device/audio_test_freertos: without the midi edge their + # intersection is empty and they drop out of the selection entirely + boards = on_roster(self, 'metro_m4_express', 'nrf54lm20dk') + s = hil_select.classify(['src/class/audio/audio.h'], REPO, real_rosters()) + for board in boards: + self.assertEqual(s['boards'].get(board), ['device/midi_test'], board) + + def test_edge_is_per_header_not_per_class(self): + # midi includes audio.h, not audio_device.h: an audio_device change must + # not drag midi's examples in + s = hil_select.classify(['src/class/audio/audio_device.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for tests in s['boards'].values(): + if tests != 'all': + self.assertNotIn('device/midi_test', tests) + + +class TestFallbackRules(unittest.TestCase): + def test_unknown_tool_is_full(self): + s = sel(['tools/random_new_script.py']) + self.assertTrue(s['full']) + + def test_docs_only_is_empty_not_full(self): + s = sel(['docs/info/contributing.rst', 'README.rst']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + def test_bsp_family_selects_family_boards(self): + s = sel(['hw/bsp/rp2040/family.cmake']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico', s['boards']) + self.assertEqual(s['boards']['raspberry_pi_pico'], 'all') + self.assertNotIn('stm32f407disco', s['boards']) + + def test_bsp_board_narrows_to_board(self): + s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) + self.assertFalse(s['full']) + self.assertEqual(list(s['boards'].keys()), ['raspberry_pi_pico']) + + def test_example_change_selects_that_example(self): + s = sel(['examples/device/cdc_msc/src/main.c']) + self.assertFalse(s['full']) + self.assertEqual(s['boards']['raspberry_pi_pico'], ['device/cdc_msc']) + + def test_core_common_is_full(self): + for f in ['src/tusb.c', 'src/common/tusb_fifo.c', 'src/osal/osal_freertos.h']: + self.assertTrue(sel([f])['full'], f) + + def test_board_test_example_is_full(self): + # board_test is the park/teardown firmware hil_test.py flashes on every board, + # not an unlisted example: a regression there must not skip the whole rig + for f in ['examples/device/board_test/src/main.c', + 'examples/device/board_test/CMakeLists.txt']: + self.assertTrue(sel([f])['full'], f) + + def test_harness_is_full(self): + for f in ['test/hil/hil_test.py', '.github/workflows/build.yml', 'hw/mcu/nxp/x.c', 'lib/foo/x.c']: + self.assertTrue(sel([f])['full'], f) + + def test_mixed_roles_no_pruning(self): + s = sel(['src/device/usbd.c', 'src/host/usbh.c']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico2', s['boards']) + self.assertIn('stm32f407disco', s['boards']) + + def test_cmakelists_and_requirements_are_full(self): + for f in ['src/CMakeLists.txt', 'examples/CMakeLists.txt', + 'examples/device/CMakeLists.txt', 'test/hil/requirements.txt']: + self.assertTrue(sel([f])['full'], f) + + def test_docs_txt_is_noncode(self): + s = sel(['docs/info/changelog.txt']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + +class TestArgsEmission(unittest.TestCase): + def test_args_for_scoped_selection(self): + s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + args = hil_select.selection_args(s, ROSTERS) + a = args['tinyusb.json'] + self.assertIn('-b raspberry_pi_pico', a) + self.assertNotIn('stm32f407disco', a) + self.assertIn('-bt raspberry_pi_pico:', a) # device-only subset of a device+host board + + def test_args_full_is_empty(self): + s = sel(['tools/random_new_script.py']) + self.assertEqual(hil_select.selection_args(s, ROSTERS), {'tinyusb.json': ''}) + + def test_args_all_board_gets_bare_b(self): + s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) + a = hil_select.selection_args(s, ROSTERS)['tinyusb.json'] + self.assertIn('-b raspberry_pi_pico', a) + self.assertNotIn('-bt', a) + + def test_args_by_flasher_splits_esp_from_the_rest(self): + s = sel(['src/device/usbd.c']) + per = hil_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json'] + self.assertIn('espressif_s3_devkitm', per['esptool']) + self.assertIn('raspberry_pi_pico', per['openocd']) + self.assertNotIn('espressif_s3_devkitm', per.get('openocd', '') + per.get('jlink', '')) + + def test_args_by_flasher_omits_a_flasher_with_no_selected_board(self): + # the esp CI leg must see no args at all here, not a filter matching zero boards + s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) + per = hil_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json'] + self.assertEqual(per, {'openocd': '-b raspberry_pi_pico'}) + + def test_args_by_flasher_full_is_empty(self): + s = sel(['tools/random_new_script.py']) + self.assertEqual(hil_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}}) + + def test_cli_diff_file(self): + import subprocess, tempfile, json as j + with tempfile.NamedTemporaryFile('w', suffix='.txt', delete=False) as f: + f.write('src/class/cdc/cdc_device.c\n') + path = f.name + r = subprocess.run([sys.executable, os.path.join(REPO, 'test/hil/helper/hil_select.py'), + '--diff-file', path, os.path.join(REPO, 'test/hil/tinyusb.json')], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + out = j.loads(r.stdout) + self.assertFalse(out['full']) + self.assertIn('tinyusb.json', out['args']) + self.assertTrue(any('cdc_device' in line for line in out['reasons'])) + # A core-class diff must select boards THROUGH THE CLI: the in-process tests + # inject their own repo root, so only this subprocess path catches a broken + # repo_root derivation -- which once made every repo-relative glob match + # nothing and turned this exact diff into a silent full-HIL skip. + self.assertTrue(out['boards'], + 'CLI selected zero boards for a src/class change: repo_root broken?') + os.unlink(path) + + +class TestRealRosterPortFamilies(unittest.TestCase): + """Regression for port_families() missing espressif's dwc2 reference, which + lives in a component CMakeLists.txt rather than family.cmake/family.mk.""" + def test_dwc2_change_selects_espressif_boards(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = hil_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertIn(board, s['boards']) + + +class TestOptionGatedPort(unittest.TestCase): + """Regression: family_support.cmake compiles some ports from a build option + (MAX3421_HOST=1 -> hcd_max3421.c), so a board's family file never names them.""" + # host-side option board (max3421 as host controller), off any max3421 family + OPT_ROSTER = [('test/hil/opt.json', [ + {'name': 'fake_dual_board', 'uid': 'o1', 'flasher': {'name': 'jlink'}, + 'build': {'args': ['MAX3421_HOST=1']}, + 'tests': {'device': True, 'host': False, 'dual': True}}, + {'name': 'fake_host_board', 'uid': 'o2', 'flasher': {'name': 'jlink'}, + 'variant': [{'name': 'fake_host_board', 'flags': '-DMAX3421_HOST=1'}], + 'tests': {'device': False, 'host': True, 'dual': False}}, + {'name': 'fake_off_board', 'uid': 'o3', 'flasher': {'name': 'jlink'}, + 'variant': [{'name': 'fake_off_board', 'defines': ['MAX3421_HOST=0']}], + 'tests': {'device': True, 'host': True, 'dual': True}}, + ])] + + def test_real_roster_max3421_selects_option_board(self): + boards = on_roster(self, 'metro_m4_express') + s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertIn(board, s['boards']) + + def test_option_selects_via_args_defines_and_flags(self): + s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, self.OPT_ROSTER) + self.assertFalse(s['full']) + self.assertIn('fake_dual_board', s['boards']) # build.args + self.assertIn('fake_host_board', s['boards']) # variant flags + self.assertNotIn('fake_off_board', s['boards']) # variant defines, but =0 + + def test_device_role_port_does_not_pull_host_only_option_board(self): + s = hil_select.classify(['src/portable/analog/max3421/dcd_max3421.c'], REPO, self.OPT_ROSTER) + self.assertFalse(s['full']) + self.assertNotIn('fake_host_board', s['boards']) # host-only board, device change + self.assertIn('fake_dual_board', s['boards']) # device-capable option board + + def test_gates_parsed_from_family_support(self): + self.assertEqual(hil_select.port_option_gates(REPO).get('analog/max3421'), + {'MAX3421_HOST'}) + + def test_board_cmake_option_counts(self): + """A board can enable a gated port in its own BSP rather than via the roster + (hw/bsp/espressif/boards/*/board.cmake -> set(MAX3421_HOST 1)); board_options() + must see those too, or such a board joining the roster is silently dropped.""" + self.assertIn('MAX3421_HOST', + hil_select.bsp_board_options('adafruit_feather_esp32s3', REPO)) + self.assertIn('CFG_TUH_RPI_PIO_USB', + hil_select.bsp_board_options('adafruit_fruit_jam', REPO)) + # commented-out `# set(MAX3421_HOST 1)` must not count + self.assertNotIn('MAX3421_HOST', + hil_select.bsp_board_options('feather_nrf52840_express', REPO)) + + def test_board_cmake_option_selects_off_family_board(self): + # adafruit_feather_esp32s3 is not on any rig roster; stand it in as one to + # prove the BSP-sourced option alone pulls a max3421 change onto the board + roster = [('test/hil/opt.json', [ + {'name': 'adafruit_feather_esp32s3', 'uid': 'o1', 'flasher': {'name': 'esptool'}, + 'tests': {'device': False, 'host': True, 'dual': False}}])] + s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster) + self.assertFalse(s['full']) + self.assertIn('adafruit_feather_esp32s3', s['boards']) + + def test_board_mk_option_is_ignored(self): + """Make-only options must not select: HIL CI builds with CMake exclusively, so + hw/bsp/nrf/boards/nrf5340dk/board.mk's MAX3421_HOST compiles nothing here.""" + roster = [('test/hil/opt.json', [ + {'name': 'nrf5340dk', 'uid': 'o1', 'flasher': {'name': 'jlink'}, + 'tests': {'device': False, 'host': True, 'dual': False}}])] + s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + +class TestPortFamiliesCmakeOnly(unittest.TestCase): + """port_families() is CMake-only (HIL CI never builds with Make) and matches on + 'port_dir/' so a port dir is not a prefix of a sibling.""" + def test_make_only_family_is_not_a_family(self): + # hw/bsp/pic32mz has family.mk but no family.cmake + self.assertEqual(hil_select.port_families('microchip/pic32mz', REPO), set()) + + def test_prefix_port_does_not_inherit_sibling_families(self): + # bare-substring matching let 'microchip/pic' match '.../microchip/pic32mz/...' + self.assertEqual(hil_select.port_families('microchip/pic', REPO), set()) + + def test_make_only_port_forces_full(self): + s = sel(['src/portable/microchip/pic32mz/dcd_pic32mz.c']) + self.assertTrue(s['full']) + self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons']) + + def test_cmake_families_still_found(self): + self.assertEqual(hil_select.port_families('raspberrypi/rp2040', REPO), {'rp2040'}) + self.assertIn('stm32f4', hil_select.port_families('synopsys/dwc2', REPO)) + + +class TestPortFamiliesCoverage(unittest.TestCase): + """Systematic guard: every real dcd_*/hcd_* port directory should map to at + least one board family, so a future family.cmake/CMakeLists.txt layout that + port_families() doesn't scan fails loudly instead of silently dropping boards + (as espressif's dwc2 reference did - see TestRealRosterPortFamilies).""" + # Ports with no board family: not a bug, just not wired into any rig board. + # Add here (with a reason) only if port_families() legitimately can't find one. + # A port listed here force-fulls (fail-open), so it is never under-selected. + NO_FAMILY = { + 'template', # reference/example port, not built by any board + # hw/bsp/pic32mz has family.mk only (no family.cmake), and port_families() + # is CMake-only because HIL CI builds every board with CMake - so this port + # is compiled for no HIL board. + 'microchip/pic32mz', + 'microchip/pic', # same: only ever referenced from pic32mz's family.mk + } + + @staticmethod + def _dcd_hcd_ports(): + portable_root = os.path.join(REPO, 'src/portable') + ports = [] + for entry in sorted(os.listdir(portable_root)): + d = os.path.join(portable_root, entry) + if not os.path.isdir(d): + continue + if glob.glob(os.path.join(d, 'dcd_*.c')) or glob.glob(os.path.join(d, 'hcd_*.c')): + ports.append(entry) + continue + for sub in sorted(os.listdir(d)): + sd = os.path.join(d, sub) + if os.path.isdir(sd) and (glob.glob(os.path.join(sd, 'dcd_*.c')) or + glob.glob(os.path.join(sd, 'hcd_*.c'))): + ports.append(f'{entry}/{sub}') + return ports + + def test_every_port_maps_to_a_family(self): + ports = self._dcd_hcd_ports() + self.assertTrue(ports) # sanity: the scan itself found something + for port in ports: + if port in self.NO_FAMILY: + continue + fams = hil_select.port_families(port, REPO) + self.assertTrue(fams, f'{port}: no family references this port ' + f'(port_families() scan gap, or add to NO_FAMILY)') + + +class TestRealRosterOnlyListTests(unittest.TestCase): + """Regression for roster-only-list tests (e.g. espressif's hid_composite_freertos) + being invisible to the selector because it only knew the shared hil_util lists.""" + def test_only_list_example_change_selects_it(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = hil_select.classify(['examples/device/hid_composite_freertos/src/main.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertEqual(s['boards'][board], ['device/hid_composite_freertos']) + + def test_class_change_includes_only_list_boards(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = hil_select.classify(['src/class/hid/hid_device.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertIn(board, s['boards']) + + +class TestPortAndCoreRoleUseExtras(unittest.TestCase): + """Regression: the port rule and core-role rule must thread the roster-only + test universe (extras) the same way the class rule already does, so a DCD + or device-stack change doesn't silently drop espressif's only-list tests + (e.g. hid_composite_freertos) that aren't in the shared device_tests list.""" + def test_dcd_change_includes_only_list_test(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = hil_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + tests = s['boards'][board] + self.assertIn('device/hid_composite_freertos', tests) + self.assertIn('device/cdc_msc_freertos', tests) + self.assertIn('device/audio_test_freertos', tests) + self.assertIn('device/usbtest', tests) + + def test_core_device_change_includes_only_list_test(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = hil_select.classify(['src/device/usbd.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + tests = s['boards'][board] + self.assertIn('device/hid_composite_freertos', tests) + self.assertIn('device/cdc_msc_freertos', tests) + self.assertIn('device/audio_test_freertos', tests) + self.assertIn('device/usbtest', tests) + + def test_host_change_does_not_leak_device_only_list_test(self): + s = hil_select.classify(['src/host/usbh.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board, tests in s['boards'].items(): + if tests == 'all': + continue + self.assertNotIn('device/hid_composite_freertos', tests, board) + + +class TestFamilies(unittest.TestCase): + """`families` exists for consumers that build (not just test) the diff: most + families have no rig board, so `boards` alone would compile nothing for them.""" + def test_off_rig_port_still_reports_family(self): + s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) # no same7x board on the rig + self.assertEqual(s['families'], ['same7x']) + + def test_port_families_are_reported(self): + s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + self.assertIn('rp2040', s['families']) + + def test_bsp_family_and_board_report_family(self): + self.assertEqual(sel(['hw/bsp/rp2040/family.cmake'])['families'], ['rp2040']) + self.assertEqual(sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h'])['families'], + ['rp2040']) + + def test_docs_only_has_no_families(self): + self.assertEqual(sel(['docs/info/contributing.rst'])['families'], []) + + def test_full_selection_still_reports_families(self): + """A full-matrix file must not hide the families of the other changed files: + consumers that build from `families` (e.g. /pre-pr) ignore `boards` when full.""" + s = sel(['src/common/tusb_fifo.c', 'src/portable/microchip/samx7x/dcd_samx7x.c']) + self.assertTrue(s['full']) + self.assertIn('same7x', s['families']) + # full stays full: every roster board, and no args to narrow the run + self.assertEqual(set(s['boards']), {b['name'] for b in ROSTER}) + self.assertTrue(all(v == 'all' for v in s['boards'].values())) + self.assertEqual(hil_select.selection_args(s, ROSTERS), {'tinyusb.json': ''}) + self.assertEqual(hil_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}}) + + def test_family_order_does_not_matter(self): + # same as above with the full-matrix file last (was the only order that worked) + s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c', 'src/common/tusb_fifo.c']) + self.assertTrue(s['full']) + self.assertIn('same7x', s['families']) + + +class TestGitDiffArgv(unittest.TestCase): + def test_diff_disables_rename_detection(self): + """Without --no-renames git reports only a rename's destination, so moving an + HIL-relevant file to a non-code path would be classified as non-code only.""" + self.assertIn('--no-renames', hil_select.GIT_DIFF_ARGV) + + +class TestPortWithoutFamilyIsFull(unittest.TestCase): + """A port dir no family file references must widen (full matrix), not silently + contribute zero boards — the fail-open contract.""" + def test_unreferenced_port_forces_full(self): + orig = hil_select.port_families + hil_select.port_families = lambda port_dir, repo_root: set() + try: + s = sel(['src/portable/vendor/newip/dcd_newip.c']) + finally: + hil_select.port_families = orig + self.assertTrue(s['full']) + self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons']) + + +class TestOpenocdVidPid(unittest.TestCase): + """The roster's optional flasher `vid_pid` field (openocd-verbatim, e.g. + "0x1a86 0x8010", more pairs appended) pins openocd's probe discovery so it + never opens foreign usbfs nodes. It must be emitted BEFORE the args: the + rescue cfgs run `init` internally (rp2350-rescue.cfg errors on any + config-stage command after its init; rp2040.cfg under RESCUE scans before a + trailing flag is even parsed), and no rig cfg sets a competing list + (the 2026-08-10 convoy mechanism).""" + + def test_vid_pid_flag_precedes_args(self): + cmd = hil_flash._openocd_cmd_base( + {'uid': 'S1', 'args': '-f target/wch-riscv.cfg', 'vid_pid': '0x1a86 0x8010'}) + self.assertIn('-c "adapter usb vid_pid 0x1a86 0x8010" -f target/wch-riscv.cfg', cmd) + self.assertTrue(cmd.endswith('-f target/wch-riscv.cfg'), cmd) + + def test_rescue_cfg_command_keeps_vid_pid_before_init(self): + """rescue_openocd swaps the target cfg for one that runs `init` internally; + a vid_pid flag after the args would error there (rp2350) or be skipped + (rp2040) -- in exactly the wedged-rig scenario the pin exists for.""" + flasher = {'name': 'openocd', 'uid': 'S1', 'vid_pid': '0x2e8a 0x000c', + 'args': '-c "set RESCUE 1" -f target/rp2040.cfg'} + cmd = hil_flash._openocd_cmd_base(flasher) + self.assertLess(cmd.index('adapter usb vid_pid'), cmd.index('-f target/'), cmd) + + def test_vid_pid_multiple_pairs(self): + cmd = hil_flash._openocd_cmd_base( + {'uid': 'S1', 'args': '-f i.cfg', 'vid_pid': '0x2e8a 0x000c 0x2e8a 0x000d'}) + self.assertIn('-c "adapter usb vid_pid 0x2e8a 0x000c 0x2e8a 0x000d"', cmd) + + def test_no_field_no_flag_but_warns(self): + # the roster lint only covers the committed rosters; a dev PC's local.json entry + # without the field must at least say what it is giving up -- on STDERR, since + # hil_test captures stdout per test and would swallow it on a passing run + import io + from contextlib import redirect_stderr + hil_flash._VID_PID_WARNED.discard('S-warn') + cap = io.StringIO() + with redirect_stderr(cap): + cmd = hil_flash._openocd_cmd_base({'uid': 'S-warn', 'args': '-f i.cfg'}) + self.assertNotIn('vid_pid', cmd) + self.assertIn('vid_pid', cap.getvalue()) + + def test_roster_openocd_entries_all_pin_vid_pid(self): + # every openocd probe on the rig has a known VID/PID; a new entry without the + # pin silently reintroduces open-everything discovery + for path, board in roster_flashers(): + f = board['flasher'] + # tinyusb.json only: hfp.json is the hifiphile rig owner's file, and a + # blocking repo-wide lint over someone else's roster would red every PR the + # moment they add an openocd board (hil_flash treats the field as optional) + if f['name'] == 'openocd' and path.endswith('tinyusb.json'): + self.assertIn('vid_pid', f, + f"{path}: {board['name']} openocd flasher lacks vid_pid") + self.assertNotIn('vid_pid', f.get('args', ''), + f"{path}: {board['name']} packs vid_pid into args; use the field") + + +class TestRosterFlashersDispatch(unittest.TestCase): + """hil_test and hil_pool_check resolve a board's flasher with a bare + getattr(hil_flash, f'flash_{name}'), and hil_test does it inside a redirect_stdout — + so a renamed or typo'd roster name raises an AttributeError whose output is swallowed, + with nothing pointing at the roster as the thing to edit. Renaming a flash_*/reset_* + pair without updating every roster must fail here instead.""" + + def test_flash_and_reset_exist_for_every_roster_flasher(self): + for path, board in roster_flashers(): + name = board['flasher']['name'].lower() + for fn in (f'flash_{name}', f'reset_{name}'): + self.assertTrue(callable(getattr(hil_flash, fn, None)), + f'{path}: {board["name"]} uses flasher "{name}" ' + f'but hil_flash.{fn} does not exist') + + def test_firmware_suffix_known_for_every_roster_flasher(self): + """find_firmware falls back to accepting .elf-or-.bin when a flasher is missing + from FLASHER_SUFFIX, silently restoring the mismatch that map exists to catch.""" + for path, board in roster_flashers(): + name = board['flasher']['name'].lower() + self.assertIn(name, hil_flash.FLASHER_SUFFIX, + f'{path}: {board["name"]} uses flasher "{name}" ' + f'with no hil_flash.FLASHER_SUFFIX entry') + + +class FlasherRecoverEntry(unittest.TestCase): + """Optional roster key: a SECOND flasher used only to deliver recovery while a usbfs + node is poisoned. Boards whose primary flasher cannot get past a convoy (jlink, + stlink, lm4flash) name an openocd entry here instead of changing how they are + normally flashed.""" + + def test_recover_flasher_prefers_the_optional_entry(self): + prim = {'name': 'jlink', 'uid': 'X', 'args': '-device MIMXRT1064xxx6A'} + rec = {'name': 'openocd', 'uid': 'X', 'args': '-f interface/jlink.cfg -f target/foo.cfg'} + self.assertEqual(hil_flash.recover_flasher({'flasher': prim, 'flasher_recover': rec}), rec) + self.assertEqual(hil_flash.recover_flasher({'flasher': prim}), prim) + + def test_openocd_over_jlink_is_convoy_safe_without_a_pin(self): + """libjaylink discovery returns early unless idVendor == 0x1366 (SEGGER) and the PID + is in its table, and only THEN calls libusb_open (discovery_usb.c) -- it never opens + a foreign node. `adapter usb vid_pid` is a no-op for this driver: jlink.c reads + adapter_serial / usb address / usb location, never the vid/pid.""" + self.assertTrue(hil_flash.convoy_safe( + {'name': 'openocd', 'args': '-f interface/jlink.cfg -f target/stm32f4x.cfg'})) + + def test_openocd_with_neither_a_pin_nor_jlink_is_not_safe(self): + self.assertFalse(hil_flash.convoy_safe( + {'name': 'openocd', 'args': '-f interface/stlink.cfg -f target/stm32h7x.cfg'})) + + def test_the_existing_rules_are_unchanged(self): + self.assertTrue(hil_flash.convoy_safe( + {'name': 'openocd', 'vid_pid': '0x2e8a 0x000c', 'args': '-f interface/cmsis-dap.cfg'})) + self.assertFalse(hil_flash.convoy_safe({'name': 'jlink', 'uid': 'X'})) + self.assertTrue(hil_flash.convoy_safe({'name': 'esptool'})) + + +if __name__ == '__main__': + unittest.main(verbosity=1) diff --git a/test/hil/test/test_hil_util.py b/test/hil/test/test_hil_util.py new file mode 100644 index 000000000..9c3d5edef --- /dev/null +++ b/test/hil/test/test_hil_util.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests for hil_util.run_cmd's binary/split_stderr/quiet modes — real subprocesses, no +# hardware. Stdlib + hil_util only (hil_util is stdlib-only), so the pre-commit hil-test +# hook can run this on GitHub's bare runner. Run directly: +# python3 test/hil/test/test_hil_util.py +import io +import os +import shutil +import tempfile +import sys +import time +import unittest +from contextlib import redirect_stdout +from pathlib import Path + +# the module under test lives in the parent dir's helper/ package +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from helper import hil_util + + +@unittest.skipIf(os.name == 'nt', 'POSIX shell commands') +class RunCmdModes(unittest.TestCase): + def test_default_mode_unchanged(self): + r = hil_util.run_cmd('printf out; printf err >&2') + self.assertEqual(r.returncode, 0) + self.assertIsInstance(r.stdout, str) + # stderr merged into stdout, as every existing caller expects + self.assertIn('out', r.stdout) + self.assertIn('err', r.stdout) + + def test_binary_stdout_is_exact_bytes(self): + # \xff is not valid UTF-8: text mode would mangle it via errors='replace' + r = hil_util.run_cmd(r"printf 'a\377\000b'", binary=True) + self.assertEqual(r.returncode, 0) + self.assertEqual(r.stdout, b'a\xff\x00b') + + def test_split_stderr_keeps_stdout_clean(self): + r = hil_util.run_cmd('printf out; printf err >&2', split_stderr=True) + self.assertEqual(r.returncode, 0) + self.assertEqual(r.stdout, 'out') + self.assertEqual(r.stderr, 'err') + + def test_binary_split_stderr_timeout_returns_124(self): + t0 = time.monotonic() + r = hil_util.run_cmd(r"printf 'p\377re'; printf warn >&2; sleep 30", + binary=True, split_stderr=True, timeout=1) + self.assertEqual(r.returncode, 124) + # killpg + bounded communicate: well under sleep 30 + self.assertLess(time.monotonic() - t0, 15) + self.assertIn(b'p\xffre', r.stdout or b'') + # stderr collected before the timeout must survive the kill + self.assertIn(b'warn', r.stderr or b'') + + def test_text_mode_timeout_stdout_stays_str(self): + r = hil_util.run_cmd('sleep 30', timeout=1) + self.assertEqual(r.returncode, 124) + # a text-mode caller must never get bytes back, even empty + self.assertIsInstance(r.stdout, str) + + def test_failed_banner_includes_split_stderr(self): + # with split_stderr the diagnostic is in .stderr; the banner must not go blank. + # The text travels via env, not the command string — the banner title echoes the + # command, which would make a literal assertion pass vacuously. + os.environ['RUN_CMD_TEST_ERR'] = 'diagnostic-xyzzy' + self.addCleanup(os.environ.pop, 'RUN_CMD_TEST_ERR', None) + cap = io.StringIO() + with redirect_stdout(cap): + r = hil_util.run_cmd('printf "$RUN_CMD_TEST_ERR" >&2; exit 3', split_stderr=True) + self.assertEqual(r.returncode, 3) + self.assertIn('COMMAND FAILED', cap.getvalue()) + self.assertIn('diagnostic-xyzzy', cap.getvalue()) + + def test_no_group_markers_when_stdout_is_captured(self): + # GitHub folds ::group:: only at line start of the JOB's real stdout. Pool + # workers run tests under redirect_stdout and compact the capture into one + # row line, where the markers land mid-line and render as literal noise. + saved_ci = os.environ.get('CI') # pre-exists on GitHub runners: restore, not pop + os.environ['CI'] = '1' + self.addCleanup(lambda: os.environ.update({'CI': saved_ci}) if saved_ci is not None + else os.environ.pop('CI', None)) + cap = io.StringIO() + with redirect_stdout(cap): + r = hil_util.run_cmd('printf boom; exit 3') + self.assertEqual(r.returncode, 3) + self.assertIn('COMMAND FAILED', cap.getvalue()) + self.assertNotIn('::group::', cap.getvalue()) + self.assertNotIn('::endgroup::', cap.getvalue()) + + def test_quiet_suppresses_failed_banner(self): + # retry-loop callers report failures themselves; per-poll banners are noise + cap = io.StringIO() + with redirect_stdout(cap): + r = hil_util.run_cmd('printf boom >&2; exit 3', quiet=True) + self.assertEqual(r.returncode, 3) + self.assertNotIn('COMMAND FAILED', cap.getvalue()) + + +class BottomLayer(unittest.TestCase): + def test_bad_timeout_env_falls_back(self): + # hil_select (the PR-diff selector) imports hil_util for the example rosters; + # a malformed HIL_CMD_TIMEOUT must not crash the selector at import and knock + # CI back to the full-matrix fallback + import subprocess + r = subprocess.run( + [sys.executable, '-c', 'from helper import hil_util; print(hil_util.CMD_TIMEOUT)'], + cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + env={**os.environ, 'HIL_CMD_TIMEOUT': 'bogus'}, + capture_output=True, text=True, timeout=30) + self.assertEqual(r.returncode, 0, r.stderr) + # the warning must NOT be on stdout: hil_select's stdout is machine-read JSON + self.assertEqual(r.stdout.strip(), '180') + self.assertIn('warning', r.stderr) # but a silent fallback hides the misconfiguration + + def test_tinyusb_root_is_the_repo_root(self): + # the constant is derived from __file__ parents[N]; moving hil_util.py without + # adjusting N silently re-points every firmware/build path (it happened) + self.assertTrue((hil_util.TINYUSB_ROOT / 'examples').is_dir(), hil_util.TINYUSB_ROOT) + self.assertTrue((hil_util.TINYUSB_ROOT / 'test' / 'hil').is_dir(), hil_util.TINYUSB_ROOT) + + def test_hil_util_is_a_single_module_instance(self): + # helper modules must be imported via the helper package everywhere: a plain + # `import hil_util` from inside helper/ creates a SECOND module object, and + # state like `verbose` set on one copy never reaches the other + import hil_flash + from helper import hil_pool_check + self.assertIs(hil_flash.hil_util, hil_util) + self.assertIs(hil_pool_check.hil_util, hil_util) + self.assertIs(hil_pool_check.hil_flash, hil_flash) + + def test_bare_runner_modules_stay_stdlib_only(self): + # hil_examples.py used to make this structural (a list of strings cannot grow a + # dependency); with the rosters folded into hil_util the invariant needs teeth: + # everything the bare GitHub runner imports (selector + this suite) must stay + # stdlib + local. Adding pyserial/pymtp here breaks hil_select on CI. + import ast + hil_dir = Path(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + # ONLY the modules the bare runner can import -- not every stem in the tree. + # Globbing the directory allowed `import pymtp` (and hil_test, usbtest, + # mtp_test) through, so the pymtp case this test names could never fail: that + # module runs ctypes.CDLL(find_library('mtp')) at import and raises where there + # is no libmtp, taking hil_select down with it. + local = {'helper', 'hil_util', 'hil_select', 'hil_flash', + 'hil_health', 'hil_lock', 'hil_pool_check'} + allowed = set(sys.stdlib_module_names) | local + # hil_pool_check included: test_hil_util_is_a_single_module_instance imports it + # on the bare runner, and its `import serial` is function-local for exactly + # this reason -- hoisting it must fail HERE, not on every PR's pre-commit CI + for mod in ('helper/hil_util', 'hil_flash', 'helper/hil_select', + 'helper/hil_health', 'helper/hil_lock', 'helper/hil_pool_check'): + tree = ast.parse((hil_dir / f'{mod}.py').read_text()) + # module level only: a deferred import inside a function cannot break + # importability (hil_pool_check keeps `import serial` function-local + # for exactly that reason) + for node in tree.body: + roots = [] + if isinstance(node, ast.Import): + roots = [a.name.split('.')[0] for a in node.names] + elif isinstance(node, ast.ImportFrom) and node.module: + roots = [node.module.split('.')[0]] + for root in roots: + self.assertIn(root, allowed, + f'{mod}.py imports {root}, not stdlib/local - breaks the bare CI runner') + + +class BoundedReadBookkeeping(unittest.TestCase): + """Two ways the strand accounting lied, both of which cost a blindness credit -- and + the process goes blind after four.""" + + def test_a_value_that_arrived_at_the_deadline_is_not_a_strand(self): + """join() returns, is_alive() is still True, but the reader HAS deposited its + value. read_sysfs booked a strand from is_alive() alone, so a merely-slow healthy + read was memoised as unreadable forever. bounded_open already gets this right.""" + import threading, time as _t + before = hil_util._sysfs_stuck + self.addCleanup(setattr, hil_util, '_sysfs_stuck', before) + real_thread = threading.Thread + + class Lingering(real_thread): + """Deposits the value, then outlives the join by a hair.""" + def run(self): + super().run() + _t.sleep(0.6) # still alive when join(grace) returns + + self.addCleanup(setattr, threading, 'Thread', real_thread) + threading.Thread = Lingering + with tempfile.NamedTemporaryFile('w', suffix='_attr', delete=False) as fh: + fh.write('cafe\n') + path = fh.name + self.addCleanup(os.unlink, path) + hil_util.read_sysfs(path, grace=0.2) + self.assertEqual(hil_util._sysfs_stuck, before, + 'a value that arrived was still counted as a strand') + + def test_bounded_open_does_not_re_strand_a_known_path(self): + """Same rule read_sysfs has: re-opening a path known to hang costs another thread, + another fd and another blindness credit to learn what we already know. The printer + test re-opens ONE lp node on every retry.""" + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + fifo = os.path.join(d, 'lp0') + os.mkfifo(fifo) # open() blocks: no writer, ever + self.addCleanup(setattr, hil_util, '_sysfs_stuck', hil_util._sysfs_stuck) + self.addCleanup(setattr, hil_util, '_sysfs_stranded', dict(hil_util._sysfs_stranded)) + before = hil_util._sysfs_stuck + for _ in range(3): + hil_util.bounded_open(fifo, os.O_WRONLY, 0.3) + self.assertLessEqual(hil_util._sysfs_stuck - before, 1, + 'each retry spent another blindness credit on the same path') + + +class RunAlongsideKeepsStderrOffThePayload(unittest.TestCase): + """test_device_printer_to_cdc byte-compares run_alongside's stdout against the payload + it wrote. Merging stderr into that stream turns any stray child stderr byte -- a + PYTHONWARNINGS chirp, a sitecustomize print, a venv .pth deprecation -- into + 'CDC->Printer wrong data', sending a maintainer after the printer class driver for an + interpreter warning. hil_ci.sh runs python3 with no isolating flags.""" + + def test_child_stderr_does_not_contaminate_stdout(self): + from helper import hil_util + argv = [sys.executable, '-c', + 'import sys; sys.stderr.write("noise\\n"); sys.stdout.write("PAYLOAD")'] + r = hil_util.run_alongside(argv, lambda: time.sleep(0.2), timeout=20) + self.assertEqual(r.returncode, 0) + self.assertEqual(r.stdout, b'PAYLOAD', + 'child stderr leaked into the payload stream') + + +if __name__ == '__main__': + unittest.main() diff --git a/test/hil/test_hil_select.py b/test/hil/test_hil_select.py deleted file mode 100644 index 5e6b16759..000000000 --- a/test/hil/test_hil_select.py +++ /dev/null @@ -1,581 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MIT -# Unit tests for hil_select.py — pure logic, no hardware, no git. Run directly: -# python3 test/hil/test_hil_select.py -import glob -import json -import os -import sys -import unittest - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import hil_flash -import hil_select -from hil_examples import device_tests, dual_tests, host_test - -REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - - -def real_rosters(): - """The actual rig rosters, for regression tests that need real-world data - (a specific board/family/only-list) rather than the synthetic ROSTER above.""" - rosters = [] - for name in ('tinyusb.json', 'hfp.json'): - path = os.path.join(REPO, 'test/hil', name) - with open(path) as f: - rosters.append((f'test/hil/{name}', json.load(f)['boards'])) - return rosters - - -def roster_flashers(): - """(roster path, board) for every board in the live rosters, `boards-skip` - included: a parked board's flasher name must still dispatch, so that unparking it - is not what discovers the name went stale.""" - for name in ('tinyusb.json', 'hfp.json'): - path = os.path.join(REPO, 'test/hil', name) - with open(path) as f: - cfg = json.load(f) - for key in ('boards', 'boards-skip'): - for b in cfg.get(key, []): - yield f'test/hil/{name}', b - - -def on_roster(tc, *names): - """The subset of `names` currently in the live rig rosters, skipping the test - when none are. Parking/unparking a board is routine rig maintenance and must not - fail this suite: CI runs it right before the selector and treats a failure as - 'selector unusable', dropping PR scoping and annotating the run.""" - have = {b['name'] for _, boards in real_rosters() for b in boards} - got = [n for n in names if n in have] - if not got: - tc.skipTest(f'not in the rig roster: {", ".join(names)}') - return got - - -ROSTER = [ - # device-only, rp2040 family - {'name': 'raspberry_pi_pico', 'uid': 'u1', 'flasher': {'name': 'openocd'}, - 'tests': {'device': True, 'host': True, 'dual': True}}, - # device-only, stm32f4 family - {'name': 'stm32f407disco', 'uid': 'u2', 'flasher': {'name': 'jlink'}, - 'tests': {'device': True, 'host': False, 'dual': False}}, - # host-only board - {'name': 'raspberry_pi_pico2', 'uid': 'u3', 'flasher': {'name': 'openocd'}, - 'tests': {'device': False, 'host': True, 'dual': False}}, - # only-list board (espressif-style), flashed by the CI leg that splits on esptool - {'name': 'espressif_s3_devkitm', 'uid': 'u4', 'flasher': {'name': 'esptool'}, - 'tests': {'only': ['device/cdc_msc_freertos', 'host/device_info']}}, -] -ROSTERS = [('test/hil/tinyusb.json', ROSTER)] - - -def sel(files): - return hil_select.classify(files, REPO, ROSTERS) - - -class TestPortRule(unittest.TestCase): - def test_dcd_rp2040_selects_pico_family_only(self): - s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) - self.assertFalse(s['full']) - self.assertIn('raspberry_pi_pico', s['boards']) - self.assertNotIn('stm32f407disco', s['boards']) - self.assertNotIn('espressif_s3_devkitm', s['boards']) - # device role: no host tests in pico's list - self.assertTrue(all(not t.startswith('host/') for t in s['boards']['raspberry_pi_pico'])) - # host-only boards drop out entirely on a device-role change - self.assertNotIn('raspberry_pi_pico2', s['boards']) - - def test_shared_port_file_is_both_roles(self): - s = sel(['src/portable/synopsys/dwc2/dwc2_common.c']) - self.assertFalse(s['full']) - self.assertNotIn('raspberry_pi_pico', s['boards']) # rp2040 is not a dwc2 family - self.assertIn('stm32f407disco', s['boards']) # stm32f4 is - - -class TestCoreRoleRule(unittest.TestCase): - def test_usbd_selects_all_device_tests_everywhere(self): - s = sel(['src/device/usbd.c']) - self.assertFalse(s['full']) - self.assertNotIn('raspberry_pi_pico2', s['boards']) # host-only board dropped - pico = s['boards']['raspberry_pi_pico'] - self.assertTrue(set(device_tests).issubset(set(pico))) - self.assertTrue(set(dual_tests).issubset(set(pico))) # dual survives device role - self.assertTrue(all(not t.startswith('host/') for t in pico)) - # only-list board: selection intersects its only-list - esp = s['boards']['espressif_s3_devkitm'] - self.assertEqual(esp, ['device/cdc_msc_freertos']) - - def test_host_change_drops_device(self): - s = sel(['src/host/usbh.c']) - self.assertFalse(s['full']) - self.assertIn('raspberry_pi_pico2', s['boards']) - self.assertNotIn('stm32f407disco', s['boards']) # device-only board dropped - - -class TestClassRule(unittest.TestCase): - def test_cdc_device_selects_cdc_examples_only(self): - s = sel(['src/class/cdc/cdc_device.c']) - self.assertFalse(s['full']) - pico = s['boards']['raspberry_pi_pico'] - self.assertIn('device/cdc_msc', pico) - self.assertIn('device/cdc_dual_ports', pico) - self.assertNotIn('device/msc_dual_lun', pico) # CFG_TUD_CDC 0 there - self.assertNotIn('device/usbtest', pico) # CFG_TUD_CDC 0 there - self.assertTrue(all(not t.startswith('host/') for t in pico)) - - def test_msc_host_selects_host_side(self): - s = sel(['src/class/msc/msc_host.c']) - self.assertFalse(s['full']) - self.assertNotIn('stm32f407disco', s['boards']) # device-only board - pico2 = s['boards']['raspberry_pi_pico2'] - self.assertIn('host/msc_file_explorer', pico2) - self.assertTrue(all(not t.startswith('device/') for t in pico2)) - - -class TestClassIncludeEdges(unittest.TestCase): - """A class header another class includes reaches that class's examples too. - src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, so - midi_test's firmware contains audio.h - but the class rule derives macros from - the directory name alone, so an audio.h change used to select only - device/audio_test_freertos. On boards that skip that example the per-board - intersection emptied and an audio.h-only PR ran ZERO HIL on them.""" - def test_edges_derived_from_includes(self): - edges = hil_select.class_include_edges(REPO) - self.assertEqual(edges.get('audio/audio.h'), {'midi'}) - self.assertEqual(edges.get('cdc/cdc.h'), {'net'}) - - def test_audio_header_selects_midi_example(self): - s = hil_select.classify(['src/class/audio/audio.h'], REPO, real_rosters()) - self.assertFalse(s['full']) - # every board that runs device/midi_test at all must run it here (boards with - # a tests.only list, e.g. espressif, run the freertos examples instead) - by_name = {b['name']: b for _, bs in real_rosters() for b in bs} - checked = 0 - for name, tests in s['boards'].items(): - if 'device/midi_test' in hil_select.board_tests(by_name[name]): - self.assertIn('device/midi_test', tests, name) - checked += 1 - self.assertTrue(checked) - - def test_audio_header_reaches_boards_that_skip_audio(self): - # both skip device/audio_test_freertos: without the midi edge their - # intersection is empty and they drop out of the selection entirely - boards = on_roster(self, 'metro_m4_express', 'nrf54lm20dk') - s = hil_select.classify(['src/class/audio/audio.h'], REPO, real_rosters()) - for board in boards: - self.assertEqual(s['boards'].get(board), ['device/midi_test'], board) - - def test_edge_is_per_header_not_per_class(self): - # midi includes audio.h, not audio_device.h: an audio_device change must - # not drag midi's examples in - s = hil_select.classify(['src/class/audio/audio_device.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for tests in s['boards'].values(): - if tests != 'all': - self.assertNotIn('device/midi_test', tests) - - -class TestFallbackRules(unittest.TestCase): - def test_unknown_tool_is_full(self): - s = sel(['tools/random_new_script.py']) - self.assertTrue(s['full']) - - def test_docs_only_is_empty_not_full(self): - s = sel(['docs/info/contributing.rst', 'README.rst']) - self.assertFalse(s['full']) - self.assertEqual(s['boards'], {}) - - def test_bsp_family_selects_family_boards(self): - s = sel(['hw/bsp/rp2040/family.cmake']) - self.assertFalse(s['full']) - self.assertIn('raspberry_pi_pico', s['boards']) - self.assertEqual(s['boards']['raspberry_pi_pico'], 'all') - self.assertNotIn('stm32f407disco', s['boards']) - - def test_bsp_board_narrows_to_board(self): - s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) - self.assertFalse(s['full']) - self.assertEqual(list(s['boards'].keys()), ['raspberry_pi_pico']) - - def test_example_change_selects_that_example(self): - s = sel(['examples/device/cdc_msc/src/main.c']) - self.assertFalse(s['full']) - self.assertEqual(s['boards']['raspberry_pi_pico'], ['device/cdc_msc']) - - def test_core_common_is_full(self): - for f in ['src/tusb.c', 'src/common/tusb_fifo.c', 'src/osal/osal_freertos.h']: - self.assertTrue(sel([f])['full'], f) - - def test_board_test_example_is_full(self): - # board_test is the park/teardown firmware hil_test.py flashes on every board, - # not an unlisted example: a regression there must not skip the whole rig - for f in ['examples/device/board_test/src/main.c', - 'examples/device/board_test/CMakeLists.txt']: - self.assertTrue(sel([f])['full'], f) - - def test_harness_is_full(self): - for f in ['test/hil/hil_test.py', '.github/workflows/build.yml', 'hw/mcu/nxp/x.c', 'lib/foo/x.c']: - self.assertTrue(sel([f])['full'], f) - - def test_mixed_roles_no_pruning(self): - s = sel(['src/device/usbd.c', 'src/host/usbh.c']) - self.assertFalse(s['full']) - self.assertIn('raspberry_pi_pico2', s['boards']) - self.assertIn('stm32f407disco', s['boards']) - - def test_cmakelists_and_requirements_are_full(self): - for f in ['src/CMakeLists.txt', 'examples/CMakeLists.txt', - 'examples/device/CMakeLists.txt', 'test/hil/requirements.txt']: - self.assertTrue(sel([f])['full'], f) - - def test_docs_txt_is_noncode(self): - s = sel(['docs/info/changelog.txt']) - self.assertFalse(s['full']) - self.assertEqual(s['boards'], {}) - - -class TestArgsEmission(unittest.TestCase): - def test_args_for_scoped_selection(self): - s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) - args = hil_select.selection_args(s, ROSTERS) - a = args['tinyusb.json'] - self.assertIn('-b raspberry_pi_pico', a) - self.assertNotIn('stm32f407disco', a) - self.assertIn('-bt raspberry_pi_pico:', a) # device-only subset of a device+host board - - def test_args_full_is_empty(self): - s = sel(['tools/random_new_script.py']) - self.assertEqual(hil_select.selection_args(s, ROSTERS), {'tinyusb.json': ''}) - - def test_args_all_board_gets_bare_b(self): - s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) - a = hil_select.selection_args(s, ROSTERS)['tinyusb.json'] - self.assertIn('-b raspberry_pi_pico', a) - self.assertNotIn('-bt', a) - - def test_args_by_flasher_splits_esp_from_the_rest(self): - s = sel(['src/device/usbd.c']) - per = hil_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json'] - self.assertIn('espressif_s3_devkitm', per['esptool']) - self.assertIn('raspberry_pi_pico', per['openocd']) - self.assertNotIn('espressif_s3_devkitm', per.get('openocd', '') + per.get('jlink', '')) - - def test_args_by_flasher_omits_a_flasher_with_no_selected_board(self): - # the esp CI leg must see no args at all here, not a filter matching zero boards - s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) - per = hil_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json'] - self.assertEqual(per, {'openocd': '-b raspberry_pi_pico'}) - - def test_args_by_flasher_full_is_empty(self): - s = sel(['tools/random_new_script.py']) - self.assertEqual(hil_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}}) - - def test_cli_diff_file(self): - import subprocess, tempfile, json as j - with tempfile.NamedTemporaryFile('w', suffix='.txt', delete=False) as f: - f.write('src/class/cdc/cdc_device.c\n') - path = f.name - r = subprocess.run([sys.executable, os.path.join(REPO, 'test/hil/hil_select.py'), - '--diff-file', path, os.path.join(REPO, 'test/hil/tinyusb.json')], - capture_output=True, text=True) - self.assertEqual(r.returncode, 0, r.stderr) - out = j.loads(r.stdout) - self.assertFalse(out['full']) - self.assertIn('tinyusb.json', out['args']) - self.assertTrue(any('cdc_device' in line for line in out['reasons'])) - os.unlink(path) - - -class TestRealRosterPortFamilies(unittest.TestCase): - """Regression for port_families() missing espressif's dwc2 reference, which - lives in a component CMakeLists.txt rather than family.cmake/family.mk.""" - def test_dwc2_change_selects_espressif_boards(self): - boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') - s = hil_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - self.assertIn(board, s['boards']) - - -class TestOptionGatedPort(unittest.TestCase): - """Regression: family_support.cmake compiles some ports from a build option - (MAX3421_HOST=1 -> hcd_max3421.c), so a board's family file never names them.""" - # host-side option board (max3421 as host controller), off any max3421 family - OPT_ROSTER = [('test/hil/opt.json', [ - {'name': 'fake_dual_board', 'uid': 'o1', 'flasher': {'name': 'jlink'}, - 'build': {'args': ['MAX3421_HOST=1']}, - 'tests': {'device': True, 'host': False, 'dual': True}}, - {'name': 'fake_host_board', 'uid': 'o2', 'flasher': {'name': 'jlink'}, - 'variant': [{'name': 'fake_host_board', 'flags': '-DMAX3421_HOST=1'}], - 'tests': {'device': False, 'host': True, 'dual': False}}, - {'name': 'fake_off_board', 'uid': 'o3', 'flasher': {'name': 'jlink'}, - 'variant': [{'name': 'fake_off_board', 'defines': ['MAX3421_HOST=0']}], - 'tests': {'device': True, 'host': True, 'dual': True}}, - ])] - - def test_real_roster_max3421_selects_option_board(self): - boards = on_roster(self, 'metro_m4_express') - s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - self.assertIn(board, s['boards']) - - def test_option_selects_via_args_defines_and_flags(self): - s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, self.OPT_ROSTER) - self.assertFalse(s['full']) - self.assertIn('fake_dual_board', s['boards']) # build.args - self.assertIn('fake_host_board', s['boards']) # variant flags - self.assertNotIn('fake_off_board', s['boards']) # variant defines, but =0 - - def test_device_role_port_does_not_pull_host_only_option_board(self): - s = hil_select.classify(['src/portable/analog/max3421/dcd_max3421.c'], REPO, self.OPT_ROSTER) - self.assertFalse(s['full']) - self.assertNotIn('fake_host_board', s['boards']) # host-only board, device change - self.assertIn('fake_dual_board', s['boards']) # device-capable option board - - def test_gates_parsed_from_family_support(self): - self.assertEqual(hil_select.port_option_gates(REPO).get('analog/max3421'), - {'MAX3421_HOST'}) - - def test_board_cmake_option_counts(self): - """A board can enable a gated port in its own BSP rather than via the roster - (hw/bsp/espressif/boards/*/board.cmake -> set(MAX3421_HOST 1)); board_options() - must see those too, or such a board joining the roster is silently dropped.""" - self.assertIn('MAX3421_HOST', - hil_select.bsp_board_options('adafruit_feather_esp32s3', REPO)) - self.assertIn('CFG_TUH_RPI_PIO_USB', - hil_select.bsp_board_options('adafruit_fruit_jam', REPO)) - # commented-out `# set(MAX3421_HOST 1)` must not count - self.assertNotIn('MAX3421_HOST', - hil_select.bsp_board_options('feather_nrf52840_express', REPO)) - - def test_board_cmake_option_selects_off_family_board(self): - # adafruit_feather_esp32s3 is not on any rig roster; stand it in as one to - # prove the BSP-sourced option alone pulls a max3421 change onto the board - roster = [('test/hil/opt.json', [ - {'name': 'adafruit_feather_esp32s3', 'uid': 'o1', 'flasher': {'name': 'esptool'}, - 'tests': {'device': False, 'host': True, 'dual': False}}])] - s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster) - self.assertFalse(s['full']) - self.assertIn('adafruit_feather_esp32s3', s['boards']) - - def test_board_mk_option_is_ignored(self): - """Make-only options must not select: HIL CI builds with CMake exclusively, so - hw/bsp/nrf/boards/nrf5340dk/board.mk's MAX3421_HOST compiles nothing here.""" - roster = [('test/hil/opt.json', [ - {'name': 'nrf5340dk', 'uid': 'o1', 'flasher': {'name': 'jlink'}, - 'tests': {'device': False, 'host': True, 'dual': False}}])] - s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster) - self.assertFalse(s['full']) - self.assertEqual(s['boards'], {}) - - -class TestPortFamiliesCmakeOnly(unittest.TestCase): - """port_families() is CMake-only (HIL CI never builds with Make) and matches on - 'port_dir/' so a port dir is not a prefix of a sibling.""" - def test_make_only_family_is_not_a_family(self): - # hw/bsp/pic32mz has family.mk but no family.cmake - self.assertEqual(hil_select.port_families('microchip/pic32mz', REPO), set()) - - def test_prefix_port_does_not_inherit_sibling_families(self): - # bare-substring matching let 'microchip/pic' match '.../microchip/pic32mz/...' - self.assertEqual(hil_select.port_families('microchip/pic', REPO), set()) - - def test_make_only_port_forces_full(self): - s = sel(['src/portable/microchip/pic32mz/dcd_pic32mz.c']) - self.assertTrue(s['full']) - self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons']) - - def test_cmake_families_still_found(self): - self.assertEqual(hil_select.port_families('raspberrypi/rp2040', REPO), {'rp2040'}) - self.assertIn('stm32f4', hil_select.port_families('synopsys/dwc2', REPO)) - - -class TestPortFamiliesCoverage(unittest.TestCase): - """Systematic guard: every real dcd_*/hcd_* port directory should map to at - least one board family, so a future family.cmake/CMakeLists.txt layout that - port_families() doesn't scan fails loudly instead of silently dropping boards - (as espressif's dwc2 reference did - see TestRealRosterPortFamilies).""" - # Ports with no board family: not a bug, just not wired into any rig board. - # Add here (with a reason) only if port_families() legitimately can't find one. - # A port listed here force-fulls (fail-open), so it is never under-selected. - NO_FAMILY = { - 'template', # reference/example port, not built by any board - # hw/bsp/pic32mz has family.mk only (no family.cmake), and port_families() - # is CMake-only because HIL CI builds every board with CMake - so this port - # is compiled for no HIL board. - 'microchip/pic32mz', - 'microchip/pic', # same: only ever referenced from pic32mz's family.mk - } - - @staticmethod - def _dcd_hcd_ports(): - portable_root = os.path.join(REPO, 'src/portable') - ports = [] - for entry in sorted(os.listdir(portable_root)): - d = os.path.join(portable_root, entry) - if not os.path.isdir(d): - continue - if glob.glob(os.path.join(d, 'dcd_*.c')) or glob.glob(os.path.join(d, 'hcd_*.c')): - ports.append(entry) - continue - for sub in sorted(os.listdir(d)): - sd = os.path.join(d, sub) - if os.path.isdir(sd) and (glob.glob(os.path.join(sd, 'dcd_*.c')) or - glob.glob(os.path.join(sd, 'hcd_*.c'))): - ports.append(f'{entry}/{sub}') - return ports - - def test_every_port_maps_to_a_family(self): - ports = self._dcd_hcd_ports() - self.assertTrue(ports) # sanity: the scan itself found something - for port in ports: - if port in self.NO_FAMILY: - continue - fams = hil_select.port_families(port, REPO) - self.assertTrue(fams, f'{port}: no family references this port ' - f'(port_families() scan gap, or add to NO_FAMILY)') - - -class TestRealRosterOnlyListTests(unittest.TestCase): - """Regression for roster-only-list tests (e.g. espressif's hid_composite_freertos) - being invisible to the selector because it only knew the shared hil_examples lists.""" - def test_only_list_example_change_selects_it(self): - boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') - s = hil_select.classify(['examples/device/hid_composite_freertos/src/main.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - self.assertEqual(s['boards'][board], ['device/hid_composite_freertos']) - - def test_class_change_includes_only_list_boards(self): - boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') - s = hil_select.classify(['src/class/hid/hid_device.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - self.assertIn(board, s['boards']) - - -class TestPortAndCoreRoleUseExtras(unittest.TestCase): - """Regression: the port rule and core-role rule must thread the roster-only - test universe (extras) the same way the class rule already does, so a DCD - or device-stack change doesn't silently drop espressif's only-list tests - (e.g. hid_composite_freertos) that aren't in the shared device_tests list.""" - def test_dcd_change_includes_only_list_test(self): - boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') - s = hil_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - tests = s['boards'][board] - self.assertIn('device/hid_composite_freertos', tests) - self.assertIn('device/cdc_msc_freertos', tests) - self.assertIn('device/audio_test_freertos', tests) - self.assertIn('device/usbtest', tests) - - def test_core_device_change_includes_only_list_test(self): - boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') - s = hil_select.classify(['src/device/usbd.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - tests = s['boards'][board] - self.assertIn('device/hid_composite_freertos', tests) - self.assertIn('device/cdc_msc_freertos', tests) - self.assertIn('device/audio_test_freertos', tests) - self.assertIn('device/usbtest', tests) - - def test_host_change_does_not_leak_device_only_list_test(self): - s = hil_select.classify(['src/host/usbh.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board, tests in s['boards'].items(): - if tests == 'all': - continue - self.assertNotIn('device/hid_composite_freertos', tests, board) - - -class TestFamilies(unittest.TestCase): - """`families` exists for consumers that build (not just test) the diff: most - families have no rig board, so `boards` alone would compile nothing for them.""" - def test_off_rig_port_still_reports_family(self): - s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c']) - self.assertFalse(s['full']) - self.assertEqual(s['boards'], {}) # no same7x board on the rig - self.assertEqual(s['families'], ['same7x']) - - def test_port_families_are_reported(self): - s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) - self.assertIn('rp2040', s['families']) - - def test_bsp_family_and_board_report_family(self): - self.assertEqual(sel(['hw/bsp/rp2040/family.cmake'])['families'], ['rp2040']) - self.assertEqual(sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h'])['families'], - ['rp2040']) - - def test_docs_only_has_no_families(self): - self.assertEqual(sel(['docs/info/contributing.rst'])['families'], []) - - def test_full_selection_still_reports_families(self): - """A full-matrix file must not hide the families of the other changed files: - consumers that build from `families` (e.g. /pre-pr) ignore `boards` when full.""" - s = sel(['src/common/tusb_fifo.c', 'src/portable/microchip/samx7x/dcd_samx7x.c']) - self.assertTrue(s['full']) - self.assertIn('same7x', s['families']) - # full stays full: every roster board, and no args to narrow the run - self.assertEqual(set(s['boards']), {b['name'] for b in ROSTER}) - self.assertTrue(all(v == 'all' for v in s['boards'].values())) - self.assertEqual(hil_select.selection_args(s, ROSTERS), {'tinyusb.json': ''}) - self.assertEqual(hil_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}}) - - def test_family_order_does_not_matter(self): - # same as above with the full-matrix file last (was the only order that worked) - s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c', 'src/common/tusb_fifo.c']) - self.assertTrue(s['full']) - self.assertIn('same7x', s['families']) - - -class TestGitDiffArgv(unittest.TestCase): - def test_diff_disables_rename_detection(self): - """Without --no-renames git reports only a rename's destination, so moving an - HIL-relevant file to a non-code path would be classified as non-code only.""" - self.assertIn('--no-renames', hil_select.GIT_DIFF_ARGV) - - -class TestPortWithoutFamilyIsFull(unittest.TestCase): - """A port dir no family file references must widen (full matrix), not silently - contribute zero boards — the fail-open contract.""" - def test_unreferenced_port_forces_full(self): - orig = hil_select.port_families - hil_select.port_families = lambda port_dir, repo_root: set() - try: - s = sel(['src/portable/vendor/newip/dcd_newip.c']) - finally: - hil_select.port_families = orig - self.assertTrue(s['full']) - self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons']) - - -class TestRosterFlashersDispatch(unittest.TestCase): - """hil_test and hil_pool_check resolve a board's flasher with a bare - getattr(hil_flash, f'flash_{name}'), and hil_test does it inside a redirect_stdout — - so a renamed or typo'd roster name raises an AttributeError whose output is swallowed, - with nothing pointing at the roster as the thing to edit. Renaming a flash_*/reset_* - pair without updating every roster must fail here instead.""" - - def test_flash_and_reset_exist_for_every_roster_flasher(self): - for path, board in roster_flashers(): - name = board['flasher']['name'].lower() - for fn in (f'flash_{name}', f'reset_{name}'): - self.assertTrue(callable(getattr(hil_flash, fn, None)), - f'{path}: {board["name"]} uses flasher "{name}" ' - f'but hil_flash.{fn} does not exist') - - def test_firmware_suffix_known_for_every_roster_flasher(self): - """find_firmware falls back to accepting .elf-or-.bin when a flasher is missing - from FLASHER_SUFFIX, silently restoring the mismatch that map exists to catch.""" - for path, board in roster_flashers(): - name = board['flasher']['name'].lower() - self.assertIn(name, hil_flash.FLASHER_SUFFIX, - f'{path}: {board["name"]} uses flasher "{name}" ' - f'with no hil_flash.FLASHER_SUFFIX entry') - - -if __name__ == '__main__': - unittest.main(verbosity=1) diff --git a/test/hil/tinyusb.json b/test/hil/tinyusb.json index c9b38992c..549a17cd0 100644 --- a/test/hil/tinyusb.json +++ b/test/hil/tinyusb.json @@ -149,6 +149,7 @@ "flasher": { "name": "openocd", "uid": "E6614C311B597D32", + "vid_pid": "0x2e8a 0x000c", "args": "-f interface/cmsis-dap.cfg -f target/max32665.cfg", "verify": true } @@ -240,6 +241,7 @@ "flasher": { "name": "openocd", "uid": "E6614103E72C1D2F", + "vid_pid": "0x2e8a 0x000c", "args": "-f interface/cmsis-dap.cfg -f target/rp2040.cfg -c \"adapter speed 5000\"", "verify": true } @@ -270,6 +272,7 @@ "flasher": { "name": "openocd", "uid": "E6633861A3819D38", + "vid_pid": "0x2e8a 0x000c", "args": "-f interface/cmsis-dap.cfg -f target/rp2040.cfg -c \"adapter speed 5000\"", "verify": true }, @@ -296,6 +299,7 @@ "flasher": { "name": "openocd", "uid": "E6633861A3978538", + "vid_pid": "0x2e8a 0x000c", "args": "-f interface/cmsis-dap.cfg -f target/rp2350.cfg -c \"adapter speed 5000\"", "verify": true } @@ -326,6 +330,7 @@ "flasher": { "name": "openocd", "uid": "E663AC91D3359B38", + "vid_pid": "0x2e8a 0x000c", "args": "-f interface/cmsis-dap.cfg -f target/rp2350.cfg -c \"adapter speed 5000\"", "verify": true } @@ -407,9 +412,8 @@ "dual": false }, "flasher": { - "name": "openocd", + "name": "stlink", "uid": "004C00343137510F39383538", - "args": "-f interface/stlink.cfg -f target/stm32h7x.cfg", "verify": true } }, @@ -422,9 +426,8 @@ "dual": false }, "flasher": { - "name": "openocd", + "name": "stlink", "uid": "066FFF495087534867063844", - "args": "-f interface/stlink.cfg -f target/stm32g0x.cfg", "verify": true }, "comment": "32-bit scheme, 2KB USB SRAM" @@ -472,6 +475,7 @@ "flasher": { "name": "openocd", "uid": "A76D8F062C2A", + "vid_pid": "0x1a86 0x8010", "args": "-f target/wch-riscv.cfg", "verify": false } @@ -488,6 +492,7 @@ "flasher": { "name": "openocd", "uid": "BC4954081051", + "vid_pid": "0x1a86 0x8010", "args": "-f target/wch-riscv.cfg", "verify": false } @@ -508,6 +513,7 @@ "flasher": { "name": "openocd", "uid": "BC5DA47360D0", + "vid_pid": "0x1a86 0x8010", "args": "-f target/wch-riscv.cfg", "verify": false } @@ -524,6 +530,7 @@ "flasher": { "name": "openocd", "uid": "57468F06DC03", + "vid_pid": "0x1a86 0x8010", "args": "-f target/wch-riscv.cfg", "verify": false } diff --git a/test/hil/usbtest.py b/test/hil/usbtest.py index 83ea3e24c..485e9e0e4 100755 --- a/test/hil/usbtest.py +++ b/test/hil/usbtest.py @@ -25,7 +25,9 @@ capability flags only unlock cases, they don't require the endpoints to exist. import argparse import json +from contextlib import redirect_stdout import os +import pathlib import re import shutil import subprocess @@ -33,16 +35,47 @@ import sys import time from pathlib import Path +sys.path.append(os.path.dirname(os.path.abspath(__file__))) # PYTHONSAFEPATH drops it + VID = 'cafe' PID = '4010' GZ_REF = '0525 a4a0' # copy Gadget Zero's capability profile (ctrl_out+iso+intr) SYS_USB = Path('/sys/bus/usb/devices') DRIVER = Path('/sys/bus/usb/drivers/usbtest') -USB_RECOVER = Path(__file__).resolve().parents[2] / '.claude/skills/usb-kernel-recover/scripts/usb_recover.sh' PATTERN_PARAM = Path('/sys/module/usbtest/parameters/pattern') - -# Battery per tier, in run order: control sanity first, then simple bulk, -# queued, unaligned, unlink, halt/toggle, throughput last. +RECOVER_FLASH_TIMEOUT = 90 # bound on the post-hang reflash; typical flash is 10-20s +RECOVER_RESET_TIMEOUT = 30 # bound on the post-hang probe reset; ResetTarget measures ~130ms + + +def recovery_steps(flasher_name: str, time_left: float) -> list: + """Ordered (kind, bound) recovery attempts that fit in `time_left`. + + RESET FIRST, reflash second. A probe reset fails the in-flight URB at the source just + as a park-flash does, but it is non-destructive -- the firmware under test survives, so + the wedge can still be autopsied -- writes no flash, and cannot brick SWD the way a bad + park image has on mimxrt1064_evk and max32666fthr (survived a power cycle). Measured + 128-129 ms against a full erase+program, and it works on i.MX RT and on DWC2 alike + (stm32f407disco, 2026-08-16: `r; g` -> USB disconnect, re-enumerated 325 ms later). + + The reset also fits budgets a reflash does not: the old gate skipped recovery entirely + when RECOVER_FLASH_TIMEOUT did not fit, which left the holder in place for the next + job. Whether either worked is decided by wedged_pids(), never by the exit code -- a + clean flash only proves the probe wrote the MCU. + """ + import hil_flash + steps = [] + reset_fn = getattr(hil_flash, f'reset_{flasher_name.lower()}', None) + if getattr(reset_fn, 'no_op', False): + reset_fn = None # a stub that returns rc 0 without resetting: do not claim it + if reset_fn and time_left >= RECOVER_RESET_TIMEOUT: + steps.append(('reset', RECOVER_RESET_TIMEOUT)) + if time_left >= RECOVER_FLASH_TIMEOUT: + steps.append(('flash', RECOVER_FLASH_TIMEOUT)) + return steps +HELPER_TIMEOUT = 30 # default bound for sudo helpers (dmesg/modprobe/setpci/tee) + +# Battery per tier, in run order: control sanity, simple bulk, queued, unaligned, unlink, +# halt/toggle, throughput last. TIER_CASES = { 1: [0, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 17, 18, 19, 20, 11, 12, 24, 13, 29, 27, 28], 2: [14, 21], @@ -50,10 +83,10 @@ TIER_CASES = { 4: [15, 16, 22, 23], } -# Per-case testusb parameters (full speed / high speed). All -s/-v values are -# multiples of 512 so transfers stay packet-aligned at both speeds: the device -# streams whole max-size packets and a non-aligned IN length would babble. -# 14/21 must never run with defaults (vary >= length is -EINVAL in the kernel). +# Per-case testusb parameters (full speed / high speed). All -s/-v values are multiples +# of 512 so transfers stay packet-aligned at both speeds: the device streams whole max-size +# packets and a non-aligned IN length would babble. 14/21 must never run with defaults +# (vary >= length is -EINVAL in the kernel). PARAMS = { 0: ('-c 1', '-c 1'), 9: ('-c 256', '-c 1000'), @@ -88,9 +121,40 @@ RE_FAIL = re.compile(r'test (\d+) --> (\d+) \((.*)\)') def run(cmd, **kw): - kw.setdefault('capture_output', True) + # NOT subprocess.run(timeout=): CPython's post-timeout path is an UNBOUNDED wait() that + # never returns on a D-state child -- the hang sysfs_write's timeout exists to catch. + timeout = kw.pop('timeout', None) + data = kw.pop('input', None) # subprocess.run-only kwarg; Popen takes stdin + kw.pop('capture_output', None) # ditto: expressed by the PIPEs below kw.setdefault('text', True) - return subprocess.run(cmd, **kw) + kw.setdefault('encoding', 'utf-8') + kw.setdefault('errors', 'replace') # strict decode would raise out of _sudo_soft + # NO start_new_session: these helpers (dmesg, modprobe, setpci, tee) must stay in our + # process group so hil_test's outer killpg reaps them with us. + timeout = timeout if timeout is not None else HELPER_TIMEOUT + proc = subprocess.Popen(cmd, stdin=subprocess.PIPE if data is not None else None, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kw) + try: + out, err = proc.communicate(input=data, timeout=timeout) + return subprocess.CompletedProcess(cmd, proc.returncode, out, err) + except subprocess.TimeoutExpired: + # Under sudo our child is only the wrapper; the root grandchild survives this and + # is left for the report and hil_pool_check to name. Close our pipe ends so an + # abandoned child costs no fds. + try: + proc.kill() # same group as us: never killpg, that would kill us too + except OSError: + pass + try: + proc.communicate(timeout=5) + except subprocess.TimeoutExpired: + for pipe in (proc.stdout, proc.stderr, proc.stdin): + try: + if pipe is not None: + pipe.close() + except OSError: + pass # unkillable: abandon it, the caller reports the timeout + raise def sudo(cmd, **kw): @@ -104,29 +168,106 @@ def sudo(cmd, **kw): def sysfs_write(path, data, check=True): - # A driver-registry write (new_id/remove_id/bind) blocks in D state when a wedged device - # holds its lock (driver_attach walks the bus): fail fast and loud instead of piling up - # unkillable writers and hanging the whole run -- the rig needs USB recovery first. + # A driver-registry write (new_id/remove_id/bind) blocks in D state when a wedged + # device holds its lock: fail fast instead of piling up unkillable writers -- the rig + # needs USB recovery first. + # + # Verified in v6.12.96: unbind_store -> device_driver_detach -> + # device_release_driver_internal -> __device_driver_lock (drivers/base/dd.c), which + # takes device_lock() -- the UNINTERRUPTIBLE variant, unlike the sysfs read path -- and + # ALSO device_lock(parent), because usb_bus_type sets need_parent_lock = true + # (drivers/usb/core/driver.c:2048). So one such write against a wedged device blocks + # unkillably while holding the HUB's lock: that is the mechanism by which a single + # wedged port takes its whole bus down, and why this fails fast instead. try: r = sudo(['tee', str(path)], input=data, timeout=15) except subprocess.TimeoutExpired: sys.exit(f'write "{data}" > {path} blocked >15s: USB subsystem is wedged ' - '(a D-state device lock exists). Recover the rig (usb_recover.sh) ' + '(a D-state device lock exists). Recover the rig (usb-kernel-recover skill) ' 'before running batteries.') if check and r.returncode != 0: sys.exit(f'write "{data}" > {path} failed: {r.stderr.strip()}') return r.returncode == 0 +def _read_sysfs_bounded(path, grace=1.0): + """Bounded sysfs attribute read. The value, or None, or hil_util.SYSFS_UNKNOWN. + + Delegates to hil_util.read_sysfs (imported here, like every helper import in this + file) so both properties hold: the strand cap -- find_device re-scans after EVERY + case, so a 30-case battery against a wedged peer would otherwise strand dozens of + threads and fds -- and UNKNOWN kept distinct from None. Folding UNKNOWN into None made + a blinded scan read as "device dropped off the bus", which aborts down a path that + skips the HUNG recovery entirely. + """ + from helper import hil_util + return hil_util.read_sysfs(str(path), grace) + + +_DEV_CACHE: dict = {} # serial -> sysname, see find_device + + +def _reread(sysname, serial): + """Re-describe an already-resolved device, CONFIRMING its serial. + + idVendor/idProduct/busnum/devnum/speed are lock-free (sysfs.c:688-705), so they cannot + block on a wedged peer -- but every identical board answers them the same, so they + prove nothing about identity. `serial` does, at one bounded read: a sysname is a + topology path, and after a renumber (controller reset, reboot) it can name a DIFFERENT + cafe:4010 board whose verdicts would be filed under this one. Returns None when the + serial is gone, mismatched or unconfirmed -- caller falls back to a full scan. + """ + d = SYS_USB / sysname + try: + if ((d / 'idVendor').read_text().strip() != VID + or (d / 'idProduct').read_text().strip() != PID): + return None + dev_serial = _read_sysfs_bounded(d / 'serial') + if not isinstance(dev_serial, str) or dev_serial.lower() != serial.lower(): + return None # gone, mismatched, or unconfirmable -> full scan decides + return { + 'sysname': sysname, + 'serial': dev_serial, + 'node': '/dev/bus/usb/%03d/%03d' % (int((d / 'busnum').read_text()), + int((d / 'devnum').read_text())), + 'speed': (d / 'speed').read_text().strip(), + 'tier': int((d / 'bcdDevice').read_text().strip()[-2:], 16), + } + except (OSError, ValueError): + return None + + def find_device(serial, first=False): - """Locate the usbtest device in sysfs, return info dict or None.""" - matches = [] + """Locate the usbtest device in sysfs, return info dict or None. + + Cached by serial: this is called after EVERY case, and a full scan pays a bounded + but real `serial` read for every cafe:4010 peer on the rig. With another board + wedged that cost lands on a HEALTHY battery ~30 times over, truncating it into + BUDGET entries. The fast path pays ONE bounded read -- our own device's serial, the + only attribute that tells identical boards apart (see _reread). + """ + if serial: + sysname = _DEV_CACHE.get(serial.lower()) + if sysname: + hit = _reread(sysname, serial) + if hit: + return hit + _DEV_CACHE.pop(serial.lower(), None) + matches, inconclusive = [], [] for dev in SYS_USB.iterdir(): try: if (dev / 'idVendor').read_text().strip() != VID or \ (dev / 'idProduct').read_text().strip() != PID: continue - dev_serial = (dev / 'serial').read_text().strip() + # BOUNDED: idVendor/idProduct are cached descriptors, but `serial` is served + # under device_lock(), so an unbounded read blocks us in D state on exactly the + # DUT whose hang we are here to report, losing every verdict collected so far. + dev_serial = _read_sysfs_bounded(dev / 'serial') + if dev_serial is not None and not isinstance(dev_serial, str): + inconclusive.append(dev.name) # unknown: NOT proof it is not ours + continue + if dev_serial is None: + continue if serial and dev_serial.lower() != serial.lower(): continue matches.append({ @@ -140,12 +281,17 @@ def find_device(serial, first=False): except (OSError, ValueError): continue if not matches: - return None + # "could not tell" is not "gone". The caller aborts the battery on a falsy return + # and that path skips the HUNG reflash, so a blinded scan would report the wedge + # we exist to recover from as a physical disconnect. + return {'inconclusive': inconclusive} if inconclusive else None + if serial and len(matches) == 1: + _DEV_CACHE[serial.lower()] = matches[0]['sysname'] if len(matches) > 1 and not first: if serial: - # Dual-port parts (nanoch32v203 fsdev/usbfs, ch32v307 usbhs/usbfs) briefly enumerate - # BOTH ports with the same serial around a variant reflash; picking one arbitrarily - # could bind the stale port. Report ambiguity so the caller retries until it drops. + # Dual-port parts (nanoch32v203, ch32v307) briefly enumerate BOTH ports with + # one serial around a variant reflash, and picking one could bind the stale + # port -- report ambiguity so the caller retries until it drops. return {'ambiguous': sorted(m['sysname'] for m in matches)} sys.exit(f'multiple {VID}:{PID} devices found, use --serial: ' + ', '.join(m["serial"] for m in matches)) @@ -165,8 +311,8 @@ def check_host_compat(dev): vid_did = ((pci / 'vendor').read_text().strip(), (pci / 'device').read_text().strip()) break except (OSError, ValueError): - # transient sysfs error (e.g. racing a re-enumeration): retry so a blip doesn't - # silently pass an incompatible host; if the probe truly fails, fail open but say so + # transient sysfs error (racing a re-enumeration): retry so a blip does not + # silently pass an incompatible host, then fail open but say so if attempt == 2: print('warning: cannot probe the upstream host controller; ' 'skipping the host compatibility check', file=sys.stderr) @@ -178,19 +324,16 @@ def check_host_compat(dev): 'placed in the EHCI periodic schedule and unlinked reads complete as short ' 'transfers (EREMOTEIO). Move the DUT to an xHCI port.') if drv.startswith('xhci') and vid_did in (('0x1912', '0x0014'), ('0x1912', '0x0015')): - # The Renesas uPD720201/uPD720202 must run its latest firmware (>= 2.0.2.6, - # K2026090.mem; RAM-uploaded, so it reverts to ROM on every power cycle unless - # re-loaded). On the ROM firmware its command ring intermittently dies under unlink - # stress: a Configure Endpoint command stops completing, the hub worker deadlocks - # holding the device lock (needs a host power cycle). Three separate boards killed - # it this way (ch32v307 2026-07-10; ra6m5 test 24, mimxrt1015 2026-07-11). Both - # parts expose the FW version register at PCI config offset 0x6c. NOTE this check - # is necessary, not sufficient: board-specific batteries have killed the controller - # on current firmware too (mimxrt1015, stop-endpoint timeout) - those are handled - # by per-board skips in the rig config. + # The Renesas uPD720201/uPD720202 must run firmware >= 2.0.2.6 (K2026090.mem; + # RAM-uploaded, so it reverts to ROM on every power cycle): on ROM firmware its + # command ring dies under unlink stress and the hub worker deadlocks holding the + # device lock, needing a host power cycle (ch32v307 2026-07-10; ra6m5 test 24, + # mimxrt1015 2026-07-11). Both parts expose the FW version at PCI config 0x6c. + # Necessary, not sufficient -- batteries have killed the controller on current + # firmware too, which per-board skips in the rig config handle. fw = None try: - r = sudo(['setpci', '-s', pci.name, '0x6c.l'], capture_output=True, text=True) + r = _sudo_soft(['setpci', '-s', pci.name, '0x6c.l'], capture_output=True, text=True) if r.returncode == 0: fw = int(r.stdout.strip(), 16) except (OSError, ValueError): @@ -211,7 +354,7 @@ def check_host_compat(dev): def bind_usbtest(dev): """Bind the device's interface 0 to the usbtest driver.""" if not DRIVER.exists(): - r = sudo(['modprobe', 'usbtest']) + r = _sudo_soft(['modprobe', 'usbtest']) if r.returncode != 0 or not DRIVER.exists(): sys.exit(f'cannot load usbtest module: {r.stderr.strip()}') @@ -222,8 +365,8 @@ def bind_usbtest(dev): sysfs_write(DRIVER / 'remove_id', f'{VID} {PID}', check=False) sysfs_write(DRIVER / 'new_id', f'{VID} {PID} 0 {GZ_REF}') if stale_binding: - # bound before the re-registration: that probe captured the OLD dynamic id's capability - # profile; unbind once (device is idle here) so the loop below reprobes the fresh one + # it probed against the OLD dynamic id's capability profile; unbind once (the + # device is idle here) so the loop below reprobes the fresh one sysfs_write(drv / 'unbind', intf, check=False) deadline = time.monotonic() + 3 @@ -248,51 +391,64 @@ def set_pattern(value): 'the "pattern" param, or it is not readable') +def _sudo_soft(cmd, **kw): + """sudo() for calls whose failure must never abort the battery: run() re-raises + TimeoutExpired, and two of these are evaluated inside run_case's own timeout handler + -- a raise there loses the HUNG verdict, the recovery and the JSON report.""" + try: + return sudo(cmd, **kw) + except (OSError, ValueError, subprocess.SubprocessError, SystemExit) as e: + # SystemExit too: sudo() sys.exit()s on 'a password is required', unwinding out of + # run_case's timeout handler before the HUNG verdict is recorded -- which leaves + # unrecovered_hang False and lets the finally run the remove_id/unbind that must + # never happen while a D-state device lock is held + print(f'{cmd[0]}: {type(e).__name__}: {e}', file=sys.stderr) + return subprocess.CompletedProcess(cmd, 1, '', '') + + def dmesg_tail(): - r = sudo(['dmesg']) + r = _sudo_soft(['dmesg']) lines = [l for l in r.stdout.splitlines() if 'usbtest' in l] return '\n'.join(lines[-8:]) -def wedged_pids(devnode): - """Return (pids, complete): PIDs in uninterruptible sleep whose cmdline names devnode, i.e. - still holding its usbfs device lock, and whether every /proc entry could actually be read. - Matched by device node rather than by our child's pid because run_case() may wrap testusb in - sudo, in which case the Popen pid is the wrapper and the blocked process is its child -- - killing the wrapper would make a pid-based check look clean while the real holder is stuck. - complete is False when a PermissionError hid an entry (a hidepid/ProtectProc mount, or the - root-owned child of that same sudo). An entry we could not read might be the holder, so the - caller must treat that as unrecovered rather than as an all-clear.""" +def wedged_pids(devnode): + """(pids, complete): pids still in D state on `devnode` after a recovery reflash. + + Matched by device node rather than by our child's pid because run_case() may wrap + testusb in sudo: the Popen pid is then the wrapper and the blocked process is its + child. A clean flash only proves the probe wrote the MCU, not that the D-state holder + let go -- this is what tells the two apart. + + FAIL CLOSED. `complete` is False when an entry could be HIDDEN from us, and the caller + must then keep treating the hang as unrecovered: the holder is root-owned (run_case + uses `sudo -n` whenever the node is not writable) and a hidepid/ProtectProc mount + hides exactly that entry. Reporting "no holder" from a scan that could not see it + clears unrecovered_hang and lets cleanup run remove_id/unbind against a device whose + usbfs lock is still held -- which deadlocks the bus, not just this board. + + Self-contained: /proc is plain text and this is one pass over it, so importing a + helper to do it would only add a failure mode on the recovery path. + """ stuck, complete = [], True - # hidepid=2 and systemd's ProtectProc=invisible omit other users' processes from iterdir() - # entirely -- no entry at all, so no PermissionError to catch -- and testusb runs under sudo - # whenever the device node is not writable. The scan would then look clean while hiding the - # very holder it exists to find. pid 1 is always root-owned, so being unable to read it means - # enumeration is restricted and no result from this scan can be trusted as complete. + # A restricted /proc hides other users' entries ENTIRELY -- no entry, so no + # PermissionError to catch -- and testusb runs under sudo, so the holder is exactly + # what is hidden. Detect the restriction itself rather than its symptom. if os.geteuid() != 0 and not os.access('/proc/1/cmdline', os.R_OK): complete = False - for entry in Path('/proc').iterdir(): - if not entry.name.isdigit(): - continue + for d in pathlib.Path('/proc').glob('[0-9]*'): try: - cmdline = (entry / 'cmdline').read_bytes() - except PermissionError: - complete = False # cannot rule this pid out - continue - except OSError: - continue # raced with process exit: genuinely gone, not hidden - if devnode.encode() not in cmdline: - continue - try: - stat = (entry / 'stat').read_text() - if stat[stat.rindex(')') + 2] == 'D': # comm may contain ')', so scan from the right - stuck.append(int(entry.name)) + st = (d / 'stat').read_bytes() + if st[st.rindex(b')') + 2:st.rindex(b')') + 3] != b'D': + continue + if devnode.encode() in (d / 'cmdline').read_bytes(): + stuck.append(int(d.name)) except PermissionError: - complete = False - except (OSError, ValueError, IndexError): - continue + complete = False # cannot rule this pid out + except (OSError, ValueError): + continue # raced with exit return stuck, complete @@ -306,17 +462,29 @@ def run_case(num, dev, testusb, quick, timeout): cmd = ['sudo', '-n'] + cmd result = {'num': num, 'name': CASE_NAMES[num], 'params': fs_hs} - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + # NO start_new_session: testusb must stay in OUR process group so the caller's outer + # killpg still reaps it; a sudo-wrapped child is escalated through sudo below instead. + # errors='replace': testusb output is not guaranteed UTF-8, and a strict decode would + # raise out of here and out of main(), printing no JSON at all (battery '0/30'). + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, encoding='utf-8', errors='replace') try: out, _ = p.communicate(timeout=timeout) except subprocess.TimeoutExpired: - p.kill() + # Under sudo we only kill the wrapper; its root-owned testusb keeps the inherited + # stdout pipe, so the reap below times out and the overrun is reported as HUNG. + # Accepted rather than escalated: the rig's udev rules make the device node + # writable, so sudo is the exception, and the harness must never sudo-kill a pid + # it cannot prove is its own. + try: + p.kill() + except OSError: + pass try: out, _ = p.communicate(timeout=5) except subprocess.TimeoutExpired: - # SIGKILL had no effect: the child is in uninterruptible sleep on an - # in-kernel usbfs ioctl (device stopped responding mid-transfer). - # Abandon it — waiting or re-signalling can never succeed. + # SIGKILL had no effect: the child is in uninterruptible sleep on an in-kernel + # usbfs ioctl. Abandon it — waiting or re-signalling can never succeed. result.update(status='HUNG', detail=f'testusb stuck in D state after {timeout}s', dmesg=dmesg_tail()) return result @@ -362,7 +530,18 @@ def main(): p.add_argument('--keep-binding', action='store_true', help='leave usbtest dynamic id registered') p.add_argument('--testusb', default=None, help='path to testusb binary') p.add_argument('--timeout', type=int, default=120, help='per-case timeout in seconds') + p.add_argument('--recover-board', help='board JSON (name + flasher) for the post-hang ' + 'reflash recovery; without it a HUNG case leaves the device wedged') + p.add_argument('--recover-fw', help='firmware path reflashed by the post-hang recovery') + p.add_argument('--outer-timeout', type=int, default=0, + help='the caller\'s total bound on this process; a reflash that cannot ' + 'finish before it is skipped rather than orphaned mid-flash') + p.add_argument('--budget', type=int, default=0, + help='stop starting new cases after this many seconds (0 = no limit). ' + 'Callers that impose their own outer timeout set this to reserve ' + 'the remainder for the post-hang recovery path') args = p.parse_args() + t_start = time.monotonic() sys.stdout.reconfigure(line_buffering=True) # per-case results visible when piped/logged testusb = args.testusb or shutil.which('testusb') or os.path.expanduser('~/testusb') @@ -370,22 +549,32 @@ def main(): sys.exit('testusb binary not found: build kernel tools/usb/testusb.c ' 'and install it, or pass --testusb') - # retry briefly: right after a flash the enumeration may still be settling, and on dual-port - # parts the other port's stale same-serial node takes a moment to drop off (see find_device) + # retry briefly: after a flash the enumeration may still be settling, and a dual-port + # part's stale same-serial node takes a moment to drop off (see find_device) deadline = time.monotonic() + 8 while True: dev = find_device(args.serial) - if dev and 'ambiguous' not in dev: + # find_device is THREE-valued: a device, {'ambiguous': [...]}, or + # {'inconclusive': [...]} when bounded reads could not rule a device out. Screening + # only for 'ambiguous' let the inconclusive marker through as if it were a device, + # and the next statement subscripts dev['tier'] -> KeyError, no JSON on stdout, and + # hil_test reports "usbtest did not run / 0-30" for a merely-unreadable bus. + if dev and not ({'ambiguous', 'inconclusive'} & dev.keys()): break if time.monotonic() > deadline: - if dev: + if dev and 'ambiguous' in dev: sys.exit(f"multiple devices with serial {args.serial}: {', '.join(dev['ambiguous'])} " '— stale enumeration from another port? replug or retry') + if dev and 'inconclusive' in dev: + from helper import hil_util as _hu + sys.exit(f"cannot tell whether {VID}:{PID} is present: bounded sysfs reads " + f"did not answer for {', '.join(dev['inconclusive'])}" + f"{_hu.sysfs_blind_note()}") sys.exit(f'no {VID}:{PID} device' + (f' with serial {args.serial}' if args.serial else '')) time.sleep(0.5) - # tier drives which cases run; a stale/foreign device advertising an out-of-range tier - # must not silently run an empty battery ('0/0 passed' would read as green in CI) + # a stale/foreign device advertising an out-of-range tier must not silently run an + # empty battery ('0/0 passed' would read as green in CI) tier = args.tier or dev['tier'] if not 1 <= tier <= max(TIER_CASES): sys.exit(f"device advertises tier {tier} (bcdDevice ...{tier:02x}); reflash a usbtest build " @@ -404,8 +593,7 @@ def main(): if not args.json: print(info) - # probe the upstream controller before touching the device: an incompatible host - # (MosChip MCS9990, or uPD720201 on pre-2.0.2.6 firmware) exits here, before any bind + # before touching the device: an incompatible host exits here, before any bind check_host_compat(dev) results = [] @@ -414,7 +602,15 @@ def main(): bind_usbtest(dev) set_pattern(0) # tier 1 firmware sources zeros; also required by perf cases 27/28 - for num in cases: + abort_reason = None # set on any early exit; drives the BUDGET back-fill below + for idx, num in enumerate(cases): + # Only a HUNG case aborts the battery; an ordinary case timeout is a FAIL and + # the loop continues, each burning --timeout+5s, so without this the run can + # still be in the case loop when the outer timeout SIGKILLs it before it emits + # JSON. Checked before dispatch: worst overshoot is one case. + if args.budget and time.monotonic() - t_start > args.budget: + abort_reason = f'battery budget {args.budget}s exhausted' + break results.append(run_case(num, dev, testusb, args.quick, args.timeout)) r = results[-1] if not args.json: @@ -422,112 +618,255 @@ def main(): extra += f" {r['mbps']} MB/s" if 'mbps' in r else '' print(f"test {num:2d} {r['name']:22s} {r['status']:6s}{extra}") if r['status'] == 'HUNG': - print(f'aborting battery: kernel-side hang, device wedged mid-transfer.\n' - f'auto-recovering: {USB_RECOVER.name} root-cycle {dev["sysname"]} ' - f'(see .claude/skills/usb-kernel-recover)', file=sys.stderr) - # Cutting VBUS at the root port fails the in-flight URB so the usbfs ioctl returns. - # Must run BEFORE any unbind/remove_id, which would take the device lock the stuck - # ioctl holds and deadlock the bus. + abort_reason = 'battery aborted on a kernel-side hang' + # Reflash, NEVER a root-port cycle: resetting the MCU through the DUT's own + # debug probe fails the in-flight URB at the source, so the ioctl returns, + # the queued kill lands and the cleanup below is lock-safe -- and it reaches + # exactly one board, where a root-port cycle bounces every fixture under the + # port (and could never remove power anyway; see usb-kernel-recover). + # Deliberately not gated on a hub-worker check: our own stuck testusb is + # what drives a hub worker into usb_lock_device(), so a pre-check reads + # wedged by construction. # - # Assume unrecovered until proven otherwise, so that any early exit from this block - # -- an OSError spawning the helper, a KeyboardInterrupt, a sudo prompt killing the - # run -- still reaches the finally cleanup with the flag set, instead of running - # the remove_id/unbind the comments there forbid while a device lock is held. + # Assume unrecovered until proven otherwise, so any early exit from this + # block reaches the finally with the flag set instead of running the + # remove_id/unbind that must not happen while a device lock is held. unrecovered_hang = True - # Pass the serial so the helper refuses a stale busport rather than cutting power - # to whatever else now occupies that path. Popen rather than sudo()/subprocess.run: - # run() would kill() then wait() unbounded on timeout, which never returns if - # uhubctl is itself in D state -- the case the timeout exists for. Merge stderr - # into stdout so the helper's target-identity and action lines are not lost. - # Only pass the serial when we actually have one: an empty third argument reads as - # "no expectation" and would silently disable the helper's stale-busport guard. - cmd = [str(USB_RECOVER), 'root-cycle', dev['sysname']] - if dev['serial']: - cmd.append(dev['serial']) - if os.geteuid() != 0: - cmd = ['sudo', '-n'] + cmd - try: - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - text=True) - except OSError as e: - # helper missing or not executable, or sudo unavailable. unrecovered_hang is - # already True so the finally block still skips the unsafe cleanup -- this only - # replaces a traceback with a message that says what to fix. - print(f'cannot run {USB_RECOVER}: {e}', file=sys.stderr) + print('aborting battery: kernel-side hang, device wedged mid-transfer', + file=sys.stderr) + if not (args.recover_board and args.recover_fw): + print('no --recover-board/--recover-fw: the device stays wedged and ' + 'cleanup is skipped', file=sys.stderr) + break + # The reflash is bounded to RECOVER_FLASH_TIMEOUT and skipped when the + # caller's outer bound cannot contain it: the flasher runs in its own + # session, so an outer killpg mid-flash would ORPHAN it on the probe. Gate + # each step on the time actually LEFT -- reserving for the worst case up + # front skipped recovery for nearly every real hang, since the hang-prone + # cases run late in the tier order. + def _time_left(): + if not args.outer_timeout: + return float('inf') + # what still runs after a step: run_cmd's post-kill reap (10s), + # the settle (5s), the sudo-escalated descendant reap run_case may + # have just paid (up to 7s) and the JSON write + return args.outer_timeout - (time.monotonic() - t_start) - 35 + + if _time_left() < RECOVER_RESET_TIMEOUT: + print('insufficient time before the outer bound for even a bounded ' + 'reset; the device stays wedged and cleanup is skipped', + file=sys.stderr) break - rc = None try: - out, _ = p.communicate(timeout=60) # normal run is ~8s - rc = p.returncode - except subprocess.TimeoutExpired: - p.kill() + board = json.loads(args.recover_board) + bname, fname = board['name'], board['flasher']['name'] + import hil_flash # deferred: stdlib-only unless recovery actually runs + flash_fn = getattr(hil_flash, f'flash_{fname.lower()}') + reset_fn = getattr(hil_flash, f'reset_{fname.lower()}', None) + except Exception as e: # malformed/short json, import failure, unknown flasher + print(f'reflash recovery unavailable ({e})', file=sys.stderr) + break + # DELIVERY must be convoy-safe or the recovery makes things worse: our own + # testusb is D-state on this DUT's node, so a flasher that enumerates by + # OPENING usbfs nodes blocks on it, survives SIGKILL and is abandoned -- + # a SECOND stray, the budget spent, the device still wedged. On 2026-08-12 + # a vid_pid-pinned openocd was the only flasher that still reached its + # probe; JLinkExe's ShowEmuList returned zero. See hil_flash.convoy_safe. + if not hil_flash.convoy_safe(board['flasher']): + print(f'{fname} is not convoy-safe for delivery (it enumerates by ' + f'opening usbfs nodes, and this DUT has a D-state holder on ' + f'its own node): skipping the reflash rather than adding a ' + f'second stray. Pin the roster entry with vid_pid on an ' + f'openocd flasher to enable recovery for this board.', + file=sys.stderr) + break + # RESET FIRST (see recovery_steps). Non-destructive, ~130 ms, and it + # clears the wedge by the same mechanism as the reflash. wedged_pids is the + # arbiter: reset_esptool is a stub that returns rc 0 without resetting + # anything, so an exit code here proves nothing. + steps = recovery_steps(fname, _time_left()) + if reset_fn and any(k == 'reset' for k, _ in steps): + print(f'auto-recovering: resetting {bname} via {fname} probe ' + f'(non-destructive; reflash only if this does not clear it)', + file=sys.stderr) try: - out, _ = p.communicate(timeout=5) - rc = p.returncode - except subprocess.TimeoutExpired: - out = ('root-cycle abandoned after 60s: uhubctl did not die to SIGKILL, so ' - 'it is wedged too and the convoy has spread beyond this device') - if out: - print(out.strip(), file=sys.stderr) - if rc is not None: - time.sleep(5) # let the bus settle and the freed ioctl unwind - # Authoritative either way. A non-zero exit only means the device did not come - # back within the poll (a slow bootloader will do that) -- if nothing still - # holds the lock, the bus is usable and cleanup is safe. Conversely a zero exit - # only proves re-enumeration, not that the D-state holder let go. + with redirect_stdout(sys.stderr): + reset_fn(board, timeout=RECOVER_RESET_TIMEOUT) + except TypeError: + with redirect_stdout(sys.stderr): + reset_fn(board) # older primitives take no bound + except Exception as e: + print(f'probe reset raised: {e}; falling through to the reflash', + file=sys.stderr) + time.sleep(5) # let the freed ioctl unwind stuck, complete = wedged_pids(dev['node']) - if stuck: - print(f'{dev["sysname"]}: pid(s) {stuck} still in D state on ' - f'{dev["node"]} — the device lock was never released', file=sys.stderr) - elif not complete: - print('cannot confirm recovery: /proc is only partly readable, so a ' - 'hidden D-state holder cannot be ruled out', file=sys.stderr) - else: + if complete and not stuck: + print('probe reset cleared the wedge; skipping the reflash ' + '(firmware under test left intact for autopsy)', + file=sys.stderr) unrecovered_hang = False + break + if _time_left() < RECOVER_FLASH_TIMEOUT: + print('reset did not clear it and no budget left for a reflash; ' + 'the device stays wedged', file=sys.stderr) + break + print(f'auto-recovering: reflashing {bname} via ' + f'{fname} (see .claude/skills/usb-kernel-recover). ' + f'Unbudgeted by flash_permit, like the root-cycle it replaced: the ' + f'per-controller semaphores live in hil_test\'s process.', + file=sys.stderr) + # run_cmd bounds the flash; its banners go to stdout, which in --json mode + # carries the result object -- keep them off it. A raising flasher (missing + # serial node, unwritable CWD) must not cost the battery its JSON report. + try: + with redirect_stdout(sys.stderr): + ret = flash_fn(board, args.recover_fw, timeout=RECOVER_FLASH_TIMEOUT) + except Exception as e: + print(f'reflash raised: {e}; the device may still be wedged', file=sys.stderr) + break + if ret.returncode != 0: + # a wedged RP DAP answers nothing and the probe has no reset line; + # POR it via the Rescue DP and retry once, exactly as the normal + # flash path does (no-op for every other board/failure) + out_txt = ret.stdout if isinstance(ret.stdout, str) else '' + # inside the redirect like its siblings (hil_test slices the result + # object from the first '{' on stdout), and only if a POR + retry + # still fits before the outer kill + rescued = False + try: + if _time_left() >= 2 * RECOVER_FLASH_TIMEOUT: + with redirect_stdout(sys.stderr): + rescued = hil_flash.rescue_openocd( + board, out_txt, timeout=RECOVER_FLASH_TIMEOUT) + if rescued: + print('DAP wedged; rescued via Rescue DP, retrying reflash', + file=sys.stderr) + with redirect_stdout(sys.stderr): + ret = flash_fn(board, args.recover_fw, + timeout=RECOVER_FLASH_TIMEOUT) + except Exception as e: + # guarded like the first flash: a raise here would unwind past the + # BUDGET back-fill and the JSON print + print(f'rescue/retry raised: {e}', file=sys.stderr) + if ret.returncode != 0: + print(f'reflash failed (rc {ret.returncode}); the device may still ' + f'be wedged', file=sys.stderr) + # settle even on a non-zero exit: the reset may have landed before the + # flasher failed, and the freed ioctl needs a moment to unwind before + # wedged_pids samples + time.sleep(5) + # Authoritative either way: a clean flash only proves the probe wrote the + # MCU, not that the D-state holder let go. + stuck, complete = wedged_pids(dev['node']) + if stuck: + print(f'{dev["sysname"]}: pid(s) {stuck} still in D state on ' + f'{dev["node"]} — the device lock was never released', file=sys.stderr) + # No hub-worker verdict here: our own testusb still holds the DUT's + # device lock, which is what drives a hub worker into usb_lock_device() + # -- any verdict from here is confounded by construction. + elif not complete: + print('cannot confirm recovery: /proc is only partly readable, so a ' + 'hidden D-state holder cannot be ruled out', file=sys.stderr) + else: + unrecovered_hang = False + break + # re-resolve: a mid-battery re-enumeration changes the devnum and so the node + # path. Match on the concrete serial (not args.serial, which may be None) so + # this can never retarget to another device sharing the VID:PID. + # first=False: the ambiguity guard exists because ONE serial can match two + # sysfs nodes on the dual-port WCH parts, and `dev = live` below makes any + # mistake stick for the rest of the battery -- including wedged_pids() then + # scanning the wrong node and clearing unrecovered_hang on a device it never + # checked. Ambiguous comes back as {'ambiguous': [...]}, handled below. + live = find_device(dev['serial']) + if live and live.get('ambiguous'): + # two nodes now answer to one serial (the dual-port WCH parts do this + # around a re-enumeration). Picking either would file the rest of the + # battery's verdicts under a device we cannot identify, so stop here and + # keep the recovery in play rather than guess. + abort_reason = (f'serial {dev["serial"]} matches more than one device ' + f'({", ".join(live["ambiguous"])}) after case {num}') + unrecovered_hang = True + break + if live and live.get('inconclusive'): + # bounded reads stopped answering, so we cannot say the device left -- + # treat it as the wedge it probably is, which keeps the HUNG reflash and + # the lock-safe cleanup in play + from helper import hil_util as _hu + abort_reason = ('cannot tell whether the device is still present: bounded ' + 'sysfs reads stopped answering' + _hu.sysfs_blind_note()) + unrecovered_hang = True break - # re-resolve: after a mid-battery re-enumeration the devnum (and thus the node - # path) changes; keep testing the live node instead of the stale one. Match on the - # concrete serial (not args.serial, which may be None) so this can never retarget to - # a different device that happens to share the VID:PID. - live = find_device(dev['serial'], first=True) if not live: - results.append({'num': num, 'status': 'FAIL', - 'detail': f'device dropped off the bus after case {num}'}) + # no second entry for `num`: run_case already recorded it, and a duplicate + # inflates the denominator (31/30) and reports a PASSing case as failed + abort_reason = f'device dropped off the bus after case {num}' break dev = live + if abort_reason and all(c in {r['num'] for r in results} for c in cases) \ + and 'dropped off the bus' in abort_reason and results: + # nothing left to back-fill (the drop happened during/after the LAST case), + # so the run would report a clean pass; the case it died on is not a pass + if results[-1].get('status') == 'PASS': + # only a PASS: a real FAIL/NOTRUN verdict names the actual regression + # (errno, dmesg) and must not be overwritten by the drop message + results[-1] = dict(results[-1], status='FAIL', detail=abort_reason) + if abort_reason: + # One BUDGET entry per case never dispatched, on EVERY abort path: a shrunken + # denominator (4/5 instead of 4/30) hides that most of the battery never + # executed and makes a regression in the skipped range read as "not the + # problem". + ran = {r['num'] for r in results} + results += [{'num': n, 'status': 'BUDGET', 'detail': f'not run: {abort_reason}'} + for n in cases if n not in ran] finally: # best-effort cleanup: a sudo/sysfs failure here (sudo() may sys.exit) must not replace # an exception propagating out of the try body with a less useful one try: if unrecovered_hang: - # testusb is still stuck in a usbfs ioctl holding the device lock; remove_id/unbind - # would join the convoy and deadlock the bus (see usb-kernel-recover skill) — leave it be + # testusb still holds the device lock in a usbfs ioctl: remove_id/unbind + # would join the convoy and deadlock the bus (see usb-kernel-recover) print('skipping cleanup after unrecovered hang: ask the operator for a full PVE host ' 'power cycle (a VM reboot is not reliable — hubs latch up across the PCIe reset)', file=sys.stderr) elif not args.keep_binding: sysfs_write(DRIVER / 'remove_id', f'{VID} {PID}', check=False) - # release every claimed interface: other devices sharing the VID:PID (stale example - # firmware on a test rig) may have been grabbed on probe and would otherwise stay + # release every claimed interface: another device sharing the VID:PID + # (stale example firmware) may have been grabbed on probe and would stay # bound to usbtest until re-plugged, hijacking the next test's device for intf in DRIVER.glob('*:*'): sysfs_write(DRIVER / 'unbind', intf.name, check=False) except SystemExit: pass - failed = [r for r in results if r['status'] != 'PASS'] + # BUDGET, not NOTRUN: NOTRUN is taken, for a case the KERNEL gated off (-EOPNOTSUPP, + # see run_case) -- a real result that must stay in `failed` and keep its case number. + # BUDGET keeps the denominator honest without lying about the numerator: naming cases + # that never executed as failures sends a maintainer bisecting one of them. + notrun = [r for r in results if r['status'] == 'BUDGET'] + failed = [r for r in results if r['status'] not in ('PASS', 'BUDGET')] ran = len(results) if args.json: + # `wedged` is the verdict this process ALREADY computed; without it the caller had + # to infer one from 'HUNG' in our stdout, which misses a recovery that ran and + # failed, the inconclusive abort (no case reaches status HUNG), and any battery + # killed before it printed. print(json.dumps({'serial': dev['serial'], 'speed': dev['speed'], 'tier': tier, - 'passed': ran - len(failed), 'failed': len(failed), + 'passed': ran - len(failed) - len(notrun), + 'failed': len(failed), 'notrun': len(notrun), + 'wedged': bool(unrecovered_hang), 'cases': results}, indent=2)) else: - print(f"{ran - len(failed)}/{ran} passed") + print(f"{ran - len(failed) - len(notrun)}/{ran} passed" + + (f", {len(notrun)} not run" if notrun else "")) for r in failed: print(f" FAILED test {r['num']}: {r.get('detail', '')}") if r.get('dmesg'): print(' ' + r['dmesg'].replace('\n', '\n ')) - return len(failed) + # NOTRUN counts toward the exit status even though it is reported separately: a + # standalone run whose cases were all skipped has NOT passed, and returning 0 hands a + # false success to any script driving this directly. + return len(failed) + len(notrun) if __name__ == '__main__': diff --git a/tools/metrics_compare_base.py b/tools/metrics_compare_base.py index 799a96800..844130097 100644 --- a/tools/metrics_compare_base.py +++ b/tools/metrics_compare_base.py @@ -81,7 +81,7 @@ def symlink_deps(main_root, worktree_dir): def ci_first_boards(): """Return the first board (alphabetical) of each arm-gcc CI family.""" - matrix_py = os.path.join(TINYUSB_ROOT, '.github', 'workflows', 'ci_set_matrix.py') + matrix_py = os.path.join(TINYUSB_ROOT, '.github', 'scripts', 'ci_set_matrix.py') if not os.path.isfile(matrix_py): return [] ret = run([sys.executable, matrix_py]) @@ -188,7 +188,7 @@ def main(): args.combined = True ci_boards = ci_first_boards() if not ci_boards: - parser.error('--ci: failed to derive boards from .github/workflows/ci_set_matrix.py') + parser.error('--ci: failed to derive boards from .github/scripts/ci_set_matrix.py') # Append, dedup, preserve order seen = set(args.board) for b in ci_boards: -- cgit v1.3.1 From b925231216eabf277938607ba50f1f4b78c0ce7d Mon Sep 17 00:00:00 2001 From: hathach Date: Mon, 17 Aug 2026 01:02:39 +0700 Subject: bsp(lpc55): run lpcxpresso55s28 as a high-speed device, add it to the ci pool Flip the board to device-highspeed/host-fullspeed, matching lpcxpresso55s69 and the way it is cabled on the test rig, and add it to the rig pool with the unique id read from its flash PFR. This is the first hardware coverage the ip3511 high-speed device path has ever had, and it immediately exposed the clear-stall type-bit bug fixed separately. The port swap also exposed a build gap: family.mk only linked a host controller for port 1, so make host builds on port 0 failed with undefined references - mirror family.cmake and link the OHCI driver there. The board's rhport defaults now come from family.cmake's guarded ones rather than a duplicate copy, so a -D override on the command line wins. --- hw/bsp/lpc55/boards/lpcxpresso55s28/board.cmake | 4 ---- hw/bsp/lpc55/boards/lpcxpresso55s28/board.mk | 6 +++--- hw/bsp/lpc55/family.mk | 2 ++ test/hil/tinyusb.json | 14 ++++++++++++++ 4 files changed, 19 insertions(+), 7 deletions(-) (limited to 'test/hil/tinyusb.json') diff --git a/hw/bsp/lpc55/boards/lpcxpresso55s28/board.cmake b/hw/bsp/lpc55/boards/lpcxpresso55s28/board.cmake index b3d6ec722..d7992eec6 100644 --- a/hw/bsp/lpc55/boards/lpcxpresso55s28/board.cmake +++ b/hw/bsp/lpc55/boards/lpcxpresso55s28/board.cmake @@ -8,10 +8,6 @@ set(JLINK_OPTION "-USB 000727031389") set(PYOCD_TARGET LPC55S28) set(NXPLINK_DEVICE LPC55S28:LPCXpresso55S28) -# device fullspeed, host highspeed -set(RHPORT_DEVICE 0) -set(RHPORT_HOST 1) - function(update_board TARGET) target_compile_definitions(${TARGET} PUBLIC CPU_LPC55S28JBD100 diff --git a/hw/bsp/lpc55/boards/lpcxpresso55s28/board.mk b/hw/bsp/lpc55/boards/lpcxpresso55s28/board.mk index db2e11fd7..aecb5a100 100644 --- a/hw/bsp/lpc55/boards/lpcxpresso55s28/board.mk +++ b/hw/bsp/lpc55/boards/lpcxpresso55s28/board.mk @@ -2,9 +2,9 @@ MCU_VARIANT = LPC55S28 MCU_CORE = LPC55S28 MCU_DRIVER_VARIANT = LPC55S69 -# device fullspeed, host highspeed -RHPORT_DEVICE ?= 0 -RHPORT_HOST ?= 1 +# device highspeed, host fullspeed +RHPORT_DEVICE ?= 1 +RHPORT_HOST ?= 0 CFLAGS += -DCPU_LPC55S28JBD100 diff --git a/hw/bsp/lpc55/family.mk b/hw/bsp/lpc55/family.mk index a9b6f6af1..a640cc793 100644 --- a/hw/bsp/lpc55/family.mk +++ b/hw/bsp/lpc55/family.mk @@ -36,6 +36,8 @@ ifeq ($(RHPORT_HOST), 1) SRC_C += $(TOP)/src/portable/nxp/lpc_ip3516/hcd_lpc_ip3516.c else CFLAGS += -DBOARD_TUH_MAX_SPEED=OPT_MODE_FULL_SPEED + # host on port 0 uses the OHCI controller (mirrors family.cmake) + SRC_C += $(TOP)/src/portable/ohci/ohci.c endif # mcu driver cause following warnings diff --git a/test/hil/tinyusb.json b/test/hil/tinyusb.json index 549a17cd0..6f552f126 100644 --- a/test/hil/tinyusb.json +++ b/test/hil/tinyusb.json @@ -196,6 +196,20 @@ "args": "-device LPC11U37/401" } }, + { + "name": "lpcxpresso55s28", + "uid": "2BF1839A7D51F553A15AB03FD08F70AB", + "tests": { + "device": true, + "host": false, + "dual": false + }, + "flasher": { + "name": "jlink", + "uid": "000727031389", + "args": "-device LPC55S28" + } + }, { "name": "ra4m1_ek", "uid": "152E163038303131393346E46F26574B", -- cgit v1.3.1