From 3963a1b70a572132aced1c1a0033e1c8249a0c7e Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 14 Aug 2026 01:08:40 +0700 Subject: test/hil, ci: contain a wedged USB stack instead of stranding the runner A wedged USB device used to take the whole HIL run with it. Every worker that touched the poisoned node blocked uninterruptibly, the pool could not be joined, map_async discarded every board's result, and the job ran to the GitHub ceiling with no report at all -- while the self-hosted runner's single job slot stayed occupied and every queued job waited behind it. Bound the calls a worker makes itself. read_sysfs, bounded_open and run_cmd all answer within a wall clock; read_sysfs distinguishes "absent" from "unknown", because a blocked read is not evidence of absence, and caps stranded readers at four (each costs a thread and an fd for the life of the process) after which the worker declares itself blind. mtype, the gio unmount, the libmtp session and the arecord/iperf reaps go through those bounds; the MTP session runs in a disposable subprocess, since libmtp's ctypes calls block unkillably in D state. Bound the run. A pool guard (HIL_POOL_TIMEOUT, 60 min) fires before any job ceiling and still writes a report. When the pool will not shut down, the sweep kills what the workers spawned -- descendants, not just direct children, since flashers run in their own session -- confirms each kill actually landed, and exits early so the runner is freed. Whatever survived is named in the report. Deliberately shallow past that point. We do not re-scan process groups, prove pid ownership, or escalate through sudo: a root-owned survivor is reported, not force-killed, because signalling a pid we cannot prove is ours is the worse failure, and the job ceiling backstops whatever this misses. A D-state holder was never killable anyway. Recover instead of reporting a wedge. A HUNG usbtest case reflashes its own DUT through its roster flasher, but only where the flasher can reach its probe past a poisoned node -- openocd pinned to a validated vid_pid, or esptool. Where it cannot, the run says so rather than reserving budget for a path that cannot fire. Raise the CI ceilings above the pool guard so the guard fires first and still writes its report, and pin --retry 1 on every HIL leg: the guard is a flat constant and does not scale with max_retry, so argparse's default of 3 would triple the serialized usbtest tail against an unchanged guard. Split the module: execution in hil_test/hil_flash/usbtest, infrastructure in helper/ (locking, health, selection, shared bounded IO), and the two matrix generators into .github/scripts/ -- ci_set_matrix.py sat in workflows/, where GitHub treats every file as a workflow definition. 193 tests cover the bounded paths, the kill ladder, the guard and the selector against synthetic /proc trees and PATH-injected fakes; a real wedge cannot be manufactured on demand. --- test/hil/tinyusb.json | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'test/hil/tinyusb.json') diff --git a/test/hil/tinyusb.json b/test/hil/tinyusb.json index c9b38992c..549a17cd0 100644 --- a/test/hil/tinyusb.json +++ b/test/hil/tinyusb.json @@ -149,6 +149,7 @@ "flasher": { "name": "openocd", "uid": "E6614C311B597D32", + "vid_pid": "0x2e8a 0x000c", "args": "-f interface/cmsis-dap.cfg -f target/max32665.cfg", "verify": true } @@ -240,6 +241,7 @@ "flasher": { "name": "openocd", "uid": "E6614103E72C1D2F", + "vid_pid": "0x2e8a 0x000c", "args": "-f interface/cmsis-dap.cfg -f target/rp2040.cfg -c \"adapter speed 5000\"", "verify": true } @@ -270,6 +272,7 @@ "flasher": { "name": "openocd", "uid": "E6633861A3819D38", + "vid_pid": "0x2e8a 0x000c", "args": "-f interface/cmsis-dap.cfg -f target/rp2040.cfg -c \"adapter speed 5000\"", "verify": true }, @@ -296,6 +299,7 @@ "flasher": { "name": "openocd", "uid": "E6633861A3978538", + "vid_pid": "0x2e8a 0x000c", "args": "-f interface/cmsis-dap.cfg -f target/rp2350.cfg -c \"adapter speed 5000\"", "verify": true } @@ -326,6 +330,7 @@ "flasher": { "name": "openocd", "uid": "E663AC91D3359B38", + "vid_pid": "0x2e8a 0x000c", "args": "-f interface/cmsis-dap.cfg -f target/rp2350.cfg -c \"adapter speed 5000\"", "verify": true } @@ -407,9 +412,8 @@ "dual": false }, "flasher": { - "name": "openocd", + "name": "stlink", "uid": "004C00343137510F39383538", - "args": "-f interface/stlink.cfg -f target/stm32h7x.cfg", "verify": true } }, @@ -422,9 +426,8 @@ "dual": false }, "flasher": { - "name": "openocd", + "name": "stlink", "uid": "066FFF495087534867063844", - "args": "-f interface/stlink.cfg -f target/stm32g0x.cfg", "verify": true }, "comment": "32-bit scheme, 2KB USB SRAM" @@ -472,6 +475,7 @@ "flasher": { "name": "openocd", "uid": "A76D8F062C2A", + "vid_pid": "0x1a86 0x8010", "args": "-f target/wch-riscv.cfg", "verify": false } @@ -488,6 +492,7 @@ "flasher": { "name": "openocd", "uid": "BC4954081051", + "vid_pid": "0x1a86 0x8010", "args": "-f target/wch-riscv.cfg", "verify": false } @@ -508,6 +513,7 @@ "flasher": { "name": "openocd", "uid": "BC5DA47360D0", + "vid_pid": "0x1a86 0x8010", "args": "-f target/wch-riscv.cfg", "verify": false } @@ -524,6 +530,7 @@ "flasher": { "name": "openocd", "uid": "57468F06DC03", + "vid_pid": "0x1a86 0x8010", "args": "-f target/wch-riscv.cfg", "verify": false } -- cgit v1.3.1