From 3963a1b70a572132aced1c1a0033e1c8249a0c7e Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 14 Aug 2026 01:08:40 +0700 Subject: test/hil, ci: contain a wedged USB stack instead of stranding the runner A wedged USB device used to take the whole HIL run with it. Every worker that touched the poisoned node blocked uninterruptibly, the pool could not be joined, map_async discarded every board's result, and the job ran to the GitHub ceiling with no report at all -- while the self-hosted runner's single job slot stayed occupied and every queued job waited behind it. Bound the calls a worker makes itself. read_sysfs, bounded_open and run_cmd all answer within a wall clock; read_sysfs distinguishes "absent" from "unknown", because a blocked read is not evidence of absence, and caps stranded readers at four (each costs a thread and an fd for the life of the process) after which the worker declares itself blind. mtype, the gio unmount, the libmtp session and the arecord/iperf reaps go through those bounds; the MTP session runs in a disposable subprocess, since libmtp's ctypes calls block unkillably in D state. Bound the run. A pool guard (HIL_POOL_TIMEOUT, 60 min) fires before any job ceiling and still writes a report. When the pool will not shut down, the sweep kills what the workers spawned -- descendants, not just direct children, since flashers run in their own session -- confirms each kill actually landed, and exits early so the runner is freed. Whatever survived is named in the report. Deliberately shallow past that point. We do not re-scan process groups, prove pid ownership, or escalate through sudo: a root-owned survivor is reported, not force-killed, because signalling a pid we cannot prove is ours is the worse failure, and the job ceiling backstops whatever this misses. A D-state holder was never killable anyway. Recover instead of reporting a wedge. A HUNG usbtest case reflashes its own DUT through its roster flasher, but only where the flasher can reach its probe past a poisoned node -- openocd pinned to a validated vid_pid, or esptool. Where it cannot, the run says so rather than reserving budget for a path that cannot fire. Raise the CI ceilings above the pool guard so the guard fires first and still writes its report, and pin --retry 1 on every HIL leg: the guard is a flat constant and does not scale with max_retry, so argparse's default of 3 would triple the serialized usbtest tail against an unchanged guard. Split the module: execution in hil_test/hil_flash/usbtest, infrastructure in helper/ (locking, health, selection, shared bounded IO), and the two matrix generators into .github/scripts/ -- ci_set_matrix.py sat in workflows/, where GitHub treats every file as a workflow definition. 193 tests cover the bounded paths, the kill ladder, the guard and the selector against synthetic /proc trees and PATH-injected fakes; a real wedge cannot be manufactured on demand. --- .github/workflows/ci_set_matrix.py | 111 ------------------------------------- 1 file changed, 111 deletions(-) delete mode 100755 .github/workflows/ci_set_matrix.py (limited to '.github/workflows/ci_set_matrix.py') diff --git a/.github/workflows/ci_set_matrix.py b/.github/workflows/ci_set_matrix.py deleted file mode 100755 index 50ada5964..000000000 --- a/.github/workflows/ci_set_matrix.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python3 -import json - -# toolchain, url -toolchain_list = [ - "aarch64-gcc", - "arm-clang", - "arm-iar", - "arm-gcc", - "esp-idf", - "ft9xx-gcc", - "msp430-gcc", - "riscv-gcc", - "rx-gcc" -] - -# family: [supported toolchain] -family_list = { - "apm32f0xx": ["arm-gcc"], - "at32f402_405": ["arm-gcc"], - "at32f403a_407": ["arm-gcc"], - "at32f413": ["arm-gcc"], - "at32f415": ["arm-gcc"], - "at32f423": ["arm-gcc"], - "at32f425": ["arm-gcc"], - "at32f435_437": ["arm-gcc"], - "at32f45x": ["arm-gcc"], - "broadcom_32bit": ["arm-gcc"], - "broadcom_64bit": ["aarch64-gcc"], - "ch32f20x": ["arm-gcc"], - "ch32v10x": ["riscv-gcc"], - "ch32v20x": ["riscv-gcc"], - "ch32v30x": ["riscv-gcc"], - "ch583": ["riscv-gcc"], - "da1469x": ["arm-gcc"], - "fomu": ["riscv-gcc"], - "ft9xx": ["ft9xx-gcc"], - "gd32vf103": ["riscv-gcc"], - "hpmicro": ["riscv-gcc"], - "imxrt": ["arm-gcc", "arm-clang"], - "kinetis_k": ["arm-gcc"], - "kinetis_k32l": ["arm-gcc"], - "kinetis_kl": ["arm-gcc"], - "lpc11": ["arm-gcc", "arm-clang"], - "lpc13": ["arm-gcc", "arm-clang"], - "lpc15": ["arm-gcc", "arm-clang"], - "lpc17": ["arm-gcc", "arm-clang"], - "lpc18": ["arm-gcc", "arm-clang"], - "lpc40": ["arm-gcc", "arm-clang"], - "lpc43": ["arm-gcc", "arm-clang"], - "lpc51": ["arm-gcc", "arm-clang"], - "lpc54": ["arm-gcc", "arm-clang"], - "lpc55": ["arm-gcc", "arm-clang"], - "maxim": ["arm-gcc"], - "mcx": ["arm-gcc"], - "mm32": ["arm-gcc"], - "msp430": ["msp430-gcc"], - "msp432e4": ["arm-gcc"], - "nrf": ["arm-gcc", "arm-clang"], - "nuc100_120": ["arm-gcc"], - "nuc121_125": ["arm-gcc"], - "nuc126": ["arm-gcc"], - "nuc505": ["arm-gcc"], - "ra": ["arm-gcc"], - "rp2040": ["arm-gcc"], - "rw61x": ["arm-gcc"], - "rx": ["rx-gcc"], - "samd11": ["arm-gcc", "arm-clang"], - "samd2x_l2x": ["arm-gcc", "arm-clang"], - "samd5x_e5x": ["arm-gcc", "arm-clang"], - "samg": ["arm-gcc", "arm-clang"], - "stm32c0": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32c5": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32f0": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32f1": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32f2": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32f3": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32f4": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32f7": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32g0": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32g4": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32h5": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32h7": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32h7rs": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32l0": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32l4": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32n6": ["arm-gcc"], - "stm32u0": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32u5": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32wb": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32wba": ["arm-gcc", "arm-clang", "arm-iar"], - "tm4c": ["arm-gcc"], - "xmc4000": ["arm-gcc"], - # S3, P4 will be built by hil test - # "-bespressif_s3_devkitm": ["esp-idf"], - # "-bespressif_p4_function_ev": ["esp-idf"], -} - - -def set_matrix_json(): - matrix = {} - for toolchain in toolchain_list: - filtered_families = [family for family, supported_toolchain in family_list.items() if - toolchain in supported_toolchain] - matrix[toolchain] = filtered_families - - print(json.dumps(matrix)) - - -if __name__ == '__main__': - set_matrix_json() -- cgit v1.3.1