From 90d48c2a632751c1b38831655e256f1bc1918a47 Mon Sep 17 00:00:00 2001 From: hathach Date: Thu, 9 Jul 2026 23:38:48 +0700 Subject: bsp(ch32): naked fsdev ISRs so nested USBD IRQs return safely The three USBD lines nest under QingKe HWSTK; gcc's interrupt prologue corrupts the return, so rely on the hardware stack and bare mret. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HeF2gZ1M7GWkz6Av4BpKPg --- hw/bsp/ch32v20x/family.c | 38 +++++++++++++++++++------------------- hw/bsp/ch32v30x/family.c | 9 +++++++++ 2 files changed, 28 insertions(+), 19 deletions(-) (limited to 'hw/bsp') diff --git a/hw/bsp/ch32v20x/family.c b/hw/bsp/ch32v20x/family.c index 76024cfde..8875e4faa 100644 --- a/hw/bsp/ch32v20x/family.c +++ b/hw/bsp/ch32v20x/family.c @@ -27,25 +27,25 @@ manufacturer: WCH * - CFG_TUD_WCH_USBIP_USBFS */ -// Port0: USBD (fsdev) -__attribute__((interrupt)) __attribute__((used)) void USB_LP_CAN1_RX0_IRQHandler(void) { - #if CFG_TUD_WCH_USBIP_FSDEV - tud_int_handler(0); - #endif -} - -__attribute__((interrupt)) __attribute__((used)) void USB_HP_CAN1_TX_IRQHandler(void) { - #if CFG_TUD_WCH_USBIP_FSDEV - tud_int_handler(0); - #endif - -} - -__attribute__((interrupt)) __attribute__((used)) void USBWakeUp_IRQHandler(void) { - #if CFG_TUD_WCH_USBIP_FSDEV - tud_int_handler(0); - #endif -} +// Port0: USBD (fsdev). The USBD raises three IRQ lines (LP/HP/WakeUp) that all funnel into the +// non-reentrant tud_int_handler and can nest (HP preempts LP) with QingKe HWSTK enabled. The +// mainline toolchain's plain __attribute__((interrupt)) emits a software prologue that fights the +// hardware context stack and corrupts the return on nesting. Emit naked handlers that rely on +// HWSTK for context save/restore (equivalent to WCH's "WCH-Interrupt-fast"), which nests safely. +#if CFG_TUD_ENABLED && CFG_TUD_WCH_USBIP_FSDEV + // The `call dcd_int_handler` below lives inside naked asm where LTO cannot see it; without a + // compiler-visible reference, -flto builds (make) internalize/drop the symbol and the link fails. + TU_ATTR_USED static void (*const fsdev_isr_keep)(uint8_t) = dcd_int_handler; + #define FSDEV_NAKED_ISR(name) \ + __attribute__((naked)) __attribute__((used)) void name(void) { \ + __asm volatile("li a0, 0\n\t call dcd_int_handler\n\t mret"); } +#else + #define FSDEV_NAKED_ISR(name) \ + __attribute__((naked)) __attribute__((used)) void name(void) { __asm volatile("mret"); } +#endif +FSDEV_NAKED_ISR(USB_LP_CAN1_RX0_IRQHandler) +FSDEV_NAKED_ISR(USB_HP_CAN1_TX_IRQHandler) +FSDEV_NAKED_ISR(USBWakeUp_IRQHandler) // Port1: USBFS __attribute__((interrupt)) __attribute__((used)) void USBHD_IRQHandler(void) { diff --git a/hw/bsp/ch32v30x/family.c b/hw/bsp/ch32v30x/family.c index aee4e7d4f..02c3c7d44 100644 --- a/hw/bsp/ch32v30x/family.c +++ b/hw/bsp/ch32v30x/family.c @@ -29,6 +29,7 @@ */ #include "stdio.h" +#include // https://github.com/openwch/ch32v307/pull/90 // https://github.com/openwch/ch32v20x/pull/12 @@ -166,6 +167,14 @@ uint32_t board_button_read(void) { #endif } +size_t board_get_unique_id(uint8_t id[], size_t max_len) { + volatile uint32_t* ch32_uuid = ((volatile uint32_t*) 0x1FFFF7E8UL); // ESIG unique ID + uint32_t uid[3] = { ch32_uuid[0], ch32_uuid[1], ch32_uuid[2] }; + const size_t len = max_len < sizeof(uid) ? max_len : sizeof(uid); + memcpy(id, uid, len); // byte copy: id[] need not be 4-byte aligned + return len; +} + int board_uart_read(uint8_t* buf, int len) { (void) buf; (void) len; -- cgit v1.3.1 From 24f8bce0bc4a07a69f242ff1e790da90719e984d Mon Sep 17 00:00:00 2001 From: hathach Date: Mon, 13 Jul 2026 15:30:44 +0700 Subject: rusb2: EP0 OUT reliability, HS UTMI PHY power-up, FS-only build support - EP0 OUT: park a back-to-back data-stage packet the DCP accepted before PID could go NAK and deliver it into the next armed chunk; flow-control the single-buffer control pipe between chunks (usbtest ctrl_out corruption); discard a packet parked while an OUT pipe was halted so BOT reset recovery's fresh CBW read can't receive stale WRITE data - HS UTMI PHY power-up per the FSP sequence, shared by dcd/hcd: CLKSEL programmed from the board XTAL (EK-RA8M1 runs 20 MHz; the 24 MHz reset default never locks) while DIRPD holds the PHY down, then timed release - hw/bsp(ra8m1_ek): fix U60CK divider macro - BSP_CFG_U60CK_DIV used the generic USB_CLOCK_DIV_8 encoding (7), which USB60CKDIVCR rejects, leaving the USBHS link domain at 480 MHz; the USB60-specific BSP_CLOCKS_USB60_CLOCK_DIV_8 (4) sticks and yields the required 60 MHz from PLL1P - support FS-only builds on the high-speed port: gate SYSCFG.HSE on TUD_OPT_HIGH_SPEED (RHPORT_DEVICE_SPEED=OPT_MODE_FULL_SPEED was a silent no-op) and always compile both hwfifo access widths - the FIFO width belongs to the module, not the link speed (FS builds corrupted odd-length tails: 16-bit access against MBW-32) - iso activate: reset stale pipe bookkeeping so a BRDY firing before the class re-arms can't replay a pre-SET_INTERFACE transfer; write PIPEBUF after PIPESEL selects the pipe (PIPESEL-windowed register) - clear-halt: re-assert BUF on a still-armed OUT pipe (usbtest case 29) - bound the D0FIFO ready spin so an undrained double-buffered IN pipe can't freeze the stack with the IRQ masked - usbtest example: cap interrupt mps at 64 on RUSB2 high speed (pipes 6-9 have a fixed 64-byte buffer, RA6M5 UM 29.1) Verified: usbtest 30/30 on ra6m5_ek (HS), ra4m1_ek (FS) and ra8m1_ek (FS-forced build on the HS port). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HeF2gZ1M7GWkz6Av4BpKPg --- examples/device/usbtest/src/usb_descriptors.h | 8 ++- hw/bsp/ra/boards/ra8m1_ek/ra_gen/bsp_clock_cfg.h | 5 +- src/portable/renesas/rusb2/dcd_rusb2.c | 90 ++++++++++++++++-------- src/portable/renesas/rusb2/hcd_rusb2.c | 8 +-- src/portable/renesas/rusb2/rusb2_ra.h | 42 +++++++++++ src/tusb_option.h | 2 +- 6 files changed, 116 insertions(+), 39 deletions(-) (limited to 'hw/bsp') diff --git a/examples/device/usbtest/src/usb_descriptors.h b/examples/device/usbtest/src/usb_descriptors.h index 8a033eca9..61b931bd0 100644 --- a/examples/device/usbtest/src/usb_descriptors.h +++ b/examples/device/usbtest/src/usb_descriptors.h @@ -57,7 +57,13 @@ #define USBTEST_INT_EP_MPS_FS 64 #define USBTEST_ISO_EP_MPS_FS 128 #endif -#define USBTEST_INT_EP_MPS_HS 512 +// RUSB2 (Renesas RA) interrupt pipes 6-9 have a fixed 64-byte single buffer at any speed +// (RA6M5 UM R01UH0891 sec 29.1: "Pipes 6 to 9: Interrupt transfer with 64-byte single buffer"). +#if TU_CHECK_MCU(OPT_MCU_RAXXX) + #define USBTEST_INT_EP_MPS_HS 64 +#else + #define USBTEST_INT_EP_MPS_HS 512 +#endif #define USBTEST_ISO_EP_MPS_HS 512 // Compile-time capability maximum: sizes the source buffers / vendor epbufs for the largest diff --git a/hw/bsp/ra/boards/ra8m1_ek/ra_gen/bsp_clock_cfg.h b/hw/bsp/ra/boards/ra8m1_ek/ra_gen/bsp_clock_cfg.h index f2f1ae0c9..25638f02b 100644 --- a/hw/bsp/ra/boards/ra8m1_ek/ra_gen/bsp_clock_cfg.h +++ b/hw/bsp/ra/boards/ra8m1_ek/ra_gen/bsp_clock_cfg.h @@ -51,6 +51,9 @@ #define BSP_CFG_CANFDCLK_DIV (BSP_CLOCKS_CANFD_CLOCK_DIV_8) /* CANFDCLK Div /8 */ #define BSP_CFG_I3CCLK_DIV (BSP_CLOCKS_I3C_CLOCK_DIV_3) /* I3CCLK Div /3 */ #define BSP_CFG_UCK_DIV (BSP_CLOCKS_USB_CLOCK_DIV_5) /* UCK Div /5 */ -#define BSP_CFG_U60CK_DIV (BSP_CLOCKS_USB_CLOCK_DIV_8) /* U60CK Div /8 */ +/* U60CK Div /8: PLL1P 480 MHz -> 60 MHz. Hand-fixed: Smart Configurator emitted the USB_ macro + * namespace (BSP_CLOCKS_USB_CLOCK_DIV_8 = 7, rejected by USB60CKDIVCR -> link clock ran at + * 480 MHz); configuration.xml already says u60ck.div.8, so keep the USB60_ macro if regenerating. */ +#define BSP_CFG_U60CK_DIV (BSP_CLOCKS_USB60_CLOCK_DIV_8) #define BSP_CFG_OCTA_DIV (BSP_CLOCKS_OCTA_CLOCK_DIV_4) /* OCTASPICLK Div /4 */ #endif /* BSP_CLOCK_CFG_H_ */ diff --git a/src/portable/renesas/rusb2/dcd_rusb2.c b/src/portable/renesas/rusb2/dcd_rusb2.c index 5e42f63f5..c93bab05e 100644 --- a/src/portable/renesas/rusb2/dcd_rusb2.c +++ b/src/portable/renesas/rusb2/dcd_rusb2.c @@ -43,6 +43,7 @@ typedef struct static dcd_data_t _dcd; + //--------------------------------------------------------------------+ // INTERNAL OBJECT & FUNCTION DECLARATION //--------------------------------------------------------------------+ @@ -190,6 +191,15 @@ static bool pipe0_xfer_out(rusb2_reg_t *rusb) { pipe_state_t *pipe = &_dcd.pipe[0]; const unsigned rem = pipe->remaining; + // BRDY with no armed transfer: a back-to-back data-stage packet beat the PID=NAK below (the + // host has already ACKed it). Park it in the DCP buffer — an unread buffer NAKs further OUTs — + // and let process_pipe0_xfer deliver it when usbd arms the next chunk. BCLR here would silently + // drop the packet and shift every later chunk by one (usbtest ctrl_out corruption at ra4m1). + if (pipe->buf == NULL && rem == 0) { + rusb->DCPCTR = RUSB2_PIPE_CTR_PID_NAK; + return false; + } + const uint16_t mps = edpt0_max_packet_size(rusb); const uint16_t vld = rusb->CFIFOCTR_b.DTLN; const uint16_t len = tu_min16(tu_min16(rem, mps), vld); @@ -360,7 +370,16 @@ static void process_status_completion(uint8_t rhport) dcd_event_xfer_complete(rhport, ep_addr, 0, XFER_RESULT_SUCCESS, true); } -static bool process_pipe0_xfer(rusb2_reg_t *rusb, int buffer_type, uint8_t ep_addr, void *buffer, +// Report a completed transfer on `num` and reset its bookkeeping. Single completion path for the +// BRDY handler and the EP0 parked-packet drain, so they can't diverge (e.g. on clearing `queued`). +static void pipe_xfer_complete(uint8_t rhport, unsigned num, bool in_isr) { + pipe_state_t *pipe = &_dcd.pipe[num]; + pipe->queued = false; + dcd_event_xfer_complete(rhport, pipe->ep, pipe->length - pipe->remaining, + XFER_RESULT_SUCCESS, in_isr); +} + +static bool process_pipe0_xfer(uint8_t rhport, rusb2_reg_t *rusb, int buffer_type, uint8_t ep_addr, void *buffer, uint16_t total_bytes) { uint16_t fifo_sel = (rusb2_is_highspeed_reg(rusb) ? RUSB2_FIFOSEL_MBW_32BIT : RUSB2_FIFOSEL_MBW_16BIT) | FIFOSEL_BIGEND; @@ -386,6 +405,15 @@ static bool process_pipe0_xfer(rusb2_reg_t *rusb, int buffer_type, uint8_t ep_ad /* IN */ TU_ASSERT(rusb->DCPCTR_b.BSTS && (rusb->USBREQ & 0x80)); pipe0_xfer_in(rusb); + } else if (rusb->CFIFOCTR_b.DTLN > 0) { + /* OUT: a back-to-back packet parked by pipe0_xfer_out already sits in the DCP buffer (its + BRDY has fired and been cleared) — deliver it into this chunk now; no new BRDY will come + for it. Runs with the USB IRQ masked (dcd_edpt_xfer). Detected via the hardware DTLN + rather than a driver flag: the BCLR at SETUP/bus-reset then self-heals any parked state. */ + if (pipe0_xfer_out(rusb)) { + pipe_xfer_complete(rhport, 0, false); + return true; // PID stays NAK (set by pipe0_xfer_out) until the next chunk is armed + } } rusb->DCPCTR = RUSB2_PIPE_CTR_PID_BUF; } else { @@ -460,11 +488,11 @@ static bool process_pipe_xfer(rusb2_reg_t* rusb, int buffer_type, uint8_t ep_add return true; } -static bool process_edpt_xfer(rusb2_reg_t* rusb, int buffer_type, uint8_t ep_addr, void* buffer, uint16_t total_bytes) +static bool process_edpt_xfer(uint8_t rhport, rusb2_reg_t* rusb, int buffer_type, uint8_t ep_addr, void* buffer, uint16_t total_bytes) { const unsigned epn = tu_edpt_number(ep_addr); if (0 == epn) { - return process_pipe0_xfer(rusb, buffer_type, ep_addr, buffer, total_bytes); + return process_pipe0_xfer(rhport, rusb, buffer_type, ep_addr, buffer, total_bytes); } else { return process_pipe_xfer(rusb, buffer_type, ep_addr, buffer, total_bytes); } @@ -508,10 +536,7 @@ static void process_pipe_brdy(uint8_t rhport, unsigned num) } } if (completed) { - pipe->queued = false; - dcd_event_xfer_complete(rhport, pipe->ep, - pipe->length - pipe->remaining, - XFER_RESULT_SUCCESS, true); + pipe_xfer_complete(rhport, num, true); // TU_LOG1("C %d %d\r\n", num, pipe->length - pipe->remaining); } } @@ -636,19 +661,8 @@ bool dcd_init(uint8_t rhport, const tusb_rhport_init_t* rh_init) { #ifdef RUSB2_SUPPORT_HIGHSPEED if ( rusb2_is_highspeed_rhport(rhport) ) { - rusb->SYSCFG_b.HSE = 1; - - // leave CLKSEL as default (0x11) 24Mhz - - // Power and reset UTMI Phy - uint16_t physet = (rusb->PHYSET | RUSB2_PHYSET_PLLRESET_Msk) & ~RUSB2_PHYSET_DIRPD_Msk; - rusb->PHYSET = physet; - R_BSP_SoftwareDelay((uint32_t) 1, BSP_DELAY_UNITS_MILLISECONDS); - rusb->PHYSET_b.PLLRESET = 0; - - // set UTMI to operating mode and wait for PLL lock confirmation - rusb->LPSTS_b.SUSPENDM = 1; - while (!rusb->PLLSTA_b.PLLLOCK) {} + rusb->SYSCFG_b.HSE = TUD_OPT_HIGH_SPEED ? 1 : 0; // FS-only build: no HS chirp + rusb2_utmi_phy_powerup(rusb); rusb->SYSCFG_b.DRPD = 0; rusb->SYSCFG_b.USBE = 1; @@ -753,6 +767,10 @@ bool dcd_edpt_open(uint8_t rhport, tusb_desc_endpoint_t const * ep_desc) if ( !rusb2_is_highspeed_rhport(rhport) && mps > 256) { return false; } + } else if (xfer == TUSB_XFER_INTERRUPT) { + // Interrupt pipes (6-9) have a fixed 64-byte buffer even in high speed (RA6M5 UM 29.1); + // a larger PIPEMAXP would enumerate, then silently truncate every transfer + TU_ASSERT(mps <= 64); } // Re-opening an endpoint must reuse its pipe: usbd_edpt_close() is a no-op on ISO_ALLOC ports, @@ -770,13 +788,12 @@ bool dcd_edpt_open(uint8_t rhport, tusb_desc_endpoint_t const * ep_desc) /* setup pipe */ dcd_int_disable(rhport); + rusb->PIPESEL = num; if ( rusb2_is_highspeed_rhport(rhport) ) { - // FIXME shouldn't be after pipe selection and config, also the BUFNMB should be changed - // depending on the allocation scheme + // PIPEBUF is PIPESEL-windowed (RA6M5 UM 29.2.35): write it after selecting the pipe. + // FIXME BUFNMB is a fixed 0x08 for every pipe; a real per-pipe allocation scheme is needed. rusb->PIPEBUF = 0x7C08; } - - rusb->PIPESEL = num; rusb->PIPEMAXP = mps; volatile uint16_t *ctr = get_pipectr(rusb, num); *ctr = RUSB2_PIPE_CTR_ACLRM_Msk | RUSB2_PIPE_CTR_SQCLR_Msk; @@ -860,14 +877,12 @@ bool dcd_edpt_iso_alloc(uint8_t rhport, uint8_t ep_addr, uint16_t largest_packet _dcd.ep[dir][epn] = num; dcd_int_disable(rhport); + rusb->PIPESEL = (uint16_t) num; if (rusb2_is_highspeed_rhport(rhport)) { - // FIXME (as in dcd_edpt_open): PIPEBUF is a PIPESEL-windowed register (RA6M5 UM §29.2.35) so it - // must be written AFTER PIPESEL selects this pipe, and the fixed BUFNMB=0x08 overlaps every - // HS pipe — a real per-pipe buffer allocator is needed. Left as-is: no RA6M5 HS board on - // the HIL rig to validate a change, and the current mis-ordered write is inert on FS/RA4M1. + // PIPEBUF is PIPESEL-windowed (RA6M5 UM 29.2.35): write it after selecting the pipe. + // FIXME (as in dcd_edpt_open): BUFNMB is a fixed 0x08 for every pipe; a real allocator is needed. rusb->PIPEBUF = 0x7C08; } - rusb->PIPESEL = (uint16_t) num; rusb->PIPEMAXP = largest_packet_size; volatile uint16_t *ctr = get_pipectr(rusb, num); *ctr = RUSB2_PIPE_CTR_ACLRM_Msk | RUSB2_PIPE_CTR_SQCLR_Msk; @@ -893,6 +908,13 @@ bool dcd_edpt_iso_activate(uint8_t rhport, const tusb_desc_endpoint_t *desc_ep) volatile uint16_t *ctr = get_pipectr(rusb, num); *ctr = RUSB2_PIPE_CTR_ACLRM_Msk | RUSB2_PIPE_CTR_SQCLR_Msk; // abort in-flight + reset data toggle *ctr = 0; + // a transfer armed before SET_INTERFACE survives to here (no dcd close on this port): drop the + // stale bookkeeping so a BRDY firing before the class re-arms can't replay it + pipe_state_t *pipe = &_dcd.pipe[num]; + pipe->buf = NULL; + pipe->remaining = 0; + pipe->queued = false; + pipe->zlp_pending = false; *ctr = RUSB2_PIPE_CTR_PID_BUF; // enable dcd_int_enable(rhport); return true; @@ -904,7 +926,7 @@ bool dcd_edpt_xfer(uint8_t rhport, uint8_t ep_addr, uint8_t * buffer, uint16_t t rusb2_reg_t* rusb = RUSB2_REG(rhport); dcd_int_disable(rhport); - bool r = process_edpt_xfer(rusb, 0, ep_addr, buffer, total_bytes); + bool r = process_edpt_xfer(rhport, rusb, 0, ep_addr, buffer, total_bytes); dcd_int_enable(rhport); return r; @@ -917,7 +939,7 @@ bool dcd_edpt_xfer_fifo(uint8_t rhport, uint8_t ep_addr, tu_fifo_t * ff, uint16_ rusb2_reg_t* rusb = RUSB2_REG(rhport); dcd_int_disable(rhport); - bool r = process_edpt_xfer(rusb, 1, ep_addr, ff, total_bytes); + bool r = process_edpt_xfer(rhport, rusb, 1, ep_addr, ff, total_bytes); dcd_int_enable(rhport); return r; @@ -952,6 +974,12 @@ void dcd_edpt_clear_stall(uint8_t rhport, uint8_t ep_addr) } else { const unsigned num = _dcd.ep[0][tu_edpt_number(ep_addr)]; rusb->PIPESEL = (uint16_t)num; + // Drop any packet parked in the buffer while halted: a data-OUT packet the host sent before + // aborting its transfer would otherwise be delivered into the next read after recovery + // (BOT reset + clear-halt re-arms a 31-byte CBW read which then receives stale WRITE data, + // "SCSI CBW is not valid" -> stall -> reset loop; ra6m5 msc write wedge). + *ctr = RUSB2_PIPE_CTR_ACLRM_Msk; + *ctr = 0; // Non-bulk OUT re-enables straight away. Bulk OUT is normally armed together with its transaction // counter (TRE) by process_pipe_xfer(), so we don't blindly re-enable it here — but if a receive // was already armed (still queued), SQCLR above just left it NAKing. Re-assert BUF so it keeps diff --git a/src/portable/renesas/rusb2/hcd_rusb2.c b/src/portable/renesas/rusb2/hcd_rusb2.c index 849551d27..489162e54 100644 --- a/src/portable/renesas/rusb2/hcd_rusb2.c +++ b/src/portable/renesas/rusb2/hcd_rusb2.c @@ -454,11 +454,9 @@ bool hcd_init(uint8_t rhport, const tusb_rhport_init_t* rh_init) { if (rusb2_is_highspeed_rhport(rhport) ) { rusb->SYSCFG_b.HSE = 1; rusb->PHYSET_b.HSEB = 0; - rusb->PHYSET_b.DIRPD = 0; - R_BSP_SoftwareDelay((uint32_t) 1, BSP_DELAY_UNITS_MILLISECONDS); - rusb->PHYSET_b.PLLRESET = 0; - rusb->LPSTS_b.SUSPENDM = 1; - while ( !rusb->PLLSTA_b.PLLLOCK ); + // same PHY reference-clock + power-up requirements as dcd_init: without CLKSEL matching the + // board XTAL the PLL never locks and the wait below would spin forever (e.g. EK-RA8M1, 20 MHz) + rusb2_utmi_phy_powerup(rusb); rusb->SYSCFG_b.DRPD = 1; rusb->SYSCFG_b.DCFM = 1; rusb->SYSCFG_b.DPRPU = 0; diff --git a/src/portable/renesas/rusb2/rusb2_ra.h b/src/portable/renesas/rusb2/rusb2_ra.h index e5945ffe2..0954d0d25 100644 --- a/src/portable/renesas/rusb2/rusb2_ra.h +++ b/src/portable/renesas/rusb2/rusb2_ra.h @@ -49,6 +49,23 @@ typedef struct { #define rusb2_is_highspeed_rhport(_p) (_p == 1) #define rusb2_is_highspeed_reg(_reg) (_reg == RUSB2_REG(1)) + + // UTMI PHY reference clock is the main oscillator: PHYSET.CLKSEL must match the board XTAL + // before the PHY PLL is released (RA6M5 UM R01UH0891 29.2.17: 00=12 MHz, 10=20 MHz, + // 11=24 MHz reset default). EK-RA6M5 runs 24 MHz (default works); EK-RA8M1 runs 20 MHz and + // never locks/chirps on the default. A board with a non-standard USB clocking scheme can + // pre-define RUSB2_PHYSET_CLKSEL_VALUE to override this selection. + #ifndef RUSB2_PHYSET_CLKSEL_VALUE + #if BSP_CFG_XTAL_HZ == 12000000 + #define RUSB2_PHYSET_CLKSEL_VALUE 0u + #elif BSP_CFG_XTAL_HZ == 20000000 + #define RUSB2_PHYSET_CLKSEL_VALUE 2u + #elif BSP_CFG_XTAL_HZ == 24000000 + #define RUSB2_PHYSET_CLKSEL_VALUE 3u + #else + #error "USBHS UTMI PHY: no PHYSET.CLKSEL encoding for this BSP_CFG_XTAL_HZ; define RUSB2_PHYSET_CLKSEL_VALUE" + #endif + #endif #else #define RUSB2_CONTROLLER_COUNT 1 @@ -84,6 +101,31 @@ TU_ATTR_ALWAYS_INLINE static inline void rusb2_int_disable(uint8_t rhport) { TU_ATTR_ALWAYS_INLINE static inline void rusb2_phy_init(void) { } +#ifdef RUSB2_SUPPORT_HIGHSPEED +// UTMI PHY power-up per the FSP reference sequence (r_usb_preg_access.c), shared by dcd_init and +// hcd_init: program CLKSEL to the board XTAL while the PHY is powered down (DIRPD=1), 1 us, +// release DIRPD, 1 ms, release PLLRESET, then wait for PLL lock. Changing CLKSEL as the PHY +// powers up gets mis-sampled (EK-RA8M1, 20 MHz). +static inline void rusb2_utmi_phy_powerup(rusb2_reg_t* rusb) { + uint16_t physet = rusb->PHYSET | RUSB2_PHYSET_DIRPD_Msk; + rusb->PHYSET = physet; + #ifdef RUSB2_PHYSET_CLKSEL_VALUE + physet = (uint16_t) ((physet & ~RUSB2_PHYSET_CLKSEL_Msk) | + (RUSB2_PHYSET_CLKSEL_VALUE << RUSB2_PHYSET_CLKSEL_Pos)); + rusb->PHYSET = physet; + #endif + R_BSP_SoftwareDelay((uint32_t) 1, BSP_DELAY_UNITS_MICROSECONDS); + physet &= (uint16_t) ~RUSB2_PHYSET_DIRPD_Msk; + rusb->PHYSET = physet; + R_BSP_SoftwareDelay((uint32_t) 1, BSP_DELAY_UNITS_MILLISECONDS); + rusb->PHYSET_b.PLLRESET = 0; + + // set UTMI to operating mode and wait for PLL lock confirmation + rusb->LPSTS_b.SUSPENDM = 1; + while (!rusb->PLLSTA_b.PLLLOCK) {} +} +#endif + #ifdef __cplusplus } #endif diff --git a/src/tusb_option.h b/src/tusb_option.h index e19ee1629..65cf747e2 100644 --- a/src/tusb_option.h +++ b/src/tusb_option.h @@ -376,7 +376,7 @@ //------------ RUSB2 --------------// #if defined(TUP_USBIP_RUSB2) #define CFG_TUD_EDPT_DEDICATED_HWFIFO 1 - #define CFG_TUSB_FIFO_HWFIFO_DATA_STRIDE (2 | (TUD_OPT_HIGH_SPEED ? 4 : 0)) // 16 bit and 32 bit if highspeed + #define CFG_TUSB_FIFO_HWFIFO_DATA_STRIDE (2 | 4) // HS module uses 32-bit access at any link speed (e.g. FS-forced build) #define CFG_TUSB_FIFO_HWFIFO_ADDR_STRIDE 0 #define CFG_TUSB_FIFO_HWFIFO_CUSTOM_WRITE // custom write since rusb2 can change access width 32 -> 16 and can write // odd byte with byte access -- cgit v1.3.1 From 83577c42134fc5ef271e5e80655512bcd6cb3e8d Mon Sep 17 00:00:00 2001 From: hathach Date: Wed, 15 Jul 2026 20:13:55 +0700 Subject: bsp/stm32h5: add missing IAR linker script for stm32h533 stm32h533nucleo could never link with IAR: family.cmake points LD_FILE_IAR at linker/stm32h533xx_flash.icf, which did not exist (every sibling H5 variant has one). Surfaced by CircleCI's one-random job picking stm32h533nucleo+IAR (Fatal error[Lc002]). H533 and H523 have identical memory maps (512K flash / 272K RAM; their GCC .ld files differ only in a comment), so the icf is a copy of the H523 one. --- hw/bsp/stm32h5/linker/stm32h533xx_flash.icf | 32 +++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 hw/bsp/stm32h5/linker/stm32h533xx_flash.icf (limited to 'hw/bsp') diff --git a/hw/bsp/stm32h5/linker/stm32h533xx_flash.icf b/hw/bsp/stm32h5/linker/stm32h533xx_flash.icf new file mode 100644 index 000000000..dc97788ae --- /dev/null +++ b/hw/bsp/stm32h5/linker/stm32h533xx_flash.icf @@ -0,0 +1,32 @@ +/*###ICF### Section handled by ICF editor, don't touch! ****/ +/*-Editor annotation file-*/ +/* IcfEditorFile="$TOOLKIT_DIR$\config\ide\IcfEditor\cortex_v1_0.xml" */ +/*-Specials-*/ +define symbol __ICFEDIT_intvec_start__ = 0x08000000; +/*-Memory Regions-*/ +define symbol __ICFEDIT_region_ROM_start__ = 0x08000000; +define symbol __ICFEDIT_region_ROM_end__ = 0x0807FFFF; +define symbol __ICFEDIT_region_RAM_start__ = 0x20000000; +define symbol __ICFEDIT_region_RAM_end__ = 0x20043FFF; + +/*-Sizes-*/ +define symbol __ICFEDIT_size_cstack__ = 0x1000; +define symbol __ICFEDIT_size_heap__ = 0x200; +/**** End of ICF editor section. ###ICF###*/ + + +define memory mem with size = 4G; +define region ROM_region = mem:[from __ICFEDIT_region_ROM_start__ to __ICFEDIT_region_ROM_end__]; +define region RAM_region = mem:[from __ICFEDIT_region_RAM_start__ to __ICFEDIT_region_RAM_end__]; + +define block CSTACK with alignment = 8, size = __ICFEDIT_size_cstack__ { }; +define block HEAP with alignment = 8, size = __ICFEDIT_size_heap__ { }; + +initialize by copy { readwrite }; +do not initialize { section .noinit }; + +place at address mem:__ICFEDIT_intvec_start__ { readonly section .intvec }; + +place in ROM_region { readonly }; +place in RAM_region { readwrite, + block CSTACK, block HEAP }; -- cgit v1.3.1 From 6173d87ef13db5af42889700e5f3885b5cbc6985 Mon Sep 17 00:00:00 2001 From: hathach Date: Thu, 16 Jul 2026 14:11:24 +0700 Subject: lpc15, lpc40: board_get_unique_id via IAP ReadUID MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Real 128-bit chip UID as the board serial (IAP cmd 58, status checked against IAP_CMD_SUCCESS), replacing the shared placeholder — required for HIL board identification by serial. lpc40's lpcopen Chip_IAP_ReadUID() returns only the first UID word, hence the direct iap_entry() call. Verified on ea4088_quickstart and lpcxpresso1549: both enumerate with their chip UID and are selected by it in the HIL configs. --- hw/bsp/lpc15/family.c | 13 +++++++++++++ hw/bsp/lpc40/family.c | 13 +++++++++++++ 2 files changed, 26 insertions(+) (limited to 'hw/bsp') diff --git a/hw/bsp/lpc15/family.c b/hw/bsp/lpc15/family.c index bbfee1b51..5178ad68b 100644 --- a/hw/bsp/lpc15/family.c +++ b/hw/bsp/lpc15/family.c @@ -122,6 +122,19 @@ uint32_t board_button_read(void) return Chip_GPIO_GetPinState(LPC_GPIO, BUTTON_PORT, BUTTON_PIN) ? 0 : 1; } +size_t board_get_unique_id(uint8_t id[], size_t max_len) +{ + // IAP ReadUID (cmd 58) returns status + 4 words = full 128-bit UID + unsigned int command[5] = { IAP_READ_UID_CMD, 0, 0, 0, 0 }; + unsigned int result[5]; + iap_entry(command, result); + TU_ASSERT(result[0] == IAP_CMD_SUCCESS, 0); + + size_t const len = tu_min32(max_len, 16); + memcpy(id, &result[1], len); + return len; +} + int board_uart_read(uint8_t* buf, int len) { (void) buf; (void) len; diff --git a/hw/bsp/lpc40/family.c b/hw/bsp/lpc40/family.c index d8a63576b..750bd2660 100644 --- a/hw/bsp/lpc40/family.c +++ b/hw/bsp/lpc40/family.c @@ -135,6 +135,19 @@ uint32_t board_button_read(void) { return BUTTON_ACTIV_STATE == Chip_GPIO_GetPinState(LPC_GPIO, BUTTON_PORT, BUTTON_PIN); } +size_t board_get_unique_id(uint8_t id[], size_t max_len) { + // IAP ReadUID (cmd 58) returns status + 4 words = full 128-bit UID + // (lpcopen's Chip_IAP_ReadUID() only returns the first word) + unsigned int command[5] = { IAP_READ_UID_CMD, 0, 0, 0, 0 }; + unsigned int result[5]; + iap_entry(command, result); + TU_ASSERT(result[0] == IAP_CMD_SUCCESS, 0); + + size_t const len = tu_min32(max_len, 16); + memcpy(id, &result[1], len); + return len; +} + int board_uart_read(uint8_t *buf, int len) { //return UART_ReceiveByte(BOARD_UART_PORT); (void) buf; -- cgit v1.3.1 From fa1fee0a5f82b5a78ace26ee1722c1d636c5b32a Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 10 Jul 2026 00:17:18 +0700 Subject: migrate NXP Kinetis khci to chipidea ci_fs driver (device + host) Complete the khci -> chipidea ci_fs migration that was started for device (commit d70403f1f "host is not yet"): - device: switch kinetis_k/kl/k32l (Makefiles + k32l CMake) to dcd_ci_fs.c - host: add hcd_ci_fs.c (port of hcd_khci.c onto ci_fs_regs_t) and switch all Kinetis families to it; remove src/portable/nxp/khci entirely - enable host examples (device_info, cdc_msc_hid) for mcu:KINETIS_K - README: merge the KL and K32L2 rows into a single "KL, K32L" ci_fs row hcd_ci_fs.c also fixes two pre-existing host bugs found via HIL on frdm_k64f (present in the old hcd_khci.c too): - data toggle was flipped on a NAK in suspend_transfer; a NAK transfers no data so the toggle must be preserved, else the retried bulk packet is silently discarded by the device (MSC CBW/CSW hang). See comment in file. - prepare_packets asserted and dropped a transfer when the single shared BDT was still owned by an in-flight transfer under concurrent activity; now it returns busy and resume_transfer defers/retries on the next SOF. HIL verified on frdm_k64f: device 13/13, host cdc_msc_hid (CDC mount + echo + MSC mount, through a hub). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ExGPLP5eU43LR7o6yYLpNi --- README.rst | 4 +- examples/host/cdc_msc_hid/only.txt | 1 + examples/host/device_info/only.txt | 1 + hw/bsp/kinetis_k/family.cmake | 2 +- hw/bsp/kinetis_k/family.mk | 4 +- hw/bsp/kinetis_k32l/family.cmake | 4 +- hw/bsp/kinetis_k32l/family.mk | 4 +- hw/bsp/kinetis_kl/family.cmake | 2 +- hw/bsp/kinetis_kl/family.mk | 4 +- src/portable/chipidea/ci_fs/hcd_ci_fs.c | 649 ++++++++++++++++++++++++++++++++ src/portable/nxp/khci/dcd_khci.c | 560 --------------------------- src/portable/nxp/khci/hcd_khci.c | 628 ------------------------------ 12 files changed, 662 insertions(+), 1201 deletions(-) create mode 100644 src/portable/chipidea/ci_fs/hcd_ci_fs.c delete mode 100644 src/portable/nxp/khci/dcd_khci.c delete mode 100644 src/portable/nxp/khci/hcd_khci.c (limited to 'hw/bsp') diff --git a/README.rst b/README.rst index e5806bafc..205f3f544 100644 --- a/README.rst +++ b/README.rst @@ -239,9 +239,7 @@ Supported CPUs +--------------+---------+-------------------+--------+------+-----------+------------------------+--------------------+ | NXP | iMXRT | RT 10xx, 11xx | ✅ | ✅ | ✅ | ci_hs, ehci | | | +---------+-------------------+--------+------+-----------+------------------------+--------------------+ -| | Kinetis | KL | ✅ | 🟡 | ❌ | ci_fs, khci | | -| | +-------------------+--------+------+-----------+------------------------+--------------------+ -| | | K32L2 | ✅ | | ❌ | khci | ci_fs variant | +| | Kinetis | KL, K32L | ✅ | 🟡 | ❌ | ci_fs | | | +---------+-------------------+--------+------+-----------+------------------------+--------------------+ | | LPC | 11u, 13, 15 | ✅ | ❌ | ❌ | lpc_ip3511 | | | | +-------------------+--------+------+-----------+------------------------+--------------------+ diff --git a/examples/host/cdc_msc_hid/only.txt b/examples/host/cdc_msc_hid/only.txt index a2ff93be5..a2f4f273a 100644 --- a/examples/host/cdc_msc_hid/only.txt +++ b/examples/host/cdc_msc_hid/only.txt @@ -3,6 +3,7 @@ family:samd21 family:samd5x_e5x mcu:CH32V20X mcu:KINETIS_KL +mcu:KINETIS_K mcu:LPC175X_6X mcu:LPC177X_8X mcu:LPC18XX diff --git a/examples/host/device_info/only.txt b/examples/host/device_info/only.txt index 4c2cb0f35..7f30218df 100644 --- a/examples/host/device_info/only.txt +++ b/examples/host/device_info/only.txt @@ -4,6 +4,7 @@ family:samd21 family:samd5x_e5x mcu:CH32V20X mcu:KINETIS_KL +mcu:KINETIS_K mcu:LPC175X_6X mcu:LPC177X_8X mcu:LPC18XX diff --git a/hw/bsp/kinetis_k/family.cmake b/hw/bsp/kinetis_k/family.cmake index e1b5c221e..e408c0f4a 100644 --- a/hw/bsp/kinetis_k/family.cmake +++ b/hw/bsp/kinetis_k/family.cmake @@ -63,7 +63,7 @@ function(family_configure_example TARGET RTOS) ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/family.c ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/../board.c ${TOP}/src/portable/chipidea/ci_fs/dcd_ci_fs.c - ${TOP}/src/portable/nxp/khci/hcd_khci.c + ${TOP}/src/portable/chipidea/ci_fs/hcd_ci_fs.c ${STARTUP_FILE_${CMAKE_C_COMPILER_ID}} ) target_include_directories(${TARGET} PUBLIC diff --git a/hw/bsp/kinetis_k/family.mk b/hw/bsp/kinetis_k/family.mk index b1e1fb3aa..5d0e4a702 100644 --- a/hw/bsp/kinetis_k/family.mk +++ b/hw/bsp/kinetis_k/family.mk @@ -17,8 +17,8 @@ LDFLAGS += \ --specs=nosys.specs --specs=nano.specs \ SRC_C += \ - src/portable/nxp/khci/dcd_khci.c \ - src/portable/nxp/khci/hcd_khci.c \ + src/portable/chipidea/ci_fs/dcd_ci_fs.c \ + src/portable/chipidea/ci_fs/hcd_ci_fs.c \ $(MCU_DIR)/system_${MCU_VARIANT}.c \ $(MCU_DIR)/drivers/fsl_clock.c \ $(SDK_DIR)/drivers/gpio/fsl_gpio.c \ diff --git a/hw/bsp/kinetis_k32l/family.cmake b/hw/bsp/kinetis_k32l/family.cmake index 020695589..950682363 100644 --- a/hw/bsp/kinetis_k32l/family.cmake +++ b/hw/bsp/kinetis_k32l/family.cmake @@ -64,8 +64,8 @@ function(family_configure_example TARGET RTOS) target_sources(${TARGET} PUBLIC ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/family.c ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/../board.c - ${TOP}/src/portable/nxp/khci/dcd_khci.c - ${TOP}/src/portable/nxp/khci/hcd_khci.c + ${TOP}/src/portable/chipidea/ci_fs/dcd_ci_fs.c + ${TOP}/src/portable/chipidea/ci_fs/hcd_ci_fs.c ${STARTUP_FILE_${CMAKE_C_COMPILER_ID}} ) target_include_directories(${TARGET} PUBLIC diff --git a/hw/bsp/kinetis_k32l/family.mk b/hw/bsp/kinetis_k32l/family.mk index a99fb5dbe..357128aa5 100644 --- a/hw/bsp/kinetis_k32l/family.mk +++ b/hw/bsp/kinetis_k32l/family.mk @@ -13,8 +13,8 @@ LDFLAGS += \ -specs=nosys.specs -specs=nano.specs SRC_C += \ - src/portable/nxp/khci/dcd_khci.c \ - src/portable/nxp/khci/hcd_khci.c \ + src/portable/chipidea/ci_fs/dcd_ci_fs.c \ + src/portable/chipidea/ci_fs/hcd_ci_fs.c \ $(MCUX_DEVICES)/K32L/$(MCU_VARIANT)/system_$(MCU_VARIANT).c \ $(MCUX_DEVICES)/K32L/$(MCU_VARIANT)/drivers/fsl_clock.c \ $(MCUX_CORE)/drivers/gpio/fsl_gpio.c \ diff --git a/hw/bsp/kinetis_kl/family.cmake b/hw/bsp/kinetis_kl/family.cmake index 230a3057d..b74f4f8b9 100644 --- a/hw/bsp/kinetis_kl/family.cmake +++ b/hw/bsp/kinetis_kl/family.cmake @@ -62,7 +62,7 @@ function(family_configure_example TARGET RTOS) ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/family.c ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/../board.c ${TOP}/src/portable/chipidea/ci_fs/dcd_ci_fs.c - ${TOP}/src/portable/nxp/khci/hcd_khci.c + ${TOP}/src/portable/chipidea/ci_fs/hcd_ci_fs.c ${STARTUP_FILE_${CMAKE_C_COMPILER_ID}} ) target_include_directories(${TARGET} PUBLIC diff --git a/hw/bsp/kinetis_kl/family.mk b/hw/bsp/kinetis_kl/family.mk index 201ab99dc..9c780a868 100644 --- a/hw/bsp/kinetis_kl/family.mk +++ b/hw/bsp/kinetis_kl/family.mk @@ -17,8 +17,8 @@ LDFLAGS += \ -specs=nosys.specs -specs=nano.specs \ SRC_C += \ - src/portable/nxp/khci/dcd_khci.c \ - src/portable/nxp/khci/hcd_khci.c \ + src/portable/chipidea/ci_fs/dcd_ci_fs.c \ + src/portable/chipidea/ci_fs/hcd_ci_fs.c \ $(MCU_DIR)/system_$(MCU).c \ $(MCU_DIR)/drivers/fsl_clock.c \ $(SDK_DIR)/drivers/gpio/fsl_gpio.c \ diff --git a/src/portable/chipidea/ci_fs/hcd_ci_fs.c b/src/portable/chipidea/ci_fs/hcd_ci_fs.c new file mode 100644 index 000000000..44a68a8d6 --- /dev/null +++ b/src/portable/chipidea/ci_fs/hcd_ci_fs.c @@ -0,0 +1,649 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2021 Koji Kitayama + * SPDX-FileCopyrightText: Copyright (c) 2021 Ha Thach (tinyusb.org) + * SPDX-License-Identifier: MIT + * + * This file is part of the TinyUSB stack. + */ + +#include "tusb_option.h" + +#if CFG_TUH_ENABLED && defined(TUP_USBIP_CHIPIDEA_FS) + +#include "host/hcd.h" +#include "host/usbh.h" +#include "ci_fs_type.h" + +// Host is currently only available on NXP Kinetis. The ChipIdea-FS host controller +// interface is register-compatible via ci_fs_regs_t. Unlike the device driver, the host +// driver does not include the ci_fs_.h header because those define the device +// dcd_int_enable()/dcd_int_disable() functions, which would collide in a dual-role build. +#if defined(TUP_USBIP_CHIPIDEA_FS_KINETIS) + #include "fsl_device_registers.h" + #define CI_FS_REG(_port) ((ci_fs_regs_t*) USB0_BASE) + #define CI_FS_IRQN USB0_IRQn +#else + #error "MCU is not supported" +#endif + +#define CI_REG CI_FS_REG(0) + +//--------------------------------------------------------------------+ +// MACRO TYPEDEF CONSTANT ENUM DECLARATION +//--------------------------------------------------------------------+ + +enum { + TOK_PID_OUT = 0x1u, + TOK_PID_IN = 0x9u, + TOK_PID_SETUP = 0xDu, + TOK_PID_DATA0 = 0x3u, + TOK_PID_DATA1 = 0xbu, + TOK_PID_ACK = 0x2u, + TOK_PID_STALL = 0xeu, + TOK_PID_NAK = 0xau, + TOK_PID_BUSTO = 0x0u, + TOK_PID_ERR = 0xfu, +}; + +typedef struct TU_ATTR_PACKED +{ + union { + uint32_t head; + struct { + union { + struct { + uint16_t : 2; + __IO uint16_t tok_pid : 4; + uint16_t data : 1; + __IO uint16_t own : 1; + uint16_t : 8; + }; + struct { + uint16_t : 2; + uint16_t bdt_stall : 1; + uint16_t dts : 1; + uint16_t ninc : 1; + uint16_t keep : 1; + uint16_t : 10; + }; + }; + __IO uint16_t bc : 10; + uint16_t : 6; + }; + }; + uint8_t *addr; +}buffer_descriptor_t; + +TU_VERIFY_STATIC( sizeof(buffer_descriptor_t) == 8, "size is not correct" ); + +typedef struct TU_ATTR_PACKED +{ + union { + uint32_t state; + struct { + uint32_t pipenum:16; + uint32_t odd : 1; + uint32_t : 0; + }; + }; + uint8_t *buffer; + uint16_t length; + uint16_t remaining; +} endpoint_state_t; + +typedef struct TU_ATTR_PACKED +{ + uint8_t dev_addr; + uint8_t ep_addr; + uint16_t max_packet_size; + union { + uint8_t flags; + struct { + uint8_t data : 1; + uint8_t xfer : 2; + uint8_t : 0; + }; + }; + uint8_t *buffer; + uint16_t length; + uint16_t remaining; +} pipe_state_t; + + +typedef struct +{ + union { + /* [OUT,IN][EVEN,ODD] */ + buffer_descriptor_t bdt[2][2]; + uint16_t bda[2*2]; + }; + endpoint_state_t endpoint[2]; + pipe_state_t pipe[CFG_TUH_ENDPOINT_MAX * 2]; + uint32_t in_progress; /* Bitmap. Each bit indicates that a transfer of the corresponding pipe is in progress */ + uint32_t pending; /* Bitmap. Each bit indicates that a transfer of the corresponding pipe will be resume the next frame */ + bool need_reset; /* The device has not been reset after connection. */ +} hcd_data_t; + +//--------------------------------------------------------------------+ +// INTERNAL OBJECT & FUNCTION DECLARATION +//--------------------------------------------------------------------+ +// BDT(Buffer Descriptor Table) must be 256-byte aligned +CFG_TUH_MEM_SECTION TU_ATTR_ALIGNED(512) static hcd_data_t _hcd; +//CFG_TUH_MEM_SECTION TU_ATTR_ALIGNED(4) static uint8_t _rx_buf[1024]; + +static int find_pipe(uint8_t dev_addr, uint8_t ep_addr) +{ + /* Find the target pipe */ + int num; + for (num = 0; num < CFG_TUH_ENDPOINT_MAX * 2; ++num) { + pipe_state_t *p = &_hcd.pipe[num]; + if ((p->dev_addr == dev_addr) && (p->ep_addr == ep_addr)) + return num; + } + return -1; +} + +static int prepare_packets(int pipenum) +{ + pipe_state_t *pipe = &_hcd.pipe[pipenum]; + unsigned const dir_tx = tu_edpt_dir(pipe->ep_addr) ? 0 : 1; + endpoint_state_t *ep = &_hcd.endpoint[dir_tx]; + unsigned const odd = ep->odd; + buffer_descriptor_t *bd = _hcd.bdt[dir_tx]; + // The host shares a single BDT set across all pipes. If it is still owned by an + // in-flight transfer on another pipe, report busy so the caller can defer & retry. + if (bd[odd].own) return -1; + + // TU_LOG1(" %p dir %d odd %d data %d\r\n", &bd[odd], dir_tx, odd, pipe->data); + + ep->pipenum = pipenum; + + bd[odd ].data = pipe->data; + bd[odd ^ 1].data = pipe->data ^ 1; + bd[odd ^ 1].own = 0; + /* reset values for a next transfer */ + + int num_tokens = 0; /* The number of prepared packets */ + unsigned const mps = pipe->max_packet_size; + unsigned const rem = pipe->remaining; + if (rem > mps) { + /* When total_bytes is greater than the max packet size, + * it prepares to the next transfer to avoid NAK in advance. */ + bd[odd ^ 1].bc = rem >= 2 * mps ? mps: rem - mps; + bd[odd ^ 1].addr = pipe->buffer + mps; + bd[odd ^ 1].own = 1; + if (dir_tx) ++num_tokens; + } + bd[odd].bc = rem >= mps ? mps: rem; + bd[odd].addr = pipe->buffer; + __DSB(); + bd[odd].own = 1; /* This bit must be set last */ + ++num_tokens; + return num_tokens; +} + +static int select_next_pipenum(int pipenum) +{ + unsigned wip = _hcd.in_progress & ~_hcd.pending; + if (!wip) return -1; + unsigned msk = TU_GENMASK(31, pipenum); + int next = __builtin_ctz(wip & msk); + if (next) return next; + msk = TU_GENMASK(pipenum, 0); + next = __builtin_ctz(wip & msk); + return next; +} + +/* When transfer is completed, return true. */ +static bool continue_transfer(int pipenum, buffer_descriptor_t *bd) +{ + pipe_state_t *pipe = &_hcd.pipe[pipenum]; + unsigned const bc = bd->bc; + unsigned const rem = pipe->remaining - bc; + + pipe->remaining = rem; + if (rem && bc == pipe->max_packet_size) { + int const next_rem = rem - pipe->max_packet_size; + if (next_rem > 0) { + /* Prepare to the after next transfer */ + bd->addr += pipe->max_packet_size * 2; + bd->bc = next_rem > pipe->max_packet_size ? pipe->max_packet_size: next_rem; + __DSB(); + bd->own = 1; /* This bit must be set last */ + while (CI_REG->CTL & USB_CTL_TXSUSPENDTOKENBUSY_MASK) ; + CI_REG->TOKEN = CI_REG->TOKEN; /* Queue the same token as the last */ + } else if (TUSB_DIR_IN == tu_edpt_dir(pipe->ep_addr)) { /* IN */ + while (CI_REG->CTL & USB_CTL_TXSUSPENDTOKENBUSY_MASK) ; + CI_REG->TOKEN = CI_REG->TOKEN; + } + return true; + } + pipe->data = bd->data ^ 1; + return false; +} + +static bool resume_transfer(int pipenum) +{ + int num_tokens = prepare_packets(pipenum); + if (num_tokens < 0) { + // Shared BDT still owned by an in-flight transfer on another pipe. Defer this + // pipe and retry on the next SOF once the BDT is free (avoids dropping the + // transfer, which stalls e.g. a 2nd device enumerating behind a hub while the + // app issues concurrent control transfers). + _hcd.pending |= TU_BIT(pipenum); + CI_REG->INT_EN |= USB_ISTAT_SOFTOK_MASK; + return true; + } + + const unsigned ie = NVIC_GetEnableIRQ(CI_FS_IRQN); + NVIC_DisableIRQ(CI_FS_IRQN); + pipe_state_t *pipe = &_hcd.pipe[pipenum]; + + unsigned flags = CI_REG->EP[0].CTL & USB_ENDPT_HOSTWOHUB_MASK; + flags |= USB_ENDPT_EPRXEN_MASK | USB_ENDPT_EPTXEN_MASK; + switch (pipe->xfer) { + case TUSB_XFER_CONTROL: + flags |= USB_ENDPT_EPHSHK_MASK; + break; + case TUSB_XFER_ISOCHRONOUS: + flags |= USB_ENDPT_EPCTLDIS_MASK | USB_ENDPT_RETRYDIS_MASK; + break; + default: + flags |= USB_ENDPT_EPHSHK_MASK | USB_ENDPT_EPCTLDIS_MASK | USB_ENDPT_RETRYDIS_MASK; + break; + } + // TU_LOG1(" resume pipenum %d flags %x\r\n", pipenum, flags); + + CI_REG->EP[0].CTL = flags; + CI_REG->ADDR = (CI_REG->ADDR & USB_ADDR_LSEN_MASK) | pipe->dev_addr; + + unsigned const token = tu_edpt_number(pipe->ep_addr) | + ((tu_edpt_dir(pipe->ep_addr) ? TOK_PID_IN: TOK_PID_OUT) << USB_TOKEN_TOKENPID_SHIFT); + do { + while (CI_REG->CTL & USB_CTL_TXSUSPENDTOKENBUSY_MASK) ; + CI_REG->TOKEN = token; + } while (--num_tokens); + if (ie) NVIC_EnableIRQ(CI_FS_IRQN); + return true; +} + +static void suspend_transfer(int pipenum, buffer_descriptor_t *bd) +{ + pipe_state_t *pipe = &_hcd.pipe[pipenum]; + pipe->buffer = bd->addr; + // A NAK transfers no data, so the data toggle must be preserved for the retry. + // (Do NOT flip pipe->data here: flipping it makes the retried packet use the wrong + // DATA0/DATA1, which the device silently discards - breaking any bulk/interrupt + // transfer that is NAKed, e.g. the MSC CBW/CSW when the device is momentarily busy.) + if ((TUSB_XFER_INTERRUPT == pipe->xfer) || + (TUSB_XFER_BULK == pipe->xfer)) { + _hcd.pending |= TU_BIT(pipenum); + CI_REG->INT_EN |= USB_ISTAT_SOFTOK_MASK; + } +} + +static void process_tokdne(uint8_t rhport) +{ + (void)rhport; + const unsigned s = CI_REG->STAT; + CI_REG->INT_STAT = USB_ISTAT_TOKDNE_MASK; /* fetch the next token if received */ + uint8_t const dir_in = (s & USB_STAT_TX_MASK) ? TUSB_DIR_OUT: TUSB_DIR_IN; + unsigned const odd = (s & USB_STAT_ODD_MASK) ? 1 : 0; + + buffer_descriptor_t *bd = (buffer_descriptor_t *)&_hcd.bda[s]; + endpoint_state_t *ep = &_hcd.endpoint[s >> 3]; + + /* fetch status before discarded by the next steps */ + const unsigned pid = bd->tok_pid; + + /* reset values for a next transfer */ + bd->bdt_stall = 0; + bd->dts = 1; + bd->ninc = 0; + bd->keep = 0; + /* Update the odd variable to prepare for the next transfer */ + ep->odd = odd ^ 1; + + int pipenum = ep->pipenum; + int next_pipenum; + // TU_LOG1("TOKDNE %x PID %x pipe %d\r\n", s, pid, pipenum); + + xfer_result_t result; + switch (pid) { + default: + if (continue_transfer(pipenum, bd)) + return; + result = XFER_RESULT_SUCCESS; + break; + case TOK_PID_NAK: + suspend_transfer(pipenum, bd); + next_pipenum = select_next_pipenum(pipenum); + if (0 <= next_pipenum) + resume_transfer(next_pipenum); + return; + case TOK_PID_STALL: + result = XFER_RESULT_STALLED; + break; + case TOK_PID_ERR: /* mismatch toggle bit */ + case TOK_PID_BUSTO: + result = XFER_RESULT_FAILED; + break; + } + _hcd.in_progress &= ~TU_BIT(pipenum); + pipe_state_t *pipe = &_hcd.pipe[ep->pipenum]; + hcd_event_xfer_complete(pipe->dev_addr, + tu_edpt_addr(CI_REG->TOKEN & USB_TOKEN_TOKENENDPT_MASK, dir_in), + pipe->length - pipe->remaining, + result, true); + next_pipenum = select_next_pipenum(pipenum); + if (0 <= next_pipenum) + resume_transfer(next_pipenum); +} + +static void process_attach(uint8_t rhport) +{ + unsigned ctl = CI_REG->CTL; + if (!(ctl & USB_CTL_JSTATE_MASK)) { + /* The attached device is a low speed device. */ + CI_REG->ADDR = USB_ADDR_LSEN_MASK; + CI_REG->EP[0].CTL = USB_ENDPT_HOSTWOHUB_MASK; + } + hcd_event_device_attach(rhport, true); +} + +static void process_bus_reset(uint8_t rhport) +{ + CI_REG->INT_STAT = USB_ISTAT_TOKDNE_MASK; + CI_REG->USBCTRL &= ~USB_USBCTRL_SUSP_MASK; + CI_REG->CTL &= ~USB_CTL_USBENSOFEN_MASK; + CI_REG->ADDR = 0; + CI_REG->EP[0].CTL = 0; + + hcd_event_device_remove(rhport, true); + + _hcd.in_progress = 0; + _hcd.pending = 0; + buffer_descriptor_t *bd = &_hcd.bdt[0][0]; + for (unsigned i = 0; i < 2; ++i, ++bd) { + bd->head = 0; + } +} + +/*------------------------------------------------------------------*/ +/* Host API + *------------------------------------------------------------------*/ +bool hcd_init(uint8_t rhport, const tusb_rhport_init_t* rh_init) { + (void) rhport; + (void) rh_init; + CI_REG->USBTRC0 |= USB_USBTRC0_USBRESET_MASK; + while (CI_REG->USBTRC0 & USB_USBTRC0_USBRESET_MASK); + + tu_memclr(&_hcd, sizeof(_hcd)); + CI_REG->USBTRC0 |= TU_BIT(6); /* software must set this bit to 1 */ + CI_REG->BDT_PAGE1 = (uint8_t)((uintptr_t)_hcd.bdt >> 8); + CI_REG->BDT_PAGE2 = (uint8_t)((uintptr_t)_hcd.bdt >> 16); + CI_REG->BDT_PAGE3 = (uint8_t)((uintptr_t)_hcd.bdt >> 24); + + CI_REG->USBCTRL &= ~USB_USBCTRL_SUSP_MASK; + CI_REG->CTL |= USB_CTL_ODDRST_MASK; + for (unsigned i = 0; i < 16; ++i) { + CI_REG->EP[i].CTL = 0; + } + CI_REG->CTL &= ~USB_CTL_ODDRST_MASK; + + CI_REG->SOF_THLD = 74; /* for 64-byte packets */ + // CI_REG->SOF_THLD = 144; /* for low speed 8-byte packets */ + CI_REG->CTL = USB_CTL_HOSTMODEEN_MASK | USB_CTL_SE0_MASK; + CI_REG->USBCTRL = USB_USBCTRL_PDE_MASK; + + NVIC_ClearPendingIRQ(CI_FS_IRQN); + CI_REG->INT_EN = USB_INTEN_ATTACHEN_MASK | USB_INTEN_TOKDNEEN_MASK | + USB_INTEN_USBRSTEN_MASK | USB_INTEN_ERROREN_MASK | USB_INTEN_STALLEN_MASK; + CI_REG->ERR_ENB = 0xff; + + return true; +} + +void hcd_int_enable(uint8_t rhport) +{ + (void)rhport; + NVIC_EnableIRQ(CI_FS_IRQN); +} + +void hcd_int_disable(uint8_t rhport) +{ + (void)rhport; + NVIC_DisableIRQ(CI_FS_IRQN); +} + +uint32_t hcd_frame_number(uint8_t rhport) +{ + (void)rhport; + /* The device must be reset at least once after connection + * in order to start the frame counter. */ + if (_hcd.need_reset) hcd_port_reset(rhport); + uint32_t frmnum = CI_REG->FRM_NUML; + frmnum |= CI_REG->FRM_NUMH << 8u; + return frmnum; +} + +/*--------------------------------------------------------------------+ + * Port API + *--------------------------------------------------------------------+ */ +bool hcd_port_connect_status(uint8_t rhport) +{ + (void)rhport; + if (CI_REG->INT_STAT & USB_ISTAT_ATTACH_MASK) + return true; + return false; +} + +void hcd_port_reset(uint8_t rhport) +{ + (void)rhport; + CI_REG->CTL &= ~USB_CTL_USBENSOFEN_MASK; + CI_REG->CTL |= USB_CTL_RESET_MASK; + unsigned cnt = SystemCoreClock / 100; + while (cnt--) __NOP(); + CI_REG->CTL &= ~USB_CTL_RESET_MASK; + CI_REG->CTL |= USB_CTL_USBENSOFEN_MASK; + _hcd.need_reset = false; +} + +void hcd_port_reset_end(uint8_t rhport) { + (void) rhport; +} + +tusb_speed_t hcd_port_speed_get(uint8_t rhport) +{ + (void)rhport; + tusb_speed_t speed = TUSB_SPEED_FULL; + const unsigned ie = NVIC_GetEnableIRQ(CI_FS_IRQN); + NVIC_DisableIRQ(CI_FS_IRQN); + if (CI_REG->ADDR & USB_ADDR_LSEN_MASK) + speed = TUSB_SPEED_LOW; + if (ie) NVIC_EnableIRQ(CI_FS_IRQN); + return speed; +} + +void hcd_device_close(uint8_t rhport, uint8_t dev_addr) +{ + (void)rhport; + const unsigned ie = NVIC_GetEnableIRQ(CI_FS_IRQN); + NVIC_DisableIRQ(CI_FS_IRQN); + pipe_state_t *p = &_hcd.pipe[0]; + pipe_state_t *end = &_hcd.pipe[CFG_TUH_ENDPOINT_MAX * 2]; + for (;p != end; ++p) { + if (p->dev_addr == dev_addr) + tu_memclr(p, sizeof(*p)); + } + if (ie) NVIC_EnableIRQ(CI_FS_IRQN); +} + +//--------------------------------------------------------------------+ +// Endpoints API +//--------------------------------------------------------------------+ +bool hcd_setup_send(uint8_t rhport, uint8_t dev_addr, uint8_t const setup_packet[8]) +{ + (void)rhport; + // TU_LOG1("SETUP %u\r\n", dev_addr); + TU_ASSERT(0 == (_hcd.in_progress & TU_BIT(0))); + + int pipenum = find_pipe(dev_addr, 0); + if (pipenum < 0) return false; + + pipe_state_t *pipe = &_hcd.pipe[pipenum]; + pipe[0].data = 0; + pipe[0].buffer = (uint8_t*)(uintptr_t)setup_packet; + pipe[0].length = 8; + pipe[0].remaining = 8; + pipe[1].data = 1; + + if (1 != prepare_packets(pipenum)) + return false; + + _hcd.in_progress |= TU_BIT(pipenum); + + unsigned hostwohub = CI_REG->EP[0].CTL & USB_ENDPT_HOSTWOHUB_MASK; + CI_REG->EP[0].CTL = hostwohub | + USB_ENDPT_EPHSHK_MASK | USB_ENDPT_EPRXEN_MASK | USB_ENDPT_EPTXEN_MASK; + CI_REG->ADDR = (CI_REG->ADDR & USB_ADDR_LSEN_MASK) | dev_addr; + while (CI_REG->CTL & USB_CTL_TXSUSPENDTOKENBUSY_MASK) ; + CI_REG->TOKEN = (TOK_PID_SETUP << USB_TOKEN_TOKENPID_SHIFT); + return true; +} + +bool hcd_edpt_open(uint8_t rhport, uint8_t dev_addr, tusb_desc_endpoint_t const * ep_desc) +{ + (void)rhport; + uint8_t const ep_addr = ep_desc->bEndpointAddress; + // TU_LOG1("O %u %x\r\n", dev_addr, ep_addr); + /* Find a free pipe */ + pipe_state_t *p = &_hcd.pipe[0]; + pipe_state_t *end = &_hcd.pipe[CFG_TUH_ENDPOINT_MAX * 2]; + if (dev_addr || ep_addr) { + p += 2; + for (; p < end && (p->dev_addr || p->ep_addr); ++p) ; + if (p == end) return false; + } + p->dev_addr = dev_addr; + p->ep_addr = ep_addr; + p->max_packet_size = ep_desc->wMaxPacketSize; + p->xfer = ep_desc->bmAttributes.xfer; + p->data = 0; + if (!ep_addr) { + /* Open one more pipe for Control IN transfer */ + TU_ASSERT(TUSB_XFER_CONTROL == p->xfer); + pipe_state_t *q = p + 1; + TU_ASSERT(!q->dev_addr && !q->ep_addr); + q->dev_addr = dev_addr; + q->ep_addr = tu_edpt_addr(0, TUSB_DIR_IN); + q->max_packet_size = ep_desc->wMaxPacketSize; + q->xfer = ep_desc->bmAttributes.xfer; + q->data = 1; + } + return true; +} + +bool hcd_edpt_close(uint8_t rhport, uint8_t daddr, uint8_t ep_addr) { + (void) rhport; (void) daddr; (void) ep_addr; + return false; // TODO not implemented yet +} + +/* The address of buffer must be aligned to 4 byte boundary. And it must be at least 4 bytes long. + * DMA writes data in 4 byte unit */ +bool hcd_edpt_xfer(uint8_t rhport, uint8_t dev_addr, uint8_t ep_addr, uint8_t * buffer, uint16_t buflen) +{ + (void)rhport; + // TU_LOG1("X %u %x %x %d\r\n", dev_addr, ep_addr, (uintptr_t)buffer, buflen); + + int pipenum = find_pipe(dev_addr, ep_addr); + TU_ASSERT(0 <= pipenum); + + TU_ASSERT(0 == (_hcd.in_progress & TU_BIT(pipenum))); + unsigned const ie = NVIC_GetEnableIRQ(CI_FS_IRQN); + NVIC_DisableIRQ(CI_FS_IRQN); + pipe_state_t *pipe = &_hcd.pipe[pipenum]; + pipe->buffer = buffer; + pipe->length = buflen; + pipe->remaining = buflen; + _hcd.in_progress |= TU_BIT(pipenum); + _hcd.pending |= TU_BIT(pipenum); /* Send at the next Frame */ + CI_REG->INT_EN |= USB_ISTAT_SOFTOK_MASK; + if (ie) NVIC_EnableIRQ(CI_FS_IRQN); + return true; +} + +bool hcd_edpt_abort_xfer(uint8_t rhport, uint8_t dev_addr, uint8_t ep_addr) { + (void) rhport; + (void) dev_addr; + (void) ep_addr; + // TODO not implemented yet + return false; +} + +bool hcd_edpt_clear_stall(uint8_t rhport, uint8_t dev_addr, uint8_t ep_addr) { + (void) rhport; + if (!tu_edpt_number(ep_addr)) return true; + int num = find_pipe(dev_addr, ep_addr); + if (num < 0) return false; + pipe_state_t *p = &_hcd.pipe[num]; + p->data = 0; /* Reset data toggle */ + return true; +} + +/*--------------------------------------------------------------------+ + * ISR + *--------------------------------------------------------------------+*/ +void hcd_int_handler(uint8_t rhport, bool in_isr) +{ + (void) in_isr; + uint32_t is = CI_REG->INT_STAT; + uint32_t msk = CI_REG->INT_EN; + + // TU_LOG1("S %lx\r\n", is); + + /* clear disabled interrupts */ + CI_REG->INT_STAT = (is & ~msk & ~USB_ISTAT_TOKDNE_MASK) | USB_ISTAT_SOFTOK_MASK; + is &= msk; + + if (is & USB_ISTAT_ERROR_MASK) { + unsigned err = CI_REG->ERR_STAT; + if (err) { + TU_LOG1(" ERR %x\r\n", err); + CI_REG->ERR_STAT = err; + } else { + CI_REG->INT_EN &= ~USB_ISTAT_ERROR_MASK; + } + } + + if (is & USB_ISTAT_USBRST_MASK) { + CI_REG->INT_EN = (msk & ~USB_INTEN_USBRSTEN_MASK) | USB_INTEN_ATTACHEN_MASK; + process_bus_reset(rhport); + return; + } + if (is & USB_ISTAT_ATTACH_MASK) { + CI_REG->INT_EN = (msk & ~USB_INTEN_ATTACHEN_MASK) | USB_INTEN_USBRSTEN_MASK; + _hcd.need_reset = true; + process_attach(rhport); + return; + } + if (is & USB_ISTAT_STALL_MASK) { + CI_REG->INT_STAT = USB_ISTAT_STALL_MASK; + } + if (is & USB_ISTAT_SOFTOK_MASK) { + msk &= ~USB_ISTAT_SOFTOK_MASK; + CI_REG->INT_EN = msk; + if (_hcd.pending) { + int pipenum = __builtin_ctz(_hcd.pending); + _hcd.pending = 0; + if (!(is & USB_ISTAT_TOKDNE_MASK)) + resume_transfer(pipenum); + } + } + if (is & USB_ISTAT_TOKDNE_MASK) { + process_tokdne(rhport); + } +} + +#endif diff --git a/src/portable/nxp/khci/dcd_khci.c b/src/portable/nxp/khci/dcd_khci.c deleted file mode 100644 index 9f61bd236..000000000 --- a/src/portable/nxp/khci/dcd_khci.c +++ /dev/null @@ -1,560 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2020 Koji Kitayama - * SPDX-FileCopyrightText: Copyright (c) 2020 Ha Thach (tinyusb.org) - * SPDX-License-Identifier: MIT - * - * This file is part of the TinyUSB stack. - */ - -#include "tusb_option.h" - -#if CFG_TUD_ENABLED && defined(TUP_USBIP_CHIPIDEA_FS) - -#ifdef TUP_USBIP_CHIPIDEA_FS_KINETIS - #include "fsl_device_registers.h" - #define KHCI USB0 -#else - #error "MCU is not supported" -#endif - -#include "device/dcd.h" - -//--------------------------------------------------------------------+ -// MACRO TYPEDEF CONSTANT ENUM DECLARATION -//--------------------------------------------------------------------+ - -enum { - TOK_PID_OUT = 0x1u, - TOK_PID_IN = 0x9u, - TOK_PID_SETUP = 0xDu, -}; - -typedef struct TU_ATTR_PACKED -{ - union { - uint32_t head; - struct { - union { - struct { - uint16_t : 2; - __IO uint16_t tok_pid : 4; - uint16_t data : 1; - __IO uint16_t own : 1; - uint16_t : 8; - }; - struct { - uint16_t : 2; - uint16_t bdt_stall : 1; - uint16_t dts : 1; - uint16_t ninc : 1; - uint16_t keep : 1; - uint16_t : 10; - }; - }; - __IO uint16_t bc : 10; - uint16_t : 6; - }; - }; - uint8_t *addr; -}buffer_descriptor_t; - -TU_VERIFY_STATIC( sizeof(buffer_descriptor_t) == 8, "size is not correct" ); - -typedef struct TU_ATTR_PACKED -{ - union { - uint32_t state; - struct { - uint32_t max_packet_size :11; - uint32_t : 5; - uint32_t odd : 1; - uint32_t :15; - }; - }; - uint16_t length; - uint16_t remaining; -}endpoint_state_t; - -TU_VERIFY_STATIC( sizeof(endpoint_state_t) == 8, "size is not correct" ); - -typedef struct -{ - union { - /* [#EP][OUT,IN][EVEN,ODD] */ - buffer_descriptor_t bdt[16][2][2]; - uint16_t bda[512]; - }; - TU_ATTR_ALIGNED(4) union { - endpoint_state_t endpoint[16][2]; - endpoint_state_t endpoint_unified[16 * 2]; - }; - uint8_t setup_packet[8]; - uint8_t addr; -}dcd_data_t; - -//--------------------------------------------------------------------+ -// INTERNAL OBJECT & FUNCTION DECLARATION -//--------------------------------------------------------------------+ -// BDT(Buffer Descriptor Table) must be 256-byte aligned -CFG_TUD_MEM_SECTION TU_ATTR_ALIGNED(512) static dcd_data_t _dcd; - -TU_VERIFY_STATIC( sizeof(_dcd.bdt) == 512, "size is not correct" ); - -static void prepare_next_setup_packet(uint8_t rhport) -{ - const unsigned out_odd = _dcd.endpoint[0][0].odd; - const unsigned in_odd = _dcd.endpoint[0][1].odd; - TU_ASSERT(0 == _dcd.bdt[0][0][out_odd].own, ); - - _dcd.bdt[0][0][out_odd].data = 0; - _dcd.bdt[0][0][out_odd ^ 1].data = 1; - _dcd.bdt[0][1][in_odd].data = 1; - _dcd.bdt[0][1][in_odd ^ 1].data = 0; - dcd_edpt_xfer(rhport, tu_edpt_addr(0, TUSB_DIR_OUT), - _dcd.setup_packet, sizeof(_dcd.setup_packet), false); -} - -static void process_stall(uint8_t rhport) -{ - for (int i = 0; i < 16; ++i) { - unsigned const endpt = KHCI->ENDPOINT[i].ENDPT; - - if (endpt & USB_ENDPT_EPSTALL_MASK) { - // prepare next setup if endpoint0 - if ( i == 0 ) prepare_next_setup_packet(rhport); - - // clear stall bit - KHCI->ENDPOINT[i].ENDPT = endpt & ~USB_ENDPT_EPSTALL_MASK; - } - } -} - -static void process_tokdne(uint8_t rhport) -{ - const unsigned s = KHCI->STAT; - KHCI->ISTAT = USB_ISTAT_TOKDNE_MASK; /* fetch the next token if received */ - - uint8_t const epnum = (s >> USB_STAT_ENDP_SHIFT); - uint8_t const dir = (s & USB_STAT_TX_MASK) >> USB_STAT_TX_SHIFT; - unsigned const odd = (s & USB_STAT_ODD_MASK) ? 1 : 0; - - buffer_descriptor_t *bd = (buffer_descriptor_t *)&_dcd.bda[s]; - endpoint_state_t *ep = &_dcd.endpoint_unified[s >> 3]; - - /* fetch pid before discarded by the next steps */ - const unsigned pid = bd->tok_pid; - - /* reset values for a next transfer */ - bd->bdt_stall = 0; - bd->dts = 1; - bd->ninc = 0; - bd->keep = 0; - /* update the odd variable to prepare for the next transfer */ - ep->odd = odd ^ 1; - if (pid == TOK_PID_SETUP) { - dcd_event_setup_received(rhport, bd->addr, true); - KHCI->CTL &= ~USB_CTL_TXSUSPENDTOKENBUSY_MASK; - return; - } - - const unsigned bc = bd->bc; - const unsigned remaining = ep->remaining - bc; - if (remaining && bc == ep->max_packet_size) { - /* continue the transferring consecutive data */ - ep->remaining = remaining; - const int next_remaining = remaining - ep->max_packet_size; - if (next_remaining > 0) { - /* prepare to the after next transfer */ - bd->addr += ep->max_packet_size * 2; - bd->bc = next_remaining > ep->max_packet_size ? ep->max_packet_size: next_remaining; - __DSB(); - bd->own = 1; /* the own bit must set after addr */ - } - return; - } - const unsigned length = ep->length; - dcd_event_xfer_complete(rhport, - tu_edpt_addr(epnum, dir), - length - remaining, XFER_RESULT_SUCCESS, true); - if (0 == epnum && 0 == length) { - /* After completion a ZLP of control transfer, - * it prepares for the next steup transfer. */ - if (_dcd.addr) { - /* When the transfer was the SetAddress, - * the device address should be updated here. */ - KHCI->ADDR = _dcd.addr; - _dcd.addr = 0; - } - prepare_next_setup_packet(rhport); - } -} - -static void process_bus_reset(uint8_t rhport) -{ - KHCI->USBCTRL &= ~USB_USBCTRL_SUSP_MASK; - KHCI->CTL |= USB_CTL_ODDRST_MASK; - KHCI->ADDR = 0; - KHCI->INTEN = USB_INTEN_USBRSTEN_MASK | USB_INTEN_TOKDNEEN_MASK | USB_INTEN_SLEEPEN_MASK | - USB_INTEN_ERROREN_MASK | USB_INTEN_STALLEN_MASK; - - KHCI->ENDPOINT[0].ENDPT = USB_ENDPT_EPHSHK_MASK | USB_ENDPT_EPRXEN_MASK | USB_ENDPT_EPTXEN_MASK; - for (unsigned i = 1; i < 16; ++i) { - KHCI->ENDPOINT[i].ENDPT = 0; - } - buffer_descriptor_t *bd = _dcd.bdt[0][0]; - for (unsigned i = 0; i < sizeof(_dcd.bdt)/sizeof(*bd); ++i, ++bd) { - bd->head = 0; - } - const endpoint_state_t ep0 = { - .max_packet_size = CFG_TUD_ENDPOINT0_SIZE, - .odd = 0, - .length = 0, - .remaining = 0, - }; - _dcd.endpoint[0][0] = ep0; - _dcd.endpoint[0][1] = ep0; - tu_memclr(_dcd.endpoint[1], sizeof(_dcd.endpoint) - sizeof(_dcd.endpoint[0])); - _dcd.addr = 0; - prepare_next_setup_packet(rhport); - KHCI->CTL &= ~USB_CTL_ODDRST_MASK; - dcd_event_bus_reset(rhport, TUSB_SPEED_FULL, true); -} - -static void process_bus_sleep(uint8_t rhport) -{ - // Enable resume & disable suspend interrupt - const unsigned inten = KHCI->INTEN; - - KHCI->INTEN = (inten & ~USB_INTEN_SLEEPEN_MASK) | USB_INTEN_RESUMEEN_MASK; - KHCI->USBTRC0 |= USB_USBTRC0_USBRESMEN_MASK; - KHCI->USBCTRL |= USB_USBCTRL_SUSP_MASK; - - dcd_event_bus_signal(rhport, DCD_EVENT_SUSPEND, true); -} - -static void process_bus_resume(uint8_t rhport) -{ - // Enable suspend & disable resume interrupt - const unsigned inten = KHCI->INTEN; - - KHCI->USBCTRL &= ~USB_USBCTRL_SUSP_MASK; // will also clear USB_USBTRC0_USB_RESUME_INT_MASK - KHCI->USBTRC0 &= ~USB_USBTRC0_USBRESMEN_MASK; - KHCI->INTEN = (inten & ~USB_INTEN_RESUMEEN_MASK) | USB_INTEN_SLEEPEN_MASK; - - dcd_event_bus_signal(rhport, DCD_EVENT_RESUME, true); -} - -/*------------------------------------------------------------------*/ -/* Device API - *------------------------------------------------------------------*/ -bool dcd_init(uint8_t rhport, const tusb_rhport_init_t* rh_init) { - (void) rhport; - (void) rh_init; - - // save crystal-less setting (if available) - #if defined(FSL_FEATURE_USB_KHCI_IRC48M_MODULE_CLOCK_ENABLED) && FSL_FEATURE_USB_KHCI_IRC48M_MODULE_CLOCK_ENABLED == 1 - uint32_t clk_recover_irc_en = KHCI->CLK_RECOVER_IRC_EN; - uint32_t clk_recover_ctrl = KHCI->CLK_RECOVER_CTRL; - #endif - - KHCI->USBTRC0 |= USB_USBTRC0_USBRESET_MASK; - while (KHCI->USBTRC0 & USB_USBTRC0_USBRESET_MASK); - - // restore crystal-less setting (if available) - #if defined(FSL_FEATURE_USB_KHCI_IRC48M_MODULE_CLOCK_ENABLED) && FSL_FEATURE_USB_KHCI_IRC48M_MODULE_CLOCK_ENABLED == 1 - KHCI->CLK_RECOVER_IRC_EN = clk_recover_irc_en; - KHCI->CLK_RECOVER_CTRL |= clk_recover_ctrl; - #endif - - tu_memclr(&_dcd, sizeof(_dcd)); - KHCI->USBTRC0 |= TU_BIT(6); /* software must set this bit to 1 */ - KHCI->BDTPAGE1 = (uint8_t)((uintptr_t)_dcd.bdt >> 8); - KHCI->BDTPAGE2 = (uint8_t)((uintptr_t)_dcd.bdt >> 16); - KHCI->BDTPAGE3 = (uint8_t)((uintptr_t)_dcd.bdt >> 24); - - KHCI->INTEN = USB_INTEN_USBRSTEN_MASK; - - dcd_connect(rhport); - NVIC_ClearPendingIRQ(USB0_IRQn); - - return true; -} - -void dcd_int_enable(uint8_t rhport) -{ - (void) rhport; - NVIC_EnableIRQ(USB0_IRQn); -} - -void dcd_int_disable(uint8_t rhport) -{ - (void) rhport; - NVIC_DisableIRQ(USB0_IRQn); -} - -void dcd_set_address(uint8_t rhport, uint8_t dev_addr) -{ - _dcd.addr = dev_addr & 0x7F; - /* Response with status first before changing device address */ - dcd_edpt_xfer(rhport, tu_edpt_addr(0, TUSB_DIR_IN), NULL, 0, false); -} - -void dcd_remote_wakeup(uint8_t rhport) -{ - (void) rhport; - - KHCI->CTL |= USB_CTL_RESUME_MASK; - - unsigned cnt = SystemCoreClock / 1000; - while (cnt--) __NOP(); - - KHCI->CTL &= ~USB_CTL_RESUME_MASK; -} - -void dcd_connect(uint8_t rhport) -{ - (void) rhport; - KHCI->USBCTRL = 0; - KHCI->CONTROL |= USB_CONTROL_DPPULLUPNONOTG_MASK; - KHCI->CTL |= USB_CTL_USBENSOFEN_MASK; -} - -void dcd_disconnect(uint8_t rhport) -{ - (void) rhport; - KHCI->CTL = 0; - KHCI->CONTROL &= ~USB_CONTROL_DPPULLUPNONOTG_MASK; -} - -void dcd_sof_enable(uint8_t rhport, bool en) -{ - (void) rhport; - (void) en; - - // TODO implement later -} - -//--------------------------------------------------------------------+ -// Endpoint API -//--------------------------------------------------------------------+ -static bool edpt_open(uint8_t rhport, uint8_t ep_addr, uint16_t max_packet_size, tusb_xfer_type_t xfer) { - (void)rhport; - - const unsigned epn = tu_edpt_number(ep_addr); - const unsigned dir = tu_edpt_dir(ep_addr); - endpoint_state_t *ep = &_dcd.endpoint[epn][dir]; - const unsigned odd = ep->odd; - buffer_descriptor_t *bd = _dcd.bdt[epn][dir]; - - /* No support for control transfer */ - TU_ASSERT(epn && (xfer != TUSB_XFER_CONTROL)); - - ep->max_packet_size = max_packet_size; - unsigned val = USB_ENDPT_EPCTLDIS_MASK; - val |= (xfer != TUSB_XFER_ISOCHRONOUS) ? USB_ENDPT_EPHSHK_MASK : 0; - val |= dir ? USB_ENDPT_EPTXEN_MASK : USB_ENDPT_EPRXEN_MASK; - KHCI->ENDPOINT[epn].ENDPT |= val; - - if (xfer != TUSB_XFER_ISOCHRONOUS) { - bd[odd].dts = 1; - bd[odd].data = 0; - bd[odd ^ 1].dts = 1; - bd[odd ^ 1].data = 1; - } - - return true; -} - -bool dcd_edpt_open(uint8_t rhport, const tusb_desc_endpoint_t *ep_desc) { - return edpt_open(rhport, ep_desc->bEndpointAddress, tu_edpt_packet_size(ep_desc), ep_desc->bmAttributes.xfer); -} - -bool dcd_edpt_iso_alloc(uint8_t rhport, uint8_t ep_addr, uint16_t largest_packet_size) { - return edpt_open(rhport, ep_addr, largest_packet_size, TUSB_XFER_ISOCHRONOUS); -} - -bool dcd_edpt_iso_activate(uint8_t rhport, const tusb_desc_endpoint_t *ep_desc) { - const unsigned epn = tu_edpt_number(ep_desc->bEndpointAddress); - const unsigned dir = tu_edpt_dir(ep_desc->bEndpointAddress); - endpoint_state_t *ep = &_dcd.endpoint[epn][dir]; - - dcd_int_disable(rhport); - ep->max_packet_size = tu_edpt_packet_size(ep_desc); - dcd_int_enable(rhport); - - return true; -} - -void dcd_edpt_close_all(uint8_t rhport) -{ - (void) rhport; - const unsigned ie = NVIC_GetEnableIRQ(USB0_IRQn); - NVIC_DisableIRQ(USB0_IRQn); - for (unsigned i = 1; i < 16; ++i) { - KHCI->ENDPOINT[i].ENDPT = 0; - } - if (ie) NVIC_EnableIRQ(USB0_IRQn); - buffer_descriptor_t *bd = _dcd.bdt[1][0]; - for (unsigned i = 2; i < sizeof(_dcd.bdt)/sizeof(*bd); ++i, ++bd) { - bd->head = 0; - } - endpoint_state_t *ep = &_dcd.endpoint[1][0]; - for (unsigned i = 2; i < sizeof(_dcd.endpoint)/sizeof(*ep); ++i, ++ep) { - /* Clear except the odd */ - ep->max_packet_size = 0; - ep->length = 0; - ep->remaining = 0; - } -} - -bool dcd_edpt_xfer(uint8_t rhport, uint8_t ep_addr, uint8_t * buffer, uint16_t total_bytes, bool is_isr) -{ - (void) rhport; - (void) is_isr; - const unsigned epn = tu_edpt_number(ep_addr); - const unsigned dir = tu_edpt_dir(ep_addr); - endpoint_state_t *ep = &_dcd.endpoint[epn][dir]; - buffer_descriptor_t *bd = &_dcd.bdt[epn][dir][ep->odd]; - TU_ASSERT(0 == bd->own); - - const unsigned ie = NVIC_GetEnableIRQ(USB0_IRQn); - NVIC_DisableIRQ(USB0_IRQn); - - ep->length = total_bytes; - ep->remaining = total_bytes; - - const unsigned mps = ep->max_packet_size; - if (total_bytes > mps) { - buffer_descriptor_t *next = ep->odd ? bd - 1: bd + 1; - /* When total_bytes is greater than the max packet size, - * it prepares to the next transfer to avoid NAK in advance. */ - next->bc = total_bytes >= 2 * mps ? mps: total_bytes - mps; - next->addr = buffer + mps; - next->own = 1; - } - bd->bc = total_bytes >= mps ? mps: total_bytes; - bd->addr = buffer; - __DSB(); - bd->own = 1; /* This bit must be set last */ - - if (ie) NVIC_EnableIRQ(USB0_IRQn); - return true; -} - -void dcd_edpt_stall(uint8_t rhport, uint8_t ep_addr) -{ - (void) rhport; - const unsigned epn = tu_edpt_number(ep_addr); - - if (0 == epn) { - KHCI->ENDPOINT[epn].ENDPT |= USB_ENDPT_EPSTALL_MASK; - } else { - const unsigned dir = tu_edpt_dir(ep_addr); - const unsigned odd = _dcd.endpoint[epn][dir].odd; - buffer_descriptor_t *bd = &_dcd.bdt[epn][dir][odd]; - TU_ASSERT(0 == bd->own,); - - const unsigned ie = NVIC_GetEnableIRQ(USB0_IRQn); - NVIC_DisableIRQ(USB0_IRQn); - - bd->bdt_stall = 1; - __DSB(); - bd->own = 1; /* This bit must be set last */ - - if (ie) NVIC_EnableIRQ(USB0_IRQn); - } -} - -void dcd_edpt_clear_stall(uint8_t rhport, uint8_t ep_addr) -{ - (void) rhport; - const unsigned epn = tu_edpt_number(ep_addr); - TU_VERIFY(epn,); - const unsigned dir = tu_edpt_dir(ep_addr); - const unsigned odd = _dcd.endpoint[epn][dir].odd; - buffer_descriptor_t *bd = _dcd.bdt[epn][dir]; - TU_VERIFY(bd[odd].own,); - - const unsigned ie = NVIC_GetEnableIRQ(USB0_IRQn); - NVIC_DisableIRQ(USB0_IRQn); - - bd[odd].own = 0; - __DSB(); - - // clear stall - bd[odd].bdt_stall = 0; - - // Reset data toggle - bd[odd ].data = 0; - bd[odd ^ 1].data = 1; - - // We already cleared this in ISR, but just clear it here to be safe - const unsigned endpt = KHCI->ENDPOINT[epn].ENDPT; - if (endpt & USB_ENDPT_EPSTALL_MASK) { - KHCI->ENDPOINT[epn].ENDPT = endpt & ~USB_ENDPT_EPSTALL_MASK; - } - - if (ie) NVIC_EnableIRQ(USB0_IRQn); -} - -//--------------------------------------------------------------------+ -// ISR -//--------------------------------------------------------------------+ -void dcd_int_handler(uint8_t rhport) -{ - uint32_t is = KHCI->ISTAT; - uint32_t msk = KHCI->INTEN; - - // clear non-enabled interrupts - KHCI->ISTAT = is & ~msk; - is &= msk; - - if (is & USB_ISTAT_ERROR_MASK) { - /* TODO: */ - uint32_t es = KHCI->ERRSTAT; - KHCI->ERRSTAT = es; - KHCI->ISTAT = is; /* discard any pending events */ - } - - if (is & USB_ISTAT_USBRST_MASK) { - KHCI->ISTAT = is; /* discard any pending events */ - process_bus_reset(rhport); - } - - if (is & USB_ISTAT_SLEEP_MASK) { - // TU_LOG3("Suspend: "); TU_LOG2_HEX(is); - - // Note Host usually has extra delay after bus reset (without SOF), which could falsely - // detected as Sleep event. Though usbd has debouncing logic so we are good - KHCI->ISTAT = USB_ISTAT_SLEEP_MASK; - process_bus_sleep(rhport); - } - -#if 0 // ISTAT_RESUME never trigger, probably for host mode ? - if (is & USB_ISTAT_RESUME_MASK) { - // TU_LOG2("ISTAT Resume: "); TU_LOG2_HEX(is); - KHCI->ISTAT = USB_ISTAT_RESUME_MASK; - process_bus_resume(rhport); - } -#endif - - if (KHCI->USBTRC0 & USB_USBTRC0_USB_RESUME_INT_MASK) { - // TU_LOG2("USBTRC0 Resume: "); TU_LOG2_HEX(is); TU_LOG2_HEX(KHCI->USBTRC0); - process_bus_resume(rhport); - } - - if (is & USB_ISTAT_SOFTOK_MASK) { - KHCI->ISTAT = USB_ISTAT_SOFTOK_MASK; - dcd_event_sof(rhport, tu_u16(KHCI->FRMNUMH, KHCI->FRMNUML), true); - } - - if (is & USB_ISTAT_STALL_MASK) { - KHCI->ISTAT = USB_ISTAT_STALL_MASK; - process_stall(rhport); - } - - if (is & USB_ISTAT_TOKDNE_MASK) { - process_tokdne(rhport); - } -} -#endif diff --git a/src/portable/nxp/khci/hcd_khci.c b/src/portable/nxp/khci/hcd_khci.c deleted file mode 100644 index 209940656..000000000 --- a/src/portable/nxp/khci/hcd_khci.c +++ /dev/null @@ -1,628 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2021 Koji Kitayama - * SPDX-FileCopyrightText: Copyright (c) 2021 Ha Thach (tinyusb.org) - * SPDX-License-Identifier: MIT - * - * This file is part of the TinyUSB stack. - */ - -#include "tusb_option.h" - -#if CFG_TUH_ENABLED && defined(TUP_USBIP_CHIPIDEA_FS) - -#ifdef TUP_USBIP_CHIPIDEA_FS_KINETIS - #include "fsl_device_registers.h" - #define KHCI USB0 -#else - #error "MCU is not supported" -#endif - -#include "host/hcd.h" -#include "host/usbh.h" - -//--------------------------------------------------------------------+ -// MACRO TYPEDEF CONSTANT ENUM DECLARATION -//--------------------------------------------------------------------+ - -enum { - TOK_PID_OUT = 0x1u, - TOK_PID_IN = 0x9u, - TOK_PID_SETUP = 0xDu, - TOK_PID_DATA0 = 0x3u, - TOK_PID_DATA1 = 0xbu, - TOK_PID_ACK = 0x2u, - TOK_PID_STALL = 0xeu, - TOK_PID_NAK = 0xau, - TOK_PID_BUSTO = 0x0u, - TOK_PID_ERR = 0xfu, -}; - -typedef struct TU_ATTR_PACKED -{ - union { - uint32_t head; - struct { - union { - struct { - uint16_t : 2; - __IO uint16_t tok_pid : 4; - uint16_t data : 1; - __IO uint16_t own : 1; - uint16_t : 8; - }; - struct { - uint16_t : 2; - uint16_t bdt_stall : 1; - uint16_t dts : 1; - uint16_t ninc : 1; - uint16_t keep : 1; - uint16_t : 10; - }; - }; - __IO uint16_t bc : 10; - uint16_t : 6; - }; - }; - uint8_t *addr; -}buffer_descriptor_t; - -TU_VERIFY_STATIC( sizeof(buffer_descriptor_t) == 8, "size is not correct" ); - -typedef struct TU_ATTR_PACKED -{ - union { - uint32_t state; - struct { - uint32_t pipenum:16; - uint32_t odd : 1; - uint32_t : 0; - }; - }; - uint8_t *buffer; - uint16_t length; - uint16_t remaining; -} endpoint_state_t; - -typedef struct TU_ATTR_PACKED -{ - uint8_t dev_addr; - uint8_t ep_addr; - uint16_t max_packet_size; - union { - uint8_t flags; - struct { - uint8_t data : 1; - uint8_t xfer : 2; - uint8_t : 0; - }; - }; - uint8_t *buffer; - uint16_t length; - uint16_t remaining; -} pipe_state_t; - - -typedef struct -{ - union { - /* [OUT,IN][EVEN,ODD] */ - buffer_descriptor_t bdt[2][2]; - uint16_t bda[2*2]; - }; - endpoint_state_t endpoint[2]; - pipe_state_t pipe[CFG_TUH_ENDPOINT_MAX * 2]; - uint32_t in_progress; /* Bitmap. Each bit indicates that a transfer of the corresponding pipe is in progress */ - uint32_t pending; /* Bitmap. Each bit indicates that a transfer of the corresponding pipe will be resume the next frame */ - bool need_reset; /* The device has not been reset after connection. */ -} hcd_data_t; - -//--------------------------------------------------------------------+ -// INTERNAL OBJECT & FUNCTION DECLARATION -//--------------------------------------------------------------------+ -// BDT(Buffer Descriptor Table) must be 256-byte aligned -CFG_TUH_MEM_SECTION TU_ATTR_ALIGNED(512) static hcd_data_t _hcd; -//CFG_TUH_MEM_SECTION TU_ATTR_ALIGNED(4) static uint8_t _rx_buf[1024]; - -static int find_pipe(uint8_t dev_addr, uint8_t ep_addr) -{ - /* Find the target pipe */ - int num; - for (num = 0; num < CFG_TUH_ENDPOINT_MAX * 2; ++num) { - pipe_state_t *p = &_hcd.pipe[num]; - if ((p->dev_addr == dev_addr) && (p->ep_addr == ep_addr)) - return num; - } - return -1; -} - -static int prepare_packets(int pipenum) -{ - pipe_state_t *pipe = &_hcd.pipe[pipenum]; - unsigned const dir_tx = tu_edpt_dir(pipe->ep_addr) ? 0 : 1; - endpoint_state_t *ep = &_hcd.endpoint[dir_tx]; - unsigned const odd = ep->odd; - buffer_descriptor_t *bd = _hcd.bdt[dir_tx]; - TU_ASSERT(0 == bd[odd].own, -1); - - // TU_LOG1(" %p dir %d odd %d data %d\r\n", &bd[odd], dir_tx, odd, pipe->data); - - ep->pipenum = pipenum; - - bd[odd ].data = pipe->data; - bd[odd ^ 1].data = pipe->data ^ 1; - bd[odd ^ 1].own = 0; - /* reset values for a next transfer */ - - int num_tokens = 0; /* The number of prepared packets */ - unsigned const mps = pipe->max_packet_size; - unsigned const rem = pipe->remaining; - if (rem > mps) { - /* When total_bytes is greater than the max packet size, - * it prepares to the next transfer to avoid NAK in advance. */ - bd[odd ^ 1].bc = rem >= 2 * mps ? mps: rem - mps; - bd[odd ^ 1].addr = pipe->buffer + mps; - bd[odd ^ 1].own = 1; - if (dir_tx) ++num_tokens; - } - bd[odd].bc = rem >= mps ? mps: rem; - bd[odd].addr = pipe->buffer; - __DSB(); - bd[odd].own = 1; /* This bit must be set last */ - ++num_tokens; - return num_tokens; -} - -static int select_next_pipenum(int pipenum) -{ - unsigned wip = _hcd.in_progress & ~_hcd.pending; - if (!wip) return -1; - unsigned msk = TU_GENMASK(31, pipenum); - int next = __builtin_ctz(wip & msk); - if (next) return next; - msk = TU_GENMASK(pipenum, 0); - next = __builtin_ctz(wip & msk); - return next; -} - -/* When transfer is completed, return true. */ -static bool continue_transfer(int pipenum, buffer_descriptor_t *bd) -{ - pipe_state_t *pipe = &_hcd.pipe[pipenum]; - unsigned const bc = bd->bc; - unsigned const rem = pipe->remaining - bc; - - pipe->remaining = rem; - if (rem && bc == pipe->max_packet_size) { - int const next_rem = rem - pipe->max_packet_size; - if (next_rem > 0) { - /* Prepare to the after next transfer */ - bd->addr += pipe->max_packet_size * 2; - bd->bc = next_rem > pipe->max_packet_size ? pipe->max_packet_size: next_rem; - __DSB(); - bd->own = 1; /* This bit must be set last */ - while (KHCI->CTL & USB_CTL_TXSUSPENDTOKENBUSY_MASK) ; - KHCI->TOKEN = KHCI->TOKEN; /* Queue the same token as the last */ - } else if (TUSB_DIR_IN == tu_edpt_dir(pipe->ep_addr)) { /* IN */ - while (KHCI->CTL & USB_CTL_TXSUSPENDTOKENBUSY_MASK) ; - KHCI->TOKEN = KHCI->TOKEN; - } - return true; - } - pipe->data = bd->data ^ 1; - return false; -} - -static bool resume_transfer(int pipenum) -{ - int num_tokens = prepare_packets(pipenum); - TU_ASSERT(0 <= num_tokens); - - const unsigned ie = NVIC_GetEnableIRQ(USB0_IRQn); - NVIC_DisableIRQ(USB0_IRQn); - pipe_state_t *pipe = &_hcd.pipe[pipenum]; - - unsigned flags = KHCI->ENDPOINT[0].ENDPT & USB_ENDPT_HOSTWOHUB_MASK; - flags |= USB_ENDPT_EPRXEN_MASK | USB_ENDPT_EPTXEN_MASK; - switch (pipe->xfer) { - case TUSB_XFER_CONTROL: - flags |= USB_ENDPT_EPHSHK_MASK; - break; - case TUSB_XFER_ISOCHRONOUS: - flags |= USB_ENDPT_EPCTLDIS_MASK | USB_ENDPT_RETRYDIS_MASK; - break; - default: - flags |= USB_ENDPT_EPHSHK_MASK | USB_ENDPT_EPCTLDIS_MASK | USB_ENDPT_RETRYDIS_MASK; - break; - } - // TU_LOG1(" resume pipenum %d flags %x\r\n", pipenum, flags); - - KHCI->ENDPOINT[0].ENDPT = flags; - KHCI->ADDR = (KHCI->ADDR & USB_ADDR_LSEN_MASK) | pipe->dev_addr; - - unsigned const token = tu_edpt_number(pipe->ep_addr) | - ((tu_edpt_dir(pipe->ep_addr) ? TOK_PID_IN: TOK_PID_OUT) << USB_TOKEN_TOKENPID_SHIFT); - do { - while (KHCI->CTL & USB_CTL_TXSUSPENDTOKENBUSY_MASK) ; - KHCI->TOKEN = token; - } while (--num_tokens); - if (ie) NVIC_EnableIRQ(USB0_IRQn); - return true; -} - -static void suspend_transfer(int pipenum, buffer_descriptor_t *bd) -{ - pipe_state_t *pipe = &_hcd.pipe[pipenum]; - pipe->buffer = bd->addr; - pipe->data = bd->data ^ 1; - if ((TUSB_XFER_INTERRUPT == pipe->xfer) || - (TUSB_XFER_BULK == pipe->xfer)) { - _hcd.pending |= TU_BIT(pipenum); - KHCI->INTEN |= USB_ISTAT_SOFTOK_MASK; - } -} - -static void process_tokdne(uint8_t rhport) -{ - (void)rhport; - const unsigned s = KHCI->STAT; - KHCI->ISTAT = USB_ISTAT_TOKDNE_MASK; /* fetch the next token if received */ - uint8_t const dir_in = (s & USB_STAT_TX_MASK) ? TUSB_DIR_OUT: TUSB_DIR_IN; - unsigned const odd = (s & USB_STAT_ODD_MASK) ? 1 : 0; - - buffer_descriptor_t *bd = (buffer_descriptor_t *)&_hcd.bda[s]; - endpoint_state_t *ep = &_hcd.endpoint[s >> 3]; - - /* fetch status before discarded by the next steps */ - const unsigned pid = bd->tok_pid; - - /* reset values for a next transfer */ - bd->bdt_stall = 0; - bd->dts = 1; - bd->ninc = 0; - bd->keep = 0; - /* Update the odd variable to prepare for the next transfer */ - ep->odd = odd ^ 1; - - int pipenum = ep->pipenum; - int next_pipenum; - // TU_LOG1("TOKDNE %x PID %x pipe %d\r\n", s, pid, pipenum); - - xfer_result_t result; - switch (pid) { - default: - if (continue_transfer(pipenum, bd)) - return; - result = XFER_RESULT_SUCCESS; - break; - case TOK_PID_NAK: - suspend_transfer(pipenum, bd); - next_pipenum = select_next_pipenum(pipenum); - if (0 <= next_pipenum) - resume_transfer(next_pipenum); - return; - case TOK_PID_STALL: - result = XFER_RESULT_STALLED; - break; - case TOK_PID_ERR: /* mismatch toggle bit */ - case TOK_PID_BUSTO: - result = XFER_RESULT_FAILED; - break; - } - _hcd.in_progress &= ~TU_BIT(pipenum); - pipe_state_t *pipe = &_hcd.pipe[ep->pipenum]; - hcd_event_xfer_complete(pipe->dev_addr, - tu_edpt_addr(KHCI->TOKEN & USB_TOKEN_TOKENENDPT_MASK, dir_in), - pipe->length - pipe->remaining, - result, true); - next_pipenum = select_next_pipenum(pipenum); - if (0 <= next_pipenum) - resume_transfer(next_pipenum); -} - -static void process_attach(uint8_t rhport) -{ - unsigned ctl = KHCI->CTL; - if (!(ctl & USB_CTL_JSTATE_MASK)) { - /* The attached device is a low speed device. */ - KHCI->ADDR = USB_ADDR_LSEN_MASK; - KHCI->ENDPOINT[0].ENDPT = USB_ENDPT_HOSTWOHUB_MASK; - } - hcd_event_device_attach(rhport, true); -} - -static void process_bus_reset(uint8_t rhport) -{ - KHCI->ISTAT = USB_ISTAT_TOKDNE_MASK; - KHCI->USBCTRL &= ~USB_USBCTRL_SUSP_MASK; - KHCI->CTL &= ~USB_CTL_USBENSOFEN_MASK; - KHCI->ADDR = 0; - KHCI->ENDPOINT[0].ENDPT = 0; - - hcd_event_device_remove(rhport, true); - - _hcd.in_progress = 0; - _hcd.pending = 0; - buffer_descriptor_t *bd = &_hcd.bdt[0][0]; - for (unsigned i = 0; i < 2; ++i, ++bd) { - bd->head = 0; - } -} - -/*------------------------------------------------------------------*/ -/* Host API - *------------------------------------------------------------------*/ -bool hcd_init(uint8_t rhport, const tusb_rhport_init_t* rh_init) { - (void) rhport; - (void) rh_init; - KHCI->USBTRC0 |= USB_USBTRC0_USBRESET_MASK; - while (KHCI->USBTRC0 & USB_USBTRC0_USBRESET_MASK); - - tu_memclr(&_hcd, sizeof(_hcd)); - KHCI->USBTRC0 |= TU_BIT(6); /* software must set this bit to 1 */ - KHCI->BDTPAGE1 = (uint8_t)((uintptr_t)_hcd.bdt >> 8); - KHCI->BDTPAGE2 = (uint8_t)((uintptr_t)_hcd.bdt >> 16); - KHCI->BDTPAGE3 = (uint8_t)((uintptr_t)_hcd.bdt >> 24); - - KHCI->USBCTRL &= ~USB_USBCTRL_SUSP_MASK; - KHCI->CTL |= USB_CTL_ODDRST_MASK; - for (unsigned i = 0; i < 16; ++i) { - KHCI->ENDPOINT[i].ENDPT = 0; - } - KHCI->CTL &= ~USB_CTL_ODDRST_MASK; - - KHCI->SOFTHLD = 74; /* for 64-byte packets */ - // KHCI->SOFTHLD = 144; /* for low speed 8-byte packets */ - KHCI->CTL = USB_CTL_HOSTMODEEN_MASK | USB_CTL_SE0_MASK; - KHCI->USBCTRL = USB_USBCTRL_PDE_MASK; - - NVIC_ClearPendingIRQ(USB0_IRQn); - KHCI->INTEN = USB_INTEN_ATTACHEN_MASK | USB_INTEN_TOKDNEEN_MASK | - USB_INTEN_USBRSTEN_MASK | USB_INTEN_ERROREN_MASK | USB_INTEN_STALLEN_MASK; - KHCI->ERREN = 0xff; - - return true; -} - -void hcd_int_enable(uint8_t rhport) -{ - (void)rhport; - NVIC_EnableIRQ(USB0_IRQn); -} - -void hcd_int_disable(uint8_t rhport) -{ - (void)rhport; - NVIC_DisableIRQ(USB0_IRQn); -} - -uint32_t hcd_frame_number(uint8_t rhport) -{ - (void)rhport; - /* The device must be reset at least once after connection - * in order to start the frame counter. */ - if (_hcd.need_reset) hcd_port_reset(rhport); - uint32_t frmnum = KHCI->FRMNUML; - frmnum |= KHCI->FRMNUMH << 8u; - return frmnum; -} - -/*--------------------------------------------------------------------+ - * Port API - *--------------------------------------------------------------------+ */ -bool hcd_port_connect_status(uint8_t rhport) -{ - (void)rhport; - if (KHCI->ISTAT & USB_ISTAT_ATTACH_MASK) - return true; - return false; -} - -void hcd_port_reset(uint8_t rhport) -{ - (void)rhport; - KHCI->CTL &= ~USB_CTL_USBENSOFEN_MASK; - KHCI->CTL |= USB_CTL_RESET_MASK; - unsigned cnt = SystemCoreClock / 100; - while (cnt--) __NOP(); - KHCI->CTL &= ~USB_CTL_RESET_MASK; - KHCI->CTL |= USB_CTL_USBENSOFEN_MASK; - _hcd.need_reset = false; -} - -void hcd_port_reset_end(uint8_t rhport) { - (void) rhport; -} - -tusb_speed_t hcd_port_speed_get(uint8_t rhport) -{ - (void)rhport; - tusb_speed_t speed = TUSB_SPEED_FULL; - const unsigned ie = NVIC_GetEnableIRQ(USB0_IRQn); - NVIC_DisableIRQ(USB0_IRQn); - if (KHCI->ADDR & USB_ADDR_LSEN_MASK) - speed = TUSB_SPEED_LOW; - if (ie) NVIC_EnableIRQ(USB0_IRQn); - return speed; -} - -void hcd_device_close(uint8_t rhport, uint8_t dev_addr) -{ - (void)rhport; - const unsigned ie = NVIC_GetEnableIRQ(USB0_IRQn); - NVIC_DisableIRQ(USB0_IRQn); - pipe_state_t *p = &_hcd.pipe[0]; - pipe_state_t *end = &_hcd.pipe[CFG_TUH_ENDPOINT_MAX * 2]; - for (;p != end; ++p) { - if (p->dev_addr == dev_addr) - tu_memclr(p, sizeof(*p)); - } - if (ie) NVIC_EnableIRQ(USB0_IRQn); -} - -//--------------------------------------------------------------------+ -// Endpoints API -//--------------------------------------------------------------------+ -bool hcd_setup_send(uint8_t rhport, uint8_t dev_addr, uint8_t const setup_packet[8]) -{ - (void)rhport; - // TU_LOG1("SETUP %u\r\n", dev_addr); - TU_ASSERT(0 == (_hcd.in_progress & TU_BIT(0))); - - int pipenum = find_pipe(dev_addr, 0); - if (pipenum < 0) return false; - - pipe_state_t *pipe = &_hcd.pipe[pipenum]; - pipe[0].data = 0; - pipe[0].buffer = (uint8_t*)(uintptr_t)setup_packet; - pipe[0].length = 8; - pipe[0].remaining = 8; - pipe[1].data = 1; - - if (1 != prepare_packets(pipenum)) - return false; - - _hcd.in_progress |= TU_BIT(pipenum); - - unsigned hostwohub = KHCI->ENDPOINT[0].ENDPT & USB_ENDPT_HOSTWOHUB_MASK; - KHCI->ENDPOINT[0].ENDPT = hostwohub | - USB_ENDPT_EPHSHK_MASK | USB_ENDPT_EPRXEN_MASK | USB_ENDPT_EPTXEN_MASK; - KHCI->ADDR = (KHCI->ADDR & USB_ADDR_LSEN_MASK) | dev_addr; - while (KHCI->CTL & USB_CTL_TXSUSPENDTOKENBUSY_MASK) ; - KHCI->TOKEN = (TOK_PID_SETUP << USB_TOKEN_TOKENPID_SHIFT); - return true; -} - -bool hcd_edpt_open(uint8_t rhport, uint8_t dev_addr, tusb_desc_endpoint_t const * ep_desc) -{ - (void)rhport; - uint8_t const ep_addr = ep_desc->bEndpointAddress; - // TU_LOG1("O %u %x\r\n", dev_addr, ep_addr); - /* Find a free pipe */ - pipe_state_t *p = &_hcd.pipe[0]; - pipe_state_t *end = &_hcd.pipe[CFG_TUH_ENDPOINT_MAX * 2]; - if (dev_addr || ep_addr) { - p += 2; - for (; p < end && (p->dev_addr || p->ep_addr); ++p) ; - if (p == end) return false; - } - p->dev_addr = dev_addr; - p->ep_addr = ep_addr; - p->max_packet_size = ep_desc->wMaxPacketSize; - p->xfer = ep_desc->bmAttributes.xfer; - p->data = 0; - if (!ep_addr) { - /* Open one more pipe for Control IN transfer */ - TU_ASSERT(TUSB_XFER_CONTROL == p->xfer); - pipe_state_t *q = p + 1; - TU_ASSERT(!q->dev_addr && !q->ep_addr); - q->dev_addr = dev_addr; - q->ep_addr = tu_edpt_addr(0, TUSB_DIR_IN); - q->max_packet_size = ep_desc->wMaxPacketSize; - q->xfer = ep_desc->bmAttributes.xfer; - q->data = 1; - } - return true; -} - -bool hcd_edpt_close(uint8_t rhport, uint8_t daddr, uint8_t ep_addr) { - (void) rhport; (void) daddr; (void) ep_addr; - return false; // TODO not implemented yet -} - -/* The address of buffer must be aligned to 4 byte boundary. And it must be at least 4 bytes long. - * DMA writes data in 4 byte unit */ -bool hcd_edpt_xfer(uint8_t rhport, uint8_t dev_addr, uint8_t ep_addr, uint8_t * buffer, uint16_t buflen) -{ - (void)rhport; - // TU_LOG1("X %u %x %x %d\r\n", dev_addr, ep_addr, (uintptr_t)buffer, buflen); - - int pipenum = find_pipe(dev_addr, ep_addr); - TU_ASSERT(0 <= pipenum); - - TU_ASSERT(0 == (_hcd.in_progress & TU_BIT(pipenum))); - unsigned const ie = NVIC_GetEnableIRQ(USB0_IRQn); - NVIC_DisableIRQ(USB0_IRQn); - pipe_state_t *pipe = &_hcd.pipe[pipenum]; - pipe->buffer = buffer; - pipe->length = buflen; - pipe->remaining = buflen; - _hcd.in_progress |= TU_BIT(pipenum); - _hcd.pending |= TU_BIT(pipenum); /* Send at the next Frame */ - KHCI->INTEN |= USB_ISTAT_SOFTOK_MASK; - if (ie) NVIC_EnableIRQ(USB0_IRQn); - return true; -} - -bool hcd_edpt_abort_xfer(uint8_t rhport, uint8_t dev_addr, uint8_t ep_addr) { - (void) rhport; - (void) dev_addr; - (void) ep_addr; - // TODO not implemented yet - return false; -} - -bool hcd_edpt_clear_stall(uint8_t rhport, uint8_t dev_addr, uint8_t ep_addr) { - (void) rhport; - if (!tu_edpt_number(ep_addr)) return true; - int num = find_pipe(dev_addr, ep_addr); - if (num < 0) return false; - pipe_state_t *p = &_hcd.pipe[num]; - p->data = 0; /* Reset data toggle */ - return true; -} - -/*--------------------------------------------------------------------+ - * ISR - *--------------------------------------------------------------------+*/ -void hcd_int_handler(uint8_t rhport, bool in_isr) -{ - (void) in_isr; - uint32_t is = KHCI->ISTAT; - uint32_t msk = KHCI->INTEN; - - // TU_LOG1("S %lx\r\n", is); - - /* clear disabled interrupts */ - KHCI->ISTAT = (is & ~msk & ~USB_ISTAT_TOKDNE_MASK) | USB_ISTAT_SOFTOK_MASK; - is &= msk; - - if (is & USB_ISTAT_ERROR_MASK) { - unsigned err = KHCI->ERRSTAT; - if (err) { - TU_LOG1(" ERR %x\r\n", err); - KHCI->ERRSTAT = err; - } else { - KHCI->INTEN &= ~USB_ISTAT_ERROR_MASK; - } - } - - if (is & USB_ISTAT_USBRST_MASK) { - KHCI->INTEN = (msk & ~USB_INTEN_USBRSTEN_MASK) | USB_INTEN_ATTACHEN_MASK; - process_bus_reset(rhport); - return; - } - if (is & USB_ISTAT_ATTACH_MASK) { - KHCI->INTEN = (msk & ~USB_INTEN_ATTACHEN_MASK) | USB_INTEN_USBRSTEN_MASK; - _hcd.need_reset = true; - process_attach(rhport); - return; - } - if (is & USB_ISTAT_STALL_MASK) { - KHCI->ISTAT = USB_ISTAT_STALL_MASK; - } - if (is & USB_ISTAT_SOFTOK_MASK) { - msk &= ~USB_ISTAT_SOFTOK_MASK; - KHCI->INTEN = msk; - if (_hcd.pending) { - int pipenum = __builtin_ctz(_hcd.pending); - _hcd.pending = 0; - if (!(is & USB_ISTAT_TOKDNE_MASK)) - resume_transfer(pipenum); - } - } - if (is & USB_ISTAT_TOKDNE_MASK) { - process_tokdne(rhport); - } -} - -#endif -- cgit v1.3.1 From 6d4c985c9a2c1d937add144bc70b5b18df33021e Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 10 Jul 2026 00:17:55 +0700 Subject: kinetis_k: non-blocking board_uart_read (RX FIFO) + SIM unique id - board_uart_read was a stub returning 0, so host examples that bridge the UART console to a CDC device (echo test) received nothing. Implement it via an RDRF-interrupt-fed tu_fifo, matching the stm32 family (non-blocking, no RX overrun). board_uart_write is already non-blocking. - implement board_get_unique_id() from the SIM 128-bit UID registers so frdm_k64f/teensy_35 report a real USB serial instead of the fixed default. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01ExGPLP5eU43LR7o6yYLpNi --- hw/bsp/kinetis_k/family.c | 41 +++++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) (limited to 'hw/bsp') diff --git a/hw/bsp/kinetis_k/family.c b/hw/bsp/kinetis_k/family.c index a5af83931..238123f30 100644 --- a/hw/bsp/kinetis_k/family.c +++ b/hw/bsp/kinetis_k/family.c @@ -35,10 +35,25 @@ #include "fsl_clock.h" #include "fsl_uart.h" #include "fsl_sysmpu.h" +#include "common/tusb_fifo.h" #include "board/clock_config.h" #include "board/pin_mux.h" +#ifdef UART_DEV +// RX ring buffer filled by the RDRF interrupt so board_uart_read() is non-blocking +// and does not drop bytes to UART overrun (see stm32 family for reference). +static uint8_t uart_rx_ff_buf[32]; +static tu_fifo_t uart_rx_ff; + +void UART0_RX_TX_IRQHandler(void) { + if (UART_DEV->S1 & UART_S1_RDRF_MASK) { + uint8_t byte = UART_DEV->D; // reading S1 then D clears RDRF (and any overrun) + tu_fifo_write(&uart_rx_ff, &byte); + } +} +#endif + //--------------------------------------------------------------------+ // Forward USB interrupt events to TinyUSB IRQ Handler //--------------------------------------------------------------------+ @@ -89,6 +104,9 @@ void board_init(void) { .enableRx = true }; UART_Init(UART_DEV, &uart_config, UART_CLOCK); + tu_fifo_config(&uart_rx_ff, uart_rx_ff_buf, sizeof(uart_rx_ff_buf), false); + UART_DEV->C2 |= UART_C2_RIE_MASK; // enable RX data register full interrupt + NVIC_EnableIRQ(UART0_RX_TX_IRQn); #endif // USB @@ -112,14 +130,11 @@ uint32_t board_button_read(void) { } int board_uart_read(uint8_t *buf, int len) { - (void) buf; - (void) len; #ifdef UART_DEV - // Read blocking will block until there is data -// UART_ReadBlocking(UART_DEV, buf, len); -// return len; - return 0; + return (int) tu_fifo_read_n(&uart_rx_ff, buf, (uint16_t) len); #else + (void) buf; + (void) len; return 0; #endif } @@ -144,6 +159,20 @@ int board_uart_write(void const *buf, int len) { #endif } +size_t board_get_unique_id(uint8_t id[], size_t max_len) { + (void) max_len; + // Kinetis 128-bit Unique Identification Register (SIM->UIDH/UIDMH/UIDML/UIDL) + uint32_t* id32 = (uint32_t*) (uintptr_t) id; + uint8_t const len = 16; + + id32[0] = SIM->UIDH; + id32[1] = SIM->UIDMH; + id32[2] = SIM->UIDML; + id32[3] = SIM->UIDL; + + return len; +} + #if CFG_TUSB_OS == OPT_OS_NONE volatile uint32_t system_ticks = 0; -- cgit v1.3.1 From 4782770e7f5f4a726f9739d940c110196807f15c Mon Sep 17 00:00:00 2001 From: hathach Date: Sat, 18 Jul 2026 00:18:27 +0700 Subject: fix(ci_fs): address code-review findings in host/device drivers Host (hcd_ci_fs.c): - Release the speculatively-armed sibling BDT on the NAK path (IN only) as well as on completion, so a NAKed multi-packet IN no longer leaks a BDT that stays own=1 and blocks every same-direction pipe. Both paths now go through a single release_sibling_bd() helper (was a copy-pasted disarm). - Clear the ENTIRE shared BDT (both directions) on bus reset; clearing only the IN half left a stale OUT/SETUP descriptor after a disconnect mid-OUT, blocking the first control transfer on re-enumeration. - Size bda[] to span the whole BDT (2*2*4) so STAT-indexed access is within the declared array bounds (was out-of-declared-bounds, benign via union). Shared (ci_fs_type.h): - Hoist buffer_descriptor_t and the TOK_PID enum out of the device and host drivers into the shared header so the identical definitions cannot drift. Board (kinetis_k): - Drop a redundant local in board_get_unique_id. Build-verified: host + kinetis k/kl/k32l + MCX. HIL: frdm_k64f host 2/2 (cdc_msc_hid + device_info); frdm_kl25z device core suite green with the relocated definitions. --- hw/bsp/kinetis_k/family.c | 3 +- src/portable/chipidea/ci_fs/ci_fs_type.h | 54 ++++++++++++++++++++ src/portable/chipidea/ci_fs/dcd_ci_fs.c | 38 +------------- src/portable/chipidea/ci_fs/hcd_ci_fs.c | 88 ++++++++++++-------------------- 4 files changed, 88 insertions(+), 95 deletions(-) (limited to 'hw/bsp') diff --git a/hw/bsp/kinetis_k/family.c b/hw/bsp/kinetis_k/family.c index 238123f30..2df3313b8 100644 --- a/hw/bsp/kinetis_k/family.c +++ b/hw/bsp/kinetis_k/family.c @@ -163,14 +163,13 @@ size_t board_get_unique_id(uint8_t id[], size_t max_len) { (void) max_len; // Kinetis 128-bit Unique Identification Register (SIM->UIDH/UIDMH/UIDML/UIDL) uint32_t* id32 = (uint32_t*) (uintptr_t) id; - uint8_t const len = 16; id32[0] = SIM->UIDH; id32[1] = SIM->UIDMH; id32[2] = SIM->UIDML; id32[3] = SIM->UIDL; - return len; + return 16; } #if CFG_TUSB_OS == OPT_OS_NONE diff --git a/src/portable/chipidea/ci_fs/ci_fs_type.h b/src/portable/chipidea/ci_fs/ci_fs_type.h index a525c96ca..857a75253 100644 --- a/src/portable/chipidea/ci_fs/ci_fs_type.h +++ b/src/portable/chipidea/ci_fs/ci_fs_type.h @@ -27,6 +27,60 @@ extern "C" { // align 4 is used to get rid of reserved fields #define _va32 volatile TU_ATTR_ALIGNED(4) +//--------------------------------------------------------------------+ +// Buffer Descriptor Table (BDT) - shared by the device (dcd) and host (hcd) drivers +// since both target the same ChipIdea-FS silicon. Keep the layout in one place so a +// fix cannot silently drift between the two drivers. +//--------------------------------------------------------------------+ + +// Token PID values reported in the BDT tok_pid field / written to the TOKEN register. +// The device driver only uses OUT/IN/SETUP; the rest are host-only. +enum { + TOK_PID_OUT = 0x1u, + TOK_PID_IN = 0x9u, + TOK_PID_SETUP = 0xDu, + TOK_PID_DATA0 = 0x3u, + TOK_PID_DATA1 = 0xbu, + TOK_PID_ACK = 0x2u, + TOK_PID_STALL = 0xeu, + TOK_PID_NAK = 0xau, + TOK_PID_BUSTO = 0x0u, + TOK_PID_ERR = 0xfu, +}; + +// Note: this header is included before the CMSIS device header, so use plain `volatile` +// rather than CMSIS `__IO`. +typedef struct TU_ATTR_PACKED +{ + union { + uint32_t head; + struct { + union { + struct { + uint16_t : 2; + volatile uint16_t tok_pid : 4; + uint16_t data : 1; + volatile uint16_t own : 1; + uint16_t : 8; + }; + struct { + uint16_t : 2; + uint16_t bdt_stall : 1; + uint16_t dts : 1; + uint16_t ninc : 1; + uint16_t keep : 1; + uint16_t : 10; + }; + }; + volatile uint16_t bc : 10; + uint16_t : 6; + }; + }; + uint8_t *addr; +}buffer_descriptor_t; + +TU_VERIFY_STATIC( sizeof(buffer_descriptor_t) == 8, "size is not correct" ); + typedef struct { _va32 uint8_t PER_ID; // [00] Peripheral ID register _va32 uint8_t ID_COMP; // [04] Peripheral ID complement register diff --git a/src/portable/chipidea/ci_fs/dcd_ci_fs.c b/src/portable/chipidea/ci_fs/dcd_ci_fs.c index b03670551..295f2e578 100644 --- a/src/portable/chipidea/ci_fs/dcd_ci_fs.c +++ b/src/portable/chipidea/ci_fs/dcd_ci_fs.c @@ -24,43 +24,7 @@ //--------------------------------------------------------------------+ // MACRO TYPEDEF CONSTANT ENUM DECLARATION //--------------------------------------------------------------------+ - -enum { - TOK_PID_OUT = 0x1u, - TOK_PID_IN = 0x9u, - TOK_PID_SETUP = 0xDu, -}; - -typedef struct TU_ATTR_PACKED -{ - union { - uint32_t head; - struct { - union { - struct { - uint16_t : 2; - __IO uint16_t tok_pid : 4; - uint16_t data : 1; - __IO uint16_t own : 1; - uint16_t : 8; - }; - struct { - uint16_t : 2; - uint16_t bdt_stall : 1; - uint16_t dts : 1; - uint16_t ninc : 1; - uint16_t keep : 1; - uint16_t : 10; - }; - }; - __IO uint16_t bc : 10; - uint16_t : 6; - }; - }; - uint8_t *addr; -}buffer_descriptor_t; - -TU_VERIFY_STATIC( sizeof(buffer_descriptor_t) == 8, "size is not correct" ); +// TOK_PID_* and buffer_descriptor_t are shared with the host driver in ci_fs_type.h typedef struct TU_ATTR_PACKED { diff --git a/src/portable/chipidea/ci_fs/hcd_ci_fs.c b/src/portable/chipidea/ci_fs/hcd_ci_fs.c index 5c5d81521..a6f5405b5 100644 --- a/src/portable/chipidea/ci_fs/hcd_ci_fs.c +++ b/src/portable/chipidea/ci_fs/hcd_ci_fs.c @@ -31,50 +31,7 @@ //--------------------------------------------------------------------+ // MACRO TYPEDEF CONSTANT ENUM DECLARATION //--------------------------------------------------------------------+ - -enum { - TOK_PID_OUT = 0x1u, - TOK_PID_IN = 0x9u, - TOK_PID_SETUP = 0xDu, - TOK_PID_DATA0 = 0x3u, - TOK_PID_DATA1 = 0xbu, - TOK_PID_ACK = 0x2u, - TOK_PID_STALL = 0xeu, - TOK_PID_NAK = 0xau, - TOK_PID_BUSTO = 0x0u, - TOK_PID_ERR = 0xfu, -}; - -typedef struct TU_ATTR_PACKED -{ - union { - uint32_t head; - struct { - union { - struct { - uint16_t : 2; - __IO uint16_t tok_pid : 4; - uint16_t data : 1; - __IO uint16_t own : 1; - uint16_t : 8; - }; - struct { - uint16_t : 2; - uint16_t bdt_stall : 1; - uint16_t dts : 1; - uint16_t ninc : 1; - uint16_t keep : 1; - uint16_t : 10; - }; - }; - __IO uint16_t bc : 10; - uint16_t : 6; - }; - }; - uint8_t *addr; -}buffer_descriptor_t; - -TU_VERIFY_STATIC( sizeof(buffer_descriptor_t) == 8, "size is not correct" ); +// TOK_PID_* and buffer_descriptor_t are shared with the device driver in ci_fs_type.h typedef struct TU_ATTR_PACKED { @@ -115,7 +72,10 @@ typedef struct union { /* [OUT,IN][EVEN,ODD] */ buffer_descriptor_t bdt[2][2]; - uint16_t bda[2*2]; + /* bda aliases bdt for STAT-register indexing: STAT gives the byte-offset/2 of the + * completed BD, so it indexes bda[] in uint16_t units. Each buffer_descriptor_t is + * 4 uint16_t, hence 2*2*4 elements to span the whole table (must equal sizeof bdt). */ + uint16_t bda[2*2*4]; }; endpoint_state_t endpoint[2]; pipe_state_t pipe[CFG_TUH_ENDPOINT_MAX * 2]; @@ -282,6 +242,22 @@ static void suspend_transfer(int pipenum, buffer_descriptor_t *bd) } } +// Release the speculatively-armed sibling BDT of a multi-packet transfer. +// prepare_packets arms the sibling (odd^1) BDT (own=1) so a multi-packet transfer can +// ping-pong without NAKs. When the transfer ends - completes early on a short IN packet, +// stalls/errors, or (for IN) is NAKed before the sibling's token is issued - that sibling +// is left owned by the SIE. Because the host shares ONE BDT set across all pipes, a +// leftover armed sibling blocks every other pipe forever (e.g. a 2nd device stuck +// enumerating behind a hub). Release it - but ONLY for a multi-packet transfer: a +// single-packet transfer never armed a sibling, so that BDT slot may legitimately belong +// to another pipe's in-flight transfer. +static inline void release_sibling_bd(unsigned s, const pipe_state_t *pipe) +{ + if (pipe->length > pipe->max_packet_size) { + ((buffer_descriptor_t *)&_hcd.bda[s ^ USB_STAT_ODD_MASK])->own = 0; + } +} + static void process_tokdne(uint8_t rhport) { (void)rhport; @@ -316,6 +292,12 @@ static void process_tokdne(uint8_t rhport) result = XFER_RESULT_SUCCESS; break; case TOK_PID_NAK: + // Release the speculatively-armed sibling so the deferred retry (and any other pipe + // sharing the single BDT) can claim it; otherwise it stays own=1 forever and every + // same-direction transfer wedges. IN only: an IN issues just one token so the sibling + // was never put on the wire, whereas an OUT issues both tokens and its sibling may + // still be in flight - touching it there would race the SIE write-back. + if (TUSB_DIR_IN == dir_in) release_sibling_bd(s, &_hcd.pipe[pipenum]); suspend_transfer(pipenum, bd); next_pipenum = select_next_pipenum(pipenum); if (0 <= next_pipenum) @@ -331,16 +313,7 @@ static void process_tokdne(uint8_t rhport) } _hcd.in_progress &= ~TU_BIT(pipenum); pipe_state_t *pipe = &_hcd.pipe[ep->pipenum]; - /* A multi-packet transfer speculatively arms the sibling (odd^1) BDT (see - * prepare_packets) to ping-pong without NAKs. When it ends early (a short IN packet) - * or fails, that sibling is still owned by the SIE; since the host shares a single - * BDT set across all pipes, a leftover armed sibling blocks every other pipe forever - * (e.g. a 2nd device stuck enumerating behind a hub). Release it - but ONLY for a - * multi-packet transfer: a single-packet transfer never armed a sibling, so that - * BDT slot may legitimately belong to another pipe's in-flight transfer. */ - if (pipe->length > pipe->max_packet_size) { - ((buffer_descriptor_t *)&_hcd.bda[s ^ USB_STAT_ODD_MASK])->own = 0; - } + release_sibling_bd(s, pipe); hcd_event_xfer_complete(pipe->dev_addr, tu_edpt_addr(CI_REG->TOKEN & USB_TOKEN_TOKENENDPT_MASK, dir_in), pipe->length - pipe->remaining, @@ -373,8 +346,11 @@ static void process_bus_reset(uint8_t rhport) _hcd.in_progress = 0; _hcd.pending = 0; + // Clear the ENTIRE shared BDT (both directions, both even/odd). Clearing only the IN + // pair left a stale OUT/SETUP descriptor (own=1) after a disconnect mid-OUT, which then + // blocks the first control transfer on re-enumeration. buffer_descriptor_t *bd = &_hcd.bdt[0][0]; - for (unsigned i = 0; i < 2; ++i, ++bd) { + for (unsigned i = 0; i < 2 * 2; ++i, ++bd) { bd->head = 0; } } -- cgit v1.3.1 From 127dd2ca26dcba18d14e745bec4c1a69b55a2c01 Mon Sep 17 00:00:00 2001 From: Jie Feng Date: Wed, 10 Jun 2026 18:25:08 +0800 Subject: add py32f0 support --- .gitignore | 1 + .../py32f0/boards/py32f071_dev_board/board.cmake | 9 ++ hw/bsp/py32f0/boards/py32f071_dev_board/board.h | 75 ++++++++++++ hw/bsp/py32f0/boards/py32f071_dev_board/board.mk | 8 ++ .../boards/py32f071_dev_board/py32f071_hal_conf.h | 105 ++++++++++++++++ .../py32f0/boards/py32f071_dev_board/py32f071xb.ld | 122 +++++++++++++++++++ hw/bsp/py32f0/family.c | 133 +++++++++++++++++++++ hw/bsp/py32f0/family.cmake | 97 +++++++++++++++ hw/bsp/py32f0/family.mk | 57 +++++++++ src/common/tusb_mcu.h | 11 ++ src/portable/mentor/musb/dcd_musb.c | 51 ++++++-- src/portable/mentor/musb/musb_max32.h | 1 + src/portable/mentor/musb/musb_py32.h | 79 ++++++++++++ src/portable/mentor/musb/musb_ti.h | 1 + src/portable/mentor/musb/musb_type.h | 71 +++++++++++ src/tusb_option.h | 13 +- tools/get_deps.py | 6 + 17 files changed, 829 insertions(+), 11 deletions(-) create mode 100644 hw/bsp/py32f0/boards/py32f071_dev_board/board.cmake create mode 100644 hw/bsp/py32f0/boards/py32f071_dev_board/board.h create mode 100644 hw/bsp/py32f0/boards/py32f071_dev_board/board.mk create mode 100644 hw/bsp/py32f0/boards/py32f071_dev_board/py32f071_hal_conf.h create mode 100644 hw/bsp/py32f0/boards/py32f071_dev_board/py32f071xb.ld create mode 100644 hw/bsp/py32f0/family.c create mode 100644 hw/bsp/py32f0/family.cmake create mode 100644 hw/bsp/py32f0/family.mk create mode 100644 src/portable/mentor/musb/musb_py32.h (limited to 'hw/bsp') diff --git a/.gitignore b/.gitignore index 145069e72..ca745ee19 100644 --- a/.gitignore +++ b/.gitignore @@ -81,6 +81,7 @@ hw/mcu/gd/ hw/mcu/hpmicro/ hw/mcu/infineon/ hw/mcu/microchip/ +hw/mcu/puya/ hw/mcu/mindmotion/ hw/mcu/nordic/nrfx/ hw/mcu/nuvoton/ diff --git a/hw/bsp/py32f0/boards/py32f071_dev_board/board.cmake b/hw/bsp/py32f0/boards/py32f071_dev_board/board.cmake new file mode 100644 index 000000000..6c9c58e3d --- /dev/null +++ b/hw/bsp/py32f0/boards/py32f071_dev_board/board.cmake @@ -0,0 +1,9 @@ +set(PYOCD_TARGET py32f071ex8) +set(LD_FILE_GNU ${CMAKE_CURRENT_LIST_DIR}/py32f071xb.ld) + +function(update_board TARGET) + target_compile_definitions(${TARGET} PUBLIC + PY32F071xB + CFG_EXAMPLE_VIDEO_READONLY + ) +endfunction() diff --git a/hw/bsp/py32f0/boards/py32f071_dev_board/board.h b/hw/bsp/py32f0/boards/py32f071_dev_board/board.h new file mode 100644 index 000000000..ddd67bb6a --- /dev/null +++ b/hw/bsp/py32f0/boards/py32f071_dev_board/board.h @@ -0,0 +1,75 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2026, Ha Thach (tinyusb.org) + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * This file is part of the TinyUSB stack. + */ + +#ifndef BOARD_H_ +#define BOARD_H_ + +#ifdef __cplusplus + extern "C" { +#endif + +#define LED_PORT GPIOB +#define LED_PIN GPIO_PIN_2 +#define LED_STATE_ON 0 + +#define BUTTON_PORT GPIOB +#define BUTTON_PIN GPIO_PIN_0 +#define BUTTON_STATE_ACTIVE 0 + +static inline void board_py32f0_clock_init(void) { + RCC_OscInitTypeDef RCC_OscInitStruct = {0}; + RCC_ClkInitTypeDef RCC_ClkInitStruct = {0}; + + RCC_OscInitStruct.OscillatorType = RCC_OSCILLATORTYPE_HSE | RCC_OSCILLATORTYPE_HSI | + RCC_OSCILLATORTYPE_LSI | RCC_OSCILLATORTYPE_LSE; + RCC_OscInitStruct.HSIState = RCC_HSI_ON; + RCC_OscInitStruct.HSIDiv = RCC_HSI_DIV1; + RCC_OscInitStruct.HSICalibrationValue = RCC_HSICALIBRATION_16MHz; + RCC_OscInitStruct.HSEState = RCC_HSE_ON; + RCC_OscInitStruct.HSEFreq = RCC_HSE_16_32MHz; + RCC_OscInitStruct.LSIState = RCC_LSI_OFF; + RCC_OscInitStruct.LSEState = RCC_LSE_OFF; + RCC_OscInitStruct.PLL.PLLState = RCC_PLL_ON; + RCC_OscInitStruct.PLL.PLLSource = RCC_PLLSOURCE_HSE; + RCC_OscInitStruct.PLL.PLLMUL = RCC_PLL_MUL2; + if (HAL_RCC_OscConfig(&RCC_OscInitStruct) != HAL_OK) { + while (1) {} + } + + RCC_ClkInitStruct.ClockType = RCC_CLOCKTYPE_HCLK | RCC_CLOCKTYPE_SYSCLK | RCC_CLOCKTYPE_PCLK1; + RCC_ClkInitStruct.SYSCLKSource = RCC_SYSCLKSOURCE_PLLCLK; + RCC_ClkInitStruct.AHBCLKDivider = RCC_SYSCLK_DIV1; + RCC_ClkInitStruct.APB1CLKDivider = RCC_HCLK_DIV1; + if (HAL_RCC_ClockConfig(&RCC_ClkInitStruct, FLASH_LATENCY_1) != HAL_OK) { + while (1) {} + } +} + +#ifdef __cplusplus + } +#endif + +#endif diff --git a/hw/bsp/py32f0/boards/py32f071_dev_board/board.mk b/hw/bsp/py32f0/boards/py32f071_dev_board/board.mk new file mode 100644 index 000000000..61a2c578c --- /dev/null +++ b/hw/bsp/py32f0/boards/py32f071_dev_board/board.mk @@ -0,0 +1,8 @@ +CFLAGS += \ + -DPY32F071xB \ + -DCFG_EXAMPLE_VIDEO_READONLY + +LD_FILE = $(BOARD_PATH)/py32f071xb.ld +PYOCD_TARGET = py32f071ex8 + +flash: flash-pyocd diff --git a/hw/bsp/py32f0/boards/py32f071_dev_board/py32f071_hal_conf.h b/hw/bsp/py32f0/boards/py32f071_dev_board/py32f071_hal_conf.h new file mode 100644 index 000000000..e513bf8cb --- /dev/null +++ b/hw/bsp/py32f0/boards/py32f071_dev_board/py32f071_hal_conf.h @@ -0,0 +1,105 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2026, Ha Thach (tinyusb.org) + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * This file is part of the TinyUSB stack. + */ + +#ifndef PY32F071_HAL_CONF_H_ +#define PY32F071_HAL_CONF_H_ + +#ifdef __cplusplus + extern "C" { +#endif + +#define HAL_MODULE_ENABLED +#define HAL_RCC_MODULE_ENABLED +#define HAL_FLASH_MODULE_ENABLED +#define HAL_GPIO_MODULE_ENABLED +#define HAL_PWR_MODULE_ENABLED +#define HAL_CORTEX_MODULE_ENABLED + +#if !defined(HSI_VALUE) + #define HSI_VALUE ((uint32_t) 8000000) +#endif + +#if !defined(HSE_VALUE) + #define HSE_VALUE ((uint32_t) 24000000) +#endif + +#if !defined(HSE_STARTUP_TIMEOUT) + #define HSE_STARTUP_TIMEOUT ((uint32_t) 200) +#endif + +#if !defined(LSI_VALUE) + #define LSI_VALUE ((uint32_t) 32768) +#endif + +#if !defined(LSE_VALUE) + #define LSE_VALUE ((uint32_t) 32768) +#endif + +#if !defined(LSE_STARTUP_TIMEOUT) + #define LSE_STARTUP_TIMEOUT ((uint32_t) 5000) +#endif + +#define VDD_VALUE ((uint32_t) 3300) +#define TICK_INT_PRIORITY ((uint32_t) 3) +#define USE_RTOS 0 +#define PREFETCH_ENABLE 0 + +#ifdef HAL_MODULE_ENABLED + #include "py32f0xx_hal.h" +#endif + +#ifdef HAL_RCC_MODULE_ENABLED + #include "py32f071_hal_rcc.h" +#endif + +#ifdef HAL_FLASH_MODULE_ENABLED + #include "py32f071_hal_flash.h" +#endif + +#ifdef HAL_GPIO_MODULE_ENABLED + #include "py32f071_hal_gpio.h" +#endif + +#ifdef HAL_PWR_MODULE_ENABLED + #include "py32f071_hal_pwr.h" +#endif + +#ifdef HAL_CORTEX_MODULE_ENABLED + #include "py32f071_hal_cortex.h" +#endif + +#ifdef USE_FULL_ASSERT +void assert_failed(uint8_t *file, uint32_t line); + #define assert_param(expr) ((expr) ? (void) 0U : assert_failed((uint8_t *) __FILE__, __LINE__)) +#else + #define assert_param(expr) ((void) 0U) +#endif + +#ifdef __cplusplus + } +#endif + +#endif diff --git a/hw/bsp/py32f0/boards/py32f071_dev_board/py32f071xb.ld b/hw/bsp/py32f0/boards/py32f071_dev_board/py32f071xb.ld new file mode 100644 index 000000000..bb6b88d2c --- /dev/null +++ b/hw/bsp/py32f0/boards/py32f071_dev_board/py32f071xb.ld @@ -0,0 +1,122 @@ +ENTRY(Reset_Handler) + +_estack = ORIGIN(RAM) + LENGTH(RAM); +_Min_Heap_Size = 0x200; +_Min_Stack_Size = 0x400; + +MEMORY +{ + RAM (xrw) : ORIGIN = 0x20000000, LENGTH = 16K + FLASH (rx) : ORIGIN = 0x08000000, LENGTH = 128K +} + +SECTIONS +{ + .isr_vector : + { + . = ALIGN(4); + KEEP(*(.isr_vector)) + . = ALIGN(4); + } >FLASH + + .text : + { + . = ALIGN(4); + *(.text) + *(.text*) + *(.glue_7) + *(.glue_7t) + *(.eh_frame) + KEEP(*(.init)) + KEEP(*(.fini)) + . = ALIGN(4); + _etext = .; + } >FLASH + + .rodata : + { + . = ALIGN(4); + *(.rodata) + *(.rodata*) + . = ALIGN(4); + } >FLASH + + .ARM.extab : + { + *(.ARM.extab* .gnu.linkonce.armextab.*) + } >FLASH + + .ARM : + { + __exidx_start = .; + *(.ARM.exidx*) + __exidx_end = .; + } >FLASH + + .preinit_array : + { + PROVIDE_HIDDEN(__preinit_array_start = .); + KEEP(*(.preinit_array*)) + PROVIDE_HIDDEN(__preinit_array_end = .); + } >FLASH + + .init_array : + { + PROVIDE_HIDDEN(__init_array_start = .); + KEEP(*(SORT(.init_array.*))) + KEEP(*(.init_array*)) + PROVIDE_HIDDEN(__init_array_end = .); + } >FLASH + + .fini_array : + { + PROVIDE_HIDDEN(__fini_array_start = .); + KEEP(*(SORT(.fini_array.*))) + KEEP(*(.fini_array*)) + PROVIDE_HIDDEN(__fini_array_end = .); + } >FLASH + + _sidata = LOADADDR(.data); + + .data : + { + . = ALIGN(4); + _sdata = .; + *(.data) + *(.data*) + . = ALIGN(4); + _edata = .; + } >RAM AT> FLASH + + . = ALIGN(4); + .bss : + { + _sbss = .; + __bss_start__ = _sbss; + *(.bss) + *(.bss*) + *(COMMON) + . = ALIGN(4); + _ebss = .; + __bss_end__ = _ebss; + } >RAM + + ._user_heap_stack : + { + . = ALIGN(8); + PROVIDE(end = .); + PROVIDE(_end = .); + . = . + _Min_Heap_Size; + . = . + _Min_Stack_Size; + . = ALIGN(8); + } >RAM + + /DISCARD/ : + { + libc.a(*) + libm.a(*) + libgcc.a(*) + } + + .ARM.attributes 0 : { *(.ARM.attributes) } +} diff --git a/hw/bsp/py32f0/family.c b/hw/bsp/py32f0/family.c new file mode 100644 index 000000000..1428b1fa9 --- /dev/null +++ b/hw/bsp/py32f0/family.c @@ -0,0 +1,133 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2026, Ha Thach (tinyusb.org) + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * This file is part of the TinyUSB stack. + */ + +/* metadata: + manufacturer: Puya +*/ + +#include "py32f0xx_hal.h" +#include "bsp/board_api.h" +#include "board.h" + +void USB_IRQHandler(void) { + tud_int_handler(0); +} + +void USBD_IRQHandler(void) { + tud_int_handler(0); +} + +void HAL_MspInit(void) { + __HAL_RCC_SYSCFG_CLK_ENABLE(); + __HAL_RCC_PWR_CLK_ENABLE(); +} + +void board_init(void) { + HAL_Init(); + board_py32f0_clock_init(); + + __HAL_RCC_SYSCFG_CLK_ENABLE(); + __HAL_RCC_PWR_CLK_ENABLE(); + __HAL_RCC_GPIOA_CLK_ENABLE(); + __HAL_RCC_GPIOB_CLK_ENABLE(); + +#if CFG_TUSB_OS == OPT_OS_NONE + SysTick_Config(SystemCoreClock / 1000); +#elif CFG_TUSB_OS == OPT_OS_FREERTOS + SysTick->CTRL &= ~1U; + NVIC_SetPriority(USB_IRQn, configLIBRARY_MAX_SYSCALL_INTERRUPT_PRIORITY); +#endif + + GPIO_InitTypeDef GPIO_InitStruct; + + GPIO_InitStruct.Pin = LED_PIN; + GPIO_InitStruct.Mode = GPIO_MODE_OUTPUT_PP; + GPIO_InitStruct.Pull = GPIO_PULLUP; + GPIO_InitStruct.Speed = GPIO_SPEED_FREQ_HIGH; + HAL_GPIO_Init(LED_PORT, &GPIO_InitStruct); + board_led_write(false); + + GPIO_InitStruct.Pin = BUTTON_PIN; + GPIO_InitStruct.Mode = GPIO_MODE_INPUT; + GPIO_InitStruct.Pull = GPIO_PULLUP; + GPIO_InitStruct.Speed = GPIO_SPEED_FREQ_HIGH; + HAL_GPIO_Init(BUTTON_PORT, &GPIO_InitStruct); + + __HAL_RCC_USB_CLK_ENABLE(); +} + +void board_led_write(bool state) { + GPIO_PinState pin_state = (GPIO_PinState) (state ? LED_STATE_ON : (1 - LED_STATE_ON)); + HAL_GPIO_WritePin(LED_PORT, LED_PIN, pin_state); +} + +uint32_t board_button_read(void) { + return BUTTON_STATE_ACTIVE == HAL_GPIO_ReadPin(BUTTON_PORT, BUTTON_PIN); +} + +size_t board_get_unique_id(uint8_t id[], size_t max_len) { + (void) max_len; + volatile uint32_t * py32_uuid = (volatile uint32_t *) UID_BASE; + uint32_t* id32 = (uint32_t*) (uintptr_t) id; + uint8_t const len = 12; + + id32[0] = py32_uuid[0]; + id32[1] = py32_uuid[1]; + id32[2] = py32_uuid[2]; + + return len; +} + +int board_uart_read(uint8_t *buf, int len) { + (void) buf; (void) len; + return 0; +} + +int board_uart_write(void const *buf, int len) { + (void) buf; (void) len; + return -1; +} + +#if CFG_TUSB_OS == OPT_OS_NONE +volatile uint32_t system_ticks = 0; + +void SysTick_Handler(void) { + HAL_IncTick(); + system_ticks++; +} + +uint32_t tusb_time_millis_api(void) { + return system_ticks; +} +#endif + +void HardFault_Handler(void) { + __asm("BKPT #0\n"); +} + +void _init(void); +void _init(void) { +} diff --git a/hw/bsp/py32f0/family.cmake b/hw/bsp/py32f0/family.cmake new file mode 100644 index 000000000..2416ace88 --- /dev/null +++ b/hw/bsp/py32f0/family.cmake @@ -0,0 +1,97 @@ +include_guard() + +include(${CMAKE_CURRENT_LIST_DIR}/boards/${BOARD}/board.cmake) + +if (NOT DEFINED PY32_SDK_NAME) + set(PY32_SDK_NAME PY32F071_Firmware) +endif () +if (NOT DEFINED PY32_SERIES) + set(PY32_SERIES PY32F071) +endif () +if (NOT DEFINED PY32_TEMPLATE) + set(PY32_TEMPLATE PY32F071xx_Templates) +endif () +if (NOT DEFINED PY32_STARTUP) + set(PY32_STARTUP startup_py32f071xx.s) +endif () +if (NOT DEFINED PY32_SYSTEM_SOURCE) + set(PY32_SYSTEM_SOURCE system_py32f071.c) +endif () +if (NOT DEFINED PY32_HAL_PREFIX) + set(PY32_HAL_PREFIX py32f071) +endif () +set(PY32_SDK ${TOP}/hw/mcu/puya/${PY32_SDK_NAME}) +set(PY32_HAL ${PY32_SDK}/Drivers/${PY32_SERIES}_HAL_Driver) +set(PY32_CMSIS ${PY32_SDK}/Drivers/CMSIS/Device/${PY32_SERIES}) + +set(CMAKE_SYSTEM_CPU cortex-m0plus CACHE INTERNAL "System Processor") +set(CMAKE_TOOLCHAIN_FILE ${TOP}/examples/build_system/cmake/toolchain/arm_${TOOLCHAIN}.cmake) + +set(FAMILY_MCUS PY32F0 CACHE INTERNAL "") + +set(STARTUP_FILE_GNU ${PY32_SDK}/Templates/${PY32_TEMPLATE}/EIDE/${PY32_STARTUP}) +set(STARTUP_FILE_Clang ${STARTUP_FILE_GNU}) +set(LD_FILE_Clang ${LD_FILE_GNU}) + +function(family_add_board BOARD_TARGET) + add_library(${BOARD_TARGET} STATIC + ${PY32_SDK}/Templates/${PY32_TEMPLATE}/Src/${PY32_SYSTEM_SOURCE} + ${PY32_HAL}/Src/${PY32_HAL_PREFIX}_hal.c + ${PY32_HAL}/Src/${PY32_HAL_PREFIX}_hal_cortex.c + ${PY32_HAL}/Src/${PY32_HAL_PREFIX}_hal_flash.c + ${PY32_HAL}/Src/${PY32_HAL_PREFIX}_hal_gpio.c + ${PY32_HAL}/Src/${PY32_HAL_PREFIX}_hal_pwr.c + ${PY32_HAL}/Src/${PY32_HAL_PREFIX}_hal_rcc.c + ${PY32_HAL}/Src/${PY32_HAL_PREFIX}_hal_rcc_ex.c + ) + target_include_directories(${BOARD_TARGET} PUBLIC + ${CMAKE_CURRENT_FUNCTION_LIST_DIR} + ${PY32_SDK}/Drivers/CMSIS/Include + ${PY32_CMSIS}/Include + ${PY32_HAL}/Inc + ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/boards/${BOARD} + ) + target_compile_definitions(${BOARD_TARGET} PRIVATE + USE_HAL_DRIVER + ) + update_board(${BOARD_TARGET}) +endfunction() + +function(family_configure_example TARGET RTOS) + family_configure_common(${TARGET} ${RTOS}) + family_add_tinyusb(${TARGET} OPT_MCU_PY32F0) + + target_sources(${TARGET} PUBLIC + ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/family.c + ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/../board.c + ${TOP}/src/portable/mentor/musb/dcd_musb.c + ${STARTUP_FILE_${CMAKE_C_COMPILER_ID}} + ) + target_include_directories(${TARGET} PUBLIC + ${CMAKE_CURRENT_FUNCTION_LIST_DIR} + ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/../../ + ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/boards/${BOARD} + ) + + if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + target_link_options(${TARGET} PUBLIC + "LINKER:--script=${LD_FILE_GNU}" + -nostartfiles + --specs=nosys.specs --specs=nano.specs + ) + elseif (CMAKE_C_COMPILER_ID STREQUAL "Clang") + message(FATAL_ERROR "Clang is not supported") + elseif (CMAKE_C_COMPILER_ID STREQUAL "IAR") + message(FATAL_ERROR "IAR is not supported") + endif () + + if (CMAKE_C_COMPILER_ID STREQUAL "GNU" OR CMAKE_C_COMPILER_ID STREQUAL "Clang") + set_source_files_properties(${CMAKE_CURRENT_FUNCTION_LIST_DIR}/family.c PROPERTIES COMPILE_FLAGS "-Wno-missing-prototypes") + endif () + set_source_files_properties(${STARTUP_FILE_${CMAKE_C_COMPILER_ID}} PROPERTIES + SKIP_LINTING ON + COMPILE_OPTIONS -w) + + family_add_bin_hex(${TARGET}) + family_flash_pyocd(${TARGET}) +endfunction() diff --git a/hw/bsp/py32f0/family.mk b/hw/bsp/py32f0/family.mk new file mode 100644 index 000000000..7ddfea5f7 --- /dev/null +++ b/hw/bsp/py32f0/family.mk @@ -0,0 +1,57 @@ +UF2_FAMILY_ID = 0x0 + +include $(TOP)/$(BOARD_PATH)/board.mk + +PY32_SDK_NAME ?= PY32F071_Firmware +PY32_SERIES ?= PY32F071 +PY32_TEMPLATE ?= PY32F071xx_Templates +PY32_STARTUP ?= startup_py32f071xx.s +PY32_SYSTEM_SOURCE ?= system_py32f071.c +PY32_HAL_PREFIX ?= py32f071 + +SDK_DIR = hw/mcu/puya/$(PY32_SDK_NAME) +HAL_DIR = $(SDK_DIR)/Drivers/$(PY32_SERIES)_HAL_Driver +CMSIS_DIR = $(SDK_DIR)/Drivers/CMSIS +CMSIS_DEVICE_DIR = $(CMSIS_DIR)/Device/$(PY32_SERIES) + +DEPS_SUBMODULES += $(SDK_DIR) + +CFLAGS += \ + -DCFG_TUSB_MCU=OPT_MCU_PY32F0 \ + -DUSE_HAL_DRIVER + +# Puya HAL leaves some CMSIS-compatible callback parameters intentionally unused. +CFLAGS += -Wno-error=unused-parameter + +MCU_DIR = $(SDK_DIR) + +SRC_C += \ + src/portable/mentor/musb/dcd_musb.c \ + hw/bsp/py32f0/family.c \ + $(SDK_DIR)/Templates/$(PY32_TEMPLATE)/Src/$(PY32_SYSTEM_SOURCE) \ + $(HAL_DIR)/Src/$(PY32_HAL_PREFIX)_hal.c \ + $(HAL_DIR)/Src/$(PY32_HAL_PREFIX)_hal_cortex.c \ + $(HAL_DIR)/Src/$(PY32_HAL_PREFIX)_hal_flash.c \ + $(HAL_DIR)/Src/$(PY32_HAL_PREFIX)_hal_gpio.c \ + $(HAL_DIR)/Src/$(PY32_HAL_PREFIX)_hal_pwr.c \ + $(HAL_DIR)/Src/$(PY32_HAL_PREFIX)_hal_rcc.c \ + $(HAL_DIR)/Src/$(PY32_HAL_PREFIX)_hal_rcc_ex.c + +SRC_S += $(SDK_DIR)/Templates/$(PY32_TEMPLATE)/EIDE/$(PY32_STARTUP) + +INC += \ + $(TOP)/hw/bsp/py32f0 \ + $(TOP)/$(CMSIS_DIR)/Include \ + $(TOP)/$(CMSIS_DEVICE_DIR)/Include \ + $(TOP)/$(HAL_DIR)/Inc \ + $(TOP)/hw/bsp/py32f0/boards/$(BOARD) + +SKIP_NANOLIB = 1 + +LDFLAGS += \ + -nostdlib -nostartfiles \ + --specs=nosys.specs --specs=nano.specs \ + -Wl,--gc-sections \ + -Wl,--print-memory-usage + +CPU_CORE ?= cortex-m0plus diff --git a/src/common/tusb_mcu.h b/src/common/tusb_mcu.h index 0959ac3a9..d599f89fb 100644 --- a/src/common/tusb_mcu.h +++ b/src/common/tusb_mcu.h @@ -703,6 +703,17 @@ #define TU_ATTR_FAST_FUNC __attribute__((section(".fast"))) +//--------------------------------------------------------------------+ +// Puya +//--------------------------------------------------------------------+ +#elif TU_CHECK_MCU(OPT_MCU_PY32F0) + #define TUP_USBIP_MUSB + #define TUP_USBIP_MUSB_PY32 + #define TUP_DCD_ENDPOINT_MAX 6 + // PY32 shares the buffer between IN and OUT of the same endpoint number. + // Possible to share IN/OUT if only one direction is armed at any one time + #define CFG_TUD_ENDPOINT_ONE_DIRECTION_ONLY 1 + #endif // External USB controller diff --git a/src/portable/mentor/musb/dcd_musb.c b/src/portable/mentor/musb/dcd_musb.c index 17993f23a..45c6fe7df 100644 --- a/src/portable/mentor/musb/dcd_musb.c +++ b/src/portable/mentor/musb/dcd_musb.c @@ -24,6 +24,8 @@ #include "musb_ti.h" #elif defined(TUP_USBIP_MUSB_ADI) #include "musb_max32.h" +#elif defined(TUP_USBIP_MUSB_PY32) + #include "musb_py32.h" #else #error "Unsupported MCU" #endif @@ -45,6 +47,7 @@ typedef struct { }; uint16_t length; /* the number of bytes in the buffer */ uint16_t remaining; /* the number of bytes remaining in the buffer */ + uint16_t mps; /* maximum packet size */ bool armed; /* true while a transfer is posted */ bool use_fifo; /* true: buf is tu_fifo_t*; false: buf is plain byte pointer. */ } pipe_state_t; @@ -159,6 +162,14 @@ TU_ATTR_ALWAYS_INLINE static inline pipe_state_t* pipe_get(uint8_t epnum, tusb_d return &_dcd.pipe[idx]; } +TU_ATTR_ALWAYS_INLINE static inline uint16_t musb_mps_to_maxp(uint16_t mps) { +#if defined(TUP_USBIP_MUSB_PY32) + return (uint8_t) ((mps + 7u) / 8u); +#else + return mps; +#endif +} + //-------------------------------------------------------------------- // HW FIFO Helper // Note: Index register is already set by caller @@ -223,7 +234,9 @@ TU_ATTR_ALWAYS_INLINE static inline bool hwfifo_config(musb_regs_t* musb, unsign bool double_packet) { (void) mps; - #if defined(TUP_USBIP_MUSB_ADI) + #if defined(TUP_USBIP_MUSB_PY32) + (void) musb; (void) epnum; (void) is_rx; (void) double_packet; + #elif defined(TUP_USBIP_MUSB_ADI) // AnalogDevice FIFO sizes: EP1..7 = 512 B, EP8..9 = 2048 B, EP10..11 = 4096 B. // DPB requires FIFO >= 2 * MPS. For HS bulk (MPS=512) only EP >= 8 qualifies. // Force single-buffered on EP < 8 even if the caller requested DPB. @@ -266,7 +279,7 @@ TU_ATTR_ALWAYS_INLINE static inline void hwfifo_flush(musb_regs_t* musb, unsigne // write to txfifo using pipe_state_t info static void pipe_write(musb_regs_t* musb_regs, pipe_state_t* pipe, uint8_t epnum) { musb_ep_csr_t* ep_csr = &musb_regs->indexed_csr; - const uint16_t mps = ep_csr->tx_maxp & MUSB_TXMAXP_PACKET_SIZE_M; + const uint16_t mps = pipe->mps; const uint16_t xact_len = tu_min16(mps, pipe->remaining); volatile void *hwfifo = &musb_regs->fifo[epnum]; if (xact_len) { @@ -320,7 +333,7 @@ static void process_epin_isr(uint8_t rhport, musb_regs_t *musb_regs, uint8_t epn // release the FIFO slot by clearing RXRDY. return true if short packet static bool pipe_read(musb_regs_t* musb_regs, pipe_state_t* pipe, uint8_t epnum) { musb_ep_csr_t* ep_csr = &musb_regs->indexed_csr; // index already set in process_epout_isr() - const uint16_t mps = ep_csr->rx_maxp & MUSB_RXMAXP_PACKET_SIZE_M; + const uint16_t mps = pipe->mps; const uint16_t rx_count = ep_csr->rx_count; const uint16_t xact_len = tu_min16(tu_min16(pipe->remaining, mps), rx_count); volatile void *hwfifo = &musb_regs->fifo[epnum]; @@ -632,7 +645,11 @@ static void process_bus_reset_isr(uint8_t rhport) { hwfifo_reset(musb, i, 0); hwfifo_reset(musb, i, 1); } +#if defined(TUP_USBIP_MUSB_PY32) + dcd_event_bus_reset(rhport, TUSB_SPEED_FULL, true); +#else dcd_event_bus_reset(rhport, (musb->power & MUSB_POWER_HSMODE) ? TUSB_SPEED_HIGH : TUSB_SPEED_FULL, true); +#endif } /*------------------------------------------------------------------ @@ -641,6 +658,11 @@ static void process_bus_reset_isr(uint8_t rhport) { #if CFG_TUSB_DEBUG >= MUSB_DEBUG static void print_musb_info(musb_regs_t* musb_regs) { +#if defined(TUP_USBIP_MUSB_PY32) + (void) musb_regs; + // musb discovery fields not present + TU_LOG1("musb py32 fixed full-speed configuration\r\n"); +#else // print version, epinfo, raminfo, config_data0, fifo_size TU_LOG1("musb version = %u.%u\r\n", musb_regs->hwvers_bit.major, musb_regs->hwvers_bit.minor); TU_LOG1("Number of endpoints: %u TX, %u RX\r\n", musb_regs->epinfo_bit.tx_ep_num, musb_regs->epinfo_bit.rx_ep_num); @@ -657,6 +679,7 @@ static void print_musb_info(musb_regs_t* musb_regs) { TU_LOG1("FIFO %u Size: TX %u RX %u\r\n", i, musb_regs->indexed_csr.fifo_size_bit.tx, musb_regs->indexed_csr.fifo_size_bit.rx); } #endif +#endif } #endif @@ -716,15 +739,23 @@ void dcd_remote_wakeup(uint8_t rhport) { void dcd_connect(uint8_t rhport) { musb_regs_t* musb_regs = MUSB_REGS(rhport); +#if defined(TUP_USBIP_MUSB_PY32) + (void) musb_regs; +#else musb_regs->power |= TUD_OPT_HIGH_SPEED ? MUSB_POWER_HSENAB : 0; musb_regs->power |= MUSB_POWER_SOFTCONN; +#endif } // Disconnect by disabling internal pull-up resistor on D+/D- void dcd_disconnect(uint8_t rhport) { musb_regs_t* musb_regs = MUSB_REGS(rhport); +#if defined(TUP_USBIP_MUSB_PY32) + (void) musb_regs; +#else musb_regs->power &= ~MUSB_POWER_SOFTCONN; +#endif } void dcd_sof_enable(uint8_t rhport, bool en) @@ -750,6 +781,7 @@ bool dcd_edpt_open(uint8_t rhport, tusb_desc_endpoint_t const * ep_desc) { pipe->buf = NULL; pipe->length = 0; pipe->remaining = 0; + pipe->mps = (uint16_t) mps; pipe->armed = false; musb_regs_t* musb = MUSB_REGS(rhport); @@ -757,7 +789,7 @@ bool dcd_edpt_open(uint8_t rhport, tusb_desc_endpoint_t const * ep_desc) { const uint8_t is_rx = (1 - epdir); musb_ep_maxp_csr_t* maxp_csr = &ep_csr->maxp_csr[is_rx]; - maxp_csr->maxp = mps; + maxp_csr->maxp = musb_mps_to_maxp((uint16_t) mps); maxp_csr->csrh = 0; #if MUSB_CFG_SHARED_FIFO if (epdir) { @@ -768,7 +800,7 @@ bool dcd_edpt_open(uint8_t rhport, tusb_desc_endpoint_t const * ep_desc) { hwfifo_flush(musb, epn, is_rx, true); TU_ASSERT(hwfifo_config(musb, epn, is_rx, mps, ep_desc->bmAttributes.xfer == TUSB_XFER_BULK)); - musb->intren_ep[is_rx] |= TU_BIT(epn); + musb->intren_ep[is_rx ^ MUSB_INTR_EP_TX_RX_SWAP] |= TU_BIT(epn); return true; } @@ -779,6 +811,8 @@ bool dcd_edpt_iso_alloc(uint8_t rhport, uint8_t ep_addr, uint16_t largest_packet musb_regs_t* musb = MUSB_REGS(rhport); musb_ep_csr_t* ep_csr = get_ep_csr(musb, epn); const uint8_t is_rx = 1 - dir_in; + pipe_state_t *pipe = pipe_get(epn, dir_in); + pipe->mps = largest_packet_size; ep_csr->maxp_csr[is_rx].csrh = 0; return hwfifo_config(musb, epn, is_rx, largest_packet_size, true); } @@ -796,6 +830,7 @@ bool dcd_edpt_iso_activate(uint8_t rhport, tusb_desc_endpoint_t const *ep_desc ) pipe->buf = NULL; pipe->length = 0; pipe->remaining = 0; + pipe->mps = (uint16_t) mps; pipe->armed = false; musb_regs_t* musb = MUSB_REGS(rhport); @@ -803,7 +838,7 @@ bool dcd_edpt_iso_activate(uint8_t rhport, tusb_desc_endpoint_t const *ep_desc ) const uint8_t is_rx = 1 - dir_in; musb_ep_maxp_csr_t* maxp_csr = &ep_csr->maxp_csr[is_rx]; - maxp_csr->maxp = mps; + maxp_csr->maxp = musb_mps_to_maxp((uint16_t) mps); maxp_csr->csrh |= MUSB_CSRH_ISO; #if MUSB_CFG_SHARED_FIFO if (dir_in) { @@ -818,7 +853,7 @@ bool dcd_edpt_iso_activate(uint8_t rhport, tusb_desc_endpoint_t const *ep_desc ) musb->fifo_size[is_rx] = hwfifo_byte2size(mps) | MUSB_FIFOSZ_DOUBLE_PACKET; #endif - musb->intren_ep[is_rx] |= TU_BIT(epn); + musb->intren_ep[is_rx ^ MUSB_INTR_EP_TX_RX_SWAP] |= TU_BIT(epn); if (ie) musb_dcd_int_enable(rhport); @@ -839,7 +874,7 @@ void dcd_edpt_close_all(uint8_t rhport) musb_ep_maxp_csr_t* maxp_csr = &ep_csr->maxp_csr[d]; hwfifo_flush(musb, i, d, true); hwfifo_reset(musb, i, d); - maxp_csr->maxp = 0; + maxp_csr->maxp = musb_mps_to_maxp(0); maxp_csr->csrh = 0; } } diff --git a/src/portable/mentor/musb/musb_max32.h b/src/portable/mentor/musb/musb_max32.h index 628454216..9cf99126e 100644 --- a/src/portable/mentor/musb/musb_max32.h +++ b/src/portable/mentor/musb/musb_max32.h @@ -28,6 +28,7 @@ extern "C" { #define MUSB_CFG_SHARED_FIFO 1 // shared FIFO for TX and RX endpoints #define MUSB_CFG_DYNAMIC_FIFO 0 // dynamic EP FIFO sizing +#define MUSB_INTR_EP_TX_RX_SWAP 0 static const uintptr_t MUSB_BASES[] = { MXC_BASE_USBHS }; diff --git a/src/portable/mentor/musb/musb_py32.h b/src/portable/mentor/musb/musb_py32.h new file mode 100644 index 000000000..68d58ccb8 --- /dev/null +++ b/src/portable/mentor/musb/musb_py32.h @@ -0,0 +1,79 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2026, Ha Thach (tinyusb.org) + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * This file is part of the TinyUSB stack. + */ + +#ifndef TUSB_MUSB_PY32_H_ +#define TUSB_MUSB_PY32_H_ + +#include "py32f0xx.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// PY32 does not expose generic MUSB shared FIFO allocation registers; this +// selects the existing TX_MODE path required for IN endpoints. +#define MUSB_CFG_SHARED_FIFO 1 +#define MUSB_CFG_DYNAMIC_FIFO 0 +#define MUSB_INTR_EP_TX_RX_SWAP 1 + +static const uintptr_t MUSB_BASES[] = { USBD_BASE }; +static const IRQn_Type musb_irqs[] = { USB_IRQn }; + +TU_ATTR_ALWAYS_INLINE static inline void musb_dcd_phy_init(uint8_t rhport) { + musb_regs_t* musb_regs = MUSB_REGS(rhport); + + musb_regs->index = 0; + musb_regs->faddr = 0; + musb_regs->intr_usben = MUSB_IE_RESET | MUSB_IE_RESUME | MUSB_IE_SUSPND; + musb_regs->intr_txen = TU_BIT(0); + musb_regs->intr_rxen = 0; +} + +TU_ATTR_ALWAYS_INLINE static inline void musb_dcd_int_enable(uint8_t rhport) { + NVIC_EnableIRQ(musb_irqs[rhport]); +} + +TU_ATTR_ALWAYS_INLINE static inline void musb_dcd_int_disable(uint8_t rhport) { + NVIC_DisableIRQ(musb_irqs[rhport]); +} + +TU_ATTR_ALWAYS_INLINE static inline void musb_dcd_int_clear(uint8_t rhport) { + NVIC_ClearPendingIRQ(musb_irqs[rhport]); +} + +TU_ATTR_ALWAYS_INLINE static inline unsigned musb_dcd_get_int_enable(uint8_t rhport) { + return NVIC_GetEnableIRQ(musb_irqs[rhport]); +} + +TU_ATTR_ALWAYS_INLINE static inline void musb_dcd_int_handler_enter(uint8_t rhport) { + (void) rhport; +} + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/src/portable/mentor/musb/musb_ti.h b/src/portable/mentor/musb/musb_ti.h index db2e237d2..1cf26b1cb 100644 --- a/src/portable/mentor/musb/musb_ti.h +++ b/src/portable/mentor/musb/musb_ti.h @@ -30,6 +30,7 @@ #define MUSB_CFG_SHARED_FIFO 0 #define MUSB_CFG_DYNAMIC_FIFO 1 #define MUSB_CFG_DYNAMIC_FIFO_SIZE 4096 +#define MUSB_INTR_EP_TX_RX_SWAP 0 static const uintptr_t MUSB_BASES[] = { USB0_BASE }; diff --git a/src/portable/mentor/musb/musb_type.h b/src/portable/mentor/musb/musb_type.h index 3c079cb25..1e8a761c0 100644 --- a/src/portable/mentor/musb/musb_type.h +++ b/src/portable/mentor/musb/musb_type.h @@ -64,6 +64,75 @@ #define __R volatile const #endif +#if defined(TUP_USBIP_MUSB_PY32) + +typedef struct TU_ATTR_PACKED { + __IO uint8_t csrh; // 0x04, 0x08: CSRH + __IO uint8_t csrl; // 0x05, 0x09: CSRL + __IO uint8_t maxp; // 0x06, 0x0A: MAXP + __I uint8_t reserved; +} musb_ep_maxp_csr_t; + +TU_VERIFY_STATIC(sizeof(musb_ep_maxp_csr_t) == 4, "size is not correct"); + +typedef struct TU_ATTR_PACKED { + __IO uint8_t csr0l; // 0x00: CSR0 + __IO uint8_t count0; // 0x01: COUNT0 + __I uint8_t reserved_0x02[2]; + + union { + struct { + __IO uint8_t tx_csrh; // 0x04: TX CSRH + __IO uint8_t tx_csrl; // 0x05: TX CSRL + __IO uint8_t tx_maxp; // 0x06: TX MAXP + __I uint8_t reserved_0x07; + + __IO uint8_t rx_csrh; // 0x08: RX CSRH + __IO uint8_t rx_csrl; // 0x09: RX CSRL + __IO uint8_t rx_maxp; // 0x0A: RX MAXP + __I uint8_t reserved_0x0b; + }; + + musb_ep_maxp_csr_t maxp_csr[2]; + }; + + __IO uint16_t rx_count; // 0x0C: RX COUNT + __I uint8_t reserved_0x0e[2]; +} musb_ep_csr_t; + +TU_VERIFY_STATIC(sizeof(musb_ep_csr_t) == 16, "size is not correct"); + +typedef struct TU_ATTR_PACKED { + __IO uint8_t faddr; // 0x00: FADDR + __IO uint8_t power; // 0x01: POWER + __I uint8_t reserved_0x02[2]; + + __IO uint8_t intr_usb; // 0x04: INTRUSB + __IO uint8_t intr_rx; // 0x05: INTRRX + __IO uint8_t intr_tx; // 0x06: INTRTX + __I uint8_t reserved_0x07; + + __IO uint8_t intr_usben; // 0x08: INTRUSBEN + union { + struct { + __IO uint8_t intr_rxen; // 0x09: INTRRXEN + __IO uint8_t intr_txen; // 0x0A: INTRTXEN + }; + + __IO uint8_t intren_ep[2]; // 0x09-0x0A: RX, TX + }; + __I uint8_t reserved_0x0b; + + __IO uint16_t frame; // 0x0C: FRAME + __IO uint8_t index; // 0x0E: INDEX + __I uint8_t reserved_0x0f; + + musb_ep_csr_t indexed_csr; // 0x10-0x1F: Indexed CSR + __IO uint32_t fifo[16]; // 0x20-0x5F: FIFO 0-15 +} musb_regs_t; + +#else + typedef struct TU_ATTR_PACKED { __IO uint16_t maxp; // 0x00, 0x04: MAXP __IO uint8_t csrl; // 0x02, 0x06: CSRL @@ -277,6 +346,8 @@ typedef struct { TU_VERIFY_STATIC(sizeof(musb_regs_t) == 0x350, "size is not correct"); +#endif + //--------------------------------------------------------------------+ // Helper //--------------------------------------------------------------------+ diff --git a/src/tusb_option.h b/src/tusb_option.h index e19ee1629..0036fbe3c 100644 --- a/src/tusb_option.h +++ b/src/tusb_option.h @@ -104,6 +104,9 @@ #define OPT_MCU_NUC120 802 #define OPT_MCU_NUC505 803 +// Puya +#define OPT_MCU_PY32F0 850 ///< Puya PY32F0 + // Espressif #define OPT_MCU_ESP32S2 900 ///< Espressif ESP32-S2 #define OPT_MCU_ESP32S3 901 ///< Espressif ESP32-S3 @@ -349,9 +352,13 @@ //------------ MUSB --------------// #if defined(TUP_USBIP_MUSB) #define CFG_TUD_EDPT_DEDICATED_HWFIFO 1 - #define CFG_TUSB_FIFO_HWFIFO_DATA_STRIDE 4 // 32 bit data - #define CFG_TUSB_FIFO_HWFIFO_DATA_ODD_16BIT_ACCESS // allow odd 16bit access - #define CFG_TUSB_FIFO_HWFIFO_DATA_ODD_8BIT_ACCESS // allow odd 8bit access + #if defined(TUP_USBIP_MUSB_PY32) + #define CFG_TUSB_FIFO_HWFIFO_DATA_STRIDE 1 // 8 bit data + #else + #define CFG_TUSB_FIFO_HWFIFO_DATA_STRIDE 4 // 32 bit data + #define CFG_TUSB_FIFO_HWFIFO_DATA_ODD_16BIT_ACCESS // allow odd 16bit access + #define CFG_TUSB_FIFO_HWFIFO_DATA_ODD_8BIT_ACCESS // allow odd 8bit access + #endif #define CFG_TUSB_FIFO_HWFIFO_ADDR_STRIDE 0 // fixed hwfifo #endif diff --git a/tools/get_deps.py b/tools/get_deps.py index 9d745ee26..e26810cac 100755 --- a/tools/get_deps.py +++ b/tools/get_deps.py @@ -82,6 +82,12 @@ deps_optional = { 'hw/mcu/nxp/mcux-devices-rt': ['https://github.com/nxp-mcuxpresso/mcux-devices-rt', 'dba2b523c9df61f3330bd186242f8210a8e47c45', 'imxrt'], + 'hw/mcu/puya/PY32F071_Firmware': ['https://github.com/OpenPuya/PY32F071_Firmware.git', + '73e384cddce63e3019de41d9b6af17dfc96fa536', + 'py32f0'], + 'hw/mcu/puya/PY32F072_Firmware': ['https://github.com/OpenPuya/PY32F072_Firmware.git', + 'bc3a6cdbece335a27abb8b51e6fc4911f5116185', + 'py32f0'], 'hw/mcu/raspberry_pi/FreeRTOS-Kernel': ['https://github.com/raspberrypi/FreeRTOS-Kernel.git', '4f7299d6ea746b27a9dd19e87af568e34bd65b15', 'rp2040'], -- cgit v1.3.1 From 6e0f455634126e7eba315ebc279e563aefbf5815 Mon Sep 17 00:00:00 2001 From: Jie Feng Date: Wed, 10 Jun 2026 18:49:07 +0800 Subject: examples now build --- examples/device/net_lwip_webserver/skip.txt | 1 + hw/bsp/py32f0/FreeRTOSConfig/FreeRTOSConfig.h | 108 +++++++++++++++++++++ .../py32f0/boards/py32f071_dev_board/board.cmake | 1 + hw/bsp/py32f0/boards/py32f071_dev_board/board.mk | 1 + 4 files changed, 111 insertions(+) create mode 100644 hw/bsp/py32f0/FreeRTOSConfig/FreeRTOSConfig.h (limited to 'hw/bsp') diff --git a/examples/device/net_lwip_webserver/skip.txt b/examples/device/net_lwip_webserver/skip.txt index 5e5562087..c3df1ee4b 100644 --- a/examples/device/net_lwip_webserver/skip.txt +++ b/examples/device/net_lwip_webserver/skip.txt @@ -10,6 +10,7 @@ mcu:SAMD11 mcu:STM32L0 mcu:STM32F0 mcu:KINETIS_KL +mcu:PY32F0 mcu:STM32H7RS mcu:STM32N6 family:broadcom_64bit diff --git a/hw/bsp/py32f0/FreeRTOSConfig/FreeRTOSConfig.h b/hw/bsp/py32f0/FreeRTOSConfig/FreeRTOSConfig.h new file mode 100644 index 000000000..6b3ae0cd5 --- /dev/null +++ b/hw/bsp/py32f0/FreeRTOSConfig/FreeRTOSConfig.h @@ -0,0 +1,108 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2026, Ha Thach (tinyusb.org) + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * This file is part of the TinyUSB stack. + */ + +#ifndef FREERTOS_CONFIG_H_ +#define FREERTOS_CONFIG_H_ + +#ifndef __IASMARM__ + #include "py32f0xx.h" +#endif + +#define configENABLE_MPU 0 +#define configENABLE_FPU 0 +#define configENABLE_TRUSTZONE 0 +#define configMINIMAL_SECURE_STACK_SIZE 1024 + +#define configUSE_PREEMPTION 1 +#define configUSE_PORT_OPTIMISED_TASK_SELECTION 0 +#define configCPU_CLOCK_HZ SystemCoreClock +#define configTICK_RATE_HZ 1000 +#define configMAX_PRIORITIES 5 +#define configMINIMAL_STACK_SIZE 128 +#define configTOTAL_HEAP_SIZE ( configSUPPORT_DYNAMIC_ALLOCATION * 4 * 1024 ) +#define configMAX_TASK_NAME_LEN 16 +#define configUSE_16_BIT_TICKS 0 +#define configIDLE_SHOULD_YIELD 1 +#define configUSE_MUTEXES 1 +#define configUSE_RECURSIVE_MUTEXES 1 +#define configUSE_COUNTING_SEMAPHORES 1 +#define configQUEUE_REGISTRY_SIZE 4 +#define configUSE_QUEUE_SETS 0 +#define configUSE_TIME_SLICING 0 +#define configUSE_NEWLIB_REENTRANT 0 +#define configENABLE_BACKWARD_COMPATIBILITY 1 +#define configSTACK_ALLOCATION_FROM_SEPARATE_HEAP 0 + +#define configSUPPORT_STATIC_ALLOCATION 1 +#define configSUPPORT_DYNAMIC_ALLOCATION 0 + +#define configUSE_IDLE_HOOK 0 +#define configUSE_TICK_HOOK 0 +#define configUSE_MALLOC_FAILED_HOOK 0 +#define configCHECK_FOR_STACK_OVERFLOW 2 +#define configCHECK_HANDLER_INSTALLATION 0 + +#define configGENERATE_RUN_TIME_STATS 0 +#define configRECORD_STACK_HIGH_ADDRESS 1 +#define configUSE_TRACE_FACILITY 1 +#define configUSE_STATS_FORMATTING_FUNCTIONS 0 + +#define configUSE_CO_ROUTINES 0 +#define configMAX_CO_ROUTINE_PRIORITIES 2 + +#define configUSE_TIMERS 1 +#define configTIMER_TASK_PRIORITY ( configMAX_PRIORITIES - 2 ) +#define configTIMER_QUEUE_LENGTH 32 +#define configTIMER_TASK_STACK_DEPTH configMINIMAL_STACK_SIZE + +#define INCLUDE_vTaskPrioritySet 0 +#define INCLUDE_uxTaskPriorityGet 0 +#define INCLUDE_vTaskDelete 0 +#define INCLUDE_vTaskSuspend 1 +#define INCLUDE_xResumeFromISR 0 +#define INCLUDE_vTaskDelayUntil 1 +#define INCLUDE_vTaskDelay 1 +#define INCLUDE_xTaskGetSchedulerState 0 +#define INCLUDE_xTaskGetCurrentTaskHandle 1 +#define INCLUDE_uxTaskGetStackHighWaterMark 0 +#define INCLUDE_xTaskGetIdleTaskHandle 0 +#define INCLUDE_xTimerGetTimerDaemonTaskHandle 0 +#define INCLUDE_pcTaskGetTaskName 0 +#define INCLUDE_eTaskGetState 0 +#define INCLUDE_xEventGroupSetBitFromISR 0 +#define INCLUDE_xTimerPendFunctionCall 0 + +#define xPortPendSVHandler PendSV_Handler +#define xPortSysTickHandler SysTick_Handler +#define vPortSVCHandler SVC_Handler + +#define configPRIO_BITS __NVIC_PRIO_BITS +#define configLIBRARY_LOWEST_INTERRUPT_PRIORITY ( ( 1 << configPRIO_BITS ) - 1 ) +#define configLIBRARY_MAX_SYSCALL_INTERRUPT_PRIORITY 2 +#define configKERNEL_INTERRUPT_PRIORITY ( configLIBRARY_LOWEST_INTERRUPT_PRIORITY << ( 8 - configPRIO_BITS ) ) +#define configMAX_SYSCALL_INTERRUPT_PRIORITY ( configLIBRARY_MAX_SYSCALL_INTERRUPT_PRIORITY << ( 8 - configPRIO_BITS ) ) + +#endif diff --git a/hw/bsp/py32f0/boards/py32f071_dev_board/board.cmake b/hw/bsp/py32f0/boards/py32f071_dev_board/board.cmake index 6c9c58e3d..194d3e6b7 100644 --- a/hw/bsp/py32f0/boards/py32f071_dev_board/board.cmake +++ b/hw/bsp/py32f0/boards/py32f071_dev_board/board.cmake @@ -4,6 +4,7 @@ set(LD_FILE_GNU ${CMAKE_CURRENT_LIST_DIR}/py32f071xb.ld) function(update_board TARGET) target_compile_definitions(${TARGET} PUBLIC PY32F071xB + CFG_EXAMPLE_MSC_DUAL_READONLY CFG_EXAMPLE_VIDEO_READONLY ) endfunction() diff --git a/hw/bsp/py32f0/boards/py32f071_dev_board/board.mk b/hw/bsp/py32f0/boards/py32f071_dev_board/board.mk index 61a2c578c..14bf27f97 100644 --- a/hw/bsp/py32f0/boards/py32f071_dev_board/board.mk +++ b/hw/bsp/py32f0/boards/py32f071_dev_board/board.mk @@ -1,5 +1,6 @@ CFLAGS += \ -DPY32F071xB \ + -DCFG_EXAMPLE_MSC_DUAL_READONLY \ -DCFG_EXAMPLE_VIDEO_READONLY LD_FILE = $(BOARD_PATH)/py32f071xb.ld -- cgit v1.3.1 From e7458103240919482f4a803f239af4ca0e1a459f Mon Sep 17 00:00:00 2001 From: Jie Feng Date: Wed, 10 Jun 2026 19:16:35 +0800 Subject: probably the right mcu target --- hw/bsp/py32f0/boards/py32f071_dev_board/board.cmake | 2 +- hw/bsp/py32f0/boards/py32f071_dev_board/board.mk | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'hw/bsp') diff --git a/hw/bsp/py32f0/boards/py32f071_dev_board/board.cmake b/hw/bsp/py32f0/boards/py32f071_dev_board/board.cmake index 194d3e6b7..4c0f227ce 100644 --- a/hw/bsp/py32f0/boards/py32f071_dev_board/board.cmake +++ b/hw/bsp/py32f0/boards/py32f071_dev_board/board.cmake @@ -1,4 +1,4 @@ -set(PYOCD_TARGET py32f071ex8) +set(PYOCD_TARGET py32f071xb) set(LD_FILE_GNU ${CMAKE_CURRENT_LIST_DIR}/py32f071xb.ld) function(update_board TARGET) diff --git a/hw/bsp/py32f0/boards/py32f071_dev_board/board.mk b/hw/bsp/py32f0/boards/py32f071_dev_board/board.mk index 14bf27f97..235465d68 100644 --- a/hw/bsp/py32f0/boards/py32f071_dev_board/board.mk +++ b/hw/bsp/py32f0/boards/py32f071_dev_board/board.mk @@ -4,6 +4,6 @@ CFLAGS += \ -DCFG_EXAMPLE_VIDEO_READONLY LD_FILE = $(BOARD_PATH)/py32f071xb.ld -PYOCD_TARGET = py32f071ex8 +PYOCD_TARGET = py32f071xb flash: flash-pyocd -- cgit v1.3.1 From ced3d0fa17ffc76e40b581fbcbf8508894ff0e70 Mon Sep 17 00:00:00 2001 From: Jie Feng Date: Thu, 11 Jun 2026 23:06:56 +0800 Subject: makefile cleanup --- .../py32f0/boards/py32f071_dev_board/board.cmake | 1 + hw/bsp/py32f0/boards/py32f071_dev_board/board.mk | 2 ++ hw/bsp/py32f0/family.cmake | 30 ++++++++-------------- hw/bsp/py32f0/family.mk | 17 ++++++------ 4 files changed, 23 insertions(+), 27 deletions(-) (limited to 'hw/bsp') diff --git a/hw/bsp/py32f0/boards/py32f071_dev_board/board.cmake b/hw/bsp/py32f0/boards/py32f071_dev_board/board.cmake index 4c0f227ce..2ecf476bd 100644 --- a/hw/bsp/py32f0/boards/py32f071_dev_board/board.cmake +++ b/hw/bsp/py32f0/boards/py32f071_dev_board/board.cmake @@ -1,4 +1,5 @@ set(PYOCD_TARGET py32f071xb) +set(PY32_SERIES PY32F071) set(LD_FILE_GNU ${CMAKE_CURRENT_LIST_DIR}/py32f071xb.ld) function(update_board TARGET) diff --git a/hw/bsp/py32f0/boards/py32f071_dev_board/board.mk b/hw/bsp/py32f0/boards/py32f071_dev_board/board.mk index 235465d68..00003f364 100644 --- a/hw/bsp/py32f0/boards/py32f071_dev_board/board.mk +++ b/hw/bsp/py32f0/boards/py32f071_dev_board/board.mk @@ -1,3 +1,5 @@ +PY32_SERIES = PY32F071 + CFLAGS += \ -DPY32F071xB \ -DCFG_EXAMPLE_MSC_DUAL_READONLY \ diff --git a/hw/bsp/py32f0/family.cmake b/hw/bsp/py32f0/family.cmake index 2416ace88..df6dec42b 100644 --- a/hw/bsp/py32f0/family.cmake +++ b/hw/bsp/py32f0/family.cmake @@ -2,27 +2,19 @@ include_guard() include(${CMAKE_CURRENT_LIST_DIR}/boards/${BOARD}/board.cmake) -if (NOT DEFINED PY32_SDK_NAME) - set(PY32_SDK_NAME PY32F071_Firmware) -endif () -if (NOT DEFINED PY32_SERIES) - set(PY32_SERIES PY32F071) -endif () -if (NOT DEFINED PY32_TEMPLATE) - set(PY32_TEMPLATE PY32F071xx_Templates) -endif () -if (NOT DEFINED PY32_STARTUP) - set(PY32_STARTUP startup_py32f071xx.s) -endif () -if (NOT DEFINED PY32_SYSTEM_SOURCE) - set(PY32_SYSTEM_SOURCE system_py32f071.c) -endif () -if (NOT DEFINED PY32_HAL_PREFIX) - set(PY32_HAL_PREFIX py32f071) +if (NOT DEFINED MCU_VARIANT) + set(MCU_VARIANT PY32F071) endif () +string(TOLOWER ${MCU_VARIANT} PY32_SERIES_LOWER) +set(PY32_SDK_NAME ${MCU_VARIANT}_Firmware) +set(PY32_TEMPLATE ${MCU_VARIANT}xx_Templates) +set(PY32_STARTUP startup_${PY32_SERIES_LOWER}xx.s) +set(PY32_SYSTEM_SOURCE system_${PY32_SERIES_LOWER}.c) +set(PY32_HAL_PREFIX ${PY32_SERIES_LOWER}) + set(PY32_SDK ${TOP}/hw/mcu/puya/${PY32_SDK_NAME}) -set(PY32_HAL ${PY32_SDK}/Drivers/${PY32_SERIES}_HAL_Driver) -set(PY32_CMSIS ${PY32_SDK}/Drivers/CMSIS/Device/${PY32_SERIES}) +set(PY32_HAL ${PY32_SDK}/Drivers/${MCU_VARIANT}_HAL_Driver) +set(PY32_CMSIS ${PY32_SDK}/Drivers/CMSIS/Device/${MCU_VARIANT}) set(CMAKE_SYSTEM_CPU cortex-m0plus CACHE INTERNAL "System Processor") set(CMAKE_TOOLCHAIN_FILE ${TOP}/examples/build_system/cmake/toolchain/arm_${TOOLCHAIN}.cmake) diff --git a/hw/bsp/py32f0/family.mk b/hw/bsp/py32f0/family.mk index 7ddfea5f7..b01a9eefa 100644 --- a/hw/bsp/py32f0/family.mk +++ b/hw/bsp/py32f0/family.mk @@ -2,17 +2,18 @@ UF2_FAMILY_ID = 0x0 include $(TOP)/$(BOARD_PATH)/board.mk -PY32_SDK_NAME ?= PY32F071_Firmware -PY32_SERIES ?= PY32F071 -PY32_TEMPLATE ?= PY32F071xx_Templates -PY32_STARTUP ?= startup_py32f071xx.s -PY32_SYSTEM_SOURCE ?= system_py32f071.c -PY32_HAL_PREFIX ?= py32f071 +MCU_VARIANT ?= PY32F071 +PY32_SERIES_LOWER = $(subst PY32F,py32f,$(MCU_VARIANT)) +PY32_SDK_NAME = $(MCU_VARIANT)_Firmware +PY32_TEMPLATE = $(MCU_VARIANT)xx_Templates +PY32_STARTUP = startup_$(PY32_SERIES_LOWER)xx.s +PY32_SYSTEM_SOURCE = system_$(PY32_SERIES_LOWER).c +PY32_HAL_PREFIX = $(PY32_SERIES_LOWER) SDK_DIR = hw/mcu/puya/$(PY32_SDK_NAME) -HAL_DIR = $(SDK_DIR)/Drivers/$(PY32_SERIES)_HAL_Driver +HAL_DIR = $(SDK_DIR)/Drivers/$(MCU_VARIANT)_HAL_Driver CMSIS_DIR = $(SDK_DIR)/Drivers/CMSIS -CMSIS_DEVICE_DIR = $(CMSIS_DIR)/Device/$(PY32_SERIES) +CMSIS_DEVICE_DIR = $(CMSIS_DIR)/Device/$(MCU_VARIANT) DEPS_SUBMODULES += $(SDK_DIR) -- cgit v1.3.1 From 9418aba918d7c4107026c894e863d9ef0ec5db81 Mon Sep 17 00:00:00 2001 From: Jie Feng Date: Thu, 18 Jun 2026 17:28:49 +0800 Subject: misc fixes --- docs/reference/device_issues.rst | 11 +++++++++++ examples/device/cdc_uac2/src/tusb_config.h | 8 +++++--- examples/device/uac2_headset/src/usb_descriptors.c | 5 +++++ hw/bsp/py32f0/boards/py32f071_dev_board/board.cmake | 1 + hw/bsp/py32f0/boards/py32f071_dev_board/board.mk | 1 + src/portable/mentor/musb/dcd_musb.c | 5 ++++- 6 files changed, 27 insertions(+), 4 deletions(-) (limited to 'hw/bsp') diff --git a/docs/reference/device_issues.rst b/docs/reference/device_issues.rst index ae9cd55f1..0850409cb 100644 --- a/docs/reference/device_issues.rst +++ b/docs/reference/device_issues.rst @@ -33,3 +33,14 @@ Reference: `CH32V30X Reference Manual`_ USBFS/USBHS controller chapter Data corruption may occur on isochronous endpoints. Due to the lacking of FIFO for interrupt status registers, later completed transfer will overwrite `INT_ST` and `RX_LEN` register if previous transfer processing is not completed. Other types of transfers are not affected. + +Puya PY32F071/072 +--------------------------------- +**Severity: Very Low** + +Reference: `PY32F07x Reference Manual` USBD chapter + +The USB device controller (MUSB-like) has 5 application endpoints EP1-EP5 with fixed FIFO sizes +shared between IN and OUT of the same endpoint number: EP1 = 512 B, EP2-4 = 128 B, EP5 = 64 B. +This is much lower than the max ISO ep size of 1024 for high EP numbers. +Place large isochronous endpoints on EP1 and size descriptors accordingly. diff --git a/examples/device/cdc_uac2/src/tusb_config.h b/examples/device/cdc_uac2/src/tusb_config.h index 358ff6747..f54a9b606 100644 --- a/examples/device/cdc_uac2/src/tusb_config.h +++ b/examples/device/cdc_uac2/src/tusb_config.h @@ -109,8 +109,10 @@ extern "C" { #define CFG_TUD_AUDIO_FUNC_1_N_FORMATS 2 // Audio format type I specifications -#if defined(__RX__) -#define CFG_TUD_AUDIO_FUNC_1_MAX_SAMPLE_RATE 48000 // 16bit/48kHz is the best quality for Renesas RX +#if defined(__RX__) || (CFG_TUSB_MCU == OPT_MCU_PY32F0) +// RX : 16bit/48kHz is the best quality for Renesas RX +// PY32F0 : 48kHz/16bit keeps ISO packets <= 98 B so both directions fit the fixed EP FIFOs (EP1 512 B, EP2 128 B) +#define CFG_TUD_AUDIO_FUNC_1_MAX_SAMPLE_RATE 48000 #else #define CFG_TUD_AUDIO_FUNC_1_MAX_SAMPLE_RATE 96000 // 24bit/96kHz is the best quality for full-speed, high-speed is needed beyond this #endif @@ -123,7 +125,7 @@ extern "C" { #define CFG_TUD_AUDIO_FUNC_1_FORMAT_1_N_BYTES_PER_SAMPLE_RX 2 #define CFG_TUD_AUDIO_FUNC_1_FORMAT_1_RESOLUTION_RX 16 -#if defined(__RX__) +#if defined(__RX__) || (CFG_TUSB_MCU == OPT_MCU_PY32F0) // 8bit in 8bit slots #define CFG_TUD_AUDIO_FUNC_1_FORMAT_2_N_BYTES_PER_SAMPLE_TX 1 #define CFG_TUD_AUDIO_FUNC_1_FORMAT_2_RESOLUTION_TX 8 diff --git a/examples/device/uac2_headset/src/usb_descriptors.c b/examples/device/uac2_headset/src/usb_descriptors.c index 1615b92ec..27b6c930c 100644 --- a/examples/device/uac2_headset/src/usb_descriptors.c +++ b/examples/device/uac2_headset/src/usb_descriptors.c @@ -98,6 +98,11 @@ uint8_t const * tud_descriptor_device_cb(void) #define EPNUM_AUDIO_OUT 0x0A #define EPNUM_AUDIO_IN 0x0B #define EPNUM_AUDIO_INT 0x01 + #elif TU_CHECK_MCU(OPT_MCU_PY32F0) + // Speaker OUT (196 B) only fits EP1 (512 B FIFO); mic IN (98 B) fits EP2 (128 B) + #define EPNUM_AUDIO_OUT 0x01 + #define EPNUM_AUDIO_IN 0x02 + #define EPNUM_AUDIO_INT 0x03 #else #define EPNUM_AUDIO_IN 0x01 #define EPNUM_AUDIO_OUT 0x02 diff --git a/hw/bsp/py32f0/boards/py32f071_dev_board/board.cmake b/hw/bsp/py32f0/boards/py32f071_dev_board/board.cmake index 2ecf476bd..e759f4af1 100644 --- a/hw/bsp/py32f0/boards/py32f071_dev_board/board.cmake +++ b/hw/bsp/py32f0/boards/py32f071_dev_board/board.cmake @@ -5,6 +5,7 @@ set(LD_FILE_GNU ${CMAKE_CURRENT_LIST_DIR}/py32f071xb.ld) function(update_board TARGET) target_compile_definitions(${TARGET} PUBLIC PY32F071xB + CFG_EXAMPLE_MSC_READONLY CFG_EXAMPLE_MSC_DUAL_READONLY CFG_EXAMPLE_VIDEO_READONLY ) diff --git a/hw/bsp/py32f0/boards/py32f071_dev_board/board.mk b/hw/bsp/py32f0/boards/py32f071_dev_board/board.mk index 00003f364..e7fbc338d 100644 --- a/hw/bsp/py32f0/boards/py32f071_dev_board/board.mk +++ b/hw/bsp/py32f0/boards/py32f071_dev_board/board.mk @@ -2,6 +2,7 @@ PY32_SERIES = PY32F071 CFLAGS += \ -DPY32F071xB \ + -DCFG_EXAMPLE_MSC_READONLY \ -DCFG_EXAMPLE_MSC_DUAL_READONLY \ -DCFG_EXAMPLE_VIDEO_READONLY diff --git a/src/portable/mentor/musb/dcd_musb.c b/src/portable/mentor/musb/dcd_musb.c index 69b7ef0de..7d79a089a 100644 --- a/src/portable/mentor/musb/dcd_musb.c +++ b/src/portable/mentor/musb/dcd_musb.c @@ -235,7 +235,10 @@ TU_ATTR_ALWAYS_INLINE static inline bool hwfifo_config(musb_regs_t* musb, unsign (void) mps; #if defined(TUP_USBIP_MUSB_PY32) - (void) musb; (void) epnum; (void) is_rx; (void) double_packet; + (void) musb; (void) epnum; (void) is_rx; (void) mps; (void) double_packet; + // Puya FIFO sizes: EP0 = 64 B, EP1 = 512 B, EP2..4 = 128 B, EP5 = 64 B, shared between IN and OUT. + //static const uint16_t py32_fifo_size[] = { 64, 512, 128, 128, 128, 64 }; + //TU_VERIFY(epnum < TU_ARRAY_SIZE(py32_fifo_size) && mps <= py32_fifo_size[epnum]); #elif defined(TUP_USBIP_MUSB_ADI) // AnalogDevice FIFO sizes: EP1..7 = 512 B, EP8..9 = 2048 B, EP10..11 = 4096 B. // DPB requires FIFO >= 2 * MPS. For HS bulk (MPS=512) only EP >= 8 qualifies. -- cgit v1.3.1 From c60e6005faa469adbb583f489d4a9c7b6c3c2f5c Mon Sep 17 00:00:00 2001 From: Jie Feng Date: Sat, 2 May 2026 12:42:07 +0800 Subject: Add support for APM32F072 --- .../boards/apm32f072_dev_board/board.cmake | 8 ++ .../apm32f0xx/boards/apm32f072_dev_board/board.h | 49 +++++++ .../apm32f0xx/boards/apm32f072_dev_board/board.mk | 7 + hw/bsp/apm32f0xx/family.c | 145 +++++++++++++++++++++ hw/bsp/apm32f0xx/family.cmake | 84 ++++++++++++ hw/bsp/apm32f0xx/family.mk | 39 ++++++ src/common/tusb_mcu.h | 8 ++ src/portable/st/stm32_fsdev/fsdev_apm32.h | 77 +++++++++++ src/portable/st/stm32_fsdev/fsdev_common.h | 2 + src/tusb_option.h | 3 + 10 files changed, 422 insertions(+) create mode 100644 hw/bsp/apm32f0xx/boards/apm32f072_dev_board/board.cmake create mode 100644 hw/bsp/apm32f0xx/boards/apm32f072_dev_board/board.h create mode 100644 hw/bsp/apm32f0xx/boards/apm32f072_dev_board/board.mk create mode 100644 hw/bsp/apm32f0xx/family.c create mode 100644 hw/bsp/apm32f0xx/family.cmake create mode 100644 hw/bsp/apm32f0xx/family.mk create mode 100644 src/portable/st/stm32_fsdev/fsdev_apm32.h (limited to 'hw/bsp') diff --git a/hw/bsp/apm32f0xx/boards/apm32f072_dev_board/board.cmake b/hw/bsp/apm32f0xx/boards/apm32f072_dev_board/board.cmake new file mode 100644 index 000000000..33148dbd4 --- /dev/null +++ b/hw/bsp/apm32f0xx/boards/apm32f072_dev_board/board.cmake @@ -0,0 +1,8 @@ +set(MCU_VARIANT APM32F072xB) +set(MCU_LINKER_NAME APM32F07xxB) + +set(JLINK_DEVICE APM32F072RB) + +function(update_board TARGET) + target_compile_definitions(${TARGET} PUBLIC ${MCU_VARIANT}) +endfunction() diff --git a/hw/bsp/apm32f0xx/boards/apm32f072_dev_board/board.h b/hw/bsp/apm32f0xx/boards/apm32f072_dev_board/board.h new file mode 100644 index 000000000..2c9e567e5 --- /dev/null +++ b/hw/bsp/apm32f0xx/boards/apm32f072_dev_board/board.h @@ -0,0 +1,49 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2024, Ha Thach (tinyusb.org) + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * This file is part of the TinyUSB stack. + */ + +/* metadata: + name: APM32F072 Dev Board + url: https://www.geehy.com +*/ + +#ifndef BOARD_H_ +#define BOARD_H_ + +#ifdef __cplusplus + extern "C" { +#endif + +// LED +#define LED_PORT GPIOC +#define LED_PIN GPIO_PIN_13 +#define LED_STATE_ON 0 +#define LED_GPIO_CLK_EN() RCM_EnableAHBPeriphClock(RCM_AHB_PERIPH_GPIOC) + +#ifdef __cplusplus + } +#endif + +#endif /* BOARD_H_ */ diff --git a/hw/bsp/apm32f0xx/boards/apm32f072_dev_board/board.mk b/hw/bsp/apm32f0xx/boards/apm32f072_dev_board/board.mk new file mode 100644 index 000000000..2e5df9947 --- /dev/null +++ b/hw/bsp/apm32f0xx/boards/apm32f072_dev_board/board.mk @@ -0,0 +1,7 @@ +MCU_VARIANT = APM32F072xB +MCU_LINKER_NAME = APM32F07xxB + +JLINK_DEVICE = APM32F072RB + +CFLAGS += \ + -D${MCU_VARIANT} diff --git a/hw/bsp/apm32f0xx/family.c b/hw/bsp/apm32f0xx/family.c new file mode 100644 index 000000000..cbc427f8c --- /dev/null +++ b/hw/bsp/apm32f0xx/family.c @@ -0,0 +1,145 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2019 Ha Thach (tinyusb.org) + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * This file is part of the TinyUSB stack. + */ + +/* metadata: + manufacturer: Geehy +*/ + +#include "apm32f0xx.h" +#include "apm32f0xx_rcm.h" +#include "apm32f0xx_gpio.h" +#include "apm32f0xx_misc.h" +#include "apm32f0xx_crs.h" +#include "bsp/board_api.h" +#include "board.h" + +//--------------------------------------------------------------------+ +// Forward USB interrupt events to TinyUSB IRQ Handler +//--------------------------------------------------------------------+ +void USBD_IRQHandler(void) { + tud_int_handler(0); +} + +//--------------------------------------------------------------------+ +// Board Init +//--------------------------------------------------------------------+ +void board_init(void) { + // Enable HSI48 for USB clock + RCM_EnableHSI48(); + while (RCM_ReadStatusFlag(RCM_FLAG_HSI48RDY) == RESET) {} + + // Select HSI48 as USB clock source + RCM_ConfigUSBCLK(RCM_USBCLK_HSI48); + + // Enable CRS for automatic HSI48 calibration from USB SOF + RCM_EnableAPB1PeriphClock(RCM_APB1_PERIPH_CRS); + CRS_ConfigSynchronizationSource(CRS_SYNC_SOURCE_USB); + CRS_EnableAutomaticCalibration(); + CRS_EnableFrequencyErrorCounter(); + + // Enable USB peripheral clock + RCM_EnableAPB1PeriphClock(RCM_APB1_PERIPH_USB); + + // SysTick 1ms tick + SysTick_Config(SystemCoreClock / 1000); + + // LED + LED_GPIO_CLK_EN(); + GPIO_Config_T gpio_config; + GPIO_ConfigStructInit(&gpio_config); + gpio_config.pin = LED_PIN; + gpio_config.mode = GPIO_MODE_OUT; + gpio_config.outtype = GPIO_OUT_TYPE_PP; + gpio_config.speed = GPIO_SPEED_50MHz; + GPIO_Config(LED_PORT, &gpio_config); + + board_led_write(false); +} + +//--------------------------------------------------------------------+ +// Board porting API +//--------------------------------------------------------------------+ +void board_led_write(bool state) { + if (state ^ (!LED_STATE_ON)) { + GPIO_SetBit(LED_PORT, LED_PIN); + } else { + GPIO_ClearBit(LED_PORT, LED_PIN); + } +} + +uint32_t board_button_read(void) { + return 0; +} + +size_t board_get_unique_id(uint8_t id[], size_t max_len) { + (void) max_len; + volatile uint32_t *apm32_uuid = ((volatile uint32_t *) 0x1FFFF7AC); + uint32_t *id32 = (uint32_t *) (uintptr_t) id; + uint8_t const len = 12; + + id32[0] = apm32_uuid[0]; + id32[1] = apm32_uuid[1]; + id32[2] = apm32_uuid[2]; + + return len; +} + +int board_uart_read(uint8_t *buf, int len) { + (void) buf; + (void) len; + return 0; +} + +int board_uart_write(void const *buf, int len) { + (void) buf; + (void) len; + return 0; +} + +#if CFG_TUSB_OS == OPT_OS_NONE +volatile uint32_t system_ticks = 0; + +void SysTick_Handler(void) { + system_ticks++; +} + +uint32_t tusb_time_millis_api(void) { + return system_ticks; +} + +void SVC_Handler(void) { +} + +void PendSV_Handler(void) { +} +#endif + +void HardFault_Handler(void) { + __asm("BKPT #0\n"); +} + +void _init(void) { +} diff --git a/hw/bsp/apm32f0xx/family.cmake b/hw/bsp/apm32f0xx/family.cmake new file mode 100644 index 000000000..99a94a7a8 --- /dev/null +++ b/hw/bsp/apm32f0xx/family.cmake @@ -0,0 +1,84 @@ +include_guard() + +set(APM32_FAMILY apm32f0xx) +set(APM32_SDK ${TOP}/hw/mcu/geehy/APM32F0xx_SDK_V1.8.6/Libraries) + +# include board specific +include(${CMAKE_CURRENT_LIST_DIR}/boards/${BOARD}/board.cmake) + +# toolchain set up +set(CMAKE_SYSTEM_CPU cortex-m0plus CACHE INTERNAL "System Processor") +set(CMAKE_TOOLCHAIN_FILE ${TOP}/examples/build_system/cmake/toolchain/arm_${TOOLCHAIN}.cmake) + +set(FAMILY_MCUS APM32F0XX CACHE INTERNAL "") + +#------------------------------------ +# Startup & Linker script +#------------------------------------ +set(STARTUP_FILE_GNU ${APM32_SDK}/Device/Geehy/APM32F0xx/Source/gcc/startup_apm32f072.S) +set(STARTUP_FILE_Clang ${STARTUP_FILE_GNU}) +if (NOT DEFINED LD_FILE_GNU) +set(LD_FILE_GNU ${APM32_SDK}/Device/Geehy/APM32F0xx/Source/gcc/gcc_${MCU_LINKER_NAME}.ld) +endif () +set(LD_FILE_Clang ${LD_FILE_GNU}) + +#------------------------------------ +# BOARD_TARGET +#------------------------------------ +function(family_add_board BOARD_TARGET) + add_library(${BOARD_TARGET} STATIC + ${APM32_SDK}/Device/Geehy/APM32F0xx/Source/system_apm32f0xx.c + ${APM32_SDK}/APM32F0xx_StdPeriphDriver/src/apm32f0xx_gpio.c + ${APM32_SDK}/APM32F0xx_StdPeriphDriver/src/apm32f0xx_misc.c + ${APM32_SDK}/APM32F0xx_StdPeriphDriver/src/apm32f0xx_rcm.c + ${APM32_SDK}/APM32F0xx_StdPeriphDriver/src/apm32f0xx_crs.c + ) + target_include_directories(${BOARD_TARGET} PUBLIC + ${CMAKE_CURRENT_FUNCTION_LIST_DIR} + ${APM32_SDK}/CMSIS/Include + ${APM32_SDK}/Device/Geehy/APM32F0xx/Include + ${APM32_SDK}/APM32F0xx_StdPeriphDriver/inc + ) + + update_board(${BOARD_TARGET}) +endfunction() + +#------------------------------------ +# Functions +#------------------------------------ +function(family_configure_example TARGET RTOS) + family_configure_common(${TARGET} ${RTOS}) + family_add_tinyusb(${TARGET} OPT_MCU_APM32F0XX) + + target_sources(${TARGET} PUBLIC + ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/family.c + ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/../board.c + ${TOP}/src/portable/st/stm32_fsdev/dcd_stm32_fsdev.c + ${TOP}/src/portable/st/stm32_fsdev/fsdev_common.c + ${STARTUP_FILE_${CMAKE_C_COMPILER_ID}} + ) + target_include_directories(${TARGET} PUBLIC + ${CMAKE_CURRENT_FUNCTION_LIST_DIR} + ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/../../ + ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/boards/${BOARD} + ) + + if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + target_link_options(${TARGET} PUBLIC + "LINKER:--script=${LD_FILE_GNU}" + -nostartfiles + --specs=nosys.specs --specs=nano.specs + ) + elseif (CMAKE_C_COMPILER_ID STREQUAL "Clang") + target_link_options(${TARGET} PUBLIC + "LINKER:--script=${LD_FILE_Clang}" + ) + endif () + set_source_files_properties(${STARTUP_FILE_${CMAKE_C_COMPILER_ID}} PROPERTIES + SKIP_LINTING ON + COMPILE_OPTIONS -w) + + # Flashing + family_add_bin_hex(${TARGET}) + family_flash_jlink(${TARGET}) +endfunction() diff --git a/hw/bsp/apm32f0xx/family.mk b/hw/bsp/apm32f0xx/family.mk new file mode 100644 index 000000000..73a762d69 --- /dev/null +++ b/hw/bsp/apm32f0xx/family.mk @@ -0,0 +1,39 @@ +APM32_FAMILY = apm32f0xx +APM32_SDK = hw/mcu/geehy/APM32F0xx_SDK_V1.8.6/Libraries + +include $(TOP)/$(BOARD_PATH)/board.mk + +CPU_CORE ?= cortex-m0plus + +CFLAGS += \ + -flto + +CFLAGS += \ + -DCFG_TUSB_MCU=OPT_MCU_APM32F0XX + +LDFLAGS += \ + -flto --specs=nosys.specs -nostdlib -nostartfiles + +SRC_C += \ + src/portable/st/stm32_fsdev/dcd_stm32_fsdev.c \ + src/portable/st/stm32_fsdev/fsdev_common.c \ + $(APM32_SDK)/APM32F0xx_StdPeriphDriver/src/apm32f0xx_gpio.c \ + $(APM32_SDK)/APM32F0xx_StdPeriphDriver/src/apm32f0xx_misc.c \ + $(APM32_SDK)/APM32F0xx_StdPeriphDriver/src/apm32f0xx_rcm.c \ + $(APM32_SDK)/APM32F0xx_StdPeriphDriver/src/apm32f0xx_crs.c \ + $(APM32_SDK)/Device/Geehy/APM32F0xx/Source/system_apm32f0xx.c + +INC += \ + $(TOP)/$(BOARD_PATH) \ + $(TOP)/$(APM32_SDK)/APM32F0xx_StdPeriphDriver/inc \ + $(TOP)/$(APM32_SDK)/CMSIS/Include \ + $(TOP)/$(APM32_SDK)/Device/Geehy/APM32F0xx/Include + +SRC_S += $(APM32_SDK)/Device/Geehy/APM32F0xx/Source/gcc/startup_apm32f072.S + +LD_FILE ?= $(APM32_SDK)/Device/Geehy/APM32F0xx/Source/gcc/gcc_${MCU_LINKER_NAME}.ld + +# For freeRTOS port source +FREERTOS_PORTABLE_SRC = $(FREERTOS_PORTABLE_PATH)/ARM_CM0 + +flash: flash-jlink diff --git a/src/common/tusb_mcu.h b/src/common/tusb_mcu.h index b8d883dde..93b4a2ee9 100644 --- a/src/common/tusb_mcu.h +++ b/src/common/tusb_mcu.h @@ -716,6 +716,14 @@ // Possible to share IN/OUT if only one direction is armed at any one time #define CFG_TUD_ENDPOINT_ONE_DIRECTION_ONLY 1 +//--------------------------------------------------------------------+ +// Geehy +//--------------------------------------------------------------------+ +#elif TU_CHECK_MCU(OPT_MCU_APM32F0XX) + #define TUP_USBIP_FSDEV + #define TUP_USBIP_FSDEV_APM32 + #define CFG_TUSB_FSDEV_PMA_SIZE 1024u + #endif // External USB controller diff --git a/src/portable/st/stm32_fsdev/fsdev_apm32.h b/src/portable/st/stm32_fsdev/fsdev_apm32.h new file mode 100644 index 000000000..5031445e8 --- /dev/null +++ b/src/portable/st/stm32_fsdev/fsdev_apm32.h @@ -0,0 +1,77 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2024, hathach (tinyusb.org) + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + */ +#ifndef TUSB_FSDEV_APM32_H +#define TUSB_FSDEV_APM32_H + +#include "common/tusb_compiler.h" + +#if CFG_TUSB_MCU == OPT_MCU_APM32F0XX + #include "apm32f0xx.h" +#endif + +#define FSDEV_USE_SBUF_ISO 0 +#define FSDEV_REG_BASE ((uint32_t)(USBD_BASE)) +#define FSDEV_PMA_BASE ((uint32_t)(USBD_BASE + 0x400UL)) + +#ifndef CFG_TUD_FSDEV_DOUBLE_BUFFERED_ISO_EP + #define CFG_TUD_FSDEV_DOUBLE_BUFFERED_ISO_EP 0 +#endif + +//--------------------------------------------------------------------+ +// +//--------------------------------------------------------------------+ + +static const IRQn_Type fsdev_irq[] = { + USBD_IRQn +}; +enum { FSDEV_IRQ_NUM = TU_ARRAY_SIZE(fsdev_irq) }; + +TU_ATTR_ALWAYS_INLINE static inline void fsdev_int_enable(uint8_t rhport) { + (void)rhport; + for (uint8_t i = 0; i < FSDEV_IRQ_NUM; i++) { + NVIC_EnableIRQ(fsdev_irq[i]); + } +} + +TU_ATTR_ALWAYS_INLINE static inline void fsdev_int_disable(uint8_t rhport) { + (void)rhport; + for (uint8_t i = 0; i < FSDEV_IRQ_NUM; i++) { + NVIC_DisableIRQ(fsdev_irq[i]); + } +} + +TU_ATTR_ALWAYS_INLINE static inline void fsdev_disconnect(uint8_t rhport) { + (void) rhport; + FSDEV_REG->CNTR |= U_CNTR_PDWN; + FSDEV_REG->BCDR &= ~U_BCDR_DPPU; +} + +TU_ATTR_ALWAYS_INLINE static inline void fsdev_connect(uint8_t rhport) { + (void) rhport; + FSDEV_REG->CNTR &= ~U_CNTR_PDWN; + FSDEV_REG->BCDR |= U_BCDR_DPPU; +} + +#endif diff --git a/src/portable/st/stm32_fsdev/fsdev_common.h b/src/portable/st/stm32_fsdev/fsdev_common.h index ebde44bf7..00d9b513a 100644 --- a/src/portable/st/stm32_fsdev/fsdev_common.h +++ b/src/portable/st/stm32_fsdev/fsdev_common.h @@ -284,6 +284,8 @@ typedef struct { #include "fsdev_ch32.h" #elif defined(TUP_USBIP_FSDEV_AT32) #include "fsdev_at32.h" +#elif defined(TUP_USBIP_FSDEV_APM32) + #include "fsdev_apm32.h" #else #error "Unknown USB IP" #endif diff --git a/src/tusb_option.h b/src/tusb_option.h index b5e910fca..24f802b73 100644 --- a/src/tusb_option.h +++ b/src/tusb_option.h @@ -207,6 +207,9 @@ // Puya #define OPT_MCU_PY32F0 2700 ///< Puya PY32F0 +// Geehy +#define OPT_MCU_APM32F0XX 2800 ///< Geehy APM32F0xx + // Check if configured MCU is one of listed // Apply TU_MCU_IS_EQUAL with || as separator to list of input #define TU_MCU_IS_EQUAL(_m) (CFG_TUSB_MCU == (_m)) -- cgit v1.3.1 From 80a000e2f027cbeeda5a1a461b7c475ec22a3c91 Mon Sep 17 00:00:00 2001 From: hathach Date: Thu, 23 Jul 2026 11:38:26 +0700 Subject: fix(rp2040): make stdio_rtt_init static LOGGER=rtt builds of any rp2040 example fail with -Werror=missing-prototypes (stdio_rtt_init has no prototype and is only called from family.c). Found by building cdc_msc -DLOG=2 -DLOGGER=rtt for raspberry_pi_pico. --- hw/bsp/rp2040/family.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'hw/bsp') diff --git a/hw/bsp/rp2040/family.c b/hw/bsp/rp2040/family.c index 55feec159..15f179656 100644 --- a/hw/bsp/rp2040/family.c +++ b/hw/bsp/rp2040/family.c @@ -153,7 +153,7 @@ static stdio_driver_t stdio_rtt = { .in_chars = stdio_rtt_read }; -void stdio_rtt_init(void) { +static void stdio_rtt_init(void) { stdio_set_driver_enabled(&stdio_rtt, true); } #endif -- cgit v1.3.1 From 3597d408a2421e72a8cf821bd8899d8b1bf4031e Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 24 Jul 2026 15:28:50 +0700 Subject: stm32h7: tune stm32h743eval ozone trace reference +100 ps sample timing at 400 MHz core / 50 MHz TRACECLK (PLL1R-fixed), width 4. Startup-burst overflow at 400 MHz is expected; board.h documents the PLLN reduction for overflow-free capture. --- hw/bsp/stm32h7/boards/stm32h743eval/ozone/stm32h743.jdebug | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'hw/bsp') diff --git a/hw/bsp/stm32h7/boards/stm32h743eval/ozone/stm32h743.jdebug b/hw/bsp/stm32h7/boards/stm32h743eval/ozone/stm32h743.jdebug index 32a8155c1..a8645c372 100644 --- a/hw/bsp/stm32h7/boards/stm32h743eval/ozone/stm32h743.jdebug +++ b/hw/bsp/stm32h7/boards/stm32h743eval/ozone/stm32h743.jdebug @@ -22,7 +22,7 @@ void OnProjectLoad (void) { Project.SetTargetIF ("SWD"); Project.SetTIFSpeed ("50 MHz"); - File.Open ("../../../../../../examples/device/cdc_msc/cmake-build-stm32h743eval/cdc_msc.elf"); + File.Open ("../../../../../../examples/cmake-build-stm32h743eval/device/cdc_msc/cdc_msc.elf"); // File.Open ("../../../../../../examples/cmake-build-stm32h743eval_host1/host/cdc_msc_hid/cdc_msc_hid.elf"); } -- cgit v1.3.1 From 6d41a1fd71f8a13c4c977fdf3919ef0165823b32 Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 24 Jul 2026 15:28:51 +0700 Subject: lpc40: validate ea4088_quickstart etm-trace reference 120 MHz TRACECLK width 4 over the fully-wired J7 (rev B schematic, TRACE_5V on pin 11). FS enumeration finishes in <100 ms - ISR analysis needs a short no-eviction window (--duration-ms 150). --- hw/bsp/lpc40/boards/ea4088_quickstart/ozone/ea4088_quickstart.jdebug | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'hw/bsp') diff --git a/hw/bsp/lpc40/boards/ea4088_quickstart/ozone/ea4088_quickstart.jdebug b/hw/bsp/lpc40/boards/ea4088_quickstart/ozone/ea4088_quickstart.jdebug index 6aaf1076d..20de1e915 100644 --- a/hw/bsp/lpc40/boards/ea4088_quickstart/ozone/ea4088_quickstart.jdebug +++ b/hw/bsp/lpc40/boards/ea4088_quickstart/ozone/ea4088_quickstart.jdebug @@ -21,7 +21,7 @@ void OnProjectLoad (void) { Project.SetTracePortWidth (4); // User settings - File.Open ("../../../../../../examples/device/cdc_msc/cmake-build-ea4088-quickstart/cdc_msc.elf"); + File.Open ("../../../../../../examples/cmake-build-ea4088_quickstart/device/cdc_msc/cdc_msc.elf"); } /********************************************************************* -- cgit v1.3.1 From b3a9b3422cf9e21d6b407940237e7c08537f782d Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 24 Jul 2026 15:28:51 +0700 Subject: lpc18: mcb1800 etm-trace - disable pull-ups on trace lines 60 MHz TRACECLK (CCLK/2) width 4 with J5 DBG_EN fitted; board.h drops the trace-line pull-ups and the ozone reference points at the device example. A badly-mated ribbon reads register-perfect yet silent - re-seat first. --- hw/bsp/lpc18/boards/mcb1800/board.h | 27 ++++++++++++++++-------- hw/bsp/lpc18/boards/mcb1800/ozone/lpc1857.jdebug | 6 +++--- hw/bsp/lpc18/family.c | 1 + 3 files changed, 22 insertions(+), 12 deletions(-) (limited to 'hw/bsp') diff --git a/hw/bsp/lpc18/boards/mcb1800/board.h b/hw/bsp/lpc18/boards/mcb1800/board.h index dba7a62a3..ec7f1fa95 100644 --- a/hw/bsp/lpc18/boards/mcb1800/board.h +++ b/hw/bsp/lpc18/boards/mcb1800/board.h @@ -48,15 +48,6 @@ static inline void board_lpc18_pinmux(void) { const PINMUX_GRP_T pinmuxing[] = { - // ETM Trace - #ifdef TRACE_ETM - { 0xF, 4, SCU_MODE_FUNC2 | SCU_MODE_HIGHSPEEDSLEW_EN }, - { 0xF, 5, SCU_MODE_FUNC3 | SCU_MODE_HIGHSPEEDSLEW_EN }, - { 0xF, 6, SCU_MODE_FUNC3 | SCU_MODE_HIGHSPEEDSLEW_EN }, - { 0xF, 7, SCU_MODE_FUNC3 | SCU_MODE_HIGHSPEEDSLEW_EN }, - { 0xF, 8, SCU_MODE_FUNC3 | SCU_MODE_HIGHSPEEDSLEW_EN }, - #endif - // LEDs { 0xD, 10, (SCU_MODE_INBUFF_EN | SCU_MODE_INACT | SCU_MODE_FUNC4) }, { 0xD, 11, (SCU_MODE_INBUFF_EN | SCU_MODE_INACT | SCU_MODE_FUNC4 | SCU_MODE_PULLDOWN) }, @@ -96,6 +87,24 @@ static inline void board_lpc18_pinmux(void) { } } +#ifdef TRACE_ETM +// Must run AFTER Chip_SetupCoreClock: muxing the trace pins earlier starts +// TRACECLK at the boot clock and the mid-init frequency switch desyncs the +// trace decoder ("Unknown trace data packet"). +static inline void board_trace_pinmux(void) { + // SCU_MODE_INACT: disable the pull-up on the 60 MHz trace lines - leaving it + // enabled degrades the edges enough for intermittent decode corruption. + const PINMUX_GRP_T trace_pinmux[] = { + { 0xF, 4, SCU_MODE_INACT | SCU_MODE_FUNC2 | SCU_MODE_HIGHSPEEDSLEW_EN }, + { 0xF, 5, SCU_MODE_INACT | SCU_MODE_FUNC3 | SCU_MODE_HIGHSPEEDSLEW_EN }, + { 0xF, 6, SCU_MODE_INACT | SCU_MODE_FUNC3 | SCU_MODE_HIGHSPEEDSLEW_EN }, + { 0xF, 7, SCU_MODE_INACT | SCU_MODE_FUNC3 | SCU_MODE_HIGHSPEEDSLEW_EN }, + { 0xF, 8, SCU_MODE_INACT | SCU_MODE_FUNC3 | SCU_MODE_HIGHSPEEDSLEW_EN }, + }; + Chip_SCU_SetPinMuxing(trace_pinmux, sizeof(trace_pinmux) / sizeof(PINMUX_GRP_T)); +} +#endif + #ifdef __cplusplus } #endif diff --git a/hw/bsp/lpc18/boards/mcb1800/ozone/lpc1857.jdebug b/hw/bsp/lpc18/boards/mcb1800/ozone/lpc1857.jdebug index f94960f09..a6dcef0eb 100644 --- a/hw/bsp/lpc18/boards/mcb1800/ozone/lpc1857.jdebug +++ b/hw/bsp/lpc18/boards/mcb1800/ozone/lpc1857.jdebug @@ -10,7 +10,7 @@ */ void OnProjectLoad (void) { Project.AddSvdFile ("Cortex-M3.svd"); - Project.AddSvdFile ("../../../../../../../cmsis-svd/data/NXP/LPC18xx.svd"); + //Project.AddSvdFile ("../../../../../../../cmsis-svd/data/NXP/LPC18xx.svd"); Project.SetDevice ("LPC1857"); Project.SetHostIF ("USB", ""); @@ -20,8 +20,8 @@ void OnProjectLoad (void) { Project.SetTraceSource ("Trace Pins"); Project.SetTracePortWidth (4); - //File.Open ("../../../../../../examples/cmake-build-mcb1800/device/cdc_msc/cdc_msc.elf"); - File.Open ("../../../../../../examples/cmake-build-mcb1800/host/cdc_msc_hid/cdc_msc_hid.elf"); + File.Open ("../../../../../../examples/cmake-build-mcb1800/device/cdc_msc/cdc_msc.elf"); + //File.Open ("../../../../../../examples/cmake-build-mcb1800/host/cdc_msc_hid/cdc_msc_hid.elf"); } /********************************************************************* * diff --git a/hw/bsp/lpc18/family.c b/hw/bsp/lpc18/family.c index 8a612d9d8..58c2193fc 100644 --- a/hw/bsp/lpc18/family.c +++ b/hw/bsp/lpc18/family.c @@ -69,6 +69,7 @@ void SystemInit(void) { #ifdef TRACE_ETM // Trace clock is limited to 60MHz, limit CPU clock to 120MHz Chip_SetupCoreClock(CLKIN_CRYSTAL, 120000000UL, true); + board_trace_pinmux(); // after clock setup so TRACECLK starts at its final frequency #else // CPU clock max to 180 Mhz Chip_SetupCoreClock(CLKIN_CRYSTAL, MAX_CLOCK_FREQ, true); -- cgit v1.3.1 From ca1b7821fa592ba7a56d0a4ff9b221e5e7a18985 Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 24 Jul 2026 15:28:51 +0700 Subject: lpc43: prepare ea4357 trace pins (bring-up blocked on SJ1 rework) BSP mux + board.h are register-proven; the module routes TRACECLK to the header only with SJ1's 0-ohm resistor moved to pads 2-3 (Lauterbach doc confirms), so hardware validation waits on that rework. --- hw/bsp/lpc43/boards/ea4357/board.h | 16 ++++++++++++++++ hw/bsp/lpc43/family.c | 6 ++++++ 2 files changed, 22 insertions(+) (limited to 'hw/bsp') diff --git a/hw/bsp/lpc43/boards/ea4357/board.h b/hw/bsp/lpc43/boards/ea4357/board.h index fca617361..0152825f5 100644 --- a/hw/bsp/lpc43/boards/ea4357/board.h +++ b/hw/bsp/lpc43/boards/ea4357/board.h @@ -83,6 +83,22 @@ static const PINMUX_GRP_T pinmuxing[] = { // { 0, 3, SCU_MODE_INACT | SCU_MODE_INBUFF_EN | SCU_MODE_ZIF_DIS | SCU_MODE_HIGHSPEEDSLEW_EN | SCU_MODE_FUNC0 }, //}; +#ifdef TRACE_ETM +// Must run AFTER Chip_SetupCoreClock: muxing the trace pins earlier starts +// TRACECLK at the boot clock and the mid-init frequency switch desyncs the +// trace decoder. SCU_MODE_INACT keeps pull-ups off the 60 MHz lines. +static inline void board_trace_pinmux(void) { + const PINMUX_GRP_T trace_pinmux[] = { + { 0xF, 4, SCU_MODE_INACT | SCU_MODE_FUNC2 | SCU_MODE_HIGHSPEEDSLEW_EN }, + { 0xF, 5, SCU_MODE_INACT | SCU_MODE_FUNC3 | SCU_MODE_HIGHSPEEDSLEW_EN }, + { 0xF, 6, SCU_MODE_INACT | SCU_MODE_FUNC3 | SCU_MODE_HIGHSPEEDSLEW_EN }, + { 0xF, 7, SCU_MODE_INACT | SCU_MODE_FUNC3 | SCU_MODE_HIGHSPEEDSLEW_EN }, + { 0xF, 8, SCU_MODE_INACT | SCU_MODE_FUNC3 | SCU_MODE_HIGHSPEEDSLEW_EN }, + }; + Chip_SCU_SetPinMuxing(trace_pinmux, sizeof(trace_pinmux) / sizeof(PINMUX_GRP_T)); +} +#endif + #ifdef __cplusplus } #endif diff --git a/hw/bsp/lpc43/family.c b/hw/bsp/lpc43/family.c index 5aff49704..411ea7d58 100644 --- a/hw/bsp/lpc43/family.c +++ b/hw/bsp/lpc43/family.c @@ -89,7 +89,13 @@ void SystemInit(void) // Chip_SCU_ClockPinMuxSet(pinclockmuxing[i].pinnum, pinclockmuxing[i].modefunc); // } +#ifdef TRACE_ETM + // Trace clock is limited to 60MHz, limit CPU clock to 120MHz + Chip_SetupCoreClock(CLKIN_CRYSTAL, 120000000UL, true); + board_trace_pinmux(); // after clock setup so TRACECLK starts at its final frequency +#else Chip_SetupXtalClocking(); +#endif } void board_init(void) -- cgit v1.3.1 From 927f12415ef09b6f590c83660c13491a644b4626 Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 24 Jul 2026 15:28:51 +0700 Subject: nrf: etm-trace for nrf52840dk and nrf5340dk nrf52840dk: 16 MHz TRACECLK (hardware cap) width 4, P25 soldered, SW7=Alt; no family code needed (J-Link arms TRACECONFIG). nrf5340dk: TRACE_ETM builds force the TAD port to 16 MHz (SystemInit's 64 MHz is marginal), +3 ns sample timing; the interface MCU's UART1 flow control drives the trace pins - SB27/SB28 must be cut (P0.10/P0.11 = TRACEDATA1/0). --- hw/bsp/nrf/boards/nrf52840dk/ozone/nrf52840.jdebug | 2 +- hw/bsp/nrf/boards/nrf5340dk/ozone/nrf5340.jdebug | 6 +++++- hw/bsp/nrf/family.c | 6 ++++++ 3 files changed, 12 insertions(+), 2 deletions(-) (limited to 'hw/bsp') diff --git a/hw/bsp/nrf/boards/nrf52840dk/ozone/nrf52840.jdebug b/hw/bsp/nrf/boards/nrf52840dk/ozone/nrf52840.jdebug index fa7ab9e23..40f28baa9 100644 --- a/hw/bsp/nrf/boards/nrf52840dk/ozone/nrf52840.jdebug +++ b/hw/bsp/nrf/boards/nrf52840dk/ozone/nrf52840.jdebug @@ -21,7 +21,7 @@ void OnProjectLoad (void) { Project.SetTracePortWidth (4); // User settings - File.Open ("../../../../../../examples/device/cdc_msc/cmake-build-pca10056/cdc_msc.elf"); + File.Open ("../../../../../../examples/cmake-build-nrf52840dk/device/cdc_msc/cdc_msc.elf"); } /********************************************************************* diff --git a/hw/bsp/nrf/boards/nrf5340dk/ozone/nrf5340.jdebug b/hw/bsp/nrf/boards/nrf5340dk/ozone/nrf5340.jdebug index 4ad0376a4..34b1841b0 100644 --- a/hw/bsp/nrf/boards/nrf5340dk/ozone/nrf5340.jdebug +++ b/hw/bsp/nrf/boards/nrf5340dk/ozone/nrf5340.jdebug @@ -29,9 +29,13 @@ void OnProjectLoad (void) { Project.SetTraceSource ("Trace Pins"); Project.SetTracePortWidth (4); + // +3 ns sample point: the DK's analog-switch/SWO stubs make TD=0 marginal + // (Ozone sends TraceSampleAdjust TD=0 when no timing is set); solid across + // the +2..+4 ns band with SB27/SB28 cut, SB57 (SWO) left intact + Project.SetTraceTiming (3000, 3000, 3000, 3000); // User settings - File.Open ("../../../../../../examples/device/cdc_msc/cmake-build-pca10095/cdc_msc.elf"); + File.Open ("../../../../../../examples/cmake-build-nrf5340dk/device/cdc_msc/cdc_msc.elf"); } /********************************************************************* diff --git a/hw/bsp/nrf/family.c b/hw/bsp/nrf/family.c index 31a6bac9e..3cef4dac3 100644 --- a/hw/bsp/nrf/family.c +++ b/hw/bsp/nrf/family.c @@ -165,6 +165,12 @@ static nrfx_gpiote_t _gpiote = NRFX_GPIOTE_INSTANCE(0); // //--------------------------------------------------------------------+ void board_init(void) { +#if defined(TRACE_ETM) && defined(NRF5340_XXAA) + // SystemInit (ENABLE_TRACE) sets the TAD trace port to 64 MHz, which is + // marginal through the DK's switch stubs - 16 MHz streams reliably (matches the validated nrf52840dk) and is + // ample bandwidth for the 64 MHz core + NRF_TAD_S->TRACEPORTSPEED = TAD_TRACEPORTSPEED_TRACEPORTSPEED_16MHz; +#endif #if !defined(NRF54H20_XXAA) && !defined(NRF54LM20A_ENGA_XXAA) // stop LF clock just in case we jump from application without reset NRF_CLOCK->TASKS_LFCLKSTOP = 1UL; -- cgit v1.3.1 From 3b7ef31593aeada32667be873f4bd1b6cfadf3cb Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 24 Jul 2026 15:28:52 +0700 Subject: stm32h5: TRACE_ETM support and stm32h563nucleo reference H5 hangs its debug AP if trace CoreSight is touched unclocked (recover = power-cycle): the reference's AfterTargetConnect clocks the DBGMCU trace domain but defers IOEN to firmware, or the mid-boot clock switch desyncs the decoder. Stock solder bridges make the CN5 path marginal: validated config is 100 MHz core, width 1, +5 ns (board.h selects the reduced clock for TRACE_ETM builds); width 4 / 250 MHz retest waits on SB removal. --- hw/bsp/stm32h5/boards/stm32h563nucleo/board.h | 5 + .../boards/stm32h563nucleo/ozone/stm32h563.jdebug | 113 +++++++++++++++++++++ hw/bsp/stm32h5/family.c | 20 ++++ 3 files changed, 138 insertions(+) create mode 100644 hw/bsp/stm32h5/boards/stm32h563nucleo/ozone/stm32h563.jdebug (limited to 'hw/bsp') diff --git a/hw/bsp/stm32h5/boards/stm32h563nucleo/board.h b/hw/bsp/stm32h5/boards/stm32h563nucleo/board.h index 18c13017a..959dc4828 100644 --- a/hw/bsp/stm32h5/boards/stm32h563nucleo/board.h +++ b/hw/bsp/stm32h5/boards/stm32h563nucleo/board.h @@ -88,7 +88,12 @@ static inline void SystemClock_Config(void) { RCC_OscInitStruct.PLL.PLLState = RCC_PLL_ON; RCC_OscInitStruct.PLL.PLLSource = RCC_PLL1_SOURCE_HSE; RCC_OscInitStruct.PLL.PLLM = 4; + #ifdef TRACE_ETM + RCC_OscInitStruct.PLL.PLLN = 100; // 100 MHz core: the Nucleo trace path (CN5 via solder bridges) + // corrupts the trace stream at higher TRACECLK (= SYSCLK/2) + #else RCC_OscInitStruct.PLL.PLLN = 250; + #endif RCC_OscInitStruct.PLL.PLLP = 2; RCC_OscInitStruct.PLL.PLLQ = 2; RCC_OscInitStruct.PLL.PLLR = 2; diff --git a/hw/bsp/stm32h5/boards/stm32h563nucleo/ozone/stm32h563.jdebug b/hw/bsp/stm32h5/boards/stm32h563nucleo/ozone/stm32h563.jdebug new file mode 100644 index 000000000..04f4e4582 --- /dev/null +++ b/hw/bsp/stm32h5/boards/stm32h563nucleo/ozone/stm32h563.jdebug @@ -0,0 +1,113 @@ +/********************************************************************* +* +* OnProjectLoad +* +* Function description +* Project load routine. Required. +* +* Notes +* Nucleo-H563ZI trace: PE2-PE6 reach the MIPI-20 connector (CN5) only +* with SB8, SB9, SB64, SB68, SB70, SB71, SB78 removed (MB1404 UM). +* Firmware must be built with TRACE_ETM=1 (trace pin + DBGMCU init). +* +* The trace path through the solder bridges is signal-marginal: reliable +* only at 100 MHz core (TRACE_ETM builds select this automatically in +* board.h), port width 1 and +5 ns sample timing (validated empirically; +* width 4 or 250 MHz core corrupts the stream within ~100 ms). +* +********************************************************************** +*/ +void OnProjectLoad (void) { + Project.SetTraceSource ("Trace Pins"); + Project.SetTracePortWidth (1); + Project.SetTraceTiming (5000, 5000, 5000, 5000); + Project.SetSWO (0); + Edit.SysVar (VAR_TRACE_CORE_CLOCK, 100000000); + Project.AddSvdFile ("$(InstallDir)/Config/CPU/Cortex-M33F.svd"); + + Project.SetDevice ("STM32H563ZI"); + Project.SetHostIF ("USB", ""); + Project.SetTargetIF ("SWD"); + Project.SetTIFSpeed ("4 MHz"); + + File.Open ("../../../../../../examples/cmake-build-stm32h563nucleo/device/cdc_msc/cdc_msc.elf"); +} + +/********************************************************************* +* +* AfterTargetConnect +* +* Function description +* Enable the trace clock domain (DBGMCU_CR: TRACE_IOEN | TRACE_CLKEN | +* TRACE_MODE=4-bit) BEFORE Ozone touches the trace CoreSight components. +* On STM32H5 an access to unclocked trace components hangs the debug AP +* ("Failed to read target status") until the board is power-cycled. +* +********************************************************************** +*/ +void AfterTargetConnect (void) { + unsigned int cr; + cr = Target.ReadU32(0x44024004); // DBGMCU_CR + // clock the domain (CLKEN|MODE=4-bit) but keep the pins OFF (clear IOEN): + // the pins must only go live via firmware trace_etm_init() AFTER the system + // clock switch, or the mid-trace frequency change desyncs the decoder. + Target.WriteU32(0x44024004, (cr & 0xFFFFFFEF) | 0xE0); +} + +/********************************************************************* +* +* AfterTargetReset +* +* Function description +* Event handler routine. +* - Sets the PC register to program reset value. +* - Sets the SP register to program reset value on Cortex-M. +* +********************************************************************** +*/ +void AfterTargetReset (void) { + unsigned int SP; + unsigned int PC; + unsigned int VectorTableAddr; + + VectorTableAddr = Elf.GetBaseAddr(); + + if (VectorTableAddr == 0xFFFFFFFF) { + Util.Log("Project file error: failed to get program base"); + } else { + SP = Target.ReadU32(VectorTableAddr); + Target.SetReg("SP", SP); + + PC = Target.ReadU32(VectorTableAddr + 4); + Target.SetReg("PC", PC); + } +} + +/********************************************************************* +* +* AfterTargetDownload +* +* Function description +* Event handler routine. +* - Sets the PC register to program reset value. +* - Sets the SP register to program reset value on Cortex-M. +* +********************************************************************** +*/ +void AfterTargetDownload (void) { + unsigned int SP; + unsigned int PC; + unsigned int VectorTableAddr; + + VectorTableAddr = Elf.GetBaseAddr(); + + if (VectorTableAddr == 0xFFFFFFFF) { + Util.Log("Project file error: failed to get program base"); + } else { + SP = Target.ReadU32(VectorTableAddr); + Target.SetReg("SP", SP); + + PC = Target.ReadU32(VectorTableAddr + 4); + Target.SetReg("PC", PC); + } +} diff --git a/hw/bsp/stm32h5/family.c b/hw/bsp/stm32h5/family.c index 867171734..36a95ac8c 100644 --- a/hw/bsp/stm32h5/family.c +++ b/hw/bsp/stm32h5/family.c @@ -99,6 +99,24 @@ static UART_HandleTypeDef UartHandle = { }; #endif +#ifdef TRACE_ETM +static void trace_etm_init(void) { + // H5 trace pins are PE2 to PE6 (Nucleo-144: requires trace solder-bridge config, see board docs) + GPIO_InitTypeDef gpio_init; + gpio_init.Pin = GPIO_PIN_2 | GPIO_PIN_3 | GPIO_PIN_4 | GPIO_PIN_5 | GPIO_PIN_6; + gpio_init.Mode = GPIO_MODE_AF_PP; + gpio_init.Pull = GPIO_PULLUP; + gpio_init.Speed = GPIO_SPEED_FREQ_VERY_HIGH; + gpio_init.Alternate = GPIO_AF0_TRACE; + HAL_GPIO_Init(GPIOE, &gpio_init); + + // Enable trace port + clock, synchronous 4-bit mode + DBGMCU->CR |= DBGMCU_CR_TRACE_IOEN | DBGMCU_CR_TRACE_CLKEN | DBGMCU_CR_TRACE_MODE; +} +#else +#define trace_etm_init() +#endif + void board_init(void) { // Cache UID before ICACHE is enabled (STM32H5 errata: reading UID_BASE with ICACHE causes hard fault) volatile uint32_t* stm32_uuid = (volatile uint32_t*) UID_BASE; @@ -127,6 +145,8 @@ void board_init(void) { __HAL_RCC_GPIOI_CLK_ENABLE(); #endif + trace_etm_init(); + #if CFG_TUSB_OS == OPT_OS_NONE // 1ms tick timer SysTick_Config(SystemCoreClock / 1000); -- cgit v1.3.1 From 34dcb9fe523f501d492bf4d7dabfd1bcd1a1c8b3 Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 24 Jul 2026 15:28:52 +0700 Subject: imxrt: TRACE_ETM for RT1011 and RT1176, validate both boards metro_m7_1011 (custom ETM-header rework): 500 MHz core, 66 MHz TRACECLK width 4, +50 ps; trace_etm_init ungates the 132 MHz trace root that BOARD_BootClockRUN leaves gated. mimxrt1170_evkb: 996 MHz CM7 at width 1, CSTRACE pinned to 50 MHz (stock 132 corrupts - the Ethernet PHY loads the CLK net) and the CM7 platform trace-funnel port enabled in firmware: J-Link does not program that funnel and everything reads register-perfect yet silent without it. FlexSPI boot needs the committed SP/PC hooks; D1-D3 stay dead pending the R1882-R1884 continuity check (width-4 TODO). --- .../metro_m7_1011/ozone/metro_m7_1011.jdebug | 2 +- .../boards/mimxrt1170_evkb/ozone/mimxrt1176.jdebug | 69 ++++++++++++++++++++++ .../ozone/mimxrt1176_trace.JLinkScript | 11 ++++ hw/bsp/imxrt/family.c | 61 +++++++++++++++++-- 4 files changed, 138 insertions(+), 5 deletions(-) create mode 100644 hw/bsp/imxrt/boards/mimxrt1170_evkb/ozone/mimxrt1176.jdebug create mode 100644 hw/bsp/imxrt/boards/mimxrt1170_evkb/ozone/mimxrt1176_trace.JLinkScript (limited to 'hw/bsp') diff --git a/hw/bsp/imxrt/boards/metro_m7_1011/ozone/metro_m7_1011.jdebug b/hw/bsp/imxrt/boards/metro_m7_1011/ozone/metro_m7_1011.jdebug index 90f9b77e5..fdb8b30a0 100644 --- a/hw/bsp/imxrt/boards/metro_m7_1011/ozone/metro_m7_1011.jdebug +++ b/hw/bsp/imxrt/boards/metro_m7_1011/ozone/metro_m7_1011.jdebug @@ -22,7 +22,7 @@ void OnProjectLoad (void) { // timing delay for trace pins in pico seconds, default is 2 nano seconds - File.Open ("../../../../../../examples/cmake-build-metro-m7-1011-sd/device/cdc_msc/cdc_msc.elf"); + File.Open ("../../../../../../examples/cmake-build-metro_m7_1011/device/cdc_msc/cdc_msc.elf"); } /********************************************************************* diff --git a/hw/bsp/imxrt/boards/mimxrt1170_evkb/ozone/mimxrt1176.jdebug b/hw/bsp/imxrt/boards/mimxrt1170_evkb/ozone/mimxrt1176.jdebug new file mode 100644 index 000000000..6931b9cd1 --- /dev/null +++ b/hw/bsp/imxrt/boards/mimxrt1170_evkb/ozone/mimxrt1176.jdebug @@ -0,0 +1,69 @@ +/********************************************************************* +* +* OnProjectLoad +* +* Function description +* Project load routine. Required. +* +* Notes +* MIMXRT1170-EVKB trace requires board rework: the TRACE pads +* (GPIO_DISP_B2_02..06) are factory-wired to the ENET_1G PHY - populate +* 0-ohm R1881-R1886 to route them to the Cortex Debug+ETM connector J58 +* (EVKB Hardware User Guide 3.2). +* Firmware must be built with TRACE_ETM=1 (trace pad mux + CSTRACE clock +* + JTAG_nTRST/DMIC_DATA1 pad fix in board_init). +* +* Validated: port width 1, 0 ns sample timing, 50 MHz CSTRACE root +* (25 MHz TRACE_CLK pin, set by trace_etm_init - the rework path corrupts +* at the stock 132 MHz root) while the CM7 runs 996 MHz. A startup-burst +* trace overflow is expected and benign; width 4 fails at any timing. +* +********************************************************************** +*/ +void OnProjectLoad (void) { + // declares the CSSYS TPIU/funnel (system APB-AP, not in the CM7 ROM table) + Project.SetJLinkScript ("./mimxrt1176_trace.JLinkScript"); + Project.SetTraceSource ("Trace Pins"); + Project.SetTracePortWidth (1); + Project.SetTraceTiming (0, 0, 0, 0); + Project.SetSWO (0); + Edit.SysVar (VAR_TRACE_CORE_CLOCK, 996000000); + Project.AddSvdFile ("$(InstallDir)/Config/CPU/Cortex-M7F.svd"); + + Project.SetDevice ("MIMXRT1176xxxA_M7"); + Project.SetHostIF ("USB", ""); + Project.SetTargetIF ("SWD"); + Project.SetTIFSpeed ("4 MHz"); + + File.Open ("../../../../../../examples/cmake-build-mimxrt1170_evkb/device/cdc_msc/cdc_msc.elf"); +} + +/********************************************************************* +* +* AfterTargetReset +* +* Function description +* JTAG_nTRST shares its pad with DMIC_DATA1, which drives it low by +* default and kills ETM trace - mux the pad to GPIO before the app runs +* (EVKB Hardware User Guide 3.2). No SP/PC init here: the app boots from +* FlexSPI NOR, so the ROM bootloader must perform it (SEGGER wiki +* "i.MXRT1176" / NXP community solution for ERR050708 boards). +* +********************************************************************** +*/ +void AfterTargetReset (void) { + Target.WriteU32 (0x40C08028, 0xA); // IOMUXC GPIO_LPSR_10 -> GPIO12_IO10 +} + +/********************************************************************* +* +* AfterTargetDownload +* +* Function description +* Intentionally empty: SP/PC init from the vector table would bypass the +* ROM bootloader's FlexSPI setup (see AfterTargetReset note). +* +********************************************************************** +*/ +void AfterTargetDownload (void) { +} diff --git a/hw/bsp/imxrt/boards/mimxrt1170_evkb/ozone/mimxrt1176_trace.JLinkScript b/hw/bsp/imxrt/boards/mimxrt1170_evkb/ozone/mimxrt1176_trace.JLinkScript new file mode 100644 index 000000000..ef39235d8 --- /dev/null +++ b/hw/bsp/imxrt/boards/mimxrt1170_evkb/ozone/mimxrt1176_trace.JLinkScript @@ -0,0 +1,11 @@ +/* RT1176 pin trace: the CSSYS TPIU and ATB funnel live on the system APB-AP + * (AP2) and are not discoverable from the CM7 ROM table - declare them, or + * J-Link aborts with "Required trace components for pin trace not found!" + * (addresses per i.MX RT1170 RM memory map: CSSYS TPIU E004_6000, CSSYS ATB + * Funnel E004_5000; same values as SEGGER's RT1176 trace example script). + */ +int ConfigTargetSettings(void) { + JLINK_ExecCommand("CORESIGHT_SetTPIUBaseAddr = 0xE0046000 ForceUnlock = 1 APIndex = 2"); + JLINK_ExecCommand("CORESIGHT_SetCSTFBaseAddr = 0xE0045000 ForceUnlock = 1 APIndex = 2"); + return 0; +} diff --git a/hw/bsp/imxrt/family.c b/hw/bsp/imxrt/family.c index 9f3297e9a..e3ef6233c 100644 --- a/hw/bsp/imxrt/family.c +++ b/hw/bsp/imxrt/family.c @@ -106,6 +106,62 @@ static void init_usb_phy(uint8_t usb_id) { usb_phy->TX = phytx; } +#ifdef TRACE_ETM +static void trace_etm_init(void) { +#if defined(CPU_MIMXRT1011DAE5A) + // Metro M7 rev A "ETM Trace" rework: 4-bit TRACE + TRACE_CLK on the added + // 2x10 header; the RT1011 has a single mux option per trace signal + IOMUXC_SetPinMux(IOMUXC_GPIO_AD_00_ARM_TRACE0, 0U); + IOMUXC_SetPinMux(IOMUXC_GPIO_13_ARM_TRACE1, 0U); + IOMUXC_SetPinMux(IOMUXC_GPIO_12_ARM_TRACE2, 0U); + IOMUXC_SetPinMux(IOMUXC_GPIO_11_ARM_TRACE3, 0U); + IOMUXC_SetPinMux(IOMUXC_GPIO_AD_02_ARM_TRACE_CLK, 0U); + // fast slew, max speed, high drive + IOMUXC_SetPinConfig(IOMUXC_GPIO_AD_00_ARM_TRACE0, 0x00F1U); + IOMUXC_SetPinConfig(IOMUXC_GPIO_13_ARM_TRACE1, 0x00F1U); + IOMUXC_SetPinConfig(IOMUXC_GPIO_12_ARM_TRACE2, 0x00F1U); + IOMUXC_SetPinConfig(IOMUXC_GPIO_11_ARM_TRACE3, 0x00F1U); + IOMUXC_SetPinConfig(IOMUXC_GPIO_AD_02_ARM_TRACE_CLK, 0x00F1U); + + // TRACE_CLK_ROOT already runs 132 MHz (PLL2/4) from BOARD_BootClockRUN, + // which leaves it gated - just ungate + CLOCK_EnableClock(kCLOCK_Trace); +#elif defined(CPU_MIMXRT1176DVMAA_cm7) + // JTAG_nTRST pad is shared with DMIC_DATA1 which drives it low by default and + // breaks ETM trace - switch the pad to GPIO (MIMXRT1170-EVKB HUG 3.2) + IOMUXC_SetPinMux(IOMUXC_GPIO_LPSR_10_GPIO12_IO10, 0U); + + // TRACE0-3 + TRACE_CLK on GPIO_DISP_B2_02..06, fast slew + high drive + IOMUXC_SetPinMux(IOMUXC_GPIO_DISP_B2_02_ARM_TRACE00, 0U); + IOMUXC_SetPinMux(IOMUXC_GPIO_DISP_B2_03_ARM_TRACE01, 0U); + IOMUXC_SetPinMux(IOMUXC_GPIO_DISP_B2_04_ARM_TRACE02, 0U); + IOMUXC_SetPinMux(IOMUXC_GPIO_DISP_B2_05_ARM_TRACE03, 0U); + IOMUXC_SetPinMux(IOMUXC_GPIO_DISP_B2_06_ARM_TRACE_CLK, 0U); + IOMUXC_SetPinConfig(IOMUXC_GPIO_DISP_B2_02_ARM_TRACE00, IOMUXC_SW_PAD_CTL_PAD_DSE_MASK); + IOMUXC_SetPinConfig(IOMUXC_GPIO_DISP_B2_03_ARM_TRACE01, IOMUXC_SW_PAD_CTL_PAD_DSE_MASK); + IOMUXC_SetPinConfig(IOMUXC_GPIO_DISP_B2_04_ARM_TRACE02, IOMUXC_SW_PAD_CTL_PAD_DSE_MASK); + IOMUXC_SetPinConfig(IOMUXC_GPIO_DISP_B2_05_ARM_TRACE03, IOMUXC_SW_PAD_CTL_PAD_DSE_MASK); + IOMUXC_SetPinConfig(IOMUXC_GPIO_DISP_B2_06_ARM_TRACE_CLK, IOMUXC_SW_PAD_CTL_PAD_DSE_MASK); + + // 50 MHz CSTRACE root (OscRc400M/8) -> 25 MHz TRACE_CLK pin (= root/2) + CLOCK_SetRootClockMux(kCLOCK_Root_Cstrace, kCLOCK_CSTRACE_ClockRoot_MuxOscRc400M); + CLOCK_SetRootClockDiv(kCLOCK_Root_Cstrace, 8); + CLOCK_EnableClock(kCLOCK_Cstrace); + + // Enable the CM7 slave port on the platform trace funnel (E004_3000): the + // debugger only programs the CSSYS funnel/TPIU it was told about, and this + // in-between funnel resets with all ports disabled, silently eating the ETM + // stream. Core-side CoreSight accesses honor the lock, hence the LAR unlock. + *(volatile uint32_t *) 0xE0043FB0 = 0xC5ACCE55; + *(volatile uint32_t *) 0xE0043000 |= 1U; +#else + #error "TRACE_ETM: no trace pin setup for this MCU variant" +#endif +} +#else + #define trace_etm_init() +#endif + void board_init(void) { // make sure the dcache is on. #if defined(__DCACHE_PRESENT) && __DCACHE_PRESENT @@ -119,10 +175,7 @@ void board_init(void) { SystemCoreClockUpdate(); BOARD_ConfigMPU(); // defined in board.h - -#ifdef TRACE_ETM - //CLOCK_EnableClock(kCLOCK_Trace); -#endif + trace_etm_init(); #if CFG_TUSB_OS == OPT_OS_NONE // 1ms tick timer -- cgit v1.3.1 From 93443d3b4686b63e2263758c17e11e66a589d8a5 Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 24 Jul 2026 15:28:52 +0700 Subject: stm32h7rs: TRACE_ETM support and stm32h7s3nucleo reference 300 MHz core, 50 MHz TRACECLK, width 2: SB11/SB12 stub TRACED2/3 onto Zio CN8 and kill width 4 under IRQ-heavy USB traffic (removal = width-4 TODO at 600 MHz). Session note: --attach while a host actively polls the device wedges its USB session. --- hw/bsp/stm32h7rs/boards/stm32h7s3nucleo/board.h | 6 ++ .../boards/stm32h7s3nucleo/ozone/stm32h7s3.jdebug | 94 ++++++++++++++++++++++ .../ozone/stm32h7s3_trace.JLinkScript | 11 +++ hw/bsp/stm32h7rs/family.c | 21 +++-- 4 files changed, 125 insertions(+), 7 deletions(-) create mode 100644 hw/bsp/stm32h7rs/boards/stm32h7s3nucleo/ozone/stm32h7s3.jdebug create mode 100644 hw/bsp/stm32h7rs/boards/stm32h7s3nucleo/ozone/stm32h7s3_trace.JLinkScript (limited to 'hw/bsp') diff --git a/hw/bsp/stm32h7rs/boards/stm32h7s3nucleo/board.h b/hw/bsp/stm32h7rs/boards/stm32h7s3nucleo/board.h index 996eb1515..098fc0bed 100644 --- a/hw/bsp/stm32h7rs/boards/stm32h7s3nucleo/board.h +++ b/hw/bsp/stm32h7rs/boards/stm32h7s3nucleo/board.h @@ -123,7 +123,13 @@ static inline void SystemClock_Config(void) RCC_OscInitStruct.PLL1.PLLState = RCC_PLL_ON; RCC_OscInitStruct.PLL1.PLLSource = RCC_PLLSOURCE_HSE; RCC_OscInitStruct.PLL1.PLLM = 12; +#ifdef TRACE_ETM + // 300 MHz core -> 50 MHz trace clock: at 600 MHz the stream survives idle + // but dies (unknown trace packet) during IRQ-heavy bursts, e.g. USB traffic + RCC_OscInitStruct.PLL1.PLLN = 150; +#else RCC_OscInitStruct.PLL1.PLLN = 300; +#endif RCC_OscInitStruct.PLL1.PLLP = 1; RCC_OscInitStruct.PLL1.PLLQ = 2; RCC_OscInitStruct.PLL1.PLLR = 2; diff --git a/hw/bsp/stm32h7rs/boards/stm32h7s3nucleo/ozone/stm32h7s3.jdebug b/hw/bsp/stm32h7rs/boards/stm32h7s3nucleo/ozone/stm32h7s3.jdebug new file mode 100644 index 000000000..f6658d2d7 --- /dev/null +++ b/hw/bsp/stm32h7rs/boards/stm32h7s3nucleo/ozone/stm32h7s3.jdebug @@ -0,0 +1,94 @@ +/********************************************************************* +* +* OnProjectLoad +* +* Function description +* Project load routine. Required. +* +* Notes +* Nucleo-H7S3L8 wires 4-bit trace to the CN1 MIPI20 natively (no rework): +* TRACE_CLK/D0 = PE2/PE3, TRACE_D1/D2/D3 = PG14/PD2/PC12 (MB1737 Table 7). +* Firmware must be built with TRACE_ETM=1 (trace pin mux + DBGMCU +* DBGCKEN/TRACECLKEN in board_init). +* +* TRACE_ETM builds run a 300 MHz core (board.h) -> 50 MHz trace clock +* (cpu/3/2). Width 2 on the stub-free D0/D1 lines: SB11/SB12 (default ON) +* stub D2/D3 onto Zio CN8 and the 4-bit stream dies under IRQ-heavy USB +* traffic (idle is clean) - remove SB11/SB12 to try width 4 / 600 MHz. +* +********************************************************************** +*/ +void OnProjectLoad (void) { + // declares the off-ROM-table CSTF/TMC/TPIU (see the script header) + Project.SetJLinkScript ("./stm32h7s3_trace.JLinkScript"); + Project.SetTraceSource ("Trace Pins"); + Project.SetTracePortWidth (2); + Project.SetSWO (0); + Edit.SysVar (VAR_TRACE_CORE_CLOCK, 300000000); + Project.AddSvdFile ("$(InstallDir)/Config/CPU/Cortex-M7F.svd"); + + Project.SetDevice ("STM32H7S3L8"); + Project.SetHostIF ("USB", ""); + Project.SetTargetIF ("SWD"); + Project.SetTIFSpeed ("4 MHz"); + + File.Open ("../../../../../../examples/cmake-build-stm32h7s3nucleo/device/cdc_msc/cdc_msc.elf"); +} + +/********************************************************************* +* +* AfterTargetReset +* +* Function description +* Event handler routine. +* - Sets the PC register to program reset value. +* - Sets the SP register to program reset value on Cortex-M. +* +********************************************************************** +*/ +void AfterTargetReset (void) { + unsigned int SP; + unsigned int PC; + unsigned int VectorTableAddr; + + VectorTableAddr = Elf.GetBaseAddr(); + + if (VectorTableAddr == 0xFFFFFFFF) { + Util.Log("Project file error: failed to get program base"); + } else { + SP = Target.ReadU32(VectorTableAddr); + Target.SetReg("SP", SP); + + PC = Target.ReadU32(VectorTableAddr + 4); + Target.SetReg("PC", PC); + } +} + +/********************************************************************* +* +* AfterTargetDownload +* +* Function description +* Event handler routine. +* - Sets the PC register to program reset value. +* - Sets the SP register to program reset value on Cortex-M. +* +********************************************************************** +*/ +void AfterTargetDownload (void) { + unsigned int SP; + unsigned int PC; + unsigned int VectorTableAddr; + + VectorTableAddr = Elf.GetBaseAddr(); + + if (VectorTableAddr == 0xFFFFFFFF) { + Util.Log("Project file error: failed to get program base"); + } else { + SP = Target.ReadU32(VectorTableAddr); + Target.SetReg("SP", SP); + + PC = Target.ReadU32(VectorTableAddr + 4); + Target.SetReg("PC", PC); + } +} diff --git a/hw/bsp/stm32h7rs/boards/stm32h7s3nucleo/ozone/stm32h7s3_trace.JLinkScript b/hw/bsp/stm32h7rs/boards/stm32h7s3nucleo/ozone/stm32h7s3_trace.JLinkScript new file mode 100644 index 000000000..5ef4f164d --- /dev/null +++ b/hw/bsp/stm32h7rs/boards/stm32h7s3nucleo/ozone/stm32h7s3_trace.JLinkScript @@ -0,0 +1,11 @@ +/* STM32H7RS pin trace: the CSTF funnel, TMC (ETF) and TPIU are not in the + * core ROM table - declare them or J-Link aborts trace init with "Required + * trace components for pin trace not found!" (addresses per SEGGER's + * NUCLEO-H7S3L8 trace example script; AP0). + */ +int ConfigTargetSettings(void) { + JLINK_ExecCommand("CORESIGHT_SetCSTFBaseAddr = 0x5C013000 ForceUnlock = 1 APIndex = 0"); + JLINK_ExecCommand("CORESIGHT_SetTMCBaseAddr = 0x5C014000 ForceUnlock = 1 APIndex = 0"); + JLINK_ExecCommand("CORESIGHT_SetTPIUBaseAddr = 0x5C015000 ForceUnlock = 1 APIndex = 0"); + return 0; +} diff --git a/hw/bsp/stm32h7rs/family.c b/hw/bsp/stm32h7rs/family.c index b0841c947..385b3c929 100644 --- a/hw/bsp/stm32h7rs/family.c +++ b/hw/bsp/stm32h7rs/family.c @@ -98,17 +98,24 @@ void OTG_HS_IRQHandler(void) { #ifdef TRACE_ETM void trace_etm_init(void) { - // H7 trace pin is PE2 to PE6 - GPIO_InitTypeDef gpio_init; - gpio_init.Pin = GPIO_PIN_2 | GPIO_PIN_3 | GPIO_PIN_4 | GPIO_PIN_5 | GPIO_PIN_6; + // Nucleo-H7S3L8 routes 4-bit trace to the CN1 MIPI20: TRACE_CLK/D0 on + // PE2/PE3, TRACE_D1/D2/D3 on the PG14/PD2/PC12 alternates (MB1737 Table 7). + // No pull: pull-ups degrade the edges at the 100 MHz trace clock (= cpu/3/2) + GPIO_InitTypeDef gpio_init; gpio_init.Mode = GPIO_MODE_AF_PP; - gpio_init.Pull = GPIO_PULLUP; + gpio_init.Pull = GPIO_NOPULL; gpio_init.Speed = GPIO_SPEED_FREQ_VERY_HIGH; gpio_init.Alternate = GPIO_AF0_TRACE; + gpio_init.Pin = GPIO_PIN_2 | GPIO_PIN_3; HAL_GPIO_Init(GPIOE, &gpio_init); - - // Enable trace clk, also in D1 and D3 domain - DBGMCU->CR |= DBGMCU_CR_DBG_TRACECKEN | DBGMCU_CR_DBG_CKD1EN | DBGMCU_CR_DBG_CKD3EN; + gpio_init.Pin = GPIO_PIN_14; + HAL_GPIO_Init(GPIOG, &gpio_init); + gpio_init.Pin = GPIO_PIN_2; + HAL_GPIO_Init(GPIOD, &gpio_init); + gpio_init.Pin = GPIO_PIN_12; + HAL_GPIO_Init(GPIOC, &gpio_init); + + DBGMCU->CR |= DBGMCU_CR_DBGCKEN | DBGMCU_CR_TRACECLKEN; } #else #define trace_etm_init() -- cgit v1.3.1 From 63bccf47c632fcd6834eaca783c5ab15622d8c47 Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 24 Jul 2026 15:28:52 +0700 Subject: stm32n6: TRACE_ETM support and stm32n657nucleo reference M55 flashless RAM image: Development boot (JP2/BOOT1=1) REQUIRED - flash boot parks the chip un-attachable. 300 MHz core (TRACE_ETM selects IC1/4; 600 MHz kills the stream in the startup burst), 18.75 MHz TRACECLK (cpu/16) width 4; N6 trace components are ROM-table-discoverable, no J-Link script. --- hw/bsp/stm32n6/boards/stm32n657nucleo/board.h | 6 ++ .../boards/stm32n657nucleo/ozone/stm32n657.jdebug | 94 ++++++++++++++++++++++ hw/bsp/stm32n6/family.c | 22 +++++ 3 files changed, 122 insertions(+) create mode 100644 hw/bsp/stm32n6/boards/stm32n657nucleo/ozone/stm32n657.jdebug (limited to 'hw/bsp') diff --git a/hw/bsp/stm32n6/boards/stm32n657nucleo/board.h b/hw/bsp/stm32n6/boards/stm32n657nucleo/board.h index be9ea7a31..873c004d9 100644 --- a/hw/bsp/stm32n6/boards/stm32n657nucleo/board.h +++ b/hw/bsp/stm32n6/boards/stm32n657nucleo/board.h @@ -152,7 +152,13 @@ static void SystemClock_Config(void) { RCC_CLOCKTYPE_PCLK1 | RCC_CLOCKTYPE_PCLK2 | RCC_CLOCKTYPE_PCLK4 | RCC_CLOCKTYPE_PCLK5); RCC_ClkInitStruct.CPUCLKSource = RCC_CPUCLKSOURCE_IC1; RCC_ClkInitStruct.IC1Selection.ClockSelection = RCC_ICCLKSOURCE_PLL1; +#ifdef TRACE_ETM + // 300 MHz CPU -> 37.5 MHz TPIU clock (fixed cpu/8): at 600 MHz the trace + // stream dies with unknown-packet decode errors in the startup burst + RCC_ClkInitStruct.IC1Selection.ClockDivider = 4; +#else RCC_ClkInitStruct.IC1Selection.ClockDivider = 2; +#endif RCC_ClkInitStruct.SYSCLKSource = RCC_SYSCLKSOURCE_IC2_IC6_IC11; RCC_ClkInitStruct.IC2Selection.ClockSelection = RCC_ICCLKSOURCE_PLL1; RCC_ClkInitStruct.IC2Selection.ClockDivider = 3; diff --git a/hw/bsp/stm32n6/boards/stm32n657nucleo/ozone/stm32n657.jdebug b/hw/bsp/stm32n6/boards/stm32n657nucleo/ozone/stm32n657.jdebug new file mode 100644 index 000000000..16eead280 --- /dev/null +++ b/hw/bsp/stm32n6/boards/stm32n657nucleo/ozone/stm32n657.jdebug @@ -0,0 +1,94 @@ +/********************************************************************* +* +* OnProjectLoad +* +* Function description +* Project load routine. Required. +* +* Notes +* Nucleo-N657X0-Q wires 4-bit trace to the CN1 MIPI20 natively (SB36/38/ +* 39/40/41 fitted by default): TRACE_CLK = PB3, D0/D1/D2/D3 = +* PE3/PB0/PB6/PB7 (MB1940 Table 5). No J-Link script needed - the N6 +* trace components sit behind a ROM table J-Link discovers natively. +* +* BOOT1 (JP2) must select Development boot (BOOT1 = 1): the N657 is +* flashless, the app is a RAM image (AXISRAM2) loaded by the debugger, +* and in flash boot the bootROM parks the chip un-attachable. +* Firmware must be built with TRACE_ETM=1 (pin mux + DBGMCU +* DBGCLKEN/TRACECLKEN). Trace clock = cpu/8. TRACE_ETM builds run a 300 MHz core (board.h) -> +* 37.5 MHz TPIU clock: 600 MHz kills the stream in the startup burst. +* +********************************************************************** +*/ +void OnProjectLoad (void) { + Project.SetTraceSource ("Trace Pins"); + Project.SetTracePortWidth (4); + Project.SetSWO (0); + Edit.SysVar (VAR_TRACE_CORE_CLOCK, 300000000); + Project.AddSvdFile ("$(InstallDir)/Config/CPU/Cortex-M55F.svd"); + + Project.SetDevice ("STM32N657X0"); + Project.SetHostIF ("USB", ""); + Project.SetTargetIF ("SWD"); + Project.SetTIFSpeed ("4 MHz"); + + File.Open ("../../../../../../examples/cmake-build-stm32n657nucleo/device/cdc_msc/cdc_msc.elf"); +} + +/********************************************************************* +* +* AfterTargetReset +* +* Function description +* Event handler routine. +* - Sets the PC register to program reset value. +* - Sets the SP register to program reset value on Cortex-M. +* +********************************************************************** +*/ +void AfterTargetReset (void) { + unsigned int SP; + unsigned int PC; + unsigned int VectorTableAddr; + + VectorTableAddr = Elf.GetBaseAddr(); + + if (VectorTableAddr == 0xFFFFFFFF) { + Util.Log("Project file error: failed to get program base"); + } else { + SP = Target.ReadU32(VectorTableAddr); + Target.SetReg("SP", SP); + + PC = Target.ReadU32(VectorTableAddr + 4); + Target.SetReg("PC", PC); + } +} + +/********************************************************************* +* +* AfterTargetDownload +* +* Function description +* Event handler routine. +* - Sets the PC register to program reset value. +* - Sets the SP register to program reset value on Cortex-M. +* +********************************************************************** +*/ +void AfterTargetDownload (void) { + unsigned int SP; + unsigned int PC; + unsigned int VectorTableAddr; + + VectorTableAddr = Elf.GetBaseAddr(); + + if (VectorTableAddr == 0xFFFFFFFF) { + Util.Log("Project file error: failed to get program base"); + } else { + SP = Target.ReadU32(VectorTableAddr); + Target.SetReg("SP", SP); + + PC = Target.ReadU32(VectorTableAddr + 4); + Target.SetReg("PC", PC); + } +} diff --git a/hw/bsp/stm32n6/family.c b/hw/bsp/stm32n6/family.c index 80de20c6a..9d3faa1f2 100644 --- a/hw/bsp/stm32n6/family.c +++ b/hw/bsp/stm32n6/family.c @@ -110,6 +110,27 @@ void USB1_OTG_HS_IRQHandler(void) { tusb_int_handler(0, true); } +#ifdef TRACE_ETM +static void trace_etm_init(void) { + // Nucleo-N657X0-Q routes 4-bit trace to the CN1 MIPI20 natively: + // TRACE_CLK = PB3, D0 = PE3, D1 = PB0, D2 = PB6, D3 = PB7 (MB1940 Table 5) + GPIO_InitTypeDef gpio_init; + gpio_init.Mode = GPIO_MODE_AF_PP; + gpio_init.Pull = GPIO_NOPULL; + gpio_init.Speed = GPIO_SPEED_FREQ_VERY_HIGH; + gpio_init.Alternate = GPIO_AF0_TRACE; + gpio_init.Pin = GPIO_PIN_0 | GPIO_PIN_3 | GPIO_PIN_6 | GPIO_PIN_7; + HAL_GPIO_Init(GPIOB, &gpio_init); + gpio_init.Pin = GPIO_PIN_3; + HAL_GPIO_Init(GPIOE, &gpio_init); + + // trace clock (ck_cpu_tpiu) is a fixed cpu/8 - just enable it + DBGMCU->CR |= DBGMCU_CR_DBGCLKEN | DBGMCU_CR_TRACECLKEN; +} +#else + #define trace_etm_init() +#endif + void board_init(void) { /* Enable BusFault and SecureFault handlers (HardFault is default) */ SCB->SHCSR |= (SCB_SHCSR_BUSFAULTENA_Msk | SCB_SHCSR_SECUREFAULTENA_Msk); @@ -148,6 +169,7 @@ void board_init(void) { for (uint8_t i = 0; i < TU_ARRAY_SIZE(board_pindef); i++) { HAL_GPIO_Init(board_pindef[i].port, &board_pindef[i].pin_init); } + trace_etm_init(); NVIC_SetPriority(UCPD1_IRQn, NVIC_EncodePriority(NVIC_GetPriorityGrouping(),5, 0)); NVIC_EnableIRQ(UCPD1_IRQn); -- cgit v1.3.1 From 4d9e4c9e3895b0c15a04763028d1becd27bbf89c Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 24 Jul 2026 15:28:53 +0700 Subject: ra: TRACE_ETM for ra6m5_ek and ra8m1_ek Generic TRCKCR setup gated on DHCSR.C_DEBUGEN (a standalone-boot TRCKCR write wedges the chip un-attachable until power-cycle), two-step write per the hardware manual. ra6m5_ek: div-4 (25 MHz pin) - div-2 is dead on this board at every width/timing; J9 must be closed. ra8m1_ek: chip-max 120 MHz TRCLK / 60 MHz pin via the committed JLinkScript whose empty OnTraceStart defers the trace clock to firmware (J-Link's from-reset enable steps the clock mid-stream at the FSP MOCO-to-PLL switch); ReadIntoTraceCache covers runtime ROM execution. J9 closed on both EKs - open = SWD contention up to apparent bricks. --- hw/bsp/ra/boards/ra6m5_ek/ozone/ra6m5.jdebug | 2 +- hw/bsp/ra/boards/ra8m1_ek/ozone/ra8m1.jdebug | 8 ++++++-- .../boards/ra8m1_ek/ozone/ra8m1_trace.JLinkScript | 20 ++++++++++++++++++++ hw/bsp/ra/family.c | 21 +++++++++++++++++++-- 4 files changed, 46 insertions(+), 5 deletions(-) create mode 100644 hw/bsp/ra/boards/ra8m1_ek/ozone/ra8m1_trace.JLinkScript (limited to 'hw/bsp') diff --git a/hw/bsp/ra/boards/ra6m5_ek/ozone/ra6m5.jdebug b/hw/bsp/ra/boards/ra6m5_ek/ozone/ra6m5.jdebug index ca18fed7c..466658f39 100644 --- a/hw/bsp/ra/boards/ra6m5_ek/ozone/ra6m5.jdebug +++ b/hw/bsp/ra/boards/ra6m5_ek/ozone/ra6m5.jdebug @@ -15,7 +15,7 @@ void OnProjectLoad (void) { Project.SetDevice ("R7FA6M5BH"); Project.SetHostIF ("USB", ""); Project.SetTargetIF ("SWD"); - Project.SetTIFSpeed ("50 MHz"); + Project.SetTIFSpeed ("4 MHz"); // 50 MHz SWD gave intermittent "Failed to initialize DAP" Project.SetTraceSource ("Trace Pins"); Project.SetTracePortWidth (4); diff --git a/hw/bsp/ra/boards/ra8m1_ek/ozone/ra8m1.jdebug b/hw/bsp/ra/boards/ra8m1_ek/ozone/ra8m1.jdebug index 242a15db9..f0a43bf5f 100644 --- a/hw/bsp/ra/boards/ra8m1_ek/ozone/ra8m1.jdebug +++ b/hw/bsp/ra/boards/ra8m1_ek/ozone/ra8m1.jdebug @@ -13,7 +13,7 @@ void OnProjectLoad (void) { Project.SetDevice ("R7FA8M1AH"); Project.SetHostIF ("USB", ""); Project.SetTargetIF ("SWD"); - Project.SetTIFSpeed ("50 MHz"); + Project.SetTIFSpeed ("4 MHz"); // 50 MHz SWD gave intermittent "Failed to initialize DAP" Project.AddSvdFile ("$(InstallDir)/Config/CPU/Cortex-M85F.svd"); Project.AddSvdFile ("../../../../../../../cmsis-svd-data/data/Renesas/R7FA6M5BH.svd"); @@ -32,7 +32,7 @@ void OnProjectLoad (void) { void BeforeTargetConnect (void) { // Trace pin init is done by J-Link script file as J-Link script files are IDE independent //Project.SetJLinkScript("../../../debug.jlinkscript"); - Project.SetJLinkScript ("$(ProjectDir)/Renesas_RA8_TracePins.pex"); + Project.SetJLinkScript ("./ra8m1_trace.JLinkScript"); } /********************************************************************* @@ -83,8 +83,12 @@ void BeforeTargetConnect (void) { */ void AfterTargetDownload (void) { _SetupTarget(); + // RA8 executes chip-ROM code at runtime (seen at ~0x3B20); without this the + // trace decoder dies there ("not covered by trace cache" -> unknown packet) + Exec.Command("ReadIntoTraceCache 0x0 0x10000"); } + /********************************************************************* * * BeforeTargetDisconnect diff --git a/hw/bsp/ra/boards/ra8m1_ek/ozone/ra8m1_trace.JLinkScript b/hw/bsp/ra/boards/ra8m1_ek/ozone/ra8m1_trace.JLinkScript new file mode 100644 index 000000000..3869c1eb7 --- /dev/null +++ b/hw/bsp/ra/boards/ra8m1_ek/ozone/ra8m1_trace.JLinkScript @@ -0,0 +1,20 @@ +/* RA8M1 pin trace: CSTF funnel and TMC are off the core ROM table (AP1) - + * declare them or trace init cannot route the M85 ETM stream (addresses per + * SEGGER's RA8 trace example script). Pins + TRCKCR belong to firmware + * (trace_etm_init): the SEGGER pex also muxed them at trace start, and its + * clock choice fought the firmware's mid-run = decoder desync. + */ +int ConfigTargetSettings(void) { + JLINK_ExecCommand("CORESIGHT_SetCSTFBaseAddr = 0x80013000 ForceUnlock = 1 APIndex = 1"); + JLINK_ExecCommand("CORESIGHT_SetTMCBaseAddr = 0x80014000 ForceUnlock = 1 APIndex = 1"); + return 0; +} + +/* Replace J-Link's built-in RA8 trace start, which enables the trace clock + * from reset - bsp_clock_init's MOCO -> 480 MHz PLL switch would then step + * TRCLK mid-stream and desync the decoder. The firmware's trace_etm_init + * enables TRCKCR at the final clock instead. + */ +int OnTraceStart(void) { + return 0; +} diff --git a/hw/bsp/ra/family.c b/hw/bsp/ra/family.c index c8a4d33d9..3c548fa7f 100644 --- a/hw/bsp/ra/family.c +++ b/hw/bsp/ra/family.c @@ -99,13 +99,30 @@ void board_init(void) { R_IOPORT_Open(&IOPORT_CFG_CTRL, &IOPORT_CFG_NAME); #ifdef TRACE_ETM + // TRCKCR is only writable while a debugger is connected (RA HUM) - a + // standalone boot must skip trace init or the write wedges the chip into + // an un-attachable crash loop (recover: power-cycle + immediate erase) + if (DCB->DHCSR & DCB_DHCSR_C_DEBUGEN_Msk) { // TRCKCR is protected by PRCR bit0 register R_SYSTEM->PRCR = (uint16_t) (BSP_PRV_PRCR_KEY | 0x01); - // Enable trace clock (max 100Mhz). Since PLL/CPU is 200Mhz, clock div = 2 - R_SYSTEM->TRCKCR = R_SYSTEM_TRCKCR_TRCKEN_Msk | 0x01; + // TCLK pin = TRCLK/2; set the divider with TRCKEN=0 first (HUM procedure). + // Values are the empirical per-board ceilings: one step below the divider + // at which the stream dies mid-run. +#if defined(BSP_MCU_GROUP_RA8M1) + // 480 MHz CPU: /8 -> 60 MHz TRCLK, 30 MHz pin (/4 = 60 MHz pin dies) + R_SYSTEM->TRCKCR = 0x02; + R_SYSTEM->TRCKCR = R_SYSTEM_TRCKCR_TRCKEN_Msk | 0x02; +#else + // RA6M5 200 MHz CPU: /4 -> 50 MHz TRCLK, 25 MHz pin. /2 (50 MHz pin) is + // silent on the EK-RA6M5 in every combination - board path ceiling, + // reconfirmed with J9 closed and the OnTraceStart override + R_SYSTEM->TRCKCR = 0x02; + R_SYSTEM->TRCKCR = R_SYSTEM_TRCKCR_TRCKEN_Msk | 0x02; +#endif R_SYSTEM->PRCR = (uint16_t) BSP_PRV_PRCR_KEY; + } #endif #if CFG_TUSB_OS == OPT_OS_FREERTOS -- cgit v1.3.1 From e90d232b7718656a627775e3ecde63c6fc7c2e84 Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 24 Jul 2026 15:28:53 +0700 Subject: rp2040: RP2350/pico2 ETM trace over fly-wired MIPI-20 J-Link's built-in RP2350 script owns the whole chip-side path (component map is not ROM-table-discoverable; a custom JLinkScript replaces the built-in one and kills pin trace), re-arming at every resume - firmware does no trace setup. TRACE_ETM builds pin clk_sys to 48 MHz from crt0 (fly-wire seating-proof; the port is DDR at clk_sys/2 and the J-Trace PRO V2 cliff sits just above 40 MHz TRACECLK - SEGGER requires V3.0+ for this chip), clear TIMER0/1 DBGPAUSE (default freezes the us-timer while any core is debug-halted and sleep_ms spins forever), and run the UART console TX-only (GPIO1 = default UART0 RX = TRACECLK). --- .../rp2040/boards/raspberry_pi_pico2/board.cmake | 14 +++++ .../boards/raspberry_pi_pico2/ozone/rp2350.jdebug | 70 ++++++++++++++++++++++ hw/bsp/rp2040/family.c | 24 +++++++- 3 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 hw/bsp/rp2040/boards/raspberry_pi_pico2/ozone/rp2350.jdebug (limited to 'hw/bsp') diff --git a/hw/bsp/rp2040/boards/raspberry_pi_pico2/board.cmake b/hw/bsp/rp2040/boards/raspberry_pi_pico2/board.cmake index 0a7dd4d23..08384b0cd 100644 --- a/hw/bsp/rp2040/boards/raspberry_pi_pico2/board.cmake +++ b/hw/bsp/rp2040/boards/raspberry_pi_pico2/board.cmake @@ -1,3 +1,17 @@ set(PICO_PLATFORM rp2350-arm-s) set(PICO_BOARD pico2) #set(OPENOCD_SERIAL E6614103E77C5A24) + +if (TRACE_ETM STREQUAL "1") + # TRACECLK is clk_sys/2 and must stay constant once trace is armed (a step + # desyncs the decoder), so the trace clock is pinned from crt0 onwards. + # 48 MHz (24 MHz TRACECLK) holds full-width trace on a typical fly-wire + # seating; a fresh, tight seating supports up to 72-80 MHz (re-qualify per + # the etm-trace skill), and >80 MHz needs a V3 probe + real trace board. + add_compile_definitions( + SYS_CLK_KHZ=48000 + PLL_SYS_VCO_FREQ_HZ=1440000000 + PLL_SYS_POSTDIV1=6 + PLL_SYS_POSTDIV2=5 + ) +endif () diff --git a/hw/bsp/rp2040/boards/raspberry_pi_pico2/ozone/rp2350.jdebug b/hw/bsp/rp2040/boards/raspberry_pi_pico2/ozone/rp2350.jdebug new file mode 100644 index 000000000..ff48eb673 --- /dev/null +++ b/hw/bsp/rp2040/boards/raspberry_pi_pico2/ozone/rp2350.jdebug @@ -0,0 +1,70 @@ +/********************************************************************* +* +* OnProjectLoad +* +* Function description +* Project load routine. Required. +* +* Notes +* Pico 2 has no trace connector - fly-wire GPIO1-5 to the MIPI20: +* TRACECLK=GPIO1->12, D0=GPIO2->14, D1=GPIO3->16, D2=GPIO4->18, +* D3=GPIO5->20 (SEGGER validates this board the same way). Firmware must +* be built with TRACE_ETM=1: it pins clk_sys to 48 MHz (board.cmake) so +* the 4-bit port never saturates and the clock never steps mid-stream, +* and keeps the us-timer free of TIMER DBGPAUSE (family.c). The whole +* chip-side trace path (ETM/funnel/TPIU/pin mux) is armed by J-Link's +* built-in RP2350 script at every resume - do NOT set a custom +* JLinkScript here: it would replace that script and J-Link then fails +* with "Required trace components for pin trace not found". +* GPIO1 is the default UART0 RX: console TX still works, RX is lost. +* +********************************************************************** +*/ +void OnProjectLoad (void) { + Project.SetTraceSource ("Trace Pins"); + Project.SetTracePortWidth (4); + Project.SetSWO (0); + Edit.SysVar (VAR_TRACE_CORE_CLOCK, 48000000); + Project.AddSvdFile ("$(InstallDir)/Config/CPU/Cortex-M33F.svd"); + + Project.SetDevice ("RP2350_M33_0"); + Project.SetHostIF ("USB", ""); + Project.SetTargetIF ("SWD"); + Project.SetTIFSpeed ("25 MHz"); + + File.Open ("../../../../../../examples/cmake-build-raspberry_pi_pico2/device/cdc_msc/cdc_msc.elf"); +} + +/********************************************************************* +* +* AfterTargetReset +* +* Function description +* Event handler routine. +* - Sets the PC register to program reset value. +* - Sets the SP register to program reset value on Cortex-M. +* +********************************************************************** +*/ +void AfterTargetReset (void) { + // intentionally empty: the RP2350 bootrom must run to validate the + // IMAGE_DEF and hand over to the app - setting SP/PC from the vector + // table bypasses it and the pico-sdk runtime never comes up +} + +/********************************************************************* +* +* AfterTargetDownload +* +* Function description +* Event handler routine. +* - Sets the PC register to program reset value. +* - Sets the SP register to program reset value on Cortex-M. +* +********************************************************************** +*/ +void AfterTargetDownload (void) { + // intentionally empty: the RP2350 bootrom must run to validate the + // IMAGE_DEF and hand over to the app - setting SP/PC from the vector + // table bypasses it and the pico-sdk runtime never comes up +} diff --git a/hw/bsp/rp2040/family.c b/hw/bsp/rp2040/family.c index 15f179656..f0d6ba245 100644 --- a/hw/bsp/rp2040/family.c +++ b/hw/bsp/rp2040/family.c @@ -161,6 +161,20 @@ static void stdio_rtt_init(void) { //--------------------------------------------------------------------+ // //--------------------------------------------------------------------+ +#if defined(TRACE_ETM) && defined(PICO_RP2350) && PICO_RP2350 == 1 +// J-Link's built-in RP2350 device script re-arms the whole chip-side trace +// path (ETM/funnel/TPIU/pins) via OnTraceStart at every resume, so firmware +// must NOT touch it - it only keeps the us-timer running while cores sit +// debug-halted (default TIMER DBGPAUSE freezes it, and sleep_ms() then spins +// forever after any debugger session). +static void trace_etm_init(void) { + *(volatile uint32_t*) 0x400B002Cu = 0; // TIMER0 DBGPAUSE + *(volatile uint32_t*) 0x400B802Cu = 0; // TIMER1 DBGPAUSE +} +#else + #define trace_etm_init() +#endif + void board_init(void) { #if (CFG_TUH_ENABLED && CFG_TUH_RPI_PIO_USB) || (CFG_TUD_ENABLED && CFG_TUD_RPI_PIO_USB) @@ -199,10 +213,18 @@ void board_init(void) #endif #ifdef UART_DEV - bi_decl(bi_2pins_with_func(UART_TX_PIN, UART_RX_PIN, GPIO_FUNC_UART)); uart_inst = uart_get_instance(UART_DEV); +#if defined(TRACE_ETM) && defined(PICO_RP2350) && PICO_RP2350 == 1 + // GPIO1 (default UART RX) is TRACECLK: TX-only console, and never touch + // GPIO1 - even a brief re-mux gaps the trace clock and desyncs the probe + bi_decl(bi_1pin_with_name(UART_TX_PIN, "UART TX")); + stdio_uart_init_full(uart_inst, CFG_BOARD_UART_BAUDRATE, UART_TX_PIN, -1); +#else + bi_decl(bi_2pins_with_func(UART_TX_PIN, UART_RX_PIN, GPIO_FUNC_UART)); stdio_uart_init_full(uart_inst, CFG_BOARD_UART_BAUDRATE, UART_TX_PIN, UART_RX_PIN); #endif +#endif + trace_etm_init(); #if defined(LOGGER_RTT) stdio_rtt_init(); -- cgit v1.3.1 From 853f7e8f70caf4df9786001e84eca048d2664de7 Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 24 Jul 2026 15:28:53 +0700 Subject: samd5x_e5x: ETM trace support for same54_xplained The populated 20-pin Cortex Debug+ETM header carries 4-bit trace (TRACECLK=PC27, D0-3=PC28/PC26/PC25/PC24, mux H). TRACE_ETM builds mux the pins and enable GCLK channel 47 (GCLK_CM4_TRACE) from GCLK0 - without that gate the port stays silent with pins and TPIU armed. Chip-max 120 MHz core / 60 MHz TRACECLK validated (3x 280M-fetch captures). --- .../boards/same54_xplained/ozone/same54.jdebug | 89 ++++++++++++++++++++++ hw/bsp/samd5x_e5x/family.c | 27 +++++++ 2 files changed, 116 insertions(+) create mode 100644 hw/bsp/samd5x_e5x/boards/same54_xplained/ozone/same54.jdebug (limited to 'hw/bsp') diff --git a/hw/bsp/samd5x_e5x/boards/same54_xplained/ozone/same54.jdebug b/hw/bsp/samd5x_e5x/boards/same54_xplained/ozone/same54.jdebug new file mode 100644 index 000000000..a0f8d5c3d --- /dev/null +++ b/hw/bsp/samd5x_e5x/boards/same54_xplained/ozone/same54.jdebug @@ -0,0 +1,89 @@ +/********************************************************************* +* +* OnProjectLoad +* +* Function description +* Project load routine. Required. +* +* Notes +* SAM E54 Xplained Pro carries a populated 20-pin Cortex Debug+ETM +* connector (Table 5-11): TRACECLK=PC27, D0=PC28, D1=PC26, D2=PC25, +* D3=PC24 - no rework needed. Firmware must be built with TRACE_ETM=1 +* (mux function H on those pins in board_init); TPIU/ETM are +* ROM-table-discoverable so no J-Link script is required. +* Trace clock is CPU/2 = 60 MHz at the stock 120 MHz core. +* +********************************************************************** +*/ +void OnProjectLoad (void) { + Project.SetTraceSource ("Trace Pins"); + Project.SetTracePortWidth (4); + Project.SetSWO (0); + Edit.SysVar (VAR_TRACE_CORE_CLOCK, 120000000); + Project.AddSvdFile ("$(InstallDir)/Config/CPU/Cortex-M4F.svd"); + + Project.SetDevice ("ATSAME54P20"); + Project.SetHostIF ("USB", ""); + Project.SetTargetIF ("SWD"); + Project.SetTIFSpeed ("4 MHz"); + + File.Open ("../../../../../../examples/cmake-build-same54_xplained/device/cdc_msc/cdc_msc.elf"); +} + +/********************************************************************* +* +* AfterTargetReset +* +* Function description +* Event handler routine. +* - Sets the PC register to program reset value. +* - Sets the SP register to program reset value on Cortex-M. +* +********************************************************************** +*/ +void AfterTargetReset (void) { + unsigned int SP; + unsigned int PC; + unsigned int VectorTableAddr; + + VectorTableAddr = Elf.GetBaseAddr(); + + if (VectorTableAddr == 0xFFFFFFFF) { + Util.Log("Project file error: failed to get program base"); + } else { + SP = Target.ReadU32(VectorTableAddr); + Target.SetReg("SP", SP); + + PC = Target.ReadU32(VectorTableAddr + 4); + Target.SetReg("PC", PC); + } +} + +/********************************************************************* +* +* AfterTargetDownload +* +* Function description +* Event handler routine. +* - Sets the PC register to program reset value. +* - Sets the SP register to program reset value on Cortex-M. +* +********************************************************************** +*/ +void AfterTargetDownload (void) { + unsigned int SP; + unsigned int PC; + unsigned int VectorTableAddr; + + VectorTableAddr = Elf.GetBaseAddr(); + + if (VectorTableAddr == 0xFFFFFFFF) { + Util.Log("Project file error: failed to get program base"); + } else { + SP = Target.ReadU32(VectorTableAddr); + Target.SetReg("SP", SP); + + PC = Target.ReadU32(VectorTableAddr + 4); + Target.SetReg("PC", PC); + } +} diff --git a/hw/bsp/samd5x_e5x/family.c b/hw/bsp/samd5x_e5x/family.c index 71ef2d6ce..a2c3b70de 100644 --- a/hw/bsp/samd5x_e5x/family.c +++ b/hw/bsp/samd5x_e5x/family.c @@ -87,6 +87,31 @@ void USB_3_Handler(void) { USB_Any_Handler(); } static void max3421_init(void); #endif +#if defined(TRACE_ETM) +// same54_xplained routes 4-bit trace to its 20-pin Cortex Debug+ETM header: +// TRACECLK=PC27, D0=PC28, D1=PC26, D2=PC25, D3=PC24 - all peripheral +// function H (CM4 trace). TPIU/ETM are ROM-table-discoverable; the debugger +// arms them, firmware only muxes the pins. +static void trace_etm_init(void) { + // the CM4 trace unit runs from its own GCLK channel (47): feed it GCLK0 + // (CPU clock) - without this the pins mux fine but the port stays silent + GCLK->PCHCTRL[47].reg = GCLK_PCHCTRL_GEN_GCLK0 | GCLK_PCHCTRL_CHEN; + while (!(GCLK->PCHCTRL[47].reg & GCLK_PCHCTRL_CHEN)) {} + + const uint8_t pin[] = {24, 25, 26, 27, 28}; + for (unsigned i = 0; i < 5; i++) { + PORT->Group[2].PINCFG[pin[i]].reg = PORT_PINCFG_PMUXEN | PORT_PINCFG_DRVSTR; + if (pin[i] & 1) { + PORT->Group[2].PMUX[pin[i] >> 1].bit.PMUXO = 7; // function H + } else { + PORT->Group[2].PMUX[pin[i] >> 1].bit.PMUXE = 7; + } + } +} +#else + #define trace_etm_init() +#endif + void board_init(void) { // Clock init ( follow hpl_init.c ) hri_nvmctrl_set_CTRLA_RWS_bf(NVMCTRL, 0); @@ -104,6 +129,8 @@ void board_init(void) { // Init 1ms tick timer (samd SystemCoreClock may not correct) SystemCoreClock = CONF_CPU_FREQUENCY; + trace_etm_init(); + #if CFG_TUSB_OS == OPT_OS_NONE SysTick_Config(CONF_CPU_FREQUENCY / 1000); #elif CFG_TUSB_OS == OPT_OS_FREERTOS -- cgit v1.3.1 From 2a2d5f65ad61dc43d20ff479db9a27510592b50c Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 24 Jul 2026 15:28:54 +0700 Subject: same7x: ETM trace support for same70_xplained J403 (bottom-side Cortex Debug+ETM footprint, header required): TRACECLK=PD8 peripheral D, TRACED0-3=PD4-7 peripheral C. TRACE_ETM builds hold the KSZ8081 PHY in reset (PD4-7 are its RMII receive outputs and it drives against the trace stream), clock the TPIU from PCK3 (MCK/2) and mux the pins; the ozone reference starts PCK3 in the post-reset/download hooks - TPIU programming while PCK3 is stopped is silently lost. Width-1 validated at the stock 300 MHz core; width 4 blocked on a dead D1 line (suspect probe channel, h743eval crosscheck pending). --- .../boards/same70_xplained/ozone/same70.jdebug | 103 +++++++++++++++++++++ hw/bsp/same7x/family.c | 28 ++++++ 2 files changed, 131 insertions(+) create mode 100644 hw/bsp/same7x/boards/same70_xplained/ozone/same70.jdebug (limited to 'hw/bsp') diff --git a/hw/bsp/same7x/boards/same70_xplained/ozone/same70.jdebug b/hw/bsp/same7x/boards/same70_xplained/ozone/same70.jdebug new file mode 100644 index 000000000..0fde09716 --- /dev/null +++ b/hw/bsp/same7x/boards/same70_xplained/ozone/same70.jdebug @@ -0,0 +1,103 @@ +/********************************************************************* +* +* OnProjectLoad +* +* Function description +* Project load routine. Required. +* +* Notes +* SAM E70 Xplained: solder a 20-pin 50-mil header on the J403 ETM footprint +* (bottom side). TRACECLK=PD8 (peripheral D), TRACED0-3=PD4-7 (peripheral C), +* shared with the Ethernet PHY - no Ethernet while tracing. TRACE_ETM=1 +* builds mux the pins and clock the TPIU from PCK3=MCK; TPIU/ETM are +* ROM-table-discoverable so no J-Link script is required. +* Trace clock pin is PCK3/2 = 75 MHz at the stock 300 MHz core (MCK 150). +* +********************************************************************** +*/ +void OnProjectLoad (void) { + Project.SetTraceSource ("Trace Pins"); + Project.SetTracePortWidth (4); + Project.SetSWO (0); + Edit.SysVar (VAR_TRACE_CORE_CLOCK, 300000000); + Project.AddSvdFile ("$(InstallDir)/Config/CPU/Cortex-M7F.svd"); + + Project.SetDevice ("ATSAME70Q21B"); + Project.SetHostIF ("USB", ""); + Project.SetTargetIF ("SWD"); + Project.SetTIFSpeed ("4 MHz"); + + File.Open ("../../../../../../examples/cmake-build-same70_xplained/device/cdc_msc/cdc_msc.elf"); +} + +/********************************************************************* +* +* AfterTargetReset +* +* Function description +* Event handler routine. +* - Sets the PC register to program reset value. +* - Sets the SP register to program reset value on Cortex-M. +* +********************************************************************** +*/ +void AfterTargetReset (void) { + unsigned int SP; + unsigned int PC; + unsigned int VectorTableAddr; + + VectorTableAddr = Elf.GetBaseAddr(); + + if (VectorTableAddr == 0xFFFFFFFF) { + Util.Log("Project file error: failed to get program base"); + } else { + SP = Target.ReadU32(VectorTableAddr); + Target.SetReg("SP", SP); + + PC = Target.ReadU32(VectorTableAddr + 4); + Target.SetReg("PC", PC); + } + + // TPIU trace clock = PCK3; it must run BEFORE Ozone arms the trace + // components (TPIU programming while PCK3 is stopped is lost and the port + // emits unformatted garbage). A target reset wipes the PMC, so this runs + // in the post-reset/post-download hooks, not AfterTargetConnect. + Target.WriteU32 (0x400E064C, 0x00000014); // PMC_PCK3: CSS=MCK, PRESS=/2 + Target.WriteU32 (0x400E0600, 0x00000800); // PMC_SCER: PCK3 on +} + +/********************************************************************* +* +* AfterTargetDownload +* +* Function description +* Event handler routine. +* - Sets the PC register to program reset value. +* - Sets the SP register to program reset value on Cortex-M. +* +********************************************************************** +*/ +void AfterTargetDownload (void) { + unsigned int SP; + unsigned int PC; + unsigned int VectorTableAddr; + + VectorTableAddr = Elf.GetBaseAddr(); + + if (VectorTableAddr == 0xFFFFFFFF) { + Util.Log("Project file error: failed to get program base"); + } else { + SP = Target.ReadU32(VectorTableAddr); + Target.SetReg("SP", SP); + + PC = Target.ReadU32(VectorTableAddr + 4); + Target.SetReg("PC", PC); + } + + // TPIU trace clock = PCK3; it must run BEFORE Ozone arms the trace + // components (TPIU programming while PCK3 is stopped is lost and the port + // emits unformatted garbage). A target reset wipes the PMC, so this runs + // in the post-reset/post-download hooks, not AfterTargetConnect. + Target.WriteU32 (0x400E064C, 0x00000014); // PMC_PCK3: CSS=MCK, PRESS=/2 + Target.WriteU32 (0x400E0600, 0x00000800); // PMC_SCER: PCK3 on +} diff --git a/hw/bsp/same7x/family.c b/hw/bsp/same7x/family.c index d02e6c5f1..6a3466354 100644 --- a/hw/bsp/same7x/family.c +++ b/hw/bsp/same7x/family.c @@ -62,6 +62,34 @@ void board_init(void) { /* Disable Watchdog */ hri_wdt_set_MR_WDDIS_bit(WDT); +#if defined(TRACE_ETM) + // same70_xplained J403 (Cortex Debug+ETM footprint, bottom side) carries + // 4-bit trace: TRACECLK=PD8 (peripheral D), TRACED0-3=PD4-7 (peripheral C). + // The trace pins double as the Ethernet PHY's RMII receive lines + // (PD4=CRS_DV, PD5/6=RXD0/1, PD7=RXER - PHY OUTPUTS): hold the KSZ8081 in + // reset (PHY_RESET=PC10 low) or it drives against the trace stream. + _pmc_enable_periph_clock(ID_PIOC); + gpio_set_pin_level(GPIO(GPIO_PORTC, 10), false); + gpio_set_pin_direction(GPIO(GPIO_PORTC, 10), GPIO_DIRECTION_OUT); + gpio_set_pin_function(GPIO(GPIO_PORTC, 10), GPIO_PIN_FUNCTION_OFF); + + // The TPIU is clocked from PCK3 (datasheet 16.7.4) - run it from MCK. + // skip if the debugger already started PCK3 (reprogramming glitches the + // clock mid-stream and desyncs the decoder) + uint32_t const pck3 = PMC_PCK_CSS_MCK | PMC_PCK_PRES(1); // MCK/2 = 75 MHz -> 37.5 MHz pin + if (PMC->PMC_PCK[3] != pck3 || !(PMC->PMC_SR & PMC_SR_PCKRDY3)) { + PMC->PMC_PCK[3] = pck3; + PMC->PMC_SCER = PMC_SCER_PCK3; + while (!(PMC->PMC_SR & PMC_SR_PCKRDY3)) {} + } + uint32_t const clk_pin = PIO_PD8D_TPIU_TRACECLK; + uint32_t const dat_pin = PIO_PD4C_TPIU_TRACED0 | PIO_PD5C_TPIU_TRACED1 | + PIO_PD6C_TPIU_TRACED2 | PIO_PD7C_TPIU_TRACED3; + PIOD->PIO_ABCDSR[0] = (PIOD->PIO_ABCDSR[0] | clk_pin) & ~dat_pin; // D=11, C=01 + PIOD->PIO_ABCDSR[1] |= clk_pin | dat_pin; + PIOD->PIO_PDR = clk_pin | dat_pin; // hand the pins to the peripheral +#endif + #ifdef LED_PIN _pmc_enable_periph_clock(LED_PORT_CLOCK); gpio_set_pin_level(LED_PIN, LED_STATE_OFF); -- cgit v1.3.1 From d3eaeb06e04519b480637f6e7cb007a3de08d923 Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 24 Jul 2026 16:20:13 +0700 Subject: imxrt: hold EVKB Ethernet PHY in reset while tracing, 50 MHz trace pin Fresh bring-up pass on mimxrt1170_evkb: holding the 100M RTL8201 in reset (ENET_RST_B = GPIO_LPSR_04) stops its RMII lines driving against the shared trace pads and doubles the clean trace-pin rate to 50 MHz (100 MHz CSTRACE root; 133 MHz root is marginal, stock 132 corrupts). Validated 3x 8 s TinyUSB captures at 11.46M fetches. D1-D3 remain silent in every configuration - the welded R1882-R1884 are electrically open; reflow is the remaining step to width 4. Board notes gain JP4 (must be shorted for an external probe on J58). --- .claude/skills/etm-trace/boards.md | 22 ++++++++++++++-------- hw/bsp/imxrt/family.c | 13 +++++++++++-- 2 files changed, 25 insertions(+), 10 deletions(-) (limited to 'hw/bsp') diff --git a/.claude/skills/etm-trace/boards.md b/.claude/skills/etm-trace/boards.md index dd628ea80..ea6c6181e 100644 --- a/.claude/skills/etm-trace/boards.md +++ b/.claude/skills/etm-trace/boards.md @@ -23,7 +23,7 @@ reference. | ea4088_quickstart | 120 MHz | 120 MHz | 4 | 0 (unset) | J7 (fully wired) | — | | nrf52840dk | 64 MHz | 16 MHz (hw cap) | 4 | 0 (unset) | solder P25, SW7 → Alt | — | | nrf5340dk (M33) | 64 MHz | 16 MHz (TAD, forced) | 4 | +3 ns | mount P25; cut SB27/SB28 | — | -| mimxrt1170_evkb | 996 MHz | 25 MHz (root/2) | 1 | 0 | populate 0 Ω R1881-R1886; J58 | verify/reflow R1882-R1884 → width 4 | +| mimxrt1170_evkb | 996 MHz | 50 MHz (root/2) | 1 | 0 | weld 0 Ω R1881-R1886; JP4 shorted; J58 (populated) | reflow R1882-R1884 (D1-D3 open) → width 4 | | ra6m5_ek (M33) | 200 MHz | 25 MHz (TRCLK/4 /2) | 4 | 0 (unset) | J9 closed; native J20 trace | — | | ra8m1_ek (M85) | 480 MHz | 60 MHz (TRCLK/4 /2) | 4 | 0 (unset) | J9 closed + Table 7 jumpers | — | | raspberry_pi_pico2 (RP2350 M33) | 48 MHz | 24 MHz (clk_sys/2) | 4 | 0 (unset) | fly-wire GPIO1-5 → MIPI20 (map in jdebug) | 72-80 MHz per seating (re-qualify); >80 needs V3 probe + trace board | @@ -66,13 +66,19 @@ Board caveats (beyond the table): builds force the TAD port to 16 MHz (SystemInit's 64 MHz is marginal). - **ea4088_quickstart**: FS enumeration ends < 100 ms — for `--isr` use `--duration-ms 150`. Boot-ROM address warning (0x1FFF1FF0) is normal. -- **mimxrt1170_evkb**: width 4 fails (D1-D3 path under investigation). - `trace_etm_init` fixes the JTAG_nTRST/DMIC_DATA1 pad, pins CSTRACE to - 50 MHz (stock 132 MHz corrupts — the 100M PHY drives the CLK net) and - enables the CM7 platform trace-funnel port, which J-Link doesn't program: - without it everything reads register-perfect yet zero data arrives. - FlexSPI apps: ROM bootloader must set SP/PC — the committed reset/download - hooks handle this. Startup-burst overflow at 996 MHz is normal. +- **mimxrt1170_evkb**: width 1 only — D1-D3 are stone silent at any config + (pinmux register-perfect, PHY quieted, funnel enabled): the welded 0402s + R1882/R1883/R1884 are electrically open — reflow to unlock width 4. + `trace_etm_init` fixes the JTAG_nTRST/DMIC_DATA1 pad, holds the 100M + RTL8201 PHY in reset (ENET_RST_B = GPIO_LPSR_04 — its RMII lines share + the trace pads; with it quiet the CLK line runs a 100 MHz root/50 MHz + pin, 2x the pre-lever rate; 133 MHz root is marginal, stock 132 corrupts) + and enables the CM7 platform trace-funnel port, which J-Link doesn't + program: without it everything reads register-perfect yet zero data + arrives. JP4 must be shorted (disables MCU-Link SWD) for the external + J-Trace on J58. FlexSPI apps: ROM bootloader must set SP/PC — the + committed reset/download hooks handle this. Startup-burst overflow at + 996 MHz is normal. No Ethernet (100M) while tracing. - **ra6m5_ek**: TRCKCR div-2 (100 MHz TRCLK = 50 MHz pin, the chip max) is unusable on this board — swept widths 1/4 across -2..+4 ns, all dead; TRACE_ETM builds use div-4 (25 MHz pin). The TCLK pin runs TRCLK/2. 50 MHz SWD TIF diff --git a/hw/bsp/imxrt/family.c b/hw/bsp/imxrt/family.c index e3ef6233c..1b1a1f1d8 100644 --- a/hw/bsp/imxrt/family.c +++ b/hw/bsp/imxrt/family.c @@ -131,6 +131,13 @@ static void trace_etm_init(void) { // breaks ETM trace - switch the pad to GPIO (MIMXRT1170-EVKB HUG 3.2) IOMUXC_SetPinMux(IOMUXC_GPIO_LPSR_10_GPIO12_IO10, 0U); + // Hold the 100M Ethernet PHY (RTL8201) in reset: its RMII lines are + // hardwired to the trace pads and drive against the stream at speed + // (ENET_RST_B = GPIO_LPSR_04) + IOMUXC_SetPinMux(IOMUXC_GPIO_LPSR_04_GPIO12_IO04, 0U); + GPIO12->GDIR |= (1U << 4); + GPIO12->DR &= ~(1U << 4); + // TRACE0-3 + TRACE_CLK on GPIO_DISP_B2_02..06, fast slew + high drive IOMUXC_SetPinMux(IOMUXC_GPIO_DISP_B2_02_ARM_TRACE00, 0U); IOMUXC_SetPinMux(IOMUXC_GPIO_DISP_B2_03_ARM_TRACE01, 0U); @@ -143,9 +150,11 @@ static void trace_etm_init(void) { IOMUXC_SetPinConfig(IOMUXC_GPIO_DISP_B2_05_ARM_TRACE03, IOMUXC_SW_PAD_CTL_PAD_DSE_MASK); IOMUXC_SetPinConfig(IOMUXC_GPIO_DISP_B2_06_ARM_TRACE_CLK, IOMUXC_SW_PAD_CTL_PAD_DSE_MASK); - // 50 MHz CSTRACE root (OscRc400M/8) -> 25 MHz TRACE_CLK pin (= root/2) + // 100 MHz CSTRACE root (OscRc400M/4) -> 50 MHz TRACE_CLK pin (= root/2). + // With the PHY held in reset the CLK line is clean here; 133 MHz root + // (66 MHz pin) is marginal on this board, stock 132 MHz corrupts. CLOCK_SetRootClockMux(kCLOCK_Root_Cstrace, kCLOCK_CSTRACE_ClockRoot_MuxOscRc400M); - CLOCK_SetRootClockDiv(kCLOCK_Root_Cstrace, 8); + CLOCK_SetRootClockDiv(kCLOCK_Root_Cstrace, 4); CLOCK_EnableClock(kCLOCK_Cstrace); // Enable the CM7 slave port on the platform trace funnel (E004_3000): the -- cgit v1.3.1 From e65368ea16975710f029f7ff7e2089b9ea90186d Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 24 Jul 2026 18:38:40 +0700 Subject: address #3787 reviews: bsp fixes, script hardening, board-note accuracy Bot findings (Copilot/Codex): no-op board_trace_pinmux stubs for lpcxpresso18s37/43s67 (TRACE_ETM otherwise broke their build), SAME70 ID_PIOD clock enable, capture-script duplicate BeforeTargetConnect on the RA references, profile-script support for --no-timestamps itraces. Deep review (whole branch): same70_xplained board row + caveat restored, stale pico2 72 MHz claim corrected to the shipped 48, explicit SetTracePortWidth(4) in the three references that relied on Ozone's default, coverage-cell guard, median-based SysTick calibration, dead session flag removed, stale RA8M1 divider comment fixed (0x02 = /4 is the validated chip max) and the debugger guard indented. EVKB bench findings: only R1884/D3 remains open (D1/D2 meter-verified); RT1176 trace width is 1 or 4 only - J-Link arms the CSSYS TPIU and its own sampler at 4-bit for any width>=2 request; a powered MCU-Link USB breaks the external probe even with JP4 shorted. --- .claude/skills/etm-trace/boards.md | 24 ++++++++-- .claude/skills/etm-trace/scripts/etm_capture.py | 11 +++-- .claude/skills/etm-trace/scripts/etm_profile.py | 54 +++++++++++++++++----- .../metro_m7_1011/ozone/metro_m7_1011.jdebug | 1 + hw/bsp/lpc18/boards/lpcxpresso18s37/board.h | 6 +++ hw/bsp/lpc43/boards/lpcxpresso43s67/board.h | 6 +++ hw/bsp/ra/boards/ra8m1_ek/ozone/ra8m1.jdebug | 1 + hw/bsp/ra/family.c | 30 ++++++------ hw/bsp/same7x/family.c | 1 + .../boards/stm32h743eval/ozone/stm32h743.jdebug | 1 + 10 files changed, 100 insertions(+), 35 deletions(-) (limited to 'hw/bsp') diff --git a/.claude/skills/etm-trace/boards.md b/.claude/skills/etm-trace/boards.md index ea6c6181e..ccebefbf2 100644 --- a/.claude/skills/etm-trace/boards.md +++ b/.claude/skills/etm-trace/boards.md @@ -23,11 +23,12 @@ reference. | ea4088_quickstart | 120 MHz | 120 MHz | 4 | 0 (unset) | J7 (fully wired) | — | | nrf52840dk | 64 MHz | 16 MHz (hw cap) | 4 | 0 (unset) | solder P25, SW7 → Alt | — | | nrf5340dk (M33) | 64 MHz | 16 MHz (TAD, forced) | 4 | +3 ns | mount P25; cut SB27/SB28 | — | -| mimxrt1170_evkb | 996 MHz | 50 MHz (root/2) | 1 | 0 | weld 0 Ω R1881-R1886; JP4 shorted; J58 (populated) | reflow R1882-R1884 (D1-D3 open) → width 4 | +| mimxrt1170_evkb | 996 MHz | 50 MHz (root/2) | 1 | 0 | weld 0 Ω R1881-R1886; JP4 shorted; J58 (populated) | re-weld R1884 (D3 open; D1/D2 meter-verified good) → width 4 | | ra6m5_ek (M33) | 200 MHz | 25 MHz (TRCLK/4 /2) | 4 | 0 (unset) | J9 closed; native J20 trace | — | | ra8m1_ek (M85) | 480 MHz | 60 MHz (TRCLK/4 /2) | 4 | 0 (unset) | J9 closed + Table 7 jumpers | — | | raspberry_pi_pico2 (RP2350 M33) | 48 MHz | 24 MHz (clk_sys/2) | 4 | 0 (unset) | fly-wire GPIO1-5 → MIPI20 (map in jdebug) | 72-80 MHz per seating (re-qualify); >80 needs V3 probe + trace board | | same54_xplained (E54 M4F) | 120 MHz | 60 MHz (CPU/2) | 4 | 0 (unset) | none — populated 20-pin ETM header | — | +| same70_xplained (E70 M7) | 300 MHz | 37.5 MHz (PCK3/2) | 1 | 0 (unset) | solder 20-pin header on J403 (bottom) | width 4 blocked: D1 (J403.16) dead at speed — probe-channel crosscheck pending | | SEGGER H7/F407 ref | demo defaults | demo | 4 | demo | probe-powered: add `--power` | — | Board caveats (beyond the table): @@ -35,7 +36,7 @@ Board caveats (beyond the table): - **stm32h743eval**: startup-burst overflow at 400 MHz is normal (reduce PLLN in board.h for overflow-free capture); timestamp ref 200 MHz. - **stm32n657nucleo** (M55, flashless): **JP2 (BOOT1) must be 1** — the app - is a RAM image the debugger loads (Development boot); in flash boot theb + is a RAM image the debugger loads (Development boot); in flash boot the bootROM parks the chip un-attachable ("Can not attach to CPU"). SEGGER's KB says BOOT0/BOOT1 = 0/0 for their example — that is flash boot and it does NOT attach; the board manual's Table 11 is right. 600 MHz core kills @@ -76,7 +77,12 @@ Board caveats (beyond the table): and enables the CM7 platform trace-funnel port, which J-Link doesn't program: without it everything reads register-perfect yet zero data arrives. JP4 must be shorted (disables MCU-Link SWD) for the external - J-Trace on J58. FlexSPI apps: ROM bootloader must set SP/PC — the + J-Trace on J58; a powered MCU-Link USB breaks the external probe's connect + even with JP4 shorted - power the board from another port. **Width is 1 or + 4 only**: a width-2 request arms the CSSYS TPIU (E004_6000) and the probe + sampler at 4-bit anyway (CSPSR reads 0x8 after a width-2 session; a live + CSPSR=2 poke with LAR unlock + Trace.Clear still captures nothing because + the probe keeps sampling 4-bit). FlexSPI apps: ROM bootloader must set SP/PC — the committed reset/download hooks handle this. Startup-burst overflow at 996 MHz is normal. No Ethernet (100M) while tracing. - **ra6m5_ek**: TRCKCR div-2 (100 MHz TRCLK = 50 MHz pin, the chip max) is @@ -118,7 +124,7 @@ Board caveats (beyond the table): trace component map (funnel/TPIU/ETM are not in the ROM table → "Required trace components for pin trace not found", 0 fetches) and re-arms the whole chip-side path via `OnTraceStart` at every resume. Firmware therefore does - no trace setup; TRACE_ETM builds only (a) pin clk_sys to 72 MHz from crt0 + no trace setup; TRACE_ETM builds only (a) pin clk_sys to 48 MHz from crt0 (board.cmake) — the fly-wire ceiling: 96/150 MHz kill the stream in the startup burst at any sample timing (and at 150 MHz the saturated probe stops answering halts, "CPU could not be halted"); any post-arm clock @@ -136,5 +142,15 @@ Board caveats (beyond the table): `trace_etm_init` feeds it GCLK0. Pins PC24-28 mux to function H. The populated 20-pin header runs chip-max 60 MHz TRACECLK width 4 with no timing adjustment - the connector-vs-flywire contrast board. +- **same70_xplained**: J403 is a bottom-side bare footprint — solder the + header. Trace pins PD4-7 double as the KSZ8081 PHY's RMII receive outputs: + TRACE_ETM builds hold it in reset (PHY_RESET=PC10) or it drives against + the stream. TPIU clock = PCK3 (datasheet 16.7.4), run at MCK/2; TPIU + programming while PCK3 is stopped is silently LOST — the reference starts + PCK3 in the post-reset/download hooks (a reset wipes the PMC, so + AfterTargetConnect is too early). Width 1 validated at the stock 300 MHz + core; width 2/4 blocked on a dead D1 line at J403.16 (clean at DC by + meter, dead at speed — probe-channel crosscheck on a known-good width-4 + board pending). No Ethernet while tracing. - **SEGGER ref boards**: run their own demo (ladder step 3 flags); `--isr` degrades gracefully without a live SysTick. diff --git a/.claude/skills/etm-trace/scripts/etm_capture.py b/.claude/skills/etm-trace/scripts/etm_capture.py index 6468be12d..af6c1ec68 100644 --- a/.claude/skills/etm-trace/scripts/etm_capture.py +++ b/.claude/skills/etm-trace/scripts/etm_capture.py @@ -126,6 +126,12 @@ def resolve_board(board): name, block = m.group(1), m.group(0) if name == "OnProjectLoad": continue + elif name == "BeforeTargetConnect": + # the generated project synthesizes its own BeforeTargetConnect + # (JLINK_SCRIPT_HOOK) from the SetJLinkScript regex below; + # inheriting the reference's copy too would emit a duplicate + # function definition + continue elif name == "AfterTargetReset": cfg["reset_hook"] = block elif name == "AfterTargetDownload": @@ -472,7 +478,6 @@ def main(): start_new_session=True) profile_out = os.path.join(outdir, "code_profile.txt") itrace_out = os.path.join(outdir, "itrace.csv") - ok = False try: ses.connect(20) ses.drain(3) # version banner @@ -513,7 +518,6 @@ def main(): ses.wait_echo(f'Export.PowerGraphs ("{outdir}/power.csv")', 60) ses.send("Debug.Stop", 5) ses.send("File.Exit", 2) - ok = True finally: for _ in range(15): if proc.poll() is not None: @@ -551,9 +555,6 @@ def main(): sys.exit("error: session completed but NO trace data was collected " "(profile totals are zero) - trace signal not reaching the " "probe: check wiring/connector, trace pinmux, sample timing.") - if not ok: - sys.exit("error: session did not complete cleanly (see session.log)") - print(f"\ncapture OK: {outdir}") print(f" code_profile.txt ({os.path.getsize(profile_out)} bytes)") if args.trace_csv: diff --git a/.claude/skills/etm-trace/scripts/etm_profile.py b/.claude/skills/etm-trace/scripts/etm_profile.py index c5ddf27a7..b00f6f1d2 100644 --- a/.claude/skills/etm-trace/scripts/etm_profile.py +++ b/.claude/skills/etm-trace/scripts/etm_profile.py @@ -66,7 +66,7 @@ def parse_profile(path): for module, name, cells in rows(lines[cov_start:prof_start]): m_src = cov_pat.match(cells[0]) if cells else None m_inst = cov_pat.match(cells[1]) if len(cells) > 1 else None - if name == "Total" and m_inst: + if name == "Total" and m_inst and m_src: totals["src_cov"] = num(m_src.group(1)), num(m_src.group(2)) totals["inst_cov"] = num(m_inst.group(1)), num(m_inst.group(2)) elif name in funcs and m_inst: @@ -120,14 +120,29 @@ def iter_itrace(path): caller reads unit separately with itrace_unit().""" with open(path, newline="", errors="replace") as f: rd = csv.reader(f) - next(rd, None) + hdr = next(rd, None) or [] + # --no-timestamps captures drop the Timestamp column entirely: locate + # the Address column from the header and yield t=None for such rows + # (consumers count instructions but skip time math) + has_ts = any("Timestamp" in c for c in hdr) + try: + addr_i = next(i for i, c in enumerate(hdr) if "Address" in c) + except StopIteration: + addr_i = 1 if has_ts else 0 for row in rd: - if not row or row[0] == "PC" or len(row) < 2: + if not row or len(row) <= addr_i: continue try: - yield float(row[0]), int(row[1], 16) + addr = int(row[addr_i], 16) except ValueError: continue + if has_ts and row[0] != "PC": + try: + yield float(row[0]), addr + continue + except ValueError: + pass + yield None, addr def itrace_unit(path): @@ -144,6 +159,8 @@ def time_by_func(path, syms): sample = [] t_prev = None for t, _ in iter_itrace(path): + if t is None: + continue if t_prev is not None and t_prev - t > 0: sample.append(t_prev - t) if len(sample) >= 200000: @@ -158,6 +175,8 @@ def time_by_func(path, syms): fn = addr_to_func(syms, a) if fn: cf[fn] = cf.get(fn, 0) + 1 + if t is None: + continue if t_prev is not None: d = t_prev - t if 0 < d < cap and fn: @@ -180,6 +199,8 @@ def isr_report(itrace, elf, isr_arg, top): usb_rows, tick_rows, tmin, tmax = [], [], None, None for t, a in iter_itrace(itrace): + if t is None: + continue # --no-timestamps capture: the <20-rows message below applies tmin = t if tmin is None else min(tmin, t) tmax = t if tmax is None else max(tmax, t) if any(lo <= a < hi for lo, hi in body): @@ -195,10 +216,12 @@ def isr_report(itrace, elf, isr_arg, top): f"({len(tick_rows)} rows) - capture with timestamps enabled") return - # rough raw-units-per-1ms from the large mode of consecutive tick deltas + # rough raw-units-per-1ms from the large mode of consecutive tick deltas; + # threshold from the median, not the max - one trace-overflow gap would + # otherwise inflate the cut and leave only outliers in the sample deltas = [b[0] - a[0] for a, b in zip(tick_rows, tick_rows[1:])] - big = [d for d in deltas if d > max(deltas) / 10] - raw_ms = statistics.median(big) + big = [d for d in deltas if d > 10 * statistics.median(deltas)] + raw_ms = statistics.median(big) if big else statistics.median(deltas) gap = 0.03 * raw_ms # 30 us in raw units edge = 0.05 * raw_ms @@ -390,11 +413,18 @@ def main(): cshare = {k: v / tc for k, v in cf.items()} unit = itrace_unit(itrace) print(f"\n## Instruction history (itrace.csv, unit '{unit}')\n") - print(f"- {n:,} instructions; top {args.top} by TIME share " - f"(vs instruction share):") - for name, ts in sorted(tshare.items(), key=lambda kv: -kv[1])[:args.top]: - print(f" - `{short(name)}`: {100 * ts:.1f}% time, " - f"{100 * cshare.get(name, 0):.1f}% instructions") + if not tf: + print(f"- {n:,} instructions, NO timestamps (--no-timestamps " + f"capture): time shares unavailable, top {args.top} by " + f"instruction share:") + for name, cs in sorted(cshare.items(), key=lambda kv: -kv[1])[:args.top]: + print(f" - `{short(name)}`: {100 * cs:.1f}% instructions") + else: + print(f"- {n:,} instructions; top {args.top} by TIME share " + f"(vs instruction share):") + for name, ts in sorted(tshare.items(), key=lambda kv: -kv[1])[:args.top]: + print(f" - `{short(name)}`: {100 * ts:.1f}% time, " + f"{100 * cshare.get(name, 0):.1f}% instructions") lines_csv = os.path.join(args.capture_dir, "profile_lines.csv") if os.path.isfile(lines_csv): diff --git a/hw/bsp/imxrt/boards/metro_m7_1011/ozone/metro_m7_1011.jdebug b/hw/bsp/imxrt/boards/metro_m7_1011/ozone/metro_m7_1011.jdebug index fdb8b30a0..80489b61c 100644 --- a/hw/bsp/imxrt/boards/metro_m7_1011/ozone/metro_m7_1011.jdebug +++ b/hw/bsp/imxrt/boards/metro_m7_1011/ozone/metro_m7_1011.jdebug @@ -10,6 +10,7 @@ */ void OnProjectLoad (void) { Project.SetTraceSource ("Trace Pins"); + Project.SetTracePortWidth (4); Project.SetTraceTiming (50, 50, 50, 50); Project.SetDevice ("MIMXRT1011xxx4A"); Project.SetHostIF ("USB", ""); diff --git a/hw/bsp/lpc18/boards/lpcxpresso18s37/board.h b/hw/bsp/lpc18/boards/lpcxpresso18s37/board.h index 2cf4dbdf8..1be07c49e 100644 --- a/hw/bsp/lpc18/boards/lpcxpresso18s37/board.h +++ b/hw/bsp/lpc18/boards/lpcxpresso18s37/board.h @@ -76,6 +76,12 @@ static inline void board_lpc18_pinmux(void) Chip_SCU_SetPinMuxing(pinmuxing, sizeof(pinmuxing) / sizeof(PINMUX_GRP_T)); } + +// TRACE_ETM builds: no trace header is wired out on the LPCXpresso18S37 - +// provide the no-op the family init expects (see mcb1800/ea4357 for a +// board that routes the trace pins) +static inline void board_trace_pinmux(void) {} + #ifdef __cplusplus } #endif diff --git a/hw/bsp/lpc43/boards/lpcxpresso43s67/board.h b/hw/bsp/lpc43/boards/lpcxpresso43s67/board.h index 4427905e8..6a317b5dc 100644 --- a/hw/bsp/lpc43/boards/lpcxpresso43s67/board.h +++ b/hw/bsp/lpc43/boards/lpcxpresso43s67/board.h @@ -71,6 +71,12 @@ static const PINMUX_GRP_T pinmuxing[] = { {0x2, 5, SCU_MODE_INBUFF_EN | SCU_MODE_PULLUP | SCU_MODE_FUNC4 }, }; + +// TRACE_ETM builds: no trace header is wired out on the LPCXpresso43S67 - +// provide the no-op the family init expects (see mcb1800/ea4357 for a +// board that routes the trace pins) +static inline void board_trace_pinmux(void) {} + #ifdef __cplusplus } #endif diff --git a/hw/bsp/ra/boards/ra8m1_ek/ozone/ra8m1.jdebug b/hw/bsp/ra/boards/ra8m1_ek/ozone/ra8m1.jdebug index f0a43bf5f..927eeda72 100644 --- a/hw/bsp/ra/boards/ra8m1_ek/ozone/ra8m1.jdebug +++ b/hw/bsp/ra/boards/ra8m1_ek/ozone/ra8m1.jdebug @@ -10,6 +10,7 @@ */ void OnProjectLoad (void) { Project.SetTraceSource ("Trace Pins"); + Project.SetTracePortWidth (4); Project.SetDevice ("R7FA8M1AH"); Project.SetHostIF ("USB", ""); Project.SetTargetIF ("SWD"); diff --git a/hw/bsp/ra/family.c b/hw/bsp/ra/family.c index 3c548fa7f..307644972 100644 --- a/hw/bsp/ra/family.c +++ b/hw/bsp/ra/family.c @@ -103,25 +103,27 @@ void board_init(void) { // standalone boot must skip trace init or the write wedges the chip into // an un-attachable crash loop (recover: power-cycle + immediate erase) if (DCB->DHCSR & DCB_DHCSR_C_DEBUGEN_Msk) { - // TRCKCR is protected by PRCR bit0 register - R_SYSTEM->PRCR = (uint16_t) (BSP_PRV_PRCR_KEY | 0x01); + // TRCKCR is protected by PRCR bit0 register + R_SYSTEM->PRCR = (uint16_t) (BSP_PRV_PRCR_KEY | 0x01); - // TCLK pin = TRCLK/2; set the divider with TRCKEN=0 first (HUM procedure). - // Values are the empirical per-board ceilings: one step below the divider - // at which the stream dies mid-run. + // TCLK pin = TRCLK/2; set the divider with TRCKEN=0 first (HUM procedure). + // Values are the empirical per-board ceilings. #if defined(BSP_MCU_GROUP_RA8M1) - // 480 MHz CPU: /8 -> 60 MHz TRCLK, 30 MHz pin (/4 = 60 MHz pin dies) - R_SYSTEM->TRCKCR = 0x02; - R_SYSTEM->TRCKCR = R_SYSTEM_TRCKCR_TRCKEN_Msk | 0x02; + // 480 MHz CPU: /4 -> 120 MHz TRCLK, 60 MHz pin - chip max, clean on + // EK-RA8M1 with the committed empty-OnTraceStart JLinkScript (which + // defers the trace clock to firmware; without it the FSP MOCO->PLL + // switch steps the clock mid-stream and any divider fails) + R_SYSTEM->TRCKCR = 0x02; + R_SYSTEM->TRCKCR = R_SYSTEM_TRCKCR_TRCKEN_Msk | 0x02; #else - // RA6M5 200 MHz CPU: /4 -> 50 MHz TRCLK, 25 MHz pin. /2 (50 MHz pin) is - // silent on the EK-RA6M5 in every combination - board path ceiling, - // reconfirmed with J9 closed and the OnTraceStart override - R_SYSTEM->TRCKCR = 0x02; - R_SYSTEM->TRCKCR = R_SYSTEM_TRCKCR_TRCKEN_Msk | 0x02; + // RA6M5 200 MHz CPU: /4 -> 50 MHz TRCLK, 25 MHz pin. /2 (50 MHz pin) is + // silent on the EK-RA6M5 in every combination - board path ceiling, + // reconfirmed with J9 closed and the OnTraceStart override + R_SYSTEM->TRCKCR = 0x02; + R_SYSTEM->TRCKCR = R_SYSTEM_TRCKCR_TRCKEN_Msk | 0x02; #endif - R_SYSTEM->PRCR = (uint16_t) BSP_PRV_PRCR_KEY; + R_SYSTEM->PRCR = (uint16_t) BSP_PRV_PRCR_KEY; } #endif diff --git a/hw/bsp/same7x/family.c b/hw/bsp/same7x/family.c index 6a3466354..d99c17efb 100644 --- a/hw/bsp/same7x/family.c +++ b/hw/bsp/same7x/family.c @@ -82,6 +82,7 @@ void board_init(void) { PMC->PMC_SCER = PMC_SCER_PCK3; while (!(PMC->PMC_SR & PMC_SR_PCKRDY3)) {} } + _pmc_enable_periph_clock(ID_PIOD); uint32_t const clk_pin = PIO_PD8D_TPIU_TRACECLK; uint32_t const dat_pin = PIO_PD4C_TPIU_TRACED0 | PIO_PD5C_TPIU_TRACED1 | PIO_PD6C_TPIU_TRACED2 | PIO_PD7C_TPIU_TRACED3; diff --git a/hw/bsp/stm32h7/boards/stm32h743eval/ozone/stm32h743.jdebug b/hw/bsp/stm32h7/boards/stm32h743eval/ozone/stm32h743.jdebug index a8645c372..f9780147d 100644 --- a/hw/bsp/stm32h7/boards/stm32h743eval/ozone/stm32h743.jdebug +++ b/hw/bsp/stm32h7/boards/stm32h743eval/ozone/stm32h743.jdebug @@ -10,6 +10,7 @@ */ void OnProjectLoad (void) { Project.SetTraceSource ("Trace Pins"); + Project.SetTracePortWidth (4); Project.SetTraceTiming (100, 100, 100, 100); Project.SetSWO (0); Edit.SysVar (VAR_TRACE_CORE_CLOCK, 200000000); -- cgit v1.3.1 From f08c8211904ac9feb2598aa354b0b7bddc245bfe Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 24 Jul 2026 21:30:42 +0700 Subject: address #3787 Codex round 2: board-gate PHY resets, session robustness The board-specific PHY-reset nets move behind a board.h opt-in (TRACE_ETM_QUIET_ENET_PHY on same70_xplained and mimxrt1170_evkb) so other boards of those families cannot inherit a foreign GPIO write; the chip-level trace pin muxes stay family-wide by design (same pattern as stm32h7). same70 reference: width 1 is the validated default until the J403.16 rework, and the hooks now wait (bounded) for PCKRDY3 before Ozone arms trace. ra8m1 reference caches the boot ROM in AfterTargetConnect so --attach sessions decode ROM execution too. etm_capture rejects an unexpanded CMake JLINK_DEVICE with a clear error; PIO-USB + TRACE_ETM on RP2350 is now a compile error (48 MHz trace clock is too slow for PIO-USB and a runtime switch would desync the stream); etm_profile keeps same-named statics from different modules as distinct rows. Build-verified: same70_xplained, mimxrt1170_evkb, raspberry_pi_pico2. --- .claude/skills/etm-trace/scripts/etm_capture.py | 4 ++++ .claude/skills/etm-trace/scripts/etm_profile.py | 4 ++++ hw/bsp/imxrt/boards/mimxrt1170_evkb/board.h | 5 +++++ hw/bsp/imxrt/family.c | 7 +++++-- hw/bsp/ra/boards/ra8m1_ek/ozone/ra8m1.jdebug | 8 +++++--- hw/bsp/rp2040/family.c | 3 +++ hw/bsp/same7x/boards/same70_xplained/board.h | 4 ++++ hw/bsp/same7x/boards/same70_xplained/ozone/same70.jdebug | 12 ++++++++++++ hw/bsp/same7x/family.c | 5 ++++- 9 files changed, 46 insertions(+), 6 deletions(-) (limited to 'hw/bsp') diff --git a/.claude/skills/etm-trace/scripts/etm_capture.py b/.claude/skills/etm-trace/scripts/etm_capture.py index af6c1ec68..ccde886f2 100644 --- a/.claude/skills/etm-trace/scripts/etm_capture.py +++ b/.claude/skills/etm-trace/scripts/etm_capture.py @@ -160,6 +160,10 @@ def resolve_board(board): for path in glob.glob(f"{REPO_ROOT}/hw/bsp/*/boards/{board}/board.cmake"): m = re.search(r'JLINK_DEVICE\s+([^\s)]+)\s*\)', open(path).read()) if m: + if "${" in m.group(1): + sys.exit(f"error: {path} defines JLINK_DEVICE via an " + f"unexpanded CMake variable ({m.group(1)}) - pass " + f"--device explicitly for this board") cfg["device"] = m.group(1) cfg["ref"] = path break diff --git a/.claude/skills/etm-trace/scripts/etm_profile.py b/.claude/skills/etm-trace/scripts/etm_profile.py index b00f6f1d2..b859328ca 100644 --- a/.claude/skills/etm-trace/scripts/etm_profile.py +++ b/.claude/skills/etm-trace/scripts/etm_profile.py @@ -61,6 +61,10 @@ def parse_profile(path): totals["run"], totals["fetch"] = run, fetch elif name == "[Unaccounted]": totals["unaccounted"] = fetch + elif name in funcs and funcs[name]["module"] != module: + # same-named static from another module: keep both rows distinct + funcs[f"{name} [{module}]"] = {"module": module, "run": run, + "fetch": fetch} else: funcs[name] = {"module": module, "run": run, "fetch": fetch} for module, name, cells in rows(lines[cov_start:prof_start]): diff --git a/hw/bsp/imxrt/boards/mimxrt1170_evkb/board.h b/hw/bsp/imxrt/boards/mimxrt1170_evkb/board.h index a6332d896..c041fd47b 100644 --- a/hw/bsp/imxrt/boards/mimxrt1170_evkb/board.h +++ b/hw/bsp/imxrt/boards/mimxrt1170_evkb/board.h @@ -35,6 +35,11 @@ // required since iMXRT MCUX-SDK include this file for board size #define BOARD_FLASH_SIZE (0x1000000U) +// TRACE_ETM: this board wires the 100M PHY reset (ENET_RST_B) to +// GPIO_LPSR_04; the family trace init holds it in reset (RMII lines share +// the trace pads) +#define TRACE_ETM_QUIET_ENET_PHY 1 + // LED: IOMUXC_GPIO_AD_04_GPIO9_IO03 #define LED_PORT BOARD_INITPINS_USER_LED_PERIPHERAL #define LED_PIN BOARD_INITPINS_USER_LED_CHANNEL diff --git a/hw/bsp/imxrt/family.c b/hw/bsp/imxrt/family.c index 1b1a1f1d8..4bd7993f0 100644 --- a/hw/bsp/imxrt/family.c +++ b/hw/bsp/imxrt/family.c @@ -131,12 +131,15 @@ static void trace_etm_init(void) { // breaks ETM trace - switch the pad to GPIO (MIMXRT1170-EVKB HUG 3.2) IOMUXC_SetPinMux(IOMUXC_GPIO_LPSR_10_GPIO12_IO10, 0U); +#ifdef TRACE_ETM_QUIET_ENET_PHY // Hold the 100M Ethernet PHY (RTL8201) in reset: its RMII lines are - // hardwired to the trace pads and drive against the stream at speed - // (ENET_RST_B = GPIO_LPSR_04) + // hardwired to the trace pads and drive against the stream at speed. The + // reset net is a BOARD property (mimxrt1170_evkb: ENET_RST_B = + // GPIO_LPSR_04), hence the board.h gate. IOMUXC_SetPinMux(IOMUXC_GPIO_LPSR_04_GPIO12_IO04, 0U); GPIO12->GDIR |= (1U << 4); GPIO12->DR &= ~(1U << 4); +#endif // TRACE0-3 + TRACE_CLK on GPIO_DISP_B2_02..06, fast slew + high drive IOMUXC_SetPinMux(IOMUXC_GPIO_DISP_B2_02_ARM_TRACE00, 0U); diff --git a/hw/bsp/ra/boards/ra8m1_ek/ozone/ra8m1.jdebug b/hw/bsp/ra/boards/ra8m1_ek/ozone/ra8m1.jdebug index 927eeda72..02d568240 100644 --- a/hw/bsp/ra/boards/ra8m1_ek/ozone/ra8m1.jdebug +++ b/hw/bsp/ra/boards/ra8m1_ek/ozone/ra8m1.jdebug @@ -41,12 +41,14 @@ void BeforeTargetConnect (void) { * AfterTargetConnect * * Function description -* Event handler routine. Optional. +* Cache the boot-ROM range for the trace decoder on --attach sessions +* too (the download hook that normally does this is skipped on attach). * ********************************************************************** */ -//void AfterTargetConnect (void) { -//} +void AfterTargetConnect (void) { + Exec.Command("ReadIntoTraceCache 0x0 0x10000"); +} /********************************************************************* * diff --git a/hw/bsp/rp2040/family.c b/hw/bsp/rp2040/family.c index f0d6ba245..e12f51b14 100644 --- a/hw/bsp/rp2040/family.c +++ b/hw/bsp/rp2040/family.c @@ -180,6 +180,9 @@ void board_init(void) #if (CFG_TUH_ENABLED && CFG_TUH_RPI_PIO_USB) || (CFG_TUD_ENABLED && CFG_TUD_RPI_PIO_USB) // Set the system clock to a multiple of 12mhz for bit-banging USB with pico-usb #if defined(PICO_RP2350) && PICO_RP2350 == 1 + #ifdef TRACE_ETM + #error "TRACE_ETM pins clk_sys to 48 MHz (board.cmake) - too slow for PIO-USB, and a runtime clock switch desyncs the trace stream" + #endif set_sys_clock_khz(156000, true); // rp2350 default is 150Mhz #else set_sys_clock_khz(120000, true); // rp2040 default is 125Mhz diff --git a/hw/bsp/same7x/boards/same70_xplained/board.h b/hw/bsp/same7x/boards/same70_xplained/board.h index 85e23deb8..86edf606f 100644 --- a/hw/bsp/same7x/boards/same70_xplained/board.h +++ b/hw/bsp/same7x/boards/same70_xplained/board.h @@ -52,6 +52,10 @@ extern "C" { #define UART_PORT_CLOCK ID_USART1 #define BOARD_USART USART1 +// TRACE_ETM: this board wires the KSZ8081 PHY reset to PC10; the family +// trace init holds it in reset (RMII rx lines share the trace pads) +#define TRACE_ETM_QUIET_ENET_PHY 1 + static inline void board_vbus_set(uint8_t rhport, bool state) { (void) rhport; (void) state; diff --git a/hw/bsp/same7x/boards/same70_xplained/ozone/same70.jdebug b/hw/bsp/same7x/boards/same70_xplained/ozone/same70.jdebug index 0fde09716..f024f0ceb 100644 --- a/hw/bsp/same7x/boards/same70_xplained/ozone/same70.jdebug +++ b/hw/bsp/same7x/boards/same70_xplained/ozone/same70.jdebug @@ -64,6 +64,12 @@ void AfterTargetReset (void) { // in the post-reset/post-download hooks, not AfterTargetConnect. Target.WriteU32 (0x400E064C, 0x00000014); // PMC_PCK3: CSS=MCK, PRESS=/2 Target.WriteU32 (0x400E0600, 0x00000800); // PMC_SCER: PCK3 on + // wait for PCKRDY3 (bounded) before Ozone arms the trace components + int i; + i = 0; + while (((Target.ReadU32 (0x400E0668) & 0x00000800) == 0) && (i < 100)) { + i = i + 1; + } } /********************************************************************* @@ -100,4 +106,10 @@ void AfterTargetDownload (void) { // in the post-reset/post-download hooks, not AfterTargetConnect. Target.WriteU32 (0x400E064C, 0x00000014); // PMC_PCK3: CSS=MCK, PRESS=/2 Target.WriteU32 (0x400E0600, 0x00000800); // PMC_SCER: PCK3 on + // wait for PCKRDY3 (bounded) before Ozone arms the trace components + int i; + i = 0; + while (((Target.ReadU32 (0x400E0668) & 0x00000800) == 0) && (i < 100)) { + i = i + 1; + } } diff --git a/hw/bsp/same7x/family.c b/hw/bsp/same7x/family.c index d99c17efb..8ec6a708b 100644 --- a/hw/bsp/same7x/family.c +++ b/hw/bsp/same7x/family.c @@ -65,13 +65,16 @@ void board_init(void) { #if defined(TRACE_ETM) // same70_xplained J403 (Cortex Debug+ETM footprint, bottom side) carries // 4-bit trace: TRACECLK=PD8 (peripheral D), TRACED0-3=PD4-7 (peripheral C). +#ifdef TRACE_ETM_QUIET_ENET_PHY // The trace pins double as the Ethernet PHY's RMII receive lines // (PD4=CRS_DV, PD5/6=RXD0/1, PD7=RXER - PHY OUTPUTS): hold the KSZ8081 in - // reset (PHY_RESET=PC10 low) or it drives against the trace stream. + // reset or it drives against the trace stream. The reset net is a BOARD + // property (same70_xplained: PHY_RESET=PC10), hence the board.h gate. _pmc_enable_periph_clock(ID_PIOC); gpio_set_pin_level(GPIO(GPIO_PORTC, 10), false); gpio_set_pin_direction(GPIO(GPIO_PORTC, 10), GPIO_DIRECTION_OUT); gpio_set_pin_function(GPIO(GPIO_PORTC, 10), GPIO_PIN_FUNCTION_OFF); +#endif // The TPIU is clocked from PCK3 (datasheet 16.7.4) - run it from MCK. // skip if the debugger already started PCK3 (reprogramming glitches the -- cgit v1.3.1 From 7b9761f1919014152d3e9ee1aaf203d7fb0e762b Mon Sep 17 00:00:00 2001 From: Jie Feng Date: Sat, 25 Jul 2026 11:25:05 +0800 Subject: Enable APM32F0 dependency fetching and CI --- .github/workflows/ci_set_matrix.py | 1 + .gitignore | 1 + examples/device/net_lwip_webserver/skip.txt | 1 + hw/bsp/apm32f0xx/FreeRTOSConfig/FreeRTOSConfig.h | 108 +++++++++++++++++++++ .../boards/apm32f072_dev_board/board.cmake | 7 +- .../apm32f0xx/boards/apm32f072_dev_board/board.mk | 5 +- hw/bsp/apm32f0xx/family.c | 9 ++ hw/bsp/apm32f0xx/family.cmake | 2 +- hw/bsp/apm32f0xx/family.mk | 2 +- tools/get_deps.py | 3 + 10 files changed, 135 insertions(+), 4 deletions(-) create mode 100644 hw/bsp/apm32f0xx/FreeRTOSConfig/FreeRTOSConfig.h (limited to 'hw/bsp') diff --git a/.github/workflows/ci_set_matrix.py b/.github/workflows/ci_set_matrix.py index dc0d3871f..50ada5964 100755 --- a/.github/workflows/ci_set_matrix.py +++ b/.github/workflows/ci_set_matrix.py @@ -16,6 +16,7 @@ toolchain_list = [ # family: [supported toolchain] family_list = { + "apm32f0xx": ["arm-gcc"], "at32f402_405": ["arm-gcc"], "at32f403a_407": ["arm-gcc"], "at32f413": ["arm-gcc"], diff --git a/.gitignore b/.gitignore index ca745ee19..8773322e4 100644 --- a/.gitignore +++ b/.gitignore @@ -78,6 +78,7 @@ hw/mcu/artery/ hw/mcu/broadcom/ hw/mcu/bridgetek/ft9xx/ft90x-sdk/ hw/mcu/gd/ +hw/mcu/geehy/ hw/mcu/hpmicro/ hw/mcu/infineon/ hw/mcu/microchip/ diff --git a/examples/device/net_lwip_webserver/skip.txt b/examples/device/net_lwip_webserver/skip.txt index c3df1ee4b..53836b581 100644 --- a/examples/device/net_lwip_webserver/skip.txt +++ b/examples/device/net_lwip_webserver/skip.txt @@ -1,3 +1,4 @@ +mcu:APM32F0XX mcu:CH32V103 mcu:CH32V20X mcu:LPC11UXX diff --git a/hw/bsp/apm32f0xx/FreeRTOSConfig/FreeRTOSConfig.h b/hw/bsp/apm32f0xx/FreeRTOSConfig/FreeRTOSConfig.h new file mode 100644 index 000000000..b8d555ebe --- /dev/null +++ b/hw/bsp/apm32f0xx/FreeRTOSConfig/FreeRTOSConfig.h @@ -0,0 +1,108 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2026, Ha Thach (tinyusb.org) + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * This file is part of the TinyUSB stack. + */ + +#ifndef FREERTOS_CONFIG_H_ +#define FREERTOS_CONFIG_H_ + +#ifndef __IASMARM__ + #include "apm32f0xx.h" +#endif + +#define configENABLE_MPU 0 +#define configENABLE_FPU 0 +#define configENABLE_TRUSTZONE 0 +#define configMINIMAL_SECURE_STACK_SIZE 1024 + +#define configUSE_PREEMPTION 1 +#define configUSE_PORT_OPTIMISED_TASK_SELECTION 0 +#define configCPU_CLOCK_HZ SystemCoreClock +#define configTICK_RATE_HZ 1000 +#define configMAX_PRIORITIES 5 +#define configMINIMAL_STACK_SIZE 128 +#define configTOTAL_HEAP_SIZE ( configSUPPORT_DYNAMIC_ALLOCATION * 4 * 1024 ) +#define configMAX_TASK_NAME_LEN 16 +#define configUSE_16_BIT_TICKS 0 +#define configIDLE_SHOULD_YIELD 1 +#define configUSE_MUTEXES 1 +#define configUSE_RECURSIVE_MUTEXES 1 +#define configUSE_COUNTING_SEMAPHORES 1 +#define configQUEUE_REGISTRY_SIZE 4 +#define configUSE_QUEUE_SETS 0 +#define configUSE_TIME_SLICING 0 +#define configUSE_NEWLIB_REENTRANT 0 +#define configENABLE_BACKWARD_COMPATIBILITY 1 +#define configSTACK_ALLOCATION_FROM_SEPARATE_HEAP 0 + +#define configSUPPORT_STATIC_ALLOCATION 1 +#define configSUPPORT_DYNAMIC_ALLOCATION 0 + +#define configUSE_IDLE_HOOK 0 +#define configUSE_TICK_HOOK 0 +#define configUSE_MALLOC_FAILED_HOOK 0 +#define configCHECK_FOR_STACK_OVERFLOW 2 +#define configCHECK_HANDLER_INSTALLATION 0 + +#define configGENERATE_RUN_TIME_STATS 0 +#define configRECORD_STACK_HIGH_ADDRESS 1 +#define configUSE_TRACE_FACILITY 1 +#define configUSE_STATS_FORMATTING_FUNCTIONS 0 + +#define configUSE_CO_ROUTINES 0 +#define configMAX_CO_ROUTINE_PRIORITIES 2 + +#define configUSE_TIMERS 1 +#define configTIMER_TASK_PRIORITY ( configMAX_PRIORITIES - 2 ) +#define configTIMER_QUEUE_LENGTH 32 +#define configTIMER_TASK_STACK_DEPTH configMINIMAL_STACK_SIZE + +#define INCLUDE_vTaskPrioritySet 0 +#define INCLUDE_uxTaskPriorityGet 0 +#define INCLUDE_vTaskDelete 0 +#define INCLUDE_vTaskSuspend 1 +#define INCLUDE_xResumeFromISR 0 +#define INCLUDE_vTaskDelayUntil 1 +#define INCLUDE_vTaskDelay 1 +#define INCLUDE_xTaskGetSchedulerState 0 +#define INCLUDE_xTaskGetCurrentTaskHandle 1 +#define INCLUDE_uxTaskGetStackHighWaterMark 0 +#define INCLUDE_xTaskGetIdleTaskHandle 0 +#define INCLUDE_xTimerGetTimerDaemonTaskHandle 0 +#define INCLUDE_pcTaskGetTaskName 0 +#define INCLUDE_eTaskGetState 0 +#define INCLUDE_xEventGroupSetBitFromISR 0 +#define INCLUDE_xTimerPendFunctionCall 0 + +#define xPortPendSVHandler PendSV_Handler +#define xPortSysTickHandler SysTick_Handler +#define vPortSVCHandler SVC_Handler + +#define configPRIO_BITS __NVIC_PRIO_BITS +#define configLIBRARY_LOWEST_INTERRUPT_PRIORITY ( ( 1 << configPRIO_BITS ) - 1 ) +#define configLIBRARY_MAX_SYSCALL_INTERRUPT_PRIORITY 2 +#define configKERNEL_INTERRUPT_PRIORITY ( configLIBRARY_LOWEST_INTERRUPT_PRIORITY << ( 8 - configPRIO_BITS ) ) +#define configMAX_SYSCALL_INTERRUPT_PRIORITY ( configLIBRARY_MAX_SYSCALL_INTERRUPT_PRIORITY << ( 8 - configPRIO_BITS ) ) + +#endif diff --git a/hw/bsp/apm32f0xx/boards/apm32f072_dev_board/board.cmake b/hw/bsp/apm32f0xx/boards/apm32f072_dev_board/board.cmake index 33148dbd4..d25f675be 100644 --- a/hw/bsp/apm32f0xx/boards/apm32f072_dev_board/board.cmake +++ b/hw/bsp/apm32f0xx/boards/apm32f072_dev_board/board.cmake @@ -4,5 +4,10 @@ set(MCU_LINKER_NAME APM32F07xxB) set(JLINK_DEVICE APM32F072RB) function(update_board TARGET) - target_compile_definitions(${TARGET} PUBLIC ${MCU_VARIANT}) + target_compile_definitions(${TARGET} PUBLIC + ${MCU_VARIANT} + CFG_EXAMPLE_MSC_READONLY + CFG_EXAMPLE_MSC_DUAL_READONLY + CFG_EXAMPLE_VIDEO_READONLY + ) endfunction() diff --git a/hw/bsp/apm32f0xx/boards/apm32f072_dev_board/board.mk b/hw/bsp/apm32f0xx/boards/apm32f072_dev_board/board.mk index 2e5df9947..f78d2091f 100644 --- a/hw/bsp/apm32f0xx/boards/apm32f072_dev_board/board.mk +++ b/hw/bsp/apm32f0xx/boards/apm32f072_dev_board/board.mk @@ -4,4 +4,7 @@ MCU_LINKER_NAME = APM32F07xxB JLINK_DEVICE = APM32F072RB CFLAGS += \ - -D${MCU_VARIANT} + -D${MCU_VARIANT} \ + -DCFG_EXAMPLE_MSC_READONLY \ + -DCFG_EXAMPLE_MSC_DUAL_READONLY \ + -DCFG_EXAMPLE_VIDEO_READONLY diff --git a/hw/bsp/apm32f0xx/family.c b/hw/bsp/apm32f0xx/family.c index cbc427f8c..9caa0207c 100644 --- a/hw/bsp/apm32f0xx/family.c +++ b/hw/bsp/apm32f0xx/family.c @@ -36,6 +36,15 @@ #include "bsp/board_api.h" #include "board.h" +void USBD_IRQHandler(void); +#if CFG_TUSB_OS == OPT_OS_NONE +void SysTick_Handler(void); +void SVC_Handler(void); +void PendSV_Handler(void); +#endif +void HardFault_Handler(void); +void _init(void); + //--------------------------------------------------------------------+ // Forward USB interrupt events to TinyUSB IRQ Handler //--------------------------------------------------------------------+ diff --git a/hw/bsp/apm32f0xx/family.cmake b/hw/bsp/apm32f0xx/family.cmake index 99a94a7a8..0cf199f26 100644 --- a/hw/bsp/apm32f0xx/family.cmake +++ b/hw/bsp/apm32f0xx/family.cmake @@ -1,7 +1,7 @@ include_guard() set(APM32_FAMILY apm32f0xx) -set(APM32_SDK ${TOP}/hw/mcu/geehy/APM32F0xx_SDK_V1.8.6/Libraries) +set(APM32_SDK ${TOP}/hw/mcu/geehy/APM32F0xx_SDK/Libraries) # include board specific include(${CMAKE_CURRENT_LIST_DIR}/boards/${BOARD}/board.cmake) diff --git a/hw/bsp/apm32f0xx/family.mk b/hw/bsp/apm32f0xx/family.mk index 73a762d69..735c83f90 100644 --- a/hw/bsp/apm32f0xx/family.mk +++ b/hw/bsp/apm32f0xx/family.mk @@ -1,5 +1,5 @@ APM32_FAMILY = apm32f0xx -APM32_SDK = hw/mcu/geehy/APM32F0xx_SDK_V1.8.6/Libraries +APM32_SDK = hw/mcu/geehy/APM32F0xx_SDK/Libraries include $(TOP)/$(BOARD_PATH)/board.mk diff --git a/tools/get_deps.py b/tools/get_deps.py index e26810cac..baaf3761f 100755 --- a/tools/get_deps.py +++ b/tools/get_deps.py @@ -46,6 +46,9 @@ deps_optional = { 'hw/mcu/gd/nuclei-sdk': ['https://github.com/Nuclei-Software/nuclei-sdk.git', '7eb7bfa9ea4fbeacfafe1d5f77d5a0e6ed3922e7', 'gd32vf103'], + 'hw/mcu/geehy/APM32F0xx_SDK': ['https://github.com/GeehySemi/APM32F0xx_SDK.git', + 'cfc1fe826e1869133de86d4c6b298fc153e6bc32', + 'apm32f0xx'], 'hw/mcu/infineon/mtb-xmclib-cat3': ['https://github.com/Infineon/mtb-xmclib-cat3.git', 'daf5500d03cba23e68c2f241c30af79cd9d63880', 'xmc4000'], -- cgit v1.3.1 From bf7a62055ec829f00bd9c3a129e327b6a67ac55b Mon Sep 17 00:00:00 2001 From: Jie Feng Date: Sat, 25 Jul 2026 11:37:33 +0800 Subject: Run APM32F0 CPU and USB from 48 MHz PLL --- hw/bsp/apm32f0xx/family.c | 18 +++++------------- hw/bsp/apm32f0xx/family.cmake | 1 - hw/bsp/apm32f0xx/family.mk | 1 - 3 files changed, 5 insertions(+), 15 deletions(-) (limited to 'hw/bsp') diff --git a/hw/bsp/apm32f0xx/family.c b/hw/bsp/apm32f0xx/family.c index 9caa0207c..643490797 100644 --- a/hw/bsp/apm32f0xx/family.c +++ b/hw/bsp/apm32f0xx/family.c @@ -32,7 +32,6 @@ #include "apm32f0xx_rcm.h" #include "apm32f0xx_gpio.h" #include "apm32f0xx_misc.h" -#include "apm32f0xx_crs.h" #include "bsp/board_api.h" #include "board.h" @@ -56,18 +55,11 @@ void USBD_IRQHandler(void) { // Board Init //--------------------------------------------------------------------+ void board_init(void) { - // Enable HSI48 for USB clock - RCM_EnableHSI48(); - while (RCM_ReadStatusFlag(RCM_FLAG_HSI48RDY) == RESET) {} - - // Select HSI48 as USB clock source - RCM_ConfigUSBCLK(RCM_USBCLK_HSI48); - - // Enable CRS for automatic HSI48 calibration from USB SOF - RCM_EnableAPB1PeriphClock(RCM_APB1_PERIPH_CRS); - CRS_ConfigSynchronizationSource(CRS_SYNC_SOURCE_USB); - CRS_EnableAutomaticCalibration(); - CRS_EnableFrequencyErrorCounter(); + // Configure HSE and PLL for a 48 MHz system clock + SystemClockConfig(); + + // Route the 48 MHz PLL clock to USB + RCM_ConfigUSBCLK(RCM_USBCLK_PLLCLK); // Enable USB peripheral clock RCM_EnableAPB1PeriphClock(RCM_APB1_PERIPH_USB); diff --git a/hw/bsp/apm32f0xx/family.cmake b/hw/bsp/apm32f0xx/family.cmake index 0cf199f26..199ba7cb5 100644 --- a/hw/bsp/apm32f0xx/family.cmake +++ b/hw/bsp/apm32f0xx/family.cmake @@ -31,7 +31,6 @@ function(family_add_board BOARD_TARGET) ${APM32_SDK}/APM32F0xx_StdPeriphDriver/src/apm32f0xx_gpio.c ${APM32_SDK}/APM32F0xx_StdPeriphDriver/src/apm32f0xx_misc.c ${APM32_SDK}/APM32F0xx_StdPeriphDriver/src/apm32f0xx_rcm.c - ${APM32_SDK}/APM32F0xx_StdPeriphDriver/src/apm32f0xx_crs.c ) target_include_directories(${BOARD_TARGET} PUBLIC ${CMAKE_CURRENT_FUNCTION_LIST_DIR} diff --git a/hw/bsp/apm32f0xx/family.mk b/hw/bsp/apm32f0xx/family.mk index 735c83f90..7e26918c2 100644 --- a/hw/bsp/apm32f0xx/family.mk +++ b/hw/bsp/apm32f0xx/family.mk @@ -20,7 +20,6 @@ SRC_C += \ $(APM32_SDK)/APM32F0xx_StdPeriphDriver/src/apm32f0xx_gpio.c \ $(APM32_SDK)/APM32F0xx_StdPeriphDriver/src/apm32f0xx_misc.c \ $(APM32_SDK)/APM32F0xx_StdPeriphDriver/src/apm32f0xx_rcm.c \ - $(APM32_SDK)/APM32F0xx_StdPeriphDriver/src/apm32f0xx_crs.c \ $(APM32_SDK)/Device/Geehy/APM32F0xx/Source/system_apm32f0xx.c INC += \ -- cgit v1.3.1 From ba9940385c23db07c35ad546a1bcf153997a80a5 Mon Sep 17 00:00:00 2001 From: Zixun LI Date: Sat, 25 Jul 2026 21:32:28 +0200 Subject: refresh presets Signed-off-by: Zixun LI --- hw/bsp/BoardPresets.json | 44 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) (limited to 'hw/bsp') diff --git a/hw/bsp/BoardPresets.json b/hw/bsp/BoardPresets.json index 93a8f2c32..a480efc3e 100644 --- a/hw/bsp/BoardPresets.json +++ b/hw/bsp/BoardPresets.json @@ -42,6 +42,10 @@ "name": "apard32690", "inherits": "default" }, + { + "name": "apm32f072_dev_board", + "inherits": "default" + }, { "name": "arduino_nano33_ble", "inherits": "default" @@ -510,6 +514,10 @@ "name": "portenta_c33", "inherits": "default" }, + { + "name": "py32f071_dev_board", + "inherits": "default" + }, { "name": "pybadge", "inherits": "default" @@ -1007,6 +1015,11 @@ "description": "Build preset for the apard32690 board", "configurePreset": "apard32690" }, + { + "name": "apm32f072_dev_board", + "description": "Build preset for the apm32f072_dev_board board", + "configurePreset": "apm32f072_dev_board" + }, { "name": "arduino_nano33_ble", "description": "Build preset for the arduino_nano33_ble board", @@ -1637,6 +1650,11 @@ "description": "Build preset for the portenta_c33 board", "configurePreset": "portenta_c33" }, + { + "name": "py32f071_dev_board", + "description": "Build preset for the py32f071_dev_board board", + "configurePreset": "py32f071_dev_board" + }, { "name": "pybadge", "description": "Build preset for the pybadge board", @@ -2257,6 +2275,19 @@ } ] }, + { + "name": "apm32f072_dev_board", + "steps": [ + { + "type": "configure", + "name": "apm32f072_dev_board" + }, + { + "type": "build", + "name": "apm32f072_dev_board" + } + ] + }, { "name": "arduino_nano33_ble", "steps": [ @@ -3895,6 +3926,19 @@ } ] }, + { + "name": "py32f071_dev_board", + "steps": [ + { + "type": "configure", + "name": "py32f071_dev_board" + }, + { + "type": "build", + "name": "py32f071_dev_board" + } + ] + }, { "name": "pybadge", "steps": [ -- cgit v1.3.1 From b3708aebd985a2c21a361903b1ca60d44e457e92 Mon Sep 17 00:00:00 2001 From: Zixun LI Date: Mon, 27 Jul 2026 15:55:20 +0200 Subject: hw/bsp/lpc43: reset peripherals on IAR restart --- hw/bsp/lpc43/family.c | 13 +++++++++++++ 1 file changed, 13 insertions(+) (limited to 'hw/bsp') diff --git a/hw/bsp/lpc43/family.c b/hw/bsp/lpc43/family.c index 411ea7d58..0f3c62fef 100644 --- a/hw/bsp/lpc43/family.c +++ b/hw/bsp/lpc43/family.c @@ -59,6 +59,19 @@ void SystemInit(void); // Invoked by startup code void SystemInit(void) { +#if defined(__ICCARM__) && !defined(DONT_RESET_ON_RESTART) + // A debugger restart resets the M4 core, but can leave LPC43 peripherals and + // pending interrupts active. Match the GCC startup sequence, which the IAR + // startup lacks, before the C runtime can reuse peripheral DMA memory. + __disable_irq(); + LPC_RGU->RESET_CTRL[0] = 0x10DF1000u; + LPC_RGU->RESET_CTRL[1] = 0x01DFF7FFu; + for (uint32_t i = 0; i < 8; i++) { + NVIC->ICPR[i] = UINT32_MAX; + } + __enable_irq(); +#endif + #ifdef __USE_LPCOPEN unsigned int *pSCB_VTOR = (unsigned int *) 0xE000ED08; -- cgit v1.3.1 From edb4a0744a1f7ce95fac35fc2798a5dea052571d Mon Sep 17 00:00:00 2001 From: Zixun LI Date: Mon, 27 Jul 2026 20:44:18 +0200 Subject: hw/bsp/lpc43: configure safe flash timing --- hw/bsp/lpc43/family.c | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'hw/bsp') diff --git a/hw/bsp/lpc43/family.c b/hw/bsp/lpc43/family.c index 0f3c62fef..7f0722a33 100644 --- a/hw/bsp/lpc43/family.c +++ b/hw/bsp/lpc43/family.c @@ -59,11 +59,22 @@ void SystemInit(void); // Invoked by startup code void SystemInit(void) { +#if defined(__ICCARM__) && !defined(DONT_RESET_ON_RESTART) + __disable_irq(); +#endif + + if (Chip_CREG_OnChipFlashIsPresent()) { + // The boot ROM configures flash for its 96 MHz clock, and debugger core + // resets can preserve it. Use safe timing before switching the M4 to 204 MHz. + Chip_CREG_SetFLASHAccess(FLASHTIM_SAFE_SETTING); + __DSB(); + __ISB(); + } + #if defined(__ICCARM__) && !defined(DONT_RESET_ON_RESTART) // A debugger restart resets the M4 core, but can leave LPC43 peripherals and // pending interrupts active. Match the GCC startup sequence, which the IAR // startup lacks, before the C runtime can reuse peripheral DMA memory. - __disable_irq(); LPC_RGU->RESET_CTRL[0] = 0x10DF1000u; LPC_RGU->RESET_CTRL[1] = 0x01DFF7FFu; for (uint32_t i = 0; i < 8; i++) { -- cgit v1.3.1 From 1d915b6b59cb88f14344521db1f9345d8c7dc9a7 Mon Sep 17 00:00:00 2001 From: Ha Thach Date: Tue, 28 Jul 2026 12:50:28 +0700 Subject: bsp, hil: flash WCH boards with the unified OpenOCD fork (#3791) bsp, hil: flash with the unified OpenOCD fork https://github.com/hathach/openocd (branch tinyusb) is mainline plus every config these boards need: RPi RP2350, ADI max32/max78, the MounRiver WCH configs, and the wlinke adapter on mainline's riscv target. It is a superset of the vendor forks, so one 'openocd' covers all boards; -DOPENOCD=/OPENOCD= still select another, msdk's when MAXIM_PATH is set. Drops family_flash_openocd_wch and the OPENOCD_WCH pair, dedups family_flash_openocd_adi, aligns ch583's work area, and points hil at the flasher's own config instead of generating one per probe. Verified: HIL green on all four WCH boards and max32666fthr. --- .idea/debugServers/wch_riscv.xml | 2 +- hw/bsp/ch32v10x/family.cmake | 2 +- hw/bsp/ch32v10x/family.mk | 2 +- hw/bsp/ch32v20x/family.cmake | 2 +- hw/bsp/ch32v20x/family.mk | 2 +- hw/bsp/ch32v30x/family.cmake | 2 +- hw/bsp/ch32v30x/family.mk | 2 +- hw/bsp/ch583/family.cmake | 2 +- hw/bsp/ch583/family.mk | 2 +- hw/bsp/ch583/wch-riscv.cfg | 2 +- hw/bsp/family_rules.mk | 16 +++++++------- hw/bsp/family_support.cmake | 44 +++++++++++++-------------------------- hw/bsp/rp2040/family.cmake | 2 ++ test/hil/hil_test.py | 45 ++++++++-------------------------------- test/hil/tinyusb.json | 10 ++++----- 15 files changed, 47 insertions(+), 90 deletions(-) (limited to 'hw/bsp') diff --git a/.idea/debugServers/wch_riscv.xml b/.idea/debugServers/wch_riscv.xml index 2e147f1b6..0b2b83b2e 100644 --- a/.idea/debugServers/wch_riscv.xml +++ b/.idea/debugServers/wch_riscv.xml @@ -4,7 +4,7 @@ - + diff --git a/hw/bsp/ch32v10x/family.cmake b/hw/bsp/ch32v10x/family.cmake index fb9ccb3a3..287b8c2ff 100644 --- a/hw/bsp/ch32v10x/family.cmake +++ b/hw/bsp/ch32v10x/family.cmake @@ -93,6 +93,6 @@ function(family_configure_example TARGET RTOS) # Flashing family_add_bin_hex(${TARGET}) - family_flash_openocd_wch(${TARGET}) + family_flash_openocd(${TARGET}) #family_flash_uf2(${TARGET} ${UF2_FAMILY_ID}) endfunction() diff --git a/hw/bsp/ch32v10x/family.mk b/hw/bsp/ch32v10x/family.mk index fb699b0bb..443509699 100644 --- a/hw/bsp/ch32v10x/family.mk +++ b/hw/bsp/ch32v10x/family.mk @@ -49,5 +49,5 @@ INC += \ FREERTOS_PORTABLE_SRC = $(FREERTOS_PORTABLE_PATH)/RISC-V -OPENOCD_WCH_OPTION=-f $(TOP)/$(FAMILY_PATH)/wch-riscv.cfg +OPENOCD_OPTION=-f $(TOP)/$(FAMILY_PATH)/wch-riscv.cfg flash: flash-openocd-wch diff --git a/hw/bsp/ch32v20x/family.cmake b/hw/bsp/ch32v20x/family.cmake index 785f5ee35..a27ff021e 100644 --- a/hw/bsp/ch32v20x/family.cmake +++ b/hw/bsp/ch32v20x/family.cmake @@ -125,7 +125,7 @@ function(family_configure_example TARGET RTOS) # Flashing family_add_bin_hex(${TARGET}) - family_flash_openocd_wch(${TARGET}) + family_flash_openocd(${TARGET}) family_flash_wlink_rs(${TARGET}) #family_flash_uf2(${TARGET} ${UF2_FAMILY_ID}) endfunction() diff --git a/hw/bsp/ch32v20x/family.mk b/hw/bsp/ch32v20x/family.mk index 1d059bcba..1889c4e26 100644 --- a/hw/bsp/ch32v20x/family.mk +++ b/hw/bsp/ch32v20x/family.mk @@ -63,6 +63,6 @@ INC += \ FREERTOS_PORTABLE_SRC = $(FREERTOS_PORTABLE_PATH)/RISC-V -OPENOCD_WCH_OPTION=-f $(TOP)/$(FAMILY_PATH)/wch-riscv.cfg +OPENOCD_OPTION=-f $(TOP)/$(FAMILY_PATH)/wch-riscv.cfg flash: flash-wlink-rs #flash: flash-openocd-wch diff --git a/hw/bsp/ch32v30x/family.cmake b/hw/bsp/ch32v30x/family.cmake index b974bd5e7..e33e4b85d 100644 --- a/hw/bsp/ch32v30x/family.cmake +++ b/hw/bsp/ch32v30x/family.cmake @@ -115,6 +115,6 @@ function(family_configure_example TARGET RTOS) # Flashing family_add_bin_hex(${TARGET}) - family_flash_openocd_wch(${TARGET}) + family_flash_openocd(${TARGET}) family_flash_wlink_rs(${TARGET}) endfunction() diff --git a/hw/bsp/ch32v30x/family.mk b/hw/bsp/ch32v30x/family.mk index 5ccdea8ae..59778ec54 100644 --- a/hw/bsp/ch32v30x/family.mk +++ b/hw/bsp/ch32v30x/family.mk @@ -62,5 +62,5 @@ LD_FILE ?= $(FAMILY_PATH)/linker/ch32v30x.ld # For freeRTOS port source FREERTOS_PORTABLE_SRC = $(FREERTOS_PORTABLE_PATH)/RISC-V -OPENOCD_WCH_OPTION=-f $(TOP)/$(FAMILY_PATH)/wch-riscv.cfg +OPENOCD_OPTION=-f $(TOP)/$(FAMILY_PATH)/wch-riscv.cfg flash: flash-openocd-wch diff --git a/hw/bsp/ch583/family.cmake b/hw/bsp/ch583/family.cmake index a379298e5..f4b874e4e 100644 --- a/hw/bsp/ch583/family.cmake +++ b/hw/bsp/ch583/family.cmake @@ -104,5 +104,5 @@ function(family_configure_example TARGET RTOS) # Flashing family_add_bin_hex(${TARGET}) - family_flash_openocd_wch(${TARGET}) + family_flash_openocd(${TARGET}) endfunction() diff --git a/hw/bsp/ch583/family.mk b/hw/bsp/ch583/family.mk index 98d0f9337..3444c4811 100644 --- a/hw/bsp/ch583/family.mk +++ b/hw/bsp/ch583/family.mk @@ -52,7 +52,7 @@ INC += \ LD_FILE ?= $(FAMILY_PATH)/linker/ch582.ld -OPENOCD_WCH_OPTION=-f $(TOP)/$(FAMILY_PATH)/wch-riscv.cfg +OPENOCD_OPTION=-f $(TOP)/$(FAMILY_PATH)/wch-riscv.cfg flash: flash-openocd-wch # For freeRTOS port source diff --git a/hw/bsp/ch583/wch-riscv.cfg b/hw/bsp/ch583/wch-riscv.cfg index 64d595d8e..aa35aa9c5 100644 --- a/hw/bsp/ch583/wch-riscv.cfg +++ b/hw/bsp/ch583/wch-riscv.cfg @@ -9,7 +9,7 @@ sdi newtap $_CHIPNAME cpu -irlen 5 -expected-id 0x00001 set _TARGETNAME $_CHIPNAME.cpu target create $_TARGETNAME.0 wch_riscv -chain-position $_TARGETNAME -$_TARGETNAME.0 configure -work-area-phys 0x20000000 -work-area-size 0x8000 -work-area-backup 1 +$_TARGETNAME.0 configure -work-area-phys 0x20000000 -work-area-size 10000 -work-area-backup 1 set _FLASHNAME $_CHIPNAME.flash flash bank $_FLASHNAME wch_riscv 0x00000000 0 0 0 $_TARGETNAME.0 diff --git a/hw/bsp/family_rules.mk b/hw/bsp/family_rules.mk index ccf49dd0e..011572888 100644 --- a/hw/bsp/family_rules.mk +++ b/hw/bsp/family_rules.mk @@ -130,20 +130,18 @@ flash-pyocd: $(BUILD)/$(PROJECT).hex #pyocd reset -t $(PYOCD_TARGET) # --------------- openocd ----------------- +# OPENOCD can name another build, e.g. one of the vendor forks, though +# https://github.com/hathach/openocd branch tinyusb covers every board here +OPENOCD ?= openocd OPENOCD_OPTION ?= flash-openocd: $(BUILD)/$(PROJECT).elf - openocd $(OPENOCD_OPTION) -c "program $< verify reset exit" + $(OPENOCD) $(OPENOCD_OPTION) -c "program $< verify reset exit" # --------------- openocd-wch ----------------- -# wch-linke is not supported yet in official openOCD yet. We need to either use -# 1. download openocd as part of mounriver studio http://www.mounriver.com/download or -# 2. compiled from https://github.com/hathach/riscv-openocd-wch or -# https://github.com/dragonlock2/miscboards/blob/main/wch/SDK/riscv-openocd.tar.xz -# with ./configure --disable-werror --enable-wlinke --enable-ch347=no -OPENOCD_WCH ?= /home/${USER}/app/riscv-openocd-wch/src/openocd -OPENOCD_WCH_OPTION ?= +# WCH parts need an openocd built with the wlinke adapter. The image is written +# without verify: WCH code flash is not readable back over the debug bus. flash-openocd-wch: $(BUILD)/$(PROJECT).elf - $(OPENOCD_WCH) $(OPENOCD_WCH_OPTION) -c init -c halt -c "flash write_image $<" -c reset -c exit + $(OPENOCD) $(OPENOCD_OPTION) -c init -c halt -c "flash write_image $<" -c reset -c exit # --------------- wlink-rs ----------------- # flash with https://github.com/ch32-rs/wlink diff --git a/hw/bsp/family_support.cmake b/hw/bsp/family_support.cmake index 1f3952205..33ceb49c2 100644 --- a/hw/bsp/family_support.cmake +++ b/hw/bsp/family_support.cmake @@ -688,7 +688,9 @@ function(family_flash_stflash TARGET) endfunction() -# Add flash openocd target +# Add flash openocd target. +# The default 'openocd' should be https://github.com/hathach/openocd (branch tinyusb): which is mainline plus +# every config the rig needs (RP2350, MAX32/MAX78, WCH) and a drop-in superset of the vendor (downstream) forks function(family_flash_openocd TARGET) if (NOT DEFINED OPENOCD) set(OPENOCD openocd) @@ -715,38 +717,20 @@ function(family_flash_openocd TARGET) #set_property(TARGET ${TARGET}-openocd PROPERTY FOLDER ${TARGET}-group) endfunction() - -# Add flash openocd-wch target -# compiled from https://github.com/hathach/riscv-openocd-wch or https://github.com/dragonlock2/miscboards/blob/main/wch/SDK/riscv-openocd.tar.xz -function(family_flash_openocd_wch TARGET) - if (NOT DEFINED OPENOCD) - set(OPENOCD $ENV{HOME}/app/riscv-openocd-wch/src/openocd) +# Add flash openocd adi (Analog Devices) target using the openocd included +# with msdk (MAXIM_PATH), otherwise the default openocd +function(family_flash_openocd_adi TARGET) + # use openocd from msdk if MAXIM_PATH is set, as cmake variable or in the + # environment. Normalize the latter since msdk can be Windows (MinGW) or Linux + if (NOT DEFINED MAXIM_PATH AND DEFINED ENV{MAXIM_PATH}) + file(TO_CMAKE_PATH "$ENV{MAXIM_PATH}" MAXIM_PATH) endif () - family_flash_openocd(${TARGET}) -endfunction() - - -# Add flash openocd adi (Analog Devices) target -# included with msdk or compiled from release branch of https://github.com/analogdevicesinc/openocd -function(family_flash_openocd_adi TARGET) - if (DEFINED MAXIM_PATH) - # use openocd from msdk with MAXIM_PATH cmake variable first if the user specified it - set(OPENOCD ${MAXIM_PATH}/Tools/OpenOCD/openocd) - set(OPENOCD_OPTION2 "-s ${MAXIM_PATH}/Tools/OpenOCD/scripts") - elseif (DEFINED ENV{MAXIM_PATH}) - # use openocd from msdk with MAXIM_PATH environment variable. Normalize - # since msdk can be Windows (MinGW) or Linux - file(TO_CMAKE_PATH "$ENV{MAXIM_PATH}" MAXIM_PATH_NORM) - set(OPENOCD ${MAXIM_PATH_NORM}/Tools/OpenOCD/openocd) - set(OPENOCD_OPTION2 "-s ${MAXIM_PATH_NORM}/Tools/OpenOCD/scripts") - else() - # compiled from source - if (NOT DEFINED OPENOCD_ADI_PATH) - set(OPENOCD_ADI_PATH $ENV{HOME}/app/openocd_adi) + if (MAXIM_PATH) + if (NOT DEFINED OPENOCD) + set(OPENOCD ${MAXIM_PATH}/Tools/OpenOCD/openocd) endif () - set(OPENOCD ${OPENOCD_ADI_PATH}/src/openocd) - set(OPENOCD_OPTION2 "-s ${OPENOCD_ADI_PATH}/tcl") + set(OPENOCD_OPTION2 "-s ${MAXIM_PATH}/Tools/OpenOCD/scripts") endif () family_flash_openocd(${TARGET}) diff --git a/hw/bsp/rp2040/family.cmake b/hw/bsp/rp2040/family.cmake index aab9a4fae..43b1dc234 100644 --- a/hw/bsp/rp2040/family.cmake +++ b/hw/bsp/rp2040/family.cmake @@ -28,6 +28,8 @@ elseif (PICO_PLATFORM STREQUAL "rp2350-arm-s" OR PICO_PLATFORM STREQUAL "rp2350" set(OPENOCD_TARGET rp2350) elseif (PICO_PLATFORM STREQUAL "rp2350-riscv") set(JLINK_DEVICE rp2350_riscv_0) + # rp2350-riscv.cfg needs the raspberrypi/openocd fork: mainline's riscv + # target does not take the -dap/-ap-num the config uses set(OPENOCD_TARGET rp2350-riscv) endif() diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py index 0efc6826f..80d1e1823 100755 --- a/test/hil/hil_test.py +++ b/test/hil/hil_test.py @@ -24,10 +24,11 @@ # Host setup (required: a missing tool fails its test rather than skipping it): # - System packages: sudo apt install mtools libmtp9 alsa-utils iperf -# mtools - read_disk_file (device/cdc_msc, device/msc_dual_lun) -# libmtp9 - pymtp ctypes load (device/mtp); Debian 13 uses libmtp9t64 -# alsa-utils - arecord (device/audio_test_freertos) -# iperf - throughput tests (device/net_lwip_*) +# mtools read_disk_file (device/cdc_msc, device/msc_dual_lun) +# libmtp9 pymtp ctypes load (device/mtp); Debian 13 uses libmtp9t64 +# alsa-utils arecord (device/audio_test_freertos) +# iperf throughput tests (device/net_lwip_*) +# openocd unified openocd from https://github.com/hathach/openocd (branch tinyusb) for wch, rp2040/rp2350, analog max32 # - device/usbtest: usbtest kernel module + testusb binary (kernel tools/usb/testusb.c) on PATH, # plus sudo for modprobe / sysfs writes # - Python packages: pip install -r requirements.txt @@ -377,25 +378,6 @@ def cmd_stdout_text(out: Any) -> str: return out.decode('utf-8', errors='ignore') return str(out) -WCH_RISCV_CONTENT = """ -adapter driver wlinke -adapter speed 6000 -transport select sdi - -wlink_set_address 0x00000000 -set _CHIPNAME wch_riscv -sdi newtap $_CHIPNAME cpu -irlen 5 -expected-id 0x00001 - -set _TARGETNAME $_CHIPNAME.cpu - -target create $_TARGETNAME.0 wch_riscv -chain-position $_TARGETNAME -$_TARGETNAME.0 configure -work-area-phys 0x20000000 -work-area-size 10000 -work-area-backup 1 -set _FLASHNAME $_CHIPNAME.flash - -flash bank $_FLASHNAME wch_riscv 0x00000000 0 0 0 $_TARGETNAME.0 - -echo "Ready for Remote Connections" -""" MSC_README_TXT = \ b"This is tinyusb's MassStorage Class demo.\r\n\r\n\ @@ -662,24 +644,15 @@ def reset_openocd(board): def flash_openocd_wch(board, firmware): flasher = board['flasher'] - f_wch = f"wch-riscv_{board['uid']}.cfg" - if not os.path.exists(f_wch): - with open(f_wch, 'w') as file: - file.write(WCH_RISCV_CONTENT) - - ret = run_cmd(f'openocd_wch -c "adapter serial {flasher["uid"]}" -f {f_wch} ' - f'-c "program {firmware}.elf reset exit"') + ret = run_cmd(f'openocd -c "tcl_port disabled" -c "gdb_port disabled" -c "telnet_port disabled" ' + f'-c "adapter serial {flasher["uid"]}" {flasher.get("args", "")} -c "program {firmware}.elf reset exit"') return ret def reset_openocd_wch(board): flasher = board['flasher'] - f_wch = f"wch-riscv_{board['uid']}.cfg" - if not os.path.exists(f_wch): - with open(f_wch, 'w') as file: - file.write(WCH_RISCV_CONTENT) - - ret = run_cmd(f'openocd_wch -c "adapter serial {flasher["uid"]}" -f {f_wch} -c "program reset exit"') + ret = run_cmd(f'openocd -c "tcl_port disabled" -c "gdb_port disabled" -c "telnet_port disabled" ' + f'-c "adapter serial {flasher["uid"]}" {flasher.get("args", "")} -c "init; reset run; exit"') return ret diff --git a/test/hil/tinyusb.json b/test/hil/tinyusb.json index 812eb7571..8316dbc33 100644 --- a/test/hil/tinyusb.json +++ b/test/hil/tinyusb.json @@ -147,7 +147,7 @@ "dual": false }, "flasher": { - "name": "openocd_adi", + "name": "openocd", "uid": "E6614C311B597D32", "args": "-f interface/cmsis-dap.cfg -f target/max32665.cfg" } @@ -465,7 +465,7 @@ "flasher": { "name": "openocd_wch", "uid": "EBCA8F0670AF", - "args": "" + "args": "-f target/wch-riscv.cfg" } }, { @@ -480,7 +480,7 @@ "flasher": { "name": "openocd_wch", "uid": "BC4954081051", - "args": "" + "args": "-f target/wch-riscv.cfg" } }, { @@ -499,7 +499,7 @@ "flasher": { "name": "openocd_wch", "uid": "BC5DA47360D0", - "args": "" + "args": "-f target/wch-riscv.cfg" } }, { @@ -514,7 +514,7 @@ "flasher": { "name": "openocd_wch", "uid": "7FD88F0604B5", - "args": "" + "args": "-f target/wch-riscv.cfg" } }, { -- cgit v1.3.1 From 8895e94b7faed15b5367dcb1e6715e8ed4c56955 Mon Sep 17 00:00:00 2001 From: Zixun LI Date: Tue, 28 Jul 2026 21:15:49 +0200 Subject: hw/bsp/stm32l4: stabilize L412 USB clock --- hw/bsp/stm32l4/boards/stm32l412nucleo/board.h | 48 +++++++++++---------------- 1 file changed, 20 insertions(+), 28 deletions(-) (limited to 'hw/bsp') diff --git a/hw/bsp/stm32l4/boards/stm32l412nucleo/board.h b/hw/bsp/stm32l4/boards/stm32l412nucleo/board.h index a5250eda9..7f63ec431 100644 --- a/hw/bsp/stm32l4/boards/stm32l412nucleo/board.h +++ b/hw/bsp/stm32l4/boards/stm32l412nucleo/board.h @@ -64,9 +64,10 @@ * AHB Prescaler = 1 * APB1 Prescaler = 1 * APB2 Prescaler = 1 - * MSI Frequency(Hz) = 8000000 - * PLL_M = 1 - * PLL_N = 10 + * MSI Frequency(Hz) = 48000000 + * LSE Frequency(Hz) = 32768 + * PLL_M = 6 + * PLL_N = 20 * PLL_Q = 2 * PLL_R = 2 * VDD(V) = 3.3 @@ -78,29 +79,35 @@ static inline void board_clock_init(void) { RCC_OscInitTypeDef RCC_OscInitStruct = {0}; RCC_ClkInitTypeDef RCC_ClkInitStruct = {0}; - RCC_CRSInitTypeDef RCC_CRSInitStruct = {0}; RCC_PeriphCLKInitTypeDef PeriphClkInitStruct = {0}; /** Configure the main internal regulator output voltage */ HAL_PWREx_ControlVoltageScaling(PWR_REGULATOR_VOLTAGE_SCALE1); + /* HAL clock setup reconfigures its tick while MSI is the reset SYSCLK. */ + HAL_InitTick((1UL << __NVIC_PRIO_BITS) - 1UL); + /** Initializes the RCC Oscillators according to the specified parameters * in the RCC_OscInitTypeDef structure. */ - RCC_OscInitStruct.OscillatorType = RCC_OSCILLATORTYPE_HSI48|RCC_OSCILLATORTYPE_HSI; - RCC_OscInitStruct.HSIState = RCC_HSI_ON; - RCC_OscInitStruct.HSI48State = RCC_HSI48_ON; - RCC_OscInitStruct.HSICalibrationValue = RCC_HSICALIBRATION_DEFAULT; + RCC_OscInitStruct.OscillatorType = RCC_OSCILLATORTYPE_LSE | RCC_OSCILLATORTYPE_MSI; + RCC_OscInitStruct.LSEState = RCC_LSE_ON; + RCC_OscInitStruct.MSIState = RCC_MSI_ON; + RCC_OscInitStruct.MSICalibrationValue = RCC_MSICALIBRATION_DEFAULT; + RCC_OscInitStruct.MSIClockRange = RCC_MSIRANGE_11; RCC_OscInitStruct.PLL.PLLState = RCC_PLL_ON; - RCC_OscInitStruct.PLL.PLLSource = RCC_PLLSOURCE_HSI; - RCC_OscInitStruct.PLL.PLLM = 1; - RCC_OscInitStruct.PLL.PLLN = 10; + RCC_OscInitStruct.PLL.PLLSource = RCC_PLLSOURCE_MSI; + RCC_OscInitStruct.PLL.PLLM = 6; + RCC_OscInitStruct.PLL.PLLN = 20; RCC_OscInitStruct.PLL.PLLQ = RCC_PLLQ_DIV2; RCC_OscInitStruct.PLL.PLLR = RCC_PLLR_DIV2; HAL_RCC_OscConfig(&RCC_OscInitStruct); + /* Stabilize MSI against the on-board 32.768 kHz LSE crystal. */ + HAL_RCCEx_EnableMSIPLLMode(); + /** Initializes the CPU, AHB and APB buses clocks */ RCC_ClkInitStruct.ClockType = RCC_CLOCKTYPE_HCLK|RCC_CLOCKTYPE_SYSCLK @@ -112,24 +119,9 @@ static inline void board_clock_init(void) HAL_RCC_ClockConfig(&RCC_ClkInitStruct, FLASH_LATENCY_4); - /** Enable the SYSCFG APB clock - */ - __HAL_RCC_CRS_CLK_ENABLE(); - - /** Configures CRS - */ - RCC_CRSInitStruct.Prescaler = RCC_CRS_SYNC_DIV1; - RCC_CRSInitStruct.Source = RCC_CRS_SYNC_SOURCE_USB; - RCC_CRSInitStruct.Polarity = RCC_CRS_SYNC_POLARITY_RISING; - RCC_CRSInitStruct.ReloadValue = __HAL_RCC_CRS_RELOADVALUE_CALCULATE(48000000,1000); - RCC_CRSInitStruct.ErrorLimitValue = 34; - RCC_CRSInitStruct.HSI48CalibrationValue = 32; - - HAL_RCCEx_CRSConfig(&RCC_CRSInitStruct); - - /* Select HSI48 output as USB clock source */ + /* Use the same LSE-trimmed MSI source for USB and the CPU PLL. */ PeriphClkInitStruct.PeriphClockSelection = RCC_PERIPHCLK_USB; - PeriphClkInitStruct.UsbClockSelection = RCC_USBCLKSOURCE_HSI48; + PeriphClkInitStruct.UsbClockSelection = RCC_USBCLKSOURCE_MSI; HAL_RCCEx_PeriphCLKConfig(&PeriphClkInitStruct); /* Select PLL output as UART clock source */ -- cgit v1.3.1 From eef5af86aa26fe3d72e41156a586a6ed3ffce9f8 Mon Sep 17 00:00:00 2001 From: Ha Thach Date: Thu, 30 Jul 2026 02:29:32 +0700 Subject: hil, ci: scope HIL builds and tests to the boards a PR affects (#3797) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hil, ci: scope HIL builds and tests to the boards a PR affects Add test/hil/hil_select.py, a stdlib-only selector that maps a PR diff to the rig boards, tests and BSP families a change can affect, and wire it into CI so pull requests build and run only those. A port change picks its families' boards, a class change picks the examples enabling that class, and device/host changes prune the other role. Anything unclassified — infra, an unmapped port, a selector error — falls back to the full matrix, and push/schedule runs are untouched. Move the shared example lists to hil_examples.py; 54 hardware-free tests cover the rules. --- .claude/skills/hil/SKILL.md | 21 + .claude/skills/pre-pr/SKILL.md | 27 +- .github/workflows/build.yml | 208 ++++- .github/workflows/build_util.yml | 15 + .github/workflows/pr_comment.yml | 9 + docs/superpowers/plans/2026-07-29-hil-select.md | 856 +++++++++++++++++++++ .../2026-07-29-hil-pr-scoped-selection-design.md | 179 +++++ hw/bsp/mcx/family.cmake | 11 +- test/hil/hil_ci.sh | 1 + test/hil/hil_ci_set_matrix.py | 15 + test/hil/hil_examples.py | 37 + test/hil/hil_select.py | 520 +++++++++++++ test/hil/hil_test.py | 102 +-- test/hil/test_hil_select.py | 542 +++++++++++++ 14 files changed, 2479 insertions(+), 64 deletions(-) create mode 100644 docs/superpowers/plans/2026-07-29-hil-select.md create mode 100644 docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md create mode 100644 test/hil/hil_examples.py create mode 100755 test/hil/hil_select.py create mode 100644 test/hil/test_hil_select.py (limited to 'hw/bsp') diff --git a/.claude/skills/hil/SKILL.md b/.claude/skills/hil/SKILL.md index 6c4d3a856..f273be120 100644 --- a/.claude/skills/hil/SKILL.md +++ b/.claude/skills/hil/SKILL.md @@ -41,6 +41,27 @@ python3 test/hil/hil_lock.py release BOARD [BOARD...] Board/probe health scanning (`test/hil/hil_pool_check.py`) has its own skill: **hil-pool-check**. Use it before a HIL campaign, after rig maintenance/reboot, or when boards fail to flash. +## PR-scoped selection + +`test/hil/hil_select.py` maps a diff to affected boards/tests (used by CI on PRs; fail-open +to the full matrix). Manual use: + +```bash +SEL=$(python3 test/hil/hil_select.py --base master test/hil/tinyusb.json) +FULL=$(printf '%s' "$SEL" | python3 -c "import json,sys; print(json.load(sys.stdin)['full'])") +ARGS=$(printf '%s' "$SEL" | python3 -c "import json,sys; print(json.load(sys.stdin)['args']['tinyusb.json'])") +if [ "$FULL" = "True" ] || [ -n "$ARGS" ]; then + python3 test/hil/hil_test.py -B examples $ARGS test/hil/tinyusb.json # $ARGS empty when full: run everything +else + echo "diff affects nothing on this rig - skip HIL" +fi +``` + +Read `full`, never `args` alone: `args` is empty for BOTH `full: true` (run the whole matrix — a broad or +unclassified change) and "nothing selected" (skip). Skip only when `full` is false AND `args` is empty. + +Unit suite: `python3 test/hil/test_hil_select.py` (no hardware). + ## Prerequisites Examples must be built for the target board(s) — see CLAUDE.md "Build" → "All examples for a board" (produces `examples/cmake-build-/`). `-B examples` points `hil_test.py` at that parent folder. (This applies to `hil_test.py`; `hil_pool_check.py` builds its own missing firmware.) diff --git a/.claude/skills/pre-pr/SKILL.md b/.claude/skills/pre-pr/SKILL.md index 428383424..3829b4b9e 100644 --- a/.claude/skills/pre-pr/SKILL.md +++ b/.claude/skills/pre-pr/SKILL.md @@ -15,12 +15,27 @@ Run the software + hardware gate for the current branch. The user invoking this ## 2. Map changes to boards -- For each changed `src/portable///` (or `src/portable//` for single-level ports): families = the `hw/bsp/` directories whose build files reference it — `grep -rl "/" hw/bsp/*/family.cmake hw/bsp/*/family.mk`, then take each matching file's directory name. -- For `src/class/*`, `src/common/*`, `src/device/*`, `src/host/*`, or `src/tusb.c`: broad change — use `stm32f407disco` + `raspberry_pi_pico` PLUS any families from portable changes. -- For `hw/bsp//...` changes: that family directly. -- Catch-all: any other C/CMake source change (`examples/*`, `test/*`, anything unmatched above) → the representative set `stm32f407disco` + `raspberry_pi_pico`. The boards list must NEVER end up empty — final fallback is `[stm32f407disco]` (full-check throws on an empty list). -- Rig roster: `python3 -c "import json;print([b['name'] for b in json.load(open('test/hil/tinyusb.json'))['boards']])"` -- Pick ONE board per affected family, preferring boards on the rig roster; otherwise the first entry in `hw/bsp//boards/`. Cap at 4 boards and tell the user which families the cap dropped. +- `python3 test/hil/hil_select.py --base $BASE test/hil/tinyusb.json` → JSON with the affected + bsp `families`, the affected rig `boards`, and per-file `reasons`. `full: true` means a + broad/infra change. +- Affected families = `families` ∪ the family of every name in `boards`. Neither half is + enough alone: only the port and bsp rules fill `families` (a class/core/example change + reports boards but no families), and `boards` only ever names rig boards (an off-rig driver + change — `dcd_samx7x.c` → `same7x`, `boards: {}` — would never be compiled). + - A board's family is the `hw/bsp//boards//` directory holding it. + - When `full: true`, `boards` names every rig board and carries no signal — use `families` + alone there, plus the representative set below. +- Sample ONE board per affected family: prefer a rig-roster board of that family, else the + first entry in `hw/bsp//boards/`. + - Rig roster: `python3 -c "import json;print([b['name'] for b in json.load(open('test/hil/tinyusb.json'))['boards']])"` +- Add the representative set `stm32f407disco` + `raspberry_pi_pico` when `full: true` (broad + change — class/core/common/infra). +- Cap at 4 boards and tell the user which families the cap dropped. A broad change affects ~20 + families, so the order matters: keep `stm32f407disco` and `raspberry_pi_pico` first whenever + their families are affected, then fill from the remaining families (spread across vendors — + don't let one vendor's family names take every slot). The list must NEVER end up empty — + final fallback is `[stm32f407disco]` (full-check throws on an empty list). +- Docs-only (`full: false`, no families, no boards) keeps §1's minimal software-only gate. ## 3. HIL boards diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 76e19ee02..bdef81553 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -48,20 +48,87 @@ jobs: outputs: json: ${{ steps.set-matrix-json.outputs.matrix }} hil_json: ${{ steps.set-matrix-json.outputs.hil_matrix }} + # one pair per rig job: hil-tinyusb (tinyusb.json minus esptool boards), + # hil-tinyusb-esp (esptool boards only), hil-tinyusb (hfp.json) + hil_args_tinyusb: ${{ steps.hil-select.outputs.args_tinyusb }} + hil_run_tinyusb: ${{ steps.hil-select.outputs.run_tinyusb }} + hil_args_tinyusb_esp: ${{ steps.hil-select.outputs.args_tinyusb_esp }} + hil_run_tinyusb_esp: ${{ steps.hil-select.outputs.run_tinyusb_esp }} + hil_args_hfp: ${{ steps.hil-select.outputs.args_hfp }} + hil_run_hfp: ${{ steps.hil-select.outputs.run_hfp }} steps: - name: Checkout TinyUSB uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: HIL selection (PR only) + id: hil-select + if: github.event_name == 'pull_request' + env: + BASE_REF: ${{ github.base_ref }} + run: | + # Best-effort by design: set-matrix gates cmake, hil-build and every rig job, + # so a missing origin/, a shallow-clone hiccup or a selector traceback + # must fall back to the FULL matrix (no --select, run=true, no args) instead + # of failing the job. Same fail-open shape as pr_comment.yml's `|| true`. + SELECT_JSON='' + if ! python3 test/hil/test_hil_select.py; then + echo "::error::hil_select unit tests failed - falling back to the full HIL matrix" + elif ! SELECT_JSON=$(python3 test/hil/hil_select.py --base "origin/$BASE_REF" test/hil/tinyusb.json test/hil/hfp.json); then + echo "::warning::hil_select failed - falling back to the full HIL matrix" + SELECT_JSON='' + fi + + # One args/run pair per rig job, split by flasher: a job whose own subset is + # empty skips explicitly instead of running a board filter that matches zero + # boards ("No tests were run." exits 0 and would read as a green HIL run). + OUT='' + if [ -n "$SELECT_JSON" ]; then + OUT=$(SELECT_JSON="$SELECT_JSON" python3 -c ' + import json, os + s = json.loads(os.environ["SELECT_JSON"]) + tin = s.get("args_flasher", {}).get("tinyusb.json", {}) + legs = (("tinyusb", " ".join(a for f, a in sorted(tin.items()) if f != "esptool" and a)), + ("tinyusb_esp", tin.get("esptool", "")), + ("hfp", s.get("args", {}).get("hfp.json", ""))) + for key, a in legs: + print("args_" + key + "=" + a) + print("run_" + key + "=" + ("true" if (s.get("full") or a) else "false")) + ') || OUT='' + if [ -z "$OUT" ]; then + echo "::warning::hil_select output unusable - falling back to the full HIL matrix" + SELECT_JSON='' + fi + fi + if [ -z "$OUT" ]; then + OUT=$(for k in tinyusb tinyusb_esp hfp; do printf 'args_%s=\nrun_%s=true\n' "$k" "$k"; done) + fi + echo "$OUT" + { echo "select=$SELECT_JSON"; echo "$OUT"; } >> $GITHUB_OUTPUT - name: Generate matrix json id: set-matrix-json + env: + SELECT: ${{ steps.hil-select.outputs.select }} run: | # build matrix MATRIX_JSON=$(python .github/workflows/ci_set_matrix.py) echo "matrix=$MATRIX_JSON" echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT - # HIL matrix (merged from tinyusb + hifiphile configs) - HIL_MATRIX_JSON=$(python test/hil/hil_ci_set_matrix.py test/hil/tinyusb.json test/hil/hfp.json) + # HIL matrix (merged from tinyusb + hifiphile configs), scoped on PRs. + # Scoping is best-effort too: fall back to the unscoped (full) matrix. + HIL_MATRIX_JSON='' + if [ -n "$SELECT" ]; then + HIL_MATRIX_JSON=$(python test/hil/hil_ci_set_matrix.py --select "$SELECT" test/hil/tinyusb.json test/hil/hfp.json) || HIL_MATRIX_JSON='' + if [ -z "$HIL_MATRIX_JSON" ]; then + echo "::warning::scoped HIL matrix failed - falling back to the full HIL matrix" + fi + fi + if [ -z "$HIL_MATRIX_JSON" ]; then + HIL_MATRIX_JSON=$(python test/hil/hil_ci_set_matrix.py test/hil/tinyusb.json test/hil/hfp.json) + fi echo "hil_matrix=$HIL_MATRIX_JSON" echo "hil_matrix=$HIL_MATRIX_JSON" >> $GITHUB_OUTPUT @@ -271,6 +338,18 @@ jobs: strategy: fail-fast: false matrix: + # These names are the bucket keys of test/hil/hil_ci_set_matrix.py: every + # non-esptool roster board must land in one of them (esptool boards go to + # 'esp-idf', built by hil-build-esp below). hil_ci_set_matrix.py rejects a + # board whose "toolchain" is not a bucket, so a new bucket must be added in + # both places. + # + # INVARIANT the PR-scoped skip cascade rests on: hil_run_tinyusb / hil_run_hfp + # true => hil-build has at least one non-empty leg. It holds because those + # flags count only non-esptool boards and every such board builds here. It + # would break if an esptool board were added to hfp.json, because + # hil_args_hfp is NOT flasher-split: the hfp leg of hil-tinyusb would want to + # run while hil-build (and therefore that leg) skipped. toolchain: - 'arm-gcc' - 'riscv-gcc' @@ -298,7 +377,7 @@ jobs: # self-hosted on local VM, for attached hardware checkout HIL_JSON # --------------------------------------- hil-tinyusb: - needs: hil-build + needs: [ hil-build, set-matrix ] name: hil-tinyusb (${{ matrix.display }}) strategy: fail-fast: false @@ -357,7 +436,31 @@ jobs: - name: Test on actual hardware # Single attempt per test (--retry 1), no in-run second pass: a broken fixture # fails fast instead of holding the runner (and other PRs' HIL jobs) for hours. - run: python3 test/hil/hil_test.py --retry 1 ${{ matrix.test_args }} ${{ env.HIL_JSON }} $RERUN_ARGS + env: + # tinyusb.json minus the esptool boards (they run in hil-tinyusb-esp): each + # job gates on its own flasher subset, never on a rig-wide flag + SEL_ARGS_TINYUSB: ${{ needs.set-matrix.outputs.hil_args_tinyusb }} + SEL_RUN_TINYUSB: ${{ needs.set-matrix.outputs.hil_run_tinyusb }} + SEL_ARGS_HFP: ${{ needs.set-matrix.outputs.hil_args_hfp }} + SEL_RUN_HFP: ${{ needs.set-matrix.outputs.hil_run_hfp }} + run: | + case "$HIL_JSON" in + *tinyusb.json) SEL_ARGS="$SEL_ARGS_TINYUSB"; SEL_RUN="$SEL_RUN_TINYUSB" ;; + *hfp.json) SEL_ARGS="$SEL_ARGS_HFP"; SEL_RUN="$SEL_RUN_HFP" ;; + esac + if [ "$SEL_RUN" = "false" ]; then + echo "HIL skipped by PR selection (no affected boards on this rig)" + # leave a marker so the combined PR comment says so instead of dropping the + # section (and leaving a stale table from an earlier push in its place) + mkdir -p "$HIL_REPORT_DIR" + echo "_Skipped by PR selection: no affected boards on this rig._" > "$HIL_REPORT_DIR/hil_report.md" + exit 0 + fi + # a re-run spec is already a subset of the selection (only the boards/tests + # that failed); -b/-bt accumulate, so keeping SEL_ARGS here would re-run the + # entire original selection instead of just what failed + if [ -n "$RERUN_ARGS" ]; then SEL_ARGS=''; fi + python3 test/hil/hil_test.py --retry 1 ${{ matrix.test_args }} $SEL_ARGS ${{ env.HIL_JSON }} $RERUN_ARGS - name: Upload HIL report if: always() && github.event_name == 'pull_request' @@ -376,7 +479,7 @@ jobs: # second slot would double the per-controller flash/usbtest budgets. # --------------------------------------- hil-tinyusb-esp: - needs: hil-build-esp + needs: [ hil-build-esp, set-matrix ] name: hil-tinyusb (tinyusb-esp.json) runs-on: [ self-hosted, X64, hathach, hardware-in-the-loop ] env: @@ -420,7 +523,25 @@ jobs: merge-multiple: true - name: Test on actual hardware - run: python3 test/hil/hil_test.py --retry 1 $TEST_ARGS ${{ env.HIL_JSON }} $RERUN_ARGS + env: + # esptool subset of tinyusb.json: this job must gate on its own boards, not + # on the rig-wide flag (which would run a filter matching zero boards) + SEL_ARGS: ${{ needs.set-matrix.outputs.hil_args_tinyusb_esp }} + SEL_RUN: ${{ needs.set-matrix.outputs.hil_run_tinyusb_esp }} + run: | + if [ "$SEL_RUN" = "false" ]; then + echo "HIL skipped by PR selection (no affected esptool boards)" + # leave a marker so the combined PR comment says so instead of dropping the + # section (and leaving a stale table from an earlier push in its place) + mkdir -p "$HIL_REPORT_DIR" + echo "_Skipped by PR selection: no affected esptool boards._" > "$HIL_REPORT_DIR/hil_report.md" + exit 0 + fi + # a re-run spec is already a subset of the selection (only the boards/tests + # that failed); -b/-bt accumulate, so keeping SEL_ARGS here would re-run the + # entire original selection instead of just what failed + if [ -n "$RERUN_ARGS" ]; then SEL_ARGS=''; fi + python3 test/hil/hil_test.py --retry 1 $TEST_ARGS $SEL_ARGS ${{ env.HIL_JSON }} $RERUN_ARGS - name: Upload HIL report if: always() && github.event_name == 'pull_request' @@ -460,23 +581,75 @@ jobs: - name: Checkout TinyUSB uses: actions/checkout@v6 + with: + # full history: the "HIL selection" step below needs + # merge-base(HEAD, origin/) for PR-scoped selection + fetch-depth: 0 + + # Computed BEFORE the build: the IAR build is four boards and up to 30 minutes on + # a runner that hil-tinyusb (hfp.json) also needs, so an unaffected PR must release + # it immediately instead of building everything and then skipping. The selection + # also narrows what gets built. + # This job has no needs: on set-matrix (it must run even if that unrelated job + # fails), so it computes its own selection instead of reading set-matrix's outputs. + - name: HIL selection (PR only) + if: github.event_name == 'pull_request' + env: + BASE_REF: ${{ github.base_ref }} + run: | + # Best-effort: this job is deliberately decoupled from set-matrix so unrelated + # failures cannot kill hfp coverage - a selector failure here must likewise + # fall back to the full hfp matrix (no hil_select.json, no SEL_* vars), never + # fail the job. + if ! python3 test/hil/hil_select.py --base "origin/$BASE_REF" test/hil/hfp.json > hil_select.json; then + echo "::warning::hil_select failed - running the full hfp matrix" + rm -f hil_select.json + exit 0 + fi + # hil_select.json is passed to hil_ci_set_matrix.py --select below to scope the + # build; it already honours full=true by ignoring the board list. + # The hil_test.py args go to a file, never to $GITHUB_ENV: they are derived + # from roster board names, which a PR can edit. Only SEL_RUN (a literal + # true/false computed here, needed by the step-level `if:`) goes to the env. + if ! SEL_RUN=$(python3 -c ' + import json + s = json.load(open("hil_select.json")) + a = s["args"]["hfp.json"] + open("hil_sel_args.txt", "w").write(a) + print("true" if (s["full"] or a) else "false") + '); then + echo "::warning::hil_select output unusable - running the full hfp matrix" + rm -f hil_select.json hil_sel_args.txt + exit 0 + fi + echo "SEL_RUN=$SEL_RUN" + echo "SEL_RUN=$SEL_RUN" >> $GITHUB_ENV - name: Get build boards + if: env.SEL_RUN != 'false' run: | - MATRIX_JSON=$(python test/hil/hil_ci_set_matrix.py test/hil/hfp.json) - BUILD_ARGS=$(echo $MATRIX_JSON | jq -r '.["arm-gcc"] | join(" ")') + if [ -f hil_select.json ]; then + MATRIX_JSON=$(python test/hil/hil_ci_set_matrix.py --select "$(cat hil_select.json)" test/hil/hfp.json) + else + MATRIX_JSON=$(python test/hil/hil_ci_set_matrix.py test/hil/hfp.json) + fi + # Each variant carries its own --build-name/--cflag, which are global to a + # single build.py invocation — so keep one matrix entry per line and build + # them one at a time (joining would leak a variant's flags onto every board). + echo "$MATRIX_JSON" | jq -r '.["arm-gcc"][]' > hil_build_entries.txt + cat hil_build_entries.txt + BUILD_ARGS=$(echo "$MATRIX_JSON" | jq -r '.["arm-gcc"] | join(" ")') echo "BUILD_ARGS=$BUILD_ARGS" echo "BUILD_ARGS=$BUILD_ARGS" >> $GITHUB_ENV - name: Get Dependencies + if: env.SEL_RUN != 'false' run: python3 tools/get_deps.py $BUILD_ARGS - name: Build + if: env.SEL_RUN != 'false' run: | - # Each variant carries its own --build-name/--cflag, which are global to a - # single build.py invocation — so build one matrix entry at a time rather - # than joining them (joining would leak a variant's flags onto every board). - readarray -t ENTRIES < <(python test/hil/hil_ci_set_matrix.py test/hil/hfp.json | jq -r '.["arm-gcc"][]') + readarray -t ENTRIES < hil_build_entries.txt for entry in "${ENTRIES[@]}"; do echo "+ tools/build.py --toolchain iar $entry" python3 tools/build.py --toolchain iar $entry @@ -484,7 +657,16 @@ jobs: - name: Test on actual hardware (hardware in the loop) run: | - python3 test/hil/hil_test.py hfp.json + if [ "$SEL_RUN" = "false" ]; then + echo "HIL skipped by PR selection (no affected boards on this rig)" + # leave a marker so the combined PR comment says so instead of dropping + # the section (and leaving a stale table from an earlier push) + echo "_Skipped by PR selection: no affected boards on this rig._" > hil_report.md + exit 0 + fi + # empty/absent on a non-PR event or a selector fallback -> full hfp matrix + SEL_ARGS=$(cat hil_sel_args.txt 2>/dev/null || true) + python3 test/hil/hil_test.py $SEL_ARGS hfp.json - name: Upload HIL report if: always() && github.event_name == 'pull_request' diff --git a/.github/workflows/build_util.yml b/.github/workflows/build_util.yml index 90115862b..02f16488a 100644 --- a/.github/workflows/build_util.yml +++ b/.github/workflows/build_util.yml @@ -39,6 +39,21 @@ on: jobs: family: + # PR-scoped HIL selection can produce an empty build-args list for a toolchain + # (e.g. a dwc2-only change with no riscv boards affected); an empty matrix + # vector fails the job outright ("Matrix vector 'arg' does not contain any + # values"), so skip cleanly instead. + # + # Why that is safe for callers: GitHub SKIPS the dependents of a skipped + # `needs:` job, so this only works because the caller (hil-build) is itself a + # *matrix* job - a matrix with one skipped leg and one successful leg + # aggregates to success, and its dependents run. + # + # NOT covered: if every leg is empty the whole caller job skips, and so does + # everything that needs it. That is fine only because an all-empty selection + # means no board was selected for those rigs, so the rig jobs would have had + # nothing to run anyway (see the invariant on hil-build in build.yml). + if: inputs.build-args != '[]' runs-on: ${{ inputs.os }} strategy: fail-fast: false diff --git a/.github/workflows/pr_comment.yml b/.github/workflows/pr_comment.yml index 4d50817b4..868e56405 100644 --- a/.github/workflows/pr_comment.yml +++ b/.github/workflows/pr_comment.yml @@ -101,7 +101,16 @@ jobs: shopt -s nullglob dirs=(hil-reports/hil-report-*) if [ ${#dirs[@]} -eq 0 ]; then + # No rig produced a report: PR selection matched no board anywhere, or the + # HIL jobs did not run at all. Post it rather than exiting, so a table from + # an earlier push is replaced instead of being left to look current. echo "No HIL reports found" + { + echo "## Hardware-in-the-loop (HIL) Test Report" + echo + echo "_No HIL run for this push (no affected boards, or hardware testing did not run)._" + } > hil_combined.md + echo "found=true" >> "$GITHUB_OUTPUT" exit 0 fi { diff --git a/docs/superpowers/plans/2026-07-29-hil-select.md b/docs/superpowers/plans/2026-07-29-hil-select.md new file mode 100644 index 000000000..a8abf9887 --- /dev/null +++ b/docs/superpowers/plans/2026-07-29-hil-select.md @@ -0,0 +1,856 @@ +# PR-Scoped HIL Selection Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** A diff→(boards, tests) selector (`test/hil/hil_select.py`) that scopes CI's HIL build+test jobs on pull requests and is reusable locally, per `docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md`. + +**Architecture:** Pure-stdlib classification engine (changed files → per-board test selection, fail-open to full) + thin CLI emitting JSON with per-rig `hil_test.py` arg strings; consumed by `hil_ci_set_matrix.py --select` (prunes hil-build) and shell steps in the three HIL jobs (prunes rig runs). Test lists shared via new `hil_examples.py`. + +**Tech Stack:** Python 3.11 stdlib only (`re`, `json`, `glob`, `subprocess` for git), `unittest` for tests, GitHub Actions YAML. + +## Global Constraints + +- Work in worktree `.claude/worktrees/hil-select` (branch `claude/hil-select`); never touch the primary checkout. +- `hil_select.py`, `hil_examples.py`, `test_hil_select.py` import NOTHING outside the stdlib and each other — in particular never `hil_test`/`hil_flash`/`hil_lock` (GitHub's bare runner has no pyserial/pymtp). +- Fail-open: any changed file matching no classification rule ⇒ `full: true`. Scoping applies to `pull_request` events only; push/scheduled runs stay full. +- Behavior-preserving for existing tools: `hil_test.py` runtime behavior unchanged (only its test-list constants move to `hil_examples.py`); `hil_ci_set_matrix.py` without `--select` emits byte-identical output to today. +- The selector only ever emits board names present in the given roster (`config['boards']`); `boards-skip` is invisible to it. +- Commit messages: imperative, scoped, NO Co-Authored-By/Claude-Session trailers. +- Every commit: `python3 -m py_compile` clean on touched python files, `python3 test/hil/test_hil_select.py` green (once it exists), `pre-commit run --files ` clean. + +--- + +### Task 1: hil_examples.py + selection engine with unit tests + +**Files:** +- Create: `test/hil/hil_examples.py` +- Create: `test/hil/hil_select.py` (engine only; CLI comes in Task 2) +- Create: `test/hil/test_hil_select.py` +- Modify: `test/hil/hil_test.py` (import test lists from hil_examples) +- Modify: `test/hil/hil_ci.sh` (scp list gains `hil_examples.py`) + +**Interfaces:** +- Produces `hil_examples.py`: `device_tests: list[str]`, `dual_tests: list[str]`, `host_test: list[str]` — the three lists moved VERBATIM (incl. comments) from `hil_test.py`. +- Produces `hil_select.py` engine API used by Task 2: + - `classify(changed_files: list[str], repo_root: str, rosters: list[tuple[str, list[dict]]]) -> dict` + returning `{'full': bool, 'boards': {board_name: 'all' | sorted list[str]}, 'reasons': list[str]}` + where `rosters` = `[(config_path, config['boards']), ...]`. + - `board_roles(board: dict) -> set[str]` — subset of `{'device', 'host'}` from the roster + entry's `tests` flags (`device`/`host`/`dual` booleans; an `only` list contributes the + roles of its entries' path prefixes; `dual` implies both roles). + - `board_family(board_name: str, repo_root: str) -> str | None` — the `` for which + `hw/bsp//boards/` exists. + - `port_families(port_dir: str, repo_root: str) -> set[str]` — directories of + `hw/bsp/*/family.cmake` and `hw/bsp/*/family.mk` whose text contains `port_dir` + (e.g. `raspberrypi/rp2040`). + - `class_examples(class_dir: str, role: str, repo_root: str) -> set[str]` — tests from + `hil_examples` lists whose example `tusb_config.h` enables the class for that role (regex + `#define\s+CFG_TUD_\s+\(?\s*0*[1-9]` / `CFG_TUH_`; exceptions per spec: + `dfu_rt_device.*`→`CFG_TUD_DFU_RUNTIME`, `dfu_device.*`→`CFG_TUD_DFU`, class dir `net` + → `CFG_TUD_ECM_RNDIS|CFG_TUD_NCM`). Test path `device/x` ⇒ config at + `examples/device/x/src/tusb_config.h`; same pattern for `host/` and `dual/`. + +- [ ] **Step 1: Move the test lists into `hil_examples.py`** + +Create `test/hil/hil_examples.py`: + +```python +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# HIL example test lists, shared by hil_test.py (runner) and hil_select.py +# (PR-diff selector). Stdlib-only: hil_select runs on bare CI runners. +``` + +then MOVE the `device_tests`, `dual_tests`, `host_test` list definitions (and their preceding +comment block "The per-board run order is shuffled...") VERBATIM from `hil_test.py` into it. +In `hil_test.py`, add `from hil_examples import device_tests, dual_tests, host_test` where the +lists were (a `from`-import of data constants is fine here — they are read-only lists used by +name throughout `test_board`). Add `"$ROOT_DIR/test/hil/hil_examples.py" \` to the +`hil_ci.sh` scp list after the `hil_lock.py` line. + +- [ ] **Step 2: Verify the move broke nothing** + +Run: `cd /home/hathach/code/tinyusb/.claude/worktrees/hil-select && python3 -m py_compile test/hil/hil_examples.py test/hil/hil_test.py && python3 test/hil/hil_test.py --help >/dev/null && echo ok` +Expected: `ok` + +- [ ] **Step 3: Write the failing unit tests (spec acceptance cases)** + +Create `test/hil/test_hil_select.py`. ROSTER is a trimmed but real-shaped fixture; tests call +the engine API directly (no git, no CLI): + +```python +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests for hil_select.py — pure logic, no hardware, no git. Run directly: +# python3 test/hil/test_hil_select.py +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import hil_select +from hil_examples import device_tests, dual_tests, host_test + +REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +ROSTER = [ + # device-only, rp2040 family + {'name': 'raspberry_pi_pico', 'uid': 'u1', + 'tests': {'device': True, 'host': True, 'dual': True}}, + # device-only, stm32f4 family + {'name': 'stm32f407disco', 'uid': 'u2', + 'tests': {'device': True, 'host': False, 'dual': False}}, + # host-only board + {'name': 'raspberry_pi_pico2', 'uid': 'u3', + 'tests': {'device': False, 'host': True, 'dual': False}}, + # only-list board (espressif-style) + {'name': 'espressif_s3_devkitm', 'uid': 'u4', + 'tests': {'only': ['device/cdc_msc_freertos', 'host/device_info']}}, +] +ROSTERS = [('test/hil/tinyusb.json', ROSTER)] + + +def sel(files): + return hil_select.classify(files, REPO, ROSTERS) + + +class TestPortRule(unittest.TestCase): + def test_dcd_rp2040_selects_pico_family_only(self): + s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico', s['boards']) + self.assertNotIn('stm32f407disco', s['boards']) + self.assertNotIn('espressif_s3_devkitm', s['boards']) + # device role: no host tests in pico's list + self.assertTrue(all(not t.startswith('host/') for t in s['boards']['raspberry_pi_pico'])) + # host-only boards drop out entirely on a device-role change + self.assertNotIn('raspberry_pi_pico2', s['boards']) + + def test_shared_port_file_is_both_roles(self): + s = sel(['src/portable/synopsys/dwc2/dwc2_common.c']) + self.assertFalse(s['full']) + self.assertNotIn('raspberry_pi_pico', s['boards']) # rp2040 is not a dwc2 family + self.assertIn('stm32f407disco', s['boards']) # stm32f4 is + + +class TestCoreRoleRule(unittest.TestCase): + def test_usbd_selects_all_device_tests_everywhere(self): + s = sel(['src/device/usbd.c']) + self.assertFalse(s['full']) + self.assertNotIn('raspberry_pi_pico2', s['boards']) # host-only board dropped + pico = s['boards']['raspberry_pi_pico'] + self.assertTrue(set(device_tests).issubset(set(pico))) + self.assertTrue(set(dual_tests).issubset(set(pico))) # dual survives device role + self.assertTrue(all(not t.startswith('host/') for t in pico)) + # only-list board: selection intersects its only-list + esp = s['boards']['espressif_s3_devkitm'] + self.assertEqual(esp, ['device/cdc_msc_freertos']) + + def test_host_change_drops_device(self): + s = sel(['src/host/usbh.c']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico2', s['boards']) + self.assertNotIn('stm32f407disco', s['boards']) # device-only board dropped + + +class TestClassRule(unittest.TestCase): + def test_cdc_device_selects_cdc_examples_only(self): + s = sel(['src/class/cdc/cdc_device.c']) + self.assertFalse(s['full']) + pico = s['boards']['raspberry_pi_pico'] + self.assertIn('device/cdc_msc', pico) + self.assertIn('device/cdc_dual_ports', pico) + self.assertNotIn('device/msc_dual_lun', pico) # CFG_TUD_CDC 0 there + self.assertNotIn('device/usbtest', pico) # CFG_TUD_CDC 0 there + self.assertTrue(all(not t.startswith('host/') for t in pico)) + + def test_msc_host_selects_host_side(self): + s = sel(['src/class/msc/msc_host.c']) + self.assertFalse(s['full']) + self.assertNotIn('stm32f407disco', s['boards']) # device-only board + pico2 = s['boards']['raspberry_pi_pico2'] + self.assertIn('host/msc_file_explorer', pico2) + self.assertTrue(all(not t.startswith('device/') for t in pico2)) + + +class TestFallbackRules(unittest.TestCase): + def test_unknown_tool_is_full(self): + s = sel(['tools/random_new_script.py']) + self.assertTrue(s['full']) + + def test_docs_only_is_empty_not_full(self): + s = sel(['docs/info/contributing.rst', 'README.rst']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + def test_bsp_family_selects_family_boards(self): + s = sel(['hw/bsp/rp2040/family.cmake']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico', s['boards']) + self.assertEqual(s['boards']['raspberry_pi_pico'], 'all') + self.assertNotIn('stm32f407disco', s['boards']) + + def test_bsp_board_narrows_to_board(self): + s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) + self.assertFalse(s['full']) + self.assertEqual(list(s['boards'].keys()), ['raspberry_pi_pico']) + + def test_example_change_selects_that_example(self): + s = sel(['examples/device/cdc_msc/src/main.c']) + self.assertFalse(s['full']) + self.assertEqual(s['boards']['raspberry_pi_pico'], ['device/cdc_msc']) + + def test_core_common_is_full(self): + for f in ['src/tusb.c', 'src/common/tusb_fifo.c', 'src/osal/osal_freertos.h']: + self.assertTrue(sel([f])['full'], f) + + def test_harness_is_full(self): + for f in ['test/hil/hil_test.py', '.github/workflows/build.yml', 'hw/mcu/nxp/x.c', 'lib/foo/x.c']: + self.assertTrue(sel([f])['full'], f) + + def test_mixed_roles_no_pruning(self): + s = sel(['src/device/usbd.c', 'src/host/usbh.c']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico2', s['boards']) + self.assertIn('stm32f407disco', s['boards']) + + +if __name__ == '__main__': + unittest.main(verbosity=1) +``` + +- [ ] **Step 4: Run tests to verify they fail** + +Run: `python3 test/hil/test_hil_select.py 2>&1 | tail -2` +Expected: `ModuleNotFoundError: No module named 'hil_select'` (or import error). + +- [ ] **Step 5: Implement the engine** + +Create `test/hil/hil_select.py`: + +```python +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""PR-diff -> HIL selection: which rig boards and which tests a change can affect. + +Stdlib-only (runs on bare CI runners; never imports hil_test/hil_flash/hil_lock). +Fail-open: any file no rule classifies forces the full matrix. See +docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md. +""" +import argparse +import glob +import json +import os +import re +import subprocess +import sys + +from hil_examples import device_tests, dual_tests, host_test + +ALL_TESTS = {'device': device_tests, 'dual': dual_tests, 'host': host_test} + +# class dir -> config macro suffix exceptions (rule 3); dfu is per-file, handled inline +NET_MACROS = ('ECM_RNDIS', 'NCM') + +_NONCODE_RE = re.compile( + r'^(docs/|\.claude/|.*\.(md|rst|txt)$|LICENSE)') +_FULL_RE = re.compile( + r'^(src/common/|src/osal/|src/tusb\.c$|src/tusb\.h$|src/tusb_option\.h$|' + r'test/hil/|\.github/workflows/build.*\.yml$|\.github/actions/|' + r'tools/build\.py$|tools/get_deps\.py$|tools/cmake/|hw/mcu/|lib/|' + r'hw/bsp/(family_support\.cmake|board_api\.h|board\.c|ansi_escape\.h)$|' + r'examples/build_system/|examples/CMakeLists\.txt$)') + + +def test_role(test: str) -> str: + return test.split('/', 1)[0] # 'device' | 'dual' | 'host' + + +def board_roles(board: dict) -> set: + t = board.get('tests', {}) + roles = set() + if t.get('device'): + roles.add('device') + if t.get('host'): + roles.add('host') + if t.get('dual'): + roles.update(('device', 'host')) + for only in t.get('only', []): + r = test_role(only) + roles.update(('device', 'host') if r == 'dual' else (r,)) + return roles + + +def board_tests(board: dict) -> list: + """Every test this board would run today (mirrors hil_test.test_board's default).""" + t = board.get('tests', {}) + if 'only' in t: + run = list(t['only']) + else: + run = [] + if t.get('device'): + run += device_tests + if t.get('dual'): + run += dual_tests + if t.get('host'): + run += host_test + return [x for x in run if x not in t.get('skip', [])] + + +def board_family(board_name: str, repo_root: str): + hits = glob.glob(os.path.join(repo_root, 'hw/bsp/*/boards', board_name)) + return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None + + +def port_families(port_dir: str, repo_root: str) -> set: + fams = set() + for f in glob.glob(os.path.join(repo_root, 'hw/bsp/*/family.cmake')) + \ + glob.glob(os.path.join(repo_root, 'hw/bsp/*/family.mk')): + try: + if port_dir in open(f).read(): + fams.add(os.path.basename(os.path.dirname(f))) + except OSError: + pass + return fams + + +def _config_enables(cfg_path: str, macros) -> bool: + try: + text = open(cfg_path).read() + except OSError: + return False + return any(re.search(rf'#define\s+{m}\s+\(?\s*0*[1-9]', text) for m in macros) + + +def class_examples(macros, role: str, repo_root: str) -> set: + """Tests (from role's + dual lists) whose example config enables any macro.""" + pools = {'device': device_tests + dual_tests, 'host': host_test + dual_tests} + out = set() + for test in pools[role]: + cfg = os.path.join(repo_root, 'examples', test, 'src', 'tusb_config.h') + if _config_enables(cfg, macros): + out.add(test) + return out + + +class _Sel: + """Accumulates contributions. board->set(tests) plus 'all-board' markers.""" + def __init__(self): + self.full = False + self.by_board = {} # name -> set of tests, or 'all' + self.roles = set() # roles touched by any contribution + self.reasons = [] + + def add(self, boards, tests, reason): + """tests: 'all' or iterable of test paths.""" + self.reasons.append(reason) + for b in boards: + cur = self.by_board.get(b) + if tests == 'all' or cur == 'all': + self.by_board[b] = 'all' + else: + self.by_board[b] = (cur or set()) | set(tests) + + def force_full(self, reason): + self.full = True + self.reasons.append(reason) + + +def _classify_one(path, repo_root, roster_boards, s: _Sel): + base = os.path.basename(path) + if _NONCODE_RE.match(path): + s.reasons.append(f'{path}: non-code, no contribution') + return + if _FULL_RE.match(path): + s.force_full(f'{path}: core/infra -> full matrix') + return + + m = re.match(r'src/portable/((?:[^/]+/)?[^/]+)/', path) + if m: + port = m.group(1) + if re.match(r'(dcd_|.*_device)', base): + roles = {'device'} + elif re.match(r'(hcd_|.*_host)', base): + roles = {'host'} + else: + roles = {'device', 'host'} + fams = port_families(port, repo_root) + boards = [b['name'] for b in roster_boards + if board_family(b['name'], repo_root) in fams and (board_roles(b) & roles)] + tests = [t for r in roles for t in ALL_TESTS[r]] + dual_tests + s.roles.update(roles) + s.add(boards, tests, f'{path}: port {port} -> families {sorted(fams)} -> boards {boards} ({"/".join(sorted(roles))})') + return + + m = re.match(r'src/class/([^/]+)/', path) + if m: + cls = m.group(1) + if re.search(r'_device\.[ch]$', base): + roles = {'device'} + elif re.search(r'_host\.[ch]$', base): + roles = {'host'} + else: + roles = {'device', 'host'} + # macro names per role + def macros(prefix): + if cls == 'net': + return [f'CFG_{prefix}_{m2}' for m2 in NET_MACROS] + if cls == 'dfu': + if base.startswith('dfu_rt'): + return [f'CFG_{prefix}_DFU_RUNTIME'] + if base.startswith('dfu_device') or base.startswith('dfu_host'): + return [f'CFG_{prefix}_DFU'] + return [f'CFG_{prefix}_DFU', f'CFG_{prefix}_DFU_RUNTIME'] + return [f'CFG_{prefix}_{cls.upper()}'] + tests = set() + if 'device' in roles: + tests |= class_examples(macros('TUD'), 'device', repo_root) + if 'host' in roles: + tests |= class_examples(macros('TUH'), 'host', repo_root) + boards = [b['name'] for b in roster_boards if board_roles(b) & roles] + s.roles.update(roles) + s.add(boards, tests, f'{path}: class {cls} -> {sorted(tests)} ({"/".join(sorted(roles))})') + return + + m = re.match(r'src/(device|host)/', path) + if m: + role = m.group(1) + boards = [b['name'] for b in roster_boards if role in board_roles(b)] + s.roles.add(role) + s.add(boards, ALL_TESTS[role] + dual_tests, f'{path}: core {role} stack -> all {role} tests') + return + + m = re.match(r'hw/bsp/([^/]+)/(?:boards/([^/]+)/)?', path) + if m: + fam, brd = m.group(1), m.group(2) + if brd: + boards = [b['name'] for b in roster_boards if b['name'] == brd] + why = f'{path}: bsp board {brd}' + else: + boards = [b['name'] for b in roster_boards + if board_family(b['name'], repo_root) == fam] + why = f'{path}: bsp family {fam}' + s.roles.update(('device', 'host')) + s.add(boards, 'all', f'{why} -> boards {boards}') + return + + m = re.match(r'examples/(device|host|dual)/([^/]+)/', path) + if m: + test = f'{m.group(1)}/{m.group(2)}' + known = any(test in pool for pool in ALL_TESTS.values()) + if known: + boards = [b['name'] for b in roster_boards] + role = test_role(test) + s.roles.update(('device', 'host') if role == 'dual' else (role,)) + s.add(boards, [test], f'{path}: example -> {test} on all boards') + else: + s.reasons.append(f'{path}: example not in HIL lists, no contribution') + return + + s.force_full(f'{path}: unclassified -> full matrix') + + +def classify(changed_files, repo_root, rosters): + all_boards = [] + seen = set() + for _, boards in rosters: + for b in boards: + if b['name'] not in seen: + seen.add(b['name']) + all_boards.append(b) + + s = _Sel() + for path in changed_files: + _classify_one(path, repo_root, all_boards, s) + if s.full: + break + + if s.full: + return {'full': True, 'boards': {b['name']: 'all' for b in all_boards}, + 'reasons': s.reasons} + + # role pruning: single-role selections drop the other role's tests and boards + by_name = {b['name']: b for b in all_boards} + out = {} + for name, tests in s.by_board.items(): + allowed = board_tests(by_name[name]) + if tests == 'all': + kept = list(allowed) + else: + kept = [t for t in allowed if t in tests] + if s.roles and s.roles != {'device', 'host'}: + role = next(iter(s.roles)) + kept = [t for t in kept if test_role(t) in (role, 'dual')] + if kept: + out[name] = 'all' if set(kept) == set(allowed) else sorted(kept) + return {'full': False, 'boards': out, 'reasons': s.reasons} +``` + +- [ ] **Step 6: Run tests to verify they pass** + +Run: `python3 test/hil/test_hil_select.py` +Expected: all tests PASS (OK line). Iterate on the engine (not the tests) until green; if a +test premise contradicts the repo (e.g. a family name), verify against the tree and fix the +test only with evidence noted in your report. + +- [ ] **Step 7: Commit** + +```bash +git add test/hil/hil_examples.py test/hil/hil_select.py test/hil/test_hil_select.py test/hil/hil_test.py test/hil/hil_ci.sh +git commit -m "hil: add PR-diff selection engine (hil_select) with shared example lists" +``` + +--- + +### Task 2: CLI + args emission + +**Files:** +- Modify: `test/hil/hil_select.py` (add `selection_args`, `main`) +- Modify: `test/hil/test_hil_select.py` (add CLI/args tests) + +**Interfaces:** +- Consumes: Task 1's `classify` and roster shapes. +- Produces: + - `selection_args(sel: dict, rosters) -> dict` mapping each config path's basename to the + `hil_test.py` argument string for that rig: for each selected board ON that roster, + `-b `, plus `-bt :,` when the board's entry is a list (not 'all'). + Empty string when no selected board is on that roster. When `sel['full']`, every roster + board gets bare `-b`? NO — full means "today's behavior": `selection_args` returns `''` + for every config (no filtering args at all). + - CLI: `python3 test/hil/hil_select.py [--base REF | --diff-file PATH] CONFIG...` printing + the JSON `{'full', 'boards', 'args', 'reasons'}` to stdout, reasons also to stderr + (one line each, prefixed `hil_select: `). Non-zero exit only on operational errors + (bad ref, unreadable config) — never on an empty selection. + +- [ ] **Step 1: Add failing CLI/args tests to `test_hil_select.py`** + +```python +class TestArgsEmission(unittest.TestCase): + def test_args_for_scoped_selection(self): + s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + args = hil_select.selection_args(s, ROSTERS) + a = args['tinyusb.json'] + self.assertIn('-b raspberry_pi_pico', a) + self.assertNotIn('stm32f407disco', a) + self.assertIn('-bt raspberry_pi_pico:', a) # device-only subset of a device+host board + + def test_args_full_is_empty(self): + s = sel(['tools/random_new_script.py']) + self.assertEqual(hil_select.selection_args(s, ROSTERS), {'tinyusb.json': ''}) + + def test_args_all_board_gets_bare_b(self): + s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) + a = hil_select.selection_args(s, ROSTERS)['tinyusb.json'] + self.assertIn('-b raspberry_pi_pico', a) + self.assertNotIn('-bt', a) + + def test_cli_diff_file(self): + import subprocess, tempfile, json as j + with tempfile.NamedTemporaryFile('w', suffix='.txt', delete=False) as f: + f.write('src/class/cdc/cdc_device.c\n') + path = f.name + r = subprocess.run([sys.executable, os.path.join(REPO, 'test/hil/hil_select.py'), + '--diff-file', path, os.path.join(REPO, 'test/hil/tinyusb.json')], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + out = j.loads(r.stdout) + self.assertFalse(out['full']) + self.assertIn('tinyusb.json', out['args']) + self.assertTrue(any('cdc_device' in line for line in out['reasons'])) + os.unlink(path) +``` + +- [ ] **Step 2: Run to verify the new tests fail** + +Run: `python3 test/hil/test_hil_select.py 2>&1 | tail -3` +Expected: failures/errors mentioning `selection_args`. + +- [ ] **Step 3: Implement `selection_args` and `main`** + +Append to `hil_select.py`: + +```python +def selection_args(sel, rosters): + args = {} + for cfg_path, boards in rosters: + key = os.path.basename(cfg_path) + if sel['full']: + args[key] = '' + continue + parts = [] + for b in boards: + chosen = sel['boards'].get(b['name']) + if chosen is None: + continue + parts.append(f'-b {b["name"]}') + if chosen != 'all': + parts.append(f'-bt {b["name"]}:{",".join(chosen)}') + args[key] = ' '.join(parts) + return args + + +def changed_files_from_git(base, repo_root): + mb = subprocess.run(['git', 'merge-base', 'HEAD', base], cwd=repo_root, + capture_output=True, text=True, check=True).stdout.strip() + diff = subprocess.run(['git', 'diff', '--name-only', f'{mb}..HEAD'], cwd=repo_root, + capture_output=True, text=True, check=True).stdout + return [l for l in diff.splitlines() if l.strip()] + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + g = ap.add_mutually_exclusive_group(required=True) + g.add_argument('--base', help='git ref to diff against (merge-base..HEAD)') + g.add_argument('--diff-file', help='newline-separated changed-file list') + ap.add_argument('configs', nargs='+', help='rig roster JSON file(s)') + a = ap.parse_args() + + repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + rosters = [] + for c in a.configs: + with open(c) as f: + rosters.append((c, json.load(f)['boards'])) + + files = (open(a.diff_file).read().splitlines() if a.diff_file + else changed_files_from_git(a.base, repo_root)) + files = [f for f in files if f.strip()] + + s = classify(files, repo_root, rosters) + s['args'] = selection_args(s, rosters) + for r in s['reasons']: + print(f'hil_select: {r}', file=sys.stderr) + print(json.dumps(s)) + + +if __name__ == '__main__': + main() +``` + +(The `parts.append f'...'` line above is pseudo-highlighted; write valid Python: +`parts.append(f'-bt {b["name"]}:{",".join(chosen)}')`.) + +- [ ] **Step 4: Run the full suite** + +Run: `python3 test/hil/test_hil_select.py && chmod +x test/hil/hil_select.py` +Expected: OK. + +- [ ] **Step 5: Smoke against the real repo state** + +Run: `python3 test/hil/hil_select.py --base HEAD test/hil/tinyusb.json test/hil/hfp.json` +Expected: empty diff ⇒ `{"full": false, "boards": {}, "args": {"tinyusb.json": "", "hfp.json": ""}, ...}` exit 0. +Then: `printf 'src/portable/wch/dcd_ch32_usbfs.c\n' > /tmp/d.txt && python3 test/hil/hil_select.py --diff-file /tmp/d.txt test/hil/tinyusb.json | python3 -m json.tool | head -20` +Expected: only WCH-family boards (nanoch32v203, ch32v103r_r1_1v0, ch32v307v_r1_1v0 — whichever reference that port) with device tests. + +- [ ] **Step 6: Commit** + +```bash +git add test/hil/hil_select.py test/hil/test_hil_select.py +git commit -m "hil: hil_select CLI with per-rig hil_test argument emission" +``` + +--- + +### Task 3: hil_ci_set_matrix --select + build.yml wiring + +**Files:** +- Modify: `test/hil/hil_ci_set_matrix.py` +- Modify: `.github/workflows/build.yml` (set-matrix job; hil-build consumers unchanged; hil-tinyusb + hil-tinyusb-esp steps) + +**Interfaces:** +- Consumes: Task 2's CLI JSON (`full`, `boards`, `args`). +- Produces: + - `hil_ci_set_matrix.py [--select JSON_STRING] CONFIG...`: with `--select` and + `full == false`, boards not in `select['boards']` are skipped when building the toolchain + buckets; otherwise identical behavior. Buckets stay present (possibly `[]`) so + `fromJSON(...)[toolchain]` keeps resolving. + - set-matrix outputs: `hil_select_json` (compact selection), `hil_args_tinyusb`, + `hil_args_hfp`, `hil_run_tinyusb`, `hil_run_hfp` (string 'true'/'false'). + +- [ ] **Step 1: Add `--select` to `hil_ci_set_matrix.py`** + +In `main()` add: + +```python + parser.add_argument('--select', help='hil_select.py JSON; scopes boards when full=false') +``` + +and after parsing: + +```python + selected = None + sel = json.loads(args.select) if args.select else None + if sel and not sel.get('full'): + selected = set(sel.get('boards', {})) +``` + +then inside the per-board loop, first line: + +```python + if selected is not None and board['name'] not in selected: + continue +``` + +- [ ] **Step 2: Verify byte-identical without --select and scoped with it** + +Run: `python3 test/hil/hil_ci_set_matrix.py test/hil/tinyusb.json test/hil/hfp.json > /tmp/m1.json && git stash -q && python3 test/hil/hil_ci_set_matrix.py test/hil/tinyusb.json test/hil/hfp.json > /tmp/m0.json && git stash pop -q && diff /tmp/m0.json /tmp/m1.json && echo identical` +Expected: `identical`. +Then: `python3 test/hil/hil_ci_set_matrix.py --select '{"full": false, "boards": {"raspberry_pi_pico": "all"}}' test/hil/tinyusb.json test/hil/hfp.json` +Expected: JSON whose `arm-gcc` list contains only the raspberry_pi_pico entry, `riscv-gcc`/`esp-idf` = []. + +- [ ] **Step 3: Wire set-matrix in `.github/workflows/build.yml`** + +In the `set-matrix` job: give the checkout full history and add the selection step between +checkout and matrix generation; make the HIL matrix use it: + +```yaml + - name: Checkout TinyUSB + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: HIL selection (PR only) + id: hil-select + if: github.event_name == 'pull_request' + run: | + python3 test/hil/test_hil_select.py + SELECT_JSON=$(python3 test/hil/hil_select.py --base "origin/${{ github.base_ref }}" test/hil/tinyusb.json test/hil/hfp.json) + echo "select=$SELECT_JSON" >> $GITHUB_OUTPUT + python3 - "$SELECT_JSON" >> $GITHUB_OUTPUT <<'EOF' + import json, sys + s = json.loads(sys.argv[1]) + args = s.get('args', {}) + for cfg, key in (('tinyusb.json', 'tinyusb'), ('hfp.json', 'hfp')): + a = args.get(cfg, '') + run = 'true' if (s['full'] or a) else 'false' + print(f'args_{key}={a}') + print(f'run_{key}={run}') + EOF +``` + +and in the existing "Generate matrix json" step, change the HIL line to: + +```yaml + # HIL matrix (merged from tinyusb + hifiphile configs), scoped on PRs + SELECT='${{ steps.hil-select.outputs.select }}' + HIL_MATRIX_JSON=$(python test/hil/hil_ci_set_matrix.py ${SELECT:+--select "$SELECT"} test/hil/tinyusb.json test/hil/hfp.json) +``` + +Add to the job's `outputs:` block: + +```yaml + hil_args_tinyusb: ${{ steps.hil-select.outputs.args_tinyusb }} + hil_args_hfp: ${{ steps.hil-select.outputs.args_hfp }} + hil_run_tinyusb: ${{ steps.hil-select.outputs.run_tinyusb }} + hil_run_hfp: ${{ steps.hil-select.outputs.run_hfp }} +``` + +(On non-PR events the step is skipped: outputs are empty strings — the consumers below treat +empty `run_*` as 'true' and empty args as no filtering, i.e. today's behavior.) + +- [ ] **Step 4: Wire the rig jobs** + +In the `hil-tinyusb` job (the matrixed one covering both rigs), find the step that runs +`hil_test.py --retry 1 ${{ matrix.test_args }} ${{ env.HIL_JSON }} $RERUN_ARGS` (~line 360) +and change the step's `run:` to select per-rig args and honor the skip flag: + +```yaml + run: | + case "$HIL_JSON" in + *tinyusb.json) SEL_ARGS='${{ needs.set-matrix.outputs.hil_args_tinyusb }}'; SEL_RUN='${{ needs.set-matrix.outputs.hil_run_tinyusb }}' ;; + *hfp.json) SEL_ARGS='${{ needs.set-matrix.outputs.hil_args_hfp }}'; SEL_RUN='${{ needs.set-matrix.outputs.hil_run_hfp }}' ;; + esac + if [ "$SEL_RUN" = "false" ]; then echo "HIL skipped by PR selection (no affected boards on this rig)"; exit 0; fi + python3 test/hil/hil_test.py --retry 1 ${{ matrix.test_args }} $SEL_ARGS ${{ env.HIL_JSON }} $RERUN_ARGS +``` + +Apply the same pattern to the second `hil_test.py` invocation at ~line 423 (`hil-tinyusb-esp`, +which is tinyusb-rig only: use the `hil_args_tinyusb`/`hil_run_tinyusb` outputs directly, no +case needed) and to the hfp job's direct `python3 test/hil/hil_test.py hfp.json` call at +~line 487 (use `hil_args_hfp`/`hil_run_hfp`). Preserve each step's existing surrounding lines +(report-dir env, RERUN_ARGS logic) — only inject the SEL_ARGS/SEL_RUN mechanics. + +- [ ] **Step 5: Validate the YAML and the exact shell locally** + +Run: `pre-commit run check-yaml --files .github/workflows/build.yml && python3 -c "import yaml; yaml.safe_load(open('.github/workflows/build.yml')); print('yaml ok')"` +Expected: `yaml ok` (pyyaml is available; if not, `pip install --user pyyaml` first). +Also simulate the selection step's python inline script: +`SELECT_JSON=$(python3 test/hil/hil_select.py --diff-file /tmp/d.txt test/hil/tinyusb.json test/hil/hfp.json) && python3 -c "import json,sys; s=json.loads(sys.argv[1]); print(s['args'])" "$SELECT_JSON"` +Expected: the args dict prints. + +- [ ] **Step 6: Commit** + +```bash +git add test/hil/hil_ci_set_matrix.py .github/workflows/build.yml +git commit -m "ci: scope HIL build+test matrix by PR diff via hil_select" +``` + +--- + +### Task 4: pre-pr + hil skill docs, final validation + +**Files:** +- Modify: `.claude/skills/pre-pr/SKILL.md` (mapping section delegates to the selector) +- Modify: `.claude/skills/hil/SKILL.md` (document the selector for manual runs) + +**Interfaces:** +- Consumes: Task 2's CLI. + +- [ ] **Step 1: Rewrite pre-pr's "2. Map changes to boards" section** + +Replace the section's grep heuristics (keep its numbered-section structure and the roster/cap +policy) with: + +```markdown +## 2. Map changes to boards + +- `python3 test/hil/hil_select.py --base $BASE test/hil/tinyusb.json` → JSON with the affected + rig boards (`boards`) and per-file `reasons`. `full: true` means a broad/infra change. +- Build-board sampling: from the selection's boards (or, when `full`, the representative set + `stm32f407disco` + `raspberry_pi_pico`), pick ONE board per family, preferring rig-roster + boards; cap at 4 and tell the user which families the cap dropped. The boards list must + NEVER end up empty — final fallback is `[stm32f407disco]`. +- A `full: true` selection or an empty one (docs-only) keeps today's behavior: minimal + software-only gate for docs-only, representative set otherwise. +``` + +- [ ] **Step 2: Add a short "PR-scoped selection" note to the hil skill** + +Append to `.claude/skills/hil/SKILL.md` after the pool-check section: + +```markdown +## PR-scoped selection + +`test/hil/hil_select.py` maps a diff to affected boards/tests (used by CI on PRs; fail-open +to the full matrix). Manual use: + +```bash +ARGS=$(python3 test/hil/hil_select.py --base master test/hil/tinyusb.json | python3 -c "import json,sys; print(json.load(sys.stdin)['args']['tinyusb.json'])") +python3 test/hil/hil_test.py -B examples $ARGS test/hil/tinyusb.json +``` + +Unit suite: `python3 test/hil/test_hil_select.py` (no hardware). +``` + +- [ ] **Step 3: Full validation sweep** + +Run: `python3 test/hil/test_hil_select.py && python3 -m py_compile test/hil/hil_select.py test/hil/hil_examples.py test/hil/hil_ci_set_matrix.py test/hil/hil_test.py && python3 test/hil/hil_test.py --help >/dev/null && pre-commit run --files $(git diff --name-only claude/hil-pool-check..HEAD) && echo ALL-GREEN` +Expected: `ALL-GREEN`. + +- [ ] **Step 4: Real-diff spot checks (acceptance)** + +Run each and eyeball the JSON (record outputs in your report): +```bash +for f in 'src/portable/raspberrypi/rp2040/dcd_rp2040.c' 'src/device/usbd.c' 'src/class/cdc/cdc_device.c' 'src/host/usbh.c'; do + printf '%s\n' "$f" > /tmp/d.txt + echo "=== $f"; python3 test/hil/hil_select.py --diff-file /tmp/d.txt test/hil/tinyusb.json test/hil/hfp.json 2>/dev/null | python3 -m json.tool | sed -n '1,25p' +done +``` +Expected: matches the spec's acceptance examples (pico-family only / all-device / CDC examples +only / host side only, with hfp.json args populated only where hfp boards qualify). + +- [ ] **Step 5: Commit** + +```bash +git add .claude/skills/pre-pr/SKILL.md .claude/skills/hil/SKILL.md +git commit -m "docs: pre-pr and hil skill use hil_select for PR-scoped boards" +``` diff --git a/docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md b/docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md new file mode 100644 index 000000000..8158758bc --- /dev/null +++ b/docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md @@ -0,0 +1,179 @@ +# PR-scoped HIL selection: hil_select.py + +**Date:** 2026-07-29 +**Branch:** `claude/hil-select` (based on `claude/hil-pool-check`, which carries the +hil_lock/hil_flash split and the current rig rosters) + +## Motivation + +Every PR currently builds and runs the full HIL matrix (both rigs, every roster board, every +test). Most PRs touch one port or one class: a `dcd_rp2040` change cannot affect an STM32 board, +a `cdc_device.c` change cannot affect an MSC-only example, and a device-stack change cannot +affect host tests. Scoping HIL to the affected boards/tests cuts CI wall time and rig wear +without losing relevant coverage. + +## Goal / non-goals + +**Goal:** a shared selector that maps a PR diff to (boards, per-board test lists), wired into +CI's `set-matrix` on `pull_request` events (pruning both `hil-build` and the rig jobs) and +callable locally (pre-pr, manual runs). Scoping may only shrink coverage when the mapping is +confident; every uncertainty widens to the full matrix. + +**Non-goals:** +- Variant-level selection (all variants of a selected board run). +- Scoping the non-HIL build jobs (cmake/CircleCI one-per-family builds are independent build + coverage and stay untouched). +- Scoping push/master/scheduled runs (always full). +- Changing hil_test.py behavior (the selector only *composes* existing `-b`/`-bt` args). + +## Component: `test/hil/hil_select.py` + +Stdlib-only, importable and CLI. Lives beside the harness so `hil_ci.sh` copies are unaffected +(it runs on the GitHub runner / dev PC, not on the rig). It must NOT import `hil_test.py` +(which drags pyserial/pymtp onto the bare GitHub runner): the three test lists +(`device_tests`, `dual_tests`, `host_test`) move verbatim into a tiny stdlib-only +`test/hil/hil_examples.py` that both `hil_test.py` and `hil_select.py` import (behavior +preserving; `hil_ci.sh` scp list gains the new file). + +``` +python3 test/hil/hil_select.py --base [--diff-file ] CONFIG.json [CONFIG.json...] +``` + +- `--base REF`: changed files = `git diff --name-only $(git merge-base HEAD REF)..HEAD` + (mirrors pre-pr). `--diff-file`: newline-separated file list instead of git (unit tests, CI + reuse of a precomputed diff). +- Output (stdout, JSON): + +```json +{ + "full": false, + "boards": {"raspberry_pi_pico": "all", "stm32f407disco": ["device/cdc_msc", "device/cdc_dual_ports"]}, + "args": {"tinyusb.json": "-b raspberry_pi_pico -b stm32f407disco -bt stm32f407disco:device/cdc_msc,device/cdc_dual_ports", + "hfp.json": ""}, + "reasons": ["src/portable/raspberrypi/rp2040/dcd_rp2040.c: port rp2040 -> family rp2040 -> boards [raspberry_pi_pico, ...] (device role)"] +} +``` + +- `full: true` ⇒ `boards`/`args` cover the entire rosters (identical to today's behavior). +- `args` maps each input config file to the hil_test.py argument string for that rig: `-b` per + selected board on that roster, plus `-bt BOARD:t1,t2` for boards with a restricted test list + ("all" boards get bare `-b`). An empty string means: nothing on this rig is affected — the + rig job is skipped for this PR. +- Per-file reasoning lines (`file → rule → contribution`) go in `reasons` and to stderr, so the + CI log answers "why did/didn't HIL run X" without archaeology. + +## Classification rules + +Each changed file yields a contribution; the selection is the union. Any file matching no rule +sets `full: true` (fail-open). Rules, first match wins: + +1. **Non-code:** `docs/**`, `.claude/**` (except the workflows below via rule 8), `*.md`, + `*.rst`, `LICENSE*` → contributes nothing. +2. **Port:** `src/portable///**` (or single-level `src/portable//**`). + Role from basename: `dcd_*`/`*_device*` → device; `hcd_*`/`*_host*` → host; anything else + (shared port files, e.g. `dwc2/dwc2_common.c`) → both. Families = directories of + `hw/bsp/*/family.cmake|family.mk` whose text references `/` (pre-pr's grep), + boards = those families' entries on the input rosters. Tests = all tests of that role + (device_tests / host_test from hil_test.py's lists; dual_tests count as both roles). +3. **Class:** `src/class//*_device.*` → all device-capable roster boards; tests = the + device/dual examples in hil_test.py's lists whose `examples///src/tusb_config.h` + defines `CFG_TUD_` with a nonzero value (derived at runtime; `` = upper-cased class + dir, with the map `musb→n/a`-style exceptions NOT needed — class dirs and config macros + share names: cdc, msc, hid, midi, audio, video, vendor, usbtmc, mtp, printer. Two + exceptions: in class dir `dfu`, `dfu_rt_device.*` maps to CFG_TUD_DFU_RUNTIME and + `dfu_device.*` to CFG_TUD_DFU; class dir `net` maps to CFG_TUD_ECM_RNDIS|CFG_TUD_NCM.) `*_host.*` analogously via `CFG_TUH_`. Shared class files (e.g. `cdc.h`) → + both roles' matching examples. A class with zero matching examples contributes nothing + (known path, does not force full). +4. **Core role:** `src/device/**` → all device-capable boards, all device tests (+dual); + `src/host/**` → all host-capable boards, all host tests (+dual). +5. **Core common:** `src/common/**`, `src/osal/**`, `src/tusb.c`, `src/tusb.h`, + `src/tusb_option.h` → full. +6. **BSP:** `hw/bsp//**` → that family's roster boards, all their tests; + `hw/bsp//boards//**` narrows to that board if it is on a roster, and + contributes nothing when it is not (an off-rig board cannot be HIL-tested; known path, + does not force full). + Family-agnostic BSP files (`hw/bsp/board_api.h`, `hw/bsp/board.c`, ansi_escape.h) → full. +7. **Example:** `examples///**` → all roster boards, tests = that example if present + in hil_test.py's lists, else contributes nothing. `examples/build_system/**`, top-level + `examples/CMakeLists.txt` → full. `examples/device/board_test/**` → full: it is the park + firmware hil_test.py flashes on every board (variant boundary + teardown), not a test. +8. **Harness/infra:** `test/hil/**`, `.github/workflows/build*.yml`, + `.github/actions/**`, `tools/build.py`, `tools/get_deps.py`, `tools/cmake/**`, + `hw/mcu/**`, `lib/**` → full. +9. **Everything else** (`test/unit-test/**`, `tools/**` not above, unknown paths) → full. + (Unit-test-only changes could safely skip HIL, but per the fail-open stance anything not + explicitly classified widens; narrowing rule 9 is a later refinement.) + +**Role pruning:** after the union, if only device-role contributions exist, host-only boards +drop out and host tests are stripped from mixed boards (vice versa for host-only changes). +Dual tests survive either role. Board capability (device/host) comes from the roster entry's +`tests` flags/only-list, same logic hil_test.py uses. + +**No-rig-coverage case:** a cleanly classified change whose boards intersect a roster to the +empty set yields an empty `args` string for that rig and a stderr line saying so — the rig job +is skipped, not widened (running unrelated boards would test nothing relevant). + +**Roster source:** `config['boards']` only (boards-skip stays parked). + +## CI wiring (`.github/workflows/build.yml`) + +- `set-matrix` (PR events only): after generating today's matrices, run + `hil_select.py --base origin/${{ github.base_ref }} test/hil/tinyusb.json test/hil/hfp.json` + (checkout with enough history to reach the merge base: `fetch-depth: 0` on this one job, or + an explicit `git fetch origin $BASE_REF`). New job outputs: `hil_select_full`, + `hil_args_tinyusb`, `hil_args_hfp`, plus the selected-board list consumed by the matrix + generator. Non-PR events: skip the selector, outputs default to full/empty-args-means-all. +- `hil_ci_set_matrix.py` gains `--select ''`: when given and `full` is false, it emits + build entries only for selected boards (per config). Untouched otherwise. +- `hil-tinyusb` job (one matrixed job covering both rigs, selected by `matrix.hil_json`): a + step picks the rig's selector args in shell (`case "$HIL_JSON" in ...`) from the set-matrix + outputs and either appends them to the `hil_test.py` invocation or exits the step early with + a "HIL skipped by selection" log line when that rig has nothing to run (`run` flag output + false). The separate `hil-tinyusb-esp` job (esptool split) gets the same treatment with the + tinyusb args. Non-PR events: outputs default to run=true with empty args (today's behavior). +- The `--flasher`/`--exclude-flasher` split in the existing matrix `test_args` composes fine + with `-b` (hil_test.py applies both filters). + +## Local use + +- pre-pr's "Map changes to boards" step delegates to + `python3 test/hil/hil_select.py --base $BASE test/hil/tinyusb.json` and derives its + one-board-per-family sample from the selector's board set (its capping/sampling policy is + unchanged — the selector provides the affected set, pre-pr samples it). +- Manual: `python3 test/hil/hil_test.py -B examples $(python3 test/hil/hil_select.py --base master test/hil/tinyusb.json | jq -r '.args["tinyusb.json"]') test/hil/tinyusb.json` + — documented in the hil skill. + +## Testing + +`test/hil/test_hil_select.py` — stdlib `unittest`, no hardware, injected diffs via +`--diff-file`/API. Cases (the acceptance examples): +1. `src/portable/raspberrypi/rp2040/dcd_rp2040.c` → only rp2040-family roster boards, device + tests only, host-only boards absent, `full` false. +2. `src/device/usbd.c` → every device-capable board on both rosters, all device tests + dual, + no host-only board, no host tests. +3. `src/class/cdc/cdc_device.c` → only examples with CFG_TUD_CDC enabled (must include + device/cdc_msc and device/cdc_dual_ports; must exclude device/msc_dual_lun and all + host tests). +4. `src/class/msc/msc_host.c` → host-capable boards only, host examples with CFG_TUH_MSC. +5. `tools/random_new_script.py` → `full: true`. +6. `docs/foo.rst` alone → contributes nothing ⇒ empty selection, `full` false, all `args` + empty (CI additionally has check-paths gating; the selector's answer is still honest). +7. `hw/bsp/rp2040/family.cmake` → rp2040-family boards, all their tests. +8. Mixed device+host diff → no pruning (both roles present). +The suite runs in `set-matrix` before the selector is used, and locally via +`python3 test/hil/test_hil_select.py`. + +## Safety properties + +- Fail-open: unknown/infra paths ⇒ full matrix; selector crash in CI ⇒ job fails visibly + (never silently skips HIL). +- Only `pull_request` events are scoped. +- The selection JSON + per-file reasons are printed in the job log for audit. +- hil_test.py errors on `-b` names not in the config — the selector only emits roster names, + and the unit suite locks that invariant. + +## Sequencing + +Lands on `claude/hil-select` on top of the pool-check/split stack. Follow-ups it does not +include: narrowing rule 9 for unit-test-only changes; variant-level selection; pre-pr skill +text update ships in the same change (its mapping section shrinks to a selector call). diff --git a/hw/bsp/mcx/family.cmake b/hw/bsp/mcx/family.cmake index 89e2aadf2..60f43e152 100644 --- a/hw/bsp/mcx/family.cmake +++ b/hw/bsp/mcx/family.cmake @@ -94,10 +94,19 @@ function(family_configure_example TARGET RTOS) family_add_tinyusb(${TARGET} OPT_MCU_MCXA15) endif() + # PORT is set per board (board.cmake), so pick the driver at configure time. Spelled out + # rather than $ so the port path stays greppable: test/hil/hil_select.py + # maps a portable-driver change to the families whose build file names that directory. + if (PORT) + set(PORT_SRC ${TOP}/src/portable/chipidea/ci_hs/dcd_ci_hs.c) + else () + set(PORT_SRC ${TOP}/src/portable/chipidea/ci_fs/dcd_ci_fs.c) + endif () + target_sources(${TARGET} PUBLIC ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/family.c ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/../board.c - ${TOP}/src/portable/chipidea/$ + ${PORT_SRC} ${STARTUP_FILE_${CMAKE_C_COMPILER_ID}} ) target_include_directories(${TARGET} PUBLIC diff --git a/test/hil/hil_ci.sh b/test/hil/hil_ci.sh index 3384b4e2e..ef93bcb49 100644 --- a/test/hil/hil_ci.sh +++ b/test/hil/hil_ci.sh @@ -55,6 +55,7 @@ scp -q "$ROOT_DIR/test/hil/hil_test.py" \ "$ROOT_DIR/test/hil/hil_flash.py" \ "$ROOT_DIR/test/hil/hil_lock.py" \ "$ROOT_DIR/test/hil/usbtest.py" \ + "$ROOT_DIR/test/hil/hil_examples.py" \ "$ROOT_DIR/test/hil/pymtp.py" \ "$CONFIG" \ "$REMOTE:$REMOTE_DIR/test/hil/" diff --git a/test/hil/hil_ci_set_matrix.py b/test/hil/hil_ci_set_matrix.py index 13f7f1882..bca989bd1 100644 --- a/test/hil/hil_ci_set_matrix.py +++ b/test/hil/hil_ci_set_matrix.py @@ -17,8 +17,14 @@ def _resolve_config_path(config_file): def main(): parser = argparse.ArgumentParser() parser.add_argument('config_files', nargs='+', help='Configuration JSON file(s)') + parser.add_argument('--select', help='hil_select.py JSON; scopes boards when full=false') args = parser.parse_args() + selected = None + sel = json.loads(args.select) if args.select else None + if sel and not sel.get('full'): + selected = set(sel.get('boards', {})) + # Toolchain buckets must match the toolchains instantiated by the hil-build # job in .github/workflows/build.yml. Keep all keys present (even if empty) # so `fromJSON(hil_json)[toolchain]` always resolves to a list. @@ -40,6 +46,8 @@ def main(): config = json.load(f) for board in config['boards']: + if selected is not None and board['name'] not in selected: + continue name = board['name'] flasher = board['flasher'] # esptool boards must build under esp-idf; others default to arm-gcc @@ -49,6 +57,13 @@ def main(): toolchain = 'esp-idf' else: toolchain = board.get('toolchain', 'arm-gcc') + if toolchain not in matrix: + # a board in no bucket would never be built, and the bare KeyError + # below would only say so as a traceback from the set-matrix job + raise SystemExit( + f'{name}: toolchain {toolchain!r} is not a build bucket ' + f'({", ".join(matrix)}); add it here and to the hil-build / ' + f'hil-build-esp jobs in .github/workflows/build.yml') build_board = f'-b {name}' if 'build' in board and 'args' in board['build']: diff --git a/test/hil/hil_examples.py b/test/hil/hil_examples.py new file mode 100644 index 000000000..4c8b6918b --- /dev/null +++ b/test/hil/hil_examples.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# HIL example test lists, shared by hil_test.py (runner) and hil_select.py +# (PR-diff selector). Stdlib-only: hil_select runs on bare CI runners. + +# The per-board run order is shuffled (see test_board). +# Every example carries a unique hardcoded idProduct (see its usb_descriptors.c) + +# device tests +device_tests = [ + 'device/cdc_dual_ports', + 'device/cdc_msc', + 'device/dfu', + 'device/cdc_msc_throughput', + 'device/audio_test_freertos', + 'device/dfu_runtime', + 'device/cdc_msc_freertos', + 'device/hid_boot_interface', + 'device/msc_dual_lun', + 'device/hid_generic_inout', + 'device/printer_to_cdc', + 'device/midi_test', + 'device/mtp', + 'device/usbtest', # cafe:4010, unique PID; runs the Linux testusb tier-4 battery via usbtest.py + # 'device/net_lwip_webserver', # disabled for PR #3605: USB net iface enum is flaky on the CI HIL host +] + +dual_tests = [ + 'dual/host_info_to_device_cdc', +] + +host_test = [ + 'host/cdc_msc_hid', + 'host/msc_file_explorer', + 'host/msc_file_explorer_freertos', + 'host/device_info', +] diff --git a/test/hil/hil_select.py b/test/hil/hil_select.py new file mode 100755 index 000000000..3ac3f1fdb --- /dev/null +++ b/test/hil/hil_select.py @@ -0,0 +1,520 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""PR-diff -> HIL selection: which rig boards and which tests a change can affect. + +Stdlib-only (runs on bare CI runners; never imports hil_test/hil_flash/hil_lock). +Fail-open: any file no rule classifies forces the full matrix. See +docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md. + +JSON: full, boards (name -> 'all' | [tests]), families (bsp families the diff +touches, including ones with no rig board - build-only consumers such as /pre-pr +sample from these), args (hil_test.py args per config) and args_flasher (the same +args split by each board's flasher, for CI legs that split one rig by flasher). +""" +import argparse +import functools +import glob +import json +import os +import re +import subprocess +import sys + +from hil_examples import device_tests, dual_tests, host_test + +ALL_TESTS = {'device': device_tests, 'dual': dual_tests, 'host': host_test} + +# class dir -> config macro suffix exceptions (rule 3); dfu is per-file, handled inline +NET_MACROS = ('ECM_RNDIS', 'NCM') + +_NONCODE_RE = re.compile( + r'^(docs/|\.claude/|.*\.(md|rst)$|LICENSE)') +_FULL_RE = re.compile( + r'^(src/common/|src/osal/|src/tusb\.c$|src/tusb\.h$|src/tusb_option\.h$|' + r'test/hil/|\.github/workflows/build.*\.yml$|\.github/actions/|' + r'tools/build\.py$|tools/get_deps\.py$|tools/cmake/|hw/mcu/|lib/|' + r'hw/bsp/(family_support\.cmake|board_api\.h|board\.c|ansi_escape\.h)$|' + r'examples/build_system/|examples/CMakeLists\.txt$|' + # board_test is HIL infrastructure, not a test: hil_test.py flashes it to park + # every board (variant boundary + end-of-board teardown), so every board depends on it + r'examples/device/board_test/)') + +# --no-renames: with rename detection git reports only a rename's destination, so code +# moved out of an HIL-relevant path would be classified by its new path alone +GIT_DIFF_ARGV = ['git', 'diff', '--no-renames', '--name-only'] + + +def test_role(test: str) -> str: + return test.split('/', 1)[0] # 'device' | 'dual' | 'host' + + +def board_roles(board: dict) -> set: + t = board.get('tests', {}) + roles = set() + if t.get('device'): + roles.add('device') + if t.get('host'): + roles.add('host') + if t.get('dual'): + roles.update(('device', 'host')) + for only in t.get('only', []): + r = test_role(only) + roles.update(('device', 'host') if r == 'dual' else (r,)) + return roles + + +def board_tests(board: dict) -> list: + """Every test this board would run today (mirrors hil_test.test_board's default).""" + t = board.get('tests', {}) + if 'only' in t: + run = list(t['only']) + else: + run = [] + if t.get('device'): + run += device_tests + if t.get('dual'): + run += dual_tests + if t.get('host'): + run += host_test + return [x for x in run if x not in t.get('skip', [])] + + +# cached: called per changed file x roster board, and the tree doesn't change mid-run +@functools.lru_cache(maxsize=None) +def board_family(board_name: str, repo_root: str): + hits = glob.glob(os.path.join(repo_root, 'hw/bsp/*/boards', board_name)) + return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None + + +# `if (OPTION STREQUAL "1")` guards in family_support.cmake, and the option tokens +# a roster entry passes to the build (NAME=VALUE / -DNAME=VALUE) +_CM_IF_RE = re.compile(r'if\s*\(') +_CM_ELSE_RE = re.compile(r'else(if)?\s*\(') +_CM_ENDIF_RE = re.compile(r'endif\s*\(') +_CM_OPT_RE = re.compile(r'if\s*\(\s*\$?\{?([A-Za-z_]\w*)\}?\s+STREQUAL\s+"?1"?\s*\)') +_CM_PORT_RE = re.compile(r'src/portable/((?:[^/\s]+/)?[^/\s]+)/') +_FALSY = ('', '0', 'off', 'false', 'no') + + +@functools.lru_cache(maxsize=None) +def port_option_gates(repo_root: str) -> dict: + """port dir -> build options that compile it regardless of the board's family + file, e.g. {'analog/max3421': {'MAX3421_HOST'}} from family_support.cmake.""" + gates = {} + try: + text = open(os.path.join(repo_root, 'hw/bsp/family_support.cmake')).read() + except OSError: + return gates + stack = [] # one entry per open if(): its option, or None + for line in text.splitlines(): + line = line.strip() + if _CM_IF_RE.match(line): + m = _CM_OPT_RE.match(line) + stack.append(m.group(1) if m else None) + elif _CM_ELSE_RE.match(line): + if stack: + stack[-1] = None # the guard doesn't hold in this branch + elif _CM_ENDIF_RE.match(line): + if stack: + stack.pop() + opts = {o for o in stack if o} + m = _CM_PORT_RE.search(line) + if opts and m: + gates.setdefault(m.group(1), set()).update(opts) + return gates + + +_CM_SET_RE = re.compile(r'set\s*\(\s*([A-Za-z_]\w*)\s+([^)\s]+)\s*\)') + + +# cached: called per changed portable file x roster board +@functools.lru_cache(maxsize=None) +def bsp_board_options(board_name: str, repo_root: str) -> frozenset: + """Build options a board turns on in its own BSP: `set( )` in + hw/bsp//boards//board.cmake, e.g. MAX3421_HOST on the espressif + and rp2040 max3421 boards. CMake only - HIL CI builds nothing with Make, so a + board.mk-only option (e.g. nrf5340dk's MAX3421_HOST) compiles no port here.""" + fam = board_family(board_name, repo_root) + if not fam: + return frozenset() + path = os.path.join(repo_root, 'hw/bsp', fam, 'boards', board_name, 'board.cmake') + try: + text = open(path).read() + except OSError: + return frozenset() + out = set() + for line in text.splitlines(): + line = line.strip() + if line.startswith('#'): + continue + m = _CM_SET_RE.match(line) + if m and m.group(2).strip('"').lower() not in _FALSY: + out.add(m.group(1)) + return frozenset(out) + + +def board_options(board: dict, repo_root: str) -> set: + """Build options a board has truthy: the roster entry's build.args plus each + variant's defines (NAME=VALUE) and raw CFLAGS (-DNAME=VALUE), plus whatever its + own board.cmake sets (a board can enable a gated port without the roster saying so).""" + toks = list(board.get('build', {}).get('args', [])) + for v in board.get('variant', []): + toks += list(v.get('defines', [])) + toks += v.get('flags', '').split() + out = set(bsp_board_options(board['name'], repo_root)) + for t in toks: + name, _, val = (t[2:] if t.startswith('-D') else t).partition('=') + if name and val.strip().strip('"').lower() not in _FALSY: + out.add(name.strip()) + return out + + +@functools.lru_cache(maxsize=None) +def port_families(port_dir: str, repo_root: str) -> set: + """Board families that compile this src/portable dir. CMake only: HIL CI builds + every board with CMake, so a port wired up in family.mk alone is compiled for no + HIL board and must not select one. family.cmake lists portable sources directly + for most families; espressif instead references them from a nested component + CMakeLists.txt (hw/bsp/espressif/components/tinyusb_src/CMakeLists.txt).""" + fams = set() + bsp_root = os.path.join(repo_root, 'hw/bsp') + # trailing '/' so a port dir is not a prefix of a sibling: bare 'microchip/pic' + # would otherwise match '.../microchip/pic32mz/...' and inherit its families + needle = port_dir + '/' + for f in glob.glob(os.path.join(bsp_root, '*/family.cmake')) + \ + glob.glob(os.path.join(bsp_root, '*/components/*/CMakeLists.txt')): + try: + if needle in open(f).read(): + fam = os.path.relpath(f, bsp_root).split(os.sep, 1)[0] + fams.add(fam) + except OSError: + pass + return fams + + +_CLS_INC_RE = re.compile(r'#\s*include\s*[<"]class/([^/"<>]+)/([^"<>]+)[">]') + + +@functools.lru_cache(maxsize=None) +def class_include_edges(repo_root: str) -> dict: + """'/
' -> the other class dirs that include it. A class header + pulled in by a second class ships in every firmware enabling that second class: + src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, and + net_device.h includes class/cdc/cdc.h. The class rule derives macros from the + directory name alone, so without this edge a change to the included header + selects only its own class's examples - and on a board that skips those (e.g. + metro_m4_express skips audio_test_freertos), nothing at all. + + Derived from the actual #include lines rather than a hand-written table so it + cannot rot when a class picks up or drops a cross-class include.""" + edges = {} + for f in sorted(glob.glob(os.path.join(repo_root, 'src/class/*/*.[ch]'))): + cls = os.path.basename(os.path.dirname(f)) + try: + text = open(f).read() + except OSError: + continue + for inc_cls, inc_hdr in _CLS_INC_RE.findall(text): + if inc_cls != cls: + edges.setdefault(f'{inc_cls}/{inc_hdr}', set()).add(cls) + return edges + + +def class_macros(cls: str, base: str, prefix: str) -> list: + """Config macros that compile a class dir's code, for role prefix TUD/TUH. + `base` refines dfu only (it splits DFU from DFU_RUNTIME per file); pass '' for + a class reached through an include edge, where the widest set is correct.""" + if cls == 'net': + return [f'CFG_{prefix}_{m}' for m in NET_MACROS] + if cls == 'dfu': + if base.startswith('dfu_rt'): + return [f'CFG_{prefix}_DFU_RUNTIME'] + if base.startswith('dfu_device') or base.startswith('dfu_host'): + return [f'CFG_{prefix}_DFU'] + return [f'CFG_{prefix}_DFU', f'CFG_{prefix}_DFU_RUNTIME'] + return [f'CFG_{prefix}_{cls.upper()}'] + + +def _config_enables(cfg_path: str, macros) -> bool: + try: + text = open(cfg_path).read() + except OSError: + return False + return any(re.search(rf'#define\s+{m}\s+\(?\s*0*[1-9]', text) for m in macros) + + +def roster_only_tests(all_boards) -> set: + """Test paths that only appear in a roster board's tests.only list (e.g. + espressif boards), not in the shared device/dual/host_test lists.""" + out = set() + for b in all_boards: + out.update(b.get('tests', {}).get('only', [])) + return out + + +def class_examples(macros, role: str, repo_root: str, extra_tests: set) -> set: + """Tests (from role's + dual lists, plus roster-only-list tests of that role) + whose example config enables any macro.""" + pool = role_tests({role}, extra_tests) + out = set() + for test in pool: + cfg = os.path.join(repo_root, 'examples', test, 'src', 'tusb_config.h') + if _config_enables(cfg, macros): + out.add(test) + return out + + +def role_tests(roles: set, extras: set) -> set: + """Every test for the given role(s): each role's own list + dual tests, + plus roster-only-list tests (extras) matching those roles or 'dual'.""" + pool = set(dual_tests) + for r in roles: + pool |= set(ALL_TESTS[r]) + pool |= {t for t in extras if test_role(t) in roles or test_role(t) == 'dual'} + return pool + + +class _Sel: + """Accumulates contributions. board->set(tests) plus 'all-board' markers.""" + def __init__(self): + self.full = False + self.by_board = {} # name -> set of tests, or 'all' + self.roles = set() # roles touched by any contribution + self.families = set() # bsp families touched (incl. off-rig ones: build-only consumers) + self.reasons = [] + + def add(self, boards, tests, reason): + """tests: 'all' or iterable of test paths.""" + self.reasons.append(reason) + for b in boards: + cur = self.by_board.get(b) + if tests == 'all' or cur == 'all': + self.by_board[b] = 'all' + else: + self.by_board[b] = (cur or set()) | set(tests) + + def force_full(self, reason): + self.full = True + self.reasons.append(reason) + + +def _classify_one(path, repo_root, roster_boards, extras: set, s: _Sel): + base = os.path.basename(path) + if _NONCODE_RE.match(path): + s.reasons.append(f'{path}: non-code, no contribution') + return + if _FULL_RE.match(path): + s.force_full(f'{path}: core/infra -> full matrix') + return + + m = re.match(r'src/portable/((?:[^/]+/)?[^/]+)/', path) + if m: + port = m.group(1) + if re.match(r'(dcd_|.*_device)', base): + roles = {'device'} + elif re.match(r'(hcd_|.*_host)', base): + roles = {'host'} + else: + roles = {'device', 'host'} + fams = port_families(port, repo_root) + if not fams: + # no family references this port: either a new/renamed port dir or a + # family.cmake layout the scan misses - widen instead of contributing nothing + s.force_full(f'{path}: port {port} maps to no board family -> full matrix') + return + s.families.update(fams) + # a board can also pull the port in through a build option (e.g. MAX3421_HOST=1 + # from the roster on metro_m4_express, or from its own board.cmake), which its + # family file never names + gates = port_option_gates(repo_root).get(port, set()) + boards = [b['name'] for b in roster_boards + if (board_family(b['name'], repo_root) in fams or + (gates and board_options(b, repo_root) & gates)) and (board_roles(b) & roles)] + tests = role_tests(roles, extras) + s.roles.update(roles) + why = f'{path}: port {port} -> families {sorted(fams)}' + if gates: + why += f' + option {sorted(gates)}' + s.add(boards, tests, f'{why} -> boards {boards} ({"/".join(sorted(roles))})') + return + + m = re.match(r'src/class/([^/]+)/', path) + if m: + cls = m.group(1) + if re.search(r'_device\.[ch]$', base): + roles = {'device'} + elif re.search(r'_host\.[ch]$', base): + roles = {'host'} + else: + roles = {'device', 'host'} + # this file's own class, plus any class whose headers include it + via = sorted(class_include_edges(repo_root).get(f'{cls}/{base}', ())) + + def macros(prefix): + return (class_macros(cls, base, prefix) + + [m2 for c in via for m2 in class_macros(c, '', prefix)]) + tests = set() + if 'device' in roles: + tests |= class_examples(macros('TUD'), 'device', repo_root, extras) + if 'host' in roles: + tests |= class_examples(macros('TUH'), 'host', repo_root, extras) + boards = [b['name'] for b in roster_boards if board_roles(b) & roles] + s.roles.update(roles) + why = f'{path}: class {cls}' + (f' (+ included by {via})' if via else '') + s.add(boards, tests, f'{why} -> {sorted(tests)} ({"/".join(sorted(roles))})') + return + + m = re.match(r'src/(device|host)/', path) + if m: + role = m.group(1) + boards = [b['name'] for b in roster_boards if role in board_roles(b)] + s.roles.add(role) + s.add(boards, role_tests({role}, extras), f'{path}: core {role} stack -> all {role} tests') + return + + m = re.match(r'hw/bsp/([^/]+)/(?:boards/([^/]+)/)?', path) + if m: + fam, brd = m.group(1), m.group(2) + s.families.add(fam) + if brd: + boards = [b['name'] for b in roster_boards if b['name'] == brd] + why = f'{path}: bsp board {brd}' + else: + boards = [b['name'] for b in roster_boards + if board_family(b['name'], repo_root) == fam] + why = f'{path}: bsp family {fam}' + s.roles.update(('device', 'host')) + s.add(boards, 'all', f'{why} -> boards {boards}') + return + + m = re.match(r'examples/(device|host|dual)/([^/]+)/', path) + if m: + test = f'{m.group(1)}/{m.group(2)}' + known = any(test in pool for pool in ALL_TESTS.values()) or test in extras + if known: + boards = [b['name'] for b in roster_boards] + role = test_role(test) + s.roles.update(('device', 'host') if role == 'dual' else (role,)) + s.add(boards, [test], f'{path}: example -> {test} on all boards') + else: + s.reasons.append(f'{path}: example not in HIL lists, no contribution') + return + + s.force_full(f'{path}: unclassified -> full matrix') + + +def classify(changed_files, repo_root, rosters): + all_boards = [] + seen = set() + for _, boards in rosters: + for b in boards: + if b['name'] not in seen: + seen.add(b['name']) + all_boards.append(b) + + extras = roster_only_tests(all_boards) + s = _Sel() + # no early exit once full: keep classifying so `families` still reports every + # family the diff touches (build-only consumers need it). Nothing after the first + # force_full can change full/boards/args - the full branch below ignores by_board. + for path in changed_files: + _classify_one(path, repo_root, all_boards, extras, s) + + if s.full: + return {'full': True, 'boards': {b['name']: 'all' for b in all_boards}, + 'families': sorted(s.families), 'reasons': s.reasons} + + # role pruning: single-role selections drop the other role's tests and boards + by_name = {b['name']: b for b in all_boards} + out = {} + for name, tests in s.by_board.items(): + allowed = board_tests(by_name[name]) + if tests == 'all': + kept = list(allowed) + else: + kept = [t for t in allowed if t in tests] + if s.roles and s.roles != {'device', 'host'}: + role = next(iter(s.roles)) + kept = [t for t in kept if test_role(t) in (role, 'dual')] + if kept: + out[name] = 'all' if set(kept) == set(allowed) else sorted(kept) + return {'full': False, 'boards': out, 'families': sorted(s.families), + 'reasons': s.reasons} + + +def _board_args(name, chosen) -> list: + parts = [f'-b {name}'] + if chosen != 'all': + parts.append(f'-bt {name}:{",".join(chosen)}') + return parts + + +def selection_args(sel, rosters): + """hil_test.py args per config. Empty means either 'full matrix' or 'nothing + selected' - callers must read sel['full'] to tell them apart.""" + args = {} + for cfg_path, boards in rosters: + parts = [] + if not sel['full']: + for b in boards: + chosen = sel['boards'].get(b['name']) + if chosen is not None: + parts += _board_args(b['name'], chosen) + args[os.path.basename(cfg_path)] = ' '.join(parts) + return args + + +def selection_args_by_flasher(sel, rosters): + """{config: {flasher name: args}}. CI runs one rig as several jobs split by + flasher (esptool vs the rest); each must gate on its own subset, otherwise the + other leg runs a filter matching zero boards and reports a vacuous green.""" + out = {} + for cfg_path, boards in rosters: + per = {} + if not sel['full']: + for b in boards: + chosen = sel['boards'].get(b['name']) + if chosen is None: + continue + per.setdefault(b.get('flasher', {}).get('name', ''), []).extend( + _board_args(b['name'], chosen)) + out[os.path.basename(cfg_path)] = {f: ' '.join(p) for f, p in per.items()} + return out + + +def changed_files_from_git(base, repo_root): + mb = subprocess.run(['git', 'merge-base', 'HEAD', base], cwd=repo_root, + capture_output=True, text=True, check=True).stdout.strip() + diff = subprocess.run(GIT_DIFF_ARGV + [f'{mb}..HEAD'], cwd=repo_root, + capture_output=True, text=True, check=True).stdout + return [l for l in diff.splitlines() if l.strip()] + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + g = ap.add_mutually_exclusive_group(required=True) + g.add_argument('--base', help='git ref to diff against (merge-base..HEAD)') + g.add_argument('--diff-file', help='newline-separated changed-file list') + ap.add_argument('configs', nargs='+', help='rig roster JSON file(s)') + a = ap.parse_args() + + repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + rosters = [] + for c in a.configs: + with open(c) as f: + rosters.append((c, json.load(f)['boards'])) + + files = (open(a.diff_file).read().splitlines() if a.diff_file + else changed_files_from_git(a.base, repo_root)) + files = [f for f in files if f.strip()] + + s = classify(files, repo_root, rosters) + s['args'] = selection_args(s, rosters) + s['args_flasher'] = selection_args_by_flasher(s, rosters) + for r in s['reasons']: + print(f'hil_select: {r}', file=sys.stderr) + print(json.dumps(s)) + + +if __name__ == '__main__': + main() diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py index 71e85f55f..96d52e601 100755 --- a/test/hil/hil_test.py +++ b/test/hil/hil_test.py @@ -59,6 +59,7 @@ from multiprocessing import TimeoutError as MpTimeoutError import hil_flash import hil_lock +from hil_examples import device_tests, dual_tests, host_test # Raw Lock/Semaphore objects passed via Pool initargs are inheritable only under the fork # start method (spawn/forkserver pickle them and fail at Pool creation) — pin it so a @@ -1351,39 +1352,6 @@ def test_device_usbtest(board): # Main # ------------------------------------------------------------- -# The per-board run order is shuffled (see test_board). -# Every example carries a unique hardcoded idProduct (see its usb_descriptors.c) - -# device tests -device_tests = [ - 'device/cdc_dual_ports', - 'device/cdc_msc', - 'device/dfu', - 'device/cdc_msc_throughput', - 'device/audio_test_freertos', - 'device/dfu_runtime', - 'device/cdc_msc_freertos', - 'device/hid_boot_interface', - 'device/msc_dual_lun', - 'device/hid_generic_inout', - 'device/printer_to_cdc', - 'device/midi_test', - 'device/mtp', - 'device/usbtest', # cafe:4010, unique PID; runs the Linux testusb tier-4 battery via usbtest.py - # 'device/net_lwip_webserver', # disabled for PR #3605: USB net iface enum is flaky on the CI HIL host -] - -dual_tests = [ - 'dual/host_info_to_device_cdc', -] - -host_test = [ - 'host/cdc_msc_hid', - 'host/msc_file_explorer', - 'host/msc_file_explorer_freertos', - 'host/device_info', -] - def test_example(board: Board, variant: str, example: str) -> tuple[int, str, str | None]: """ @@ -1517,6 +1485,10 @@ def build_board(board: Board) -> tuple[str, int]: return name, failed +# pseudo-test column for a variant boundary the park-flash could not clear (see below) +BOUNDARY_CELL = 'same-PID boundary' + + def test_board(board: Board) -> tuple[str, int, list[str], list, float]: name = board['name'] flasher = board['flasher'] @@ -1568,6 +1540,7 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: err_count = 0 failed_tests = [] + board_wide_fail = False # re-run the whole board, not a subset of its tests rows = [] # list of (row_label, {example: status}, duration) — one row per build variant # a -t/-bt filtered run times only a subset; report no duration so an accumulate # re-run keeps the previous full-run value @@ -1587,10 +1560,36 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: random.Random(f'{shuffle_seed}:{name}:{vname}').shuffle(run_list) if run_list[0] == prev_last: run_list[0], run_list[-1] = run_list[-1], run_list[0] + cells = {} + if run_list and run_list[0] == prev_last and not skip_flash: + # Same example (same PID) still repeats across the boundary: a one-test + # list (the common case for a -bt scoped run) leaves nothing to swap + # with. Park on board_test first - it disables the board's USB, so the + # PID goes away and the next flash must re-enumerate to be seen. + t_park = time.monotonic() + park_ec, park_status, _ = test_example(board, vname, 'device/board_test') + if park_ec or park_status == 'skip': + # Boundary not cleared: the previous variant's device may still be + # enumerated under the same PID, so this variant's tests could pass + # against its firmware. Skip them - a false green proves nothing and + # is worse than a gap - and record the boundary itself as the failure + # (a visible ❌ cell, mirroring the board-lock row above) so the report + # matches the exit code instead of rendering all-green. + why = 'no board_test binary' if park_status == 'skip' else 'park flash failed' + log_line(f'{vname:40} {"same-PID boundary":30} {STATUS_FAILED}: not cleared ({why}); ' + f'skipping {len(run_list)} test(s) on this variant') + err_count += 1 + cells[BOUNDARY_CELL] = 'fail' + # blaming run_list[0] would re-run an innocent test that then passes, + # leaving the boundary unretested; re-run the whole board instead + board_wide_fail = True + # leave prev_last alone: the board still holds the previous variant's + # firmware, so the next variant must attempt the park again + run_list = [] + t_board += time.monotonic() - t_park # park is teardown, not board cost if run_list: prev_last = run_list[-1] t_variant = time.monotonic() - cells = {} for test in run_list: ec, status, metric = test_example(board, vname, test) err_count += ec @@ -1609,7 +1608,7 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: if not skip_flash: test_example(board, variants[0]['name'], 'device/board_test') - return name, err_count, sorted(set(failed_tests)), rows, t_total + return name, err_count, [] if board_wide_fail else sorted(set(failed_tests)), rows, t_total finally: if _lock_fh: try: @@ -1704,11 +1703,13 @@ def render_matrix(rows_all: list) -> str: return summary + '\n\n' + '\n'.join([header, sep] + body) -def accumulate_report(mret: list, report_dir: Path, fresh: bool) -> str: +def accumulate_report(mret: list, report_dir: Path, fresh: bool, scope: str = '') -> str: """Merge this run's results into hil_report.json in report_dir, then (re)write - the markdown matrix to hil_report.md. `fresh` (a full run, no --accumulate/-bt) + the markdown matrix to hil_report.md. `fresh` (a first run, no --accumulate) starts a new report; otherwise a re-run accumulates so boards/tests that - already passed are preserved while re-run cells are updated. Returns the md.""" + already passed are preserved while re-run cells are updated. `scope` names the + board filter, if any, so a scoped table is not mistaken for a full one. + Returns the md.""" acc = {} # ordered {row_label: [cells dict, duration str|None]} jpath = report_dir / REPORT_JSON if not fresh and jpath.is_file(): @@ -1736,6 +1737,10 @@ def accumulate_report(mret: list, report_dir: Path, fresh: bool) -> str: del acc[name] for row_label, cells, dur in rows: row = acc.setdefault(row_label, [{}, None]) + # the boundary cell is only ever written on failure, so a re-run of this + # variant that cleared the boundary must drop the previous attempt's ❌ + if BOUNDARY_CELL not in cells: + row[0].pop(BOUNDARY_CELL, None) row[0].update(cells) if dur is not None: row[1] = dur @@ -1745,6 +1750,10 @@ def accumulate_report(mret: list, report_dir: Path, fresh: bool) -> str: for k, (c, d) in acc.items()]}, indent=2) + '\n') md = render_matrix([(k, c, d) for k, (c, d) in acc.items()]) + if scope: + # a scoped run's small table is otherwise indistinguishable from a full one, + # and it replaces the previous full table in the sticky PR comment + md = f'_Scoped run: {scope}. Boards/tests not listed were not run._\n\n' + md (report_dir / REPORT_MD).write_text(md + '\n', encoding='utf-8') return md @@ -1831,13 +1840,14 @@ def main() -> None: # HIL report sidecar (hil_report.json/.md) and the .failed re-run spec live in # report_dir (CI keys it by run id, so it persists across run attempts but is - # private to one run). A full run starts fresh; a re-run (--accumulate / -bt, - # i.e. the .failed file) merges so already-passed boards/tests are preserved. - # Clear prior state up front on a fresh run so a crash mid-run can't leave a - # stale report or re-run spec to be consumed by a retry. + # private to one run). A full run starts fresh; a re-run (--accumulate, which + # the generated .failed spec always starts with) merges so already-passed + # boards/tests are preserved. Clear prior state up front on a fresh run so a + # crash mid-run can't leave a stale report or re-run spec for a retry. + # -bt alone is not a re-run marker: PR-scoped first attempts pass -bt too. report_dir = Path(os.environ.get('HIL_REPORT_DIR', '.')) failed_fname = report_dir / (config_file.name + '.failed') - fresh = not (args.accumulate or args.board_test) + fresh = not args.accumulate if fresh: report_dir.mkdir(parents=True, exist_ok=True) for f in (REPORT_JSON, REPORT_MD): @@ -1935,7 +1945,11 @@ def main() -> None: print(f'warning: cannot persist controller hints to {CONTROLLER_CACHE}: {e}') # board x test result matrix -> hil_report.md (accumulates across re-runs) + stdout - report = accumulate_report(mret, report_dir, fresh) + # -b/-bt in play means a filtered run (PR selection or a re-run spec): say so in the + # report, which otherwise looks exactly like a full run that happened to be small + scoped = sorted(set(args.board) | set(board_test)) + scope = f'{len(scoped)} board(s) — {", ".join(scoped)}' if scoped else '' + report = accumulate_report(mret, report_dir, fresh, scope) print() print(report) print(f'\nReport written to {(report_dir / REPORT_MD).resolve()}') diff --git a/test/hil/test_hil_select.py b/test/hil/test_hil_select.py new file mode 100644 index 000000000..6a2bf6210 --- /dev/null +++ b/test/hil/test_hil_select.py @@ -0,0 +1,542 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests for hil_select.py — pure logic, no hardware, no git. Run directly: +# python3 test/hil/test_hil_select.py +import glob +import json +import os +import sys +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import hil_select +from hil_examples import device_tests, dual_tests, host_test + +REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + +def real_rosters(): + """The actual rig rosters, for regression tests that need real-world data + (a specific board/family/only-list) rather than the synthetic ROSTER above.""" + rosters = [] + for name in ('tinyusb.json', 'hfp.json'): + path = os.path.join(REPO, 'test/hil', name) + with open(path) as f: + rosters.append((f'test/hil/{name}', json.load(f)['boards'])) + return rosters + + +def on_roster(tc, *names): + """The subset of `names` currently in the live rig rosters, skipping the test + when none are. Parking/unparking a board is routine rig maintenance and must not + fail this suite: CI runs it right before the selector and treats a failure as + 'selector unusable', dropping PR scoping and annotating the run.""" + have = {b['name'] for _, boards in real_rosters() for b in boards} + got = [n for n in names if n in have] + if not got: + tc.skipTest(f'not in the rig roster: {", ".join(names)}') + return got + + +ROSTER = [ + # device-only, rp2040 family + {'name': 'raspberry_pi_pico', 'uid': 'u1', 'flasher': {'name': 'openocd'}, + 'tests': {'device': True, 'host': True, 'dual': True}}, + # device-only, stm32f4 family + {'name': 'stm32f407disco', 'uid': 'u2', 'flasher': {'name': 'jlink'}, + 'tests': {'device': True, 'host': False, 'dual': False}}, + # host-only board + {'name': 'raspberry_pi_pico2', 'uid': 'u3', 'flasher': {'name': 'openocd'}, + 'tests': {'device': False, 'host': True, 'dual': False}}, + # only-list board (espressif-style), flashed by the CI leg that splits on esptool + {'name': 'espressif_s3_devkitm', 'uid': 'u4', 'flasher': {'name': 'esptool'}, + 'tests': {'only': ['device/cdc_msc_freertos', 'host/device_info']}}, +] +ROSTERS = [('test/hil/tinyusb.json', ROSTER)] + + +def sel(files): + return hil_select.classify(files, REPO, ROSTERS) + + +class TestPortRule(unittest.TestCase): + def test_dcd_rp2040_selects_pico_family_only(self): + s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico', s['boards']) + self.assertNotIn('stm32f407disco', s['boards']) + self.assertNotIn('espressif_s3_devkitm', s['boards']) + # device role: no host tests in pico's list + self.assertTrue(all(not t.startswith('host/') for t in s['boards']['raspberry_pi_pico'])) + # host-only boards drop out entirely on a device-role change + self.assertNotIn('raspberry_pi_pico2', s['boards']) + + def test_shared_port_file_is_both_roles(self): + s = sel(['src/portable/synopsys/dwc2/dwc2_common.c']) + self.assertFalse(s['full']) + self.assertNotIn('raspberry_pi_pico', s['boards']) # rp2040 is not a dwc2 family + self.assertIn('stm32f407disco', s['boards']) # stm32f4 is + + +class TestCoreRoleRule(unittest.TestCase): + def test_usbd_selects_all_device_tests_everywhere(self): + s = sel(['src/device/usbd.c']) + self.assertFalse(s['full']) + self.assertNotIn('raspberry_pi_pico2', s['boards']) # host-only board dropped + pico = s['boards']['raspberry_pi_pico'] + self.assertTrue(set(device_tests).issubset(set(pico))) + self.assertTrue(set(dual_tests).issubset(set(pico))) # dual survives device role + self.assertTrue(all(not t.startswith('host/') for t in pico)) + # only-list board: selection intersects its only-list + esp = s['boards']['espressif_s3_devkitm'] + self.assertEqual(esp, ['device/cdc_msc_freertos']) + + def test_host_change_drops_device(self): + s = sel(['src/host/usbh.c']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico2', s['boards']) + self.assertNotIn('stm32f407disco', s['boards']) # device-only board dropped + + +class TestClassRule(unittest.TestCase): + def test_cdc_device_selects_cdc_examples_only(self): + s = sel(['src/class/cdc/cdc_device.c']) + self.assertFalse(s['full']) + pico = s['boards']['raspberry_pi_pico'] + self.assertIn('device/cdc_msc', pico) + self.assertIn('device/cdc_dual_ports', pico) + self.assertNotIn('device/msc_dual_lun', pico) # CFG_TUD_CDC 0 there + self.assertNotIn('device/usbtest', pico) # CFG_TUD_CDC 0 there + self.assertTrue(all(not t.startswith('host/') for t in pico)) + + def test_msc_host_selects_host_side(self): + s = sel(['src/class/msc/msc_host.c']) + self.assertFalse(s['full']) + self.assertNotIn('stm32f407disco', s['boards']) # device-only board + pico2 = s['boards']['raspberry_pi_pico2'] + self.assertIn('host/msc_file_explorer', pico2) + self.assertTrue(all(not t.startswith('device/') for t in pico2)) + + +class TestClassIncludeEdges(unittest.TestCase): + """A class header another class includes reaches that class's examples too. + src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, so + midi_test's firmware contains audio.h - but the class rule derives macros from + the directory name alone, so an audio.h change used to select only + device/audio_test_freertos. On boards that skip that example the per-board + intersection emptied and an audio.h-only PR ran ZERO HIL on them.""" + def test_edges_derived_from_includes(self): + edges = hil_select.class_include_edges(REPO) + self.assertEqual(edges.get('audio/audio.h'), {'midi'}) + self.assertEqual(edges.get('cdc/cdc.h'), {'net'}) + + def test_audio_header_selects_midi_example(self): + s = hil_select.classify(['src/class/audio/audio.h'], REPO, real_rosters()) + self.assertFalse(s['full']) + # every board that runs device/midi_test at all must run it here (boards with + # a tests.only list, e.g. espressif, run the freertos examples instead) + by_name = {b['name']: b for _, bs in real_rosters() for b in bs} + checked = 0 + for name, tests in s['boards'].items(): + if 'device/midi_test' in hil_select.board_tests(by_name[name]): + self.assertIn('device/midi_test', tests, name) + checked += 1 + self.assertTrue(checked) + + def test_audio_header_reaches_boards_that_skip_audio(self): + # both skip device/audio_test_freertos: without the midi edge their + # intersection is empty and they drop out of the selection entirely + boards = on_roster(self, 'metro_m4_express', 'nrf54lm20dk') + s = hil_select.classify(['src/class/audio/audio.h'], REPO, real_rosters()) + for board in boards: + self.assertEqual(s['boards'].get(board), ['device/midi_test'], board) + + def test_edge_is_per_header_not_per_class(self): + # midi includes audio.h, not audio_device.h: an audio_device change must + # not drag midi's examples in + s = hil_select.classify(['src/class/audio/audio_device.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for tests in s['boards'].values(): + if tests != 'all': + self.assertNotIn('device/midi_test', tests) + + +class TestFallbackRules(unittest.TestCase): + def test_unknown_tool_is_full(self): + s = sel(['tools/random_new_script.py']) + self.assertTrue(s['full']) + + def test_docs_only_is_empty_not_full(self): + s = sel(['docs/info/contributing.rst', 'README.rst']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + def test_bsp_family_selects_family_boards(self): + s = sel(['hw/bsp/rp2040/family.cmake']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico', s['boards']) + self.assertEqual(s['boards']['raspberry_pi_pico'], 'all') + self.assertNotIn('stm32f407disco', s['boards']) + + def test_bsp_board_narrows_to_board(self): + s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) + self.assertFalse(s['full']) + self.assertEqual(list(s['boards'].keys()), ['raspberry_pi_pico']) + + def test_example_change_selects_that_example(self): + s = sel(['examples/device/cdc_msc/src/main.c']) + self.assertFalse(s['full']) + self.assertEqual(s['boards']['raspberry_pi_pico'], ['device/cdc_msc']) + + def test_core_common_is_full(self): + for f in ['src/tusb.c', 'src/common/tusb_fifo.c', 'src/osal/osal_freertos.h']: + self.assertTrue(sel([f])['full'], f) + + def test_board_test_example_is_full(self): + # board_test is the park/teardown firmware hil_test.py flashes on every board, + # not an unlisted example: a regression there must not skip the whole rig + for f in ['examples/device/board_test/src/main.c', + 'examples/device/board_test/CMakeLists.txt']: + self.assertTrue(sel([f])['full'], f) + + def test_harness_is_full(self): + for f in ['test/hil/hil_test.py', '.github/workflows/build.yml', 'hw/mcu/nxp/x.c', 'lib/foo/x.c']: + self.assertTrue(sel([f])['full'], f) + + def test_mixed_roles_no_pruning(self): + s = sel(['src/device/usbd.c', 'src/host/usbh.c']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico2', s['boards']) + self.assertIn('stm32f407disco', s['boards']) + + def test_cmakelists_and_requirements_are_full(self): + for f in ['src/CMakeLists.txt', 'examples/CMakeLists.txt', + 'examples/device/CMakeLists.txt', 'test/hil/requirements.txt']: + self.assertTrue(sel([f])['full'], f) + + def test_docs_txt_is_noncode(self): + s = sel(['docs/info/changelog.txt']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + +class TestArgsEmission(unittest.TestCase): + def test_args_for_scoped_selection(self): + s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + args = hil_select.selection_args(s, ROSTERS) + a = args['tinyusb.json'] + self.assertIn('-b raspberry_pi_pico', a) + self.assertNotIn('stm32f407disco', a) + self.assertIn('-bt raspberry_pi_pico:', a) # device-only subset of a device+host board + + def test_args_full_is_empty(self): + s = sel(['tools/random_new_script.py']) + self.assertEqual(hil_select.selection_args(s, ROSTERS), {'tinyusb.json': ''}) + + def test_args_all_board_gets_bare_b(self): + s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) + a = hil_select.selection_args(s, ROSTERS)['tinyusb.json'] + self.assertIn('-b raspberry_pi_pico', a) + self.assertNotIn('-bt', a) + + def test_args_by_flasher_splits_esp_from_the_rest(self): + s = sel(['src/device/usbd.c']) + per = hil_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json'] + self.assertIn('espressif_s3_devkitm', per['esptool']) + self.assertIn('raspberry_pi_pico', per['openocd']) + self.assertNotIn('espressif_s3_devkitm', per.get('openocd', '') + per.get('jlink', '')) + + def test_args_by_flasher_omits_a_flasher_with_no_selected_board(self): + # the esp CI leg must see no args at all here, not a filter matching zero boards + s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) + per = hil_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json'] + self.assertEqual(per, {'openocd': '-b raspberry_pi_pico'}) + + def test_args_by_flasher_full_is_empty(self): + s = sel(['tools/random_new_script.py']) + self.assertEqual(hil_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}}) + + def test_cli_diff_file(self): + import subprocess, tempfile, json as j + with tempfile.NamedTemporaryFile('w', suffix='.txt', delete=False) as f: + f.write('src/class/cdc/cdc_device.c\n') + path = f.name + r = subprocess.run([sys.executable, os.path.join(REPO, 'test/hil/hil_select.py'), + '--diff-file', path, os.path.join(REPO, 'test/hil/tinyusb.json')], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + out = j.loads(r.stdout) + self.assertFalse(out['full']) + self.assertIn('tinyusb.json', out['args']) + self.assertTrue(any('cdc_device' in line for line in out['reasons'])) + os.unlink(path) + + +class TestRealRosterPortFamilies(unittest.TestCase): + """Regression for port_families() missing espressif's dwc2 reference, which + lives in a component CMakeLists.txt rather than family.cmake/family.mk.""" + def test_dwc2_change_selects_espressif_boards(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = hil_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertIn(board, s['boards']) + + +class TestOptionGatedPort(unittest.TestCase): + """Regression: family_support.cmake compiles some ports from a build option + (MAX3421_HOST=1 -> hcd_max3421.c), so a board's family file never names them.""" + # host-side option board (max3421 as host controller), off any max3421 family + OPT_ROSTER = [('test/hil/opt.json', [ + {'name': 'fake_dual_board', 'uid': 'o1', 'flasher': {'name': 'jlink'}, + 'build': {'args': ['MAX3421_HOST=1']}, + 'tests': {'device': True, 'host': False, 'dual': True}}, + {'name': 'fake_host_board', 'uid': 'o2', 'flasher': {'name': 'jlink'}, + 'variant': [{'name': 'fake_host_board', 'flags': '-DMAX3421_HOST=1'}], + 'tests': {'device': False, 'host': True, 'dual': False}}, + {'name': 'fake_off_board', 'uid': 'o3', 'flasher': {'name': 'jlink'}, + 'variant': [{'name': 'fake_off_board', 'defines': ['MAX3421_HOST=0']}], + 'tests': {'device': True, 'host': True, 'dual': True}}, + ])] + + def test_real_roster_max3421_selects_option_board(self): + boards = on_roster(self, 'metro_m4_express') + s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertIn(board, s['boards']) + + def test_option_selects_via_args_defines_and_flags(self): + s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, self.OPT_ROSTER) + self.assertFalse(s['full']) + self.assertIn('fake_dual_board', s['boards']) # build.args + self.assertIn('fake_host_board', s['boards']) # variant flags + self.assertNotIn('fake_off_board', s['boards']) # variant defines, but =0 + + def test_device_role_port_does_not_pull_host_only_option_board(self): + s = hil_select.classify(['src/portable/analog/max3421/dcd_max3421.c'], REPO, self.OPT_ROSTER) + self.assertFalse(s['full']) + self.assertNotIn('fake_host_board', s['boards']) # host-only board, device change + self.assertIn('fake_dual_board', s['boards']) # device-capable option board + + def test_gates_parsed_from_family_support(self): + self.assertEqual(hil_select.port_option_gates(REPO).get('analog/max3421'), + {'MAX3421_HOST'}) + + def test_board_cmake_option_counts(self): + """A board can enable a gated port in its own BSP rather than via the roster + (hw/bsp/espressif/boards/*/board.cmake -> set(MAX3421_HOST 1)); board_options() + must see those too, or such a board joining the roster is silently dropped.""" + self.assertIn('MAX3421_HOST', + hil_select.bsp_board_options('adafruit_feather_esp32s3', REPO)) + self.assertIn('CFG_TUH_RPI_PIO_USB', + hil_select.bsp_board_options('adafruit_fruit_jam', REPO)) + # commented-out `# set(MAX3421_HOST 1)` must not count + self.assertNotIn('MAX3421_HOST', + hil_select.bsp_board_options('feather_nrf52840_express', REPO)) + + def test_board_cmake_option_selects_off_family_board(self): + # adafruit_feather_esp32s3 is not on any rig roster; stand it in as one to + # prove the BSP-sourced option alone pulls a max3421 change onto the board + roster = [('test/hil/opt.json', [ + {'name': 'adafruit_feather_esp32s3', 'uid': 'o1', 'flasher': {'name': 'esptool'}, + 'tests': {'device': False, 'host': True, 'dual': False}}])] + s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster) + self.assertFalse(s['full']) + self.assertIn('adafruit_feather_esp32s3', s['boards']) + + def test_board_mk_option_is_ignored(self): + """Make-only options must not select: HIL CI builds with CMake exclusively, so + hw/bsp/nrf/boards/nrf5340dk/board.mk's MAX3421_HOST compiles nothing here.""" + roster = [('test/hil/opt.json', [ + {'name': 'nrf5340dk', 'uid': 'o1', 'flasher': {'name': 'jlink'}, + 'tests': {'device': False, 'host': True, 'dual': False}}])] + s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + +class TestPortFamiliesCmakeOnly(unittest.TestCase): + """port_families() is CMake-only (HIL CI never builds with Make) and matches on + 'port_dir/' so a port dir is not a prefix of a sibling.""" + def test_make_only_family_is_not_a_family(self): + # hw/bsp/pic32mz has family.mk but no family.cmake + self.assertEqual(hil_select.port_families('microchip/pic32mz', REPO), set()) + + def test_prefix_port_does_not_inherit_sibling_families(self): + # bare-substring matching let 'microchip/pic' match '.../microchip/pic32mz/...' + self.assertEqual(hil_select.port_families('microchip/pic', REPO), set()) + + def test_make_only_port_forces_full(self): + s = sel(['src/portable/microchip/pic32mz/dcd_pic32mz.c']) + self.assertTrue(s['full']) + self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons']) + + def test_cmake_families_still_found(self): + self.assertEqual(hil_select.port_families('raspberrypi/rp2040', REPO), {'rp2040'}) + self.assertIn('stm32f4', hil_select.port_families('synopsys/dwc2', REPO)) + + +class TestPortFamiliesCoverage(unittest.TestCase): + """Systematic guard: every real dcd_*/hcd_* port directory should map to at + least one board family, so a future family.cmake/CMakeLists.txt layout that + port_families() doesn't scan fails loudly instead of silently dropping boards + (as espressif's dwc2 reference did - see TestRealRosterPortFamilies).""" + # Ports with no board family: not a bug, just not wired into any rig board. + # Add here (with a reason) only if port_families() legitimately can't find one. + # A port listed here force-fulls (fail-open), so it is never under-selected. + NO_FAMILY = { + 'template', # reference/example port, not built by any board + # hw/bsp/pic32mz has family.mk only (no family.cmake), and port_families() + # is CMake-only because HIL CI builds every board with CMake - so this port + # is compiled for no HIL board. + 'microchip/pic32mz', + 'microchip/pic', # same: only ever referenced from pic32mz's family.mk + } + + @staticmethod + def _dcd_hcd_ports(): + portable_root = os.path.join(REPO, 'src/portable') + ports = [] + for entry in sorted(os.listdir(portable_root)): + d = os.path.join(portable_root, entry) + if not os.path.isdir(d): + continue + if glob.glob(os.path.join(d, 'dcd_*.c')) or glob.glob(os.path.join(d, 'hcd_*.c')): + ports.append(entry) + continue + for sub in sorted(os.listdir(d)): + sd = os.path.join(d, sub) + if os.path.isdir(sd) and (glob.glob(os.path.join(sd, 'dcd_*.c')) or + glob.glob(os.path.join(sd, 'hcd_*.c'))): + ports.append(f'{entry}/{sub}') + return ports + + def test_every_port_maps_to_a_family(self): + ports = self._dcd_hcd_ports() + self.assertTrue(ports) # sanity: the scan itself found something + for port in ports: + if port in self.NO_FAMILY: + continue + fams = hil_select.port_families(port, REPO) + self.assertTrue(fams, f'{port}: no family references this port ' + f'(port_families() scan gap, or add to NO_FAMILY)') + + +class TestRealRosterOnlyListTests(unittest.TestCase): + """Regression for roster-only-list tests (e.g. espressif's hid_composite_freertos) + being invisible to the selector because it only knew the shared hil_examples lists.""" + def test_only_list_example_change_selects_it(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = hil_select.classify(['examples/device/hid_composite_freertos/src/main.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertEqual(s['boards'][board], ['device/hid_composite_freertos']) + + def test_class_change_includes_only_list_boards(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = hil_select.classify(['src/class/hid/hid_device.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertIn(board, s['boards']) + + +class TestPortAndCoreRoleUseExtras(unittest.TestCase): + """Regression: the port rule and core-role rule must thread the roster-only + test universe (extras) the same way the class rule already does, so a DCD + or device-stack change doesn't silently drop espressif's only-list tests + (e.g. hid_composite_freertos) that aren't in the shared device_tests list.""" + def test_dcd_change_includes_only_list_test(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = hil_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + tests = s['boards'][board] + self.assertIn('device/hid_composite_freertos', tests) + self.assertIn('device/cdc_msc_freertos', tests) + self.assertIn('device/audio_test_freertos', tests) + self.assertIn('device/usbtest', tests) + + def test_core_device_change_includes_only_list_test(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = hil_select.classify(['src/device/usbd.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + tests = s['boards'][board] + self.assertIn('device/hid_composite_freertos', tests) + self.assertIn('device/cdc_msc_freertos', tests) + self.assertIn('device/audio_test_freertos', tests) + self.assertIn('device/usbtest', tests) + + def test_host_change_does_not_leak_device_only_list_test(self): + s = hil_select.classify(['src/host/usbh.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board, tests in s['boards'].items(): + if tests == 'all': + continue + self.assertNotIn('device/hid_composite_freertos', tests, board) + + +class TestFamilies(unittest.TestCase): + """`families` exists for consumers that build (not just test) the diff: most + families have no rig board, so `boards` alone would compile nothing for them.""" + def test_off_rig_port_still_reports_family(self): + s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) # no same7x board on the rig + self.assertEqual(s['families'], ['same7x']) + + def test_port_families_are_reported(self): + s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + self.assertIn('rp2040', s['families']) + + def test_bsp_family_and_board_report_family(self): + self.assertEqual(sel(['hw/bsp/rp2040/family.cmake'])['families'], ['rp2040']) + self.assertEqual(sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h'])['families'], + ['rp2040']) + + def test_docs_only_has_no_families(self): + self.assertEqual(sel(['docs/info/contributing.rst'])['families'], []) + + def test_full_selection_still_reports_families(self): + """A full-matrix file must not hide the families of the other changed files: + consumers that build from `families` (e.g. /pre-pr) ignore `boards` when full.""" + s = sel(['src/common/tusb_fifo.c', 'src/portable/microchip/samx7x/dcd_samx7x.c']) + self.assertTrue(s['full']) + self.assertIn('same7x', s['families']) + # full stays full: every roster board, and no args to narrow the run + self.assertEqual(set(s['boards']), {b['name'] for b in ROSTER}) + self.assertTrue(all(v == 'all' for v in s['boards'].values())) + self.assertEqual(hil_select.selection_args(s, ROSTERS), {'tinyusb.json': ''}) + self.assertEqual(hil_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}}) + + def test_family_order_does_not_matter(self): + # same as above with the full-matrix file last (was the only order that worked) + s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c', 'src/common/tusb_fifo.c']) + self.assertTrue(s['full']) + self.assertIn('same7x', s['families']) + + +class TestGitDiffArgv(unittest.TestCase): + def test_diff_disables_rename_detection(self): + """Without --no-renames git reports only a rename's destination, so moving an + HIL-relevant file to a non-code path would be classified as non-code only.""" + self.assertIn('--no-renames', hil_select.GIT_DIFF_ARGV) + + +class TestPortWithoutFamilyIsFull(unittest.TestCase): + """A port dir no family file references must widen (full matrix), not silently + contribute zero boards — the fail-open contract.""" + def test_unreferenced_port_forces_full(self): + orig = hil_select.port_families + hil_select.port_families = lambda port_dir, repo_root: set() + try: + s = sel(['src/portable/vendor/newip/dcd_newip.c']) + finally: + hil_select.port_families = orig + self.assertTrue(s['full']) + self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons']) + + +if __name__ == '__main__': + unittest.main(verbosity=1) -- cgit v1.3.1 From 853cbff468cde5821447ecc92063acf11d3e696e Mon Sep 17 00:00:00 2001 From: hathach Date: Wed, 12 Aug 2026 23:04:33 +0700 Subject: hw/bsp/lpc55: implement board_get_unique_id from flash PFR UUID Read the 128-bit device UUID from the flash PFR region at 0x0009FC70 (UM11126 rev 2.1, section 48.8) rather than falling back to the fixed weak default in hw/bsp/board.c. Verified on lpcxpresso55s69: cdc_msc enumerates with SerialNumber E059C3E208F9B955B3BA4C5CC7F3D13D, matching the uid already recorded for that board in test/hil/local.json. --- hw/bsp/lpc55/family.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'hw/bsp') diff --git a/hw/bsp/lpc55/family.c b/hw/bsp/lpc55/family.c index e021caf35..11bf86827 100644 --- a/hw/bsp/lpc55/family.c +++ b/hw/bsp/lpc55/family.c @@ -170,6 +170,14 @@ uint32_t board_button_read(void) { return BUTTON_STATE_ACTIVE == GPIO_PinRead(GPIO, BUTTON_PORT, BUTTON_PIN); } +size_t board_get_unique_id(uint8_t id[], size_t max_len) { + // 128-bit UUID in the flash PFR region at 0x0009FC70 (UM11126 rev 2.1 section 48.8) + const uint8_t* uuid = (const uint8_t*) 0x0009FC70; + size_t const len = tu_min32(max_len, 16); + memcpy(id, uuid, len); + return len; +} + int board_uart_read(uint8_t* buf, int len) { (void) buf; (void) len; -- cgit v1.3.1 From 3963a1b70a572132aced1c1a0033e1c8249a0c7e Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 14 Aug 2026 01:08:40 +0700 Subject: test/hil, ci: contain a wedged USB stack instead of stranding the runner A wedged USB device used to take the whole HIL run with it. Every worker that touched the poisoned node blocked uninterruptibly, the pool could not be joined, map_async discarded every board's result, and the job ran to the GitHub ceiling with no report at all -- while the self-hosted runner's single job slot stayed occupied and every queued job waited behind it. Bound the calls a worker makes itself. read_sysfs, bounded_open and run_cmd all answer within a wall clock; read_sysfs distinguishes "absent" from "unknown", because a blocked read is not evidence of absence, and caps stranded readers at four (each costs a thread and an fd for the life of the process) after which the worker declares itself blind. mtype, the gio unmount, the libmtp session and the arecord/iperf reaps go through those bounds; the MTP session runs in a disposable subprocess, since libmtp's ctypes calls block unkillably in D state. Bound the run. A pool guard (HIL_POOL_TIMEOUT, 60 min) fires before any job ceiling and still writes a report. When the pool will not shut down, the sweep kills what the workers spawned -- descendants, not just direct children, since flashers run in their own session -- confirms each kill actually landed, and exits early so the runner is freed. Whatever survived is named in the report. Deliberately shallow past that point. We do not re-scan process groups, prove pid ownership, or escalate through sudo: a root-owned survivor is reported, not force-killed, because signalling a pid we cannot prove is ours is the worse failure, and the job ceiling backstops whatever this misses. A D-state holder was never killable anyway. Recover instead of reporting a wedge. A HUNG usbtest case reflashes its own DUT through its roster flasher, but only where the flasher can reach its probe past a poisoned node -- openocd pinned to a validated vid_pid, or esptool. Where it cannot, the run says so rather than reserving budget for a path that cannot fire. Raise the CI ceilings above the pool guard so the guard fires first and still writes its report, and pin --retry 1 on every HIL leg: the guard is a flat constant and does not scale with max_retry, so argparse's default of 3 would triple the serialized usbtest tail against an unchanged guard. Split the module: execution in hil_test/hil_flash/usbtest, infrastructure in helper/ (locking, health, selection, shared bounded IO), and the two matrix generators into .github/scripts/ -- ci_set_matrix.py sat in workflows/, where GitHub treats every file as a workflow definition. 193 tests cover the bounded paths, the kill ladder, the guard and the selector against synthetic /proc trees and PATH-injected fakes; a real wedge cannot be manufactured on demand. --- .circleci/config.yml | 2 +- .github/scripts/ci_set_matrix.py | 111 ++ .github/scripts/hil_ci_set_matrix.py | 92 ++ .github/workflows/build.yml | 71 +- .github/workflows/ci_set_matrix.py | 111 -- .github/workflows/pre-commit.yml | 1 + .pre-commit-config.yaml | 21 + CLAUDE.md | 3 +- .../2026-07-29-hil-pr-scoped-selection-design.md | 22 +- hw/bsp/mcx/family.cmake | 2 +- test/hil/helper/__init__.py | 4 + test/hil/helper/hil_health.py | 380 +++++ test/hil/helper/hil_lock.py | 525 ++++++ test/hil/helper/hil_pool_check.py | 1019 ++++++++++++ test/hil/helper/hil_select.py | 524 ++++++ test/hil/helper/hil_util.py | 585 +++++++ test/hil/hil_ci.sh | 85 +- test/hil/hil_ci_set_matrix.py | 90 -- test/hil/hil_examples.py | 37 - test/hil/hil_flash.py | 365 +++-- test/hil/hil_lock.py | 479 ------ test/hil/hil_pool_check.py | 1015 ------------ test/hil/hil_select.py | 520 ------ test/hil/hil_test.py | 1604 ++++++++++++------ test/hil/mtp_test.py | 246 +++ test/hil/test/stubs/pymtp.py | 125 ++ test/hil/test/test_hil_bounded.py | 1701 ++++++++++++++++++++ test/hil/test/test_hil_health.py | 636 ++++++++ test/hil/test/test_hil_select.py | 689 ++++++++ test/hil/test/test_hil_util.py | 230 +++ test/hil/test_hil_select.py | 581 ------- test/hil/tinyusb.json | 15 +- test/hil/usbtest.py | 647 ++++++-- tools/metrics_compare_base.py | 4 +- 34 files changed, 8821 insertions(+), 3721 deletions(-) create mode 100755 .github/scripts/ci_set_matrix.py create mode 100644 .github/scripts/hil_ci_set_matrix.py delete mode 100755 .github/workflows/ci_set_matrix.py create mode 100644 test/hil/helper/__init__.py create mode 100644 test/hil/helper/hil_health.py create mode 100755 test/hil/helper/hil_lock.py create mode 100644 test/hil/helper/hil_pool_check.py create mode 100755 test/hil/helper/hil_select.py create mode 100644 test/hil/helper/hil_util.py delete mode 100644 test/hil/hil_ci_set_matrix.py delete mode 100644 test/hil/hil_examples.py delete mode 100755 test/hil/hil_lock.py delete mode 100644 test/hil/hil_pool_check.py delete mode 100755 test/hil/hil_select.py create mode 100644 test/hil/mtp_test.py create mode 100644 test/hil/test/stubs/pymtp.py create mode 100644 test/hil/test/test_hil_bounded.py create mode 100644 test/hil/test/test_hil_health.py create mode 100644 test/hil/test/test_hil_select.py create mode 100644 test/hil/test/test_hil_util.py delete mode 100644 test/hil/test_hil_select.py (limited to 'hw/bsp') diff --git a/.circleci/config.yml b/.circleci/config.yml index 66799910d..48fa87899 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,7 +15,7 @@ jobs: - run: name: Set matrix command: | - MATRIX_JSON=$(python .github/workflows/ci_set_matrix.py) + MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py) echo "MATRIX_JSON=$MATRIX_JSON" BUILDSYSTEM_LIST=( diff --git a/.github/scripts/ci_set_matrix.py b/.github/scripts/ci_set_matrix.py new file mode 100755 index 000000000..50ada5964 --- /dev/null +++ b/.github/scripts/ci_set_matrix.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +import json + +# toolchain, url +toolchain_list = [ + "aarch64-gcc", + "arm-clang", + "arm-iar", + "arm-gcc", + "esp-idf", + "ft9xx-gcc", + "msp430-gcc", + "riscv-gcc", + "rx-gcc" +] + +# family: [supported toolchain] +family_list = { + "apm32f0xx": ["arm-gcc"], + "at32f402_405": ["arm-gcc"], + "at32f403a_407": ["arm-gcc"], + "at32f413": ["arm-gcc"], + "at32f415": ["arm-gcc"], + "at32f423": ["arm-gcc"], + "at32f425": ["arm-gcc"], + "at32f435_437": ["arm-gcc"], + "at32f45x": ["arm-gcc"], + "broadcom_32bit": ["arm-gcc"], + "broadcom_64bit": ["aarch64-gcc"], + "ch32f20x": ["arm-gcc"], + "ch32v10x": ["riscv-gcc"], + "ch32v20x": ["riscv-gcc"], + "ch32v30x": ["riscv-gcc"], + "ch583": ["riscv-gcc"], + "da1469x": ["arm-gcc"], + "fomu": ["riscv-gcc"], + "ft9xx": ["ft9xx-gcc"], + "gd32vf103": ["riscv-gcc"], + "hpmicro": ["riscv-gcc"], + "imxrt": ["arm-gcc", "arm-clang"], + "kinetis_k": ["arm-gcc"], + "kinetis_k32l": ["arm-gcc"], + "kinetis_kl": ["arm-gcc"], + "lpc11": ["arm-gcc", "arm-clang"], + "lpc13": ["arm-gcc", "arm-clang"], + "lpc15": ["arm-gcc", "arm-clang"], + "lpc17": ["arm-gcc", "arm-clang"], + "lpc18": ["arm-gcc", "arm-clang"], + "lpc40": ["arm-gcc", "arm-clang"], + "lpc43": ["arm-gcc", "arm-clang"], + "lpc51": ["arm-gcc", "arm-clang"], + "lpc54": ["arm-gcc", "arm-clang"], + "lpc55": ["arm-gcc", "arm-clang"], + "maxim": ["arm-gcc"], + "mcx": ["arm-gcc"], + "mm32": ["arm-gcc"], + "msp430": ["msp430-gcc"], + "msp432e4": ["arm-gcc"], + "nrf": ["arm-gcc", "arm-clang"], + "nuc100_120": ["arm-gcc"], + "nuc121_125": ["arm-gcc"], + "nuc126": ["arm-gcc"], + "nuc505": ["arm-gcc"], + "ra": ["arm-gcc"], + "rp2040": ["arm-gcc"], + "rw61x": ["arm-gcc"], + "rx": ["rx-gcc"], + "samd11": ["arm-gcc", "arm-clang"], + "samd2x_l2x": ["arm-gcc", "arm-clang"], + "samd5x_e5x": ["arm-gcc", "arm-clang"], + "samg": ["arm-gcc", "arm-clang"], + "stm32c0": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32c5": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32f0": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32f1": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32f2": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32f3": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32f4": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32f7": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32g0": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32g4": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32h5": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32h7": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32h7rs": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32l0": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32l4": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32n6": ["arm-gcc"], + "stm32u0": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32u5": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32wb": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32wba": ["arm-gcc", "arm-clang", "arm-iar"], + "tm4c": ["arm-gcc"], + "xmc4000": ["arm-gcc"], + # S3, P4 will be built by hil test + # "-bespressif_s3_devkitm": ["esp-idf"], + # "-bespressif_p4_function_ev": ["esp-idf"], +} + + +def set_matrix_json(): + matrix = {} + for toolchain in toolchain_list: + filtered_families = [family for family, supported_toolchain in family_list.items() if + toolchain in supported_toolchain] + matrix[toolchain] = filtered_families + + print(json.dumps(matrix)) + + +if __name__ == '__main__': + set_matrix_json() diff --git a/.github/scripts/hil_ci_set_matrix.py b/.github/scripts/hil_ci_set_matrix.py new file mode 100644 index 000000000..65f50788e --- /dev/null +++ b/.github/scripts/hil_ci_set_matrix.py @@ -0,0 +1,92 @@ +import argparse +import json +import os + + +def _resolve_config_path(config_file): + if os.path.exists(config_file): + return config_file + + # bare roster names resolve against the repo's test/hil (this script lives in + # .github/scripts); build.yml passes explicit paths, this is for hand-runs + repo_relative = os.path.join(os.path.dirname(__file__), '..', '..', 'test', 'hil', config_file) + if os.path.exists(repo_relative): + return repo_relative + + raise FileNotFoundError(f'Config file not found: {config_file}') + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('config_files', nargs='+', help='Configuration JSON file(s)') + parser.add_argument('--select', help='hil_select.py JSON; scopes boards when full=false') + args = parser.parse_args() + + selected = None + sel = json.loads(args.select) if args.select else None + if sel and not sel.get('full'): + selected = set(sel.get('boards', {})) + + # Toolchain buckets must match the toolchains instantiated by the hil-build + # job in .github/workflows/build.yml. Keep all keys present (even if empty) + # so `fromJSON(hil_json)[toolchain]` always resolves to a list. + matrix = { + 'arm-gcc': [], + 'riscv-gcc': [], + 'esp-idf': [] + } + + seen = {toolchain: set() for toolchain in matrix} + + def append_build_arg(toolchain, build_arg): + if build_arg not in seen[toolchain]: + seen[toolchain].add(build_arg) + matrix[toolchain].append(build_arg) + + for config_file in args.config_files: + with open(_resolve_config_path(config_file)) as f: + config = json.load(f) + + for board in config['boards']: + if selected is not None and board['name'] not in selected: + continue + name = board['name'] + flasher = board['flasher'] + # esptool boards must build under esp-idf; others default to arm-gcc + # but may opt into another bucket via an explicit "toolchain" field + # (e.g. RISC-V boards like ch32v20x need "riscv-gcc"). + if flasher['name'] == 'esptool': + toolchain = 'esp-idf' + else: + toolchain = board.get('toolchain', 'arm-gcc') + if toolchain not in matrix: + # a board in no bucket would never be built, and the bare KeyError + # below would only say so as a traceback from the set-matrix job + raise SystemExit( + f'{name}: toolchain {toolchain!r} is not a build bucket ' + f'({", ".join(matrix)}); add it here and to the hil-build / ' + f'hil-build-esp jobs in .github/workflows/build.yml') + + build_board = f'-b {name}' + if 'build' in board and 'args' in board['build']: + build_board += ' ' + ' '.join(f'-D{a}' for a in board['build']['args']) + + # Each variant builds into cmake-build- with its own cmake + # -D defines and raw CFLAGS. No 'variant' -> a single build named after + # the board. + variants = board.get('variant') or [{'name': name, 'flags': ''}] + for v in variants: + arg = build_board + if v['name'] != name: + arg += f' --build-name {v["name"]}' + for d in v.get('defines', []): + arg += f' -D{d}' + for tok in v.get('flags', '').split(): + arg += f' --cflag={tok}' + append_build_arg(toolchain, arg) + + print(json.dumps(matrix)) + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bdef81553..8f6014f48 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -41,7 +41,7 @@ jobs: - '.github/actions/**' - '.github/workflows/build.yml' - '.github/workflows/build_util.yml' - - '.github/workflows/ci_set_matrix.py' + - '.github/scripts/**' set-matrix: runs-on: ubuntu-latest @@ -72,10 +72,16 @@ jobs: # so a missing origin/, a shallow-clone hiccup or a selector traceback # must fall back to the FULL matrix (no --select, run=true, no args) instead # of failing the job. Same fail-open shape as pr_comment.yml's `|| true`. + # + # The selector's own unit suite gates it (stdlib-only, seconds): a selector + # whose tests fail can still exit 0 with valid-but-WRONG JSON -- fail-open alone + # never catches that class, and the pre-commit hil-test hook is a separate, + # advisory workflow that nothing here can `needs:`. Test-failing selector => + # full matrix, same as a crashing one. SELECT_JSON='' - if ! python3 test/hil/test_hil_select.py; then - echo "::error::hil_select unit tests failed - falling back to the full HIL matrix" - elif ! SELECT_JSON=$(python3 test/hil/hil_select.py --base "origin/$BASE_REF" test/hil/tinyusb.json test/hil/hfp.json); then + if ! python3 test/hil/test/test_hil_select.py; then + echo "::warning::hil_select unit suite failed - falling back to the full HIL matrix" + elif ! SELECT_JSON=$(python3 test/hil/helper/hil_select.py --base "origin/$BASE_REF" test/hil/tinyusb.json test/hil/hfp.json); then echo "::warning::hil_select failed - falling back to the full HIL matrix" SELECT_JSON='' fi @@ -113,7 +119,7 @@ jobs: SELECT: ${{ steps.hil-select.outputs.select }} run: | # build matrix - MATRIX_JSON=$(python .github/workflows/ci_set_matrix.py) + MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py) echo "matrix=$MATRIX_JSON" echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT @@ -121,13 +127,13 @@ jobs: # Scoping is best-effort too: fall back to the unscoped (full) matrix. HIL_MATRIX_JSON='' if [ -n "$SELECT" ]; then - HIL_MATRIX_JSON=$(python test/hil/hil_ci_set_matrix.py --select "$SELECT" test/hil/tinyusb.json test/hil/hfp.json) || HIL_MATRIX_JSON='' + HIL_MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py --select "$SELECT" test/hil/tinyusb.json test/hil/hfp.json) || HIL_MATRIX_JSON='' if [ -z "$HIL_MATRIX_JSON" ]; then echo "::warning::scoped HIL matrix failed - falling back to the full HIL matrix" fi fi if [ -z "$HIL_MATRIX_JSON" ]; then - HIL_MATRIX_JSON=$(python test/hil/hil_ci_set_matrix.py test/hil/tinyusb.json test/hil/hfp.json) + HIL_MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py test/hil/tinyusb.json test/hil/hfp.json) fi echo "hil_matrix=$HIL_MATRIX_JSON" echo "hil_matrix=$HIL_MATRIX_JSON" >> $GITHUB_OUTPUT @@ -338,7 +344,7 @@ jobs: strategy: fail-fast: false matrix: - # These names are the bucket keys of test/hil/hil_ci_set_matrix.py: every + # These names are the bucket keys of .github/scripts/hil_ci_set_matrix.py: every # non-esptool roster board must land in one of them (esptool boards go to # 'esp-idf', built by hil-build-esp below). hil_ci_set_matrix.py rejects a # board whose "toolchain" is not a bucket, so a new bucket must be added in @@ -379,6 +385,14 @@ jobs: hil-tinyusb: needs: [ hil-build, set-matrix ] name: hil-tinyusb (${{ matrix.display }}) + # Above hil_test.py's pool guard (HIL_POOL_TIMEOUT, 60 min) so the guard fires first + # and still gets to write its report. The 30 min on top is what the job pays OUTSIDE + # the guard clock: workspace cleanup, checkout, the multi-board artifact merge and + # the D-state note before it; kill_worker_children, shutdown_pool's 30 s grace, the + # report write and the upload after it. On a multi-stray convoy that tail alone is + # minutes, and a ceiling below guard+tail cancels the job before hil_report.md exists + # -- the inversion this branch removes. Both legs share the script and the guard. + timeout-minutes: 90 strategy: fail-fast: false matrix: @@ -395,6 +409,9 @@ jobs: test_args: '' runs-on: ${{ matrix.runner }} env: + # HIL_POOL_TIMEOUT deliberately unset: hil_test.py's 60 min default is below every + # ceiling here, so ceiling > guard holds by construction. Pin it to SHORTEN a run + # only -- pinning it above a ceiling re-inverts the two. HIL_JSON: ${{ matrix.hil_json }} steps: - name: Set HIL report dir (per run+job; persists across run attempts) @@ -482,6 +499,11 @@ jobs: needs: [ hil-build-esp, set-matrix ] name: hil-tinyusb (tinyusb-esp.json) runs-on: [ self-hosted, X64, hathach, hardware-in-the-loop ] + # above hil_test.py's pool guard (60 min) with room for the pre-pool checkout + # and the post-guard sweep + report upload, so its own guard still writes a report; + # only a job wedged past that (unkillable D-state worker) hits this ceiling, which + # must exist because the runner has one job slot and holds every queued job hostage + timeout-minutes: 90 env: HIL_JSON: test/hil/tinyusb.json TEST_ARGS: '--flasher esptool' @@ -564,7 +586,13 @@ jobs: github.repository_owner == 'hathach' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == true) runs-on: [ self-hosted, Linux, X64, hifiphile ] - timeout-minutes: 30 + # Unlike the hil-tinyusb jobs, this one BUILDS with IAR in the same job before running + # hil_test.py -- hfp.json's 3 boards, 4 variant entries, "up to 30 minutes" (see the + # comment above the selection step). The ceiling has to cover build + the 60 min pool + # guard + overhead, or GitHub cancels before the guard can write its report -- the + # inversion this branch removes. 30 + 60 = 90; the remaining 30 is the full-history + # checkout, get_deps, the post-guard sweep and the report upload. + timeout-minutes: 120 env: IAR_LMS_BEARER_TOKEN: ${{ secrets.IAR_LMS_BEARER_TOKEN }} PYTHONUNBUFFERED: '1' @@ -601,7 +629,12 @@ jobs: # failures cannot kill hfp coverage - a selector failure here must likewise # fall back to the full hfp matrix (no hil_select.json, no SEL_* vars), never # fail the job. - if ! python3 test/hil/hil_select.py --base "origin/$BASE_REF" test/hil/hfp.json > hil_select.json; then + if ! python3 test/hil/test/test_hil_select.py; then + echo "::warning::hil_select unit suite failed - running the full hfp matrix" + rm -f hil_select.json + exit 0 + fi + if ! python3 test/hil/helper/hil_select.py --base "origin/$BASE_REF" test/hil/hfp.json > hil_select.json; then echo "::warning::hil_select failed - running the full hfp matrix" rm -f hil_select.json exit 0 @@ -629,9 +662,9 @@ jobs: if: env.SEL_RUN != 'false' run: | if [ -f hil_select.json ]; then - MATRIX_JSON=$(python test/hil/hil_ci_set_matrix.py --select "$(cat hil_select.json)" test/hil/hfp.json) + MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py --select "$(cat hil_select.json)" test/hil/hfp.json) else - MATRIX_JSON=$(python test/hil/hil_ci_set_matrix.py test/hil/hfp.json) + MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py test/hil/hfp.json) fi # Each variant carries its own --build-name/--cflag, which are global to a # single build.py invocation — so keep one matrix entry per line and build @@ -648,6 +681,13 @@ jobs: - name: Build if: env.SEL_RUN != 'false' + # Bounded SEPARATELY from the job. This is the only HIL job that builds inline + # (hil-tinyusb downloads artifacts), and the job ceiling went 30 -> 120 to give the + # HIL step room -- which would hand a stalled IAR build the whole two hours on the + # shared self-hosted runner, never reaching hil_test.py or the report upload. That + # is the stranded-runner-with-no-report failure this branch exists to prevent. + # Typical full build here is a few minutes; 30 leaves generous headroom. + timeout-minutes: 30 run: | readarray -t ENTRIES < hil_build_entries.txt for entry in "${ENTRIES[@]}"; do @@ -666,7 +706,12 @@ jobs: fi # empty/absent on a non-PR event or a selector fallback -> full hfp matrix SEL_ARGS=$(cat hil_sel_args.txt 2>/dev/null || true) - python3 test/hil/hil_test.py $SEL_ARGS hfp.json + # --retry 1, like the other two HIL legs. The pool guard is a FLAT 3600s and + # does NOT scale with max_retry, so argparse's default of 3 would multiply the + # serialized usbtest tail (hfp.json runs four batteries) by three against an + # unchanged guard -- on a runner with a single job slot that queues every other + # job behind it. + python3 test/hil/hil_test.py --retry 1 $SEL_ARGS hfp.json - name: Upload HIL report if: always() && github.event_name == 'pull_request' diff --git a/.github/workflows/ci_set_matrix.py b/.github/workflows/ci_set_matrix.py deleted file mode 100755 index 50ada5964..000000000 --- a/.github/workflows/ci_set_matrix.py +++ /dev/null @@ -1,111 +0,0 @@ -#!/usr/bin/env python3 -import json - -# toolchain, url -toolchain_list = [ - "aarch64-gcc", - "arm-clang", - "arm-iar", - "arm-gcc", - "esp-idf", - "ft9xx-gcc", - "msp430-gcc", - "riscv-gcc", - "rx-gcc" -] - -# family: [supported toolchain] -family_list = { - "apm32f0xx": ["arm-gcc"], - "at32f402_405": ["arm-gcc"], - "at32f403a_407": ["arm-gcc"], - "at32f413": ["arm-gcc"], - "at32f415": ["arm-gcc"], - "at32f423": ["arm-gcc"], - "at32f425": ["arm-gcc"], - "at32f435_437": ["arm-gcc"], - "at32f45x": ["arm-gcc"], - "broadcom_32bit": ["arm-gcc"], - "broadcom_64bit": ["aarch64-gcc"], - "ch32f20x": ["arm-gcc"], - "ch32v10x": ["riscv-gcc"], - "ch32v20x": ["riscv-gcc"], - "ch32v30x": ["riscv-gcc"], - "ch583": ["riscv-gcc"], - "da1469x": ["arm-gcc"], - "fomu": ["riscv-gcc"], - "ft9xx": ["ft9xx-gcc"], - "gd32vf103": ["riscv-gcc"], - "hpmicro": ["riscv-gcc"], - "imxrt": ["arm-gcc", "arm-clang"], - "kinetis_k": ["arm-gcc"], - "kinetis_k32l": ["arm-gcc"], - "kinetis_kl": ["arm-gcc"], - "lpc11": ["arm-gcc", "arm-clang"], - "lpc13": ["arm-gcc", "arm-clang"], - "lpc15": ["arm-gcc", "arm-clang"], - "lpc17": ["arm-gcc", "arm-clang"], - "lpc18": ["arm-gcc", "arm-clang"], - "lpc40": ["arm-gcc", "arm-clang"], - "lpc43": ["arm-gcc", "arm-clang"], - "lpc51": ["arm-gcc", "arm-clang"], - "lpc54": ["arm-gcc", "arm-clang"], - "lpc55": ["arm-gcc", "arm-clang"], - "maxim": ["arm-gcc"], - "mcx": ["arm-gcc"], - "mm32": ["arm-gcc"], - "msp430": ["msp430-gcc"], - "msp432e4": ["arm-gcc"], - "nrf": ["arm-gcc", "arm-clang"], - "nuc100_120": ["arm-gcc"], - "nuc121_125": ["arm-gcc"], - "nuc126": ["arm-gcc"], - "nuc505": ["arm-gcc"], - "ra": ["arm-gcc"], - "rp2040": ["arm-gcc"], - "rw61x": ["arm-gcc"], - "rx": ["rx-gcc"], - "samd11": ["arm-gcc", "arm-clang"], - "samd2x_l2x": ["arm-gcc", "arm-clang"], - "samd5x_e5x": ["arm-gcc", "arm-clang"], - "samg": ["arm-gcc", "arm-clang"], - "stm32c0": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32c5": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32f0": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32f1": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32f2": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32f3": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32f4": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32f7": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32g0": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32g4": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32h5": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32h7": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32h7rs": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32l0": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32l4": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32n6": ["arm-gcc"], - "stm32u0": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32u5": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32wb": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32wba": ["arm-gcc", "arm-clang", "arm-iar"], - "tm4c": ["arm-gcc"], - "xmc4000": ["arm-gcc"], - # S3, P4 will be built by hil test - # "-bespressif_s3_devkitm": ["esp-idf"], - # "-bespressif_p4_function_ev": ["esp-idf"], -} - - -def set_matrix_json(): - matrix = {} - for toolchain in toolchain_list: - filtered_families = [family for family, supported_toolchain in family_list.items() if - toolchain in supported_toolchain] - matrix[toolchain] = filtered_families - - print(json.dumps(matrix)) - - -if __name__ == '__main__': - set_matrix_json() diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 70dd3894d..09f912bd3 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -30,6 +30,7 @@ jobs: #cd test/unit-test #ceedling test:all + # runs --all-files, so the hil-test hook fires here regardless of its `files:` scope - name: Run pre-commit uses: pre-commit/action@v3.0.1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e87b935dd..3d9c8482b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,6 +48,27 @@ repos: types_or: [c, header] language: system + # Two hooks, split by what each suite actually reads. The full discovery run costs + # ~55s (deliberate hang/timeout simulations); only test_hil_select (~0.1s) reads + # hw/bsp (board.cmake), src (portable dirs + class include graph) and examples + # (tusb_config.h per test) -- renaming a board, port dir or example breaks it without + # touching test/hil, and catching that here beats waiting for pre-commit CI. + # No types_or: the rig rosters (*.json) are inputs too. + # examples/device/mtp/src is in scope: test_hil_bounded parses README_TXT_CONTENT + # and md5-checks the logo header from there as its MTP fixtures. + - id: hil-test + name: hil-test + files: ^(test/hil/|examples/device/mtp/src/) + entry: python3 -m unittest discover -s test/hil/test + pass_filenames: false + language: system + - id: hil-select-test + name: hil-select-test + files: ^(hw/bsp/|src/|examples/) + entry: python3 test/hil/test/test_hil_select.py + pass_filenames: false + language: system + # - id: build-fuzzer # name: build-fuzzer # files: ^(src/|test/fuzz/) diff --git a/CLAUDE.md b/CLAUDE.md index 94b8192b7..fd4b9b8e0 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -99,7 +99,8 @@ Use the `pvs` skill (`.claude/skills/pvs/SKILL.md`) — it builds the examples w ## Validation After Changes -1. `pre-commit run --all-files` — format, spell, unit tests (10-15 s). +1. `pre-commit run --all-files` — format, spell, unit tests, HIL suites (~55 s; the + HIL hooks deliberately exercise real timeouts and hangs). 2. Build at least one board's full example set (Build → "All examples for a board") for modules you touched. 3. Run relevant unit tests; add fuzz/HIL coverage for parsers or protocol state machines. diff --git a/docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md b/docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md index 8158758bc..898b3c8ab 100644 --- a/docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md +++ b/docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md @@ -1,4 +1,4 @@ -# PR-scoped HIL selection: hil_select.py +# PR-scoped HIL selection: helper/hil_select.py **Date:** 2026-07-29 **Branch:** `claude/hil-select` (based on `claude/hil-pool-check`, which carries the @@ -26,17 +26,17 @@ confident; every uncertainty widens to the full matrix. - Scoping push/master/scheduled runs (always full). - Changing hil_test.py behavior (the selector only *composes* existing `-b`/`-bt` args). -## Component: `test/hil/hil_select.py` +## Component: `test/hil/helper/hil_select.py` Stdlib-only, importable and CLI. Lives beside the harness so `hil_ci.sh` copies are unaffected (it runs on the GitHub runner / dev PC, not on the rig). It must NOT import `hil_test.py` (which drags pyserial/pymtp onto the bare GitHub runner): the three test lists -(`device_tests`, `dual_tests`, `host_test`) move verbatim into a tiny stdlib-only -`test/hil/hil_examples.py` that both `hil_test.py` and `hil_select.py` import (behavior -preserving; `hil_ci.sh` scp list gains the new file). +(`device_tests`, `dual_tests`, `host_test`) move verbatim into the stdlib-only +`test/hil/helper/hil_util.py` that both `hil_test.py` and `hil_select.py` import (behavior +preserving; `hil_ci.sh` copies the whole `helper/` directory). ``` -python3 test/hil/hil_select.py --base [--diff-file ] CONFIG.json [CONFIG.json...] +python3 test/hil/helper/hil_select.py --base [--diff-file ] CONFIG.json [CONFIG.json...] ``` - `--base REF`: changed files = `git diff --name-only $(git merge-base HEAD REF)..HEAD` @@ -118,7 +118,7 @@ is skipped, not widened (running unrelated boards would test nothing relevant). ## CI wiring (`.github/workflows/build.yml`) - `set-matrix` (PR events only): after generating today's matrices, run - `hil_select.py --base origin/${{ github.base_ref }} test/hil/tinyusb.json test/hil/hfp.json` + `helper/hil_select.py --base origin/${{ github.base_ref }} test/hil/tinyusb.json test/hil/hfp.json` (checkout with enough history to reach the merge base: `fetch-depth: 0` on this one job, or an explicit `git fetch origin $BASE_REF`). New job outputs: `hil_select_full`, `hil_args_tinyusb`, `hil_args_hfp`, plus the selected-board list consumed by the matrix @@ -137,15 +137,15 @@ is skipped, not widened (running unrelated boards would test nothing relevant). ## Local use - pre-pr's "Map changes to boards" step delegates to - `python3 test/hil/hil_select.py --base $BASE test/hil/tinyusb.json` and derives its + `python3 test/hil/helper/hil_select.py --base $BASE test/hil/tinyusb.json` and derives its one-board-per-family sample from the selector's board set (its capping/sampling policy is unchanged — the selector provides the affected set, pre-pr samples it). -- Manual: `python3 test/hil/hil_test.py -B examples $(python3 test/hil/hil_select.py --base master test/hil/tinyusb.json | jq -r '.args["tinyusb.json"]') test/hil/tinyusb.json` +- Manual: `python3 test/hil/hil_test.py -B examples $(python3 test/hil/helper/hil_select.py --base master test/hil/tinyusb.json | jq -r '.args["tinyusb.json"]') test/hil/tinyusb.json` — documented in the hil skill. ## Testing -`test/hil/test_hil_select.py` — stdlib `unittest`, no hardware, injected diffs via +`test/hil/test/test_hil_select.py` — stdlib `unittest`, no hardware, injected diffs via `--diff-file`/API. Cases (the acceptance examples): 1. `src/portable/raspberrypi/rp2040/dcd_rp2040.c` → only rp2040-family roster boards, device tests only, host-only boards absent, `full` false. @@ -161,7 +161,7 @@ is skipped, not widened (running unrelated boards would test nothing relevant). 7. `hw/bsp/rp2040/family.cmake` → rp2040-family boards, all their tests. 8. Mixed device+host diff → no pruning (both roles present). The suite runs in `set-matrix` before the selector is used, and locally via -`python3 test/hil/test_hil_select.py`. +`python3 test/hil/test/test_hil_select.py`. ## Safety properties diff --git a/hw/bsp/mcx/family.cmake b/hw/bsp/mcx/family.cmake index 60f43e152..b2b4fd45b 100644 --- a/hw/bsp/mcx/family.cmake +++ b/hw/bsp/mcx/family.cmake @@ -95,7 +95,7 @@ function(family_configure_example TARGET RTOS) endif() # PORT is set per board (board.cmake), so pick the driver at configure time. Spelled out - # rather than $ so the port path stays greppable: test/hil/hil_select.py + # rather than $ so the port path stays greppable: test/hil/helper/hil_select.py # maps a portable-driver change to the families whose build file names that directory. if (PORT) set(PORT_SRC ${TOP}/src/portable/chipidea/ci_hs/dcd_ci_hs.c) diff --git a/test/hil/helper/__init__.py b/test/hil/helper/__init__.py new file mode 100644 index 000000000..a080a2f55 --- /dev/null +++ b/test/hil/helper/__init__.py @@ -0,0 +1,4 @@ +# Marks helper/ as a REGULAR package. Without this it is only a PEP 420 namespace portion, +# and a regular package named `helper` anywhere on sys.path wins over it even though +# test/hil is sys.path[0] -- one transitive pip install would break every HIL entry point +# at import. `helper` is a real distribution name on PyPI. diff --git a/test/hil/helper/hil_health.py b/test/hil/helper/hil_health.py new file mode 100644 index 000000000..92f0accc8 --- /dev/null +++ b/test/hil/helper/hil_health.py @@ -0,0 +1,380 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Shutting a wedged HIL run down: kill what the workers spawned, then report. + +A device whose usbfs node is held by a D-state process cannot be freed -- SIGKILL is not +delivered in uninterruptible sleep -- so the goal is never to fix the rig from here. It is +to free the runner's single job slot and leave a report naming what survived, instead of +letting the job sit until GitHub cancels it with nothing to show. + +Deliberately shallow. We SIGKILL the process groups the workers spawned, wait a grace, +and report whoever is still alive; we do not re-scan groups, prove pid ownership or +escalate through sudo. A root-owned survivor is named in the report for hil_pool_check +and the usb-kernel-recover skill to deal with -- signalling a pid we cannot prove is ours +is the worse failure, and the job ceiling backstops whatever this misses. + +Everything here is stdlib-only and reads /proc unprivileged (dmesg is restricted on the +rig), which keeps it importable -- and testable -- on a bare runner. +""" +import os +import signal +import threading +import time +from pathlib import Path + +PROC = Path('/proc') + + +# How long to let a SIGKILL land before calling a process a survivor. Generous enough to +# cover scheduling delay on a loaded rig, short enough that a fleet-wide sweep stays quick. +CONFIRM_KILL_GRACE = 2.0 + + +def _p(*args, **kwargs) -> None: + # These run on the free-the-runner path, where stdout can already be a dead pipe (a + # dropped ssh session). An unguarded print would raise BrokenPipeError out of + # hil_test's inner finally, skipping shutdown_pool AND the report writing. + try: + print(*args, **kwargs) + except (OSError, ValueError): + # ValueError, not just OSError: printing to a CLOSED stream raises + # "ValueError: I/O operation on closed file", and both hil_pool_check and + # hil_test redirect stdout into a StringIO that can be closed under us. Escaping + # here skips shutdown_pool/kill_pool_children/os._exit -- stranding the runner, + # the exact failure this wrapper exists to prevent. + pass + + +def _state(pid_dir: Path) -> str: + """The state letter from /proc//stat. comm can contain ')', so the field is + located from the right rather than by splitting.""" + # bytes, not read_text(): read_text decodes with the LOCALE encoding, so under LANG=C + # (systemd services, self-hosted runners) a non-ASCII comm raises UnicodeDecodeError + # and the entry silently vanishes from the scan. + stat = (pid_dir / 'stat').read_bytes() + return chr(stat[stat.rindex(b')') + 2]) + + +def _pids(): + """/proc pid entries. Yields nothing rather than raising if /proc is unreadable.""" + try: + entries = list(PROC.iterdir()) + except OSError: + return + for entry in entries: + if entry.name.isdigit(): + yield entry + + +def d_state_note() -> str: + """Pids in uninterruptible sleep, for the report. Never aborts, never blocks. + + A D-state process at start-up is NOT a fault on its own -- a healthy in-flight testusb + looks exactly like this, and the rig supports a dev run alongside CI. It is a hint for + whoever reads a red cell below. Diagnosis proper is hil_pool_check and the + usb-kernel-recover skill; this is one line, not a probe.""" + stuck = [] + for d in PROC.glob('[0-9]*'): + try: + if _state(d) == 'D': + stuck.append(d.name) + except (OSError, ValueError, IndexError): + pass # raced with exit, or /proc is restricted: not our problem here + if not stuck: + return '' + return (f'{len(stuck)} process(es) in D state when this run started: ' + f'{sorted(stuck)[:10]}') + + +def shutdown_pool(pool, grace: float = 30) -> bool: + """terminate() a worker Pool without ever blocking forever. + + multiprocessing joins its workers unbounded (util.py _exit_function terminate()s the + daemonic ones, then calls p.join() -- no timeout -- on every remaining active child, + CPython 3.13.5), and a worker in uninterruptible sleep never + reaps -- so terminate() itself hangs, taking the runner's only job slot with it. False + when the pool refuses to die within `grace` (the caller must then abandon it); a + terminate() that *raises* counts as failure too, the pool being just as alive.""" + outcome = {} + + def _term(): + try: + pool.terminate() + outcome['ok'] = True + except BaseException as e: # noqa: BLE001 - any failure means the pool is still up + # Say what happened: Pool._terminate_pool really can raise (CPython: + # AssertionError 'Cannot have cache with result_handler not alive'), and a + # swallowed one is indistinguishable from an unkillable D-state worker. + outcome['err'] = e + _p(f'warning: Pool.terminate() raised {type(e).__name__}: {e}', flush=True) + + t = threading.Thread(target=_term, daemon=True) + t.start() + t.join(grace) + # Decide on the thread, not the dict: _term may set outcome['ok'] after join(grace) + # expired, reporting a merely-slow terminate as success on one read and abandoned on + # another. Still inside terminate() == not shut down. + if t.is_alive(): + return False + return outcome.get('ok', False) + + +def child_procs(pids) -> dict: + """{ancestor pid in `pids`: [(descendant pid, its pgid), ...]}, from ONE walk of /proc. + + DESCENDANTS, not direct children: a worker's usbtest.py spawns its recovery flasher + through run_cmd (own session), so it is a GRANDCHILD that a direct-child sweep misses + and a kill mid-recovery would orphan on the probe. pgid comes back too because the two + kinds of child need different signals (see kill_pool_children).""" + wanted = set(pids) + by_parent: dict = {} # ppid -> [(pid, pgid), ...] for EVERY process + for entry in _pids(): + try: + stat = (entry / 'stat').read_bytes() + except OSError: + continue # exited between the scan and the read, or not readable + # comm (field 2) is parenthesised and may contain spaces and ')' -- so split only + # what follows the LAST ')': state, ppid, pgrp, ... + try: + fields = stat[stat.rindex(b')') + 2:].split() + ppid, pgid = int(fields[1]), int(fields[2]) + except (ValueError, IndexError): + continue # truncated or unparsable stat line + by_parent.setdefault(ppid, []).append((int(entry.name), pgid)) + out: dict = {} + for root in wanted: + todo = list(by_parent.get(root, [])) + while todo: + pid, pgid = todo.pop() + out.setdefault(root, []).append((pid, pgid)) + todo += by_parent.get(pid, []) + return out + + +def _pool_procs(pool, extra) -> list: + """The pool's worker Process objects, plus each extra's own process. + + Manager() runs in its own child process and inherits the same descriptors as the + workers, so leaving it behind defeats the point: os._exit skips its finalizer.""" + procs = list(getattr(pool, '_pool', []) or []) + for e in extra: + procs.append(getattr(e, '_process', e)) + return procs + + + + +def kill_worker_children(pool, *extra) -> int: + """SIGKILL what the pool's workers spawned; returns how many SURVIVED. + + For the TIMEOUT path only. On the normal path each worker has already run + kill_own_children() and retired (maxtasksperchild=1), so this walks fresh idle workers + and finds nothing -- measured: 4 tasks, zero overlap with the pool at sweep time. + + Call it BEFORE shutdown_pool(): terminate() reaps the (interruptible) worker and its + flasher is reparented to init, erasing the ppid link this matches on. Signalling the + worker's own group instead cannot work -- a forked pool worker inherits OUR group + (CPython 3.13.5 multiprocessing never setsid/setpgid) and run_cmd gives every flasher + a session of its own. + + TWO passes because our SIGKILL can fail an in-flight flash and the worker then retries + in a fresh session, which one /proc snapshot misses. `seen` stops a pid signalled in + pass 1 being confirmed twice. + """ + seen: set = set() + total = 0 + for i in range(2): + if i: + time.sleep(0.5) + procs = _pool_procs(pool, extra) + total += _kill_kids( + child_procs(getattr(p, 'pid', None) for p in procs if p is not None), seen) + return total + + +def kill_own_children() -> int: + """SIGKILL what THIS process spawned. Returns how many survived. + + For the worker to call before it returns. maxtasksperchild=1 retires it the moment the + task ends, reparenting its children to init, so main()'s sweep walks fresh idle workers + and finds nothing (measured over 4 tasks: zero overlap, sweep 0, 4 strays alive). + Inside the worker the ppid link is still live. + """ + return _kill_kids(child_procs([os.getpid()]), set()) + + +def _kill_kids(kids: dict, seen: set) -> int: + """SIGKILL every pid in a ppid-tree snapshot; return how many survived. + + Every pid here is a DESCENDANT of a process we own, so it is ours by construction -- no + argv identity check, because we never signal anything we did not discover through our + own ppid tree. + """ + try: + own = os.getpgid(0) + except OSError: + own = None # cannot tell our own group apart: never killpg, signal pids only + # One list: every pid here is a DESCENDANT of one of our own workers, so it is ours by + # construction -- no argv identity check needed, because we never signal anything we + # did not discover through our own ppid tree. + touched: list = [] + for children in kids.values(): + for cpid, cpgid in children: + if cpid in seen: + continue # a previous pass already signalled it + seen.add(cpid) + try: + if own is not None and cpgid != own: + # A run_cmd child: its own session, so one killpg also reaps what it + # spawned. Recorded because killpg cannot report a partial kill. + os.killpg(cpgid, signal.SIGKILL) + else: + # Shares our group (a plain subprocess.run), so killpg would take + # down the whole run -- it is signalled by pid in _kill_and_confirm. + pass + touched.append(cpid) + except PermissionError: + # NOT "already gone": the signal did not land, so this pid MUST still be + # confirmed, or the one case this handler exists for (an all-root session: + # the sudo wrapper died, its root members did not) is the one case that + # never reaches the report. + touched.append(cpid) + except ProcessLookupError: + pass # already gone + except OSError: + pass + # Both paths need confirming: a killpg'd flasher and a same-group mtype blocked on a + # wedged device are both in D state, and os.kill reported success on either. + denied = _kill_and_confirm(touched) + if denied: + _p(f'warning: could not kill {sorted(denied)}; they still hold whatever they ' + f'had open (probe, usbfs node) into the next job', flush=True) + # SURVIVORS, not the signalled-child count: the caller needs to know the rig is dirty + # for the next job, and a count of what we successfully signalled cannot tell it that. + # (They are different units anyway -- a killpg is counted once per child sharing the + # group -- so the old return was never comparable to anything.) + return len(denied) + + +def _kill_and_confirm(pids) -> list: + """SIGKILL every pid, then return those STILL alive after ONE grace window. + + SIGKILL is QUEUED, not delivered, for a task in uninterruptible sleep -- and testusb + waits in a plain wait_for_completion() with no timeout (v6.12.96 usbtest.c:1404; + usb_sg_wait, message.c:765), so that is the normal state of a healthy in-flight case + too. os.kill returning success proves nothing; only the recheck does. It is also + asynchronous, so probing immediately reports a process we just killed as a survivor + (measured: 11 of 20 plain `sleep`s with no grace). + + Signal all, then poll the set against ONE shared deadline: per-pid windows made this + scale with stray count, minutes on a convoy. A pid we cannot signal is reported, never + sudo-killed. + """ + pending = [] + for pid in pids: + try: + os.kill(pid, signal.SIGKILL) + except ProcessLookupError: + continue # already gone + except OSError: + pass # EPERM (root-owned): it stays, and the poll below reports it + pending.append(pid) + + deadline = time.monotonic() + CONFIRM_KILL_GRACE + while True: + alive = [] + for pid in pending: + try: + os.kill(pid, 0) + except ProcessLookupError: + continue # ESRCH: genuinely gone + except OSError: + pass # EPERM: it exists; the state check decides + try: + # a ZOMBIE answers kill(pid, 0) too: dead, merely unreaped. Not a survivor. + if _state(PROC / str(pid)) == 'Z': + continue + except (OSError, ValueError, IndexError): + continue # unreadable: assume gone rather than cry wolf + alive.append(pid) + pending = alive + if not pending or time.monotonic() >= deadline: + return pending # outlasted SIGKILL: D state, or not ours to kill + time.sleep(0.02) + + +def kill_pool_children(pool, *extra) -> int: + """SIGKILL the pool's worker processes themselves. Returns how many are STILL ALIVE + after the grace -- not how many were signalled. + + Survivors, not signals: the caller turns this number into "power-cycle the host", so + counting signals would send someone to a hypervisor over workers that all died. + + Call after a shutdown_pool() that returned False, and after kill_worker_children(). + A D-state worker ignores SIGKILL, but every other worker dies and drops the inherited + descriptors -- a survivor holds the runner's stdout pipe open and the runner waits for + EOF even after we exit, so the early exit would not free the job slot.""" + killed_procs: list = [] + for proc in _pool_procs(pool, extra): + try: + # Process.kill(), never a raw pid: multiprocessing's _send_signal re-checks + # `self.returncode is None` first, so once shutdown_pool's thread has reaped a + # worker this is a no-op instead of signalling a pid the OS may have recycled. + # os.pidfd_open(proc.pid) is worse: it skips that guard entirely. + if proc is None or not proc.is_alive(): + continue + proc.kill() + killed_procs.append(proc) + except (OSError, AttributeError, ValueError): + continue # already reaped, never started, or not a real process + # Re-check the Process objects, never the pids collected a moment ago: shutdown_pool's + # thread is STILL join()ing workers, so a pid killed here can be reaped and RECYCLED + # before _kill_and_confirm signals it -- and on EPERM that escalates to `sudo -n kill + # -9 `, killing an unrelated ROOT process as the last act before os._exit. + killed_pids = [] + for proc in killed_procs: + try: + if proc.is_alive() and proc.pid is not None: + killed_pids.append(proc.pid) + except (OSError, AttributeError, ValueError): + continue + # SIGKILL is asynchronous and a D-state task ignores it: only a confirmed survivor + # justifies the caller's power-cycle wording + return len(_kill_and_confirm(killed_pids)) if killed_pids else 0 + + +def write_timeout_report(report_dir: Path, boards, secs: int, md_name: str, + banner: str = '', prefix: str = '') -> None: + """Leave a report behind when the worker pool has to be abandoned. + + map_async is all-or-nothing, so a timeout loses every per-board result and the report + dir would stay empty with no reason for the failure. Any prior attempt's markdown is + kept below the banner.""" + # `prefix` carries the preflight rig-health verdict: the timeout aborts before + # accumulate_report, so without it the report loses the one line saying WHY the pool + # never finished. The '\n' stops Markdown lazy continuation pulling the banner into + # the blockquote. + try: + # Built INSIDE the try: a roster entry without a 'name' key raises KeyError while + # assembling the board list, and outside the try that escaped and stranded the + # runner -- which is exactly what the broad handler below exists to prevent. + head = (prefix + '\n' if prefix else '') + (banner or ( + f'**HIL run abandoned: worker pool timed out after {secs}s.**\n\n' + f'No per-board results could be collected for this attempt, so the ' + f'table below (if any) is from an earlier one. Boards dispatched:\n\n' + + '\n'.join(f'- {b.get("name", "?")}' for b in boards) + '\n')) + report_dir.mkdir(parents=True, exist_ok=True) + md_path = report_dir / md_name + # Its own handler so it cannot take the write down with it: a report torn by an + # attempt killed mid-write raises UnicodeDecodeError (a ValueError, and prior + # reports always contain status emoji), which under a shared try skipped the write + # entirely. Losing the old table is a nicety; losing the banner is the failure. + try: + prior = md_path.read_text(encoding='utf-8') if md_path.is_file() else '' + except (OSError, ValueError): + prior = '' + md_path.write_text(head + (f'\n{prior}' if prior else ''), encoding='utf-8') + except Exception as e: # noqa: BLE001 + # Deliberately broad: this is the first statement of the pool-abandon path, so ANY + # escape skips kill_pool_children and os._exit and strands the runner. + _p(f'warning: cannot write {md_name} to {report_dir}: {e}', flush=True) diff --git a/test/hil/helper/hil_lock.py b/test/hil/helper/hil_lock.py new file mode 100755 index 000000000..7757ef17d --- /dev/null +++ b/test/hil/helper/hil_lock.py @@ -0,0 +1,525 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""Board locks + controller permits for the TinyUSB HIL rig. + +Board locks are kernel flocks in BOARD_LOCK_DIR arbitrating hardware access +between dev sessions and CI's hil_test.py (never stop the actions-runner). +Controller permits are in-process semaphores budgeting flashes and usbtest +batteries per host controller; they have no CLI meaning. The CLI below +(hold/release/status) manages board locks only. +""" +import argparse +import fcntl +import json +import os +import re +import select +import signal +import sys +import time + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # helper/ scripts import via the test/hil root +from helper import hil_util + +BOARD_LOCK_DIR = '/tmp/tinyusb-hil-locks' +CI_REASON = 'hil_test.py' # release-protected holder tag (release refuses to kill it) +PROTECTED_REASONS = {CI_REASON, 'pool_check'} # cmd_release refuses to SIGTERM these holders +PROFILE = os.environ.get('HIL_PROFILE') == '1' + + +def lock_path(board: str) -> str: + return os.path.join(BOARD_LOCK_DIR, f'{board}.lock') + + +def flock_nb(board: str): + """Open-or-create the lock file WITHOUT truncating (a losing racer must not + wipe the winner's record) and take LOCK_EX|LOCK_NB. Returns the open handle; + raises OSError when the flock is held elsewhere (handle already closed).""" + fd = os.open(lock_path(board), os.O_RDWR | os.O_CREAT, 0o666) + fh = os.fdopen(fd, 'r+') + try: + fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + fh.close() + raise + return fh + + +def write_record(fh, reason: str) -> bool: + """Holder record; the flock itself is already held. Returns False on a write failure: + acquire_board_lock stays best-effort (the flock is the authority), but cmd_hold aborts + -- a hold whose record is missing is invisible to status/release.""" + try: + fh.truncate(0) + fh.seek(0) + json.dump({'pid': os.getpid(), 'reason': reason, + 'since': time.strftime('%Y-%m-%dT%H:%M:%S%z')}, fh) + fh.flush() + return True + except OSError: + return False + + +def clear_record(fh) -> None: + """Clear our record before dropping the flock so records stay truthful.""" + try: + fh.truncate(0) + except OSError: + pass + + +def read_record(board: str): + try: + with open(lock_path(board)) as f: + return json.load(f) + except (OSError, ValueError): + return None + + +# --- per-board dev-session locks ------------------------------------------ +def acquire_board_lock(board_name, reason=CI_REASON): + """Take this board's flock for the duration of its flash+test. + Returns an open file handle (keep it referenced; closing releases it), + or None when HIL_NO_BOARD_LOCK=1 or the lock dir is unusable (fail-open: + locking must never break a test run by itself). + Raises RuntimeError only when another session holds the board.""" + import fcntl + if os.environ.get('HIL_NO_BOARD_LOCK') == '1': + return None # user-authorized bypass — see hil skill + try: + os.makedirs(BOARD_LOCK_DIR, exist_ok=True) + fd = os.open(os.path.join(BOARD_LOCK_DIR, f'{board_name}.lock'), + os.O_RDWR | os.O_CREAT, 0o666) + fh = os.fdopen(fd, 'r+') + except OSError as e: + # odd lock dir (perms, path collision): proceed unlocked, but say so — + # a silent fail-open is indistinguishable from the intentional bypass + print(f'warning: board lock unavailable for {board_name} ({e}); proceeding unlocked', + flush=True) + return None + try: + fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + try: + info = fh.read(500).strip() + except (OSError, UnicodeDecodeError): + info = '' + fh.close() + raise RuntimeError(f'board locked: {info or "unknown holder"}') + # announce ourselves so the other side's conflict message is truthful; + # best-effort — the flock itself is already held + try: + fh.truncate(0) + fh.seek(0) + json.dump({'pid': os.getpid(), 'reason': reason, + 'since': time.strftime('%Y-%m-%dT%H:%M:%S%z')}, fh) + fh.flush() + except OSError: + pass + return fh + + +# Per-host-controller concurrency (see controller_of/controller_slot below): a usbtest +# battery saturates its DUT's host controller, so batteries and flashes are budgeted per +# controller. The 4/2 defaults trade ~3.5 min on the usbtest leg for bandwidth margin on +# the shared leaf-hub uplinks, where battery case failures were observed from 12/8 +# (profiled 2026-07-13/14: 22.2/14.3/12.5/10.8 min at usbtest width 1/2/3/4, plateau +# after). Raise per run via HIL_FLASH_PARALLEL/HIL_USBTEST_PARALLEL. +# - uPD720201 cards need firmware >= 2.0.2.6 (RAM-uploaded, reloads every power cycle): +# the ROM firmware dies under battery + re-enumeration churn. +# - a marginal DUT port bouncing during concurrent batteries can kill a uPD720201 ("xHCI +# host not responding to stop endpoint command"): fix the port/cable or pull the board +# -- lowering the widths does not fix a bad port (2026-07-16, every death). +FLASH_PARALLEL = hil_util.pos_int_env('HIL_FLASH_PARALLEL', 4) +USBTEST_PARALLEL = hil_util.pos_int_env('HIL_USBTEST_PARALLEL', 2) +CONTROLLER_SLOTS = 12 # lock slots; controllers are assigned to slots on first sight +# Bound on ONE permit wait. Generous: a real queue behind a slow board is normal, +# and this only has to beat the pool guard so a leaked permit cannot consume it. +PERMIT_TIMEOUT = hil_util.pos_int_env('HIL_PERMIT_TIMEOUT', 900) +# CONTROLLER_SLOTS + 1 entries each, built by make_permit_sems: UNKNOWN_SLOT indexes the +# extra one. Sized to CONTROLLER_SLOTS instead, the first unresolved board IndexErrors +# inside a pool worker -- which now surfaces through drain_pool as a worker-raise (the +# finished boards survive), but still loses this board and aborts the run. +usbtest_sems = None # per-slot usbtest-battery permits +flash_sems = None # per-slot flash permits +controller_map = None # shared dict: 'pci:' -> slot, 'uid:' -> pci addr cache +controller_meta = None # guards slot assignment in controller_map +controller_hints = {} # static uid -> pci from the last run's cache (read-only per worker) + + +log = print # hil_test.init_worker points this at log_line via init_scheduling + + +def init_scheduling(b_sems, f_sems, cmap, cmeta, hints, log_fn=None): + """Install per-worker scheduling state (called from hil_test.init_worker).""" + global usbtest_sems, flash_sems, controller_map, controller_meta, controller_hints, log + usbtest_sems, flash_sems = b_sems, f_sems + controller_map, controller_meta, controller_hints = cmap, cmeta, hints + if log_fn is not None: + log = log_fn + + +# ------------------------------------------------------------- +# Per-controller scheduling +# ------------------------------------------------------------- +def controller_of(uid: str): + """Resolve a DUT uid to its root host controller's PCI address, or None when it cannot + be resolved — the device is not enumerated (e.g. parked in board_test firmware with USB + off), or sysfs would not answer. Successful resolutions are cached — cabling does not + change mid-run. Dual-port parts (e.g. CH32V307 usbhs/usbfs variants) share one uid and + one cache entry: budgeting is only exact when both ports sit on the same controller + (true on this rig).""" + if controller_map is None: + return None + cached = controller_map.get(f'uid:{uid}') + if cached: + return cached + # vid='cafe' first: the target is always a TinyUSB DUT, and the VID is a lock-free + # descriptor field. Without it this read every probe's and hub's `serial` -- the + # attribute served under device_lock -- so a HEALTHY peer mid-usbtest would strand a + # reader here and spend one of this worker's four blindness credits. + devs, _ = hil_util.usb_scan(vid='cafe', serial=uid) + for dev in devs: + busnum = hil_util.read_sysfs(os.path.join(dev['dir'], 'busnum')) + if busnum is None or busnum is hil_util.SYSFS_UNKNOWN: + continue + try: + root = os.path.realpath(f'/sys/bus/usb/devices/usb{int(busnum)}') + except ValueError: + continue + m = re.findall(r'[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]', root) + if m: + controller_map[f'uid:{uid}'] = m[-1] + return m[-1] + return None + + +def controller_slot(pci: str) -> int: + """Map a controller PCI address to a lock slot (assigned on first sight).""" + key = f'pci:{pci}' + with controller_meta: + slot = controller_map.get(key) + if slot is None: + slot = controller_map.get('nslots', 0) + if slot >= CONTROLLER_SLOTS: + slot = 0 # more controllers than slots: overflow shares slot 0 (safe, over-serialized) + else: + controller_map['nslots'] = slot + 1 + controller_map[key] = slot + return slot + + +# Unresolved boards budget in a slot of their OWN, one past the real ones, and that slot +# holds exactly ONE permit whatever the per-controller width is. Neither neighbour works: +# a permit on every slot (the old fail-closed rule) serialized the whole fleet the moment +# a worker went blind, while a full private budget let unknown boards run a second +# controller's worth of batteries on top of the resolved ones -- doubling the load on +# whichever physical controller they actually sit on, which is the saturation the +# uPD720201 deaths above are attributed to. Width 1 caps the over-subscription at +1. +UNKNOWN_SLOT = CONTROLLER_SLOTS + + +def make_permit_sems(semaphore, width: int) -> list: + """One semaphore per controller slot at `width`, plus the unknown bucket at 1.""" + return [semaphore(width) for _ in range(CONTROLLER_SLOTS)] + [semaphore(1)] + + +class controller_permit: + """Context manager: one permit from `sems` on the board's controller slot. An + unresolved controller budgets in UNKNOWN_SLOT, which admits one at a time: unresolved + boards serialize against each other, never against the whole rig, and never add a + second full budget to a controller. `warn_unknown` logs that fallback (used by + usbtest, where the device is expected to be enumerated by the caller).""" + def __init__(self, sems, uid: str, warn_unknown: bool = False): + self.sems = sems + self.slots = None + self.uid = uid + # what __enter__ actually ACQUIRED. Not the same as self.slots: a bounded acquire + # that times out is skipped on purpose, and releasing it anyway would add a permit + # that was never taken -- multiprocessing semaphores are unbounded, so the width + # grows for the rest of the run, on the controller throttle that exists to keep + # concurrent batteries from killing the uPD720201 xHCI. + self.taken: list = [] + if sems is None: + return + # Hint FIRST for flash budgeting: a mis-budgeted flash is harmless, and the board + # is usually parked in board_test with USB off at this point, so controller_of + # cannot resolve it anyway -- it just walks the whole bus to say so, once per + # flash permit (~14 examples x ~21 boards a leg), each walk spawning a bounded + # reader per device. usbtest still resolves for real (warn_unknown), and by then + # the DUT is enumerated, so that walk succeeds and caches. + pci = None if warn_unknown else controller_hints.get(uid) + if pci is None: + pci = controller_of(uid) + if pci is None and warn_unknown: + log(f'warning: cannot resolve {uid} to a host controller' + f'{hil_util.sysfs_blind_note()}; budgeting it in the unknown bucket') + self.slots = [controller_slot(pci) if pci else UNKNOWN_SLOT] + + def __enter__(self): + if self.slots: + t0 = time.monotonic() + taken = self.taken = [] + try: + for s in self.slots: + # BOUNDED. multiprocessing semaphores are NOT released when a holder + # dies, and the pool sweep SIGKILLs workers -- so a permit lost that + # way would block every later worker on this controller forever, and + # boards unrelated to the wedge would burn the whole pool guard. On + # expiry proceed over-subscribed and say so: a slower controller is a + # far better failure than a hung run. + if not self.sems[s].acquire(timeout=PERMIT_TIMEOUT): + log(f'warning: waited {PERMIT_TIMEOUT}s for a permit on slot {s} ' + f'(uid {self.uid}); a holder probably died without releasing ' + f'it -- proceeding over-subscribed') + continue + taken.append(s) + # inside the try: a failed __enter__ never gets its __exit__, so a raise + # here (e.g. broken stdout) must still release the permits + if PROFILE and time.monotonic() - t0 > 1.0: + log(f'[prof] permit wait {time.monotonic() - t0:.1f}s ' + f'(uid {self.uid}, slots {self.slots})') + except BaseException: + for s in reversed(taken): + self.sems[s].release() + raise + return self + + def __exit__(self, *exc): + if self.slots: + for s in reversed(self.taken): + self.sems[s].release() + self.taken = [] + return False + + +def flash_permit(uid: str) -> controller_permit: + return controller_permit(flash_sems, uid) + + +def usbtest_permit(uid: str) -> controller_permit: + return controller_permit(usbtest_sems, uid, warn_unknown=True) + + +# --- operator CLI (hold/release/status) ------------------------------------ +def boards_from_config(config: str) -> list: + """All board names, INCLUDING boards-skip: `hold --all` guards rig-wide + operations, and parked boards can still be touched (pool_check -b names them + explicitly), so a rig-wide hold that skipped them would leave a gap.""" + try: + with open(config) as f: + cfg = json.load(f) + return [b['name'] for b in cfg['boards'] + cfg.get('boards-skip', [])] + except (OSError, ValueError, KeyError) as e: + print(f'ERROR: cannot read board roster {config}: {e}', file=sys.stderr) + sys.exit(1) + + +def is_locked(board: str) -> bool: + """True if the recorded holder process is still alive. + + Deliberately never touches the flock: even a momentary probe lock would + make a concurrent acquirer's LOCK_NB attempt fail spuriously. The flock + taken by acquirers themselves stays the only authority.""" + info = read_record(board) + pid = info.get('pid') if isinstance(info, dict) else None + if not isinstance(pid, int) or pid <= 0: + return False + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True # alive but owned by another user (e.g. the CI runner) + return True + + +def cmd_hold(boards, reason): + os.makedirs(BOARD_LOCK_DIR, exist_ok=True) + # No pre-check: the holder's own LOCK_NB flock is the only authority, since a recorded + # pid may be stale or recycled. The holder signals success through this pipe because a + # generic is_locked() poll would be fooled by a RIVAL invocation's flock — only the + # holder knows whether it won every board. + r_fd, w_fd = os.pipe() + pid = os.fork() + if pid > 0: + os.close(w_fd) + os.waitpid(pid, 0) # reap intermediate child + ready, _, _ = select.select([r_fd], [], [], 10) + ok = bool(ready) and os.read(r_fd, 1) == b'1' + os.close(r_fd) + if ok: + print(f'held: {", ".join(boards)}') + return 0 + for b in boards: + info = read_record(b) + if info: + print(f'ERROR: {b} locked: {info}', file=sys.stderr) + print('ERROR: holder failed to acquire locks', file=sys.stderr) + return 1 + # intermediate child: detach, then spawn the actual holder + os.setsid() + if os.fork() > 0: + os._exit(0) + # holder (grandchild): acquire all flocks, signal the parent, sleep until killed + os.close(r_fd) + # Keep the success pipe clear of fds 0-2: invoked with stdio closed, os.pipe() can + # land there and the dup2 loop below would clobber it. + if w_fd <= 2: + w_fd = fcntl.fcntl(w_fd, fcntl.F_DUPFD, 3) + # Detach stdio: a `hold` whose output is captured must see EOF when the front-end + # exits — the immortal holder must not keep that pipe open. + devnull = os.open(os.devnull, os.O_RDWR) + for std_fd in (0, 1, 2): + os.dup2(devnull, std_fd) + if devnull > 2: + os.close(devnull) + try: + handles = [] + for b in boards: + fh = flock_nb(b) + if not write_record(fh, reason): + raise OSError(f'cannot write holder record for {b}') + handles.append(fh) + except OSError: + try: + os.write(w_fd, b'0') + except OSError: + pass + os._exit(1) # lost a race; parent reports the failure + os.write(w_fd, b'1') + os.close(w_fd) + + def _bow_out(*_): + # clear the records before dying so read_record/status stay truthful (the kernel + # drops the flocks themselves on exit either way) + for h in handles: + clear_record(h) + os._exit(0) + + signal.signal(signal.SIGTERM, _bow_out) + while True: + signal.pause() + + +def cmd_release(boards): + rc = 0 + victims = set() + for b in boards: + try: + fd = os.open(lock_path(b), os.O_RDWR) + except OSError: + continue # no lock file (or another user's): nothing we can release + fh = os.fdopen(fd, 'r+') + try: + fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + # flock genuinely held — never SIGTERM on a mere pid record: the pid may be + # recycled, or a live worker that already moved on. + fh.close() + info = read_record(b) or {} + pid = info.get('pid') + reason = info.get('reason') + if reason in PROTECTED_REASONS: + print(f'ERROR: {b} is mid-test by {reason} (pid {pid}) — not killing it; ' + 'wait for it to finish', file=sys.stderr) + rc = 1 + elif isinstance(pid, int) and pid > 0: + victims.add(pid) + else: + print(f'ERROR: {b} is held but its record is unreadable', file=sys.stderr) + rc = 1 + continue + # flock was free: only a stale record remained — clear it + clear_record(fh) + fh.close() + for holder in sorted(victims): + try: + os.kill(holder, signal.SIGTERM) + print(f'released holder pid {holder}') + except ProcessLookupError: + pass + except PermissionError: + print(f'ERROR: holder pid {holder} belongs to another user — cannot signal it', + file=sys.stderr) + rc = 1 + time.sleep(0.3) + still = [b for b in boards if is_locked(b)] + if still: + print(f'ERROR: still locked: {", ".join(still)}', file=sys.stderr) + return 1 + return rc + + +def cmd_status(): + if not os.path.isdir(BOARD_LOCK_DIR): + print('no locks') + return 0 + any_locked = False + for fn in sorted(os.listdir(BOARD_LOCK_DIR)): + if not fn.endswith('.lock'): + continue + b = fn[:-5] + if is_locked(b): + any_locked = True + print(f'{b}: {read_record(b)}') + if not any_locked: + print('no locks') + return 0 + + +_CLI_USAGE = """Per-board advisory locks for the HIL rig. + +Arbitrates board access between dev sessions and CI's hil_test.py without +stopping the actions-runner. Locks are kernel flocks: the kernel releases +them automatically when the holder process dies, and holders clear their +lock-file record on release so records stay truthful (/tmp also clears on +reboot). + +Usage: + hil_lock.py hold BOARD [BOARD...] --reason TEXT + hil_lock.py hold --all [--config CONFIG.json] --reason TEXT + hil_lock.py release BOARD [BOARD...] | release --all + hil_lock.py status + +A holder process holds ALL boards given in one `hold` call; releasing any of +them kills that holder and releases all of its boards. +""" + + +def main(): + ap = argparse.ArgumentParser(description=_CLI_USAGE, + formatter_class=argparse.RawDescriptionHelpFormatter) + sub = ap.add_subparsers(dest='cmd', required=True) + p_hold = sub.add_parser('hold') + p_hold.add_argument('boards', nargs='*') + p_hold.add_argument('--all', action='store_true') + p_hold.add_argument('--config', + default=os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + 'tinyusb.json'), + help='board roster JSON (default: tinyusb.json in test/hil, one level above this script)') + p_hold.add_argument('--reason', required=True) + p_rel = sub.add_parser('release') + p_rel.add_argument('boards', nargs='*') + p_rel.add_argument('--all', action='store_true') + sub.add_parser('status') + a = ap.parse_args() + if a.cmd == 'hold': + boards = boards_from_config(a.config) if a.all else a.boards + if not boards: + ap.error('no boards given (name boards or use --all)') + sys.exit(cmd_hold(boards, a.reason)) + if a.cmd == 'release': + if a.all: + boards = ([fn[:-5] for fn in os.listdir(BOARD_LOCK_DIR) if fn.endswith('.lock')] + if os.path.isdir(BOARD_LOCK_DIR) else []) + else: + boards = a.boards + if not boards: + ap.error('no boards given (name boards or use --all)') + sys.exit(cmd_release(boards)) + sys.exit(cmd_status()) + + +if __name__ == '__main__': + main() diff --git a/test/hil/helper/hil_pool_check.py b/test/hil/helper/hil_pool_check.py new file mode 100644 index 000000000..371aff1e1 --- /dev/null +++ b/test/hil/helper/hil_pool_check.py @@ -0,0 +1,1019 @@ +#!/usr/bin/env python3 +"""Quick HIL pool health check. + +For every board in the rig's HIL config: is the flash probe on the USB bus, does a +light example flash, and does the board's USB device (uid) come back up? Missing +firmware is BUILT on the spot (tools/build.py, idf.py for espressif; one get_deps +retry) — never skipped; --no-build opts out. Applies only per-device-safe recovery +(probe authorized-toggle, board reset/re-flash) and prints a markdown summary +table. Row statuses: ok (flashed and verified; under --scan-only: probe present — +the scan checks presence only), flash-failed (firmware delivery failed: probe +missing, build failed, flasher error, silent flash no-op, park not verified), +failed (the check ran but did not verify: flashed with no enumeration/serial, or +the check itself errored), locked (board flock held by another process — +reported, never waited on or bypassed). + +Config is picked by hostname unless given: ci -> tinyusb.json, tusb (hifiphile +rig) -> hfp.json, anything else is a dev PC -> local.json. + +Lives in test/hil/helper/ beside hil_lock.py; imports it and hil_flash; board +recovery uses the repo's .claude/skills/usb-kernel-recover/scripts/usb_recover.sh. +""" + +import argparse +import io +import json +import glob +import os +import re +import shlex +import shutil +import socket +import sys +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) # hil_flash + the helper package +import hil_flash +from helper import hil_lock, hil_util + +REPO_ROOT = hil_util.TINYUSB_ROOT +USB_RECOVER = REPO_ROOT / '.claude' / 'skills' / 'usb-kernel-recover' / 'scripts' / 'usb_recover.sh' +SEEN_CACHE = Path.home() / '.cache' / 'tinyusb-hil' / 'pool_seen.json' +CONFIG_BY_HOST = {'ci': 'tinyusb.json', 'tusb': 'hfp.json'} # anything else: dev PC -> local.json + +# light-example preference; first built wins +DEVICE_CANDIDATES = ['device/dfu_runtime', 'device/cdc_msc', 'device/cdc_msc_freertos', + 'device/hid_composite_freertos', 'device/cdc_dual_ports'] +HOST_CANDIDATES = ['host/device_info', 'host/cdc_msc_hid', 'host/msc_file_explorer_freertos'] + +ENUM_WAIT = 12 # s, uid wait after flash +ENUM_WAIT_RETRY = 8 # s, uid wait after a recovery reset/re-flash +SERIAL_WAIT = 6 # s, host-board serial-output wait + +print_mutex = threading.Lock() +_UNKNOWN_WARNED = False # scan_usb's caveat: once per process, not once per poll +t0 = time.monotonic() + + +def say(msg: str) -> None: + with print_mutex: + print(f'[{time.monotonic() - t0:6.1f}s] {msg}', file=sys.__stdout__, flush=True) + + +def scan_usb() -> dict: + """busport -> {'serial', 'vidpid', 'ino'} for every enumerated USB device. Only + -[....] dirs match; root hubs ('usbN', no dash) are excluded because + their 'serial' is a fabricated PCI address, and including them measured 6-7s/scan slower + (an observation; NOT an autosuspend wake -- that read is cached and does no I/O). + Keyed by busport, not serial: two devices can share a serial (an Espressif + USB-Serial-JTAG bridge and the cafe device it flashes both derive it from the same + MAC), and one dict slot would silently drop whichever lost the race.""" + found = {} + # `unknown` matters BEFORE the blindness latch trips: one wedged device is the normal + # reason this tool is run, and its serial read stranding makes it absent from `devs`. + # Reported as fact, that is "probe MISSING" for hardware that is physically present. + devs, unknown = hil_util.usb_scan() + # ONCE per process: this is called from 0.5s poll loops across 4 worker threads and + # ~26 boards, so warning per call buried the table it exists to qualify under 600+ + # identical lines. The memo in read_sysfs makes the condition sticky, so one line is + # as true as six hundred. + global _UNKNOWN_WARNED + if unknown and not _UNKNOWN_WARNED: + _UNKNOWN_WARNED = True + say('WARNING: at least one device did not answer a bounded read; rows below that ' + 'say a probe or board is missing may be this scan losing sight of healthy ' + 'hardware. Find the wedged device (usb-kernel-recover) and re-run.') + for dev in devs: + try: + found[dev['busport']] = { + 'serial': dev['serial'].lower(), + 'vidpid': f"{dev['vid']}:{dev['pid']}", + 'ino': os.stat(dev['dir'] + '/').st_ino} + except OSError: + continue + return found + + +def find_usb(uid: str, devs: dict | None = None): + """Locate a flasher probe by uid, excluding VID cafe (TinyUSB DUT firmware): a + probe's uid can coincidentally equal its DUT's (Espressif USB-Serial-JTAG + bridges derive both from the same MAC), and the DUT is never the probe. + + J-Link zero-pads numeric serials (681295394 -> 000681295394): an all-digit uid + matches an all-digit serial only when that serial equals the uid zero-padded to + the serial's own length (leading zeros only) — never when the zero-stripped uid + is empty, so a placeholder serial (metro_m4_express's probe legitimately reports + '123456') can't be mistaken for an unrelated device.""" + devs = devs if devs is not None else scan_usb() + u = uid.lower() + candidates = [(bp, dev) for bp, dev in devs.items() if not dev['vidpid'].startswith('cafe:')] + for bp, dev in candidates: + if dev['serial'] == u: + return bp, dev['vidpid'], dev['ino'] + stripped = u.lstrip('0') + if u.isdigit() and stripped: + for bp, dev in candidates: + s = dev['serial'] + if s.isdigit() and s == stripped.zfill(len(s)): + return bp, dev['vidpid'], dev['ino'] + return None + + +def find_device(uid: str, pid: str | None): + """Board-online check: TinyUSB device (idVendor cafe) with this uid, optionally + PID-pinned. VID cafe keeps an Espressif USB-Serial-JTAG (303a) that shares the MAC + serial from false-passing.""" + for busport, dev in scan_usb().items(): + if (dev['serial'] == uid.lower() and dev['vidpid'].startswith('cafe:') + and (pid is None or dev['vidpid'].endswith(pid))): + return busport, dev['vidpid'], dev['ino'] + return None + + +def wait_device(uid: str, pid: str | None, old_ino, budget: float): + """Wait for the board's device with a NEW sysfs inode (flash resets the MCU, so a + genuine flash must re-enumerate; the inode is the re-enumeration marker).""" + deadline = time.monotonic() + budget + while time.monotonic() < deadline: + hit = find_device(uid, pid) + if hit and hit[2] != old_ino: + return hit + time.sleep(0.5) + return None + + +def lock_board(name: str): + """Nonblocking flock per hil_lock.py protocol. Returns the handle, or a str with the + holder's info when the board is locked elsewhere. Board locks are ALWAYS respected: a + held board is reported and skipped, never waited on, and there is no bypass here.""" + os.makedirs(hil_lock.BOARD_LOCK_DIR, exist_ok=True) + try: + fh = hil_lock.flock_nb(name) + except OSError: + # NB: conflates a held flock with open() failures (EACCES/EROFS/ENOSPC) — benign + # while everything on the rig runs as one uid + info = hil_lock.read_record(name) + return json.dumps(info) if info else 'unknown holder' + if not hil_lock.write_record(fh, 'pool_check'): + # an invisible lock (flock held, no record) is worse than no lock: status cannot + # show us and release cannot recognize the protected holder + hil_lock.clear_record(fh) + fh.close() + return 'ERROR: holder record write failed (lock dir unwritable?)' + return fh + + +def unlock_board(fh) -> None: + hil_lock.clear_record(fh) + fh.close() + + +def can_recover() -> bool: + if not USB_RECOVER.is_file(): + return False + try: + # run_cmd, not subprocess.run: run's post-timeout reap is an UNBOUNDED wait(), and + # our kill bounces off a setuid-root sudo with EPERM, leaving communicate() on a + # pipe that never closes. run_cmd killpgs, escalates through sudo, reaps bounded. + r = hil_util.run_cmd('sudo -n true', timeout=10, quiet=True) + except OSError: # sudo not installed + return False + return r.returncode == 0 + + +def recover_probe(uid: str, busport: str) -> bool: + """Soft-replug an enumerated-but-wedged probe: deauthorize+reauthorize (no VBUS cut, + touches only this device). Success = the probe re-enumerated (new sysfs inode), not the + helper's exit code, which flakes while the toggle works. J-Links respond with a full + disconnect and can stay off the bus for >8 s.""" + pre = find_usb(uid) + # Bounded through run_cmd (same reason as can_recover): the sysfs authorized store can + # block in D state on a wedged device, and this runs while the board's release- + # PROTECTED flock is held -- a hang here would lock the board until the host reboots. + cmd = ' '.join(shlex.quote(a) for a in + ['sudo', '-n', str(USB_RECOVER), 'authorized', busport]) + if hil_util.run_cmd(cmd, timeout=30, quiet=True).returncode == 124: + return False + deadline = time.monotonic() + 20 + while time.monotonic() < deadline: + post = find_usb(uid) + if post and (pre is None or post[2] != pre[2]): + return True + time.sleep(0.5) + return False + + +def resolve_variant(board: dict, example: str, note: list | None = None) -> str: + """Build-dir variant name for `example`: the first of the board's variants with + already-built firmware, falling back to the board name. Notes the pick when it + differs from the board name (e.g. nanoch32v203's build dir is variant + 'nanoch32v203-fsdev', not the board name).""" + name = board['name'] + for v in board.get('variant') or [{'name': name}]: + vn = v['name'] + if hil_flash.find_firmware(vn, example, flasher=board['flasher']['name']): + if vn != name and note is not None and f'variant: {vn}' not in note: + note.append(f'variant: {vn}') + return vn + return name + + +def pick_example(board: dict, note: list, build_missing: bool = True): + """(example, kind, variant, fw) with built firmware for this board; kind is + 'device' (uid check) or 'host' (serial-output check); variant is the resolved + build-dir variant that has it (see resolve_variant); fw is the firmware path to + flash, extension included. When nothing is built and build_missing is set (the default — + never skip a board for lack of a build), the preferred candidate is built on + the spot via ensure_fw.""" + tests = board.get('tests', {}) + only = tests.get('only', []) + skip = set(tests.get('skip', [])) # config's known-broken examples: never pick one + is_device = tests.get('device') or any(t.startswith('device/') for t in only) + if is_device: + cand = DEVICE_CANDIDATES + [t for t in only if t.startswith('device/') and t != 'device/usbtest'] + kind = 'device' + else: + cand = HOST_CANDIDATES + [t for t in only if t.startswith('host/')] + kind = 'host' + for ex in dict.fromkeys(cand): + if ex in skip: + continue + variant = resolve_variant(board, ex, note) + fw = hil_flash.find_firmware(variant, ex, flasher=board['flasher']['name']) + if fw: + return ex, kind, variant, fw + if not build_missing: + return None, kind, None, None + # nothing built anywhere: build the preferred candidate (an only-list board + # must get one of its own examples — dfu_runtime etc. may not even configure) + pref = [c for c in dict.fromkeys(cand) if c not in skip and (not only or c in only)] + if not pref: + return None, kind, None, None + variant = (board.get('variant') or [{'name': board['name']}])[0]['name'] + for ex in pref[:2]: # the second candidate covers a preferred example that fails to build + fw = ensure_fw(board, variant, ex, note) + if fw: + return ex, kind, variant, fw + return None, kind, None, None + + +_pid_cache: dict[str, str | None] = {} + + +def get_expected_pid(example: str) -> str | None: + """USB_PID for `example`'s device descriptor (examples//src/ + usb_descriptors.c, '#define USB_PID 0x....'), lowercased and without the 0x + prefix to match sysfs idProduct. Cached per example; None (also cached) when + the file or define isn't there — host examples have no usb_descriptors.c, and + the caller must stay quiet rather than false-warn.""" + if example not in _pid_cache: + pid = None + try: + text = (REPO_ROOT / 'examples' / example / 'src' / 'usb_descriptors.c').read_text() + # optional parens as in tools/check_example_pids.py's parser + m = re.search(r'#define\s+USB_PID\s+\(?\s*(0x[0-9a-fA-F]+)', text) + if m: + pid = m.group(1)[2:].lower() + except OSError: + pass + _pid_cache[example] = pid + return _pid_cache[example] + + +def call_flasher(fn, *fn_args) -> tuple[int, str]: + """Run a hil_flash flash_*/reset_* backend, normalizing raises to a failure: several + backends raise instead of returning nonzero (get_serial_dev when a bridge's + /dev/serial/by-id node vanishes, a missing config.env, a .jlink script OSError), and an + exception must not skip the caller's retry/recovery ladder. Returns (rc, error line).""" + try: + ret = fn(*fn_args) + if ret.returncode == 0: + return 0, '' + err = flash_error_line(hil_util.cmd_stdout_text(ret.stdout)) + return ret.returncode, err or f'rc={ret.returncode}' + except Exception as e: + return -1, repr(e)[:90] + + +def flash(board: dict, fw, allow_recovery: bool, probe_port: str, note: list) -> bool: + """Flash the resolved firmware with one retry; on repeated failure soft-replug the + probe and always make one final attempt afterward, confirmed replug or not — some + probes (WCH-Link, ST-Link, CP210x, picoprobe) keep their sysfs kobject across an + authorized toggle instead of dropping off the bus. Returns True on success. + + `fw` comes from pick_example: a re-resolve here would use the global search policy and + miss a firmware ensure_fw just built into cmake-build/ under an exclusive -B.""" + fn = getattr(hil_flash, f'flash_{board["flasher"]["name"].lower()}') + for attempt in range(3): + if attempt == 2: + if not (allow_recovery and probe_port): + return False + cur = find_usb(board['flasher']['uid']) + if cur is None: + # probe gone from the bus: its old busport may now hold an UNRELATED device + # (bus renumbering) and the helper only checks occupancy, so toggling would + # deauthorize an innocent fixture + note.append('probe vanished before toggle') + else: + say(f'{board["name"]:26} recovery: replugging probe {cur[0]} (authorized toggle)') + if recover_probe(board['flasher']['uid'], cur[0]): + note.append('probe replugged') + time.sleep(2) # udev recreates /dev/serial/by-id symlinks after re-enumeration + else: + note.append('probe toggle unconfirmed') + rc, err = call_flasher(fn, board, str(fw)) + if rc == 0: + return True + if rc == 127: # flasher binary missing: retries/probe recovery can't fix env + note.append(f'flasher tool missing ({err}) — esptool needs the ESP-IDF env (get-idf)' + if board['flasher']['name'].lower() == 'esptool' else + f'flasher tool missing: {err}') + return False + if attempt == 0: + say(f'{board["name"]:26} flash retry: {err}') + else: + note.append(f'flash: {err}') + return False + + +def flash_error_line(out: str) -> str: + """Most informative line of a failed flash's output: last error-looking line, + else the last non-empty one.""" + lines = [l.strip() for l in out.splitlines() if l.strip()] + for l in reversed(lines): + if any(k in l.lower() for k in ('error', 'fail', 'unknown', 'cannot', 'timeout', + 'no valid', 'not found', 'unable')): + return l[:90] + return lines[-1][:90] if lines else '' + + +def check_host_serial(board: dict, do_reset: bool = True, want_hello: bool = False) -> bytes | None: + """Host-only boards never enumerate their uid (their USB port is the host side); + aliveness = output on the flasher's UART bridge after a reset. A probe byte is + written each poll so an echo-only firmware (board_test) also answers. Returns + the first output chunk (b'' when silent, None when the port is absent/drops) so + the caller can also judge WHAT answered — see boardtest_output(). + + do_reset=False listens to the firmware as-is: used right after a flash whose + own reset already started it — a second openocd/JLink session back-to-back on + the same probe can fail transiently and leave the target halted.""" + import serial + try: + port = hil_util.get_serial_dev(board['flasher']['uid'], None, None, 0) + ser = serial.Serial(port, baudrate=115200, timeout=0.3, write_timeout=1) + except Exception as e: + say(f'{board["name"]:26} no flasher serial port: {e}') + return None + try: + # flush BEFORE the reset: this drops the pre-reset CDC backlog (which must not + # count as life) while keeping the post-reset boot banner, which prints while the + # reset tool is still tearing down and a post-reset flush would eat + ser.reset_input_buffer() + if do_reset: + getattr(hil_flash, f'reset_{board["flasher"]["name"].lower()}')(board) + # judge the WHOLE window, not the first chunk: the probe's CDC bridge has its own + # FIFO, so stale pre-flash output (e.g. board_test hellos) can arrive after our + # host-side flush and must not decide the verdict alone. + data = b'' + deadline = time.monotonic() + SERIAL_WAIT + while time.monotonic() < deadline: + try: + ser.write(b'U') + data += ser.read(256) + except serial.SerialTimeoutException: + pass + except serial.SerialException: + return None # port dropped mid-poll (bridge re-enumerating) + # early-exit on the caller's positive signal (board_test hello for park + # verification, any non-board_test output for example liveness): stale + # bridge-FIFO backlog of the OTHER kind must not end the window + if want_hello: + if b'Hello from TinyUSB' in data: + return data + elif data and not boardtest_output(data): + return data + return data + finally: + ser.close() + + +def boardtest_output(data: bytes) -> bool: + """True when (non-empty) serial output is recognizably ONLY board_test's: its + periodic HELLO_STR and echoes of our b'U' pokes, nothing else. Any residue + beyond that (an example banner, log lines) proves other firmware is talking, + however much stale board_test backlog surrounds it. Used as a negative + identity marker — after flashing a host example, board_test-only chatter + means the flash silently didn't take (the host analog of the PID check).""" + residue = data.replace(b'Hello from TinyUSB', b'') + for junk in (b'U', b'\r', b'\n'): + residue = residue.replace(junk, b'') + return len(residue) == 0 + + +def build_example(board: dict, variant: str, example: str) -> int: + """Build one example for this board: tools/build.py (same invocation shape as + hil_test.build_board), or idf.py directly for espressif (tools/build.py's esp branch + ignores -T and builds everything; variant flags travel as -DCFLAGS_CLI, the channel + tools/build.py uses). Bounded and process-group-killed via run_cmd; 600 s covers a + first configure+build of an SDK-heavy family (pico, nrf, esp). Builds normally run + pre-lock, so a board flock is not held here except on rare recovery paths. Per-build + compile parallelism is capped at cpu/-j so -j concurrent builds cannot swamp sibling + workers' verification windows. Returns the returncode (127 = ESP-IDF env missing).""" + name = board['name'] + variants = board.get('variant') or [{'name': name}] + vcfg = next((v for v in variants if v['name'] == variant), variants[0]) + if board['flasher']['name'].lower() == 'esptool': + if not shutil.which('idf.py'): + return 127 # ESP-IDF env not sourced in this shell + # -B keyed off the VARIANT so ensure_fw's post-build lookup finds it + cmd = ['idf.py', '-C', f'examples/{example}', + '-B', f'cmake-build/cmake-build-{vcfg["name"]}/{example}', + '-G', 'Ninja', f'-DBOARD={name}', 'build'] + for d in board.get('build', {}).get('args', []) + vcfg.get('defines', []): + cmd.insert(-1, f'-D{d}') + if vcfg.get('flags'): + cmd.insert(-1, f'-DCFLAGS_CLI={vcfg["flags"]}') + # the IDF component manager writes examples//dependencies.lock in the + # SOURCE tree (idf.py -B relocates only the build dir), so concurrent esp + # builds of one example for different targets corrupt each other's solve + with _esp_lock, _build_sem: + return hil_util.run_cmd(shlex.join(cmd), cwd=str(hil_util.TINYUSB_ROOT), + timeout=600).returncode + cmd = [sys.executable, str(hil_util.TINYUSB_ROOT / 'tools' / 'build.py'), + '-b', name, '-T', Path(example).name, + '-j', str(max(1, (os.cpu_count() or _jobs) // _jobs))] + for d in board.get('build', {}).get('args', []): + cmd += ['-D', d] + if vcfg['name'] != name: + cmd += ['--build-name', vcfg['name']] + for d in vcfg.get('defines', []): + cmd += ['-D', d] + for tok in vcfg.get('flags', '').split(): + cmd += [f'--cflag={tok}'] + with _build_sem: + return hil_util.run_cmd(shlex.join(cmd), cwd=str(hil_util.TINYUSB_ROOT), + timeout=600).returncode + + +_deps_lock = threading.Lock() # one get_deps at a time (it also drains _build_sem) +_esp_lock = threading.Lock() # idf.py mutates source-tree dependencies.lock per example +_no_build = False # --no-build: ensure_fw never invokes a build +_jobs = 4 # mirrors -j; set in main before the pool starts +_build_sem = threading.BoundedSemaphore(4) # build slots; get_deps drains ALL (exclusive) +_builds: dict = {} # (variant, example) -> (fw|None, reason): one attempt per run + + +def ensure_fw(board: dict, variant: str, example: str, note: list): + """Firmware for `example`, building it when absent — never skip a board for lack of a + build (--no-build opts out). One retry with deps fetched and the CMake caches dropped + when the first build fails (fresh checkouts lack the family deps; a cache configured + in a broken env poisons every later attempt). Returns the firmware path, or None with + the failure noted. Call BEFORE taking the board lock: builds are long. One attempt per + (variant, example) per run, memoized in _builds, so a repeat call (park, under the + held flock) resolves instantly even when an exclusive -B hides the fresh artifact.""" + fw = hil_flash.find_firmware(variant, example, flasher=board['flasher']['name']) + if fw: + return fw + key, base = (variant, example), Path(example).name + if key in _builds: + return _builds[key][0] + if _no_build: + _builds[key] = (None, 'disabled') + note.append(f'build skipped (--no-build): {base}') + return None + rc = build_example(board, variant, example) + if rc == 127 and board['flasher']['name'].lower() == 'esptool': + _builds[key] = (None, 'no-env') + note.append(f'cannot build {base}: ESP-IDF env missing (get-idf)') + return None + if rc == 124: # hung build: a deps/cache retry cannot cure it, don't double the stall + _builds[key] = (None, 'timeout') + note.append(f'build timeout: {base}') + return None + if rc != 0: + # retry once with deps fetched and the CMake caches dropped (cache only — a tree + # wipe would destroy every other example's firmware). get_deps git-resets shared + # deps that are already present, so it drains ALL build slots first. + with _deps_lock: + for _ in range(_jobs): + _build_sem.acquire() + try: + r = hil_util.run_cmd(shlex.join([sys.executable, str(hil_util.TINYUSB_ROOT / 'tools' / 'get_deps.py'), + '-b', board['name']]), + cwd=str(hil_util.TINYUSB_ROOT), timeout=600) + finally: + for _ in range(_jobs): + _build_sem.release() + if r.returncode != 0: + note.append('get_deps failed') + bd = hil_util.TINYUSB_ROOT / 'cmake-build' / f'cmake-build-{variant}' + # esp configures one level deeper (//): wipe both layouts + for d in (bd, bd / example): + shutil.rmtree(d / 'CMakeFiles', ignore_errors=True) + (d / 'CMakeCache.txt').unlink(missing_ok=True) + rc = build_example(board, variant, example) + if rc != 0: + _builds[key] = (None, 'fail') + note.append(f'build failed: {base}') + return None + # both build paths write to cmake-build/, so look there even when an explicit -B + # narrowed the global search — this is OUR fresh build, not a stale fallback + fw = hil_flash.find_firmware(variant, example, + roots=[hil_flash.build_dir, 'cmake-build'], + flasher=board['flasher']['name']) + _builds[key] = (fw, 'ok' if fw else 'no-fw') + note.append(f'built {base}' if fw else f'build produced no firmware: {base}') + return fw + + +def ensure_board_test(board: dict, variant: str, note: list): + """board_test firmware for parking, building it if absent (via ensure_fw). + Espressif included — tools/build.py builds board_test for that family too; + the build just needs the ESP-IDF env (127 → noted, park is then skipped).""" + fw = hil_flash.find_firmware(variant, 'device/board_test', flasher=board['flasher']['name']) + if fw: + return fw + variants = board.get('variant') or [{'name': board['name']}] + if not any(v['name'] == variant for v in variants): + variant = variants[0]['name'] + return ensure_fw(board, variant, 'device/board_test', note) + + +def verdict(row: dict, ok: bool) -> str: + """Row status for a verification result, preserving a 'flash-failed' a deeper + layer already recorded (silent flash no-op, board_test delivery failure).""" + return 'ok' if ok else ('flash-failed' if row['status'] == 'flash-failed' else 'failed') + + +def host_alive(board: dict, note: list, row: dict, flashed_example: bool = False) -> bool: + """Serial aliveness with recovery: silent -> (build and) flash board_test (it + hellos every second and echoes) -> recheck. Also cures a silent flash no-op + that left the board crashed. + + With flashed_example=True (a host example was just flashed), board_test-shaped + output FAILS the check: the parked image still talking means the example flash + silently didn't take — the host analog of the device path's PID check. + + Side effect: delivery-class failures (silent no-op, board_test build/flash + failure) set row['status'] = 'flash-failed' so verdict() preserves the cause; + the caller derives the final status from the return value via verdict().""" + data = check_host_serial(board) + if data: + if flashed_example and boardtest_output(data): + note.append('board_test output after example flash: silent flash no-op') + row['status'] = 'flash-failed' + return False + return True + variant = resolve_variant(board, 'device/board_test', note) + fw = ensure_board_test(board, variant, note) + if fw is None: + note.append('serial silent; board_test unavailable') + row['status'] = 'flash-failed' + return False + say(f'{board["name"]:26} recovery: serial silent, flashing board_test') + rc, err = call_flasher(getattr(hil_flash, f'flash_{board["flasher"]["name"].lower()}'), board, str(fw)) + if rc != 0: + note.append(f'serial silent; board_test flash failed: {err}') + row['status'] = 'flash-failed' + return False + if not check_host_serial(board): + return False + if flashed_example: + # board_test talking proves the BOARD is alive, but the just-flashed + # example never produced serial — that verification still fails + note.append('example silent; board alive via board_test reflash') + return False + note.append('recovered via board_test reflash') + return True + + +def device_recover_and_check(board: dict, example: str, variant: str, old_ino, note: list, row: dict, seen: dict) -> bool: + """Wait for the flashed board's uid to re-enumerate; on timeout, try one board + reset (skipped for flashers with no hardware reset — see hil_flash.RESET_NOOP, + it would just burn the wait) and wait again. + + The PID policy is deliberately asymmetric. Pre-reset, the re-enumeration was + caused by the flash itself, so a PID mismatch most likely means the build dir + is stale (the flash DID write what find_firmware found) — warn, don't fail — + UNLESS the firmware was built this very run: then 'stale build' is impossible + and the mismatch can only be a silent flash no-op, which fails. Post-reset, + the re-enumeration proves nothing about the flash (the reset alone explains + it), so a mismatch is treated as a silent flash no-op and fails; an unknown + expected PID scores ok with a 'pid unverified' note in both paths.""" + name = board['name'] + expected_pid = get_expected_pid(example) + built_this_run = _builds.get((variant, example), (None, ''))[1] == 'ok' + + def seen_hit(hit): + seen[board['uid']] = {'name': name, 'busport': hit[0], 'when': time.strftime('%Y-%m-%d %H:%M')} + + hit = wait_device(board['uid'], None, old_ino, ENUM_WAIT) + if hit: + if expected_pid is not None and not hit[1].endswith(expected_pid): + if built_this_run: + row['device'] = f'❌ {hit[1]}' + note.append(f'pid {hit[1]}, this run built {expected_pid}: silent flash no-op') + row['status'] = 'flash-failed' + return False + note.append(f'⚠ pid {hit[1]}, source says {expected_pid}: stale build or silent flash no-op') + elif expected_pid is None: + note.append('pid unverified') + row['device'] = f'✅ {hit[1]}' + seen_hit(hit) + return True + + flasher_name = board['flasher']['name'].lower() + if flasher_name in hil_flash.RESET_NOOP: + note.append(f'no hardware reset available for {flasher_name}') + row['device'] = '❌ not enumerated' + return False + + say(f'{name:26} recovery: uid not up, resetting board') + rc, err = call_flasher(getattr(hil_flash, f'reset_{flasher_name}'), board) + if rc != 0: + note.append(f'reset failed: {err}') + hit = wait_device(board['uid'], None, old_ino, ENUM_WAIT_RETRY) + if not hit: + row['device'] = '❌ not enumerated' + note.append('reset did not help') + return False + if expected_pid is None: + row['device'] = f'✅ {hit[1]}' + note.append('reset recovered (pid unverified)') + seen_hit(hit) + return True + if hit[1].endswith(expected_pid): + row['device'] = f'✅ {hit[1]}' + note.append('reset recovered') + seen_hit(hit) + return True + row['device'] = f'❌ {hit[1]}' + note.append(f'reset recovered wrong pid, expected {expected_pid}: silent flash no-op') + row['status'] = 'flash-failed' + return False + + +def check_board(board: dict, args, allow_recovery: bool, seen: dict) -> dict: + name = board['name'] + row = {'name': name, 'probe': '❌ missing', 'flash': '–', 'device': '–', 'note': [], 'status': 'failed'} + note = row['note'] + + probe = find_usb(board['flasher']['uid']) + if probe: + row['probe'] = f'✅ {probe[0]}' + seen[board['flasher']['uid']] = {'name': f'{name} probe', 'busport': probe[0], + 'when': time.strftime('%Y-%m-%d %H:%M')} + else: + last = seen.get(board['flasher']['uid']) + note.append(f'probe last seen {last["busport"]} {last["when"]}' if last + else 'probe never seen by pool_check') + say(f'{name:26} probe MISSING ({board["flasher"]["name"]} {board["flasher"]["uid"]})') + + # existing firmware only; a missing build is built further down (after a lock peek), + # except in scan/no-build modes and never for a missing probe + example, kind, variant, fw = pick_example(board, note, build_missing=False) + if kind == 'host': + note.append('host-only board') + + if args.scan_only: + hit = find_device(board['uid'], None) + # report the BOARD's usb state too: enumerated (with busport), off-bus (normal + # when parked in board_test), or n/a for host-only boards + if hit: + row['device'] = f'✅ {hit[1]} @{hit[0]}' + elif kind == 'host': + row['device'] = '– n/a (host-only)' + else: + row['device'] = '⚫ off bus (parked?)' + # scan verifies probe presence only, so probe present is ok; a missing probe means + # no firmware could be delivered → flash-failed + row['status'] = 'ok' if probe else 'flash-failed' + if probe: + say(f'{name:26} probe ✅ {probe[0]}' + (f' device {hit[1]}' if hit else '')) + return row + if not probe: + row['status'] = 'flash-failed' + return row + + bt_variant = resolve_variant(board, 'device/board_test', note) + need_example = example is None and not args.no_build + # board_test is also host_alive's recovery image, so host boards pre-build it + # even under --no-park; --no-build gates EVERY build, board_test included + need_bt = (not args.no_build + and (not args.no_park or kind == 'host') + and hil_flash.find_firmware(bt_variant, 'device/board_test', + flasher=board['flasher']['name']) is None) + if need_example or need_bt: + # builds are long and run BEFORE locking (park must never hold the flock through + # one); peek first so minutes of building are not wasted on — or a rebuilt tree + # swapped under — a board CI holds right now + peek = lock_board(name) + if isinstance(peek, str): + if peek.startswith('ERROR:'): # environment failure, not a held lock + row['flash'] = '❌ lock' + row['status'] = 'failed' + else: + row['flash'] = '🔒 locked' + row['status'] = 'locked' + note.append(peek) + say(f'{name:26} locked: {peek}') + return row + unlock_board(peek) + if need_example: + example, kind, variant, fw = pick_example(board, note, build_missing=True) + if need_bt and (example is not None or kind == 'host'): + # skip the park build when the example build already failed on a device board: + # the row returns before any flash/park could use it + ensure_board_test(board, bt_variant, note) + + if example is None: + if not any(n.startswith(('build failed', 'build timeout', 'build produced', + 'build skipped', 'cannot build')) for n in note): + note.append('no firmware built') + if kind != 'host': + row['status'] = 'flash-failed' + say(f'{name:26} probe ✅ {probe[0]} (no firmware to flash)') + return row + # host-only board: aliveness is still checkable without flashing — reset and listen + # to whatever is on it (parked board_test echoes and hellos on the flasher UART) + + lk = lock_board(name) + if isinstance(lk, str): + if lk.startswith('ERROR:'): # environment failure, not a held lock + row['flash'] = '❌ lock' + row['status'] = 'failed' + else: + row['flash'] = '🔒 locked' + row['status'] = 'locked' + note.append(lk) + say(f'{name:26} locked: {lk}') + return row + try: + if example is None: # host-only without firmware: UART-only aliveness check + ok = host_alive(board, note, row) + row['device'] = '✅ serial out' if ok else '❌ no serial out' + row['status'] = verdict(row, ok) + say(f'{name:26} – {row["device"]} (existing firmware)') + return row + + pre = find_device(board['uid'], None) + old_ino = pre[2] if pre else None + + try: + if not flash(board, fw, allow_recovery, probe[0], note): + row['flash'] = f'❌ {Path(example).name}' + row['status'] = 'flash-failed' + say(f'{name:26} flash FAILED ({example})') + return row + row['flash'] = f'✅ {Path(example).name}' + + if kind == 'host': + ok = host_alive(board, note, row, flashed_example=True) + row['device'] = '✅ serial out' if ok else '❌ no serial out' + else: + ok = device_recover_and_check(board, example, variant, old_ino, note, row, seen) + row['status'] = verdict(row, ok) + say(f'{name:26} {row["flash"]} {row["device"]}') + return row + finally: + # teardown for EVERY path that attempted a flash (a failed programmer op can + # still have erased/half-written the target), while the lock is still held + if not args.no_park: + park_board(board, kind, row, note) + finally: + unlock_board(lk) + + +def park_board(board: dict, kind: str, row: dict, note: list) -> None: + """Re-park with board_test, building it if absent (ensure_board_test), and + VERIFY it took: board_test never enumerates USB, so a device board's cafe + device must drop off the bus, and a host board must answer with board_test's + own output — a rc=0 park that changed nothing (silent no-op) must not pass. + A board left unparked marks an ok row flash-failed (never downgrading a + 'failed' verify verdict — that is the more diagnostic signal), with one + exception: an espressif board without the ESP-IDF env cannot build + board_test — noted, not a board fault.""" + # capture BEFORE the park flash: uid-disappearance only verifies the park if the + # device was on the bus to begin with + on_bus_before = kind != 'host' and find_device(board['uid'], None) is not None + variant = resolve_variant(board, 'device/board_test', note) + fw = ensure_board_test(board, variant, note) + if fw is None: + if any(n.startswith('cannot build board_test') for n in note): + note.append('park skipped (no ESP-IDF env)') + else: + # --no-build disables builds, not parking (--no-park is that opt-out): + # a board left running a USB-active image is unparked either way + note.append('unparked: board_test not built (--no-build)' + if any(n.startswith('build skipped (--no-build): board_test') for n in note) + else 'unparked: board_test unavailable (build failed/timed out)') + if row['status'] == 'ok': + row['status'] = 'flash-failed' + return + rc, err = call_flasher(getattr(hil_flash, f'flash_{board["flasher"]["name"].lower()}'), + board, str(fw)) + if rc != 0: + note.append(f'park flash failed: {err}') + if row['status'] == 'ok': + row['status'] = 'flash-failed' + return + if kind == 'host': + # no second reset (the park flash's own reset started board_test); POSITIVE + # marker: its hello must appear, and stale bridge-FIFO output alongside it is not + # disqualifying + data = check_host_serial(board, do_reset=False, want_hello=True) + if not (data and b'Hello from TinyUSB' in data): + note.append('park unverified: no board_test output') + if row['status'] == 'ok': + row['status'] = 'flash-failed' + return + if not on_bus_before: + # never enumerated this run: uid-disappearance cannot tell a verified park from a + # silent no-op — say so instead of passing vacuously + note.append('park unverified (device already off bus)') + return + deadline = time.monotonic() + 6 + while time.monotonic() < deadline: + if find_device(board['uid'], None) is None: + return + time.sleep(0.5) + note.append('park unverified: device still enumerated') + if row['status'] == 'ok': + row['status'] = 'flash-failed' + + +def check_board_safe(board: dict, args, allow_recovery: bool, seen: dict) -> dict: + """Isolate one board's exceptions: a crashing worker must not discard every + other board's row, the table, the topology, and the seen-cache write.""" + try: + return check_board(board, args, allow_recovery, seen) + except Exception as e: + name = board.get('name', '?') + say(f'{name:26} INTERNAL ERROR: {e!r}') + return {'name': name, 'probe': '–', 'flash': '–', 'device': '❌ error', + 'note': [repr(e)[:120]], 'status': 'failed'} + + +def controller_summary() -> list[str]: + """USB topology: controller (PCI addr, vendor) -> bus -> root-port subtree device + counts (hubs included, interfaces/root hubs not). Bus numbers renumber every boot; + PCI addresses and root-port numbers are stable.""" + vendor_names = {'0x1022': 'AMD', '0x1912': 'Renesas', '0x8086': 'Intel', '0x1b21': 'ASMedia'} + ctrl = {} + for root in glob.glob('/sys/bus/usb/devices/usb*'): + bus = int(os.path.basename(root)[3:]) + m = re.findall(r'[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]', os.path.realpath(root)) + pci = m[-1] if m else '?' + c = ctrl.setdefault(pci, {'vendor': '?', 'buses': {}}) + subtrees = {} + for d in glob.glob(f'/sys/bus/usb/devices/{bus}-*'): + b = os.path.basename(d) + if ':' in b: + continue + subtrees[b.split('.')[0]] = subtrees.get(b.split('.')[0], 0) + 1 + c['buses'][bus] = subtrees + try: + vid = open(f'/sys/bus/pci/devices/{pci}/vendor').read().strip() + c['vendor'] = vendor_names.get(vid, vid) + except OSError: + pass + + lines = [] + for pci, c in sorted(ctrl.items()): + lines.append(f'{pci} ({c["vendor"]})') + for bus, subtrees in sorted(c['buses'].items()): + detail = ' '.join(f'{k}: {n} dev' for k, n in + sorted(subtrees.items(), key=lambda i: int(i[0].split('-')[1]))) + lines.append(f' bus {bus}: {sum(subtrees.values())} devices' + + (f' {detail}' if detail else '')) + return lines + + +def main() -> None: + # toolchain/flasher CLIs live in the user bin dirs, which non-login shells may lack -- + # the same PATH shim hil_ci.sh applies on the remote side + for d in (Path.home() / 'bin', Path.home() / '.local' / 'bin'): + if d.is_dir() and str(d) not in os.environ.get('PATH', '').split(os.pathsep): + os.environ['PATH'] = f'{d}{os.pathsep}{os.environ.get("PATH", "")}' + + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument('config', nargs='?', help='HIL config json (default: by hostname)') + parser.add_argument('-b', '--board', action='append', default=[], help='only these boards') + parser.add_argument('-B', '--build-dir', default=None, + help='firmware parent dir, searched EXCLUSIVELY when given ' + '(default: examples, plus cmake-build as fallback)') + parser.add_argument('--scan-only', action='store_true', + help='USB presence scan only: no locks, no flashing') + parser.add_argument('--no-build', action='store_true', + help='do not build missing firmware (default: build the light example on the spot)') + parser.add_argument('--no-park', action='store_true', + help='leave the light example running (default: park with board_test)') + # no cross-process flash budget against a concurrent hil_test.py run (its semaphores + # are in-process), so keep this modest + parser.add_argument('-j', '--jobs', type=int, default=4) + parser.add_argument('-v', '--verbose', action='store_true') + args = parser.parse_args() + global _no_build, _jobs, _build_sem + _no_build = args.no_build + _jobs = max(1, args.jobs) + _build_sem = threading.BoundedSemaphore(_jobs) + + host = socket.gethostname() + cfg_name = args.config or CONFIG_BY_HOST.get(host, 'local.json') + cfg_path = Path(cfg_name) + if not cfg_path.exists(): + cfg_path = REPO_ROOT / 'test' / 'hil' / cfg_name + if not cfg_path.exists(): + sys.exit(f'config not found: {cfg_name} (host {host}; dev PCs need test/hil/local.json)') + with cfg_path.open() as f: + config = json.load(f) + + boards = list(config['boards']) # boards-skip (parked hardware) is not scanned by default + if args.board: + boards += config.get('boards-skip', []) # explicitly named parked boards are fair game + unknown = set(args.board) - {b['name'] for b in boards} + if unknown: + sys.exit(f'board(s) not in {cfg_path.name}: {", ".join(sorted(unknown))}') + boards = [b for b in boards if b['name'] in args.board] + + hil_flash.build_dir = args.build_dir or 'examples' + hil_util.verbose = args.verbose + if args.build_dir is None: + # default mode: search both standard layouts (cmake-build/ from tools/build.py and + # ESP-IDF, examples/ from manual builds). An EXPLICIT -B stays exclusive: the caller + # named an artifact tree, so a miss must report rather than flash an older build. + hil_flash.EXTRA_BUILD_DIRS = ['cmake-build', 'examples'] + allow_recovery = not args.scan_only and can_recover() + seen = {} + try: + loaded = json.loads(SEEN_CACHE.read_text()) + if isinstance(loaded, dict): # tolerate a torn/hand-edited cache + seen = {k: v for k, v in loaded.items() if isinstance(v, dict)} + except (OSError, ValueError): + pass + + roots = ' + '.join(dict.fromkeys([hil_flash.build_dir, *hil_flash.EXTRA_BUILD_DIRS])) + say(f'pool check: host {host}, config {cfg_path.name}, {len(boards)} boards, ' + f'{"scan-only" if args.scan_only else f"flash via {{{roots}}}/cmake-build-"}' + f'{"" if allow_recovery or args.scan_only else ", recovery unavailable (no sudo -n / usb_recover.sh)"}') + + if args.verbose: + rows = [check_board_safe(b, args, allow_recovery, seen) for b in boards] + else: + with io.StringIO() as spool, ThreadPoolExecutor(max_workers=args.jobs) as pool: + sys.stdout = spool # silence hil_util.run_cmd's COMMAND FAILED dumps; say() uses __stdout__ + try: + rows = list(pool.map(lambda b: check_board_safe(b, args, allow_recovery, seen), boards)) + finally: + sys.stdout = sys.__stdout__ + + try: + SEEN_CACHE.parent.mkdir(parents=True, exist_ok=True) + tmp = SEEN_CACHE.with_suffix('.json.tmp') + tmp.write_text(json.dumps(seen, indent=1, sort_keys=True) + '\n') + tmp.replace(SEEN_CACHE) # atomic: a killed run can't tear the cache + except OSError: + pass + + status_mark = {'ok': '✅ ok', 'flash-failed': '❌ flash-failed', 'failed': '❌ failed', + 'locked': '🔒 locked'} + headers = ['Board', 'Probe', 'Flash', 'Device', 'Status', 'Note'] + cells = [[r['name'], r['probe'], r['flash'], r['device'], + status_mark.get(r['status'], r['status']), '; '.join(r['note'])] for r in rows] + widths = [max(len(h), *(len(c[i]) for c in cells)) if cells else len(h) + for i, h in enumerate(headers)] + line = lambda vals: '| ' + ' | '.join(v.ljust(w) for v, w in zip(vals, widths)) + ' |' + print() + print(line(headers)) + print('|' + '|'.join('-' * (w + 2) for w in widths) + '|') + for c in cells: + print(line(c)) + + print('\nUSB topology (controller → root-port subtree):') + for line in controller_summary(): + print(f' {line}') + + counts = {'ok': 0, 'flash-failed': 0, 'failed': 0, 'locked': 0} + for r in rows: + counts[r.get('status', 'failed')] += 1 + print(f'\n{counts["ok"]} ok · {counts["flash-failed"]} flash-failed · {counts["failed"]} failed ' + f'· {counts["locked"]} locked · in {time.monotonic() - t0:.0f}s') + if hil_util.sysfs_blind(): + # Without this the table is the worst kind of wrong: once the process latches + # blind, every read answers SYSFS_UNKNOWN, scan_usb() returns {}, and EVERY board + # prints "probe MISSING"/"off bus" -- a clean-looking report declaring the whole + # fleet dead, produced during exactly the incident this tool is run to diagnose, + # and it sends the operator to power-cycle a rig where one device is wedged. + print('WARNING: this scan lost sight of the bus' + f'{hil_util.sysfs_blind_note()}. Rows above that say a probe or board is ' + f'missing may be this tool losing sight of healthy hardware, not absent ' + f'hardware. Find the wedged device (see the usb-kernel-recover skill) and ' + f're-run before acting on the table.') + sys.exit(min(counts['flash-failed'] + counts['failed'], 125)) + + +if __name__ == '__main__': + main() diff --git a/test/hil/helper/hil_select.py b/test/hil/helper/hil_select.py new file mode 100755 index 000000000..f0d4f0b9f --- /dev/null +++ b/test/hil/helper/hil_select.py @@ -0,0 +1,524 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""PR-diff -> HIL selection: which rig boards and which tests a change can affect. + +Stdlib-only (runs on bare CI runners; imports hil_util for the example rosters, +never hil_test/pyserial — test_hil_util.BottomLayer enforces the stdlib closure). +Fail-open: any file no rule classifies forces the full matrix. See +docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md. + +JSON: full, boards (name -> 'all' | [tests]), families (bsp families the diff +touches, including ones with no rig board - build-only consumers such as /pre-pr +sample from these), args (hil_test.py args per config) and args_flasher (the same +args split by each board's flasher, for CI legs that split one rig by flasher). +""" +import argparse +import functools +import glob +import json +import os +import re +import subprocess +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # helper/ scripts import via the test/hil root +from helper.hil_util import device_tests, dual_tests, host_test + +ALL_TESTS = {'device': device_tests, 'dual': dual_tests, 'host': host_test} + +# class dir -> config macro suffix exceptions (rule 3); dfu is per-file, handled inline +NET_MACROS = ('ECM_RNDIS', 'NCM') + +_NONCODE_RE = re.compile( + r'^(docs/|\.claude/|.*\.(md|rst)$|LICENSE)') +_FULL_RE = re.compile( + r'^(src/common/|src/osal/|src/tusb\.c$|src/tusb\.h$|src/tusb_option\.h$|' + r'test/hil/|\.github/workflows/build.*\.yml$|\.github/actions/|\.github/scripts/|' + r'tools/build\.py$|tools/get_deps\.py$|tools/cmake/|hw/mcu/|lib/|' + r'hw/bsp/(family_support\.cmake|board_api\.h|board\.c|ansi_escape\.h)$|' + r'examples/build_system/|examples/CMakeLists\.txt$|' + # board_test is HIL infrastructure, not a test: hil_test.py flashes it to park + # every board (variant boundary + end-of-board teardown), so every board depends on it + r'examples/device/board_test/)') + +# --no-renames: with rename detection git reports only a rename's destination, so code +# moved out of an HIL-relevant path would be classified by its new path alone +GIT_DIFF_ARGV = ['git', 'diff', '--no-renames', '--name-only'] + + +def test_role(test: str) -> str: + return test.split('/', 1)[0] # 'device' | 'dual' | 'host' + + +def board_roles(board: dict) -> set: + t = board.get('tests', {}) + roles = set() + if t.get('device'): + roles.add('device') + if t.get('host'): + roles.add('host') + if t.get('dual'): + roles.update(('device', 'host')) + for only in t.get('only', []): + r = test_role(only) + roles.update(('device', 'host') if r == 'dual' else (r,)) + return roles + + +def board_tests(board: dict) -> list: + """Every test this board would run today (mirrors hil_test.test_board's default).""" + t = board.get('tests', {}) + if 'only' in t: + run = list(t['only']) + else: + run = [] + if t.get('device'): + run += device_tests + if t.get('dual'): + run += dual_tests + if t.get('host'): + run += host_test + return [x for x in run if x not in t.get('skip', [])] + + +# cached: called per changed file x roster board, and the tree doesn't change mid-run +@functools.lru_cache(maxsize=None) +def board_family(board_name: str, repo_root: str): + hits = glob.glob(os.path.join(repo_root, 'hw/bsp/*/boards', board_name)) + return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None + + +# `if (OPTION STREQUAL "1")` guards in family_support.cmake, and the option tokens +# a roster entry passes to the build (NAME=VALUE / -DNAME=VALUE) +_CM_IF_RE = re.compile(r'if\s*\(') +_CM_ELSE_RE = re.compile(r'else(if)?\s*\(') +_CM_ENDIF_RE = re.compile(r'endif\s*\(') +_CM_OPT_RE = re.compile(r'if\s*\(\s*\$?\{?([A-Za-z_]\w*)\}?\s+STREQUAL\s+"?1"?\s*\)') +_CM_PORT_RE = re.compile(r'src/portable/((?:[^/\s]+/)?[^/\s]+)/') +_FALSY = ('', '0', 'off', 'false', 'no') + + +@functools.lru_cache(maxsize=None) +def port_option_gates(repo_root: str) -> dict: + """port dir -> build options that compile it regardless of the board's family + file, e.g. {'analog/max3421': {'MAX3421_HOST'}} from family_support.cmake.""" + gates = {} + try: + text = open(os.path.join(repo_root, 'hw/bsp/family_support.cmake')).read() + except OSError: + return gates + stack = [] # one entry per open if(): its option, or None + for line in text.splitlines(): + line = line.strip() + if _CM_IF_RE.match(line): + m = _CM_OPT_RE.match(line) + stack.append(m.group(1) if m else None) + elif _CM_ELSE_RE.match(line): + if stack: + stack[-1] = None # the guard doesn't hold in this branch + elif _CM_ENDIF_RE.match(line): + if stack: + stack.pop() + opts = {o for o in stack if o} + m = _CM_PORT_RE.search(line) + if opts and m: + gates.setdefault(m.group(1), set()).update(opts) + return gates + + +_CM_SET_RE = re.compile(r'set\s*\(\s*([A-Za-z_]\w*)\s+([^)\s]+)\s*\)') + + +# cached: called per changed portable file x roster board +@functools.lru_cache(maxsize=None) +def bsp_board_options(board_name: str, repo_root: str) -> frozenset: + """Build options a board turns on in its own BSP: `set( )` in + hw/bsp//boards//board.cmake, e.g. MAX3421_HOST on the espressif + and rp2040 max3421 boards. CMake only - HIL CI builds nothing with Make, so a + board.mk-only option (e.g. nrf5340dk's MAX3421_HOST) compiles no port here.""" + fam = board_family(board_name, repo_root) + if not fam: + return frozenset() + path = os.path.join(repo_root, 'hw/bsp', fam, 'boards', board_name, 'board.cmake') + try: + text = open(path).read() + except OSError: + return frozenset() + out = set() + for line in text.splitlines(): + line = line.strip() + if line.startswith('#'): + continue + m = _CM_SET_RE.match(line) + if m and m.group(2).strip('"').lower() not in _FALSY: + out.add(m.group(1)) + return frozenset(out) + + +def board_options(board: dict, repo_root: str) -> set: + """Build options a board has truthy: the roster entry's build.args plus each + variant's defines (NAME=VALUE) and raw CFLAGS (-DNAME=VALUE), plus whatever its + own board.cmake sets (a board can enable a gated port without the roster saying so).""" + toks = list(board.get('build', {}).get('args', [])) + for v in board.get('variant', []): + toks += list(v.get('defines', [])) + toks += v.get('flags', '').split() + out = set(bsp_board_options(board['name'], repo_root)) + for t in toks: + name, _, val = (t[2:] if t.startswith('-D') else t).partition('=') + if name and val.strip().strip('"').lower() not in _FALSY: + out.add(name.strip()) + return out + + +@functools.lru_cache(maxsize=None) +def port_families(port_dir: str, repo_root: str) -> set: + """Board families that compile this src/portable dir. CMake only: HIL CI builds + every board with CMake, so a port wired up in family.mk alone is compiled for no + HIL board and must not select one. family.cmake lists portable sources directly + for most families; espressif instead references them from a nested component + CMakeLists.txt (hw/bsp/espressif/components/tinyusb_src/CMakeLists.txt).""" + fams = set() + bsp_root = os.path.join(repo_root, 'hw/bsp') + # trailing '/' so a port dir is not a prefix of a sibling: bare 'microchip/pic' + # would otherwise match '.../microchip/pic32mz/...' and inherit its families + needle = port_dir + '/' + for f in glob.glob(os.path.join(bsp_root, '*/family.cmake')) + \ + glob.glob(os.path.join(bsp_root, '*/components/*/CMakeLists.txt')): + try: + if needle in open(f).read(): + fam = os.path.relpath(f, bsp_root).split(os.sep, 1)[0] + fams.add(fam) + except OSError: + pass + return fams + + +_CLS_INC_RE = re.compile(r'#\s*include\s*[<"]class/([^/"<>]+)/([^"<>]+)[">]') + + +@functools.lru_cache(maxsize=None) +def class_include_edges(repo_root: str) -> dict: + """'/
' -> the other class dirs that include it. A class header + pulled in by a second class ships in every firmware enabling that second class: + src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, and + net_device.h includes class/cdc/cdc.h. The class rule derives macros from the + directory name alone, so without this edge a change to the included header + selects only its own class's examples - and on a board that skips those (e.g. + metro_m4_express skips audio_test_freertos), nothing at all. + + Derived from the actual #include lines rather than a hand-written table so it + cannot rot when a class picks up or drops a cross-class include.""" + edges = {} + for f in sorted(glob.glob(os.path.join(repo_root, 'src/class/*/*.[ch]'))): + cls = os.path.basename(os.path.dirname(f)) + try: + text = open(f).read() + except OSError: + continue + for inc_cls, inc_hdr in _CLS_INC_RE.findall(text): + if inc_cls != cls: + edges.setdefault(f'{inc_cls}/{inc_hdr}', set()).add(cls) + return edges + + +def class_macros(cls: str, base: str, prefix: str) -> list: + """Config macros that compile a class dir's code, for role prefix TUD/TUH. + `base` refines dfu only (it splits DFU from DFU_RUNTIME per file); pass '' for + a class reached through an include edge, where the widest set is correct.""" + if cls == 'net': + return [f'CFG_{prefix}_{m}' for m in NET_MACROS] + if cls == 'dfu': + if base.startswith('dfu_rt'): + return [f'CFG_{prefix}_DFU_RUNTIME'] + if base.startswith('dfu_device') or base.startswith('dfu_host'): + return [f'CFG_{prefix}_DFU'] + return [f'CFG_{prefix}_DFU', f'CFG_{prefix}_DFU_RUNTIME'] + return [f'CFG_{prefix}_{cls.upper()}'] + + +def _config_enables(cfg_path: str, macros) -> bool: + try: + text = open(cfg_path).read() + except OSError: + return False + return any(re.search(rf'#define\s+{m}\s+\(?\s*0*[1-9]', text) for m in macros) + + +def roster_only_tests(all_boards) -> set: + """Test paths that only appear in a roster board's tests.only list (e.g. + espressif boards), not in the shared device/dual/host_test lists.""" + out = set() + for b in all_boards: + out.update(b.get('tests', {}).get('only', [])) + return out + + +def class_examples(macros, role: str, repo_root: str, extra_tests: set) -> set: + """Tests (from role's + dual lists, plus roster-only-list tests of that role) + whose example config enables any macro.""" + pool = role_tests({role}, extra_tests) + out = set() + for test in pool: + cfg = os.path.join(repo_root, 'examples', test, 'src', 'tusb_config.h') + if _config_enables(cfg, macros): + out.add(test) + return out + + +def role_tests(roles: set, extras: set) -> set: + """Every test for the given role(s): each role's own list + dual tests, + plus roster-only-list tests (extras) matching those roles or 'dual'.""" + pool = set(dual_tests) + for r in roles: + pool |= set(ALL_TESTS[r]) + pool |= {t for t in extras if test_role(t) in roles or test_role(t) == 'dual'} + return pool + + +class _Sel: + """Accumulates contributions. board->set(tests) plus 'all-board' markers.""" + def __init__(self): + self.full = False + self.by_board = {} # name -> set of tests, or 'all' + self.roles = set() # roles touched by any contribution + self.families = set() # bsp families touched (incl. off-rig ones: build-only consumers) + self.reasons = [] + + def add(self, boards, tests, reason): + """tests: 'all' or iterable of test paths.""" + self.reasons.append(reason) + for b in boards: + cur = self.by_board.get(b) + if tests == 'all' or cur == 'all': + self.by_board[b] = 'all' + else: + self.by_board[b] = (cur or set()) | set(tests) + + def force_full(self, reason): + self.full = True + self.reasons.append(reason) + + +def _classify_one(path, repo_root, roster_boards, extras: set, s: _Sel): + base = os.path.basename(path) + if _NONCODE_RE.match(path): + s.reasons.append(f'{path}: non-code, no contribution') + return + if _FULL_RE.match(path): + s.force_full(f'{path}: core/infra -> full matrix') + return + + m = re.match(r'src/portable/((?:[^/]+/)?[^/]+)/', path) + if m: + port = m.group(1) + if re.match(r'(dcd_|.*_device)', base): + roles = {'device'} + elif re.match(r'(hcd_|.*_host)', base): + roles = {'host'} + else: + roles = {'device', 'host'} + fams = port_families(port, repo_root) + if not fams: + # no family references this port: either a new/renamed port dir or a + # family.cmake layout the scan misses - widen instead of contributing nothing + s.force_full(f'{path}: port {port} maps to no board family -> full matrix') + return + s.families.update(fams) + # a board can also pull the port in through a build option (e.g. MAX3421_HOST=1 + # from the roster on metro_m4_express, or from its own board.cmake), which its + # family file never names + gates = port_option_gates(repo_root).get(port, set()) + boards = [b['name'] for b in roster_boards + if (board_family(b['name'], repo_root) in fams or + (gates and board_options(b, repo_root) & gates)) and (board_roles(b) & roles)] + tests = role_tests(roles, extras) + s.roles.update(roles) + why = f'{path}: port {port} -> families {sorted(fams)}' + if gates: + why += f' + option {sorted(gates)}' + s.add(boards, tests, f'{why} -> boards {boards} ({"/".join(sorted(roles))})') + return + + m = re.match(r'src/class/([^/]+)/', path) + if m: + cls = m.group(1) + if re.search(r'_device\.[ch]$', base): + roles = {'device'} + elif re.search(r'_host\.[ch]$', base): + roles = {'host'} + else: + roles = {'device', 'host'} + # this file's own class, plus any class whose headers include it + via = sorted(class_include_edges(repo_root).get(f'{cls}/{base}', ())) + + def macros(prefix): + return (class_macros(cls, base, prefix) + + [m2 for c in via for m2 in class_macros(c, '', prefix)]) + tests = set() + if 'device' in roles: + tests |= class_examples(macros('TUD'), 'device', repo_root, extras) + if 'host' in roles: + tests |= class_examples(macros('TUH'), 'host', repo_root, extras) + boards = [b['name'] for b in roster_boards if board_roles(b) & roles] + s.roles.update(roles) + why = f'{path}: class {cls}' + (f' (+ included by {via})' if via else '') + s.add(boards, tests, f'{why} -> {sorted(tests)} ({"/".join(sorted(roles))})') + return + + m = re.match(r'src/(device|host)/', path) + if m: + role = m.group(1) + boards = [b['name'] for b in roster_boards if role in board_roles(b)] + s.roles.add(role) + s.add(boards, role_tests({role}, extras), f'{path}: core {role} stack -> all {role} tests') + return + + m = re.match(r'hw/bsp/([^/]+)/(?:boards/([^/]+)/)?', path) + if m: + fam, brd = m.group(1), m.group(2) + s.families.add(fam) + if brd: + boards = [b['name'] for b in roster_boards if b['name'] == brd] + why = f'{path}: bsp board {brd}' + else: + boards = [b['name'] for b in roster_boards + if board_family(b['name'], repo_root) == fam] + why = f'{path}: bsp family {fam}' + s.roles.update(('device', 'host')) + s.add(boards, 'all', f'{why} -> boards {boards}') + return + + m = re.match(r'examples/(device|host|dual)/([^/]+)/', path) + if m: + test = f'{m.group(1)}/{m.group(2)}' + known = any(test in pool for pool in ALL_TESTS.values()) or test in extras + if known: + boards = [b['name'] for b in roster_boards] + role = test_role(test) + s.roles.update(('device', 'host') if role == 'dual' else (role,)) + s.add(boards, [test], f'{path}: example -> {test} on all boards') + else: + s.reasons.append(f'{path}: example not in HIL lists, no contribution') + return + + s.force_full(f'{path}: unclassified -> full matrix') + + +def classify(changed_files, repo_root, rosters): + all_boards = [] + seen = set() + for _, boards in rosters: + for b in boards: + if b['name'] not in seen: + seen.add(b['name']) + all_boards.append(b) + + extras = roster_only_tests(all_boards) + s = _Sel() + # no early exit once full: keep classifying so `families` still reports every + # family the diff touches (build-only consumers need it). Nothing after the first + # force_full can change full/boards/args - the full branch below ignores by_board. + for path in changed_files: + _classify_one(path, repo_root, all_boards, extras, s) + + if s.full: + return {'full': True, 'boards': {b['name']: 'all' for b in all_boards}, + 'families': sorted(s.families), 'reasons': s.reasons} + + # role pruning: single-role selections drop the other role's tests and boards + by_name = {b['name']: b for b in all_boards} + out = {} + for name, tests in s.by_board.items(): + allowed = board_tests(by_name[name]) + if tests == 'all': + kept = list(allowed) + else: + kept = [t for t in allowed if t in tests] + if s.roles and s.roles != {'device', 'host'}: + role = next(iter(s.roles)) + kept = [t for t in kept if test_role(t) in (role, 'dual')] + if kept: + out[name] = 'all' if set(kept) == set(allowed) else sorted(kept) + return {'full': False, 'boards': out, 'families': sorted(s.families), + 'reasons': s.reasons} + + +def _board_args(name, chosen) -> list: + parts = [f'-b {name}'] + if chosen != 'all': + parts.append(f'-bt {name}:{",".join(chosen)}') + return parts + + +def selection_args(sel, rosters): + """hil_test.py args per config. Empty means either 'full matrix' or 'nothing + selected' - callers must read sel['full'] to tell them apart.""" + args = {} + for cfg_path, boards in rosters: + parts = [] + if not sel['full']: + for b in boards: + chosen = sel['boards'].get(b['name']) + if chosen is not None: + parts += _board_args(b['name'], chosen) + args[os.path.basename(cfg_path)] = ' '.join(parts) + return args + + +def selection_args_by_flasher(sel, rosters): + """{config: {flasher name: args}}. CI runs one rig as several jobs split by + flasher (esptool vs the rest); each must gate on its own subset, otherwise the + other leg runs a filter matching zero boards and reports a vacuous green.""" + out = {} + for cfg_path, boards in rosters: + per = {} + if not sel['full']: + for b in boards: + chosen = sel['boards'].get(b['name']) + if chosen is None: + continue + per.setdefault(b.get('flasher', {}).get('name', ''), []).extend( + _board_args(b['name'], chosen)) + out[os.path.basename(cfg_path)] = {f: ' '.join(p) for f, p in per.items()} + return out + + +def changed_files_from_git(base, repo_root): + mb = subprocess.run(['git', 'merge-base', 'HEAD', base], cwd=repo_root, + capture_output=True, text=True, check=True).stdout.strip() + diff = subprocess.run(GIT_DIFF_ARGV + [f'{mb}..HEAD'], cwd=repo_root, + capture_output=True, text=True, check=True).stdout + return [l for l in diff.splitlines() if l.strip()] + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + g = ap.add_mutually_exclusive_group(required=True) + g.add_argument('--base', help='git ref to diff against (merge-base..HEAD)') + g.add_argument('--diff-file', help='newline-separated changed-file list') + ap.add_argument('configs', nargs='+', help='rig roster JSON file(s)') + a = ap.parse_args() + + # test/hil/helper/ -> repo root is FOUR levels up; three left this at /test + # after the helper/ move and every repo-relative glob silently matched nothing + repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) + rosters = [] + for c in a.configs: + with open(c) as f: + rosters.append((c, json.load(f)['boards'])) + + files = (open(a.diff_file).read().splitlines() if a.diff_file + else changed_files_from_git(a.base, repo_root)) + files = [f for f in files if f.strip()] + + s = classify(files, repo_root, rosters) + s['args'] = selection_args(s, rosters) + s['args_flasher'] = selection_args_by_flasher(s, rosters) + for r in s['reasons']: + print(f'hil_select: {r}', file=sys.stderr) + print(json.dumps(s)) + + +if __name__ == '__main__': + main() diff --git a/test/hil/helper/hil_util.py b/test/hil/helper/hil_util.py new file mode 100644 index 000000000..54984d20f --- /dev/null +++ b/test/hil/helper/hil_util.py @@ -0,0 +1,585 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Bottom layer of the HIL harness: the bounded command runner plus the shared helpers and +# data every other module needs. Stays stdlib-only and imports nothing local -- everything +# else imports this, including the unit tests on GitHub's bare runner; never import them +# from here. Callers set the module global `verbose`. + +from __future__ import annotations + +import glob +import os +import signal +import subprocess +import threading +import sys +from pathlib import Path +from typing import Any + + +# ------------------------------------------------------------- +# HIL example test lists, shared by hil_test.py (runner) and hil_select.py (PR-diff +# selector). Run order is shuffled per board (see test_board); every example carries a +# unique hardcoded idProduct (see its usb_descriptors.c). +# ------------------------------------------------------------- + +# device tests +device_tests = [ + 'device/cdc_dual_ports', + 'device/cdc_msc', + 'device/dfu', + 'device/cdc_msc_throughput', + 'device/audio_test_freertos', + 'device/dfu_runtime', + 'device/cdc_msc_freertos', + 'device/hid_boot_interface', + 'device/msc_dual_lun', + 'device/hid_generic_inout', + 'device/printer_to_cdc', + 'device/midi_test', + 'device/mtp', + 'device/usbtest', # cafe:4010, unique PID; runs the Linux testusb tier-4 battery via usbtest.py + # 'device/net_lwip_webserver', # disabled for PR #3605: USB net iface enum is flaky on the CI HIL host +] + +dual_tests = [ + 'dual/host_info_to_device_cdc', +] + +host_test = [ + 'host/cdc_msc_hid', + 'host/msc_file_explorer', + 'host/msc_file_explorer_freertos', + 'host/device_info', +] + +verbose = False + +def pos_int_env(name: str, default: int) -> int: + # One parsing policy for every HIL_* knob: a bare int() crashes every run at import + # on a malformed value, and 0/negative silently removes the bound the knob enforces. + try: + v = int(os.getenv(name, str(default))) + except ValueError: + print(f'warning: {name} is not an integer; using {default}', + file=sys.stderr, flush=True) + return default + if v <= 0: + print(f'warning: {name}={v} is not usable; using {default}', + file=sys.stderr, flush=True) + return default + return v + + +def pos_float_env(name: str, default: float) -> float: + try: + v = float(os.getenv(name, str(default))) + except ValueError: + print(f'warning: {name} is not a number; using {default}', + file=sys.stderr, flush=True) + return default + # float() accepts 'inf'/'nan': an infinite serial timeout is an unbounded read, the + # very thing these knobs exist to prevent, and nan fails every comparison silently + if not (v > 0 and v < float('inf')): + print(f'warning: {name}={v} is not usable; using {default}', + file=sys.stderr, flush=True) + return default + return v + + +CMD_TIMEOUT = pos_int_env('HIL_CMD_TIMEOUT', 180) + +TINYUSB_ROOT = Path(__file__).resolve().parents[3] # test/hil/helper/ -> repo root + + +def cmd_stdout_text(out: Any) -> str: + if out is None: + return '' + if isinstance(out, bytes): + return out.decode('utf-8', errors='ignore') + return str(out) + + +def _banner_body(out: Any, err: Any) -> str: + # split_stderr callers keep the diagnostic in stderr — a banner of stdout alone + # would be blank exactly when something went wrong + body = cmd_stdout_text(out) + err_text = cmd_stdout_text(err) + if err_text: + body = f'{body}\n{err_text}' if body else err_text + return body + + +# Shared with compact_output's stripper in hil_test: duplicated literals let the two +# layers drift and reintroduce literal marker noise mid-row in the GitHub log. +GROUP_MARK, ENDGROUP_MARK = '::group::', '::endgroup::' + + +def strip_workflow_markers(line: str) -> str: + # run_cmd only ever emits markers at line start; mid-line is not a real case. + return line.removeprefix(GROUP_MARK).removeprefix(ENDGROUP_MARK) + + +def _ci_log_groups() -> bool: + # GitHub folds ::group::/::endgroup:: only at line start of the JOB's real stdout; a + # pool worker's capture is compacted into one row line, where they render literally. + return bool(os.getenv('CI')) and sys.stdout is sys.__stdout__ + + +def _print_banner(title: str, out: Any, err: Any) -> None: + print() + if _ci_log_groups(): + print(f'{GROUP_MARK}{title}') + print(_banner_body(out, err)) + print(ENDGROUP_MARK) + else: + print(title) + print(_banner_body(out, err)) + + +SYSFS_READ_GRACE = 2.0 # bound on one attribute read of a possibly-wedged device +SYSFS_STUCK_MAX = 4 # stranded readers tolerated before read_sysfs goes blind +_sysfs_stuck = 0 # each costs a thread + an fd for the life of the process +_sysfs_stuck_lock = threading.Lock() +_sysfs_blind_logged = False + + +class _SysfsUnknown: + """Sentinel: the read did not answer. NOT "the attribute is absent" -- reading it as + absence turns a healthy board into a firmware regression in the report.""" + __slots__ = () + + def __bool__(self) -> bool: + return False + + def __repr__(self) -> str: + return 'SYSFS_UNKNOWN' + + +SYSFS_UNKNOWN = _SysfsUnknown() + + +def sysfs_blind() -> bool: + """True once this process has stranded SYSFS_STUCK_MAX readers: every later read + answers SYSFS_UNKNOWN, so nothing it reports about a device is a fact any more.""" + return _sysfs_stuck >= SYSFS_STUCK_MAX + + +def sysfs_blind_note() -> str: + """Suffix for a failure message, so a blind worker's verdict never reads as hardware.""" + return (f' (this worker is blind: {SYSFS_STUCK_MAX} sysfs reads stranded on a wedged ' + f'device, so the check could not see the bus)') if sysfs_blind() else '' + + +def read_sysfs(path: str, grace: float = SYSFS_READ_GRACE) -> str | None | _SysfsUnknown: + """Read a sysfs attribute with a WALL-CLOCK bound. + + The value, None when the attribute is genuinely unreadable (OSError), or SYSFS_UNKNOWN + when the read did not answer -- it timed out, or this process is already blind. Callers + MUST keep those apart: absence is a fact, unknown is not. + + usb_string_attr (serial/product/manufacturer) is served under the device lock a wedged + usbfs ioctl holds, so a plain open().read() blocks for as long as the wedge lasts, on + exactly the board an incident is about. The reader sleeps INTERRUPTIBLY (every read + takes usb_lock_device_interruptible, v6.12.96 sysfs.c:124-139 -- uninterruptible is the + ioctl holder, not us), so it dies with a SIGKILLed worker; what it costs meanwhile is a + thread and an fd for this process's life, because on sysfs the open() SUCCEEDS and only + the read blocks. Measured: 20 blocking reads leave 20 live threads. + + Hence the cap: callers rescan (hil_lock's controller_of re-reads every unresolved + device on EVERY permit), and hitting RLIMIT_NOFILE or the thread ceiling raises inside + the worker and loses every board's result -- worse than the hang this prevents. + """ + if sysfs_blind(): + return SYSFS_UNKNOWN + # Known-stranded? Re-reading costs another permanent thread+fd and a blindness credit + # to learn what we already know. Lives HERE, not at the call sites: a call-site memo + # has to be remembered by every new scanner, and twice it was not. + was = _sysfs_stranded.get(path, _STRAND_MISS) + if was is not _STRAND_MISS: + if was is None: + return SYSFS_UNKNOWN # stranded, inode unknown: never re-read it + try: + if os.stat(path).st_ino == was: + return SYSFS_UNKNOWN # same node, still wedged + except OSError: + pass # gone: fall through, the read reports it + _sysfs_stranded.pop(path, None) # replaced or gone -> re-read it + out: dict = {} + + def _read(): + try: + with open(path) as f: + out['v'] = f.read().strip() + except (OSError, ValueError): + pass # no such attribute, or not text: unreadable, and that IS a fact + + t = threading.Thread(target=_read, daemon=True) + t.start() + t.join(grace) + # `out` FIRST, not is_alive() alone: a reader can deposit its value and still be alive + # for a moment afterwards, and counting that as a strand memoises a healthy attribute as + # unreadable and spends one of four blindness credits. bounded_open has always checked + # its box for the same reason. + if t.is_alive() and 'v' not in out: + # Count the PATH once, not once per reader. hil_pool_check runs -j4 by default, + # which equals SYSFS_STUCK_MAX, so four threads hitting ONE wedged device used to + # spend the entire blindness budget between them -- latching blind on the single + # wedge the tool was run to find. The strand is real for each thread, but the + # DEVICE is what the cap is about. + # Under the SAME lock as the counter: check-then-act here is a race, and + # hil_pool_check runs a ThreadPoolExecutor of exactly SYSFS_STUCK_MAX workers in + # ONE process, so four threads on one wedged path could each see `first` before any + # of them recorded it -- spending the whole blindness budget on a single device, + # which is what this memo exists to prevent. note_sysfs_strand takes the lock + # itself, so call it after releasing. + with _sysfs_stuck_lock: + first = path not in _sysfs_stranded + if first: + try: + # stat, never the thread's own open(): stat does not call ->show(), so + # it cannot block on the device lock the reader is stuck behind + _sysfs_stranded[path] = os.stat(path).st_ino + except OSError: + _sysfs_stranded[path] = None # unstattable, but still known-stranded + if first: + note_sysfs_strand() + return SYSFS_UNKNOWN + return out.get('v') + + +def note_sysfs_strand() -> None: + """Record ONE stranded sysfs reader. Shared by read_sysfs and bounded_open so both + account against a single counter -- the report caveat keys off it.""" + global _sysfs_stuck, _sysfs_blind_logged + with _sysfs_stuck_lock: + _sysfs_stuck += 1 + announce = sysfs_blind() and not _sysfs_blind_logged + _sysfs_blind_logged = _sysfs_blind_logged or announce + if announce: + # once per process, on stderr: a worker's stdout is compacted into one report + # row, where this would be lost among the test output + print(f'warning: {SYSFS_STUCK_MAX} sysfs reads stranded on a wedged device; ' + f'this process is now blind and answers SYSFS_UNKNOWN for every ' + f'attribute -- its verdicts about device presence are not evidence', + file=sys.stderr, flush=True) + + +# path -> the inode it had when its read stranded. A stranded attribute stays +# stranded until the DEVICE is replaced, and a re-enumeration destroys the kernfs +# node and makes a new one -- so a changed inode is the all-clear. Keyed by path +# alone it would outlive the wedge: a busport does not change when a board comes +# back on the same port, so the HUNG reflash this branch performs would recover a +# board the harness could then never see again. +_sysfs_stranded: dict = {} +# A stranded path whose inode could not be read is stored as None, so a plain .get() cannot +# tell 'known stranded, inode unknown' from 'never seen' -- and treating the first as the +# second re-reads it, stranding another permanent thread and fd every call. Distinct miss +# sentinel, so None keeps its own meaning. +_STRAND_MISS = object() + + +def usb_scan(vid_pid=None, serial=None, vid=None) -> tuple[list, bool]: + """Enumerated USB devices matching the filters, and whether anything is unknown. + + Returns ([{busport, dir, vid, pid, serial}], unknown). `unknown` True means a bounded + read did not answer, so absence is NOT proven -- the same contract as read_sysfs. + + Three rules, one implementation for every caller: + + * Root hubs excluded (glob `*-*`): no DUT is one, and scans including them measured + seconds slower (observation, no mechanism -- the "autosuspend wake" explanation was + wrong; usb_string_attr reads a cached string, sysfs.c:124-139). + * idVendor/idProduct first: lock-free `sysfs_emit` from udev->descriptor + (sysfs.c:688-705), so they rule out nearly every device for free. + * `serial` last and bounded: it is served under the lock a wedged ioctl holds, and a + path that already stranded is never re-read (each strand costs a thread and an fd + for this process's life). + """ + out = [] + unknown = False + for d in glob.glob('/sys/bus/usb/devices/*-*'): + # Interfaces are ':.' (e.g. 2-4:1.0) -- they CONTAIN the + # colon, they do not end with it, so the original endswith() never fired and every + # scan opened idVendor/idProduct on all of them (measured: 31 of 44 matches). + if ':' in os.path.basename(d): + continue + try: + with open(os.path.join(d, 'idVendor')) as f: + dev_vid = f.read().strip() + with open(os.path.join(d, 'idProduct')) as f: + dev_pid = f.read().strip() + except OSError: + continue # vanished mid-walk, or not a device dir: a fact, not unknown + if vid_pid is not None and (dev_vid, dev_pid) != tuple(vid_pid): + continue # ruled out for free, without touching the locked attribute + if vid is not None and dev_vid != vid: + continue # same, for callers that know the VID but not the PID + sn = read_sysfs(os.path.join(d, 'serial')) + if sn is SYSFS_UNKNOWN: + unknown = True # read_sysfs memoises it; a repeat scan costs nothing + continue + if sn is None: + continue # no serial attribute: a fact + if serial is not None and sn.lower() != serial.lower(): + continue + out.append({'busport': os.path.basename(d), 'dir': d, + 'vid': dev_vid, 'pid': dev_pid, 'serial': sn}) + return out, unknown + + +def bounded_open(path: str, flags: int, timeout: float = SYSFS_READ_GRACE): + """os.open() with a wall-clock bound. + + The fd, None when the open genuinely FAILED (OSError: EBUSY, ENOENT, EACCES), or + SYSFS_UNKNOWN when it did not answer -- the same three-valued contract as read_sysfs, + and for the same reason: folding a fact into an unknown made an ordinary EBUSY read as + a wedged device and sent the operator hunting hardware that is healthy. + + An open CAN block on a wedged device -- not on O_NONBLOCK, which usblp_open never + consults, but on usb_autopm_get_interface(), a runtime-PM resume that does I/O + (v6.12.96 drivers/usb/class/usblp.c). It holds usblp_mutex while it waits, and that + mutex is driver-GLOBAL, so one wedged printer blocks opens of every usblp node. + + Unlike read_sysfs the stranded thread cleans up after itself: if we have given up it + closes the fd it eventually got, so only the thread leaks. Both sides take `handoff` + -- "store or close" and "abandon and drain" are a check-then-act pair that can + interleave into an fd stored after the box was drained, which would leak it into a + node that allows a SINGLE opener (usblp_open returns -EBUSY when usblp->used). + """ + # Same short-circuit as read_sysfs: once blind, another stranded thread buys nothing + # and the cap exists precisely to stop them accumulating. + if sysfs_blind(): + return SYSFS_UNKNOWN + # Known-stranded? Re-opening costs another thread, another fd and another blindness + # credit to learn what we already know -- and the printer test re-opens ONE lp node on + # every retry. Same memo and same inode check as read_sysfs. + was = _sysfs_stranded.get(path, _STRAND_MISS) + if was is not _STRAND_MISS: + if was is None: + return SYSFS_UNKNOWN # stranded, inode unknown: never re-read it + try: + if os.stat(path).st_ino == was: + return SYSFS_UNKNOWN + except OSError: + pass + _sysfs_stranded.pop(path, None) + box: dict = {} + done, abandoned = threading.Event(), threading.Event() + handoff = threading.Lock() + + def _open(): + try: + fd = os.open(path, flags) + except OSError: + done.set() + return + with handoff: + stored = not abandoned.is_set() + if stored: + box['fd'] = fd + if not stored: + try: + os.close(fd) + except OSError: + pass + done.set() + + threading.Thread(target=_open, daemon=True).start() + if not done.wait(timeout): + with handoff: + abandoned.set() + fd = box.pop('fd', None) # completed in the gap between timeout and flag + if fd is not None: + # It DID open, just after our deadline -- the thread finished, so nothing is + # stranded. Report unknown (we already gave up on it) but do not spend a + # blindness credit, and do not call a merely-slow node wedged. + try: + os.close(fd) + except OSError: + pass + return SYSFS_UNKNOWN + # counted like a stranded read_sysfs: the thread and (eventually) its fd are gone + # for the life of the process, and the cap exists to stop that reaching the + # thread/fd ceiling -- an exception there escapes the worker and loses every board. + # Memoised by inode so a retry of the same node does not pay again. + # same lock as read_sysfs, same reason + with _sysfs_stuck_lock: + first = path not in _sysfs_stranded + if first: + try: + _sysfs_stranded[path] = os.stat(path).st_ino + except OSError: + _sysfs_stranded[path] = None + if first: + note_sysfs_strand() + return SYSFS_UNKNOWN + return box.get('fd') + + +def _close_pipes(p: subprocess.Popen) -> None: + """Close OUR ends of an abandoned child's pipes. Never raises.""" + for pipe in (p.stdout, p.stderr, p.stdin): + try: + if pipe is not None: + pipe.close() + except OSError: + pass + + +def run_alongside(argv: list, work, timeout: int) -> subprocess.CompletedProcess: + """Run `argv` alongside `work()`, which runs in THIS thread, then reap it -- bounded. + + The read-while-we-write shape run_cmd cannot express: the caller needs the child + RUNNING while it does something else. Everything else about the contract is run_cmd's + -- own session, killpg, bounded reap, our pipe ends closed, rc 124 on the kill. + + A PROCESS, not a thread: an abandoned thread keeps the fd, and usblp_open returns + -EBUSY while usblp->used (v6.12.96 usblp.c), so every later open in this long-lived + worker would read as a wedged device. A killed process takes its fd with it. + + stdout is captured as BYTES and kept CLEAN -- a caller byte-compares it against the + payload it sent, so a single stderr byte (a PYTHONWARNINGS chirp, a sitecustomize + print, a .pth deprecation from a venv) would read as USB data corruption. stderr gets + its own pipe; communicate() drains both, so the split cannot deadlock. + `work` runs even if the child dies immediately -- the caller's own asserts decide. + """ + p = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + start_new_session=True) + + def _reap() -> subprocess.CompletedProcess: + try: + out, err = p.communicate(timeout=timeout) + return subprocess.CompletedProcess(argv, p.returncode, out, err) + except subprocess.TimeoutExpired: + try: + os.killpg(p.pid, signal.SIGKILL) + except OSError: + p.kill() + try: + out, err = p.communicate(timeout=5) + except subprocess.TimeoutExpired: + # Outlasted SIGKILL: uninterruptible, still holding whatever it opened. + # Abandoned like any other stray -- but as a real child in its own + # session, so the containment sweep FINDS it (child_procs walks the ppid + # tree) and the report names it. That is the whole difference from a + # blocked thread, which no sweep can see and no signal can reach. + out, err = b'', b'' + _close_pipes(p) # our own fds must not leak either + return subprocess.CompletedProcess(argv, 124, out, err) + + try: + work() + except BaseException: + # Reap first so the child never outlives us, then let the caller's error through. + # A `return` inside a `finally` would SWALLOW it -- an assert in `work` would + # vanish and the caller would compare data it never finished sending. + _reap() + raise + return _reap() + + +def run_cmd(cmd: str, cwd: str | None = None, timeout: int | None = None, + binary: bool = False, split_stderr: bool = False, + quiet: bool = False) -> subprocess.CompletedProcess: + if timeout is None: + timeout = CMD_TIMEOUT + # binary: raw bytes (text mode's errors='replace' mangles non-UTF-8 file content). + # split_stderr: keep stderr out of stdout, for callers that parse stdout. quiet: no + # COMMAND FAILED banner, for retry loops that report failures themselves (timeouts + # still print: a killed child is always noteworthy). + popen_kwargs = { + 'cwd': cwd, + 'shell': True, + 'stdout': subprocess.PIPE, + 'stderr': subprocess.PIPE if split_stderr else subprocess.STDOUT, + } + if not binary: + popen_kwargs.update({'text': True, 'encoding': 'utf-8', 'errors': 'replace'}) + if os.name != 'nt': + # C-level setsid, same process-group semantics as preexec_fn=os.setsid but + # safe when called from threads (pool_check runs flashes from a thread pool) + popen_kwargs['start_new_session'] = True + + p = subprocess.Popen(cmd, **popen_kwargs) + try: + out, err = p.communicate(timeout=timeout) + r = subprocess.CompletedProcess(args=cmd, returncode=p.returncode, stdout=out, stderr=err) + except subprocess.TimeoutExpired as ex: + if os.name != 'nt': + try: + os.killpg(p.pid, signal.SIGKILL) + except OSError: + # ProcessLookupError: already gone. PermissionError: an all-root group + # refuses the group kill -- letting either escape would skip the bounded + # reap, the pipe close and the rc-124 return this handler exists for. + pass + else: + p.kill() + try: + out, err = p.communicate(timeout=10) + except subprocess.TimeoutExpired: + # Something in the group outlived SIGKILL: D state (truly unkillable), or + # root-owned because sudo FORKS rather than execs, so the wrapper dies and its + # root child does not. Abandon it and let the report name it; the harness never + # sudo-kills its way out. Our ends of its pipes must not leak, though: a pool + # worker lives for the whole run, so every wedged command would cost it two fds. + out, err = None, None + _close_pipes(p) + # prefer the post-kill buffers (supersets of the exception's), falling back to ex.* + # when the child was unkillable. TimeoutExpired carries BYTES even for a text-mode + # Popen, so the fallbacks must be decoded or a text-mode caller gets bytes exactly + # when the child wedged in D state. + def _typed(v): + if not binary and isinstance(v, bytes): + return v.decode('utf-8', errors='replace') + return v + + timeout_out = _typed(out or ex.stdout) or (b'' if binary else '') + # ...and never None: with split_stderr the SUCCESS path always yields a str/bytes, + # so a caller that does `r.stderr.strip()` works everywhere except the timeout -- + # the one path it was written for. Without split_stderr stderr stays None, as on + # the success path (it was merged into stdout). + timeout_err = _typed(err if err is not None else ex.stderr) + if split_stderr and timeout_err is None: + timeout_err = b'' if binary else '' + _print_banner(f'COMMAND TIMEOUT ({timeout}s): {cmd}', timeout_out, timeout_err) + return subprocess.CompletedProcess(args=cmd, returncode=124, stdout=timeout_out, stderr=timeout_err) + except BaseException: + # BaseException, not Exception (as in CPython's own subprocess.run): + # KeyboardInterrupt is the case that matters, and start_new_session put the child in + # its OWN group, so it never got the terminal's SIGINT -- without this, Ctrl-C + # leaves the flasher or testusb holding the probe and its usbfs node. Kill and + # close, never wait: this path must not add a hang of its own. + if os.name != 'nt': + try: + os.killpg(p.pid, signal.SIGKILL) + except OSError: + pass + else: + p.kill() + _close_pipes(p) + raise + + if r.returncode != 0 and not quiet: + _print_banner(f'COMMAND FAILED: {cmd}', r.stdout, r.stderr) + elif verbose: + print(cmd) + print(cmd_stdout_text(r.stdout)) + return r + + +# get usb serial by id +def get_serial_dev(id, vendor_str, product_str, ifnum): + if vendor_str and product_str: + # known vendor and product + vendor_str = vendor_str.replace(' ', '_') + product_str = product_str.replace(' ', '_') + return f'/dev/serial/by-id/usb-{vendor_str}_{product_str}_{id}-if{ifnum:02d}' + else: + # just use id: mostly for cp210x/ftdi flasher + pattern = f'/dev/serial/by-id/usb-*_{id}-if*' + port_list = glob.glob(pattern) + if len(port_list) == 0: + raise RuntimeError(f'No serial device found for {pattern}') + return port_list[0] diff --git a/test/hil/hil_ci.sh b/test/hil/hil_ci.sh index ef93bcb49..c7dfa95df 100644 --- a/test/hil/hil_ci.sh +++ b/test/hil/hil_ci.sh @@ -20,6 +20,30 @@ CONFIG=${CONFIG:-$ROOT_DIR/test/hil/tinyusb.json} exit 1 } +# REMOTE_DIR reaches the rig as `rm -rf` input, an scp remote path and an rsync remote +# path -- the remote shell re-splits and expands all three, so no amount of LOCAL quoting +# protects them (and %q would escape the ~ that REMOTE_DIR=~/dir needs). Screen it once. +# The tilde is the whole hazard: the REMOTE shell expands it, so `~/` alone -- one typo +# away from the documented ~/dir override -- means `rm -rf` on that account's HOME. Hence +# `/` or `~/` followed by at least one named component, ending in a name character. +[[ $REMOTE_DIR =~ ^(/|~/)[A-Za-z0-9_.~/-]*[A-Za-z0-9_-]$ && $REMOTE_DIR != *..* + && $REMOTE_DIR != *//* ]] || { + echo "error: REMOTE_DIR must be /path or ~/path of [A-Za-z0-9_.~/-], no '..', no" \ + "trailing slash -- it is an rm -rf target on $REMOTE: $REMOTE_DIR" >&2 + exit 1 +} + +# --build would run tools/build.py ON THE RIG, and this script stages binaries, not the +# build tree -- it is not copied, so the run dies there with a confusing missing-file +# error. Building is the local half of this workflow by design. +for a in "$@"; do + [ "$a" = "--build" ] || continue + echo "error: --build builds on the REMOTE, but this script copies prebuilt binaries" >&2 + echo " (tools/build.py is not staged). Build locally first, then re-run:" >&2 + echo " cd examples && cmake --preset && cmake --build --preset " >&2 + exit 1 +done + # Parse -b BOARD from arguments to know which build to copy BOARD="" ARGS=() @@ -38,29 +62,35 @@ while [[ $# -gt 0 ]]; do esac done -# Setup remote directory. Use `bash -s` + heredoc so REMOTE_DIR (user-overridable) -# is passed as a positional parameter and never reinterpreted by the remote shell. +# Setup remote directory. `bash -s` + heredoc so REMOTE_DIR arrives as a positional +# parameter, keeping the `rm -rf` target out of the command string the heredoc runs. echo "==> Setting up remote $REMOTE:$REMOTE_DIR" ssh "$REMOTE" bash -s -- "$REMOTE_DIR" <<'REMOTE' set -e +# Second gate, on the side that knows what ~ expanded to: only here is $HOME a value +# rather than a guess, and this is the line that actually runs rm -rf. +case "$1" in + ''|/|"$HOME"|"$HOME"/) echo "refusing to rm -rf '$1'" >&2; exit 1 ;; +esac rm -rf -- "$1" -# .claude path: usbtest.py's HUNG recovery resolves usb_recover.sh relative to the -# staged repo root — without it, recovery ENOENTs and the wedge is left in place -mkdir -p -- "$1/test/hil" "$1/examples" "$1/.claude/skills/usb-kernel-recover/scripts" +mkdir -p -- "$1/test/hil/helper" "$1/examples" REMOTE # Copy HIL test script and config echo "==> Copying test scripts" scp -q "$ROOT_DIR/test/hil/hil_test.py" \ "$ROOT_DIR/test/hil/hil_flash.py" \ - "$ROOT_DIR/test/hil/hil_lock.py" \ "$ROOT_DIR/test/hil/usbtest.py" \ - "$ROOT_DIR/test/hil/hil_examples.py" \ "$ROOT_DIR/test/hil/pymtp.py" \ + "$ROOT_DIR/test/hil/mtp_test.py" \ "$CONFIG" \ "$REMOTE:$REMOTE_DIR/test/hil/" -scp -q "$ROOT_DIR/.claude/skills/usb-kernel-recover/scripts/usb_recover.sh" \ - "$REMOTE:$REMOTE_DIR/.claude/skills/usb-kernel-recover/scripts/" +scp -q "$ROOT_DIR/test/hil/helper/__init__.py" \ + "$ROOT_DIR/test/hil/helper/hil_util.py" \ + "$ROOT_DIR/test/hil/helper/hil_health.py" \ + "$ROOT_DIR/test/hil/helper/hil_lock.py" \ + "$ROOT_DIR/test/hil/helper/hil_select.py" \ + "$REMOTE:$REMOTE_DIR/test/hil/helper/" # Copy only firmware binaries (elf/bin/hex) plus esptool metadata # (config.env + flash_args needed by the esptool flasher), preserving structure @@ -90,16 +120,25 @@ if [ -n "$BOARD" ]; then add_build_dir "$d" done shopt -u nullglob - while IFS= read -r v; do - add_build_dir "$ROOT_DIR/examples/cmake-build-$v" - done < <(python3 -c ' + # to a file, not a process substitution: `set -e`/pipefail cannot see the exit + # status of the latter, so a malformed roster silently yielded zero variant dirs + VARIANTS_FILE=$(mktemp) + python3 -c ' import json, sys cfg = json.load(open(sys.argv[1])) for b in cfg.get("boards", []): if b["name"] == sys.argv[2]: for v in b.get("variant") or []: print(v["name"]) -' "$CONFIG" "$BOARD") +' "$CONFIG" "$BOARD" > "$VARIANTS_FILE" || { + echo "Error: could not read variants for $BOARD from $CONFIG" + rm -f "$VARIANTS_FILE" + exit 1 + } + while IFS= read -r v; do + add_build_dir "$ROOT_DIR/examples/cmake-build-$v" + done < "$VARIANTS_FILE" + rm -f "$VARIANTS_FILE" if [ ${#BUILD_DIRS[@]} -eq 0 ]; then echo "Error: no build directory found for $BOARD under $ROOT_DIR/examples/" echo "Build first with: cd examples && cmake --preset $BOARD && cmake --build --preset $BOARD" @@ -119,12 +158,24 @@ else done fi -# Run test. Use `bash -s` so REMOTE_DIR + ARGS reach the remote shell as positional -# parameters; quoting and metacharacters in args are preserved. -CONFIG_BASENAME="$(basename "$CONFIG")" +# Run test via `bash -s`, so REMOTE_DIR and the args arrive as positional parameters. +# %q the ARGS -- ssh joins its argv into ONE string that the remote shell re-splits, so +# `-t 'host/cdc msc'` would arrive as two arguments and hil_test.py would see a stray +# word where it expects the config path. REMOTE_DIR is deliberately NOT quoted here: it +# is screened above precisely so it can keep its ~ expansion. +ARGS_Q=() +for a in ${ARGS[@]+"${ARGS[@]}"}; do ARGS_Q+=("$(printf '%q' "$a")"); done +# same re-split, same fix: CONFIG is a user-supplied path and its basename lands in the +# command string too +CONFIG_Q="$(printf '%q' "test/hil/$(basename "$CONFIG")")" echo "==> Running HIL test on $REMOTE" rc=0 -ssh "$REMOTE" bash -s -- "$REMOTE_DIR" "${ARGS[@]}" "test/hil/$CONFIG_BASENAME" <<'REMOTE' || rc=$? +# --retry 1 FIRST, before the user's args: this targets the same shared rig CI uses, and +# the pool guard is a flat constant that does not scale with max_retry -- argparse's +# default of 3 lets a few flaky boards re-pay 510s each until the 3600s guard fires, +# abandoning the pool and holding board flocks against concurrent CI. Placed first, not +# appended, so argparse's last-wins means `hil_ci.sh -r 3` still gets 3. +ssh "$REMOTE" bash -s -- "$REMOTE_DIR" --retry 1 ${ARGS_Q[@]+"${ARGS_Q[@]}"} "$CONFIG_Q" <<'REMOTE' || rc=$? cd -- "$1" shift # Flasher CLIs live in the user bin dirs on ci.lan (esptool/idf in ~/.local/bin, diff --git a/test/hil/hil_ci_set_matrix.py b/test/hil/hil_ci_set_matrix.py deleted file mode 100644 index bca989bd1..000000000 --- a/test/hil/hil_ci_set_matrix.py +++ /dev/null @@ -1,90 +0,0 @@ -import argparse -import json -import os - - -def _resolve_config_path(config_file): - if os.path.exists(config_file): - return config_file - - script_relative = os.path.join(os.path.dirname(__file__), config_file) - if os.path.exists(script_relative): - return script_relative - - raise FileNotFoundError(f'Config file not found: {config_file}') - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument('config_files', nargs='+', help='Configuration JSON file(s)') - parser.add_argument('--select', help='hil_select.py JSON; scopes boards when full=false') - args = parser.parse_args() - - selected = None - sel = json.loads(args.select) if args.select else None - if sel and not sel.get('full'): - selected = set(sel.get('boards', {})) - - # Toolchain buckets must match the toolchains instantiated by the hil-build - # job in .github/workflows/build.yml. Keep all keys present (even if empty) - # so `fromJSON(hil_json)[toolchain]` always resolves to a list. - matrix = { - 'arm-gcc': [], - 'riscv-gcc': [], - 'esp-idf': [] - } - - seen = {toolchain: set() for toolchain in matrix} - - def append_build_arg(toolchain, build_arg): - if build_arg not in seen[toolchain]: - seen[toolchain].add(build_arg) - matrix[toolchain].append(build_arg) - - for config_file in args.config_files: - with open(_resolve_config_path(config_file)) as f: - config = json.load(f) - - for board in config['boards']: - if selected is not None and board['name'] not in selected: - continue - name = board['name'] - flasher = board['flasher'] - # esptool boards must build under esp-idf; others default to arm-gcc - # but may opt into another bucket via an explicit "toolchain" field - # (e.g. RISC-V boards like ch32v20x need "riscv-gcc"). - if flasher['name'] == 'esptool': - toolchain = 'esp-idf' - else: - toolchain = board.get('toolchain', 'arm-gcc') - if toolchain not in matrix: - # a board in no bucket would never be built, and the bare KeyError - # below would only say so as a traceback from the set-matrix job - raise SystemExit( - f'{name}: toolchain {toolchain!r} is not a build bucket ' - f'({", ".join(matrix)}); add it here and to the hil-build / ' - f'hil-build-esp jobs in .github/workflows/build.yml') - - build_board = f'-b {name}' - if 'build' in board and 'args' in board['build']: - build_board += ' ' + ' '.join(f'-D{a}' for a in board['build']['args']) - - # Each variant builds into cmake-build- with its own cmake - # -D defines and raw CFLAGS. No 'variant' -> a single build named after - # the board. - variants = board.get('variant') or [{'name': name, 'flags': ''}] - for v in variants: - arg = build_board - if v['name'] != name: - arg += f' --build-name {v["name"]}' - for d in v.get('defines', []): - arg += f' -D{d}' - for tok in v.get('flags', '').split(): - arg += f' --cflag={tok}' - append_build_arg(toolchain, arg) - - print(json.dumps(matrix)) - - -if __name__ == '__main__': - main() diff --git a/test/hil/hil_examples.py b/test/hil/hil_examples.py deleted file mode 100644 index 4c8b6918b..000000000 --- a/test/hil/hil_examples.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MIT -# HIL example test lists, shared by hil_test.py (runner) and hil_select.py -# (PR-diff selector). Stdlib-only: hil_select runs on bare CI runners. - -# The per-board run order is shuffled (see test_board). -# Every example carries a unique hardcoded idProduct (see its usb_descriptors.c) - -# device tests -device_tests = [ - 'device/cdc_dual_ports', - 'device/cdc_msc', - 'device/dfu', - 'device/cdc_msc_throughput', - 'device/audio_test_freertos', - 'device/dfu_runtime', - 'device/cdc_msc_freertos', - 'device/hid_boot_interface', - 'device/msc_dual_lun', - 'device/hid_generic_inout', - 'device/printer_to_cdc', - 'device/midi_test', - 'device/mtp', - 'device/usbtest', # cafe:4010, unique PID; runs the Linux testusb tier-4 battery via usbtest.py - # 'device/net_lwip_webserver', # disabled for PR #3605: USB net iface enum is flaky on the CI HIL host -] - -dual_tests = [ - 'dual/host_info_to_device_cdc', -] - -host_test = [ - 'host/cdc_msc_hid', - 'host/msc_file_explorer', - 'host/msc_file_explorer_freertos', - 'host/device_info', -] diff --git a/test/hil/hil_flash.py b/test/hil/hil_flash.py index da81fcc97..f4bed45a6 100755 --- a/test/hil/hil_flash.py +++ b/test/hil/hil_flash.py @@ -1,138 +1,48 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: MIT -# Firmware flashing for the TinyUSB HIL rig: run_cmd, one flash_*/reset_* pair per -# flasher type (dispatched by config name via getattr), find_firmware, and the -# fixture serial-port resolver get_serial_dev (here, not hil_test: flash_esptool -# needs it and helpers must not import hil_test). -# Callers set module globals `build_dir` and `verbose` (hil_test.main from argparse, -# pool_check directly) exactly as they set hil_test's globals today. -# -# from __future__ import annotations (below): some moved function signatures use -# type hints (Any, Board) not defined in this module; postponed evaluation (PEP -# 563) keeps those as unevaluated strings so the verbatim-moved defs still load. +# Firmware flashing for the TinyUSB HIL rig: one flash_*/reset_* pair per flasher type +# (dispatched by config name via getattr) plus find_firmware. The bounded runner run_cmd +# lives in hil_util (never import hil_test here). Callers set the module global +# `build_dir`. `from __future__ import annotations` keeps the Board hints below +# unevaluated: the type is not defined in this module. from __future__ import annotations -import glob import json -import os -import signal +import re import subprocess from pathlib import Path -verbose = False -build_dir = 'cmake-build' +import os +import sys -CMD_TIMEOUT = int(os.getenv('HIL_CMD_TIMEOUT', '180')) +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # PYTHONSAFEPATH drops it +from helper import hil_util + +build_dir = 'cmake-build' # flasher names (dispatch key, board['flasher']['name'].lower()) whose reset_* is a no-op RESET_NOOP = {'esptool', 'lm4flash'} # extra parents find_firmware ALSO searches after build_dir. Empty by default so -# hil_test's -B stays authoritative (a board missing there must report "Skip (no -# binary)", never silently flash a stale binary from another tree); pool_check -# opts in to cover both standard layouts. +# hil_test's -B stays authoritative: a board missing there must report "Skip (no +# binary)", never silently flash a stale binary from another tree. EXTRA_BUILD_DIRS: list = [] - -def cmd_stdout_text(out: Any) -> str: - if out is None: - return '' - if isinstance(out, bytes): - return out.decode('utf-8', errors='ignore') - return str(out) - - -# ------------------------------------------------------------- -# Path -# ------------------------------------------------------------- -TINYUSB_ROOT = Path(__file__).resolve().parents[2] - -# get usb serial by id -def get_serial_dev(id, vendor_str, product_str, ifnum): - if vendor_str and product_str: - # known vendor and product - vendor_str = vendor_str.replace(' ', '_') - product_str = product_str.replace(' ', '_') - return f'/dev/serial/by-id/usb-{vendor_str}_{product_str}_{id}-if{ifnum:02d}' - else: - # just use id: mostly for cp210x/ftdi flasher - pattern = f'/dev/serial/by-id/usb-*_{id}-if*' - port_list = glob.glob(pattern) - if len(port_list) == 0: - raise RuntimeError(f'No serial device found for {pattern}') - return port_list[0] +_VID_PID_WARNED: set = set() # one warning per probe, not per command # ------------------------------------------------------------- # Flashing firmware # ------------------------------------------------------------- -def run_cmd(cmd: str, cwd: str | None = None, timeout: int = CMD_TIMEOUT) -> subprocess.CompletedProcess: - popen_kwargs = { - 'cwd': cwd, - 'shell': True, - 'stdout': subprocess.PIPE, - 'stderr': subprocess.STDOUT, - 'text': True, - 'encoding': 'utf-8', - 'errors': 'replace', - } - if os.name != 'nt': - # C-level setsid, same process-group semantics as preexec_fn=os.setsid but - # safe when called from threads (pool_check runs flashes from a thread pool) - popen_kwargs['start_new_session'] = True - - p = subprocess.Popen(cmd, **popen_kwargs) - try: - out, _ = p.communicate(timeout=timeout) - r = subprocess.CompletedProcess(args=cmd, returncode=p.returncode, stdout=out) - except subprocess.TimeoutExpired as ex: - if os.name != 'nt': - try: - os.killpg(p.pid, signal.SIGKILL) - except ProcessLookupError: - pass - else: - p.kill() - try: - out, _ = p.communicate(timeout=10) - except subprocess.TimeoutExpired: # unkillable (e.g. D-state on wedged USB) - out = None - timeout_out = ex.stdout or out or b'' - title = f'COMMAND TIMEOUT ({timeout}s): {cmd}' - print() - if os.getenv('CI'): - print(f"::group::{title}") - print(cmd_stdout_text(timeout_out)) - print(f"::endgroup::") - else: - print(title) - print(cmd_stdout_text(timeout_out)) - return subprocess.CompletedProcess(args=cmd, returncode=124, stdout=timeout_out) - - if r.returncode != 0: - title = f'COMMAND FAILED: {cmd}' - print() - if os.getenv('CI'): - print(f"::group::{title}") - print(cmd_stdout_text(r.stdout)) - print(f"::endgroup::") - else: - print(title) - print(cmd_stdout_text(r.stdout)) - elif verbose: - print(cmd) - print(cmd_stdout_text(r.stdout)) - return r - - -def flash_jlink(board: Board, firmware: str) -> subprocess.CompletedProcess: +def flash_jlink(board: Board, firmware: str, timeout=None) -> subprocess.CompletedProcess: flasher = board['flasher'] script = ['halt', 'r', f'loadfile {firmware}', 'r', 'go', 'exit'] f_jlink = Path(f'{board["name"]}_{Path(firmware).name}.jlink') with f_jlink.open('w') as f: f.writelines(f'{s}\n' for s in script) - ret = run_cmd(f'JLinkExe -USB {flasher["uid"]} {flasher["args"]} -if swd -JTAGConf -1,-1 -speed auto -NoGui 1 -ExitOnError 1 -CommandFile {f_jlink}') + ret = hil_util.run_cmd(f'JLinkExe -USB {flasher["uid"]} {flasher["args"]} -if swd -JTAGConf -1,-1 -speed auto -NoGui 1 -ExitOnError 1 -CommandFile {f_jlink}', + timeout=timeout) f_jlink.unlink(missing_ok=True) return ret @@ -144,89 +54,208 @@ def reset_jlink(board: Board) -> subprocess.CompletedProcess: if not f_jlink.exists(): with f_jlink.open('w') as f: f.writelines(f'{s}\n' for s in script) - ret = run_cmd(f'JLinkExe -USB {flasher["uid"]} {flasher["args"]} -if swd -JTAGConf -1,-1 -speed auto -NoGui 1 -ExitOnError 1 -CommandFile {f_jlink}') + ret = hil_util.run_cmd(f'JLinkExe -USB {flasher["uid"]} {flasher["args"]} -if swd -JTAGConf -1,-1 -speed auto -NoGui 1 -ExitOnError 1 -CommandFile {f_jlink}') return ret -def flash_stlink(board, firmware): +def flash_stlink(board, firmware, timeout=None): + # --verify catches the partial/corrupt write that exits 0 and sends the test phase + # off to exercise bad firmware. Opt-IN here ("verify": true), unlike flash_openocd's + # opt-out: a default-on read-back silently changes every roster entry that lacks the + # key, including boards on rigs this was never validated against. flasher = board['flasher'] - return run_cmd(f'STM32_Programmer_CLI --connect port=swd sn={flasher["uid"]} --write {firmware} --go') + verify = ' --verify' if flasher.get('verify', False) else '' + return hil_util.run_cmd(f'STM32_Programmer_CLI --connect port=swd sn={flasher["uid"]} --write {firmware}{verify} --go', + timeout=timeout) def reset_stlink(board): flasher = board['flasher'] - return run_cmd(f'STM32_Programmer_CLI --connect port=swd sn={flasher["uid"]} --rst --go') + return hil_util.run_cmd(f'STM32_Programmer_CLI --connect port=swd sn={flasher["uid"]} --rst --go') def _openocd_cmd_base(flasher): + # Optional roster field vid_pid, openocd-verbatim (e.g. "0x1a86 0x8010"), pins probe + # discovery to the probe's IDs so openocd never opens foreign usbfs nodes to read + # strings -- a wedged node makes that open hang unkillably (the 2026-08-10 convoy). + # BEFORE args, because the rescue cfgs run `init` internally and reject (or never see) + # a config command that follows it. + vid_pid = '' + if 'vid_pid' in flasher: + # Validated HERE too, not just in convoy_safe: openocd only warns ("incomplete + # vid_pid configuration directive") and exits 0 on a malformed value, so the pin + # silently does not apply and discovery goes back to opening every usbfs node -- + # the convoy this field exists to stop. The same key name carries a DIFFERENT + # syntax under tests.dev_attached ('1a86_55d4'), so the typo is one copy away. + if valid_vid_pid(flasher['vid_pid']): + vid_pid = f'-c "adapter usb vid_pid {flasher["vid_pid"]}" ' + else: + # stderr + once-per-probe, like the missing-pin branch below: stdout here is + # captured by test_example's redirect_stdout (shown only when the test FAILS) + # and by hil_pool_check's StringIO spool, so on a PASSING run the operator + # would never learn the pin was silently dropped. + uid = flasher.get('uid', '?') + if uid not in _VID_PID_WARNED: + _VID_PID_WARNED.add(uid) + print(f'warning: {uid} has a malformed vid_pid {flasher["vid_pid"]!r} ' + f'(want "0xVVVV 0xPPPP"); probe pin DROPPED, so discovery will open ' + f'foreign usbfs nodes', file=sys.stderr, flush=True) + elif flasher.get('uid') not in _VID_PID_WARNED: + # stderr, once per probe: test_example captures stdout, so a passing run would + # swallow this and the operator would never learn discovery still opens every + # usbfs node + _VID_PID_WARNED.add(flasher.get('uid')) + print(f'warning: openocd flasher {flasher.get("uid", "?")} has no vid_pid pin; ' + f'probe discovery will open every usbfs node (hangs on a wedged one)', + file=sys.stderr, flush=True) return (f'openocd -c "tcl_port disabled" -c "gdb_port disabled" -c "telnet_port disabled" ' - f'-c "adapter serial {flasher["uid"]}" {flasher["args"]}') + f'-c "adapter serial {flasher["uid"]}" {vid_pid}{flasher["args"]}') -# `verify` is on by default and opted out per board with "verify": false in the roster. -# WCH targets must opt out: flash read-back over the WCH-Link sdi transport returns a -# repeated word instead of memory contents, so verification always reports a mismatch and -# fails the flash (measured on ch32v103r and ch32v307v, 2026-07-30). Do NOT drop verify -# fleet-wide to accommodate them — every other openocd board can read back, and without it -# a partial or corrupt write exits 0 and the test phase runs bad firmware. -def flash_openocd(board, firmware): +# `verify` is on by default, opted out per board with "verify": false. WCH targets must +# opt out: read-back over the WCH-Link sdi transport returns a repeated word instead of +# memory contents, so verification always mismatches (measured on ch32v103r and ch32v307v, +# 2026-07-30). Do NOT drop verify fleet-wide for them — every other openocd board reads +# back, and without it a partial or corrupt write exits 0 and the tests run bad firmware. +def flash_openocd(board, firmware, timeout=None): flasher = board['flasher'] verify = ' verify' if flasher.get('verify', True) else '' - ret = run_cmd(f'{_openocd_cmd_base(flasher)} -c "program {firmware}{verify} reset exit"') + ret = hil_util.run_cmd(f'{_openocd_cmd_base(flasher)} -c "program {firmware}{verify} reset exit"', + timeout=timeout) return ret -def reset_openocd(board): +def reset_openocd(board, timeout=None): + # timeout: usbtest's post-hang recovery bounds this (RECOVER_RESET_TIMEOUT); an + # unbounded reset there would outlive the caller's outer kill and orphan openocd on + # the probe, which is the stray the recovery exists to avoid. flasher = board['flasher'] - ret = run_cmd(f'{_openocd_cmd_base(flasher)} -c "init; reset run; exit"') + ret = hil_util.run_cmd(f'{_openocd_cmd_base(flasher)} -c "init; reset run; exit"', + timeout=timeout) return ret # OpenOCD's messages for "the target's debug port did not answer". The probe is fine when -# these appear (the log still shows "CMSIS-DAP: Interface ready"); the chip's debug clock -# is gone, which no reset the probe can drive would fix -- the CMSIS-DAP Debug Probe has no -# nRESET line at all. Which message you get depends on the DAP topology, NOT on the board: -# rp2040.cfg creates three multidrop DAPs (cores 0/1 and the Rescue DP at instance 0xf) so -# it fails in swd_multidrop_select, while rp2350.cfg creates a single plain ADIv6 DAP that -# fails earlier in swd_connect. A dead RP2040 can also produce the second one if the very -# first DP read never gets through, so both are accepted for both chips -- it is the target -# cfg in the roster args, below, that picks how to rescue. +# these appear ("CMSIS-DAP: Interface ready" is still logged); the chip's debug clock is +# gone, which no probe-driven reset fixes -- the CMSIS-DAP probe has no nRESET line. Which +# message appears depends on DAP topology, not the board, so both are accepted for both +# chips; RESCUE_CFG below picks the rescue. DAP_WEDGED = ('Failed to connect multidrop', 'Error connecting DP: cannot read IDR') -# How each RP target reaches its Rescue DP, keyed by the target cfg named in flasher args. -# (cfg substitution, extra args): rp2040.cfg drives the Rescue DP itself behind a RESCUE -# flag and calls init/shutdown on its own; rp2350 has a separate cfg that pokes the rescue -# bit via an AP register but never shuts down, so it would sit in the server loop until -# CMD_TIMEOUT without an explicit one. +# How each RP target reaches its Rescue DP, keyed by the target cfg named in flasher args: +# (cfg substitution, pre args, post args). rp2040.cfg drives the Rescue DP behind a RESCUE +# flag and init/shutdowns itself; rp2350-rescue.cfg never shuts down, so it needs an +# explicit one or it sits in the server loop until CMD_TIMEOUT. RESCUE_CFG = { 'target/rp2040.cfg': ('target/rp2040.cfg', '-c "set RESCUE 1" ', ''), 'target/rp2350.cfg': ('target/rp2350-rescue.cfg', '', ' -c "shutdown"'), } -def rescue_openocd(board, flash_out: str = '') -> bool: +def rescue_openocd(board, flash_out: str = '', timeout=None) -> bool: """Power-on-reset a wedged RP2040/RP2350 through its Rescue DP, the one debug port not - gated by the system clock (RP2040 datasheet 2.3.4.2): setting CDBGPWRUPREQ hard-resets - the chip, and the bootrom halts it in a safe state ready to be flashed. This is the - only way back for a target whose cores have stopped answering -- otherwise the board - needs a physical replug, since the probe carries no reset line. - - No-op (returns False) unless this is an openocd RP board AND the flash output shows the - wedge, so a flash that failed for any other reason still just retries. Returns True - when a rescue was attempted; the caller should retry the flash afterwards.""" + gated by the system clock (RP2040 datasheet 2.3.4.2): CDBGPWRUPREQ hard-resets the + chip and the bootrom halts it ready to be flashed. Without it the board needs a + physical replug -- the probe carries no reset line. + + No-op (False) unless this is an openocd RP board AND the flash output shows the wedge, + so a flash that failed for any other reason still just retries. True when a rescue was + attempted; the caller should retry the flash afterwards.""" flasher = board['flasher'] if flasher['name'].lower() != 'openocd' or not any(m in flash_out for m in DAP_WEDGED): return False for cfg, (rescue_cfg, pre, post) in RESCUE_CFG.items(): if cfg in flasher['args']: args = flasher['args'].replace(cfg, rescue_cfg) - return run_cmd(f'{_openocd_cmd_base({**flasher, "args": pre + args})}{post}').returncode == 0 + return hil_util.run_cmd(f'{_openocd_cmd_base({**flasher, "args": pre + args})}{post}', + timeout=timeout).returncode == 0 return False -def flash_esptool(board: Board, firmware: str) -> subprocess.CompletedProcess: +# openocd's own syntax: one or more "0xVVVV 0xPPPP" pairs. Validated rather than merely +# tested for truthiness -- `vid_pid` is a hand-edited roster field whose NAME is also used, +# with a different syntax, by tests.dev_attached, and convoy_safe reads a non-empty value +# as PROOF the flasher can deliver a recovery past a poisoned node. A typo there silently +# promised a recovery that openocd would reject at startup. +_VID_PID_RE = re.compile(r'^0x[0-9a-fA-F]{4}(\s+0x[0-9a-fA-F]{4})+$') + + +def valid_vid_pid(value) -> bool: + return isinstance(value, str) and bool(_VID_PID_RE.match(value.strip())) + + +def recover_flasher(board: dict) -> dict: + """The flasher that delivers RECOVERY for this board. + + Optional roster key `flasher_recover`, else the primary. It exists because delivery and + normal flashing have different requirements: a board flashed by jlink/stlink/lm4flash + cannot reach its probe past a poisoned usbfs node, but the same probe driven by openocd + often can (see convoy_safe). Keeping it a separate key rather than a list means the + primary's shape never changes, so nothing that reads board['flasher'] has to care. + """ + return board.get('flasher_recover') or board['flasher'] + + +def convoy_safe(flasher: dict) -> bool: + """Can this flasher DELIVER a recovery while a usbfs node on the rig is poisoned? + + A post-HUNG reflash only helps if the flasher reaches its probe without opening the + wedged node. Two shapes qualify: + + * openocd pinned with the roster's `vid_pid` -- the match is made from the cached + descriptor and the loop `continue`s BEFORE libusb_open, so a foreign node is never + opened. On 2026-08-12 it was the only flasher that still reached its probe. + * esptool -- delivery is `-p `, a named port; it never enumerates usbfs. + + Everything else enumerates by OPENING nodes, would block in D state on the poisoned + one, survive SIGKILL and become a second stray. JLinkExe cannot be pinned: selection + is serial-only (-USB/-SelectEmuBySN) and reading a serial requires the open (J-Link + Commander V9.66 exposes no VID/PID filter), so those boards can only become + convoy-safe by moving to openocd. + + Verified against openocd 0ce743125 (the rig's build), because the INVERSE is what + bites: cmsis_dap_usb_bulk.c:107 skips on `id_filter && !id_match`, and `id_filter` is + only `vids[0] || pids[0]` -- so without the pin nothing is skipped and every device on + the bus is opened, which the code itself expects to mostly fail. Enumeration cannot + block: libusb reads the `descriptors` sysfs attribute, and descriptors_read (v6.12.101 + drivers/usb/core/sysfs.c) is a memcpy from udev->rawdescriptors under no lock. + + The pin gates the BULK backend, which is the one that runs: `auto` tries usb_bulk -> + hid -> tcp (cmsis_dap.c:62) and stops at the first that opens, so a CMSIS-DAP v2 probe + never reaches the rest. It does NOT cover the HID fallback that a v1 probe or a failed + bulk open takes -- cmsis_dap_usb_hid.c:91 calls hid_enumerate(0x0, 0x0), pin ignored, + and filters afterwards, while hidapi's hidraw backend reads `manufacturer` and + `product` for every HID device it lists (linux/hid.c:744), both usb_string_attr and so + served under the device lock. A wedged DUT running hid_generic_inout, + hid_boot_interface or hid_composite_freertos is a HID device and would stall that walk + -- interruptibly, so it hangs rather than joining the D-state convoy and run_cmd's + timeout ends it, but "never opens a foreign node" is true of the bulk path, not of + every path openocd can take. + """ + name = (flasher.get('name') or '').lower() + if name == 'esptool': + return True + # EXACT, not startswith: rescue_openocd and usbtest's + # getattr(hil_flash, f'flash_{name}') both require the exact name, so an + # 'openocd_wch'-style entry would pass this gate, reserve USBTEST_RECOVERY_BUDGET, + # and then find no recovery path at all -- paying for a path that cannot fire, which + # is the precise cost this gate exists to avoid. + if name != 'openocd': + return False + if valid_vid_pid(flasher.get('vid_pid')): + return True + # openocd over the JLINK driver is safe WITHOUT a pin, and cannot use one: jlink.c + # never reads adapter_usb_get_vids/pids (selection is adapter serial / usb address / + # usb location), but libjaylink's discovery returns early unless idVendor == 0x1366 and + # the PID is in its table, and only THEN calls libusb_open (discovery_usb.c). So it + # never opens a foreign node -- which is exactly what JLinkExe, SEGGER's own tool, + # does do. Verified against openocd 0ce743125 and libjaylink master. + return 'interface/jlink.cfg' in (flasher.get('args') or '') + + +def flash_esptool(board: Board, firmware: str, timeout=None) -> subprocess.CompletedProcess: flasher = board['flasher'] - port = get_serial_dev(flasher["uid"], None, None, 0) + port = hil_util.get_serial_dev(flasher["uid"], None, None, 0) fw_dir = Path(firmware).parent with (fw_dir / 'config.env').open() as f: idf_target = json.load(f)['IDF_TARGET'] @@ -234,32 +263,39 @@ def flash_esptool(board: Board, firmware: str) -> subprocess.CompletedProcess: flash_args = f.read().strip().replace('\n', ' ') command = (f'esptool --chip {idf_target} -p {port} {flasher["args"]} ' f'--before=default_reset --after=hard_reset write_flash {flash_args}') - ret = run_cmd(command, cwd=str(fw_dir)) + ret = hil_util.run_cmd(command, cwd=str(fw_dir), timeout=timeout) return ret def reset_esptool(board): - flasher = board['flasher'] + # NO-OP, and marked as one: esptool's reset would be `--after hard_reset`, which is not + # wired here. Returning rc 0 without resetting is why callers must never read the exit + # code as proof -- recovery_steps skips a primitive carrying `no_op`. return subprocess.CompletedProcess(args=['dummy'], returncode=0) -def flash_lm4flash(board, firmware): +reset_esptool.no_op = True + + +def flash_lm4flash(board, firmware, timeout=None): # TI Tiva-C / Stellaris ICDI: lightweight lm4flash, resets and runs after write flasher = board['flasher'] - ret = run_cmd(f'lm4flash -s {flasher["uid"]} {flasher["args"]} {firmware}') + ret = hil_util.run_cmd(f'lm4flash -s {flasher["uid"]} {flasher["args"]} {firmware}', + timeout=timeout) return ret def reset_lm4flash(board): # lm4flash has no reset-only mode; it resets+runs on flash, so reset is a no-op - flasher = board['flasher'] return subprocess.CompletedProcess(args=['dummy'], returncode=0) -# The one place a flasher's firmware extension is decided: find_firmware resolves the -# path with it and the flash_* functions pass that path through untouched. A flasher -# added here without an entry falls back to .elf-or-.bin and can be handed the wrong -# file — test_hil_select's TestRosterFlashersDispatch fails if a roster names one. +reset_lm4flash.no_op = True + + +# The one place a flasher's firmware extension is decided. A flasher with no entry falls +# back to .elf-or-.bin and can be handed the wrong file — test_hil_select's +# TestRosterFlashersDispatch fails if a roster names one. FLASHER_SUFFIX = { 'esptool': '.bin', 'jlink': '.elf', @@ -271,13 +307,12 @@ FLASHER_SUFFIX = { def find_firmware(variant: str, example: str, roots: list | None = None, flasher: str | None = None): """Locate a built example's firmware under /cmake-build-//, - then under EXTRA_BUILD_DIRS (empty unless the caller opts in — see its comment). - `roots` overrides that search list entirely for one call (e.g. to find a build just - produced by tools/build.py in its fixed cmake-build/ layout without widening the - global policy). `flasher` is the roster flasher name: it selects which extension - counts (see FLASHER_SUFFIX), so a build that produced only the other one is reported - missing — a clean "Skip (no binary)" — instead of being handed to the flasher, which - would fail opaquely on the absent file and burn every retry plus the board lock. + then under EXTRA_BUILD_DIRS. `roots` overrides that search list entirely for one call + (e.g. a build just produced by tools/build.py in its fixed cmake-build/ layout) + without widening the global policy. `flasher` is the roster flasher name and selects + which extension counts (FLASHER_SUFFIX), so a build that produced only the other one + is reported missing — a clean "Skip (no binary)" — instead of being handed to the + flasher, which would fail opaquely and burn every retry plus the board lock. Accepts the single-config layout (firmware directly in the example dir) or Ninja Multi-Config (a per-config subdir like RelWithDebInfo/). Returns the full Path INCLUDING extension, or None if not built.""" @@ -286,7 +321,7 @@ def find_firmware(variant: str, example: str, roots: list | None = None, flasher if not suffixes or suffixes == [None]: suffixes = ['.elf', '.bin'] for bd in dict.fromkeys(roots if roots is not None else [build_dir, *EXTRA_BUILD_DIRS]): - fw_dir = TINYUSB_ROOT / bd / f'cmake-build-{variant}' / example + fw_dir = hil_util.TINYUSB_ROOT / bd / f'cmake-build-{variant}' / example if not fw_dir.is_dir(): continue for cand in [fw_dir / base, fw_dir / 'RelWithDebInfo' / base, diff --git a/test/hil/hil_lock.py b/test/hil/hil_lock.py deleted file mode 100755 index e570da16a..000000000 --- a/test/hil/hil_lock.py +++ /dev/null @@ -1,479 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MIT -"""Board locks + controller permits for the TinyUSB HIL rig. - -Board locks are kernel flocks in BOARD_LOCK_DIR arbitrating hardware access -between dev sessions and CI's hil_test.py (never stop the actions-runner). -Controller permits are in-process semaphores budgeting flashes and usbtest -batteries per host controller; they have no CLI meaning. The CLI below -(hold/release/status) manages board locks only. -""" -import argparse -import fcntl -import glob -import json -import os -import re -import select -import signal -import sys -import time - -BOARD_LOCK_DIR = '/tmp/tinyusb-hil-locks' -CI_REASON = 'hil_test.py' # release-protected holder tag (release refuses to kill it) -PROTECTED_REASONS = {CI_REASON, 'pool_check'} # cmd_release refuses to SIGTERM these holders -PROFILE = os.environ.get('HIL_PROFILE') == '1' - - -def lock_path(board: str) -> str: - return os.path.join(BOARD_LOCK_DIR, f'{board}.lock') - - -def flock_nb(board: str): - """Open-or-create the lock file WITHOUT truncating (a losing racer must not - wipe the winner's record) and take LOCK_EX|LOCK_NB. Returns the open handle; - raises OSError when the flock is held elsewhere (handle already closed).""" - fd = os.open(lock_path(board), os.O_RDWR | os.O_CREAT, 0o666) - fh = os.fdopen(fd, 'r+') - try: - fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError: - fh.close() - raise - return fh - - -def write_record(fh, reason: str) -> bool: - """Holder record; the flock itself is already held. Returns False on a write - failure — acquire_board_lock stays best-effort (the flock is the authority), - but cmd_hold aborts on it like board_lock.py did (a hold whose record is - missing is invisible to status/release).""" - try: - fh.truncate(0) - fh.seek(0) - json.dump({'pid': os.getpid(), 'reason': reason, - 'since': time.strftime('%Y-%m-%dT%H:%M:%S%z')}, fh) - fh.flush() - return True - except OSError: - return False - - -def clear_record(fh) -> None: - """Clear our record before dropping the flock so records stay truthful.""" - try: - fh.truncate(0) - except OSError: - pass - - -def read_record(board: str): - try: - with open(lock_path(board)) as f: - return json.load(f) - except (OSError, ValueError): - return None - - -# --- per-board dev-session locks ------------------------------------------ -def acquire_board_lock(board_name, reason=CI_REASON): - """Take this board's flock for the duration of its flash+test. - Returns an open file handle (keep it referenced; closing releases it), - or None when HIL_NO_BOARD_LOCK=1 or the lock dir is unusable (fail-open: - locking must never break a test run by itself). - Raises RuntimeError only when another session holds the board.""" - import fcntl - if os.environ.get('HIL_NO_BOARD_LOCK') == '1': - return None # user-authorized bypass — see hil skill - try: - os.makedirs(BOARD_LOCK_DIR, exist_ok=True) - fd = os.open(os.path.join(BOARD_LOCK_DIR, f'{board_name}.lock'), - os.O_RDWR | os.O_CREAT, 0o666) - fh = os.fdopen(fd, 'r+') - except OSError as e: - # odd lock dir (perms, path collision): proceed unlocked, but say so — - # a silent fail-open is indistinguishable from the intentional bypass - print(f'warning: board lock unavailable for {board_name} ({e}); proceeding unlocked', - flush=True) - return None - try: - fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError: - try: - info = fh.read(500).strip() - except (OSError, UnicodeDecodeError): - info = '' - fh.close() - raise RuntimeError(f'board locked: {info or "unknown holder"}') - # announce ourselves so the other side's conflict message is truthful; - # best-effort — the flock itself is already held - try: - fh.truncate(0) - fh.seek(0) - json.dump({'pid': os.getpid(), 'reason': reason, - 'since': time.strftime('%Y-%m-%dT%H:%M:%S%z')}, fh) - fh.flush() - except OSError: - pass - return fh - - -# Per-host-controller concurrency (see controller_of/controller_slot below): a usbtest battery -# saturates its DUT's host controller, so batteries and flashes are budgeted per controller. -# - uPD720201 cards need their latest firmware (>= 2.0.2.6; RAM-uploaded, reloads every -# power cycle): ROM firmware dies under battery + re-enumeration churn, and usbtest.py -# refuses the unlink-stress cases on it. -# - widths (profiled 2026-07-13/14): wall time 22.2/14.3/12.5/10.8 min at usbtest width -# 1/2/3/4, plateau after; flash width beyond 8 only adds flasher-hub contention; -# battery case failures start at 12/8 (bandwidth stretch on shared leaf-hub uplinks). -# - a marginal DUT port bouncing during concurrent batteries can wedge/kill a uPD720201 -# ("xHCI host not responding to stop endpoint command"): fix the port/cable or pull -# the board, don't lower the widths (2026-07-16: every death traced to one board's port). -FLASH_PARALLEL = int(os.getenv('HIL_FLASH_PARALLEL', '8')) -USBTEST_PARALLEL = int(os.getenv('HIL_USBTEST_PARALLEL', '4')) -CONTROLLER_SLOTS = 12 # lock slots; controllers are assigned to slots on first sight -usbtest_sems = None # CONTROLLER_SLOTS semaphores: per-slot usbtest-battery permits -flash_sems = None # CONTROLLER_SLOTS semaphores: per-slot flash permits -controller_map = None # shared dict: 'pci:' -> slot, 'uid:' -> pci addr cache -controller_meta = None # guards slot assignment in controller_map -controller_hints = {} # static uid -> pci from the last run's cache (read-only per worker) - - -log = print # hil_test.init_worker points this at log_line via init_scheduling - - -def init_scheduling(b_sems, f_sems, cmap, cmeta, hints, log_fn=None): - """Install per-worker scheduling state (called from hil_test.init_worker).""" - global usbtest_sems, flash_sems, controller_map, controller_meta, controller_hints, log - usbtest_sems, flash_sems = b_sems, f_sems - controller_map, controller_meta, controller_hints = cmap, cmeta, hints - if log_fn is not None: - log = log_fn - - -# ------------------------------------------------------------- -# Per-controller scheduling -# ------------------------------------------------------------- -def controller_of(uid: str): - """Resolve a DUT uid to its root host controller's PCI address, or None if the device - is not enumerated (e.g. parked in board_test firmware with USB off). Successful - resolutions are cached — cabling does not change mid-run. Dual-port parts (e.g. - CH32V307 usbhs/usbfs variants) share one uid and one cache entry: budgeting is only - exact when both ports sit on the same controller (true on this rig).""" - if controller_map is None: - return None - cached = controller_map.get(f'uid:{uid}') - if cached: - return cached - for f in glob.glob('/sys/bus/usb/devices/*/serial'): - d = os.path.dirname(f) - try: - if open(f).read().strip().lower() != uid.lower(): - continue - bus = int(open(os.path.join(d, 'busnum')).read()) - root = os.path.realpath(f'/sys/bus/usb/devices/usb{bus}') - m = re.findall(r'[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]', root) - if m: - controller_map[f'uid:{uid}'] = m[-1] - return m[-1] - except (OSError, ValueError): - continue - return None - - -def controller_slot(pci: str) -> int: - """Map a controller PCI address to a lock slot (assigned on first sight).""" - key = f'pci:{pci}' - with controller_meta: - slot = controller_map.get(key) - if slot is None: - slot = controller_map.get('nslots', 0) - if slot >= CONTROLLER_SLOTS: - slot = 0 # more controllers than slots: overflow shares slot 0 (safe, over-serialized) - else: - controller_map['nslots'] = slot + 1 - controller_map[key] = slot - return slot - - -class controller_permit: - """Context manager: one permit from `sems` on the board's controller slot. If the - controller is unknown, fail closed: take one permit from EVERY slot, in order, so the - operation respects the budget wherever it might land. `warn_unknown` logs that fallback - (used by usbtest, where the device is expected to be enumerated by the caller).""" - def __init__(self, sems, uid: str, warn_unknown: bool = False): - self.sems = sems - self.slots = None - self.uid = uid - if sems is None: - return - pci = controller_of(uid) - if pci is None and not warn_unknown: - # last-run cabling hint, flash budgeting only: a mis-budgeted flash is harmless, - # but a battery must never trust a stale hint (it could stack two batteries on - # one controller). In practice only a board's first flash lands here - batteries - # assert enumeration before taking their permit. - pci = controller_hints.get(uid) - if pci is None and warn_unknown: - log(f'warning: cannot resolve {uid} to a host controller; ' - 'taking a permit on every slot (over-serialized)') - self.slots = [controller_slot(pci)] if pci else list(range(CONTROLLER_SLOTS)) - - def __enter__(self): - if self.slots: - t0 = time.monotonic() - taken = [] - try: - for s in self.slots: - self.sems[s].acquire() - taken.append(s) - # stays inside the try: if this raises (e.g. broken stdout), the permits - # must be released - a failed __enter__ never gets its __exit__ - if PROFILE and time.monotonic() - t0 > 1.0: - log(f'[prof] permit wait {time.monotonic() - t0:.1f}s ' - f'(uid {self.uid}, slots {self.slots})') - except BaseException: - for s in reversed(taken): - self.sems[s].release() - raise - return self - - def __exit__(self, *exc): - if self.slots: - for s in reversed(self.slots): - self.sems[s].release() - return False - - -def flash_permit(uid: str) -> controller_permit: - return controller_permit(flash_sems, uid) - - -def usbtest_permit(uid: str) -> controller_permit: - return controller_permit(usbtest_sems, uid, warn_unknown=True) - - -# --- operator CLI (hold/release/status) ------------------------------------ -def boards_from_config(config: str) -> list: - """All board names, INCLUDING boards-skip: `hold --all` guards rig-wide - operations, and parked boards can still be touched (pool_check -b names them - explicitly), so a rig-wide hold that skipped them would leave a gap.""" - try: - with open(config) as f: - cfg = json.load(f) - return [b['name'] for b in cfg['boards'] + cfg.get('boards-skip', [])] - except (OSError, ValueError, KeyError) as e: - print(f'ERROR: cannot read board roster {config}: {e}', file=sys.stderr) - sys.exit(1) - - -def is_locked(board: str) -> bool: - """True if the recorded holder process is still alive. - - Deliberately never touches the flock: even a momentary probe lock would - make a concurrent acquirer's LOCK_NB attempt fail spuriously. The flock - taken by acquirers themselves stays the only authority.""" - info = read_record(board) - pid = info.get('pid') if isinstance(info, dict) else None - if not isinstance(pid, int) or pid <= 0: - return False - try: - os.kill(pid, 0) - except ProcessLookupError: - return False - except PermissionError: - return True # alive but owned by another user (e.g. the CI runner) - return True - - -def cmd_hold(boards, reason): - os.makedirs(BOARD_LOCK_DIR, exist_ok=True) - # No pre-check: the holder's own LOCK_NB flock is the only authority — a - # recorded pid may be stale or recycled (e.g. a live hil_test.py worker - # that already released this board's flock but not its record). - # The holder signals success through this pipe. A generic is_locked() - # poll would be fooled by a RIVAL invocation's flock — only the holder - # itself knows whether it won every board. - r_fd, w_fd = os.pipe() - pid = os.fork() - if pid > 0: - os.close(w_fd) - os.waitpid(pid, 0) # reap intermediate child - ready, _, _ = select.select([r_fd], [], [], 10) - ok = bool(ready) and os.read(r_fd, 1) == b'1' - os.close(r_fd) - if ok: - print(f'held: {", ".join(boards)}') - return 0 - for b in boards: - info = read_record(b) - if info: - print(f'ERROR: {b} locked: {info}', file=sys.stderr) - print('ERROR: holder failed to acquire locks', file=sys.stderr) - return 1 - # intermediate child: detach, then spawn the actual holder - os.setsid() - if os.fork() > 0: - os._exit(0) - # holder (grandchild): acquire all flocks, signal the parent, sleep until killed - os.close(r_fd) - # Keep the success pipe clear of fds 0-2: invoked with stdio closed, - # os.pipe() can land there and the dup2 loop below would clobber it. - if w_fd <= 2: - w_fd = fcntl.fcntl(w_fd, fcntl.F_DUPFD, 3) - # Detach stdio: a `hold` whose output is captured must see EOF when the - # front-end exits — the immortal holder must not keep that pipe open. - devnull = os.open(os.devnull, os.O_RDWR) - for std_fd in (0, 1, 2): - os.dup2(devnull, std_fd) - if devnull > 2: - os.close(devnull) - try: - handles = [] - for b in boards: - fh = flock_nb(b) - if not write_record(fh, reason): - raise OSError(f'cannot write holder record for {b}') - handles.append(fh) - except OSError: - try: - os.write(w_fd, b'0') - except OSError: - pass - os._exit(1) # lost a race; parent reports the failure - os.write(w_fd, b'1') - os.close(w_fd) - - def _bow_out(*_): - # clear the records before dying so read_record/status stay truthful - # (the kernel drops the flocks themselves on exit either way) - for h in handles: - clear_record(h) - os._exit(0) - - signal.signal(signal.SIGTERM, _bow_out) - while True: - signal.pause() - - -def cmd_release(boards): - rc = 0 - victims = set() - for b in boards: - try: - fd = os.open(lock_path(b), os.O_RDWR) - except OSError: - continue # no lock file (or another user's): nothing we can release - fh = os.fdopen(fd, 'r+') - try: - fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB) - except OSError: - # flock genuinely held — never SIGTERM on a mere pid record: the - # pid may be recycled, or a live worker that already moved on. - fh.close() - info = read_record(b) or {} - pid = info.get('pid') - reason = info.get('reason') - if reason in PROTECTED_REASONS: - print(f'ERROR: {b} is mid-test by {reason} (pid {pid}) — not killing it; ' - 'wait for it to finish', file=sys.stderr) - rc = 1 - elif isinstance(pid, int) and pid > 0: - victims.add(pid) - else: - print(f'ERROR: {b} is held but its record is unreadable', file=sys.stderr) - rc = 1 - continue - # flock was free: only a stale record remained — clear it - clear_record(fh) - fh.close() - for holder in sorted(victims): - try: - os.kill(holder, signal.SIGTERM) - print(f'released holder pid {holder}') - except ProcessLookupError: - pass - except PermissionError: - print(f'ERROR: holder pid {holder} belongs to another user — cannot signal it', - file=sys.stderr) - rc = 1 - time.sleep(0.3) - still = [b for b in boards if is_locked(b)] - if still: - print(f'ERROR: still locked: {", ".join(still)}', file=sys.stderr) - return 1 - return rc - - -def cmd_status(): - if not os.path.isdir(BOARD_LOCK_DIR): - print('no locks') - return 0 - any_locked = False - for fn in sorted(os.listdir(BOARD_LOCK_DIR)): - if not fn.endswith('.lock'): - continue - b = fn[:-5] - if is_locked(b): - any_locked = True - print(f'{b}: {read_record(b)}') - if not any_locked: - print('no locks') - return 0 - - -_CLI_USAGE = """Per-board advisory locks for the HIL rig. - -Arbitrates board access between dev sessions and CI's hil_test.py without -stopping the actions-runner. Locks are kernel flocks: the kernel releases -them automatically when the holder process dies, and holders clear their -lock-file record on release so records stay truthful (/tmp also clears on -reboot). - -Usage: - hil_lock.py hold BOARD [BOARD...] --reason TEXT - hil_lock.py hold --all [--config CONFIG.json] --reason TEXT - hil_lock.py release BOARD [BOARD...] | release --all - hil_lock.py status - -A holder process holds ALL boards given in one `hold` call; releasing any of -them kills that holder and releases all of its boards. -""" - - -def main(): - ap = argparse.ArgumentParser(description=_CLI_USAGE, - formatter_class=argparse.RawDescriptionHelpFormatter) - sub = ap.add_subparsers(dest='cmd', required=True) - p_hold = sub.add_parser('hold') - p_hold.add_argument('boards', nargs='*') - p_hold.add_argument('--all', action='store_true') - p_hold.add_argument('--config', - default=os.path.join(os.path.dirname(os.path.abspath(__file__)), - 'tinyusb.json'), - help='board roster JSON (default: tinyusb.json beside this script)') - p_hold.add_argument('--reason', required=True) - p_rel = sub.add_parser('release') - p_rel.add_argument('boards', nargs='*') - p_rel.add_argument('--all', action='store_true') - sub.add_parser('status') - a = ap.parse_args() - if a.cmd == 'hold': - boards = boards_from_config(a.config) if a.all else a.boards - if not boards: - ap.error('no boards given (name boards or use --all)') - sys.exit(cmd_hold(boards, a.reason)) - if a.cmd == 'release': - if a.all: - boards = ([fn[:-5] for fn in os.listdir(BOARD_LOCK_DIR) if fn.endswith('.lock')] - if os.path.isdir(BOARD_LOCK_DIR) else []) - else: - boards = a.boards - if not boards: - ap.error('no boards given (name boards or use --all)') - sys.exit(cmd_release(boards)) - sys.exit(cmd_status()) - - -if __name__ == '__main__': - main() diff --git a/test/hil/hil_pool_check.py b/test/hil/hil_pool_check.py deleted file mode 100644 index 98f24288a..000000000 --- a/test/hil/hil_pool_check.py +++ /dev/null @@ -1,1015 +0,0 @@ -#!/usr/bin/env python3 -"""Quick HIL pool health check. - -For every board in the rig's HIL config: is the flash probe on the USB bus, does a -light example flash, and does the board's USB device (uid) come back up? Missing -firmware is BUILT on the spot (tools/build.py, idf.py for espressif; one get_deps -retry) — never skipped; --no-build opts out. Applies only per-device-safe recovery -(probe authorized-toggle, board reset/re-flash) and prints a markdown summary -table. Row statuses: ok (flashed and verified; under --scan-only: probe present — -the scan checks presence only), flash-failed (firmware delivery failed: probe -missing, build failed, flasher error, silent flash no-op, park not verified), -failed (the check ran but did not verify: flashed with no enumeration/serial, or -the check itself errored), locked (board flock held by another process — -reported, never waited on or bypassed). - -Config is picked by hostname unless given: ci -> tinyusb.json, tusb (hifiphile -rig) -> hfp.json, anything else is a dev PC -> local.json. - -Lives in test/hil/ beside hil_lock.py and hil_flash.py, which it imports; board -recovery uses the repo's .claude/skills/usb-kernel-recover/scripts/usb_recover.sh. -""" - -import argparse -import io -import json -import glob -import os -import re -import shlex -import shutil -import socket -import subprocess -import sys -import threading -import time -from concurrent.futures import ThreadPoolExecutor -from pathlib import Path - -REPO_ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(Path(__file__).resolve().parent)) # for import-as-module callers - -import hil_lock -import hil_flash - -USB_RECOVER = REPO_ROOT / '.claude' / 'skills' / 'usb-kernel-recover' / 'scripts' / 'usb_recover.sh' -SEEN_CACHE = Path.home() / '.cache' / 'tinyusb-hil' / 'pool_seen.json' -CONFIG_BY_HOST = {'ci': 'tinyusb.json', 'tusb': 'hfp.json'} # anything else: dev PC -> local.json - -# light-example preference; first built wins -DEVICE_CANDIDATES = ['device/dfu_runtime', 'device/cdc_msc', 'device/cdc_msc_freertos', - 'device/hid_composite_freertos', 'device/cdc_dual_ports'] -HOST_CANDIDATES = ['host/device_info', 'host/cdc_msc_hid', 'host/msc_file_explorer_freertos'] - -ENUM_WAIT = 12 # s, uid wait after flash -ENUM_WAIT_RETRY = 8 # s, uid wait after a recovery reset/re-flash -SERIAL_WAIT = 6 # s, host-board serial-output wait - -print_mutex = threading.Lock() -t0 = time.monotonic() - - -def say(msg: str) -> None: - with print_mutex: - print(f'[{time.monotonic() - t0:6.1f}s] {msg}', file=sys.__stdout__, flush=True) - - -def scan_usb() -> dict: - """busport -> {'serial', 'vidpid', 'ino'} for every enumerated USB device. Only - -[....] dirs match (root hubs, named 'usbN' with no dash, are - excluded: their fabricated PCI-address 'serial' and slow autosuspend-wake read - cost 6-7s/scan on this rig). Keyed by busport, not serial: a serial can be - shared by two different devices (e.g. an Espressif USB-Serial-JTAG bridge and - the cafe TinyUSB device it flashes derive both from the same MAC) — collapsing - them into one dict slot would silently drop whichever lost the race.""" - found = {} - for f in glob.glob('/sys/bus/usb/devices/*-*/serial'): - d = os.path.dirname(f) - busport = os.path.basename(d) - try: - sn = open(f).read().strip().lower() - vidpid = f'{open(d + "/idVendor").read().strip()}:{open(d + "/idProduct").read().strip()}' - found[busport] = {'serial': sn, 'vidpid': vidpid, 'ino': os.stat(d + '/').st_ino} - except OSError: - continue - return found - - -def find_usb(uid: str, devs: dict | None = None): - """Locate a flasher probe by uid, excluding VID cafe (TinyUSB DUT firmware): a - probe's uid can coincidentally equal its DUT's (Espressif USB-Serial-JTAG - bridges derive both from the same MAC), and the DUT is never the probe. - - J-Link zero-pads numeric serials (681295394 -> 000681295394): an all-digit uid - matches an all-digit serial only when that serial equals the uid zero-padded to - the serial's own length (leading zeros only) — never when the zero-stripped uid - is empty, so a placeholder serial (metro_m4_express's probe legitimately reports - '123456') can't be mistaken for an unrelated device.""" - devs = devs if devs is not None else scan_usb() - u = uid.lower() - candidates = [(bp, dev) for bp, dev in devs.items() if not dev['vidpid'].startswith('cafe:')] - for bp, dev in candidates: - if dev['serial'] == u: - return bp, dev['vidpid'], dev['ino'] - stripped = u.lstrip('0') - if u.isdigit() and stripped: - for bp, dev in candidates: - s = dev['serial'] - if s.isdigit() and s == stripped.zfill(len(s)): - return bp, dev['vidpid'], dev['ino'] - return None - - -def find_device(uid: str, pid: str | None): - """Board-online check: TinyUSB device (idVendor cafe) with this uid, optionally - PID-pinned. VID cafe keeps an Espressif USB-Serial-JTAG (303a) sharing the MAC - serial from false-passing.""" - for busport, dev in scan_usb().items(): - if (dev['serial'] == uid.lower() and dev['vidpid'].startswith('cafe:') - and (pid is None or dev['vidpid'].endswith(pid))): - return busport, dev['vidpid'], dev['ino'] - return None - - -def wait_device(uid: str, pid: str | None, old_ino, budget: float): - """Wait for the board's device with a NEW sysfs inode (flash resets the MCU, so a - genuine flash must re-enumerate; the inode is the re-enumeration marker).""" - deadline = time.monotonic() + budget - while time.monotonic() < deadline: - hit = find_device(uid, pid) - if hit and hit[2] != old_ino: - return hit - time.sleep(0.5) - return None - - -def lock_board(name: str): - """Nonblocking flock per hil_lock.py protocol. Returns handle, or a str with - the holder's info when the board is locked elsewhere. Board locks are ALWAYS - respected: a held board is reported as locked and skipped — never waited on, - and there is deliberately no bypass here.""" - os.makedirs(hil_lock.BOARD_LOCK_DIR, exist_ok=True) - try: - fh = hil_lock.flock_nb(name) - except OSError: - # NB: conflates a held flock with open() failures (EACCES/EROFS/ENOSPC) — - # benign while everything on the rig runs as one uid; a cross-uid setup - # would need flock_nb to distinguish the two - info = hil_lock.read_record(name) - return json.dumps(info) if info else 'unknown holder' - if not hil_lock.write_record(fh, 'pool_check'): - # an invisible lock (flock held, no record) is worse than no lock: status - # can't show us and release can't recognize the protected holder — bail out - hil_lock.clear_record(fh) - fh.close() - return 'ERROR: holder record write failed (lock dir unwritable?)' - return fh - - -def unlock_board(fh) -> None: - hil_lock.clear_record(fh) - fh.close() - - -def can_recover() -> bool: - if not USB_RECOVER.is_file(): - return False - try: - r = subprocess.run(['sudo', '-n', 'true'], capture_output=True) - except OSError: # sudo not installed (bare dev PC/container): recovery off, not fatal - return False - return r.returncode == 0 - - -def recover_probe(uid: str, busport: str) -> bool: - """Soft-replug an enumerated-but-wedged probe: deauthorize+reauthorize (no VBUS - cut, touches only this device). Success = the probe re-enumerated (new sysfs - inode), not the helper's exit code (observed to flake while the toggle worked). - J-Links respond with a full disconnect and can stay off the bus for >8 s.""" - pre = find_usb(uid) - try: - # bounded: the sysfs authorized store can block in D state on a wedged - # device, and this runs while the board's (release-protected) flock is held - subprocess.run(['sudo', '-n', str(USB_RECOVER), 'authorized', busport], - capture_output=True, text=True, timeout=30) - except subprocess.TimeoutExpired: - return False - deadline = time.monotonic() + 20 - while time.monotonic() < deadline: - post = find_usb(uid) - if post and (pre is None or post[2] != pre[2]): - return True - time.sleep(0.5) - return False - - -def resolve_variant(board: dict, example: str, note: list | None = None) -> str: - """Build-dir variant name for `example`: the first of the board's variants with - already-built firmware, falling back to the board name. Notes the pick when it - differs from the board name (e.g. nanoch32v203's build dir is variant - 'nanoch32v203-fsdev', not the board name).""" - name = board['name'] - for v in board.get('variant') or [{'name': name}]: - vn = v['name'] - if hil_flash.find_firmware(vn, example, flasher=board['flasher']['name']): - if vn != name and note is not None and f'variant: {vn}' not in note: - note.append(f'variant: {vn}') - return vn - return name - - -def pick_example(board: dict, note: list, build_missing: bool = True): - """(example, kind, variant, fw) with built firmware for this board; kind is - 'device' (uid check) or 'host' (serial-output check); variant is the resolved - build-dir variant that has it (see resolve_variant); fw is the firmware path to - flash, extension included. When nothing is built and build_missing is set (the default — - never skip a board for lack of a build), the preferred candidate is built on - the spot via ensure_fw.""" - tests = board.get('tests', {}) - only = tests.get('only', []) - skip = set(tests.get('skip', [])) # config's known-broken examples: never pick one - is_device = tests.get('device') or any(t.startswith('device/') for t in only) - if is_device: - cand = DEVICE_CANDIDATES + [t for t in only if t.startswith('device/') and t != 'device/usbtest'] - kind = 'device' - else: - cand = HOST_CANDIDATES + [t for t in only if t.startswith('host/')] - kind = 'host' - for ex in dict.fromkeys(cand): - if ex in skip: - continue - variant = resolve_variant(board, ex, note) - fw = hil_flash.find_firmware(variant, ex, flasher=board['flasher']['name']) - if fw: - return ex, kind, variant, fw - if not build_missing: - return None, kind, None, None - # nothing built anywhere: build the preferred candidate (an only-list board - # must get one of its own examples — dfu_runtime etc. may not even configure) - pref = [c for c in dict.fromkeys(cand) if c not in skip and (not only or c in only)] - if not pref: - return None, kind, None, None - variant = (board.get('variant') or [{'name': board['name']}])[0]['name'] - for ex in pref[:2]: # the second candidate covers a preferred example that fails to build - fw = ensure_fw(board, variant, ex, note) - if fw: - return ex, kind, variant, fw - return None, kind, None, None - - -_pid_cache: dict[str, str | None] = {} - - -def get_expected_pid(example: str) -> str | None: - """USB_PID for `example`'s device descriptor (examples//src/ - usb_descriptors.c, '#define USB_PID 0x....'), lowercased and without the 0x - prefix to match sysfs idProduct. Cached per example; None (also cached) when - the file or define isn't there — host examples have no usb_descriptors.c, and - the caller must stay quiet rather than false-warn.""" - if example not in _pid_cache: - pid = None - try: - text = (REPO_ROOT / 'examples' / example / 'src' / 'usb_descriptors.c').read_text() - # optional parens as in tools/check_example_pids.py's parser - m = re.search(r'#define\s+USB_PID\s+\(?\s*(0x[0-9a-fA-F]+)', text) - if m: - pid = m.group(1)[2:].lower() - except OSError: - pass - _pid_cache[example] = pid - return _pid_cache[example] - - -def call_flasher(fn, *fn_args) -> tuple[int, str]: - """Run a hil_flash flash_*/reset_* backend, normalizing raises to a failure: - several backends raise instead of returning nonzero (get_serial_dev - RuntimeError when a bridge's /dev/serial/by-id node vanishes, config.env - FileNotFoundError, .jlink script OSError) and an exception must not skip the - caller's retry/recovery ladder. Returns (returncode, error line).""" - try: - ret = fn(*fn_args) - if ret.returncode == 0: - return 0, '' - err = flash_error_line(hil_flash.cmd_stdout_text(ret.stdout)) - return ret.returncode, err or f'rc={ret.returncode}' - except Exception as e: - return -1, repr(e)[:90] - - -def flash(board: dict, fw, allow_recovery: bool, probe_port: str, note: list) -> bool: - """Flash the resolved firmware with one retry; on repeated failure soft-replug - the probe and always make one final flash attempt afterward, regardless of - whether the replug is confirmed — some probes (WCH-Link, ST-Link, CP210x, - picoprobe) leave their sysfs kobject intact across an authorized toggle - instead of dropping off the bus. Returns True on success. - - `fw` comes from pick_example: a re-resolve here would use the global search - policy and miss a firmware ensure_fw just built into cmake-build/ under an - exclusive -B.""" - fn = getattr(hil_flash, f'flash_{board["flasher"]["name"].lower()}') - for attempt in range(3): - if attempt == 2: - if not (allow_recovery and probe_port): - return False - cur = find_usb(board['flasher']['uid']) - if cur is None: - # probe gone from the bus: its old busport may now hold an UNRELATED - # device (bus renumbering) and the helper only checks occupancy, so - # toggling would deauthorize an innocent fixture — skip the toggle - note.append('probe vanished before toggle') - else: - say(f'{board["name"]:26} recovery: replugging probe {cur[0]} (authorized toggle)') - if recover_probe(board['flasher']['uid'], cur[0]): - note.append('probe replugged') - time.sleep(2) # udev recreates /dev/serial/by-id symlinks after re-enumeration - else: - note.append('probe toggle unconfirmed') - rc, err = call_flasher(fn, board, str(fw)) - if rc == 0: - return True - if rc == 127: # flasher binary missing: retries/probe recovery can't fix env - note.append(f'flasher tool missing ({err}) — esptool needs the ESP-IDF env (get-idf)' - if board['flasher']['name'].lower() == 'esptool' else - f'flasher tool missing: {err}') - return False - if attempt == 0: - say(f'{board["name"]:26} flash retry: {err}') - else: - note.append(f'flash: {err}') - return False - - -def flash_error_line(out: str) -> str: - """Most informative line of a failed flash's output: last error-looking line, - else the last non-empty one.""" - lines = [l.strip() for l in out.splitlines() if l.strip()] - for l in reversed(lines): - if any(k in l.lower() for k in ('error', 'fail', 'unknown', 'cannot', 'timeout', - 'no valid', 'not found', 'unable')): - return l[:90] - return lines[-1][:90] if lines else '' - - -def check_host_serial(board: dict, do_reset: bool = True, want_hello: bool = False) -> bytes | None: - """Host-only boards never enumerate their uid (their USB port is the host side); - aliveness = output on the flasher's UART bridge after a reset. A probe byte is - written each poll so an echo-only firmware (board_test) also answers. Returns - the first output chunk (b'' when silent, None when the port is absent/drops) so - the caller can also judge WHAT answered — see boardtest_output(). - - do_reset=False listens to the firmware as-is: used right after a flash whose - own reset already started it — a second openocd/JLink session back-to-back on - the same probe can fail transiently and leave the target halted.""" - import serial - try: - port = hil_flash.get_serial_dev(board['flasher']['uid'], None, None, 0) - ser = serial.Serial(port, baudrate=115200, timeout=0.3, write_timeout=1) - except Exception as e: - say(f'{board["name"]:26} no flasher serial port: {e}') - return None - try: - # flush BEFORE issuing the reset: pyserial's open-time flush is long past, - # so this drops the pre-reset CDC backlog (which must not count as life) - # while keeping the board's post-reset boot banner, which prints while the - # reset tool is still tearing down and would be eaten by a post-reset flush - ser.reset_input_buffer() - if do_reset: - getattr(hil_flash, f'reset_{board["flasher"]["name"].lower()}')(board) - # collect the WHOLE window and judge content, not the first chunk: the - # probe's CDC bridge has its own FIFO, so stale pre-flash output (e.g. - # board_test hellos) can arrive after our host-side flush and must not - # decide the verdict alone. Early-exit once non-board_test output proves - # a real example is talking. - data = b'' - deadline = time.monotonic() + SERIAL_WAIT - while time.monotonic() < deadline: - try: - ser.write(b'U') - data += ser.read(256) - except serial.SerialTimeoutException: - pass - except serial.SerialException: - return None # port dropped mid-poll (bridge re-enumerating) - # early-exit on the caller's positive signal: fresh board_test hello - # (park verification) vs any non-board_test output (example liveness); - # stale bridge-FIFO backlog of the OTHER kind must not end the window - if want_hello: - if b'Hello from TinyUSB' in data: - return data - elif data and not boardtest_output(data): - return data - return data - finally: - ser.close() - - -def boardtest_output(data: bytes) -> bool: - """True when (non-empty) serial output is recognizably ONLY board_test's: its - periodic HELLO_STR and echoes of our b'U' pokes, nothing else. Any residue - beyond that (an example banner, log lines) proves other firmware is talking, - however much stale board_test backlog surrounds it. Used as a negative - identity marker — after flashing a host example, board_test-only chatter - means the flash silently didn't take (the host analog of the PID check).""" - residue = data.replace(b'Hello from TinyUSB', b'') - for junk in (b'U', b'\r', b'\n'): - residue = residue.replace(junk, b'') - return len(residue) == 0 - - -def build_example(board: dict, variant: str, example: str) -> int: - """Build one example for this board: tools/build.py (same invocation shape as - hil_test.build_board: -T target, -D per build.args, variant defines/flags, - --build-name), or idf.py directly for espressif (tools/build.py's esp branch - ignores -T and builds everything; variant flags travel as -DCFLAGS_CLI, the - same channel tools/build.py uses). Bounded and process-group-killed via - run_cmd; 600 s: a first configure+build of an SDK-heavy family (pico, nrf, - esp) exceeds the old 300. Builds normally run pre-lock (pick_example / the - pre-park ensure), so a board flock is not held here except on rare recovery - paths. Per-build compile parallelism is capped at cpu/-j so -j concurrent - builds cannot swamp sibling workers' verification windows. Returns the - build's returncode (127 = ESP-IDF env missing).""" - name = board['name'] - variants = board.get('variant') or [{'name': name}] - vcfg = next((v for v in variants if v['name'] == variant), variants[0]) - if board['flasher']['name'].lower() == 'esptool': - if not shutil.which('idf.py'): - return 127 # ESP-IDF env not sourced in this shell - # -B keyed off the VARIANT so ensure_fw's post-build lookup finds it - cmd = ['idf.py', '-C', f'examples/{example}', - '-B', f'cmake-build/cmake-build-{vcfg["name"]}/{example}', - '-G', 'Ninja', f'-DBOARD={name}', 'build'] - for d in board.get('build', {}).get('args', []) + vcfg.get('defines', []): - cmd.insert(-1, f'-D{d}') - if vcfg.get('flags'): - cmd.insert(-1, f'-DCFLAGS_CLI={vcfg["flags"]}') - # the IDF component manager writes examples//dependencies.lock in the - # SOURCE tree (idf.py -B relocates only the build dir), so concurrent esp - # builds of one example for different targets corrupt each other's solve - with _esp_lock, _build_sem: - return hil_flash.run_cmd(shlex.join(cmd), cwd=str(hil_flash.TINYUSB_ROOT), - timeout=600).returncode - cmd = [sys.executable, str(hil_flash.TINYUSB_ROOT / 'tools' / 'build.py'), - '-b', name, '-T', Path(example).name, - '-j', str(max(1, (os.cpu_count() or _jobs) // _jobs))] - for d in board.get('build', {}).get('args', []): - cmd += ['-D', d] - if vcfg['name'] != name: - cmd += ['--build-name', vcfg['name']] - for d in vcfg.get('defines', []): - cmd += ['-D', d] - for tok in vcfg.get('flags', '').split(): - cmd += [f'--cflag={tok}'] - with _build_sem: - return hil_flash.run_cmd(shlex.join(cmd), cwd=str(hil_flash.TINYUSB_ROOT), - timeout=600).returncode - - -_deps_lock = threading.Lock() # one get_deps at a time (it also drains _build_sem) -_esp_lock = threading.Lock() # idf.py mutates source-tree dependencies.lock per example -_no_build = False # --no-build: ensure_fw never invokes a build -_jobs = 4 # mirrors -j; set in main before the pool starts -_build_sem = threading.BoundedSemaphore(4) # build slots; get_deps drains ALL (exclusive) -_builds: dict = {} # (variant, example) -> (fw|None, reason): one attempt per run - - -def ensure_fw(board: dict, variant: str, example: str, note: list): - """Firmware for `example`, building it when absent — never skip a board for - lack of a build (--no-build opts out). One retry with deps fetched and the - CMake caches dropped when the first build fails (fresh checkouts lack the - family deps; a cache configured in a broken env poisons every later attempt). - Returns the firmware path, or None with the failure noted. Call BEFORE - taking the board lock: builds are long. One build attempt per - (variant, example) per run, success or failure — memoized in _builds, so a - repeat call (park, under the held flock) resolves instantly even when an - exclusive -B hides the fresh cmake-build/ artifact from the global search.""" - fw = hil_flash.find_firmware(variant, example, flasher=board['flasher']['name']) - if fw: - return fw - key, base = (variant, example), Path(example).name - if key in _builds: - return _builds[key][0] - if _no_build: - _builds[key] = (None, 'disabled') - note.append(f'build skipped (--no-build): {base}') - return None - rc = build_example(board, variant, example) - if rc == 127 and board['flasher']['name'].lower() == 'esptool': - _builds[key] = (None, 'no-env') - note.append(f'cannot build {base}: ESP-IDF env missing (get-idf)') - return None - if rc == 124: # hung build: a deps/cache retry cannot cure it, don't double the stall - _builds[key] = (None, 'timeout') - note.append(f'build timeout: {base}') - return None - if rc != 0: - # retry once with deps fetched and the CMake caches dropped (cache only — - # a tree wipe would destroy every other example's firmware). get_deps - # git-resets already-present shared deps (lib/fatfs's ffconf.h dance), so - # it must exclude every in-flight build, not just other get_deps calls: - # it drains ALL build slots before running. - with _deps_lock: - for _ in range(_jobs): - _build_sem.acquire() - try: - r = hil_flash.run_cmd(shlex.join([sys.executable, str(hil_flash.TINYUSB_ROOT / 'tools' / 'get_deps.py'), - '-b', board['name']]), - cwd=str(hil_flash.TINYUSB_ROOT), timeout=600) - finally: - for _ in range(_jobs): - _build_sem.release() - if r.returncode != 0: - note.append('get_deps failed') - bd = hil_flash.TINYUSB_ROOT / 'cmake-build' / f'cmake-build-{variant}' - # esp configures one level deeper (//): wipe both layouts - for d in (bd, bd / example): - shutil.rmtree(d / 'CMakeFiles', ignore_errors=True) - (d / 'CMakeCache.txt').unlink(missing_ok=True) - rc = build_example(board, variant, example) - if rc != 0: - _builds[key] = (None, 'fail') - note.append(f'build failed: {base}') - return None - # tools/build.py and the idf.py invocation above always write to cmake-build/: - # look there too even when an explicit -B narrowed the global search — this is - # OUR fresh build, not a stale-candidate fallback - fw = hil_flash.find_firmware(variant, example, - roots=[hil_flash.build_dir, 'cmake-build'], - flasher=board['flasher']['name']) - _builds[key] = (fw, 'ok' if fw else 'no-fw') - note.append(f'built {base}' if fw else f'build produced no firmware: {base}') - return fw - - -def ensure_board_test(board: dict, variant: str, note: list): - """board_test firmware for parking, building it if absent (via ensure_fw). - Espressif included — tools/build.py builds board_test for that family too; - the build just needs the ESP-IDF env (127 → noted, park is then skipped).""" - fw = hil_flash.find_firmware(variant, 'device/board_test', flasher=board['flasher']['name']) - if fw: - return fw - variants = board.get('variant') or [{'name': board['name']}] - if not any(v['name'] == variant for v in variants): - variant = variants[0]['name'] - return ensure_fw(board, variant, 'device/board_test', note) - - -def verdict(row: dict, ok: bool) -> str: - """Row status for a verification result, preserving a 'flash-failed' a deeper - layer already recorded (silent flash no-op, board_test delivery failure).""" - return 'ok' if ok else ('flash-failed' if row['status'] == 'flash-failed' else 'failed') - - -def host_alive(board: dict, note: list, row: dict, flashed_example: bool = False) -> bool: - """Serial aliveness with recovery: silent -> (build and) flash board_test (it - hellos every second and echoes) -> recheck. Also cures a silent flash no-op - that left the board crashed. - - With flashed_example=True (a host example was just flashed), board_test-shaped - output FAILS the check: the parked image still talking means the example flash - silently didn't take — the host analog of the device path's PID check. - - Side effect: delivery-class failures (silent no-op, board_test build/flash - failure) set row['status'] = 'flash-failed' so verdict() preserves the cause; - the caller derives the final status from the return value via verdict().""" - data = check_host_serial(board) - if data: - if flashed_example and boardtest_output(data): - note.append('board_test output after example flash: silent flash no-op') - row['status'] = 'flash-failed' - return False - return True - variant = resolve_variant(board, 'device/board_test', note) - fw = ensure_board_test(board, variant, note) - if fw is None: - note.append('serial silent; board_test unavailable') - row['status'] = 'flash-failed' - return False - say(f'{board["name"]:26} recovery: serial silent, flashing board_test') - rc, err = call_flasher(getattr(hil_flash, f'flash_{board["flasher"]["name"].lower()}'), board, str(fw)) - if rc != 0: - note.append(f'serial silent; board_test flash failed: {err}') - row['status'] = 'flash-failed' - return False - if not check_host_serial(board): - return False - if flashed_example: - # board_test talking proves the BOARD is alive, but the just-flashed - # example never produced serial — that verification still fails - note.append('example silent; board alive via board_test reflash') - return False - note.append('recovered via board_test reflash') - return True - - -def device_recover_and_check(board: dict, example: str, variant: str, old_ino, note: list, row: dict, seen: dict) -> bool: - """Wait for the flashed board's uid to re-enumerate; on timeout, try one board - reset (skipped for flashers with no hardware reset — see hil_flash.RESET_NOOP, - it would just burn the wait) and wait again. - - The PID policy is deliberately asymmetric. Pre-reset, the re-enumeration was - caused by the flash itself, so a PID mismatch most likely means the build dir - is stale (the flash DID write what find_firmware found) — warn, don't fail — - UNLESS the firmware was built this very run: then 'stale build' is impossible - and the mismatch can only be a silent flash no-op, which fails. Post-reset, - the re-enumeration proves nothing about the flash (the reset alone explains - it), so a mismatch is treated as a silent flash no-op and fails; an unknown - expected PID scores ok with a 'pid unverified' note in both paths.""" - name = board['name'] - expected_pid = get_expected_pid(example) - built_this_run = _builds.get((variant, example), (None, ''))[1] == 'ok' - - def seen_hit(hit): - seen[board['uid']] = {'name': name, 'busport': hit[0], 'when': time.strftime('%Y-%m-%d %H:%M')} - - hit = wait_device(board['uid'], None, old_ino, ENUM_WAIT) - if hit: - if expected_pid is not None and not hit[1].endswith(expected_pid): - if built_this_run: - row['device'] = f'❌ {hit[1]}' - note.append(f'pid {hit[1]}, this run built {expected_pid}: silent flash no-op') - row['status'] = 'flash-failed' - return False - note.append(f'⚠ pid {hit[1]}, source says {expected_pid}: stale build or silent flash no-op') - elif expected_pid is None: - note.append('pid unverified') - row['device'] = f'✅ {hit[1]}' - seen_hit(hit) - return True - - flasher_name = board['flasher']['name'].lower() - if flasher_name in hil_flash.RESET_NOOP: - note.append(f'no hardware reset available for {flasher_name}') - row['device'] = '❌ not enumerated' - return False - - say(f'{name:26} recovery: uid not up, resetting board') - rc, err = call_flasher(getattr(hil_flash, f'reset_{flasher_name}'), board) - if rc != 0: - note.append(f'reset failed: {err}') - hit = wait_device(board['uid'], None, old_ino, ENUM_WAIT_RETRY) - if not hit: - row['device'] = '❌ not enumerated' - note.append('reset did not help') - return False - if expected_pid is None: - row['device'] = f'✅ {hit[1]}' - note.append('reset recovered (pid unverified)') - seen_hit(hit) - return True - if hit[1].endswith(expected_pid): - row['device'] = f'✅ {hit[1]}' - note.append('reset recovered') - seen_hit(hit) - return True - row['device'] = f'❌ {hit[1]}' - note.append(f'reset recovered wrong pid, expected {expected_pid}: silent flash no-op') - row['status'] = 'flash-failed' - return False - - -def check_board(board: dict, args, allow_recovery: bool, seen: dict) -> dict: - name = board['name'] - row = {'name': name, 'probe': '❌ missing', 'flash': '–', 'device': '–', 'note': [], 'status': 'failed'} - note = row['note'] - - probe = find_usb(board['flasher']['uid']) - if probe: - row['probe'] = f'✅ {probe[0]}' - seen[board['flasher']['uid']] = {'name': f'{name} probe', 'busport': probe[0], - 'when': time.strftime('%Y-%m-%d %H:%M')} - else: - last = seen.get(board['flasher']['uid']) - note.append(f'probe last seen {last["busport"]} {last["when"]}' if last - else 'probe never seen by pool_check') - say(f'{name:26} probe MISSING ({board["flasher"]["name"]} {board["flasher"]["uid"]})') - - # existing firmware only here; a missing build is built on the spot further - # down (after a lock peek), except in scan/no-build modes — and never for a - # missing probe (nothing could be flashed anyway) - example, kind, variant, fw = pick_example(board, note, build_missing=False) - if kind == 'host': - note.append('host-only board') - - if args.scan_only: - hit = find_device(board['uid'], None) - # report the BOARD's usb state, not just the probe's: the enumerated device - # (with busport), off-bus (normal when parked in board_test), or n/a for - # host-only boards whose uid never enumerates - if hit: - row['device'] = f'✅ {hit[1]} @{hit[0]}' - elif kind == 'host': - row['device'] = '– n/a (host-only)' - else: - row['device'] = '⚫ off bus (parked?)' - # scan verifies probe presence only: that check DID run, so probe present - # is ok; a missing probe means no firmware could be delivered → flash-failed - row['status'] = 'ok' if probe else 'flash-failed' - if probe: - say(f'{name:26} probe ✅ {probe[0]}' + (f' device {hit[1]}' if hit else '')) - return row - if not probe: - row['status'] = 'flash-failed' - return row - - bt_variant = resolve_variant(board, 'device/board_test', note) - need_example = example is None and not args.no_build - # board_test is also host_alive's recovery image, so host boards pre-build it - # even under --no-park; --no-build gates EVERY build, board_test included - need_bt = (not args.no_build - and (not args.no_park or kind == 'host') - and hil_flash.find_firmware(bt_variant, 'device/board_test', - flasher=board['flasher']['name']) is None) - if need_example or need_bt: - # builds are long and run BEFORE locking (park must never hold the flock - # through a build); peek the lock first so minutes of building are not - # wasted on — or a rebuilt tree swapped under — a board CI holds right now - peek = lock_board(name) - if isinstance(peek, str): - if peek.startswith('ERROR:'): # environment failure, not a held lock - row['flash'] = '❌ lock' - row['status'] = 'failed' - else: - row['flash'] = '🔒 locked' - row['status'] = 'locked' - note.append(peek) - say(f'{name:26} locked: {peek}') - return row - unlock_board(peek) - if need_example: - example, kind, variant, fw = pick_example(board, note, build_missing=True) - if need_bt and (example is not None or kind == 'host'): - # skip the park-image build when the example build already failed on a - # device board: the row returns before any flash/park could use it - ensure_board_test(board, bt_variant, note) - - if example is None: - if not any(n.startswith(('build failed', 'build timeout', 'build produced', - 'build skipped', 'cannot build')) for n in note): - note.append('no firmware built') - if kind != 'host': - row['status'] = 'flash-failed' - say(f'{name:26} probe ✅ {probe[0]} (no firmware to flash)') - return row - # host-only board: aliveness is still checkable without flashing — reset and - # listen to whatever firmware is on it (the parked board_test echoes and - # prints a periodic hello on the flasher UART) - - lk = lock_board(name) - if isinstance(lk, str): - if lk.startswith('ERROR:'): # environment failure, not a held lock - row['flash'] = '❌ lock' - row['status'] = 'failed' - else: - row['flash'] = '🔒 locked' - row['status'] = 'locked' - note.append(lk) - say(f'{name:26} locked: {lk}') - return row - try: - if example is None: # host-only without firmware: UART-only aliveness check - ok = host_alive(board, note, row) - row['device'] = '✅ serial out' if ok else '❌ no serial out' - row['status'] = verdict(row, ok) - say(f'{name:26} – {row["device"]} (existing firmware)') - return row - - pre = find_device(board['uid'], None) - old_ino = pre[2] if pre else None - - try: - if not flash(board, fw, allow_recovery, probe[0], note): - row['flash'] = f'❌ {Path(example).name}' - row['status'] = 'flash-failed' - say(f'{name:26} flash FAILED ({example})') - return row - row['flash'] = f'✅ {Path(example).name}' - - if kind == 'host': - ok = host_alive(board, note, row, flashed_example=True) - row['device'] = '✅ serial out' if ok else '❌ no serial out' - else: - ok = device_recover_and_check(board, example, variant, old_ino, note, row, seen) - row['status'] = verdict(row, ok) - say(f'{name:26} {row["flash"]} {row["device"]}') - return row - finally: - # teardown for EVERY path that attempted a flash (a failed programmer op - # can still have erased/half-written the target): re-park while the - # board lock is still held - if not args.no_park: - park_board(board, kind, row, note) - finally: - unlock_board(lk) - - -def park_board(board: dict, kind: str, row: dict, note: list) -> None: - """Re-park with board_test, building it if absent (ensure_board_test), and - VERIFY it took: board_test never enumerates USB, so a device board's cafe - device must drop off the bus, and a host board must answer with board_test's - own output — a rc=0 park that changed nothing (silent no-op) must not pass. - A board left unparked marks an ok row flash-failed (never downgrading a - 'failed' verify verdict — that is the more diagnostic signal), with one - exception: an espressif board without the ESP-IDF env cannot build - board_test — noted, not a board fault.""" - # capture BEFORE the park flash: uid-disappearance only verifies the park if - # the device was on the bus to begin with (a fast park drops it immediately) - on_bus_before = kind != 'host' and find_device(board['uid'], None) is not None - variant = resolve_variant(board, 'device/board_test', note) - fw = ensure_board_test(board, variant, note) - if fw is None: - if any(n.startswith('cannot build board_test') for n in note): - note.append('park skipped (no ESP-IDF env)') - else: - # --no-build disables builds, not parking (--no-park is that opt-out): - # a board left running a USB-active image is unparked either way - note.append('unparked: board_test not built (--no-build)' - if any(n.startswith('build skipped (--no-build): board_test') for n in note) - else 'unparked: board_test unavailable (build failed/timed out)') - if row['status'] == 'ok': - row['status'] = 'flash-failed' - return - rc, err = call_flasher(getattr(hil_flash, f'flash_{board["flasher"]["name"].lower()}'), - board, str(fw)) - if rc != 0: - note.append(f'park flash failed: {err}') - if row['status'] == 'ok': - row['status'] = 'flash-failed' - return - if kind == 'host': - # no second reset (the park flash's own reset already started board_test); - # POSITIVE marker: its hello must appear — stale example output may still - # drain from the probe bridge's FIFO alongside it and is not disqualifying - data = check_host_serial(board, do_reset=False, want_hello=True) - if not (data and b'Hello from TinyUSB' in data): - note.append('park unverified: no board_test output') - if row['status'] == 'ok': - row['status'] = 'flash-failed' - return - if not on_bus_before: - # board never enumerated this run: uid-disappearance can't distinguish a - # verified park from a silent no-op — say so instead of passing vacuously - note.append('park unverified (device already off bus)') - return - deadline = time.monotonic() + 6 - while time.monotonic() < deadline: - if find_device(board['uid'], None) is None: - return - time.sleep(0.5) - note.append('park unverified: device still enumerated') - if row['status'] == 'ok': - row['status'] = 'flash-failed' - - -def check_board_safe(board: dict, args, allow_recovery: bool, seen: dict) -> dict: - """Isolate one board's exceptions: a crashing worker must not discard every - other board's row, the table, the topology, and the seen-cache write.""" - try: - return check_board(board, args, allow_recovery, seen) - except Exception as e: - name = board.get('name', '?') - say(f'{name:26} INTERNAL ERROR: {e!r}') - return {'name': name, 'probe': '–', 'flash': '–', 'device': '❌ error', - 'note': [repr(e)[:120]], 'status': 'failed'} - - -def controller_summary() -> list[str]: - """USB topology: controller (PCI addr, vendor) -> bus -> root-port subtree device - counts (hubs included, interfaces/root hubs not). Bus numbers renumber every boot; - PCI addresses and root-port numbers are stable.""" - vendor_names = {'0x1022': 'AMD', '0x1912': 'Renesas', '0x8086': 'Intel', '0x1b21': 'ASMedia'} - ctrl = {} - for root in glob.glob('/sys/bus/usb/devices/usb*'): - bus = int(os.path.basename(root)[3:]) - m = re.findall(r'[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]', os.path.realpath(root)) - pci = m[-1] if m else '?' - c = ctrl.setdefault(pci, {'vendor': '?', 'buses': {}}) - subtrees = {} - for d in glob.glob(f'/sys/bus/usb/devices/{bus}-*'): - b = os.path.basename(d) - if ':' in b: - continue - subtrees[b.split('.')[0]] = subtrees.get(b.split('.')[0], 0) + 1 - c['buses'][bus] = subtrees - try: - vid = open(f'/sys/bus/pci/devices/{pci}/vendor').read().strip() - c['vendor'] = vendor_names.get(vid, vid) - except OSError: - pass - - lines = [] - for pci, c in sorted(ctrl.items()): - lines.append(f'{pci} ({c["vendor"]})') - for bus, subtrees in sorted(c['buses'].items()): - detail = ' '.join(f'{k}: {n} dev' for k, n in - sorted(subtrees.items(), key=lambda i: int(i[0].split('-')[1]))) - lines.append(f' bus {bus}: {sum(subtrees.values())} devices' - + (f' {detail}' if detail else '')) - return lines - - -def main() -> None: - # toolchain/flasher CLIs live in the user bin dirs (arm-none-eabi-gcc + esptool - # in ~/.local/bin, STM32_Programmer_CLI in ~/bin) which non-login shells may - # lack — same PATH shim hil_ci.sh applies on the remote side - for d in (Path.home() / 'bin', Path.home() / '.local' / 'bin'): - if d.is_dir() and str(d) not in os.environ.get('PATH', '').split(os.pathsep): - os.environ['PATH'] = f'{d}{os.pathsep}{os.environ.get("PATH", "")}' - - parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) - parser.add_argument('config', nargs='?', help='HIL config json (default: by hostname)') - parser.add_argument('-b', '--board', action='append', default=[], help='only these boards') - parser.add_argument('-B', '--build-dir', default=None, - help='firmware parent dir, searched EXCLUSIVELY when given ' - '(default: examples, plus cmake-build as fallback)') - parser.add_argument('--scan-only', action='store_true', - help='USB presence scan only: no locks, no flashing') - parser.add_argument('--no-build', action='store_true', - help='do not build missing firmware (default: build the light example on the spot)') - parser.add_argument('--no-park', action='store_true', - help='leave the light example running (default: park with board_test)') - # no cross-process flash budget with a concurrent hil_test.py run yet (would need - # a file-lock budget in hil_lock; hil_test uses in-process semaphores) — keep modest - parser.add_argument('-j', '--jobs', type=int, default=4) - parser.add_argument('-v', '--verbose', action='store_true') - args = parser.parse_args() - global _no_build, _jobs, _build_sem - _no_build = args.no_build - _jobs = max(1, args.jobs) - _build_sem = threading.BoundedSemaphore(_jobs) - - host = socket.gethostname() - cfg_name = args.config or CONFIG_BY_HOST.get(host, 'local.json') - cfg_path = Path(cfg_name) - if not cfg_path.exists(): - cfg_path = REPO_ROOT / 'test' / 'hil' / cfg_name - if not cfg_path.exists(): - sys.exit(f'config not found: {cfg_name} (host {host}; dev PCs need test/hil/local.json)') - with cfg_path.open() as f: - config = json.load(f) - - boards = list(config['boards']) # boards-skip (parked hardware) is not scanned by default - if args.board: - boards += config.get('boards-skip', []) # explicitly named parked boards are fair game - unknown = set(args.board) - {b['name'] for b in boards} - if unknown: - sys.exit(f'board(s) not in {cfg_path.name}: {", ".join(sorted(unknown))}') - boards = [b for b in boards if b['name'] in args.board] - - hil_flash.build_dir = args.build_dir or 'examples' - hil_flash.verbose = args.verbose - if args.build_dir is None: - # default mode: search both standard layouts (cmake-build/ from tools/build.py - # + ESP-IDF, examples/ from manual builds). An EXPLICIT -B is exclusive — the - # caller named an artifact tree, so a miss must report, not silently flash an - # older build from elsewhere. hil_test's -B is likewise untouched by this. - hil_flash.EXTRA_BUILD_DIRS = ['cmake-build', 'examples'] - allow_recovery = not args.scan_only and can_recover() - seen = {} - try: - loaded = json.loads(SEEN_CACHE.read_text()) - if isinstance(loaded, dict): # tolerate a torn/hand-edited cache - seen = {k: v for k, v in loaded.items() if isinstance(v, dict)} - except (OSError, ValueError): - pass - - roots = ' + '.join(dict.fromkeys([hil_flash.build_dir, *hil_flash.EXTRA_BUILD_DIRS])) - say(f'pool check: host {host}, config {cfg_path.name}, {len(boards)} boards, ' - f'{"scan-only" if args.scan_only else f"flash via {{{roots}}}/cmake-build-"}' - f'{"" if allow_recovery or args.scan_only else ", recovery unavailable (no sudo -n / usb_recover.sh)"}') - - if args.verbose: - rows = [check_board_safe(b, args, allow_recovery, seen) for b in boards] - else: - with io.StringIO() as spool, ThreadPoolExecutor(max_workers=args.jobs) as pool: - sys.stdout = spool # silence hil_flash's COMMAND FAILED dumps; say() uses __stdout__ - try: - rows = list(pool.map(lambda b: check_board_safe(b, args, allow_recovery, seen), boards)) - finally: - sys.stdout = sys.__stdout__ - - try: - SEEN_CACHE.parent.mkdir(parents=True, exist_ok=True) - tmp = SEEN_CACHE.with_suffix('.json.tmp') - tmp.write_text(json.dumps(seen, indent=1, sort_keys=True) + '\n') - tmp.replace(SEEN_CACHE) # atomic: a killed run can't tear the cache - except OSError: - pass - - status_mark = {'ok': '✅ ok', 'flash-failed': '❌ flash-failed', 'failed': '❌ failed', - 'locked': '🔒 locked'} - headers = ['Board', 'Probe', 'Flash', 'Device', 'Status', 'Note'] - cells = [[r['name'], r['probe'], r['flash'], r['device'], - status_mark.get(r['status'], r['status']), '; '.join(r['note'])] for r in rows] - widths = [max(len(h), *(len(c[i]) for c in cells)) if cells else len(h) - for i, h in enumerate(headers)] - line = lambda vals: '| ' + ' | '.join(v.ljust(w) for v, w in zip(vals, widths)) + ' |' - print() - print(line(headers)) - print('|' + '|'.join('-' * (w + 2) for w in widths) + '|') - for c in cells: - print(line(c)) - - print('\nUSB topology (controller → root-port subtree):') - for line in controller_summary(): - print(f' {line}') - - counts = {'ok': 0, 'flash-failed': 0, 'failed': 0, 'locked': 0} - for r in rows: - counts[r.get('status', 'failed')] += 1 - print(f'\n{counts["ok"]} ok · {counts["flash-failed"]} flash-failed · {counts["failed"]} failed ' - f'· {counts["locked"]} locked · in {time.monotonic() - t0:.0f}s') - sys.exit(min(counts['flash-failed'] + counts['failed'], 125)) - - -if __name__ == '__main__': - main() diff --git a/test/hil/hil_select.py b/test/hil/hil_select.py deleted file mode 100755 index 3ac3f1fdb..000000000 --- a/test/hil/hil_select.py +++ /dev/null @@ -1,520 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MIT -"""PR-diff -> HIL selection: which rig boards and which tests a change can affect. - -Stdlib-only (runs on bare CI runners; never imports hil_test/hil_flash/hil_lock). -Fail-open: any file no rule classifies forces the full matrix. See -docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md. - -JSON: full, boards (name -> 'all' | [tests]), families (bsp families the diff -touches, including ones with no rig board - build-only consumers such as /pre-pr -sample from these), args (hil_test.py args per config) and args_flasher (the same -args split by each board's flasher, for CI legs that split one rig by flasher). -""" -import argparse -import functools -import glob -import json -import os -import re -import subprocess -import sys - -from hil_examples import device_tests, dual_tests, host_test - -ALL_TESTS = {'device': device_tests, 'dual': dual_tests, 'host': host_test} - -# class dir -> config macro suffix exceptions (rule 3); dfu is per-file, handled inline -NET_MACROS = ('ECM_RNDIS', 'NCM') - -_NONCODE_RE = re.compile( - r'^(docs/|\.claude/|.*\.(md|rst)$|LICENSE)') -_FULL_RE = re.compile( - r'^(src/common/|src/osal/|src/tusb\.c$|src/tusb\.h$|src/tusb_option\.h$|' - r'test/hil/|\.github/workflows/build.*\.yml$|\.github/actions/|' - r'tools/build\.py$|tools/get_deps\.py$|tools/cmake/|hw/mcu/|lib/|' - r'hw/bsp/(family_support\.cmake|board_api\.h|board\.c|ansi_escape\.h)$|' - r'examples/build_system/|examples/CMakeLists\.txt$|' - # board_test is HIL infrastructure, not a test: hil_test.py flashes it to park - # every board (variant boundary + end-of-board teardown), so every board depends on it - r'examples/device/board_test/)') - -# --no-renames: with rename detection git reports only a rename's destination, so code -# moved out of an HIL-relevant path would be classified by its new path alone -GIT_DIFF_ARGV = ['git', 'diff', '--no-renames', '--name-only'] - - -def test_role(test: str) -> str: - return test.split('/', 1)[0] # 'device' | 'dual' | 'host' - - -def board_roles(board: dict) -> set: - t = board.get('tests', {}) - roles = set() - if t.get('device'): - roles.add('device') - if t.get('host'): - roles.add('host') - if t.get('dual'): - roles.update(('device', 'host')) - for only in t.get('only', []): - r = test_role(only) - roles.update(('device', 'host') if r == 'dual' else (r,)) - return roles - - -def board_tests(board: dict) -> list: - """Every test this board would run today (mirrors hil_test.test_board's default).""" - t = board.get('tests', {}) - if 'only' in t: - run = list(t['only']) - else: - run = [] - if t.get('device'): - run += device_tests - if t.get('dual'): - run += dual_tests - if t.get('host'): - run += host_test - return [x for x in run if x not in t.get('skip', [])] - - -# cached: called per changed file x roster board, and the tree doesn't change mid-run -@functools.lru_cache(maxsize=None) -def board_family(board_name: str, repo_root: str): - hits = glob.glob(os.path.join(repo_root, 'hw/bsp/*/boards', board_name)) - return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None - - -# `if (OPTION STREQUAL "1")` guards in family_support.cmake, and the option tokens -# a roster entry passes to the build (NAME=VALUE / -DNAME=VALUE) -_CM_IF_RE = re.compile(r'if\s*\(') -_CM_ELSE_RE = re.compile(r'else(if)?\s*\(') -_CM_ENDIF_RE = re.compile(r'endif\s*\(') -_CM_OPT_RE = re.compile(r'if\s*\(\s*\$?\{?([A-Za-z_]\w*)\}?\s+STREQUAL\s+"?1"?\s*\)') -_CM_PORT_RE = re.compile(r'src/portable/((?:[^/\s]+/)?[^/\s]+)/') -_FALSY = ('', '0', 'off', 'false', 'no') - - -@functools.lru_cache(maxsize=None) -def port_option_gates(repo_root: str) -> dict: - """port dir -> build options that compile it regardless of the board's family - file, e.g. {'analog/max3421': {'MAX3421_HOST'}} from family_support.cmake.""" - gates = {} - try: - text = open(os.path.join(repo_root, 'hw/bsp/family_support.cmake')).read() - except OSError: - return gates - stack = [] # one entry per open if(): its option, or None - for line in text.splitlines(): - line = line.strip() - if _CM_IF_RE.match(line): - m = _CM_OPT_RE.match(line) - stack.append(m.group(1) if m else None) - elif _CM_ELSE_RE.match(line): - if stack: - stack[-1] = None # the guard doesn't hold in this branch - elif _CM_ENDIF_RE.match(line): - if stack: - stack.pop() - opts = {o for o in stack if o} - m = _CM_PORT_RE.search(line) - if opts and m: - gates.setdefault(m.group(1), set()).update(opts) - return gates - - -_CM_SET_RE = re.compile(r'set\s*\(\s*([A-Za-z_]\w*)\s+([^)\s]+)\s*\)') - - -# cached: called per changed portable file x roster board -@functools.lru_cache(maxsize=None) -def bsp_board_options(board_name: str, repo_root: str) -> frozenset: - """Build options a board turns on in its own BSP: `set( )` in - hw/bsp//boards//board.cmake, e.g. MAX3421_HOST on the espressif - and rp2040 max3421 boards. CMake only - HIL CI builds nothing with Make, so a - board.mk-only option (e.g. nrf5340dk's MAX3421_HOST) compiles no port here.""" - fam = board_family(board_name, repo_root) - if not fam: - return frozenset() - path = os.path.join(repo_root, 'hw/bsp', fam, 'boards', board_name, 'board.cmake') - try: - text = open(path).read() - except OSError: - return frozenset() - out = set() - for line in text.splitlines(): - line = line.strip() - if line.startswith('#'): - continue - m = _CM_SET_RE.match(line) - if m and m.group(2).strip('"').lower() not in _FALSY: - out.add(m.group(1)) - return frozenset(out) - - -def board_options(board: dict, repo_root: str) -> set: - """Build options a board has truthy: the roster entry's build.args plus each - variant's defines (NAME=VALUE) and raw CFLAGS (-DNAME=VALUE), plus whatever its - own board.cmake sets (a board can enable a gated port without the roster saying so).""" - toks = list(board.get('build', {}).get('args', [])) - for v in board.get('variant', []): - toks += list(v.get('defines', [])) - toks += v.get('flags', '').split() - out = set(bsp_board_options(board['name'], repo_root)) - for t in toks: - name, _, val = (t[2:] if t.startswith('-D') else t).partition('=') - if name and val.strip().strip('"').lower() not in _FALSY: - out.add(name.strip()) - return out - - -@functools.lru_cache(maxsize=None) -def port_families(port_dir: str, repo_root: str) -> set: - """Board families that compile this src/portable dir. CMake only: HIL CI builds - every board with CMake, so a port wired up in family.mk alone is compiled for no - HIL board and must not select one. family.cmake lists portable sources directly - for most families; espressif instead references them from a nested component - CMakeLists.txt (hw/bsp/espressif/components/tinyusb_src/CMakeLists.txt).""" - fams = set() - bsp_root = os.path.join(repo_root, 'hw/bsp') - # trailing '/' so a port dir is not a prefix of a sibling: bare 'microchip/pic' - # would otherwise match '.../microchip/pic32mz/...' and inherit its families - needle = port_dir + '/' - for f in glob.glob(os.path.join(bsp_root, '*/family.cmake')) + \ - glob.glob(os.path.join(bsp_root, '*/components/*/CMakeLists.txt')): - try: - if needle in open(f).read(): - fam = os.path.relpath(f, bsp_root).split(os.sep, 1)[0] - fams.add(fam) - except OSError: - pass - return fams - - -_CLS_INC_RE = re.compile(r'#\s*include\s*[<"]class/([^/"<>]+)/([^"<>]+)[">]') - - -@functools.lru_cache(maxsize=None) -def class_include_edges(repo_root: str) -> dict: - """'/
' -> the other class dirs that include it. A class header - pulled in by a second class ships in every firmware enabling that second class: - src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, and - net_device.h includes class/cdc/cdc.h. The class rule derives macros from the - directory name alone, so without this edge a change to the included header - selects only its own class's examples - and on a board that skips those (e.g. - metro_m4_express skips audio_test_freertos), nothing at all. - - Derived from the actual #include lines rather than a hand-written table so it - cannot rot when a class picks up or drops a cross-class include.""" - edges = {} - for f in sorted(glob.glob(os.path.join(repo_root, 'src/class/*/*.[ch]'))): - cls = os.path.basename(os.path.dirname(f)) - try: - text = open(f).read() - except OSError: - continue - for inc_cls, inc_hdr in _CLS_INC_RE.findall(text): - if inc_cls != cls: - edges.setdefault(f'{inc_cls}/{inc_hdr}', set()).add(cls) - return edges - - -def class_macros(cls: str, base: str, prefix: str) -> list: - """Config macros that compile a class dir's code, for role prefix TUD/TUH. - `base` refines dfu only (it splits DFU from DFU_RUNTIME per file); pass '' for - a class reached through an include edge, where the widest set is correct.""" - if cls == 'net': - return [f'CFG_{prefix}_{m}' for m in NET_MACROS] - if cls == 'dfu': - if base.startswith('dfu_rt'): - return [f'CFG_{prefix}_DFU_RUNTIME'] - if base.startswith('dfu_device') or base.startswith('dfu_host'): - return [f'CFG_{prefix}_DFU'] - return [f'CFG_{prefix}_DFU', f'CFG_{prefix}_DFU_RUNTIME'] - return [f'CFG_{prefix}_{cls.upper()}'] - - -def _config_enables(cfg_path: str, macros) -> bool: - try: - text = open(cfg_path).read() - except OSError: - return False - return any(re.search(rf'#define\s+{m}\s+\(?\s*0*[1-9]', text) for m in macros) - - -def roster_only_tests(all_boards) -> set: - """Test paths that only appear in a roster board's tests.only list (e.g. - espressif boards), not in the shared device/dual/host_test lists.""" - out = set() - for b in all_boards: - out.update(b.get('tests', {}).get('only', [])) - return out - - -def class_examples(macros, role: str, repo_root: str, extra_tests: set) -> set: - """Tests (from role's + dual lists, plus roster-only-list tests of that role) - whose example config enables any macro.""" - pool = role_tests({role}, extra_tests) - out = set() - for test in pool: - cfg = os.path.join(repo_root, 'examples', test, 'src', 'tusb_config.h') - if _config_enables(cfg, macros): - out.add(test) - return out - - -def role_tests(roles: set, extras: set) -> set: - """Every test for the given role(s): each role's own list + dual tests, - plus roster-only-list tests (extras) matching those roles or 'dual'.""" - pool = set(dual_tests) - for r in roles: - pool |= set(ALL_TESTS[r]) - pool |= {t for t in extras if test_role(t) in roles or test_role(t) == 'dual'} - return pool - - -class _Sel: - """Accumulates contributions. board->set(tests) plus 'all-board' markers.""" - def __init__(self): - self.full = False - self.by_board = {} # name -> set of tests, or 'all' - self.roles = set() # roles touched by any contribution - self.families = set() # bsp families touched (incl. off-rig ones: build-only consumers) - self.reasons = [] - - def add(self, boards, tests, reason): - """tests: 'all' or iterable of test paths.""" - self.reasons.append(reason) - for b in boards: - cur = self.by_board.get(b) - if tests == 'all' or cur == 'all': - self.by_board[b] = 'all' - else: - self.by_board[b] = (cur or set()) | set(tests) - - def force_full(self, reason): - self.full = True - self.reasons.append(reason) - - -def _classify_one(path, repo_root, roster_boards, extras: set, s: _Sel): - base = os.path.basename(path) - if _NONCODE_RE.match(path): - s.reasons.append(f'{path}: non-code, no contribution') - return - if _FULL_RE.match(path): - s.force_full(f'{path}: core/infra -> full matrix') - return - - m = re.match(r'src/portable/((?:[^/]+/)?[^/]+)/', path) - if m: - port = m.group(1) - if re.match(r'(dcd_|.*_device)', base): - roles = {'device'} - elif re.match(r'(hcd_|.*_host)', base): - roles = {'host'} - else: - roles = {'device', 'host'} - fams = port_families(port, repo_root) - if not fams: - # no family references this port: either a new/renamed port dir or a - # family.cmake layout the scan misses - widen instead of contributing nothing - s.force_full(f'{path}: port {port} maps to no board family -> full matrix') - return - s.families.update(fams) - # a board can also pull the port in through a build option (e.g. MAX3421_HOST=1 - # from the roster on metro_m4_express, or from its own board.cmake), which its - # family file never names - gates = port_option_gates(repo_root).get(port, set()) - boards = [b['name'] for b in roster_boards - if (board_family(b['name'], repo_root) in fams or - (gates and board_options(b, repo_root) & gates)) and (board_roles(b) & roles)] - tests = role_tests(roles, extras) - s.roles.update(roles) - why = f'{path}: port {port} -> families {sorted(fams)}' - if gates: - why += f' + option {sorted(gates)}' - s.add(boards, tests, f'{why} -> boards {boards} ({"/".join(sorted(roles))})') - return - - m = re.match(r'src/class/([^/]+)/', path) - if m: - cls = m.group(1) - if re.search(r'_device\.[ch]$', base): - roles = {'device'} - elif re.search(r'_host\.[ch]$', base): - roles = {'host'} - else: - roles = {'device', 'host'} - # this file's own class, plus any class whose headers include it - via = sorted(class_include_edges(repo_root).get(f'{cls}/{base}', ())) - - def macros(prefix): - return (class_macros(cls, base, prefix) + - [m2 for c in via for m2 in class_macros(c, '', prefix)]) - tests = set() - if 'device' in roles: - tests |= class_examples(macros('TUD'), 'device', repo_root, extras) - if 'host' in roles: - tests |= class_examples(macros('TUH'), 'host', repo_root, extras) - boards = [b['name'] for b in roster_boards if board_roles(b) & roles] - s.roles.update(roles) - why = f'{path}: class {cls}' + (f' (+ included by {via})' if via else '') - s.add(boards, tests, f'{why} -> {sorted(tests)} ({"/".join(sorted(roles))})') - return - - m = re.match(r'src/(device|host)/', path) - if m: - role = m.group(1) - boards = [b['name'] for b in roster_boards if role in board_roles(b)] - s.roles.add(role) - s.add(boards, role_tests({role}, extras), f'{path}: core {role} stack -> all {role} tests') - return - - m = re.match(r'hw/bsp/([^/]+)/(?:boards/([^/]+)/)?', path) - if m: - fam, brd = m.group(1), m.group(2) - s.families.add(fam) - if brd: - boards = [b['name'] for b in roster_boards if b['name'] == brd] - why = f'{path}: bsp board {brd}' - else: - boards = [b['name'] for b in roster_boards - if board_family(b['name'], repo_root) == fam] - why = f'{path}: bsp family {fam}' - s.roles.update(('device', 'host')) - s.add(boards, 'all', f'{why} -> boards {boards}') - return - - m = re.match(r'examples/(device|host|dual)/([^/]+)/', path) - if m: - test = f'{m.group(1)}/{m.group(2)}' - known = any(test in pool for pool in ALL_TESTS.values()) or test in extras - if known: - boards = [b['name'] for b in roster_boards] - role = test_role(test) - s.roles.update(('device', 'host') if role == 'dual' else (role,)) - s.add(boards, [test], f'{path}: example -> {test} on all boards') - else: - s.reasons.append(f'{path}: example not in HIL lists, no contribution') - return - - s.force_full(f'{path}: unclassified -> full matrix') - - -def classify(changed_files, repo_root, rosters): - all_boards = [] - seen = set() - for _, boards in rosters: - for b in boards: - if b['name'] not in seen: - seen.add(b['name']) - all_boards.append(b) - - extras = roster_only_tests(all_boards) - s = _Sel() - # no early exit once full: keep classifying so `families` still reports every - # family the diff touches (build-only consumers need it). Nothing after the first - # force_full can change full/boards/args - the full branch below ignores by_board. - for path in changed_files: - _classify_one(path, repo_root, all_boards, extras, s) - - if s.full: - return {'full': True, 'boards': {b['name']: 'all' for b in all_boards}, - 'families': sorted(s.families), 'reasons': s.reasons} - - # role pruning: single-role selections drop the other role's tests and boards - by_name = {b['name']: b for b in all_boards} - out = {} - for name, tests in s.by_board.items(): - allowed = board_tests(by_name[name]) - if tests == 'all': - kept = list(allowed) - else: - kept = [t for t in allowed if t in tests] - if s.roles and s.roles != {'device', 'host'}: - role = next(iter(s.roles)) - kept = [t for t in kept if test_role(t) in (role, 'dual')] - if kept: - out[name] = 'all' if set(kept) == set(allowed) else sorted(kept) - return {'full': False, 'boards': out, 'families': sorted(s.families), - 'reasons': s.reasons} - - -def _board_args(name, chosen) -> list: - parts = [f'-b {name}'] - if chosen != 'all': - parts.append(f'-bt {name}:{",".join(chosen)}') - return parts - - -def selection_args(sel, rosters): - """hil_test.py args per config. Empty means either 'full matrix' or 'nothing - selected' - callers must read sel['full'] to tell them apart.""" - args = {} - for cfg_path, boards in rosters: - parts = [] - if not sel['full']: - for b in boards: - chosen = sel['boards'].get(b['name']) - if chosen is not None: - parts += _board_args(b['name'], chosen) - args[os.path.basename(cfg_path)] = ' '.join(parts) - return args - - -def selection_args_by_flasher(sel, rosters): - """{config: {flasher name: args}}. CI runs one rig as several jobs split by - flasher (esptool vs the rest); each must gate on its own subset, otherwise the - other leg runs a filter matching zero boards and reports a vacuous green.""" - out = {} - for cfg_path, boards in rosters: - per = {} - if not sel['full']: - for b in boards: - chosen = sel['boards'].get(b['name']) - if chosen is None: - continue - per.setdefault(b.get('flasher', {}).get('name', ''), []).extend( - _board_args(b['name'], chosen)) - out[os.path.basename(cfg_path)] = {f: ' '.join(p) for f, p in per.items()} - return out - - -def changed_files_from_git(base, repo_root): - mb = subprocess.run(['git', 'merge-base', 'HEAD', base], cwd=repo_root, - capture_output=True, text=True, check=True).stdout.strip() - diff = subprocess.run(GIT_DIFF_ARGV + [f'{mb}..HEAD'], cwd=repo_root, - capture_output=True, text=True, check=True).stdout - return [l for l in diff.splitlines() if l.strip()] - - -def main(): - ap = argparse.ArgumentParser(description=__doc__) - g = ap.add_mutually_exclusive_group(required=True) - g.add_argument('--base', help='git ref to diff against (merge-base..HEAD)') - g.add_argument('--diff-file', help='newline-separated changed-file list') - ap.add_argument('configs', nargs='+', help='rig roster JSON file(s)') - a = ap.parse_args() - - repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - rosters = [] - for c in a.configs: - with open(c) as f: - rosters.append((c, json.load(f)['boards'])) - - files = (open(a.diff_file).read().splitlines() if a.diff_file - else changed_files_from_git(a.base, repo_root)) - files = [f for f in files if f.strip()] - - s = classify(files, repo_root, rosters) - s['args'] = selection_args(s, rosters) - s['args_flasher'] = selection_args_by_flasher(s, rosters) - for r in s['reasons']: - print(f'hil_select: {r}', file=sys.stderr) - print(json.dumps(s)) - - -if __name__ == '__main__': - main() diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py index 5d9407883..174251343 100755 --- a/test/hil/hil_test.py +++ b/test/hil/hil_test.py @@ -45,7 +45,10 @@ import os import random import re import select +import signal +import shlex import sys +import tempfile import time from contextlib import redirect_stdout from pathlib import Path @@ -53,30 +56,29 @@ from typing import TypedDict, NotRequired, cast import serial import subprocess +import traceback import json import glob import multiprocessing from multiprocessing import TimeoutError as MpTimeoutError +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # PYTHONSAFEPATH drops it import hil_flash -import hil_lock -from hil_examples import device_tests, dual_tests, host_test +from helper import hil_health, hil_lock, hil_util +from helper.hil_util import device_tests, dual_tests, host_test -# Raw Lock/Semaphore objects passed via Pool initargs are inheritable only under the fork -# start method (spawn/forkserver pickle them and fail at Pool creation) — pin it so a -# future interpreter default change cannot break the run at startup. -_mp = multiprocessing.get_context('fork') +# Raw Lock/Semaphore objects in Pool initargs are inheritable only under fork +# (spawn/forkserver pickle them and fail at Pool creation), so pin it against an +# interpreter default change. Windows has no fork: fall back so it still IMPORTS there. + +_mp = multiprocessing.get_context('fork') if os.name != 'nt' else multiprocessing.get_context() Pool, Lock, Semaphore, Manager = _mp.Pool, _mp.Lock, _mp.Semaphore, _mp.Manager -import hashlib -import ctypes -from pymtp import LIBMTP_DeviceEntry, LIBMTP_RawDevice, MTP import string -# Enumeration wait budget. The first attempt gets ENUM_TIMEOUT; retry attempts get the -# shorter ENUM_TIMEOUT_RETRY - the board was just re-flashed again, and a device that is -# going to enumerate shows up within a few seconds, so a failing test costs ~3-5x a -# passing one instead of 10-30x. Per-attempt value is set by test_example(); each pool -# worker is its own process, so a module global is safe. +# Enumeration wait budget: first attempt ENUM_TIMEOUT, retries the shorter +# ENUM_TIMEOUT_RETRY -- a device that will enumerate shows up within seconds, so a failing +# test costs ~3-5x a passing one instead of 10-30x. Set per attempt by test_example(); a +# module global is safe because each pool worker is its own process. ENUM_TIMEOUT = 8 ENUM_TIMEOUT_RETRY = 4 _enum_timeout = ENUM_TIMEOUT @@ -104,26 +106,36 @@ STATUS_OK = "\033[32mOK\033[0m" STATUS_FAILED = "\033[31mFailed\033[0m" STATUS_SKIPPED = "\033[33mSkipped\033[0m" -# Plain (non-ANSI) cell symbols for the markdown matrix report (hil_report.md). -# A missing binary is reported as skipped too. +# Plain (non-ANSI) cell symbols for hil_report.md; a missing binary counts as skipped. REPORT_CELL = {'pass': '✅', 'fail': '❌', 'skip': '⚪'} class TestFail(AssertionError): """Fail a test but still surface a metric string in its report cell (e.g. usbtest's '❌ 29/30' instead of a bare ❌). The cell metric is icon-prefixed so render/tally treat it as a failure.""" - def __init__(self, msg: str, metric: str | None = None): + def __init__(self, msg: str, metric: str | None = None, parsed: bool = False): super().__init__(msg) self.metric = metric + # parsed=True: a real per-case verdict, so a retry would only re-observe it + # (test_example skips the rest). A failure to RUN the tool stays retryable. + self.parsed = parsed verbose = False +# Set when a HUNG usbtest case could not be recovered: the DUT's usbfs node still has a +# D-state holder, so every later flash on that board enumerates into it, blocks, survives +# SIGKILL and becomes another stray. maxtasksperchild=1 gives each board its own worker, +# so this global is board-scoped; test_board resets it anyway. +board_wedged = '' +max_retry = 1 # mirrors argparse's -r default (see main); defined HERE too so + # test_example is callable (and testable) without going through main() PROFILE = os.environ.get('HIL_PROFILE') == '1' # timestamped logs + permit/flash timing + ctrl-map dump test_only = [] board_test = {} skip_flash = False print_lock = None shuffle_seed = None # per-run seed for the per-board test-order shuffle (HIL_SHUFFLE_SEED to replay) +_current_fw = None # firmware test_example resolved for the RUNNING test (set before each test fn) def init_worker(lock, seed, b_mutexes, f_sems, cmap, cmeta, hints_by_uid): @@ -147,13 +159,21 @@ def log_line(msg: str) -> None: def compact_output(raw: str) -> str: if not raw: return '' - lines = [ln.strip() for ln in raw.replace('\r', '\n').split('\n') if ln.strip()] + # Defense in depth (the emitter already suppresses them, see _ci_log_groups): markers + # piped into this capture land mid-row, where GitHub renders them literally. + lines = [] + for ln in raw.replace('\r', '\n').split('\n'): + ln = hil_util.strip_workflow_markers(ln.strip()).strip() + if ln: + lines.append(ln) return ' | '.join(lines) class FlasherCfg(TypedDict): name: str uid: str - args: str + args: NotRequired[str] # stlink entries carry no args + vid_pid: NotRequired[str] # openocd probe pin, verbatim (e.g. "0x2e8a 0x000c") + verify: NotRequired[bool] # openocd read-back verify opt-out (WCH) class AttachedDevCfg(TypedDict, total=False): @@ -197,9 +217,34 @@ class Board(TypedDict): class HilConfig(TypedDict): boards: list[Board] -POOL_TIMEOUT = int(os.getenv('HIL_POOL_TIMEOUT', '4200')) # usbtest batteries are serialized fleet-wide, lengthening the tail -SERIAL_READ_TIMEOUT = float(os.getenv('HIL_SERIAL_READ_TIMEOUT', '5')) -SERIAL_WRITE_TIMEOUT = float(os.getenv('HIL_SERIAL_WRITE_TIMEOUT', '10')) +# Below the CI job ceilings so THIS guard fires first and still writes a report, well +# above a healthy fleet run (~14 min measured), and deliberately generous: firing early +# abandons boards that were still in flight (30 min fired on 5 of the last 8 HIL jobs), +# while firing late costs minutes on an already-wedged run. The drain keeps whatever had +# already finished either way. +POOL_TIMEOUT = hil_util.pos_int_env('HIL_POOL_TIMEOUT', 3600) + + +# Headroom on top of a battery's own budget so ONE HUNG recovery (case timeout, SIGKILL +# wait, bounded reflash, settle) can finish. Only spent when cases actually time out. +USBTEST_RECOVERY_BUDGET = hil_util.pos_int_env('HIL_USBTEST_RECOVERY_BUDGET', 250) +# How long usbtest.py may keep starting new cases (--budget). The outer run_cmd timeout is +# always this PLUS the recovery headroom, never a separate literal, or lowering one eats +# the reserve the recovery needs. 0 is refused (usbtest.py reads it as "no limit"); the +# margin over a healthy battery (~200s) keeps contention from becoming BUDGET entries. +USBTEST_BATTERY_BUDGET = hil_util.pos_int_env('HIL_USBTEST_BATTERY_BUDGET', 260) + +# The battery checks its budget BEFORE dispatching a case, so it can overshoot by one +# already-started case. Our outer kill must sit ABOVE that or we SIGKILL the battery just +# as it goes to print its JSON, turning ~29 real per-case verdicts into "usbtest did not +# run" and re-paying the whole battery on retry. +# Worst case, from usbtest.py: --timeout 60 (the case) + 5s post-SIGKILL reap + +# dmesg_tail(), which is bounded by HELPER_TIMEOUT=30 and runs on BOTH the FAIL and HUNG +# timeout paths = 95s. 120 leaves a margin; 75 (my first estimate, taken before checking +# dmesg_tail) was 20s SHORT and would have killed the battery mid-print. +USBTEST_OVERSHOOT = 120 +SERIAL_READ_TIMEOUT = hil_util.pos_float_env('HIL_SERIAL_READ_TIMEOUT', 5) +SERIAL_WRITE_TIMEOUT = hil_util.pos_float_env('HIL_SERIAL_WRITE_TIMEOUT', 10) MSC_README_TXT = \ @@ -207,7 +252,6 @@ b"This is tinyusb's MassStorage Class demo.\r\n\r\n\ If you find any bugs or get any questions, feel free to file an\r\n\ issue at github.com/hathach/tinyusb" -# get usb disk by id def get_disk_dev(id, vendor_str, lun): return f'/dev/disk/by-id/usb-{vendor_str}_Mass_Storage_{id}-0:{lun}' @@ -235,8 +279,7 @@ def open_serial_dev(port: str): while timeout > 0: if os.path.exists(port): try: - # write_timeout: a wedged device otherwise blocks ser.write() forever, - # hanging the worker until the pool/job timeout kills the whole run + # write_timeout: see serial_write_all ser = serial.Serial(port, baudrate=115200, timeout=SERIAL_READ_TIMEOUT, write_timeout=SERIAL_WRITE_TIMEOUT) break @@ -252,18 +295,43 @@ def open_serial_dev(port: str): def serial_write_all(ser: serial.Serial, data: bytes): - # write_timeout is a total deadline for the whole call (pyserial keeps partial progress - # internally). A timeout means the device stopped draining — treat it as fatal: pyserial - # loses the partial-write count on raise, so retrying would duplicate bytes on the wire. + # write_timeout is a deadline for the whole call. A timeout means the device stopped + # draining, and it is fatal: pyserial loses the partial-write count on raise, so + # retrying would duplicate bytes on the wire. try: ser.write(data) except serial.SerialTimeoutException: raise AssertionError(f'Serial write timeout after {SERIAL_WRITE_TIMEOUT:.1f}s') +LP_OPEN_TIMEOUT = 5 # bound on opening the printer lp node; see test_device_printer_to_cdc +# Runs under hil_util.run_alongside as `python3 -c`. Inline rather than a file so hil_ci.sh's +# staging list does not need another entry to keep the rig working. +LP_READER = ( + 'import os, sys\n' + 'fd = os.open(sys.argv[1], os.O_RDONLY)\n' + # readiness marker: the parent must not send a byte before the node is open, or the + # bytes are lost. A blind sleep raced CPython start-up on a loaded rig. + 'open(sys.argv[3], "w").close()\n' + 'want = int(sys.argv[2])\n' + 'buf = b""\n' + 'while len(buf) < want:\n' + ' chunk = os.read(fd, min(64, want - len(buf)))\n' + ' if not chunk:\n' + ' break\n' + ' buf += chunk\n' + 'sys.stdout.buffer.write(buf)\n' +) +MTYPE_TIMEOUT = 30 # a README-sized read is <1 s; bounds a D-state hang on a wedged device + + def read_disk_file(uid: str, lun: int, fname: str) -> bytes: - # Reads a file from a FAT volume on a block device without mounting it. - # Requires mtools: `apt install mtools` (no pip dependency). + # Reads a file from an unmounted FAT volume; needs mtools. run_cmd everywhere in this + # file rather than subprocess.run/check_output: its post-timeout reap is an unbounded + # communicate() with no killpg (CPython 3.13.5 subprocess.py:558-565 -- kill(), then + # communicate() with NO timeout), which never returns on a device wedged in D state, + # where the kill is queued and never delivered. binary + # keeps the bytes exact, split_stderr keeps mtype warnings out of them. dev = get_disk_dev(uid, 'TinyUSB', lun) last_err = None @@ -271,101 +339,27 @@ def read_disk_file(uid: str, lun: int, fname: str) -> bytes: nonlocal last_err if not os.path.exists(dev): return None - try: - data = subprocess.check_output( - ['mtype', '-i', dev, f'::/{fname}'], stderr=subprocess.PIPE) - assert data, f'Cannot read file {fname} from {dev}' - return data - except subprocess.CalledProcessError as e: - last_err = e.stderr.decode(errors='replace').strip() - return None + r = hil_util.run_cmd(f"mtype -i {shlex.quote(dev)} ::/{shlex.quote(fname)}", + timeout=MTYPE_TIMEOUT, binary=True, split_stderr=True, quiet=True) + if r.returncode == 0: + if r.stdout: + return r.stdout + # rc 0 with no data is an answer (empty file, zeroed sectors), not "not + # ready" — fail now instead of spinning the budget + raise AssertionError(f'Cannot read file {fname} from {dev}: mtype returned no data') + last_err = (r.stderr or b'').decode(errors='replace').strip() or f'mtype rc {r.returncode}' + return None data = wait_until(try_read) if data is None: - raise AssertionError(f'mtype failed on {dev}: {last_err}' if last_err else f'Storage {dev} not existed') + raise AssertionError(f'Cannot read file {fname} from {dev}: {last_err}' if last_err + else f'Storage {dev} not existed') return data -def open_mtp_dev(uid: str): - mtp = MTP() - last_detail = None - deadline = time.monotonic() + 2 * enum_timeout() - - def find_ready_mtp(): - nonlocal last_detail - for marker_name in glob.glob('/dev/libmtp-*'): - marker = Path(marker_name) - serial = '' - try: - # libmtp-runtime publishes libmtp-%k only after its synchronous - # mtp-probe has accepted the device. Starting from that small, ready-only - # set avoids a broad sysfs scan racing unrelated parallel re-enumerations. - sysname = marker.name[len('libmtp-'):] - dev_path = Path('/sys/bus/usb/devices') / sysname - serial = (dev_path / 'serial').read_text().strip() - if (serial.lower() != uid.lower() - or (dev_path / 'idVendor').read_text().strip() != 'cafe' - or (dev_path / 'idProduct').read_text().strip() != '4017'): - continue - - busnum = int((dev_path / 'busnum').read_text()) - devnum = int((dev_path / 'devnum').read_text()) - usb_node = Path('/dev/bus/usb') / f'{busnum:03d}' / f'{devnum:03d}' - if marker.resolve(strict=True) != usb_node or not os.access( - usb_node, os.R_OK | os.W_OK): - last_detail = f'{marker} did not resolve to an accessible {usb_node}' - continue - return busnum, devnum - except (OSError, ValueError) as e: - # A marker can disappear while another board flashes. Only retain - # diagnostics for this board's marker, not unrelated MTP devices. - if serial.lower() == uid.lower(): - last_detail = f'{marker}: {e}' - return None - - def remaining() -> float: - return max(0.0, deadline - time.monotonic()) - - target = wait_until(find_ready_mtp, step=0.05, timeout=remaining()) - if target is None: - detail = f': {last_detail}' if last_detail else '; install libmtp-runtime' - raise AssertionError(f'MTP udev device not ready for {uid}{detail}') - - # A desktop GVFS session may claim MTP after udev probing. This is a no-op on - # headless runners, but preserves support for rigs where the mount exists. - try: - subprocess.run(['gio', 'mount', '-u', f'mtp://TinyUsb_TinyUsb_Device_{uid}/'], - stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=2) - except (FileNotFoundError, subprocess.TimeoutExpired): - pass - - # GIO can race a disconnect/re-enumeration. Resolve the completed marker again - # rather than opening a stale bus/device tuple. - target = wait_until(find_ready_mtp, step=0.05, timeout=remaining()) - if target is None: - raise AssertionError(f'MTP udev device disappeared for {uid}') - busnum, devnum = target - - # TinyUSB needs no libmtp device quirks. Construct its raw entry directly so - # this test never probes another MTP board that is still being initialized. - entry = LIBMTP_DeviceEntry(None, 0xcafe, None, 0x4017, 0) - raw = LIBMTP_RawDevice(entry, busnum, devnum) - mtp.device = mtp.mtp.LIBMTP_Open_Raw_Device(ctypes.byref(raw)) - if not mtp.device: - raise AssertionError(f'libmtp could not open MTP {uid} at {busnum:03d}/{devnum:03d}') - - try: - serial_raw = mtp.get_serialnumber() - serial = serial_raw.decode('utf-8') if serial_raw else '' - if serial.lower() != uid.lower(): - raise AssertionError(f'MTP serial mismatch at {busnum:03d}/{devnum:03d}: {serial}') - except Exception: - try: - mtp.disconnect() - except Exception: - pass - raise - return mtp +# ~5 KB of transfers plus libmtp setup takes seconds, not minutes; a larger value makes a +# wedged MTP board cost that much on every retry, all charged to the pool guard. +MTP_SESSION_MARGIN = 30 # transfer budget after enumeration; past it the session is killed def get_printer_dev(id: str, vendor_str, product_str, ifnum: int): @@ -374,10 +368,16 @@ def get_printer_dev(id: str, vendor_str, product_str, ifnum: int): product_str = product_str.replace(' ', '_') if product_str else '' for lp in glob.glob('/sys/class/usbmisc/lp*'): try: - sn = open(f'{lp}/device/../serial').read().strip() + # bounded: same device_lock() exposure as the sibling reads (see read_sysfs) + sn = hil_util.read_sysfs(f'{lp}/device/../serial') + # UNKNOWN is not None: the sentinel has no __eq__, so an unanswered read + # would fall through both tests and read as 'not this board' -- the exact + # absence/unknown conflation read_sysfs exists to prevent. + if sn is None or sn is hil_util.SYSFS_UNKNOWN: + continue if sn == id: return f'/dev/usb/{os.path.basename(lp)}' - except (FileNotFoundError, PermissionError, ValueError): + except OSError: # read_sysfs swallows its own OSError/ValueError; glob can race pass return None @@ -389,7 +389,8 @@ def open_printer_dev(id: str, vendor_str, product_str, ifnum: int) -> str: return lp_dev if lp_dev and os.path.exists(lp_dev) else None lp_dev = wait_until(try_find) - assert lp_dev, f'Printer device not found for {id} if{ifnum:02d}' + assert lp_dev, (f'Printer device not found for {id} if{ifnum:02d}' + + hil_util.sysfs_blind_note()) return lp_dev @@ -399,18 +400,16 @@ def open_printer_dev(id: str, vendor_str, product_str, ifnum: int) -> str: def test_dual_host_info_to_device_cdc(board): uid = board['uid'] declared_devs = [f'{d["vid_pid"]}_{d["serial"]}' for d in board['tests']['dev_attached']] - port = hil_flash.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0) + port = hil_util.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0) ser = open_serial_dev(port) ser.timeout = 0.1 - # read until all expected devices are enumerated data = b'' timeout = enum_timeout() while timeout > 0: new_data = ser.read(ser.in_waiting or 1) if new_data: data += new_data - # check if all devices found enum_dev_sn = [] for l in data.decode('utf-8', errors='ignore').splitlines(): vid_pid_sn = re.search(r'ID ([0-9a-fA-F]+):([0-9a-fA-F]+) SN (\w+)', l) @@ -447,7 +446,7 @@ def test_host_device_info(board): flasher = board['flasher'] declared_devs = [f'{d["vid_pid"]}_{d["serial"]}' for d in board['tests']['dev_attached']] - port = hil_flash.get_serial_dev(flasher["uid"], None, None, 0) + port = hil_util.get_serial_dev(flasher["uid"], None, None, 0) ser = open_serial_dev(port) ser.timeout = 0.1 @@ -455,14 +454,12 @@ def test_host_device_info(board): ret = getattr(hil_flash, f'reset_{flasher["name"].lower()}')(board) assert ret.returncode == 0, 'Failed to reset device' - # read until all expected devices are enumerated data = b'' timeout = enum_timeout() while timeout > 0: new_data = ser.read(ser.in_waiting or 1) if new_data: data += new_data - # check if all devices found enum_dev_sn = [] for l in data.decode('utf-8', errors='ignore').splitlines(): vid_pid_sn = re.search(r'ID ([0-9a-fA-F]+):([0-9a-fA-F]+) SN (\w+)', l) @@ -526,7 +523,7 @@ def test_host_cdc_msc_hid(board): if not cdc_devs and not msc_devs: return 'skipped' - port = hil_flash.get_serial_dev(flasher["uid"], None, None, 0) + port = hil_util.get_serial_dev(flasher["uid"], None, None, 0) ser = open_serial_dev(port) ser.timeout = 0.1 @@ -534,7 +531,6 @@ def test_host_cdc_msc_hid(board): ret = getattr(hil_flash, f'reset_{flasher["name"].lower()}')(board) assert ret.returncode == 0, 'Failed to reset device' - # Wait for all expected mount messages data = b'' timeout = enum_timeout() wait_cdc = len(cdc_devs) > 0 @@ -550,7 +546,6 @@ def test_host_cdc_msc_hid(board): time.sleep(0.1) timeout -= 0.1 - # Lookup serial chip name from vid_pid vid_pid_name = { '0403_6001': 'FTDI', '0403_6010': 'FTDI', '0403_6011': 'FTDI', '0403_6014': 'FTDI', '10c4_ea60': 'CP210x', '10c4_ea70': 'CP210x', @@ -561,7 +556,6 @@ def test_host_cdc_msc_hid(board): lines = data.decode('utf-8', errors='ignore').splitlines() - # Verify and print CDC mount if cdc_devs: assert b'CDC Interface is mounted' in data, 'CDC device not mounted on host' dev = cdc_devs[0] @@ -570,7 +564,6 @@ def test_host_cdc_msc_hid(board): if 'CDC Interface is mounted' in l: print(f'\r\n {chip_name}: {l} ', end='') - # Verify and print MSC mount (inquiry + disk size) if msc_devs: assert b'MassStorage device is mounted' in data, 'MSC device not mounted on host' assert b'Disk Size' in data, 'MSC Disk Size not reported' @@ -590,7 +583,6 @@ def test_host_cdc_msc_hid(board): packet_size = 64 - # Echo test: write random 1-packet_size chunks, wait for echo before sending next echo_len = 1024 echo_data = rand_ascii(echo_len) ser.reset_input_buffer() @@ -598,7 +590,6 @@ def test_host_cdc_msc_hid(board): while offset < echo_len: chunk_size = min(random.randint(1, packet_size), echo_len - offset) serial_write_all(ser, echo_data[offset:offset + chunk_size]) - # wait until this chunk is echoed back echo = b'' t_end = time.monotonic() + 1.0 while time.monotonic() < t_end and len(echo) < chunk_size: @@ -619,7 +610,7 @@ def test_host_msc_file_explorer(board): if not msc_devs: return 'skipped' - port = hil_flash.get_serial_dev(flasher["uid"], None, None, 0) + port = hil_util.get_serial_dev(flasher["uid"], None, None, 0) ser = open_serial_dev(port) ser.timeout = 0.1 @@ -627,7 +618,6 @@ def test_host_msc_file_explorer(board): ret = getattr(hil_flash, f'reset_{flasher["name"].lower()}')(board) assert ret.returncode == 0, 'Failed to reset device' - # Wait for MSC mount (Disk Size message) data = b'' timeout = enum_timeout() while timeout > 0: @@ -664,14 +654,12 @@ def test_host_msc_file_explorer(board): if MSC_README_TXT.decode() in resp_text: print('README.TXT matched ', end='') - # MSC throughput test: send dd command to read sectors time.sleep(0.5) ser.reset_input_buffer() for ch in 'dd 1024\r': serial_write_all(ser, ch.encode()) time.sleep(0.002) - # Read dd output until prompt resp = b'' t = 30.0 while t > 0: @@ -706,15 +694,14 @@ def test_host_msc_file_explorer_freertos(board): # Tests: device # ------------------------------------------------------------- def test_device_board_test(board): - # Dummy test pass def test_device_cdc_dual_ports(board): uid = board['uid'] port = [ - hil_flash.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0), - hil_flash.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 2) + hil_util.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0), + hil_util.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 2) ] ser = [open_serial_dev(p) for p in port] @@ -753,7 +740,7 @@ def test_device_cdc_dual_ports(board): def test_device_cdc_msc(board): uid = board['uid'] # CDC Echo test - port = hil_flash.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0) + port = hil_util.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0) ser = open_serial_dev(port) def rand_ascii(length): @@ -782,6 +769,20 @@ def test_device_cdc_msc_freertos(board): test_device_cdc_msc(board) +def link_is_fs(speed) -> bool: + """Payload scaling from a `speed` attribute. Anything not positively read as high speed + counts as FS -- including None and SYSFS_UNKNOWN: the FS payload merely tests an HS + board less, while the HS payload hard-fails a healthy FS board.""" + return speed not in ('480', '5000', '10000') + + +def dd_timeout(mib: float) -> int: + """Bound one dd by what was ASKED for: 2.5 s/MiB is the slowest rate this test has + measured (FS CDC, ~420 kB/s), over a 30 s floor. A flat bound fails a healthy board as + soon as the payload grows or the leaf-hub uplink is shared.""" + return int(30 + 2.5 * mib) + + def test_device_cdc_msc_throughput(board): uid = board['uid'] @@ -792,7 +793,6 @@ def test_device_cdc_msc_throughput(board): return f'{float(m.group(1)):.1f} {m.group(2)}ps' return '?' - # Wait for MSC disk enumeration dev = get_disk_dev(uid, 'TinyUSB', 0) timeout = enum_timeout() while timeout > 0: @@ -801,8 +801,7 @@ def test_device_cdc_msc_throughput(board): time.sleep(0.1); timeout -= 0.1 assert timeout > 0, f'Disk {dev} not found' - # Wait for CDC tty enumeration - tty = hil_flash.get_serial_dev(uid, 'TinyUSB', 'Throughput', 0) + tty = hil_util.get_serial_dev(uid, 'TinyUSB', 'Throughput', 0) timeout = enum_timeout() while timeout > 0: if os.path.exists(tty): @@ -810,41 +809,48 @@ def test_device_cdc_msc_throughput(board): time.sleep(0.1); timeout -= 0.1 assert timeout > 0, f'CDC tty {tty} not found' - # Detect speed (12 Mbps FS / 480 Mbps HS) for payload scaling - is_fs = False - for f in glob.glob('/sys/bus/usb/devices/*/serial'): - try: - if open(f).read().strip().lower() == uid.lower(): - is_fs = (open(os.path.join(os.path.dirname(f), 'speed')).read().strip() == '12') - break - except (OSError, ValueError): - pass + # Detect speed (12 Mbps FS / 480 Mbps HS) for payload scaling; a device we never find + # keeps the FS payload (see link_is_fs) + # usb_scan, not a private glob: it skips root hubs and remembers paths that already + # stranded, so one wedged peer cannot spend this worker's blindness budget four reads + # at a time. + is_fs = True + speed_known = False + devs, _ = hil_util.usb_scan(vid='cafe', serial=uid) + if devs: + speed = hil_util.read_sysfs(os.path.join(devs[0]['dir'], 'speed')) + is_fs = link_is_fs(speed) + speed_known = speed not in (None, hil_util.SYSFS_UNKNOWN) # Put tty in raw mode so dd sees pure binary throughput. - rs = hil_flash.run_cmd(f'timeout 30 stty -F {tty} raw -echo') - assert rs.returncode == 0, f'stty failed: {hil_flash.cmd_stdout_text(rs.stdout)}' + rs = hil_util.run_cmd(f'timeout 30 stty -F {tty} raw -echo') + assert rs.returncode == 0, f'stty failed: {hil_util.cmd_stdout_text(rs.stdout)}' # Payload aim: ~5 s per direction at FS (~830 kB/s), much less at HS. msc_count = 2 if is_fs else 16 # bs=1M cdc_count = 16 if is_fs else 128 # bs=64K tmp_file = f'/tmp/cdc_msc_tp_{uid}.bin' + t_cdc, t_msc = dd_timeout(cdc_count / 16), dd_timeout(msc_count) - rw = hil_flash.run_cmd(f'timeout 30 dd if=/dev/zero of={tty} bs=64K count={cdc_count} 2>&1') - assert rw.returncode == 0, f'CDC dd write failed: {hil_flash.cmd_stdout_text(rw.stdout)}' - cdc_w = parse_speed(hil_flash.cmd_stdout_text(rw.stdout)) + rw = hil_util.run_cmd(f'timeout {t_cdc} dd if=/dev/zero of={tty} bs=64K count={cdc_count} 2>&1') + assert rw.returncode == 0, f'CDC dd write failed: {hil_util.cmd_stdout_text(rw.stdout)}' + cdc_w = parse_speed(hil_util.cmd_stdout_text(rw.stdout)) - rr = hil_flash.run_cmd(f'timeout 30 dd if={tty} of=/dev/null bs=64K count={cdc_count} iflag=fullblock 2>&1') - assert rr.returncode == 0, f'CDC dd read failed: {hil_flash.cmd_stdout_text(rr.stdout)}' - cdc_r = parse_speed(hil_flash.cmd_stdout_text(rr.stdout)) + rr = hil_util.run_cmd(f'timeout {t_cdc} dd if={tty} of=/dev/null bs=64K count={cdc_count} iflag=fullblock 2>&1') + assert rr.returncode == 0, f'CDC dd read failed: {hil_util.cmd_stdout_text(rr.stdout)}' + cdc_r = parse_speed(hil_util.cmd_stdout_text(rr.stdout)) - rmr = hil_flash.run_cmd(f'dd if={dev} of={tmp_file} bs=1M count={msc_count} iflag=direct 2>&1') - assert rmr.returncode == 0, f'MSC dd read failed: {hil_flash.cmd_stdout_text(rmr.stdout)}' - msc_r = parse_speed(hil_flash.cmd_stdout_text(rmr.stdout)) + # inner bound, like the CDC pair above: run_cmd's SIGKILL is merely QUEUED against a + # dd blocked in the block layer on a half-dead device, so without one the call rides + # CMD_TIMEOUT and is abandoned holding the disk and usbfs nodes. + rmr = hil_util.run_cmd(f'timeout {t_msc} dd if={dev} of={tmp_file} bs=1M count={msc_count} iflag=direct 2>&1') + assert rmr.returncode == 0, f'MSC dd read failed: {hil_util.cmd_stdout_text(rmr.stdout)}' + msc_r = parse_speed(hil_util.cmd_stdout_text(rmr.stdout)) - rmw = hil_flash.run_cmd(f'dd if={tmp_file} of={dev} bs=1M count={msc_count} oflag=direct 2>&1') - assert rmw.returncode == 0, f'MSC dd write failed: {hil_flash.cmd_stdout_text(rmw.stdout)}' - msc_w = parse_speed(hil_flash.cmd_stdout_text(rmw.stdout)) + rmw = hil_util.run_cmd(f'timeout {t_msc} dd if={tmp_file} of={dev} bs=1M count={msc_count} oflag=direct 2>&1') + assert rmw.returncode == 0, f'MSC dd write failed: {hil_util.cmd_stdout_text(rmw.stdout)}' + msc_w = parse_speed(hil_util.cmd_stdout_text(rmw.stdout)) try: os.remove(tmp_file) @@ -853,8 +859,7 @@ def test_device_cdc_msc_throughput(board): print(f' CDC read {cdc_r} write {cdc_w}, MSC read {msc_r} write {msc_w} ', end='') - # compact read/write speeds for the report cell, e.g. "✅ C 652/422k M 1.1M/783k" - # (C=CDC, M=MSC; the unit is shown once when both sides share it) + # report cell, e.g. "✅ C 652/422k M 1.1M/783k" (C=CDC, M=MSC; shared unit shown once) def short(s): return (s.split()[0].rstrip('0').rstrip('.') + s.split()[-1][0]) if ' ' in s else s @@ -864,20 +869,29 @@ def test_device_cdc_msc_throughput(board): r = r[:-1] return f'{r}/{w}' - return f'{REPORT_CELL["pass"]} C {pair(cdc_r, cdc_w)} M {pair(msc_r, msc_w)}' + # 'FS?' when the speed could not be read: the numbers below were produced against the FS + # payload, so an HS board reads as suspiciously slow. Say so rather than publish a green + # cell whose scale is a guess. + scale = '' if speed_known else ' FS?' + return f'{REPORT_CELL["pass"]} C {pair(cdc_r, cdc_w)} M {pair(msc_r, msc_w)}{scale}' def test_device_dfu(board): uid = board['uid'] - - # Wait device enum. Deadline-based: dfu-util -l itself takes ~1 s per call, which a - # per-iteration countdown would not charge against the budget. + vid_pid = 'cafe:400b' + + # Deadline-based: dfu-util takes ~1 s per call, which a countdown would not charge + # against the budget. -d pins enumeration to THIS example's ids: a bare `-l` opens every + # DFU-capable node, and one wedged node blocks that open in D state. The pair is doubled + # because dfu-util matches run-time and DFU-mode devices against SEPARATE id pairs + # (parse_vendprod: an omitted DFU-mode pair matches ANY DFU-mode device). The deadline + # is only tested BETWEEN calls, so the per-call bound is what caps a blocked open. deadline = time.monotonic() + enum_timeout() found = False while time.monotonic() < deadline: - ret = hil_flash.run_cmd(f'dfu-util -l') - stdout = hil_flash.cmd_stdout_text(ret.stdout) - if f'serial="{uid}"' in stdout and 'Found DFU: [cafe:400b]' in stdout: + ret = hil_util.run_cmd(f'dfu-util -d {vid_pid},{vid_pid} -l', timeout=15) + stdout = hil_util.cmd_stdout_text(ret.stdout) + if f'serial="{uid}"' in stdout and f'Found DFU: [{vid_pid}]' in stdout: found = True break time.sleep(1) @@ -887,17 +901,23 @@ def test_device_dfu(board): f_dfu0 = f'dfu0_{uid}' f_dfu1 = f'dfu1_{uid}' - # Test upload try: os.remove(f_dfu0) os.remove(f_dfu1) except OSError: pass - ret = hil_flash.run_cmd(f'dfu-util -S {uid} -a 0 -U {f_dfu0}') + # -d as well as -S: dfu-util matches the SERIAL only after libusb_open() (dfu_util.c + # probes the descriptor for iSerialNumber), so -S alone still opens every DFU-capable + # node. The id filter runs BEFORE the open; -S then picks our board (see the poll). + # Each partition is one short string, so a healthy upload is ~1 s; the bound is there + # for a node that stops answering mid-transfer. + ret = hil_util.run_cmd(f'dfu-util -d {vid_pid},{vid_pid} -S {uid} -a 0 -U {f_dfu0}', + timeout=30) assert ret.returncode == 0, 'Upload failed' - ret = hil_flash.run_cmd(f'dfu-util -S {uid} -a 1 -U {f_dfu1}') + ret = hil_util.run_cmd(f'dfu-util -d {vid_pid},{vid_pid} -S {uid} -a 1 -U {f_dfu1}', + timeout=30) assert ret.returncode == 0, 'Upload failed' with open(f_dfu0) as f: @@ -912,13 +932,14 @@ def test_device_dfu(board): def test_device_dfu_runtime(board): uid = board['uid'] - # Wait device enum (deadline-based, see test_device_dfu) + vid_pid = 'cafe:400c' + # enumeration pinned to this example's ids, same per-call bound (see test_device_dfu) deadline = time.monotonic() + enum_timeout() found = False while time.monotonic() < deadline: - ret = hil_flash.run_cmd(f'dfu-util -l') - stdout = hil_flash.cmd_stdout_text(ret.stdout) - if f'serial="{uid}"' in stdout and 'Found Runtime: [cafe:400c]' in stdout: + ret = hil_util.run_cmd(f'dfu-util -d {vid_pid},{vid_pid} -l', timeout=15) + stdout = hil_util.cmd_stdout_text(ret.stdout) + if f'serial="{uid}"' in stdout and f'Found Runtime: [{vid_pid}]' in stdout: found = True break time.sleep(1) @@ -931,7 +952,6 @@ def test_device_hid_boot_interface(board): kbd = get_hid_dev(uid, 'TinyUSB', 'TinyUSB_Device', 'event-kbd') mouse1 = get_hid_dev(uid, 'TinyUSB', 'TinyUSB_Device', 'if01-event-mouse') mouse2 = get_hid_dev(uid, 'TinyUSB', 'TinyUSB_Device', 'if01-mouse') - # Wait device enum timeout = enum_timeout() while timeout > 0: if os.path.exists(kbd) and os.path.exists(mouse1) and os.path.exists(mouse2): @@ -948,12 +968,9 @@ def test_device_hid_composite_freertos(id): def test_device_printer_to_cdc(board): - import threading - uid = board['uid'] - # Wait for CDC port and printer device - cdc_port = hil_flash.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0) + cdc_port = hil_util.get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0) ser = open_serial_dev(cdc_port) lp_dev = open_printer_dev(uid, 'TinyUSB', 'TinyUSB_Device', 2) @@ -973,7 +990,6 @@ def test_device_printer_to_cdc(board): sizes = [32, 64, 128, 256, 512, random.randint(2000, 5000)] - # flush any stale data ser.reset_input_buffer() # Test 1: Printer -> CDC with multiple sizes, write in random 1-64 byte chunks @@ -983,7 +999,17 @@ def test_device_printer_to_cdc(board): ser.reset_input_buffer() rd = b'' offset = 0 - lp_fd = os.open(lp_dev, os.O_WRONLY | os.O_NONBLOCK) + # bounded: O_NONBLOCK does NOT save us -- usblp_open() takes the device mutex + # first -- and this open runs on the worker itself, with no thread to abandon + lp_fd = hil_util.bounded_open(lp_dev, os.O_WRONLY | os.O_NONBLOCK, 5) + # Three-valued on purpose: an OSError here is a FACT about the node (EBUSY from + # usblp's single-opener rule, ENOENT from a re-enumeration race, EACCES from a + # udev gap) and must not be reported as a wedge -- that sends the operator to + # usb-kernel-recover for hardware that is fine. + assert lp_fd is not hil_util.SYSFS_UNKNOWN, ( + f'printer: opening {lp_dev} for write blocked (device wedged)' + f'{hil_util.sysfs_blind_note()}') + assert lp_fd is not None, f'printer: {lp_dev} could not be opened for write' try: while offset < size: chunk_size = min(random.randint(1, 64), size - offset) @@ -1007,128 +1033,88 @@ def test_device_printer_to_cdc(board): assert rd == test_data, (f'Printer->CDC wrong data ({size} bytes):\n' f' expected: {test_data[:64]}\n received: {rd[:64]}') - # Test 2: CDC -> Printer with multiple sizes, write in random 1-64 byte chunks - # Use a thread to read from printer since /dev/usb/lp read blocks + # Test 2: CDC -> Printer with multiple sizes, write in random 1-64 byte chunks. + # The lp read runs in a PROCESS, not a thread: /dev/usb/lp* blocks on read, usblp + # allows a SINGLE opener, and a blocked thread cannot be abandoned without keeping + # that fd -- which poisoned the node for every later test this worker ran. A killed + # process takes its fd with it. ser.reset_input_buffer() time.sleep(0.5) for size in sizes: test_data = rand_ascii(size) - rd_result = [b'', None] # [data, error] - reader_ready = threading.Event() - - def lp_reader(): - try: - rd = b'' - fd = os.open(lp_dev, os.O_RDONLY) - reader_ready.set() - try: - while len(rd) < size: - chunk = os.read(fd, min(64, size - len(rd))) - if not chunk: - break - rd += chunk - finally: - os.close(fd) - rd_result[0] = rd - except Exception as e: - rd_result[1] = e - reader_ready.set() - reader = threading.Thread(target=lp_reader, daemon=True) - reader.start() - # wait for reader to open lp device before writing - reader_ready.wait(timeout=5) - time.sleep(0.1) - - # Write to CDC in small chunks with flush to avoid overflowing device FIFO - offset = 0 - while offset < size: - chunk_size = min(random.randint(1, 64), size - offset) - serial_write_all(ser, test_data[offset:offset + chunk_size]) - time.sleep(0.01) - offset += chunk_size + ready = Path(tempfile.gettempdir()) / f'hil-lp-ready-{os.getpid()}-{size}' + ready.unlink(missing_ok=True) + + def write_cdc(): + # WAIT for the reader to have the node open. The child has to fork, exec and + # boot a CPython interpreter; on a loaded rig that routinely exceeds the 0.3s + # this used to sleep, and every byte sent early is lost -- surfacing as a + # spurious data mismatch rather than a timeout. + deadline = time.monotonic() + LP_OPEN_TIMEOUT + 5 + while not ready.exists(): + if time.monotonic() > deadline: + return # reader never opened; the rc/compare below reports it + time.sleep(0.02) + offset = 0 + while offset < size: + chunk_size = min(random.randint(1, 64), size - offset) + serial_write_all(ser, test_data[offset:offset + chunk_size]) + time.sleep(0.01) + offset += chunk_size - reader.join(timeout=10) - assert not reader.is_alive(), f'CDC->Printer timeout ({size} bytes)' - assert rd_result[1] is None, f'CDC->Printer read error: {rd_result[1]}' - assert rd_result[0] == test_data, (f'CDC->Printer wrong data ({size} bytes):\n' - f' expected: {test_data[:64]}\n received: {rd_result[0][:64]}') + try: + r = hil_util.run_alongside( + [sys.executable, '-c', LP_READER, lp_dev, str(size), str(ready)], + write_cdc, LP_OPEN_TIMEOUT + 12) + finally: + ready.unlink(missing_ok=True) + # stderr, not stdout: run_alongside keeps the payload stream clean, so a traceback + # from the reader now arrives on its own pipe + assert r.returncode == 0, (f'CDC->Printer reader failed ({size} bytes, rc ' + f'{r.returncode}): {hil_util.cmd_stdout_text(r.stderr)[:200]}') + assert r.stdout == test_data, (f'CDC->Printer wrong data ({size} bytes):\n' + f' expected: {test_data[:64]}\n received: {r.stdout[:64]}') time.sleep(0.2) ser.close() def test_device_mtp(board): + # The whole session lives in mtp_test.py under run_cmd: libmtp calls are synchronous + # ctypes that block unkillably (D state) on a wedged device, so a disposable process is + # the only thing the harness can walk away from. uid = board['uid'] - - # --- BEFORE: mute C-level stderr for libmtp vid/pid warnings --- - fd = sys.stderr.fileno() - _saved = os.dup(fd) - _null = os.open(os.devnull, os.O_WRONLY) - os.dup2(_null, fd) - - try: - mtp = open_mtp_dev(uid) - finally: - # --- AFTER: restore stderr --- - os.dup2(_saved, fd) - os.close(_null) - os.close(_saved) - - try: - assert b"TinyUSB" == mtp.get_manufacturer(), 'MTP wrong manufacturer' - assert b"MTP Example" == mtp.get_modelname(), 'MTP wrong model' - assert b'1.0' == mtp.get_deviceversion(), 'MTP wrong version' - assert b'TinyUSB MTP' == mtp.get_devicename(), 'MTP wrong device name' - - # read and compare readme.txt and logo.png - f1_expect = b'TinyUSB MTP Filesystem example' - f2_md5_expect = '40ef23fc2891018d41a05d4a0d5f822f' # md5sum of logo.png - f1 = uid.encode("utf-8") + b'_file1' - f2 = uid.encode("utf-8") + b'_file2' - f3 = uid.encode("utf-8") + b'_file3' - mtp.get_file_to_file(1, f1) - with open(f1, 'rb') as file: - f1_data = file.read() - os.remove(f1) - assert f1_data == f1_expect, 'MTP file1 wrong data' - mtp.get_file_to_file(2, f2) - with open(f2, 'rb') as file: - f2_data = file.read() - os.remove(f2) - assert f2_md5_expect == hashlib.md5(f2_data).hexdigest(), 'MTP file2 wrong data' - # test send file - with open(f3, "wb") as file: - # 1524-byte payload + 12-byte MTP header = 3 full 512-byte buffers. - # This exercises delivery of the final OUT payload before its ZLP. - f3_data = bytes((i % 251) + 1 for i in range(1524)) - file.write(f3_data) - file.close() - fid = mtp.send_file_from_file(f3, b'file3') - f3_readback = f3 + b'_readback' - mtp.get_file_to_file(fid, f3_readback) - with open(f3_readback, 'rb') as f: - f3_rb_data = f.read() - os.remove(f3_readback) - assert f3_rb_data == f3_data, 'MTP file3 wrong data' - os.remove(f3) - mtp.delete_object(fid) - finally: - mtp.disconnect() + script = Path(__file__).resolve().parent / 'mtp_test.py' + # 2x, as master's in-process open_mtp_dev used: libmtp-runtime publishes + # /dev/libmtp-* only after its SYNCHRONOUS mtp-probe finishes, seconds on a freshly + # flashed FS board, and the gio unmount eats part of what is left before the first + # probe. Extracting the session into a subprocess halved this by accident (8s/4s), + # which fails healthy hardware on the retry. + t = 2 * enum_timeout() + r = hil_util.run_cmd( + f'{shlex.quote(sys.executable)} {shlex.quote(str(script))} --uid {shlex.quote(uid)} --timeout {t}', + timeout=t + MTP_SESSION_MARGIN) + if r.returncode == 124: + # "abandoned", not "killed": a session blocked in a usbfs ioctl (D state) never + # receives the SIGKILL -- it lingers until its device path clears, by design + raise AssertionError(f'MTP session wedged (abandoned after {t + MTP_SESSION_MARGIN}s; ' + f'the session process may linger unkillable in D state)') + assert r.returncode == 0, f'MTP session failed (rc {r.returncode}):\n{r.stdout}' def test_device_net_lwip_webserver(board): # MAC hard-coded in examples/device/net_lwip_webserver/src/main.c; Linux names the - # USB network interface enx. Device IP is 192.168.7.1 and - # the example runs an iperf2 TCP server on port 5001 (INCLUDE_IPERF). + # iface enx. Device IP 192.168.7.1, iperf2 TCP server on 5001 + # (INCLUDE_IPERF). import socket mac_no_colons = '0202846a9600' iface = 'enx' + mac_no_colons device_ip = '192.168.7.1' iperf_port = 5001 - # Wait for the host to get an IPv4 address in the device's subnet (DHCP served by the device). - # USB enum + DHCP serve can take longer on the CI HIL hardware than on local — give it 30s. + # Wait for an IPv4 address in the device's subnet (it serves DHCP); 30s because USB + # enum + DHCP serve is slower on the CI HIL hardware than locally. iface_timeout = 30 deadline = time.monotonic() + iface_timeout host_ip = None @@ -1142,8 +1128,7 @@ def test_device_net_lwip_webserver(board): time.sleep(0.5) assert host_ip, f'USB net iface {iface} did not come up with 192.168.7.x within {iface_timeout}s' - # Poll the iperf TCP port until the device is accepting. The net stack comes up a bit - # after DHCP completes; iperf server binding isn't instantaneous after reflash. + # Poll until the device accepts: the net stack and the iperf bind come up after DHCP. deadline = time.monotonic() + enum_timeout() last_err = None while time.monotonic() < deadline: @@ -1156,12 +1141,12 @@ def test_device_net_lwip_webserver(board): time.sleep(0.3) assert last_err is None, f'iperf TCP {device_ip}:{iperf_port} not accepting within {enum_timeout()}s: {last_err}' - # Throughput: 5-second iperf2 TCP test, CSV output for stable parsing. - # iperf2 CSV final summary line: timestamp,src_ip,src_port,dst_ip,dst_port,id,interval,bytes,bps - ret = subprocess.run(['iperf', '-c', device_ip, '-t', '5', '-y', 'C'], - capture_output=True, text=True, timeout=30) - stderr = ret.stderr.strip() - stdout = ret.stdout.strip() + # 5-second iperf2 TCP test; -y C for stable parsing (final summary line is + # timestamp,src_ip,src_port,dst_ip,dst_port,id,interval,bytes,bps). + ret = hil_util.run_cmd(f'iperf -c {device_ip} -t 5 -y C', + timeout=30, split_stderr=True, quiet=True) + stderr = (ret.stderr or '').strip() + stdout = (ret.stdout or '').strip() assert ret.returncode == 0, f'iperf rc={ret.returncode}: stderr={stderr!r} stdout={stdout!r}' lines = [l for l in stdout.splitlines() if l] assert lines, f'iperf produced no output (rc={ret.returncode}, stderr={stderr!r})' @@ -1172,19 +1157,16 @@ def test_device_net_lwip_webserver(board): mbps = bps / 1e6 print(f' iperf {mbps:5.1f} Mbps', end='') - # Reject implausibly low throughput - a working USB-net link should clear this easily. assert mbps >= 1.0, f'iperf throughput too low: {mbps:.2f} Mbps' def test_device_msc_dual_lun(board): uid = board['uid'] - # Read README from LUN 0 data0 = read_disk_file(uid, 0, 'README0.TXT') readme0 = b"LUN0: " + MSC_README_TXT assert data0 == readme0, f'MSC LUN0 wrong data in README0.TXT\n expected: {readme0}\n received: {data0}' - # Read README from LUN 1 data1 = read_disk_file(uid, 1, 'README1.TXT') readme1 = b"LUN1: " + MSC_README_TXT assert data1 == readme1, f'MSC LUN1 wrong data in README1.TXT\n expected: {readme1}\n received: {data1}' @@ -1193,7 +1175,6 @@ def test_device_msc_dual_lun(board): def test_device_midi_test(board): uid = board['uid'] - # Find MIDI device via /dev/snd/by-id using board UID timeout = enum_timeout() midi_port = None while timeout > 0: @@ -1211,7 +1192,6 @@ def test_device_midi_test(board): timeout -= 1 assert midi_port is not None, f'MIDI device not found for {uid}' - # Read MIDI messages and verify note on/off import select midi_fd = os.open(midi_port, os.O_RDONLY | os.O_NONBLOCK) try: @@ -1246,7 +1226,6 @@ def test_device_midi_test(board): i += 1 assert len(notes) >= 2, f'Expected at least 2 MIDI notes, got {len(notes)}' - # Verify notes are from the expected sequence note_sequence = [ 74, 78, 81, 86, 90, 93, 98, 102, 57, 61, 66, 69, 73, 78, 81, 85, 88, 92, 97, 100, 97, 92, 88, 85, 81, 78, 74, 69, 66, 62, 57, 62, @@ -1287,8 +1266,11 @@ def test_device_audio_test_freertos(board): raw_path, ] - ret = subprocess.run(cmd, capture_output=True, text=True, timeout=20) - assert ret.returncode == 0, f'arecord failed: {ret.stderr.strip() or ret.stdout.strip()}' + # run_cmd: ALSA capture from a wedged device blocks in D state (see read_disk_file) + ret = hil_util.run_cmd(' '.join(shlex.quote(c) for c in cmd), + timeout=20, split_stderr=True, quiet=True) + assert ret.returncode == 0, \ + f'arecord failed: {(ret.stderr or "").strip() or (ret.stdout or "").strip()}' try: with open(raw_path, 'rb') as f: @@ -1322,7 +1304,6 @@ def test_device_hid_generic_inout(board): uid = board['uid'] import hid # cython-hidapi (pip: hidapi, apt: python3-hid) - # Find HID device by UID (VID=0xCafe) timeout = enum_timeout() dev = None while timeout > 0: @@ -1339,7 +1320,6 @@ def test_device_hid_generic_inout(board): h = hid.device() h.open(dev['vendor_id'], dev['product_id'], uid) try: - # Echo test: send random data and verify echo for size in [8, 32, 63]: # Report ID (0) + payload, padded to 64 bytes payload = bytes([random.randint(1, 255) for _ in range(size)]) @@ -1356,63 +1336,177 @@ def test_device_hid_generic_inout(board): def test_device_usbtest(board): - # Run the Linux testusb tier-4 battery (test/hil/usbtest.py) against the enumerated cafe:4010 - # device; surface the pass count in the report cell ("✅ 30/30", or "❌ 29/30" on a partial). + global board_wedged + # Runs test/hil/usbtest.py against the cafe:4010 device; the pass count goes in the + # report cell ("✅ 30/30", or "❌ 29/30" on a partial). uid = board['uid'] def usbtest_enumerated(): - # match VID:PID too, not just the serial: right after flashing, the previous example's - # enumeration (same serial, different PID) can linger and would fail usbtest.py's lookup - for f in glob.glob('/sys/bus/usb/devices/*/serial'): - d = os.path.dirname(f) - try: - if (open(f).read().strip().lower() == uid.lower() - and open(os.path.join(d, 'idVendor')).read().strip() == 'cafe' - and open(os.path.join(d, 'idProduct')).read().strip() == '4010'): - return True - except OSError: - pass - return False + """True, False, or None when a bounded read did not answer -- absence unproven.""" + # vid_pid FIRST: right after flashing, the previous example's enumeration (same + # serial, different PID) can linger and would fail usbtest.py's lookup -- and + # filtering on the two lock-free descriptor fields rules out every other device + # on the bus before the one read that can block. usb_scan memoises paths that + # already stranded, so one wedged peer cannot spend the blindness budget here. + devs, unknown = hil_util.usb_scan(vid_pid=('cafe', '4010'), serial=uid) + if devs: + return True + return None if unknown else False end = time.monotonic() + enum_timeout() - while time.monotonic() < end and not usbtest_enumerated(): + seen = usbtest_enumerated() + while time.monotonic() < end and seen is not True: time.sleep(0.2) + seen = usbtest_enumerated() # fail before usbtest_permit: an absent device would otherwise queue on the battery # mutex for minutes behind real batteries just to have usbtest.py report "no device" - if not usbtest_enumerated(): + if seen is not True: # 0/30 rather than a bare cell: the battery never ran (30 = standard case count) - raise TestFail(f'no cafe:4010 device with serial {uid}', - metric=f'{REPORT_CELL["fail"]} 0/30') - # settle: right after flashing the enumeration can bounce once (and on dual-port parts like - # CH32V307 the other port's stale usbtest node — same serial and PID — lingers a moment); - # running testusb into that gap sees the device drop mid-case + raise TestFail( + f'no cafe:4010 device with serial {uid}' if seen is False else + f'cannot tell whether cafe:4010 {uid} is present: the bounded sysfs reads did ' + f'not answer{hil_util.sysfs_blind_note()}', + metric=f'{REPORT_CELL["fail"]} 0/30') + # settle: right after flashing the enumeration can bounce once (and on dual-port parts + # the other port's stale node — same serial and PID — lingers), and testusb run into + # that gap sees the device drop mid-case time.sleep(3) # --keep-binding is required for concurrent batteries: usbtest.py's cleanup unbinds - # EVERY usbtest-bound interface (releasing stale same-PID grabs), which would kill a - # peer battery mid-run under USBTEST_PARALLEL > 1; the unbind path has also wedged a - # host xHCI (usb_hcd_alloc_bandwidth) on this rig. Leaving bindings is harmless with - # unique example PIDs - the next example re-enumerates under a different PID and binds - # its normal driver. usbtest_permit budgets USBTEST_PARALLEL batteries per controller. + # EVERY usbtest-bound interface, killing a peer battery under USBTEST_PARALLEL > 1, and + # that unbind path has also wedged a host xHCI (usb_hcd_alloc_bandwidth) here. Harmless + # to leave: the next example enumerates under a different PID. script = Path(__file__).resolve().parent / 'usbtest.py' - cmd = f'python3 "{script}" --serial "{uid}" --json --keep-binding --timeout 60' + # --budget makes the battery a real bound: repeated case timeouts (a FAIL, not a HUNG, + # so the battery keeps going) can otherwise spend the whole outer timeout inside the + # case loop, leaving the recovery below nothing. + cmd = (f'{shlex.quote(sys.executable)} {shlex.quote(str(script))} ' + f'--serial {shlex.quote(uid)} --json --keep-binding ' + f'--timeout 60 --budget {USBTEST_BATTERY_BUDGET}') + # Post-hang recovery reflashes the DUT through its own probe, NEVER a root-port cycle + # (one board reached instead of every fixture under the port; see usb-kernel-recover). + # _current_fw is the artifact test_example flashed for THIS test: re-deriving it from + # board['name'] reflashes the wrong build on variant-only boards. --outer-timeout lets + # usbtest skip a reflash it cannot finish before our run_cmd kill, which would orphan + # the flasher (own session) on the probe. Never under --skip-flash -- and say so: a + # HUNG case then holds the DUT's usbfs lock for the rest of the run, and a probe reset + # is no substitute (the DWC2 pullup survives a core halt). + # ...and only when this flasher can DELIVER that reflash past a poisoned node + # (hil_flash.convoy_safe). Otherwise the flags cost twice: the delivery adds a SECOND + # stray, and the board reserves recovery budget for a path that cannot fire. + # The RECOVERY flasher, which may be the roster's optional `flasher_recover` rather + # than the primary -- a jlink/stlink board can name an openocd entry that reaches the + # same probe convoy-safely without changing how the board is normally flashed. + _rec_flasher = hil_flash.recover_flasher(board) + recovery = bool(_current_fw and not skip_flash and hil_flash.convoy_safe(_rec_flasher)) + # ONE bound, computed here and used for BOTH the child's --outer-timeout and our own + # run_cmd kill below. Three separate expressions disagreed: --skip-flash appended no + # --outer-timeout at all (usbtest reads 0 as "no limit"), and the no-recovery branch + # narrowed only the CHILD's view while run_cmd still waited the full reserve -- so a + # board that cannot recover held a pool worker AND its battery permit idle for + # USBTEST_RECOVERY_BUDGET it had no way to spend, under a usbtest width of 2. + outer = USBTEST_BATTERY_BUDGET + (USBTEST_RECOVERY_BUDGET if recovery + else USBTEST_OVERSHOOT) + if _current_fw and skip_flash: + print('note: --skip-flash disables usbtest hang recovery; a HUNG case will leave ' + 'the device wedged until it is reflashed', flush=True) + elif _current_fw and not recovery: + print(f'note: {_rec_flasher["name"]} cannot deliver a reflash past a poisoned ' + f'usbfs node, so usbtest hang recovery is disabled for {board["name"]}; a ' + f'HUNG case will leave it wedged for the rest of the run', flush=True) + if recovery: + # ship the RECOVERY flasher as `flasher`: usbtest.py, recovery_steps and + # convoy_safe all read board['flasher'], so substituting here keeps the entire + # child side unaware that a second roster entry exists + rb = json.dumps({'name': board['name'], 'flasher': _rec_flasher}) + cmd += f' --recover-board {shlex.quote(rb)} --recover-fw {shlex.quote(_current_fw)}' + cmd += f' --outer-timeout {outer}' + # The reserve above USBTEST_BATTERY_BUDGET exists because the battery can overrun by + # one already-started case, and a hang there needs room for the recovery (whose reflash + # is bounded by usbtest.RECOVER_FLASH_TIMEOUT, not HIL_CMD_TIMEOUT). Without it run_cmd + # SIGKILLs usbtest.py mid-recovery, losing the JSON and the diagnosis. with hil_lock.usbtest_permit(uid): - r = hil_flash.run_cmd(cmd, timeout=200) - out = hil_flash.cmd_stdout_text(r.stdout) + # split_stderr: the battery's final JSON is parsed from stdout, and stderr is the + # only detail left when the outer timeout kills the battery before it prints + r = hil_util.run_cmd(cmd, timeout=outer, split_stderr=True) + out = hil_util.cmd_stdout_text(r.stdout) brace = out.find('{') try: + # brace < 0 would slice from the END ('...rc 0' -> '0' -> int 0, whose subscript + # raises TypeError outside the tuple below and loses the diagnosis) + if brace < 0: + raise ValueError('no JSON object on stdout') data = json.loads(out[brace:]) passed, failed = int(data['passed']), int(data['failed']) - except (ValueError, KeyError, json.JSONDecodeError): - raise TestFail(f'usbtest did not run: {compact_output(out) or hil_flash.cmd_stdout_text(r.stderr)}', + except (ValueError, KeyError, TypeError, json.JSONDecodeError): + # compact BOTH, never `or`: a battery SIGKILLed mid-print leaves a truthy JSON + # fragment on stdout, so an `or` drops the stderr that explains the failure + parts = [compact_output(hil_util.cmd_stdout_text(r.stderr)), compact_output(out)] + detail = ' | '.join(p for p in parts if p) + # Retryable even on rc 124 (run_cmd's outer kill), though the retry re-pays the + # whole budget: 124 only says the timer expired, which a healthy battery can hit + # under load, and test_example REFLASHES before each attempt. Where usbtest's + # in-band recovery is off (--skip-flash, a flasher failing convoy_safe, a terminal + # wedge) that reflash is the only thing left to unpoison the DUT for the boards + # that share its controller. + # No JSON to read the verdict from, so fall back to the text: a battery SIGKILLed + # mid-hang still says HUNG on stdout, and this raise happens BEFORE the latch below + # -- which is why the outer-timeout case, the likeliest real wedge, never latched. + if 'HUNG' in out: + board_wedged = (f'{board["name"]}: usbtest reported a hang and was killed ' + f'before it could report a verdict') + raise TestFail(f'usbtest did not run: {detail}', metric=f'{REPORT_CELL["fail"]} 0/30') - total = passed + failed - if failed == 0 and total > 0: + # A HUNG case that recovery could not clear leaves a D-state holder on this board's + # usbfs node. Latch it: the remaining examples would each flash THROUGH that node, + # block, survive SIGKILL and add another stray -- turning one wedge into one stray per + # remaining example, which is the convoy this branch exists to contain. + # The battery's OWN verdict first: `recovery` only says the flags were passed, not that + # the reflash worked, so a convoy-safe board whose recovery failed used to come back + # unlatched and flash every remaining example through the poisoned node. + if data.get('wedged') or (not recovery and 'HUNG' in out): + # _rec_flasher, NOT board['flasher']: recovery was decided against recover_flasher() + # at the top of this function, and the two diverge as soon as a roster carries the + # optional `flasher_recover` key -- naming the wrong one sends the operator to the + # wrong probe. The wording stays on what usbtest actually reported ("still wedged"), + # because unrecovered_hang is also set by the ambiguous/inconclusive aborts, where + # nothing hung and the old text was false on both clauses. + board_wedged = (f'{board["name"]}: usbtest reports the device still wedged ' + + (f'after a recovery reflash via {_rec_flasher["name"]}' if recovery + else f'and {_rec_flasher["name"]} cannot deliver a recovery reflash')) + + # notrun counts toward the denominator but is NOT a failure: listing cases that never + # ran as failures sends a maintainer bisecting one of them. + notrun = int(data.get('notrun', 0)) + total = passed + failed + notrun + if board_wedged and failed == 0 and notrun == 0: + # Every case passed and the device STILL wedged -- usbtest's inconclusive/ambiguous + # abort fires after the last case, so nothing back-fills a BUDGET entry. Reporting + # the pass would exit 0 with a D-state holder on the rig and the board absent from + # the re-run spec. parsed=True: a retry re-pays the whole battery to re-observe a + # wedge, and flashes through the poisoned node to do it. + raise TestFail(f'usbtest {passed}/{total} but the device wedged ({board_wedged})', + metric=f'{REPORT_CELL["fail"]} {passed}/{total}', parsed=True) + if failed == 0 and notrun == 0 and total > 0: return f'{REPORT_CELL["pass"]} {passed}/{total}' - bad = [c.get('num') for c in data.get('cases', []) if c.get('status') != 'PASS'] - raise TestFail(f'usbtest {passed}/{total} (cases failed: {bad})', - metric=f'{REPORT_CELL["fail"]} {passed}/{total}') + bad = [c.get('num') for c in data.get('cases', []) + if c.get('status') not in ('PASS', 'BUDGET')] + why = f'usbtest {passed}/{total}' + if bad: + why += f' (cases failed: {bad})' + if notrun: + # the reason is per BUDGET entry: a hang or a device drop also aborts the battery, + # and blaming the budget points the maintainer at the wrong thing + reasons = {c.get('detail', '') for c in data.get('cases', []) + if c.get('status') == 'BUDGET'} + reason = (reasons.pop().replace('not run: ', '') if len(reasons) == 1 + else 'the battery stopped early') + why += f'; {notrun} case(s) never ran ({reason}), so this says nothing about them' + # parsed ONLY when every case ran: an aborted battery (budget expiry, kernel hang, bus + # drop) leaves BUDGET entries, and those are exactly what a reflash retry can fix. + raise TestFail(why, metric=f'{REPORT_CELL["fail"]} {passed}/{total}', + parsed=(notrun == 0)) # ------------------------------------------------------------- @@ -1437,42 +1531,68 @@ def test_example(board: Board, variant: str, example: str) -> tuple[int, str, st test_name = f'{variant:40} {example:30} ...' - # --skip-flash runs whatever is already on the board, so any build counts as present: - # only the flashing path needs the artifact this board's flasher actually consumes. - # Filtering there too would skip the test as "no binary" over an extension it never uses. + # --skip-flash runs whatever is already on the board, so any build counts as present; + # filtering by flasher there would skip the test over an extension it never uses. fw_name = hil_flash.find_firmware(variant, example, flasher=None if skip_flash else board['flasher']['name']) if fw_name is None: log_line(f'{test_name} Skip (no binary)') return 0, 'skip', None + # usbtest's hang recovery reflashes the exact artifact under test; re-deriving it from + # board['name'] breaks on variant-only boards + global _current_fw + _current_fw = str(fw_name) if verbose: log_line(f'Firmware {fw_name}') - # flash firmware (unless --skip-flash), then run the test. Both may fail randomly, - # retry a few times. global _enum_timeout start_s = time.time() flash_ok = True last_err = '' last_detail = '' + wedge_break = False for i in range(max_retry): + if board_wedged and i: + # The latch is set MID-attempt (a HUNG usbtest whose flasher cannot recover), + # so test_board's check between tests is too late for THIS test's own retries: + # every further attempt re-flashes into the D-state-held node, blocks, survives + # SIGKILL and leaves another stray. The wedge is not something a retry can fix. + log_line(f'{test_name} not retrying: {board_wedged}') + # COUNT it. Breaking out here skips the i == max_retry - 1 branch that would + # have incremented err_count, so the board rendered a red cell, contributed 0 + # to the exit status and was omitted from the re-run spec -- a rig left with a + # D-state holder published under sys.exit(0). Latent at CI's --retry 1, live + # for every local run and for the workflows that pass no -r. + wedge_break = True + break _enum_timeout = ENUM_TIMEOUT if i == 0 else ENUM_TIMEOUT_RETRY attempt_out = io.StringIO() with redirect_stdout(attempt_out): if not skip_flash: with hil_lock.flash_permit(board['uid']): t_flash = time.monotonic() - ret = getattr(hil_flash, f'flash_{board["flasher"]["name"].lower()}')(board, str(fw_name)) + try: + ret = getattr(hil_flash, + f'flash_{board["flasher"]["name"].lower()}')(board, str(fw_name)) + except Exception as e: + # A flasher that RAISES (esptool's get_serial_dev when the adapter + # drops off the bus, a missing config.env, an unwritable CWD) would + # propagate out of the worker and abort the whole drain, costing + # every board still in flight. + print(f'flash raised: {type(e).__name__}: {e}', flush=True) + ret = subprocess.CompletedProcess(args='flash', returncode=1, + stdout=f'{type(e).__name__}: {e}') if PROFILE: log_line(f'[prof] {variant} {example} flash attempt {i + 1}: ' f'{time.monotonic() - t_flash:.1f}s rc={ret.returncode}') flash_ok = (ret.returncode == 0) - # A wedged RP2040/RP2350 DAP answers nothing and the probe has no reset - # line, so the retry would fail identically; POR it via the Rescue DP - # first. No-op for every other board and every other flash failure. - if not flash_ok and i + 1 < max_retry and \ - hil_flash.rescue_openocd(board, hil_flash.cmd_stdout_text(ret.stdout)): + # A wedged RP2040/RP2350 DAP answers nothing and the probe has no + # reset line, so the retry fails identically; POR it via the Rescue DP + # first (no-op otherwise). NOT gated on a remaining attempt: CI HIL jobs + # run --retry 1, and this leaves the DAP POR'd for the jobs that follow. + if not flash_ok and \ + hil_flash.rescue_openocd(board, hil_util.cmd_stdout_text(ret.stdout)): log_line(f'{variant} {example}: DAP wedged, rescued via Rescue DP') if flash_ok: try: @@ -1484,7 +1604,6 @@ def test_example(board: Board, variant: str, example: str) -> tuple[int, str, st else: status = STATUS_OK result_status = 'pass' - # a test may return a string to show in its report cell (e.g. speed) metric = tret if isinstance(tret, str) else None msg = f'{test_name} {status}' if last_detail: @@ -1495,9 +1614,20 @@ def test_example(board: Board, variant: str, example: str) -> tuple[int, str, st except Exception as e: last_err = str(e) last_detail = compact_output(attempt_out.getvalue()) + if getattr(e, 'parsed', False): + # a PARSED per-case result (usbtest's "29/30"): retrying re-pays + # the whole battery, inside the fleet's usbtest permit, to + # re-observe a number the JSON already reported. Only that case. + err_count += 1 + metric = getattr(e, 'metric', None) + msg = f'{test_name} {STATUS_FAILED}: {e}' + if last_detail: + msg += f' {last_detail}' + msg += f' in {time.time() - start_s:.1f}s' + log_line(msg) + break if i == max_retry - 1: err_count += 1 - # a failing test may still carry a metric to show in its cell (e.g. "❌ 29/30") metric = getattr(e, 'metric', None) msg = f'{test_name} {STATUS_FAILED}: {e}' if last_detail: @@ -1530,13 +1660,21 @@ def test_example(board: Board, variant: str, example: str) -> tuple[int, str, st msg += f' in {time.time() - start_s:.1f}s' log_line(msg) + if wedge_break and not err_count: + # ONE error for the test, never two: a board that also failed to flash has already + # been counted just above. Without this the test returns 0 -- red cell, clean exit + # status, absent from the re-run spec. + err_count += 1 return err_count, result_status, metric def build_board(board: Board) -> tuple[str, int]: """Build firmware for this board via tools/build.py. Honors board config's variant list and build.args defines. - Output goes to cmake-build/cmake-build-/ (tools/build.py layout).""" + Output goes to cmake-build/cmake-build-/ (tools/build.py layout). + + Unbounded on purpose: --build is a local convenience (no CI workflow passes it), so + the developer watching the build is the timeout.""" name = board['name'] bcfg = cast(BuildCfg, board.get('build', {})) extra_defs = bcfg.get('args', []) @@ -1544,7 +1682,7 @@ def build_board(board: Board) -> tuple[str, int]: failed = 0 for v in variants: - cmd = [sys.executable, str(hil_flash.TINYUSB_ROOT / 'tools' / 'build.py'), '-b', name] + cmd = [sys.executable, str(hil_util.TINYUSB_ROOT / 'tools' / 'build.py'), '-b', name] for d in extra_defs: cmd += ['-D', d] if v['name'] != name: @@ -1556,8 +1694,19 @@ def build_board(board: Board) -> tuple[str, int]: if verbose: cmd.append('-v') print(f' + {" ".join(cmd)}') - r = subprocess.run(cmd, cwd=hil_flash.TINYUSB_ROOT) - if r.returncode != 0: + # stdio is inherited so the build STREAMS: a silent buffer is + # indistinguishable from a stall. + proc = subprocess.Popen(cmd, cwd=hil_util.TINYUSB_ROOT, start_new_session=True) + try: + rc = proc.wait() + except KeyboardInterrupt: + # start_new_session means the build never saw the terminal's SIGINT + try: + os.killpg(proc.pid, signal.SIGKILL) + except OSError: + proc.kill() + raise + if rc != 0: failed += 1 return name, failed @@ -1567,28 +1716,29 @@ BOUNDARY_CELL = 'same-PID boundary' def test_board(board: Board) -> tuple[str, int, list[str], list, float]: + swept = False name = board['name'] flasher = board['flasher'] + global board_wedged + board_wedged = '' try: _lock_fh = hil_lock.acquire_board_lock(name) except RuntimeError as e: log_line(f'{name:25} {STATUS_FAILED}: {e}') - # visible report row so the ❌ matches the exit code; failed-tests stays - # empty so a re-run repeats the whole board (no bogus -bt test filter) + # visible report row so the ❌ matches the exit code; failed-tests stays empty so a + # re-run repeats the whole board (no bogus -bt filter) return name, 1, [], [(name, {'board-locked': 'fail'}, None)], 0.0 # after the lock: flock wait behind a concurrent run is not board cost t_board = time.monotonic() try: - # default to all tests test_list = [] if name in board_test: test_list = board_test[name] elif len(test_only) > 0: - # Explicit -t: filter against the board's capabilities so a device-only - # board doesn't try to run host/dual tests (the test functions need a - # `dev_attached` entry in the board config that won't exist). + # Explicit -t: filter against the board's capabilities, or a device-only board + # runs host/dual tests whose `dev_attached` config entry does not exist. board_tests = board.get('tests', {}) if 'only' in board_tests: allowed = set(board_tests['only']) @@ -1618,7 +1768,7 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: err_count = 0 failed_tests = [] board_wide_fail = False # re-run the whole board, not a subset of its tests - rows = [] # list of (row_label, {example: status}, duration) — one row per build variant + rows = [] # list of (row_label, {example: status}, duration) — one per build variant # a -t/-bt filtered run times only a subset; report no duration so an accumulate # re-run keeps the previous full-run value partial = bool(test_only) or name in board_test @@ -1627,11 +1777,10 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: prev_last = None # last test of the previous variant: the variant boundary is an adjacency too for v in variants: vname = v['name'] - # Shuffle each (board, variant)'s run order — de-synchronizes the worker pool so - # usbtest batteries and flash churn spread across the timeline instead of convoying, - # and surfaces order-dependent bugs. Seeded for replay (HIL_SHUFFLE_SEED, logged by - # main). Unique per-example PIDs make any two different examples re-enumerate; only - # the variant boundary can repeat the same example (same PID) — swap it away. + # Shuffle each (board, variant)'s run order: spreads batteries and flash churn + # across the timeline instead of convoying, and surfaces order-dependent bugs. + # Seeded for replay (HIL_SHUFFLE_SEED). Unique per-example PIDs re-enumerate + # between examples; only the variant boundary can repeat one. run_list = list(test_list) if shuffle_seed is not None and len(run_list) > 1: random.Random(f'{shuffle_seed}:{name}:{vname}').shuffle(run_list) @@ -1639,23 +1788,33 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: run_list[0], run_list[-1] = run_list[-1], run_list[0] cells = {} if run_list and run_list[0] == prev_last and not skip_flash: - # Same example (same PID) still repeats across the boundary: a one-test - # list (the common case for a -bt scoped run) leaves nothing to swap - # with. Park on board_test first - it disables the board's USB, so the - # PID goes away and the next flash must re-enumerate to be seen. + # Same example (same PID) still repeats across the boundary (a one-test + # -bt run has nothing to swap with). Park on board_test first: it disables + # the board's USB, so the next flash must re-enumerate to be seen. t_park = time.monotonic() - park_ec, park_status, _ = test_example(board, vname, 'device/board_test') + # _should_park, same as the teardown park: this is attempt 0, so + # test_example's retry guard does not stop it flashing into a poisoned node + park_ec, park_status, _ = ( + test_example(board, vname, 'device/board_test') if _should_park(skip_flash) + else (0, 'skip', None)) if park_ec or park_status == 'skip': - # Boundary not cleared: the previous variant's device may still be - # enumerated under the same PID, so this variant's tests could pass - # against its firmware. Skip them - a false green proves nothing and - # is worse than a gap - and record the boundary itself as the failure - # (a visible ❌ cell, mirroring the board-lock row above) so the report - # matches the exit code instead of rendering all-green. - why = 'no board_test binary' if park_status == 'skip' else 'park flash failed' + # Boundary not cleared: the previous variant may still be enumerated + # under the same PID, so this variant's tests could pass against ITS + # firmware. Skip them and record the boundary as the failure, so the + # report matches the exit code instead of rendering all-green. + # A 'skip' here has two very different causes: no board_test build, or + # _should_park refusing to flash a WEDGED board. Reporting the latter as + # a missing binary sends the operator hunting a build that exists. + wedge_skip = park_status == 'skip' and bool(board_wedged) + why = ('the board is wedged' if wedge_skip else + 'no board_test binary' if park_status == 'skip' else + 'park flash failed') log_line(f'{vname:40} {"same-PID boundary":30} {STATUS_FAILED}: not cleared ({why}); ' f'skipping {len(run_list)} test(s) on this variant') - err_count += 1 + # the wedge already charged its own error through test_device_usbtest; + # charging again would double-count one incident in the exit code + if not wedge_skip: + err_count += 1 cells[BOUNDARY_CELL] = 'fail' # blaming run_list[0] would re-run an innocent test that then passes, # leaving the boundary unretested; re-run the whole board instead @@ -1668,31 +1827,72 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: prev_last = run_list[-1] t_variant = time.monotonic() for test in run_list: + if board_wedged: + # Do NOT flash through a poisoned node: each attempt enumerates into + # it, blocks uninterruptibly and leaves another stray behind. Report + # the skip so the cell is not mistaken for a pass. + cells[test] = f'{REPORT_CELL["skip"]} board wedged' + # ...and re-run the WHOLE board, like the boundary-failure path above: + # these tests never executed, so naming them individually in the .failed + # spec is not enough -- an --accumulate re-run that fixes only the wedged + # test would merge a green cell over it and leave these skips standing + # from the earlier attempt, forever, under a green job. + board_wide_fail = True + continue ec, status, metric = test_example(board, vname, test) err_count += ec cells[test] = metric if metric else status if ec > 0: failed_tests.append(test) + if board_wedged: + log_line(f'{vname:40} SKIPPING the rest of this board: {board_wedged}; ' + f'flashing through the poisoned node would add a stray per test') dur = f'{time.monotonic() - t_variant:.0f}s' if run_list and not partial else None rows.append((vname, cells, dur)) - # board duration excludes the teardown park-flash below; a partial (filtered) - # run reports 0.0 so it never overwrites a cached full-run duration + # excludes the teardown park-flash below; a partial (filtered) run reports 0.0 so + # it never overwrites a cached full-run duration t_total = 0.0 if partial else time.monotonic() - t_board - # flash board_test last to disable board's usb (skipped when --skip-flash is set); - # this is teardown/park, not a test — not recorded in the report - if not skip_flash: + # park: flash board_test last to disable the board's usb; teardown, not a test, + # so it is not recorded in the report. + # + # NOT on a wedged board: the latch has just skipped every remaining test precisely + # because flashing through a D-state-held node blocks, survives SIGKILL and leaves + # a stray -- and this park is a flash like any other. test_example's own guard does + # not stop it (that one only suppresses RETRIES, and this is attempt 0), so the + # containment path would add the very stray it exists to prevent. + if _should_park(skip_flash): test_example(board, variants[0]['name'], 'device/board_test') - return name, err_count, [] if board_wide_fail else sorted(set(failed_tests)), rows, t_total + # Sweep HERE, not in main()'s finally: maxtasksperchild=1 retires this process as + # soon as it returns, reparenting anything it spawned to init and off the pool's + # ppid tree, so the main-side sweep walks fresh idle workers and finds nothing. + # Measured: 4 tasks, zero overlap, sweep 0, all 4 strays alive. + stray = hil_health.kill_own_children() + swept = True + + # LAST fields: whether this worker ran out of bounded-read budget, and what it could + # not kill. Only the worker can answer either -- the blindness latch is + # process-global and this is a separate process -- and the result tuple already + # crosses back, so no Manager round-trip. + return (name, err_count, [] if board_wide_fail else sorted(set(failed_tests)), + rows, t_total, hil_util.sysfs_blind(), stray) finally: + # A raise skips the sweep above, and maxtasksperchild=1 retires this process + # immediately afterwards -- reparenting its flasher to init and erasing the ppid + # link, so main's sweep cannot see it either. The count cannot reach the report on + # this path (there is no result tuple), but the KILL still frees the probe. + if not swept: + try: + hil_health.kill_own_children() + except Exception as se: # noqa: BLE001 - never mask the original failure + print(f'warning: stray sweep failed: {type(se).__name__}: {se}', flush=True) if _lock_fh: try: - # clear our pid record before dropping the flock: this worker - # process lives on (pool reuse), so a stale record would make - # hil_lock.py's pid-liveness checks report a freed board as - # still locked for the rest of the run + # clear our pid record before dropping the flock: this worker process + # lives on (pool reuse), so a stale record would make hil_lock's + # pid-liveness checks report a freed board as locked for the rest of the run _lock_fh.truncate(0) except OSError: pass @@ -1701,10 +1901,9 @@ def test_board(board: Board) -> tuple[str, int, list[str], list, float]: REPORT_MD = 'hil_report.md' REPORT_JSON = 'hil_report.json' -# controller hints learned from previous runs: uid -> {'name', 'pci', 'duration'}. Only -# 'pci' is consumed (dispatch order and first-flash budgeting, never battery -# serialization); name/duration are informational. PCI addresses are boot-stable (bus -# numbers are not), so the cache survives reboots and only goes stale on re-cabling. +# controller hints from previous runs: uid -> {'name', 'pci', 'duration'}. Only 'pci' is +# consumed (dispatch order and first-flash budgeting, never battery serialization). PCI +# addresses are boot-stable, so the cache survives reboots and goes stale on re-cabling. CONTROLLER_CACHE = Path.home() / '.cache' / 'tinyusb-hil' / 'controller_cache.json' @@ -1729,8 +1928,8 @@ def render_matrix(rows_all: list) -> str: if not seen: return 'No tests were run.' - # metric-bearing columns pinned first (usbtest score, throughput, explorer read speed), - # the rest alphabetical by bare test name: stable regardless of the (shuffled) execution order + # metric-bearing columns pinned first, the rest alphabetical: stable regardless of the + # shuffled execution order pinned = ['usbtest', 'cdc_msc_throughput', 'msc_file_explorer', 'msc_file_explorer_freertos'] def col_key(t): @@ -1761,9 +1960,8 @@ def render_matrix(rows_all: list) -> str: sep = '| ' + '-' * board_w + ' | ' + ' | '.join(':' + '-' * (w - 2) + ':' for w in col_w) + ' |' body = [line(lbl, vals) for lbl, vals in rows_vals] - # tally run cells (blank/not-run cells are absent from the dicts). A cell is a bare status - # ('pass'/'fail'/'skip') or a metric string that carries its own icon (e.g. "❌ 29/30" is a - # fail, "✅ 30/30" / "✅ CDC …" a pass), so classify by the leading icon. + # tally run cells (not-run cells are absent from the dicts). A cell is a bare status or + # a metric string carrying its own icon ("❌ 29/30"), so classify by the leading icon. def cell_kind(v): if v == 'fail' or (isinstance(v, str) and v.startswith(REPORT_CELL['fail'])): return 'fail' @@ -1780,7 +1978,119 @@ def render_matrix(rows_all: list) -> str: return summary + '\n\n' + '\n'.join([header, sep] + body) -def accumulate_report(mret: list, report_dir: Path, fresh: bool, scope: str = '') -> str: +def _write_failed_spec(failed_fname: Path, report_dir: Path, mret: list) -> None: + """Re-run spec: only the failed boards (-b), each restricted to its own failed tests + (-bt); a board with failures but no test list re-runs entirely. + + Shared with the pool-guard path, which feeds it the boards that never reported. That + path used to leave this unwritten -- and a fresh run has already unlinked it -- so + build.yml's "Get re-run spec" step found nothing and the GitHub re-run repeated the + whole fleet to find the one board that wedged.""" + parts = ['--accumulate'] + for name, err, fts, *_ in mret: + if err > 0: + parts.append(f'-b {name}') + if fts: + parts.append(f'-bt {name}:{",".join(fts)}') + if len(parts) > 1: # build-only failures have no boards to re-run + report_dir.mkdir(parents=True, exist_ok=True) + with failed_fname.open('w') as f: + f.write(' '.join(parts)) + else: + failed_fname.unlink(missing_ok=True) + + +class PoolDrainTimeout(MpTimeoutError): + """Guard expiry, carrying the rows that DID finish. + + They ride on the exception because the raise is the containment path: losing them here + is what map_async did, and what the drain exists to stop. + """ + + def __init__(self, finished: list): + super().__init__() + self.finished = finished + + +def drain_pool(it, boards: list, deadline: float, out: list | None = None) -> list: + """Collect imap_unordered results against ONE deadline. Returns the finished rows. + + Raises PoolDrainTimeout (carrying those same rows) when the deadline passes with boards + still in flight -- the caller keeps them, names only what is missing, and writes a + re-run spec covering just those. + + A function, not an inline loop, so the tests can call THIS instead of a copy of it: the + loop's previous test built its own ThreadPool and its own drain and asserted on those, + so deleting the real one outright kept the suite green. + """ + # `out` is the CALLER's list: a worker that raises something other than a timeout + # (get_serial_dev on a dropped adapter, a Manager EOFError) propagates bare, and a + # local accumulator would take every finished board with it -- the exact loss the + # drain replaced map_async to prevent. + mret: list = out if out is not None else [] + for _ in boards: + left = deadline - time.monotonic() + if left <= 0: + raise PoolDrainTimeout(mret) + try: + mret.append(it.next(timeout=left)) + except MpTimeoutError: + raise PoolDrainTimeout(mret) from None + return mret + + +def _should_park(skip_flash: bool) -> bool: + """Flash the teardown park (device/board_test, to switch the DUT's USB off)? + + Not on a wedged board. The latch has just skipped every remaining test precisely + because flashing through a D-state-held node blocks, survives SIGKILL and leaves a + stray -- and the park is a flash like any other. test_example's own guard does not stop + it either: that one only suppresses RETRIES, and the park is always attempt 0. So the + containment path would end by adding the very stray it exists to prevent. + """ + return not skip_flash and not board_wedged + + +def _stray_note(mret: list) -> str: + """Name the strays the workers could not kill, for the report banner. + + Summed from the result tuples rather than computed in main()'s finally: that finally + runs AFTER accumulate_report on both abort paths, so a banner appended there was + written to a variable nobody read again. + """ + dirty = [(r[0], r[6]) for r in mret if len(r) > 6 and r[6]] + if not dirty: + return '' + total = sum(n for _, n in dirty) + return (f'> **Rig dirty.** {total} process(es) survived SIGKILL and still hold a probe ' + f'or usbfs node into the next job: ' + f'{", ".join(f"{b} ({n})" for b, n in dirty)}.\n') + + +def _blind_note(mret: list) -> str: + """Name the boards whose worker went blind, for the report banner. + + A blind worker answers SYSFS_UNKNOWN for every attribute, so its "device not found" is + "could not tell". That already reaches the log and the per-cell failure text, but the + TABLE is what gets quoted -- and a red cell there is read as a broken board. Seen live + (run 31794359407): four workers blind, several cells red because of it, and a report + that said nothing. + + Per-board, not global: maxtasksperchild=1 gives every board a fresh worker, so a board + that ran on a healthy one is not smeared by a neighbour's wedge. Rows synthesised by + the timeout path are 5 fields wide and have nothing to report. + """ + blind = [r[0] for r in mret if len(r) > 5 and r[5]] + if not blind: + return '' + return (f'> **Not all verdicts are evidence.** {len(blind)} board(s) ran on a worker ' + f'that went blind on sysfs -- too many bounded reads stranded on a wedged ' + f'device -- so "not found" from them means "could not tell": ' + f'{", ".join(blind)}. See the usb-kernel-recover skill.\n') + + +def accumulate_report(mret: list, report_dir: Path, fresh: bool, scope: str = '', + banner: str = '') -> str: """Merge this run's results into hil_report.json in report_dir, then (re)write the markdown matrix to hil_report.md. `fresh` (a first run, no --accumulate) starts a new report; otherwise a re-run accumulates so boards/tests that @@ -1788,29 +2098,34 @@ def accumulate_report(mret: list, report_dir: Path, fresh: bool, scope: str = '' board filter, if any, so a scoped table is not mistaken for a full one. Returns the md.""" acc = {} # ordered {row_label: [cells dict, duration str|None]} + prior_banner = '' jpath = report_dir / REPORT_JSON if not fresh and jpath.is_file(): try: saved = json.loads(jpath.read_text()) - # CI keys the report dir by run id, so the sidecar can only have been - # written by an earlier attempt of the same run + # CI keys the report dir by run id, so the sidecar is from an earlier attempt for entry in saved.get('rows', []): acc[entry['board']] = [dict(entry['cells']), entry.get('duration')] + # ... and so is the caveat those cells were collected under. A rerun on a rig + # that has since recovered contributes no banner, and the .failed spec reruns + # only FAILURES -- so the earlier attempt's passes are never re-earned and + # would be published as clean results of a rig that was not. + prior_banner = saved.get('banner', '') except (ValueError, KeyError, TypeError): pass # corrupt/old sidecar: start fresh - # merge this run: current cells override prior for boards/tests that ran; a filtered - # run reports duration None, keeping the previous full-run value - for name, _, _, rows, _ in mret: + # current cells override prior for boards/tests that ran; a filtered run reports + # duration None, keeping the previous full-run value + for name, _, _, rows, *_ in mret: if rows and not any('board-locked' in cells for _, cells, _ in rows): - # board ran for real this time: clear a stale lock-failure cell - # (its row is keyed by board name; test rows may be variant names) + # board ran for real: clear a stale lock-failure cell (its row is keyed by + # board name; test rows may be variant names) stale = acc.get(name) if stale is not None: stale[0].pop('board-locked', None) if not stale[0]: - # variant-keyed boards never repopulate the board-name row — - # drop it or it renders as a blank ghost row + # variant-keyed boards never repopulate the board-name row, so drop it + # or it renders as a blank ghost row del acc[name] for row_label, cells, dur in rows: row = acc.setdefault(row_label, [{}, None]) @@ -1823,18 +2138,100 @@ def accumulate_report(mret: list, report_dir: Path, fresh: bool, scope: str = '' row[1] = dur report_dir.mkdir(parents=True, exist_ok=True) + # by LINE, deduped: attempts repeat the same caveat far more often than they add a new + # one, and three copies of the D-state note reads as three incidents + seen, merged = set(), [] + for line in (prior_banner + banner).splitlines(): + if line.strip() and line not in seen: + seen.add(line) + merged.append(line) + banner = '\n'.join(merged) + '\n' if merged else '' jpath.write_text(json.dumps({'rows': [{'board': k, 'cells': c, 'duration': d} - for k, (c, d) in acc.items()]}, indent=2) + '\n') + for k, (c, d) in acc.items()], + 'banner': banner}, indent=2) + '\n') md = render_matrix([(k, c, d) for k, (c, d) in acc.items()]) if scope: - # a scoped run's small table is otherwise indistinguishable from a full one, - # and it replaces the previous full table in the sticky PR comment + # a scoped run's small table is otherwise indistinguishable from a full one, and + # it replaces the previous full table in the sticky PR comment md = f'_Scoped run: {scope}. Boards/tests not listed were not run._\n\n' + md + # LAST, so it is outermost: a rig-health caveat outranks the table AND the scope note, + # and the top of the report is where hil/SKILL.md tells the agent to look for it. + if banner: + md = banner + '\n' + md (report_dir / REPORT_MD).write_text(md + '\n', encoding='utf-8') return md +# containment paths print through hil_health._p: stdout may already be a dead pipe (a +# dropped ssh session), and a BrokenPipeError there would skip os._exit +_p = hil_health._p + + +def _abandon_exit(pool, mgr, abandoned: bool, err_count: int, + report: Path | None = None) -> None: + """Free the runner when the pool could not be shut down. Returns only if not abandoned. + + Must run even while an exception is propagating: multiprocessing's atexit handler + SIGTERMs its daemon workers (ignored in uninterruptible sleep) and then join()s them + with NO timeout, so an abandoned pool plus any raise between the pool's finally and + here hangs the interpreter until the job ceiling kills it. Reproduced: rc=124 at 25s + with SIGTERM-ignoring workers standing in for D state.""" + if not abandoned: + return + try: + if sys.exc_info()[0] is not None: + # os._exit below discards the traceback, and this is often the only place the + # real failure would ever be printed + traceback.print_exc() + except OSError: + pass + # Word this on evidence: shutdown_pool also returns False when terminate() RAISES, and + # a live worker after terminate() is what distinguishes a wedge from a harness bug. + # Count WORKERS only -- _pool_procs appends the Manager, our own healthy child, so + # including it made n >= 1 always and the harness-error branch unreachable. It is killed + # separately: os._exit skips its finalizer, and orphaned it holds the runner's stdout. + n = hil_health.kill_pool_children(pool) + hil_health.kill_pool_children(None, mgr) + if n: + _p(f'HIL worker pool would not terminate ({n} worker(s) still live, ' + f'uninterruptible); SIGKILLed them and abandoned the rest to free the ' + f'runner. Boards held by any leaked worker stay locked until the host is ' + f'power-cycled.', flush=True) + else: + _p('HIL worker pool shutdown failed but left no live worker behind, so this is ' + 'a harness error rather than a wedged rig -- see the Pool.terminate() ' + 'warning above. Exiting early anyway to free the runner; no board should ' + 'stay locked.', flush=True) + # A report already written by accumulate_report says nothing about the abandon, and a + # green table under a red job is how an agent ends up pasting it as this run's result. + # Prepend the caveat; best-effort, never at the cost of exiting. + if report is not None: + try: + if report.exists(): + # utf-8 explicitly (the cells are ✅/❌/⚪) and catch ValueError too: a torn + # report or a LANG=C locale raises UnicodeDecodeError -- NOT an OSError -- + # straight past os._exit, stranding the runner. + body = report.read_text(encoding='utf-8', errors='replace') + # Only when no banner is there yet, searched anywhere in the head rather + # than at char 0: write_timeout_report's banner must stay FIRST (its table + # is a PREVIOUS attempt's) and it puts the rig-health quote above itself. + if '**HIL run ab' not in body[:2000]: + report.write_text( + '**HIL run abandoned: the worker pool would not shut down.** The ' + 'table below was collected before the abandon; treat board ' + 'results as unverified.\n\n' + body, encoding='utf-8') + except (OSError, ValueError): + pass + try: + sys.stdout.flush() + except OSError: + pass + # Clamped: os._exit takes a status byte, so err_count == 256 would truncate to 0 and + # report a failing, abandoned run as green. + os._exit(min(err_count, 125) if err_count else 1) + + def main() -> None: """ Hardware test on specified boards @@ -1864,14 +2261,21 @@ def main() -> None: help='Per-board test list as BOARD:test1,test2 (overrides -t for that board); repeat for multiple boards') parser.add_argument('-B', '--build-dir', default='cmake-build', help='Build folder name (default: cmake-build)') parser.add_argument('--build', action='store_true', help='Build firmware for selected boards with cmake before running tests') - parser.add_argument('-r', '--retry', type=int, default=3, help='Retry count for failed tests (default: 3)') + # default 1, not 3: the pool guard is a FLAT 3600s that does not scale with max_retry, + # and one usbtest test at default 3 can burn 1530s of it (510s outer x3) for a single + # board. Every CI caller already pins --retry 1; the bare invocations in the hil skill + # and hil-validate.js run against the same one-slot rig and used to inherit 3. + parser.add_argument('-r', '--retry', type=int, default=1, help='Retry count for failed tests (default: 1)') parser.add_argument('-v', '--verbose', action='store_true', help='Verbose output') args = parser.parse_args() + if args.retry < 1: + # 0 would make every test loop body never run: all-red cells, exit 0 + parser.error('--retry must be >= 1') config_file = Path(args.config_file) boards = args.board verbose = args.verbose - hil_flash.verbose = args.verbose + hil_util.verbose = args.verbose test_only = args.test_only for entry in args.board_test: bname, _, tnames = entry.partition(':') @@ -1899,6 +2303,32 @@ def main() -> None: config_boards = [e for e in config['boards'] if e['name'] in boards] config_boards = [e for e in config_boards if e['flasher']['name'] not in args.exclude_flasher and (not args.flasher or e['flasher']['name'] in args.flasher)] + if not config_boards: + # same reason the unknown -b board exits 1: 'No tests were run.' with rc 0 reads as + # a green HIL leg, so a roster edit emptying a leg's filter stops testing silently + msg = (f'No boards left after the flasher filter (--flasher ' + f'{args.flasher or "-"}, --exclude-flasher {args.exclude_flasher or "-"})') + print(msg, flush=True) + # loud AND leaving evidence: exiting with no report at all lets the PR comment + # keep the previous push's stale table under a red job + try: + rd = Path(os.environ.get('HIL_REPORT_DIR', '.')) + rd.mkdir(parents=True, exist_ok=True) + (rd / REPORT_MD).write_text(f'**HIL run selected no boards.** {msg}\n', + encoding='utf-8') + except OSError: + pass + sys.exit(1) + + + # Before the build: the probe needs nothing from it, and the annotation is more useful + # early than after a multi-board cmake build has been paid for. + # One line, not a probe: a D-state pid at start-up is a hint for whoever reads a red + # cell, never a reason to refuse the run. hil_pool_check does diagnosis. + note = hil_health.d_state_note() + if note: + log_line(f'rig note: {note}') + health_banner = f'> **Rig note.** {note}. Not a fault on its own -- a healthy testusb sits in D state for most of every case.\n' if note else '' build_err = 0 if args.build: @@ -1915,26 +2345,24 @@ def main() -> None: print(f'Build phase done: {build_err} failed') print('-' * 30) - # HIL report sidecar (hil_report.json/.md) and the .failed re-run spec live in - # report_dir (CI keys it by run id, so it persists across run attempts but is - # private to one run). A full run starts fresh; a re-run (--accumulate, which - # the generated .failed spec always starts with) merges so already-passed - # boards/tests are preserved. Clear prior state up front on a fresh run so a - # crash mid-run can't leave a stale report or re-run spec for a retry. - # -bt alone is not a re-run marker: PR-scoped first attempts pass -bt too. + # The report sidecar and the .failed re-run spec live in report_dir (CI keys it by run + # id: persistent across attempts, private to one run). A full run starts fresh; a re-run + # (--accumulate, which .failed always starts with) merges so already-passed boards + # survive. -bt alone is not a re-run marker. report_dir = Path(os.environ.get('HIL_REPORT_DIR', '.')) failed_fname = report_dir / (config_file.name + '.failed') fresh = not args.accumulate - if fresh: - report_dir.mkdir(parents=True, exist_ok=True) - for f in (REPORT_JSON, REPORT_MD): - (report_dir / f).unlink(missing_ok=True) - failed_fname.unlink(missing_ok=True) + # The unlink is DEFERRED to inside the pool try/except below: wiping here leaves + # Manager() and Pool() running with the old report gone and no report-writing path + # armed, so an EAGAIN/ENOMEM on fork gives CI an EMPTY report dir with no reason. seed = os.getenv('HIL_SHUFFLE_SEED') or str(int(time.time())) log_line(f'test-order shuffle seed: {seed} (HIL_SHUFFLE_SEED={seed} to replay); ' f'flash/usbtest parallel per controller: {hil_lock.FLASH_PARALLEL}/{hil_lock.USBTEST_PARALLEL}; ' - f'enum timeout first/retry: {ENUM_TIMEOUT}/{ENUM_TIMEOUT_RETRY}s') + f'enum timeout first/retry: {ENUM_TIMEOUT}/{ENUM_TIMEOUT_RETRY}s; ' + # all three are env-tunable, so a run that dies on the guard is otherwise + # unattributable from the log alone + f'pool guard: {POOL_TIMEOUT}s') hints = {} try: @@ -1949,94 +2377,232 @@ def main() -> None: config_boards = schedule_boards(config_boards, hints_by_uid) log_line('dispatch order: ' + ', '.join(b['name'] for b in config_boards)) - mgr = Manager() - cmap = mgr.dict() - initargs = (Lock(), seed, - [Semaphore(hil_lock.USBTEST_PARALLEL) for _ in range(hil_lock.CONTROLLER_SLOTS)], - [Semaphore(hil_lock.FLASH_PARALLEL) for _ in range(hil_lock.CONTROLLER_SLOTS)], - cmap, Lock(), hints_by_uid) - with Pool(processes=os.cpu_count() or 1, initializer=init_worker, initargs=initargs) as pool: - async_ret = pool.map_async(test_board, config_boards) + # Bound BEFORE the try so the finally can name them whatever failed: Pool() forks, and + # the EAGAIN/ENOMEM the wipe comment below worries about is most likely to come from + # that fork -- after a convoy, where every stranded read holds a thread and an fd. Left + # outside, an OSError there escaped with mgr LIVE and `pool` unbound, so no report was + # written and the interpreter unwound into multiprocessing's unbounded atexit join. + pool = mgr = cmap = None + # Defined before the pool so _abandon_exit always has a value: a raise before + # `err_count = build_err + ...` would turn the containment path into a NameError. + err_count = build_err + # Fail CLOSED: only a shutdown_pool() that actually returned True clears this, and the + # assignment sits at the END of the inner finally, so anything raising before it + # (kill_worker_children, a BrokenPipeError from its print) leaves _abandon_exit armed. + pool_abandoned = True + # BEFORE Manager()/Pool(), not inside the try: hil_ci.sh reuses a persistent REMOTE_DIR + # and scp's the report back unconditionally, so if a fork failure (OSError/EAGAIN right + # after a convoy -- the case this whole block guards) skipped the wipe, the finally's + # _abandon_exit would prepend "HIL run abandoned" to the PREVIOUS run's table and + # publish last night's board results as this run's. Nothing is live yet here, so an + # OSError from the wipe itself just exits with its traceback -- it cannot strand the + # interpreter in multiprocessing's unbounded atexit join, which is what deferring it + # was protecting against. + if fresh: + report_dir.mkdir(parents=True, exist_ok=True) + for f in (REPORT_JSON, REPORT_MD): + (report_dir / f).unlink(missing_ok=True) + failed_fname.unlink(missing_ok=True) + try: + mgr = Manager() + cmap = mgr.dict() + initargs = (Lock(), seed, + hil_lock.make_permit_sems(Semaphore, hil_lock.USBTEST_PARALLEL), + hil_lock.make_permit_sems(Semaphore, hil_lock.FLASH_PARALLEL), + cmap, Lock(), hints_by_uid) + # maxtasksperchild=1: the sysfs blindness latch is process-global and permanent + # (no decrement anywhere -- see hil_util.SYSFS_STUCK_MAX), so a worker that goes + # blind on ONE wedged board would report 0/30 and "probe missing" for the 2-3 + # healthy boards it picked up afterwards. A fresh worker per board confines the + # damage to the board that caused it; the extra fork is noise against a + # flash+test cycle. + pool = Pool(processes=os.cpu_count() or 1, initializer=init_worker, + initargs=initargs, maxtasksperchild=1) + # OUTER: encloses the pool block too, not just the reporting below. An exception + # escaping async_ret.get() (a worker exception, a Ctrl-C) runs the pool finally and + # then propagates straight out of main(); with _abandon_exit in a sibling try it + # was never reached. try: - mret = async_ret.get(timeout=POOL_TIMEOUT) - except MpTimeoutError: - pool.terminate() - pool.join() - raise RuntimeError(f'HIL worker pool timed out after {POOL_TIMEOUT}s') - - err_count = build_err + sum(e[1] for e in mret) - # generate the re-run spec if anything failed: run ONLY the failed boards (-b), - # each restricted to its own failed tests (-bt); a board with failures but no - # test list (e.g. board-locked) re-runs entirely. --accumulate preserves the - # already-passed cells in the report. - parts = ['--accumulate'] - for name, err, fts, _, _ in mret: - if err > 0: - parts.append(f'-b {name}') - if fts: - parts.append(f'-bt {name}:{",".join(fts)}') - if len(parts) > 1: # build-only failures have no boards to re-run - report_dir.mkdir(parents=True, exist_ok=True) - with failed_fname.open('w') as f: - f.write(' '.join(parts)) - else: - failed_fname.unlink(missing_ok=True) + # imap_unordered, NOT map_async: map_async is all-or-nothing, so a guard expiry + # threw away every board that had already finished -- up to a worker-width of + # completed rig time -- and left the re-run spec unwritten, so CI re-tested all + # ~26 boards to find the one that wedged. Draining as results arrive keeps what + # finished and names only what was still in flight. + it = pool.imap_unordered(test_board, config_boards) + mret = [] + deadline = time.monotonic() + POOL_TIMEOUT + try: + mret = drain_pool(it, config_boards, deadline, out=mret) + except MpTimeoutError as te: + mret = te.finished + stuck = [b['name'] for b in config_boards + if b['name'] not in {r[0] for r in mret}] + # The re-run spec FIRST and before the raise: a fresh run already unlinked + # it, so leaving it unwritten is what made the GitHub re-run repeat the + # whole fleet. Only the boards that never reported go in it. + _write_failed_spec(failed_fname, report_dir, + [(n, 1, [], None, 0) for n in stuck] + + [r for r in mret if r[1] > 0]) + # Then the report, with the rows that DID finish, before anything that can + # block. Then RAISE into the ONE containment path: the inner finally runs + # the ordered sweep (kill_worker_children BEFORE terminate, or a reaped + # worker's flasher reparents out of reach), the outer one os._exit's. + banner = (f'**HIL run abandoned: worker pool timed out after ' + f'{POOL_TIMEOUT}s.** {len(mret)} board(s) below finished and ' + f'are this run\'s; {len(stuck)} never reported and are NOT in ' + f'the table: {", ".join(stuck)}. Re-run covers those.\n') + try: + accumulate_report(mret, report_dir, fresh, '', + health_banner + _blind_note(mret) + + _stray_note(mret) + banner) + except Exception as rerr: # noqa: BLE001 - the raise below must still happen + # FALL BACK, do not just warn: accumulate_report can raise on an + # unwritable/root-owned report dir or a torn JSON, and _abandon_exit + # only PREPENDS to a report that exists. Without this the artifact + # upload finds nothing (if-no-files-found: ignore) and the sticky PR + # comment keeps the previous push's green table under a red job. + print(f'warning: partial report failed: {type(rerr).__name__}: {rerr}; ' + f'falling back to the board list', flush=True) + try: + hil_health.write_timeout_report( + report_dir, [b for b in config_boards + if b['name'] in stuck], POOL_TIMEOUT, REPORT_MD, + prefix=health_banner) + except Exception as re2: # noqa: BLE001 + print(f'warning: fallback report failed too: ' + f'{type(re2).__name__}: {re2}', flush=True) + _p(f'HIL worker pool timed out after {POOL_TIMEOUT}s; sweeping and ' + f'shutting it down (abandoning it if a worker is unkillable)', + flush=True) + raise RuntimeError(f'HIL worker pool timed out after {POOL_TIMEOUT}s') + except Exception as e: + # A worker RAISED -- e.g. a flasher adapter dropping off the bus makes + # get_serial_dev raise in the worker's flash section, which no per-test + # handler guards. Same treatment as the timeout path: the drain means + # `mret` already holds every board that finished, so keep those rows and + # name only the ones still in flight. (Under map_async they were all lost, + # which is what the old banner here claimed.) + done = {r[0] for r in mret} + stuck = [b['name'] for b in config_boards if b['name'] not in done] + _write_failed_spec(failed_fname, report_dir, + [(n, 1, [], None, 0) for n in stuck] + + [r for r in mret if r[1] > 0]) + banner = (f'**HIL run aborted: a worker raised {type(e).__name__}: {e}.** ' + f'{len(mret)} board(s) below finished and are this run\'s; ' + f'{len(stuck)} did not report: {", ".join(stuck)}.\n') + try: + accumulate_report(mret, report_dir, fresh, '', + health_banner + _blind_note(mret) + + _stray_note(mret) + banner) + except Exception as re2: # noqa: BLE001 - the raise below must still happen + print(f'warning: partial report failed: {type(re2).__name__}: {re2}', + flush=True) + raise + + err_count = build_err + sum(e[1] for e in mret) + _write_failed_spec(failed_fname, report_dir, mret) + finally: + # Not `with Pool(...)`: its __exit__ joins the workers unbounded, hanging on + # any worker in uninterruptible sleep. shutdown_pool bounds the same terminate() + # by a grace period, so the pool is NOT cleanly closed/joined when it returns + # False. Record the outcome but never exit here: the report below is the only + # record of a run that otherwise passed. + # + # Same ordering as the timeout path: what the workers spawned must be + # snapshotted and killed while its parent is alive, or terminate() reparents it + # out of reach. + # + # Both calls must stay guarded: a raise here skips accumulate_report(), so a run + # whose boards ALL passed publishes an empty report dir -- and both can raise + # for reasons unrelated to the results. pool_abandoned stays fail-CLOSED, so + # _abandon_exit still arms. + try: + # Still worth running for the TIMEOUT path, where the workers are + # genuinely stuck mid-task and their children are still reachable through + # the pool's ppid tree. On the normal path every worker has already swept + # its own (kill_own_children) and retired, so this finds nothing. + # + # No banner from here: this finally runs AFTER accumulate_report on both + # abort paths, so anything appended to health_banner now is written to a + # variable nobody reads again. The report gets its count from the result + # tuples instead, via _stray_note. + hil_health.kill_worker_children(pool, mgr) + except Exception as e: + print(f'warning: worker-child sweep failed: {type(e).__name__}: {e}', + flush=True) + try: + pool_abandoned = not hil_health.shutdown_pool(pool) + except Exception as e: + print(f'warning: pool shutdown failed: {type(e).__name__}: {e}', flush=True) - # refresh controller hints: pci resolved this run, plus board durations when the - # full test list ran (a -t/-bt filtered run would understate the board's real cost) - try: - if PROFILE: - # debug snapshot of the run's live uid->PCI / PCI->slot resolutions - report_dir.mkdir(parents=True, exist_ok=True) - with (report_dir / 'hil_profile_ctrl.json').open('w') as f: - json.dump(dict(cmap), f, indent=1, sort_keys=True) - uid_of = {b['name']: b['uid'] for b in config['boards']} - for name, _, _, _, dur in mret: - uid = uid_of.get(name) - if uid is None: - continue - h = dict(hints.get(uid) or {}) - h['name'] = name # informational: cache is keyed by uid - h['pci'] = cmap.get(f'uid:{uid}') or h.get('pci') - if dur > 0: # test_board reports 0.0 for filtered (partial) runs - h['duration'] = round(dur, 1) - hints[uid] = h - # merge-on-write: another HIL job (e.g. the esp split) may have finished since - # our startup read - re-read and overlay only this run's boards so its entries - # survive, then replace atomically so a concurrent reader never sees a torn file - merged = {} + # refresh controller hints: pci resolved this run, plus durations from full runs + # only (a filtered run would understate the board's real cost) try: - with CONTROLLER_CACHE.open() as f: - cur = json.load(f) - if isinstance(cur, dict): - merged = {k: v for k, v in cur.items() if isinstance(v, dict)} - except (OSError, ValueError): - pass - merged.update({uid_of[n]: hints[uid_of[n]] for n, *_ in mret if n in uid_of}) - CONTROLLER_CACHE.parent.mkdir(parents=True, exist_ok=True) - tmp = CONTROLLER_CACHE.with_suffix('.json.tmp') - with tmp.open('w') as f: - json.dump(merged, f, indent=1, sort_keys=True) - tmp.replace(CONTROLLER_CACHE) - except OSError as e: - print(f'warning: cannot persist controller hints to {CONTROLLER_CACHE}: {e}') - - # board x test result matrix -> hil_report.md (accumulates across re-runs) + stdout - # -b/-bt in play means a filtered run (PR selection or a re-run spec): say so in the - # report, which otherwise looks exactly like a full run that happened to be small - scoped = sorted(set(args.board) | set(board_test)) - scope = f'{len(scoped)} board(s) — {", ".join(scoped)}' if scoped else '' - report = accumulate_report(mret, report_dir, fresh, scope) - print() - print(report) - print(f'\nReport written to {(report_dir / REPORT_MD).resolve()}') - - duration = time.time() - duration - print() - print("-" * 30) - print(f'Total failed: {err_count} in {duration:.1f}s') - print("-" * 30) - sys.exit(err_count) + if PROFILE: + # debug snapshot of the run's live uid->PCI / PCI->slot resolutions + report_dir.mkdir(parents=True, exist_ok=True) + with (report_dir / 'hil_profile_ctrl.json').open('w') as f: + json.dump(dict(cmap), f, indent=1, sort_keys=True) + uid_of = {b['name']: b['uid'] for b in config['boards']} + for name, _, _, _, dur, *_ in mret: + uid = uid_of.get(name) + if uid is None: + continue + h = dict(hints.get(uid) or {}) + h['name'] = name # informational: cache is keyed by uid + h['pci'] = cmap.get(f'uid:{uid}') or h.get('pci') + if dur > 0: # test_board reports 0.0 for filtered (partial) runs + h['duration'] = round(dur, 1) + hints[uid] = h + # merge-on-write: another HIL job (e.g. the esp split) may have finished since + # our startup read, so overlay only this run's boards and replace atomically + merged = {} + try: + with CONTROLLER_CACHE.open() as f: + cur = json.load(f) + if isinstance(cur, dict): + merged = {k: v for k, v in cur.items() if isinstance(v, dict)} + except (OSError, ValueError): + pass + merged.update({uid_of[n]: hints[uid_of[n]] for n, *_ in mret if n in uid_of}) + CONTROLLER_CACHE.parent.mkdir(parents=True, exist_ok=True) + tmp = CONTROLLER_CACHE.with_suffix('.json.tmp') + with tmp.open('w') as f: + json.dump(merged, f, indent=1, sort_keys=True) + tmp.replace(CONTROLLER_CACHE) + except Exception as e: + # Deliberately broad, and it must stay that way: this best-effort refresh makes + # Manager proxy RPCs that raise EOFError / BrokenPipeError / RemoteError when + # the Manager child has died, none of them OSErrors -- an OSError-only guard let + # those skip accumulate_report(). Nothing here is worth the report. + print(f'warning: cannot persist controller hints to {CONTROLLER_CACHE}: ' + f'{type(e).__name__}: {e}') + + + # board x test result matrix -> hil_report.md (accumulates across re-runs) + stdout. + # -b/-bt means a filtered run (PR selection or a re-run spec): say so, or the report + # looks exactly like a full run that happened to be small + scoped = sorted(set(args.board) | set(board_test)) + scope = f'{len(scoped)} board(s) — {", ".join(scoped)}' if scoped else '' + report = accumulate_report(mret, report_dir, fresh, scope, + health_banner + _blind_note(mret) + + _stray_note(mret)) + print() + print(report) + print(f'\nReport written to {(report_dir / REPORT_MD).resolve()}') + + duration = time.time() - duration + print() + print("-" * 30) + print(f'Total failed: {err_count} in {duration:.1f}s') + print("-" * 30) + finally: + # In the finally, not after: any raise above (accumulate_report sits outside the + # OSError handler) would skip the abandon path and unwind into multiprocessing's + # unbounded atexit join, hanging the runner. + _abandon_exit(pool, mgr, pool_abandoned, err_count, report_dir / REPORT_MD) + # Same clamp: exit status is a byte either way, so 256 failures would report green. + sys.exit(min(err_count, 125)) if __name__ == '__main__': diff --git a/test/hil/mtp_test.py b/test/hil/mtp_test.py new file mode 100644 index 000000000..92d54bdbe --- /dev/null +++ b/test/hil/mtp_test.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# One MTP test session for one board, in a disposable process. Every libmtp call is +# synchronous ctypes in our own address space and blocks in a usbfs ioctl in D state on +# a wedged device, where not even SIGKILL is delivered — so the session must be +# something the harness can abandon: hil_test.test_device_mtp runs it under +# hil_util.run_cmd (killpg + bounded reap, rc 124 on timeout). Imports stay stdlib + +# pymtp: nothing here may pull in the harness. +# +# Exit 0 on a fully passing session; 1 with the failure on stdout/stderr otherwise. +import argparse +import ctypes +import glob +import hashlib +import os +import signal +import subprocess +import sys +import threading +import time + +sys.path.append(os.path.dirname(os.path.abspath(__file__))) # PYTHONSAFEPATH drops it +# -- APPEND so PYTHONPATH still wins (the tests steer a fake pymtp that way) + +from pathlib import Path +from pymtp import LIBMTP_DeviceEntry, LIBMTP_RawDevice, MTP + +FILE1_EXPECT = b'TinyUSB MTP Filesystem example' +FILE2_MD5_EXPECT = '40ef23fc2891018d41a05d4a0d5f822f' # md5sum of logo.png + + +# Real paths by default; the offline tests point these at a fixture tree, the same way +# they steer the pymtp fake through FAKE_PYMTP_*. +# The one test seam: '' in production, a tmpdir in the offline tests, which mirror the +# real layout beneath it. This runs as a SUBPROCESS (a libmtp call blocked in a usbfs +# ioctl hangs its thread forever, so the session must be somewhere killable), and neither +# monkeypatching nor import shadowing crosses that boundary -- unlike the fake pymtp, +# which the tests inject through PYTHONPATH alone. +_ROOT = os.environ.get('HIL_MTP_FAKE_ROOT', '') +_MARKER_GLOB = f'{_ROOT}/dev/libmtp-*' +_SYS_USB = Path(f'{_ROOT}/sys/bus/usb/devices') +_USB_DEV = Path(f'{_ROOT}/dev/bus/usb') + + +def _bounded_read(path, grace: float = 2.0): + """Read a sysfs attribute with a wall-clock bound, or return None. + + `serial` is served under the device lock a wedged usbfs ioctl holds, and EVERY MTP DUT + is cafe:4017 -- so the vid/pid filter below cannot rule out a wedged NEIGHBOUR, and an + unbounded read of its serial would burn this session's whole budget and report a + healthy board as wedged. Stdlib only by design (this file never imports the harness), + so this is a small local twin of hil_util.read_sysfs. + """ + out = {} + + def _read(): + try: + out['v'] = path.read_text().strip() + except OSError: + pass + + t = threading.Thread(target=_read, daemon=True) + t.start() + t.join(grace) + return out.get('v') + + +def _ready_marker(uid: str): + """(busnum, devnum) of the udev-ready MTP device with this serial, or None. + + /dev/libmtp- is published by libmtp-runtime AFTER its synchronous mtp-probe + accepts the device, so this set is both small and ready -- unlike a sysfs-wide scan, + which races re-enumerations from other boards' jobs. Requires the libmtp-runtime + package. + """ + for marker_name in glob.glob(_MARKER_GLOB): + marker = Path(marker_name) + try: + dev = _SYS_USB / marker.name[len('libmtp-'):] + # vid/pid first: lock-free descriptor fields, so they rule out every other + # device before the `serial` read, which the kernel serves under the device + # lock a wedged usbfs ioctl would hold + if ((dev / 'idVendor').read_text().strip() != 'cafe' + or (dev / 'idProduct').read_text().strip() != '4017'): + continue + # bounded: this one CAN block, and a wedged neighbour shares the vid/pid above + serial = _bounded_read(dev / 'serial') + if serial is None or serial.lower() != uid.lower(): + continue + busnum = int((dev / 'busnum').read_text()) + devnum = int((dev / 'devnum').read_text()) + node = _USB_DEV / f'{busnum:03d}' / f'{devnum:03d}' + if marker.resolve(strict=True) != node or not os.access(node, os.R_OK | os.W_OK): + continue + return busnum, devnum + except (OSError, ValueError): + # a marker can vanish while another board flashes: not our device's problem + continue + return None + + +def _gvfs_unmount(uid: str, deadline: float) -> None: + """Drop any gvfs claim on this device, immediately before opening it. + + Called only once the udev marker exists. gvfs claims an MTP device AFTER udev + probing, so before the marker there is nothing to unmount: an earlier call is a + guaranteed no-op that still forks a process, and it leaves the gap between the + unmount and the open unprotected -- the hang this exists to prevent. Per-iteration + calls also forked one gio per second of the enumeration budget. + """ + # Popen, not run(timeout=): run's post-timeout reap is an unbounded wait(), and a gio + # blocked in D state on a wedged usbfs node does not die on SIGKILL, so run(timeout=2) + # can hang for good. Bounded by at most HALF of what is LEFT of our own budget, never + # a fixed sub-bound: the parent gives us --timeout 8 (4 on a retry), so anything larger + # collapsed the poll loop to one attempt and made a slow gio look like a wedged session. + gio_bound = max(0.5, min(3.0, (deadline - time.monotonic()) / 2)) + try: + # argv, not shell=True: uid comes from a hand-edited roster and is board firmware + # output, so a space or $(...) would unmount the wrong URI (leaving the gvfs mount + # held) or run as us. + gio = subprocess.Popen(['gio', 'mount', '-u', + f'mtp://TinyUsb_TinyUsb_Device_{uid}/'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, start_new_session=True) + except OSError: + # glib2.0-bin absent (ci.lan has no gio at all): nothing holds a gvfs mount + # either, so go straight on to the open. + return + try: + gio.wait(timeout=gio_bound) + except subprocess.TimeoutExpired: + try: + os.killpg(gio.pid, signal.SIGKILL) + except OSError: + gio.kill() + try: + gio.wait(timeout=2) # reap it: an abandoned gio leaves a zombie + except subprocess.TimeoutExpired: + pass + print('gio unmount timed out; continuing', file=sys.stderr) + + +def open_mtp_dev(uid: str, timeout: float): + mtp = MTP() + deadline = time.monotonic() + timeout + while True: + try: + # pymtp raises USB_LAYER/PTP_LAYER/GENERAL/AlreadyConnected on a board still + # settling right after a flash; an unguarded raise would skip the rest of the + # enumeration budget (and the disconnect) instead of retrying. + # + # Never detect_devices(): that PROBES every MTP device on the rig, so a board + # still initialising in a parallel job answers our scan (the race #3790 fixed). + # libmtp-runtime publishes /dev/libmtp- only after its own mtp-probe + # has accepted a device, so start from that small, ready-only set and open OUR + # device directly by bus/dev address. + target = _ready_marker(uid) + if target: + # ready first, THEN unmount, then open -- see _gvfs_unmount + _gvfs_unmount(uid, deadline) + busnum, devnum = target + # TinyUSB needs no libmtp quirks, so the raw entry can be built here + entry = LIBMTP_DeviceEntry(None, 0xcafe, None, 0x4017, 0) + raw = LIBMTP_RawDevice(entry, busnum, devnum) + mtp.device = mtp.mtp.LIBMTP_Open_Raw_Device(ctypes.byref(raw)) + if mtp.device: + serial = mtp.get_serialnumber() + if (serial.decode('utf-8') if serial else '').lower() == uid.lower(): + return mtp + mtp.disconnect() + except Exception as e: + print(f'mtp poll: {type(e).__name__}: {e}', file=sys.stderr) + # only when a device was actually opened: pymtp's `self.device == None` + # guard does NOT catch a ctypes NULL pointer (falsy, but != None), so + # disconnecting blindly calls LIBMTP_Release_Device(NULL) + if getattr(mtp, 'device', None): + try: + mtp.disconnect() + except Exception: + pass + mtp.device = None + if time.monotonic() >= deadline: + return None + time.sleep(1) + + +def run_session(uid: str, timeout: float) -> int: + mtp = open_mtp_dev(uid, timeout) + if mtp is None or mtp.device is None: + print('MTP device not found') + return 1 + + try: + assert b"TinyUSB" == mtp.get_manufacturer(), 'MTP wrong manufacturer' + assert b"MTP Example" == mtp.get_modelname(), 'MTP wrong model' + assert b'1.0' == mtp.get_deviceversion(), 'MTP wrong version' + assert b'TinyUSB MTP' == mtp.get_devicename(), 'MTP wrong device name' + + f1 = uid.encode("utf-8") + b'_file1' + f2 = uid.encode("utf-8") + b'_file2' + f3 = uid.encode("utf-8") + b'_file3' + mtp.get_file_to_file(1, f1) + with open(f1, 'rb') as file: + f1_data = file.read() + os.remove(f1) + assert f1_data == FILE1_EXPECT, 'MTP file1 wrong data' + mtp.get_file_to_file(2, f2) + with open(f2, 'rb') as file: + f2_data = file.read() + os.remove(f2) + assert FILE2_MD5_EXPECT == hashlib.md5(f2_data).hexdigest(), 'MTP file2 wrong data' + with open(f3, "wb") as file: + # 1524-byte payload + 12-byte MTP header = 3 full 512-byte buffers, so this + # exercises delivery of the final OUT payload before its ZLP. Deliberate and + # FIXED: a random size hits that boundary in ~0.2% of runs, which is not a test + # of it. Deterministic content so a mismatch is reproducible. + f3_data = bytes((i % 251) + 1 for i in range(1524)) + file.write(f3_data) + file.close() + fid = mtp.send_file_from_file(f3, b'file3') + f3_readback = f3 + b'_readback' + mtp.get_file_to_file(fid, f3_readback) + with open(f3_readback, 'rb') as f: + f3_rb_data = f.read() + os.remove(f3_readback) + assert f3_rb_data == f3_data, 'MTP file3 wrong data' + os.remove(f3) + mtp.delete_object(fid) + except AssertionError as e: + print(e) + return 1 + finally: + mtp.disconnect() + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument('--uid', required=True, help='board_get_unique_id serial to match') + parser.add_argument('--timeout', type=float, default=30, help='enumeration wait budget (s)') + args = parser.parse_args() + return run_session(args.uid, args.timeout) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/test/hil/test/stubs/pymtp.py b/test/hil/test/stubs/pymtp.py new file mode 100644 index 000000000..2720321f6 --- /dev/null +++ b/test/hil/test/stubs/pymtp.py @@ -0,0 +1,125 @@ +# SPDX-License-Identifier: MIT +# Fake pymtp for the hil unit tests — stands in both for the import (GitHub's bare +# pre-commit runner has no libmtp/pymtp) and for a scripted MTP device. Behavior is +# driven by env vars so subprocesses (mtp_test.py under run_cmd) can be steered: +# FAKE_PYMTP_MODE absent (default) | ok | hang +# FAKE_PYMTP_UID serial number the fake device reports +# FAKE_PYMTP_FILE1 text served as file id 1 (README.TXT) +# FAKE_PYMTP_LOGO path to the logo bytes served as file id 2 +# File contents come from env, not constants: the test extracts them from the example's +# own sources, so this stub cannot drift out of sync with the firmware. +# 'hang' blocks forever inside detect_devices — the in-process libmtp equivalent of a +# D-state usbfs ioctl on a wedged device. +import ctypes +import os +import time + + +class NotConnected(Exception): + pass + + +class LIBMTP_DeviceEntry(ctypes.Structure): + """Real pymtp exposes this; mtp_test builds one to open a KNOWN device instead of + probing every MTP device on the bus.""" + _fields_ = [('vendor', ctypes.c_char_p), ('vendor_id', ctypes.c_uint16), + ('product', ctypes.c_char_p), ('product_id', ctypes.c_uint16), + ('device_flags', ctypes.c_uint32)] + + +class LIBMTP_RawDevice(ctypes.Structure): + _fields_ = [('device_entry', LIBMTP_DeviceEntry), ('bus_location', ctypes.c_uint32), + ('devnum', ctypes.c_uint8)] + + +class _LibShim: + @staticmethod + def LIBMTP_Open_Raw_Device(_ref): + # mtp_test no longer calls detect_devices() (it probed every MTP device on the + # rig), so the scripted modes have to act here -- this is the only libmtp entry + # point the marker-based open goes through. + mode = os.environ.get('FAKE_PYMTP_MODE', 'absent') + if mode == 'hang': + time.sleep(10000) + if mode == 'error': + raise RuntimeError('CommandFailed: LIBMTP_ERROR_USB_LAYER') + if mode == 'error_then_ok': + flag = os.environ.get('FAKE_PYMTP_ERRED_MARKER', '/tmp/.fake_pymtp_erred') + if not os.path.exists(flag): + open(flag, 'w').close() + raise RuntimeError('CommandFailed: LIBMTP_ERROR_PTP_LAYER') + if mode == 'absent': + return ctypes.POINTER(ctypes.c_int)() # NULL: nothing to open + # the real one has restype POINTER(LIBMTP_MTPDevice): a failed open returns a + # NULL pointer, which is FALSY but compares unequal to None -- the distinction + # mtp_test's `if mtp.device:` guards depend on + if os.environ.get('FAKE_PYMTP_OPEN') == 'null': + return ctypes.POINTER(ctypes.c_int)() + return 1 + + +class MTP: + def __init__(self): + self.mtp = _LibShim() + self.device = None + self._sent = {} + self._next_id = 3 + + def detect_devices(self): + mode = os.environ.get('FAKE_PYMTP_MODE', 'absent') + if mode == 'hang': + time.sleep(10000) + if mode == 'error': + # real pymtp raises for USB_LAYER/PTP_LAYER/GENERAL/AlreadyConnected; + # the first poll after a flash routinely hits one + raise RuntimeError('CommandFailed: LIBMTP_ERROR_USB_LAYER') + if mode == 'error_then_ok': + if not getattr(self, '_erred', False): + self._erred = True + raise RuntimeError('CommandFailed: LIBMTP_ERROR_USB_LAYER') + return [ctypes.c_int(1)] + if mode != 'ok': + return [] + return [ctypes.c_int(1)] + + def get_serialnumber(self): + return os.environ.get('FAKE_PYMTP_UID', '').encode() + + def get_manufacturer(self): + return b'TinyUSB' + + def get_modelname(self): + return b'MTP Example' + + def get_deviceversion(self): + return b'1.0' + + def get_devicename(self): + return b'TinyUSB MTP' + + def get_file_to_file(self, fid, path): + if fid == 1: + data = os.environ['FAKE_PYMTP_FILE1'].encode() + elif fid == 2: + with open(os.environ['FAKE_PYMTP_LOGO'], 'rb') as f: + data = f.read() + else: + data = self._sent[fid] + with open(path, 'wb') as f: + f.write(data) + + def send_file_from_file(self, path, _name): + with open(path, 'rb') as f: + self._sent[self._next_id] = f.read() + self._next_id += 1 + return self._next_id - 1 + + def delete_object(self, fid): + del self._sent[fid] + + def disconnect(self): + # vendored pymtp raises when nothing is connected; a stub that silently accepts + # it hides a LIBMTP_Release_Device(NULL) call on real hardware + if self.device is None: + raise NotConnected('no device connected') + self.device = None diff --git a/test/hil/test/test_hil_bounded.py b/test/hil/test/test_hil_bounded.py new file mode 100644 index 000000000..908a142d5 --- /dev/null +++ b/test/hil/test/test_hil_bounded.py @@ -0,0 +1,1701 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests proving hil_test's storage and MTP helpers cannot hang the worker: a +# wedged device blocks the call in D state forever (child process or in-process ioctl), +# so these paths go through a bounded runner. Fakes stand in for the wedge (a real one +# cannot be manufactured on demand): a PATH-injected `mtype` script and a +# PYTHONPATH-injected `pymtp` module, each with a mode that blocks forever. +# Scope: mtype, the gio unmount, the libmtp session, the arecord/iperf reaps, and the +# printer read (a process now, via run_alongside, so a killed reader takes its fd with +# it -- usblp allows ONE opener, and a blocked thread kept the node for the worker's life). +# Known residue (unbounded, backstopped only by the pool guard): hid open/write and +# midi's read(64). +# +# hil_test imports pyserial, which GitHub's bare pre-commit runner does not have — so +# an inert serial module is stubbed into sys.modules BEFORE the import (nothing here +# exercises serial paths). MTP traffic never touches hil_test: it all goes through the +# mtp_test.py subprocess, which gets the fake pymtp via PYTHONPATH. +# Run directly: +# python3 test/hil/test/test_hil_bounded.py +import os +import stat +import sys +import threading +from multiprocessing import TimeoutError as MpTimeoutError +import time +import types +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory + +TEST_DIR = os.path.dirname(os.path.abspath(__file__)) +# the modules under test live in the parent dir (test/hil), not here +sys.path.insert(0, os.path.dirname(TEST_DIR)) + +serial_stub = types.ModuleType('serial') +serial_stub.Serial = type('Serial', (), {}) +serial_stub.SerialException = type('SerialException', (Exception,), {}) +serial_stub.SerialTimeoutException = type('SerialTimeoutException', (Exception,), {}) +sys.modules.setdefault('serial', serial_stub) +import hil_flash +import hil_test + + +def write_script(path: Path, body: str) -> None: + path.write_text('#!/bin/sh\n' + body + '\n') + path.chmod(path.stat().st_mode | stat.S_IEXEC) + + +def run_bounded(fn, timeout: float): + """Run fn in a daemon thread; return (finished, exception). A still-running thread is + the hang under test — leave it to die with the interpreter.""" + exc = [] + + def wrapper(): + try: + fn() + except BaseException as e: # noqa: BLE001 - tests inspect the exception + exc.append(e) + + t = threading.Thread(target=wrapper, daemon=True) + t.start() + t.join(timeout) + return not t.is_alive(), exc[0] if exc else None + + +@unittest.skipIf(os.name == 'nt', 'POSIX shell fakes') +class ReadDiskFile(unittest.TestCase): + def setUp(self): + self.tmp = TemporaryDirectory() + tmp = Path(self.tmp.name) + self.addCleanup(self.tmp.cleanup) + # fake block device node: get_disk_dev is patched to this existing path + self.dev = tmp / 'fakedev' + self.dev.write_bytes(b'') + # addCleanup, not tearDown: tearDown does NOT run when setUp raises, and a leaked + # PATH entry points at a temp bin dir this class already deleted. + for name in ('get_disk_dev', '_enum_timeout', 'MTYPE_TIMEOUT'): + self.addCleanup(setattr, hil_test, name, getattr(hil_test, name)) + hil_test.get_disk_dev = lambda uid, vendor, lun: str(self.dev) + hil_test._enum_timeout = 2 + self.bin = tmp / 'bin' + self.bin.mkdir() + self.addCleanup(os.environ.__setitem__, 'PATH', os.environ['PATH']) + os.environ['PATH'] = f'{self.bin}:{os.environ["PATH"]}' + self.pidfile = tmp / 'mtype.pid' + self.addCleanup(self._reap_mtype) + + def _reap_mtype(self): + if self.pidfile.exists(): # reap a leaked hang-mode mtype + try: + os.kill(int(self.pidfile.read_text()), 9) + except (OSError, ValueError): + pass + + def test_returns_exact_bytes_despite_stderr_noise(self): + # \377 is invalid UTF-8 and stderr noise must not leak into the data + write_script(self.bin / 'mtype', r"printf 'R\377EADME-DATA'; printf 'vfat warning' >&2") + data = hil_test.read_disk_file('uid0', 0, 'README.TXT') + self.assertEqual(data, b'R\xffEADME-DATA') + + def test_failure_message_carries_mtype_stderr_and_fname(self): + write_script(self.bin / 'mtype', "printf 'mtype: cannot read' >&2; exit 1") + with self.assertRaises(AssertionError) as cm: + hil_test.read_disk_file('uid0', 0, 'README.TXT') + self.assertIn('cannot read', str(cm.exception)) + self.assertIn('README.TXT', str(cm.exception)) + + def test_empty_read_fails_immediately_with_fname(self): + # rc 0 with no data is a real answer (bad sectors, empty file), not "not ready": + # fail at once like the old assert did, naming the file — don't spin the budget + write_script(self.bin / 'mtype', 'exit 0') + t0 = time.monotonic() + with self.assertRaises(AssertionError) as cm: + hil_test.read_disk_file('uid0', 0, 'README.TXT') + self.assertLess(time.monotonic() - t0, 1.5) + self.assertIn('README.TXT', str(cm.exception)) + + def test_hung_mtype_cannot_hang_the_worker(self): + # a D-state child never exits; the bounded runner must give up without it + write_script(self.bin / 'mtype', f'echo $$ > {self.pidfile}; exec sleep 1000') + hil_test.MTYPE_TIMEOUT = 2 + finished, exc = run_bounded(lambda: hil_test.read_disk_file('uid0', 0, 'README.TXT'), 20) + self.assertTrue(finished, 'read_disk_file hung on a stuck mtype') + self.assertIsInstance(exc, AssertionError) + + +class CompactOutput(unittest.TestCase): + def test_strips_workflow_command_markers(self): + """Defense-in-depth: the historical marker source was worker-side run_cmd + (now suppressed at the emitter); anything future that pipes markers into a + captured stdout would land them mid-row where GitHub renders them literally.""" + raw = '::group::COMMAND TIMEOUT (1s): x\nboom\n::endgroup::\ntail' + self.assertEqual(hil_test.compact_output(raw), 'COMMAND TIMEOUT (1s): x | boom | tail') + + +class UsbtestRecovery(unittest.TestCase): + def test_recovery_flags_and_flash_bound_fit_the_reserve(self): + """The post-hang reflash plumbing: the CLI flags exist, and the bounded reflash + plus the fixed recovery costs (60s case timeout + 5s kill wait + 5s settle) + fits inside USBTEST_RECOVERY_BUDGET -- otherwise the outer run_cmd kill lands + mid-flash and orphans the flasher (own session) on the probe.""" + import subprocess + hil_dir = Path(TEST_DIR).parents[0] + r = subprocess.run([sys.executable, str(hil_dir / 'usbtest.py'), '--help'], + capture_output=True, text=True, timeout=30) + self.assertEqual(r.returncode, 0, r.stderr) + for flag in ('--recover-board', '--recover-fw', '--outer-timeout'): + self.assertIn(flag, r.stdout) + + def test_the_bounded_reflash_actually_fits_the_reserve(self): + """The arithmetic the docstring above claims but never checked -- the two + constants never met in any test, so bumping either silently broke the promise. + Overrun means run_cmd's outer kill lands MID-FLASH and orphans the flasher + (start_new_session, so killpg misses it) holding the probe.""" + import re + import usbtest + hil_dir = Path(TEST_DIR).parents[0] + # read the case timeout hil_test actually passes, so this cannot drift silently + src = (hil_dir / 'hil_test.py').read_text() + m = re.search(r'--timeout (\d+) --budget', src) + self.assertIsNotNone(m, 'usbtest invocation changed shape; re-derive this bound') + case_timeout = int(m.group(1)) + kill_wait, settle, time_left_reserve = 5, 5, 35 # usbtest.py's fixed costs + worst = (case_timeout + kill_wait + usbtest.RECOVER_FLASH_TIMEOUT + + settle + time_left_reserve) + self.assertLessEqual( + worst, hil_test.USBTEST_RECOVERY_BUDGET, + f'a HUNG case needs {worst}s to recover but only ' + f'{hil_test.USBTEST_RECOVERY_BUDGET}s is reserved') + + +class UsbtestRunHelper(unittest.TestCase): + """usbtest.run() is the bounded replacement for subprocess.run: sysfs_write feeds it + input=, and every battery calls that before case 1.""" + + def setUp(self): + import usbtest + self.usbtest = usbtest + + def test_input_kwarg_is_honoured(self): + r = self.usbtest.run(['cat'], input='payload', timeout=10) + self.assertEqual(r.returncode, 0) + self.assertEqual(r.stdout, 'payload') + + def test_capture_output_kwarg_is_accepted(self): + r = self.usbtest.run(['printf', 'x'], capture_output=True, timeout=10) + self.assertEqual(r.stdout, 'x') + + def test_timeout_is_bounded_and_raises(self): + import subprocess + t0 = time.monotonic() + with self.assertRaises(subprocess.TimeoutExpired): + self.usbtest.run(['sleep', '30'], timeout=1) + self.assertLess(time.monotonic() - t0, 15) + + +class BuildBoardContract(unittest.TestCase): + def test_every_return_path_is_a_pair(self): + """main() unpacks `_, nfail = build_board(board)`; a bare int on any path + (the timeout path did) raises TypeError before the pool exists.""" + import ast + src = (Path(TEST_DIR).parents[0] / 'hil_test.py').read_text() + fn = next(n for n in ast.walk(ast.parse(src)) + if isinstance(n, ast.FunctionDef) and n.name == 'build_board') + for node in ast.walk(fn): + if isinstance(node, ast.Return) and node.value is not None: + self.assertIsInstance(node.value, ast.Tuple, + f'build_board returns a non-tuple at line {node.lineno}') + + +class RemoteStaging(unittest.TestCase): + def test_import_closure_is_staged_to_the_rig(self): + # hil_ci.sh stages an explicit scp whitelist; a module that is not on it exists + # locally and in CI checkouts but silently never reaches the remote rig (how + # mtp_test.py was first missed). Walk the local-import closure of everything + # the rig executes and require each file's exact scp entry — a bare-substring + # match would be satisfied by a mention in a comment or the run line. + import ast + hil_dir = Path(TEST_DIR).parents[0] + staged = (hil_dir / 'hil_ci.sh').read_text() + + def imported_paths(pyfile): + # ast, not regex: an earlier regex walker went silently vacuous on a + # multi-line import. ast also sees function-local deferred imports + # (usbtest.py's `import hil_flash` inside the recovery branch). + for node in ast.walk(ast.parse(pyfile.read_text())): + if isinstance(node, ast.Import): + for a in node.names: + yield a.name.replace('.', '/') + '.py' + elif isinstance(node, ast.ImportFrom) and node.module: + if node.module == 'helper': + for a in node.names: + yield f'helper/{a.name}.py' + else: + yield node.module.replace('.', '/') + '.py' + + seeds = ['hil_test.py', 'usbtest.py', 'mtp_test.py'] # CLI + spawned helpers + for f in seeds: # a renamed seed must fail loudly, not fall out of the walk + self.assertTrue((hil_dir / f).exists(), f'stale RemoteStaging seed: {f}') + todo, seen = list(seeds), set() + while todo: + f = todo.pop() + if f in seen or not (hil_dir / f).exists(): + continue # stdlib/site-packages imports have no test/hil file + seen.add(f) + todo += list(imported_paths(hil_dir / f)) + for f in sorted(seen): + self.assertIn(f'"$ROOT_DIR/test/hil/{f}"', staged, + f'{f} runs on the rig but hil_ci.sh does not scp it') + + +class _MtpFakeRig: + """The fake rig shared by the MTP cases: a udev-marker tree under one tmp root and + the scripted pymtp on PYTHONPATH. A plain mixin, NOT a TestCase -- subclassing a + TestCase to reuse a fixture re-runs every inherited test in each subclass.""" + + @classmethod + def setUpClass(cls): + # both file fixtures come from the example's sources, so drift there fails here: + # file id 1 is README.TXT (C define), file id 2 is logo.png (C byte array) + import hashlib + import re + src = Path(TEST_DIR).parents[2] / 'examples/device/mtp/src' + m = re.search(r'#define README_TXT_CONTENT "([^"]+)"', (src / 'mtp_fs_example.c').read_text()) + assert m, 'README_TXT_CONTENT define not found in mtp_fs_example.c' + cls.readme = m.group(1) + data = bytes(int(x, 16) for x in + re.findall(r'0x([0-9a-fA-F]{2})', (src / 'tinyusb_logo_png.h').read_text())) + assert hashlib.md5(data).hexdigest() == '40ef23fc2891018d41a05d4a0d5f822f' + cls.logo = data + + def setUp(self): + self.tmp = TemporaryDirectory() + tmp = Path(self.tmp.name) + self.addCleanup(self.tmp.cleanup) + logo = tmp / 'logo.bin' + logo.write_bytes(self.logo) + self.board = {'uid': 'CAFE01', 'name': 'fakeboard'} + # addCleanup, not tearDown: tearDown does NOT run when setUp raises, and a leaked + # chdir into a deleted temp dir breaks every test after it. + self.saved_env = {k: os.environ.get(k) for k in + ('FAKE_PYMTP_MODE', 'FAKE_PYMTP_UID', 'FAKE_PYMTP_LOGO', + 'FAKE_PYMTP_FILE1', 'PYTHONPATH', 'PYTHONSAFEPATH', + 'HIL_MTP_FAKE_ROOT', 'FAKE_PYMTP_ERRED_MARKER')} + self.addCleanup(self._restore_env) + # A udev-ready marker tree: libmtp-runtime publishes /dev/libmtp- only + # after mtp-probe accepts a device, and mtp_test opens THAT device directly rather + # than probing every MTP device on the rig (the parallel-probe race #3790 fixed). + # mirrors the real layout under one root, so /sys/bus/usb/devices/1-1 reads + # as the stand-in for /sys/bus/usb/devices/1-1 that it is + dev = tmp / 'sys/bus/usb/devices/1-1' + usbdev = tmp / 'dev/bus/usb/001' + markers = tmp / 'dev' # created by usbdev's parents=True + dev.mkdir(parents=True); usbdev.mkdir(parents=True) + (dev / 'idVendor').write_text('cafe\n') + (dev / 'idProduct').write_text('4017\n') + (dev / 'serial').write_text(self.board['uid'] + '\n') + (dev / 'busnum').write_text('1\n') + (dev / 'devnum').write_text('2\n') + node = usbdev / '002' + node.write_bytes(b'') + (markers / 'libmtp-1-1').symlink_to(node) + os.environ['HIL_MTP_FAKE_ROOT'] = str(tmp) + os.environ['FAKE_PYMTP_ERRED_MARKER'] = str(tmp / 'erred') + os.environ['FAKE_PYMTP_UID'] = self.board['uid'] + os.environ['FAKE_PYMTP_LOGO'] = str(logo) + os.environ['FAKE_PYMTP_FILE1'] = self.readme + stubs = os.path.join(TEST_DIR, 'stubs') + pp = self.saved_env['PYTHONPATH'] + os.environ['PYTHONPATH'] = stubs if not pp else f'{stubs}:{pp}' + # pymtp is vendored next to mtp_test.py, and a script's own dir (sys.path[0]) + # outranks PYTHONPATH — safe-path mode (3.11+) drops it so the fake wins there + os.environ['PYTHONSAFEPATH'] = '1' + for name in ('_enum_timeout', 'MTP_SESSION_MARGIN'): + self.addCleanup(setattr, hil_test, name, getattr(hil_test, name)) + hil_test._enum_timeout = 2 + # the session scratch files land in cwd + self.addCleanup(os.chdir, os.getcwd()) + os.chdir(tmp) + + def _restore_env(self): + for k, v in self.saved_env.items(): + if v is None: + os.environ.pop(k, None) + else: + os.environ[k] = v + + +@unittest.skipIf(os.name == 'nt', 'POSIX shell fakes') +@unittest.skipIf(sys.version_info < (3, 11), 'fake-pymtp steering needs PYTHONSAFEPATH') +class DeviceMtp(_MtpFakeRig, unittest.TestCase): + """test_device_mtp end to end: the real mtp_test.py subprocess under run_cmd, + with the scripted pymtp fake steered in via PYTHONPATH.""" + + def test_mtp_session_passes_against_scripted_device(self): + os.environ['FAKE_PYMTP_MODE'] = 'ok' + hil_test.test_device_mtp(self.board) # no exception + + def test_absent_device_fails_cleanly(self): + os.environ['FAKE_PYMTP_MODE'] = 'absent' + finished, exc = run_bounded(lambda: hil_test.test_device_mtp(self.board), 30) + self.assertTrue(finished) + self.assertIsInstance(exc, AssertionError) + self.assertIn('MTP device not found', str(exc)) + + def test_libmtp_error_on_one_poll_retries_instead_of_dying(self): + """pymtp raises for USB_LAYER/PTP_LAYER errors -- routine on the first poll + after a flash. An unguarded raise skipped the whole enumeration budget.""" + os.environ['FAKE_PYMTP_MODE'] = 'error_then_ok' + hil_test.test_device_mtp(self.board) # retries past the error, then passes + + def test_libmtp_error_every_poll_fails_cleanly(self): + os.environ['FAKE_PYMTP_MODE'] = 'error' + finished, exc = run_bounded(lambda: hil_test.test_device_mtp(self.board), 30) + self.assertTrue(finished) + self.assertIsInstance(exc, AssertionError) + + def test_hung_mtp_stack_cannot_hang_the_worker(self): + # in-process libmtp blocking in a usbfs ioctl (D state) hangs whatever thread + # made the call, forever — the session must be somewhere disposable + os.environ['FAKE_PYMTP_MODE'] = 'hang' + hil_test.MTP_SESSION_MARGIN = 3 + finished, exc = run_bounded(lambda: hil_test.test_device_mtp(self.board), 25) + self.assertTrue(finished, 'test_device_mtp hung on a wedged MTP stack') + self.assertIsInstance(exc, AssertionError) + + +class ConvoySafeFlasher(unittest.TestCase): + """hil_flash.convoy_safe decides whether a board gets post-HUNG recovery at all. + + It must be true ONLY for flashers that can reach their probe without opening the + poisoned usbfs node: openocd pinned with a roster vid_pid (filters on kernel-cached + sysfs descriptors) and esptool (delivers to a named tty, never enumerates usbfs). + Anything else enumerates by opening nodes, would block in D state on the wedged one + and become a second stray -- JLinkExe included, whose selection is serial-only and + so cannot be pinned at all.""" + + def setUp(self): + import hil_flash + self.f = hil_flash.convoy_safe + + def test_pinned_openocd_is_safe(self): + self.assertTrue(self.f({'name': 'openocd', 'vid_pid': '0x2e8a 0x000c'})) + + def test_unpinned_openocd_is_not(self): + self.assertFalse(self.f({'name': 'openocd'})) + self.assertFalse(self.f({'name': 'openocd', 'vid_pid': ''})) + + def test_esptool_is_safe_without_a_pin(self): + """Delivery is `-p `; there is no usbfs walk to poison.""" + self.assertTrue(self.f({'name': 'esptool'})) + + def test_enumerating_flashers_are_not(self): + for name in ('jlink', 'stlink', 'lm4flash', 'dfu-util'): + self.assertFalse(self.f({'name': name, 'vid_pid': '0x1366 0x1024'}), + f'{name} must not be treated as convoy-safe') + + def test_missing_or_odd_name_is_not_safe(self): + for flasher in ({}, {'name': None}, {'name': ''}): + self.assertFalse(self.f(flasher)) + + +class BoundedOpen(unittest.TestCase): + """hil_util.bounded_open must return rather than block, and must not leak the fd if + the open completes after we gave up (usblp_open takes the device mutex before it + consults O_NONBLOCK, so a wedged node blocks the open uninterruptibly).""" + + def setUp(self): + from helper import hil_util + self.hil_util = hil_util + self.tmp = TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + # bounded_open counts its stranded threads now, and the counter is process-global + # with no decrement: three wedged-FIFO tests here reach SYSFS_STUCK_MAX and every + # later test in this file reads SYSFS_UNKNOWN for perfectly good attributes + self.addCleanup(setattr, hil_util, '_sysfs_stuck', hil_util._sysfs_stuck) + + def test_opens_a_normal_file(self): + f = Path(self.tmp.name) / 'plain' + f.write_text('x') + fd = self.hil_util.bounded_open(str(f), os.O_RDONLY, 5) + self.assertIsNotNone(fd) + os.close(fd) + + def test_missing_path_returns_none_without_raising(self): + self.assertIsNone(self.hil_util.bounded_open( + str(Path(self.tmp.name) / 'nope'), os.O_RDONLY, 5)) + + @unittest.skipIf(os.name == 'nt', 'POSIX fifo') + def test_blocking_open_gives_up_and_does_not_leak_fds(self): + """A reader-less FIFO blocks open(O_WRONLY) forever -- the closest portable + stand-in for a wedged usblp node.""" + fifo = Path(self.tmp.name) / 'fifo' + os.mkfifo(fifo) + before = len(os.listdir('/proc/self/fd')) + t0 = time.monotonic() + for _ in range(5): + self.assertIs(self.hil_util.bounded_open(str(fifo), os.O_WRONLY, 0.2), + self.hil_util.SYSFS_UNKNOWN) + self.assertLess(time.monotonic() - t0, 10, 'bounded_open did not bound') + self.assertLessEqual(len(os.listdir('/proc/self/fd')) - before, 1, + 'bounded_open leaked fds on the blocking path') + + @unittest.skipIf(os.name == 'nt', 'POSIX fifo') + def test_open_completing_during_the_abandon_does_not_leak(self): + """The window the handoff lock exists for: the worker is at its store-or-close + decision when the caller gives up and drains the box. + + The `abandoned` Event is instrumented to park the worker there, because timing + alone never reaches that window -- 1500 tries against the unlocked version leaked + nothing, so a test that merely completes the open late proves nothing. An empty + `hit` means the instrumentation no longer bites and the window is untested.""" + hil_util = self.hil_util + fifo = Path(self.tmp.name) / 'fifo' + os.mkfifo(fifo) + caller = threading.current_thread() + drained, hit = threading.Event(), [] + + class RacingEvent(threading.Event): + def is_set(self): + v = super().is_set() + if not v and not hit and threading.current_thread() is not caller: + hit.append(True) + # bounded: the fixed bounded_open holds the lock across this call, so + # the caller cannot reach its abandon (and set drained) until we return + drained.wait(0.3) + return v + + shim = types.ModuleType('threading_shim') + shim.__dict__.update(threading.__dict__) + shim.Event = RacingEvent + hil_util.threading = shim + self.addCleanup(setattr, hil_util, 'threading', threading) + + before = len(os.listdir('/proc/self/fd')) + rd = os.open(fifo, os.O_RDONLY | os.O_NONBLOCK) # the O_WRONLY open completes at once + try: + self.assertIs(hil_util.bounded_open(str(fifo), os.O_WRONLY, 0.05), + hil_util.SYSFS_UNKNOWN) + drained.set() + time.sleep(0.1) # let an abandoned worker act on what it saw + self.assertTrue(hit, 'the abandon window was never entered') + self.assertLessEqual(len(os.listdir('/proc/self/fd')) - before, 1, + 'bounded_open stored the fd after the caller drained the box') + finally: + drained.set() + os.close(rd) + + +class SysfsUnknownIsNotAbsent(unittest.TestCase): + """read_sysfs must tell "no such attribute" (a fact) from "the read did not answer" + (not a fact). Every caller that concluded absence from the latter reported a healthy + board as a firmware regression.""" + + def setUp(self): + from helper import hil_util + self.hil_util = hil_util + self.saved = (hil_util._sysfs_stuck, hil_util._sysfs_blind_logged) + self.tmp = TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + + def tearDown(self): + # a blocked read strands a counted daemon thread; leaving the count raised would + # blind every later test in this process + self.hil_util._sysfs_stuck, self.hil_util._sysfs_blind_logged = self.saved + + def test_readable_attribute_returns_its_value(self): + p = Path(self.tmp.name) / 'serial' + p.write_text('CAFE01\n') + self.assertEqual(self.hil_util.read_sysfs(str(p)), 'CAFE01') + + def test_missing_attribute_is_none(self): + self.assertIsNone(self.hil_util.read_sysfs(str(Path(self.tmp.name) / 'nope'))) + + @unittest.skipIf(os.name == 'nt', 'POSIX fifo') + def test_blocking_read_is_unknown_not_absent(self): + """A reader-less FIFO stands in for the wedged device whose sysfs read never + returns; None here would read as "the board is gone".""" + fifo = Path(self.tmp.name) / 'fifo' + os.mkfifo(fifo) + t0 = time.monotonic() + v = self.hil_util.read_sysfs(str(fifo), grace=0.3) + self.assertLess(time.monotonic() - t0, 10, 'read_sysfs did not bound') + self.assertIs(v, self.hil_util.SYSFS_UNKNOWN) + self.assertIsNotNone(v) + + def test_blind_process_answers_unknown_for_a_readable_attribute(self): + p = Path(self.tmp.name) / 'serial' + p.write_text('CAFE01') + self.hil_util._sysfs_stuck = self.hil_util.SYSFS_STUCK_MAX + self.assertTrue(self.hil_util.sysfs_blind()) + self.assertIs(self.hil_util.read_sysfs(str(p)), self.hil_util.SYSFS_UNKNOWN) + self.assertIn('blind', self.hil_util.sysfs_blind_note()) + + def test_unknown_is_falsy_but_not_none(self): + # call sites use `(v or '')` idioms; the sentinel must keep working there while + # still being distinguishable from a real absence + self.assertFalse(self.hil_util.SYSFS_UNKNOWN) + self.assertIsNotNone(self.hil_util.SYSFS_UNKNOWN) + + +class UsbtestEnumerationVerdict(unittest.TestCase): + """test_device_usbtest must not report a healthy board as "no cafe:4010 device" just + because its own sysfs reads stopped answering.""" + + def setUp(self): + from helper import hil_util + self.hil_util = hil_util + self.td = TemporaryDirectory() + self.addCleanup(self.td.cleanup) + # a real device dir: usb_scan reads idVendor/idProduct with a plain open (they are + # lock-free descriptor fields), and only `serial` through the bounded reader + dev = Path(self.td.name) / '1-2' + dev.mkdir() + (dev / 'idVendor').write_text('cafe\n') + (dev / 'idProduct').write_text('4010\n') + (dev / 'serial').write_text('CAFE01\n') + for obj, name, val in ((hil_util, 'read_sysfs', hil_util.read_sysfs), + (hil_util, 'glob', hil_util.glob), + (hil_util, '_sysfs_stranded', {}), + (hil_test, '_enum_timeout', 1)): + self.addCleanup(setattr, obj, name, getattr(obj, name)) + setattr(obj, name, val) + hil_util.glob = types.SimpleNamespace(glob=lambda pat: [str(dev)]) + + def _fail(self, reader): + self.hil_util.read_sysfs = reader + with self.assertRaises(hil_test.TestFail) as cm: + hil_test.test_device_usbtest({'uid': 'CAFE01', 'name': 'fake', 'flasher': {}}) + return str(cm.exception) + + def test_unknown_reads_do_not_claim_the_device_is_absent(self): + msg = self._fail(lambda p, *a, **kw: self.hil_util.SYSFS_UNKNOWN) + self.assertNotIn('no cafe:4010 device', msg) + self.assertIn('did not answer', msg) + + def test_a_readable_bus_without_the_device_still_says_absent(self): + msg = self._fail(lambda p, *a, **kw: 'OTHERUID') + self.assertIn('no cafe:4010 device', msg) + + +class UnresolvedControllerBucket(unittest.TestCase): + """An unresolved controller must budget in ONE bucket. Taking a permit on every slot + serialized the whole fleet the moment a worker went blind.""" + + def setUp(self): + import threading + from helper import hil_lock + self.hil_lock = hil_lock + self.saved = (hil_lock.controller_map, hil_lock.controller_meta, + hil_lock.controller_hints, hil_lock.log) + hil_lock.controller_map, hil_lock.controller_meta = {}, threading.Lock() + hil_lock.controller_hints, hil_lock.log = {}, lambda *a, **k: None + + def tearDown(self): + (self.hil_lock.controller_map, self.hil_lock.controller_meta, + self.hil_lock.controller_hints, self.hil_lock.log) = self.saved + + def _slots(self, uid, warn): + import threading + sems = self.hil_lock.make_permit_sems(threading.Semaphore, 2) + return self.hil_lock.controller_permit(sems, uid, warn_unknown=warn).slots + + def test_unresolved_boards_share_one_slot(self): + for warn in (False, True): + slots = self._slots('NOSUCHUID', warn) + self.assertEqual(len(slots), 1, 'unresolved uid took more than one slot') + self.assertEqual(slots, self._slots('OTHERUID', warn), + 'unresolved boards must share the bucket, not spread over it') + + def test_the_semaphore_array_is_long_enough_for_the_unknown_slot(self): + """UNKNOWN_SLOT indexes one PAST the real slots. An array sized to + CONTROLLER_SLOTS IndexErrors on the first unresolved board, inside a pool worker, + which map_async turns into a total loss of every board's results.""" + import threading + sems = self.hil_lock.make_permit_sems(threading.Semaphore, 2) + self.assertGreater(len(sems), self.hil_lock.UNKNOWN_SLOT) + + def test_the_unknown_bucket_never_lends_a_controller_a_second_budget(self): + """A private FULL budget let 2 unknown batteries join 2 resolved ones on the same + physical controller -- 4 where the width is 2. One at a time caps that at +1.""" + import threading + sems = self.hil_lock.make_permit_sems(threading.Semaphore, 2) + first = self.hil_lock.controller_permit(sems, 'NOSUCHUID') + first.__enter__() + self.addCleanup(first.__exit__) + second = self.hil_lock.controller_permit(sems, 'OTHERUID') + self.assertFalse(sems[second.slots[0]].acquire(blocking=False), + 'a second unresolved board got in alongside the first') + + def test_every_real_slot_keeps_the_full_width(self): + import threading + sems = self.hil_lock.make_permit_sems(threading.Semaphore, 2) + for s in sems[:self.hil_lock.CONTROLLER_SLOTS]: + self.assertTrue(s.acquire(blocking=False) and s.acquire(blocking=False)) + self.assertFalse(s.acquire(blocking=False)) + + +class ThroughputPayloadBound(unittest.TestCase): + """An unknown link speed must pick the FS payload, and each dd must be bounded by the + payload actually requested.""" + + def test_only_a_read_high_speed_gets_the_big_payload(self): + from helper import hil_util + for speed in (None, hil_util.SYSFS_UNKNOWN, '12', '1.5'): + self.assertTrue(hil_test.link_is_fs(speed), f'{speed!r} must scale as FS') + for speed in ('480', '5000', '10000'): + self.assertFalse(hil_test.link_is_fs(speed)) + + def test_dd_bound_scales_with_the_payload_and_stays_bounded(self): + self.assertGreater(hil_test.dd_timeout(16), hil_test.dd_timeout(1)) + self.assertGreaterEqual(hil_test.dd_timeout(1), 30) # setup + flush floor + # still an INNER bound: run_cmd's own timeout must stay the outer one + self.assertLess(hil_test.dd_timeout(16), hil_test.hil_util.CMD_TIMEOUT) + + +class FindDeviceCache(unittest.TestCase): + """usbtest.find_device's cache is keyed by sysname, a bus-topology path: after a + renumber it can name a different cafe:4010 board, and idVendor/idProduct are identical + on every one of them. Only `serial` tells them apart.""" + + def setUp(self): + import usbtest + self.usbtest = usbtest + self.tmp = TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.saved_sys_usb = usbtest.SYS_USB + usbtest.SYS_USB = Path(self.tmp.name) + usbtest._DEV_CACHE.clear() + self._dev('1-2', 'AAAA', devnum=2) + self._dev('1-3', 'BBBB', devnum=3) + + def tearDown(self): + self.usbtest.SYS_USB = self.saved_sys_usb + self.usbtest._DEV_CACHE.clear() + + def _dev(self, sysname, serial, devnum): + d = Path(self.tmp.name) / sysname + d.mkdir() + for name, val in (('idVendor', self.usbtest.VID), ('idProduct', self.usbtest.PID), + ('serial', serial), ('busnum', '1'), ('devnum', str(devnum)), + ('speed', '480'), ('bcdDevice', '0104')): + (d / name).write_text(val + '\n') + + def test_cached_sysname_with_another_boards_serial_is_rejected(self): + self.usbtest._DEV_CACHE['bbbb'] = '1-2' # renumbered: 1-2 is board AAAA now + dev = self.usbtest.find_device('BBBB') + self.assertEqual(dev['sysname'], '1-3') + self.assertEqual(dev['serial'], 'BBBB') + self.assertEqual(self.usbtest._DEV_CACHE['bbbb'], '1-3') + + def test_cached_sysname_with_the_right_serial_is_kept(self): + self.usbtest._DEV_CACHE['bbbb'] = '1-3' + dev = self.usbtest.find_device('BBBB') + self.assertEqual((dev['sysname'], dev['serial']), ('1-3', 'BBBB')) + + def test_a_cached_device_that_vanished_falls_back_to_the_scan(self): + self.usbtest._DEV_CACHE['bbbb'] = '1-9' # gone from sysfs + self.assertEqual(self.usbtest.find_device('BBBB')['sysname'], '1-3') + + +class EnumPollDoesNotReReadAWedgedPath(unittest.TestCase): + """usbtest_enumerated re-globs every device each 0.2 s pass. One wedged peer therefore + strands a fresh bounded reader thread per pass, and SYSFS_STUCK_MAX=4 of those blind + the WHOLE worker for the rest of the run -- measured at 8 s of polling. A path that + already stranded is known-unknown; reading it again buys nothing and costs the + blindness budget.""" + + def test_a_stranded_path_is_read_at_most_once(self): + from contextlib import contextmanager + from helper import hil_lock, hil_util + + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + # A REAL device dir: usb_scan reads idVendor/idProduct with a plain open and + # `continue`s on OSError, so a bare FIFO is skipped before the bounded read is ever + # reached -- this test passed identically with the memo deleted until the ids were + # added. The FIFO must be the `serial` of a device that survives the cheap filter. + devdir = Path(td.name) / '1-2' + devdir.mkdir() + (devdir / 'idVendor').write_text('cafe\n') + (devdir / 'idProduct').write_text('4010\n') + wedged = devdir / 'serial' + os.mkfifo(wedged) # open() blocks forever: no writer, ever + + def patch(obj, name, value): + self.addCleanup(setattr, obj, name, getattr(obj, name)) + setattr(obj, name, value) + + def _permit(uid): + yield + + from helper import hil_util as _hu2 + patch(_hu2, 'glob', types.SimpleNamespace(glob=lambda p: [str(devdir)])) + patch(_hu2, '_sysfs_stranded', {}) + patch(hil_lock, 'usbtest_permit', contextmanager(_permit)) + # Long enough for several 2 s reads, but under the blindness cap -- past the cap + # sysfs_blind() short-circuits reads on its own and would mask the memo entirely. + patch(hil_test, '_enum_timeout', 8) + # the blindness counter is process-global and never decrements: restore it or this + # test blinds every test that runs after it + patch(hil_util, '_sysfs_stuck', hil_util._sysfs_stuck) + + # Count LEAKED THREADS, not _sysfs_stuck: a strand is booked only the first time a + # path is seen, so the counter is deduped by the memo's own bookkeeping and stays 1 + # even when the memo is broken. Each re-read blocks a fresh thread on the FIFO + # forever and leaks its fd -- which is the cost the memo exists to avoid, and the + # only thing here that actually moves when it regresses. + before = threading.active_count() + with self.assertRaises(hil_test.TestFail): # never enumerates, by construction + hil_test.test_device_usbtest({'name': 'b', 'uid': 'UID1', + 'flasher': {'name': 'openocd'}}) + self.assertLessEqual(threading.active_count() - before, 1, + 'the poll re-read a path it already knew was stranded') + + +class ReRunSpecNamesOnlyWhatFailed(unittest.TestCase): + """The pool-guard path used to leave this unwritten -- and a fresh run has already + unlinked it -- so build.yml's re-run step found nothing and GitHub re-tested all ~26 + boards to find the one that wedged.""" + + def test_only_failed_boards_and_their_failed_tests(self): + with TemporaryDirectory() as td: + d = Path(td) + spec = d / 'cfg.failed' + hil_test._write_failed_spec(spec, d, [ + ('good', 0, [], None, 1.0), + ('bad', 2, ['device/cdc_msc'], None, 1.0), + ('wedged', 1, [], None, 0.0), # never reported: no test list + ]) + got = spec.read_text() + self.assertIn('-b bad', got) + self.assertIn('-bt bad:device/cdc_msc', got) + self.assertIn('-b wedged', got) + self.assertNotIn('good', got) + + def test_an_all_green_run_removes_a_stale_spec(self): + with TemporaryDirectory() as td: + d = Path(td) + spec = d / 'cfg.failed' + spec.write_text('--accumulate -b stale') + hil_test._write_failed_spec(spec, d, [('good', 0, [], None, 1.0)]) + self.assertFalse(spec.exists(), 'a stale spec would re-run last time\'s boards') + + +class WedgedPidsFailsClosed(unittest.TestCase): + """A scan that could not SEE the holder must not report "no holder". The holder is + root-owned (run_case uses sudo -n when the node is not writable) and that is exactly + what a hidepid/ProtectProc mount hides — so an unreadable /proc reading as clear + clears unrecovered_hang and lets cleanup unbind a device whose usbfs lock is still + held, which deadlocks the bus rather than one board.""" + + def test_returns_a_completeness_flag_not_just_pids(self): + import usbtest + got = usbtest.wedged_pids('/dev/bus/usb/999/999') + self.assertIsInstance(got, tuple) + self.assertEqual(len(got), 2, 'the caller needs (pids, complete)') + + def test_a_restricted_proc_is_reported_incomplete(self): + import usbtest + self.addCleanup(setattr, usbtest.os, 'geteuid', usbtest.os.geteuid) + self.addCleanup(setattr, usbtest.os, 'access', usbtest.os.access) + usbtest.os.geteuid = lambda: 1000 # not root + usbtest.os.access = lambda p, m: False # /proc/1/cmdline unreadable + _, complete = usbtest.wedged_pids('/dev/bus/usb/999/999') + self.assertFalse(complete, 'a hidden holder was reported as absent') + + +@unittest.skipIf(os.name == 'nt', 'POSIX shell fakes') +@unittest.skipIf(sys.version_info < (3, 11), 'fake-pymtp steering needs PYTHONSAFEPATH') +class StrandMemoRemembersUnstattablePaths(unittest.TestCase): + """A stranded path whose inode could not be read is stored as None -- which dict.get() + also returns for a MISS. Testing `is not None` therefore treats 'known stranded' as + 'never seen', and every later call strands ANOTHER permanent thread and fd on a path we + already know is wedged. That is the exact unbounded growth SYSFS_STUCK_MAX exists to + stop, and it is invisible: `first = path not in _sysfs_stranded` is False, so the + blindness counter does not advance either.""" + + def setUp(self): + from helper import hil_util + self.hil_util = hil_util + self.addCleanup(hil_util._sysfs_stranded.clear) + hil_util._sysfs_stranded.clear() + self.addCleanup(setattr, hil_util, '_sysfs_stuck', hil_util._sysfs_stuck) + hil_util._sysfs_stuck = 0 + self.td = TemporaryDirectory(); self.addCleanup(self.td.cleanup) + self.fifo = os.path.join(self.td.name, 'serial') + os.mkfifo(self.fifo) # open() succeeds, read() never returns + + def test_an_unstattable_strand_is_not_re_read(self): + self.hil_util._sysfs_stranded[self.fifo] = None # as the record path stores it + before = threading.active_count() + self.assertIs(self.hil_util.read_sysfs(self.fifo, grace=0.5), + self.hil_util.SYSFS_UNKNOWN) + self.assertEqual(threading.active_count(), before, + 'a known-stranded path was re-read, stranding another thread') + + def test_a_live_strand_is_still_re_read_when_the_node_is_replaced(self): + """The memo must not become permanent blindness: a NEW inode at the same path is a + different device and has to be read.""" + self.hil_util._sysfs_stranded[self.fifo] = 999999999 # inode that is not this one + with open(os.path.join(self.td.name, 'other'), 'w') as f: + f.write('ok\n') + os.replace(os.path.join(self.td.name, 'other'), self.fifo) + self.assertEqual(self.hil_util.read_sysfs(self.fifo, grace=0.5), 'ok') + + +class MtpGioOrdering(_MtpFakeRig, unittest.TestCase): + """gio must not run until the device is READY. + + gvfs claims an MTP device only AFTER udev probing, so the mount this unmounts cannot + exist before /dev/libmtp- is published -- an unmount issued earlier is a + guaranteed no-op that still forks a process, and it leaves the window between the + unmount and the open unprotected, which is the hang it exists to prevent. Running it + per poll iteration also forks one gio per second of the enumeration budget.""" + + def setUp(self): + super().setUp() + tmp = Path(self.tmp.name) + self.gio_log = tmp / 'gio.log' + binn = tmp / 'bin'; binn.mkdir() + (binn / 'gio').write_text('#!/bin/sh\necho "$@" >> "$GIO_LOG"\n') + (binn / 'gio').chmod(0o755) + for k in ('PATH', 'GIO_LOG'): + old = os.environ.get(k) + self.addCleanup(lambda k=k, v=old: os.environ.__setitem__(k, v) + if v is not None else os.environ.pop(k, None)) + os.environ['GIO_LOG'] = str(self.gio_log) + os.environ['PATH'] = f'{binn}:{os.environ["PATH"]}' + + def _gio_calls(self): + return self.gio_log.read_text().splitlines() if self.gio_log.exists() else [] + + def test_gio_does_not_run_before_the_device_is_ready(self): + (Path(self.tmp.name) / 'dev' / 'libmtp-1-1').unlink() # never becomes ready + os.environ['FAKE_PYMTP_MODE'] = 'absent' + run_bounded(lambda: hil_test.test_device_mtp(self.board), 30) + calls = self._gio_calls() + self.assertEqual(calls, [], f'gio ran {len(calls)}x with no device ready: {calls}') + + def test_gio_still_runs_once_the_device_is_ready(self): + """The guard must delay the unmount, not delete it.""" + os.environ['FAKE_PYMTP_MODE'] = 'ok' + hil_test.test_device_mtp(self.board) + self.assertTrue(self._gio_calls(), 'gio never ran for a ready device') + + +class MtpGioFallthrough(unittest.TestCase): + """The missing-gio path must fall THROUGH to detection. `continue` there skips the + deadline check and the sleep as well, spinning at 100% CPU until the caller's outer + kill — reported as a wedged DUT for a missing apt package.""" + + def test_a_missing_gio_still_bounds_the_session(self): + import subprocess + with TemporaryDirectory() as td: + env = {**os.environ, 'PATH': td, # no gio, no anything + 'PYTHONPATH': os.path.join(TEST_DIR, 'stubs'), + 'FAKE_PYMTP_MODE': 'none', 'PYTHONSAFEPATH': '1'} + t0 = time.monotonic() + r = subprocess.run([sys.executable, + str(Path(TEST_DIR).parents[0] / 'mtp_test.py'), + '--uid', 'CAFE01', '--timeout', '3'], + capture_output=True, text=True, timeout=60, env=env) + elapsed = time.monotonic() - t0 + self.assertLess(elapsed, 30, f'did not honour --timeout 3 ({elapsed:.1f}s)') + self.assertNotEqual(r.returncode, 0) + # The assertions above are satisfied by an immediate CRASH, which is exactly what + # shipped through this test once: `pass` left gio unbound and the next line + # dereferenced it. Assert the behaviour the docstring names -- it POLLED for the + # device (so it spent its budget) and did not die on a traceback. + self.assertGreater(elapsed, 2.0, + f'exited without polling ({elapsed:.1f}s) -- it crashed') + self.assertNotIn('Traceback', r.stderr) + self.assertIn('MTP device not found', r.stdout + r.stderr) + + +class RunWhileContract(unittest.TestCase): + """The read-while-we-write runner. Its child can still outlast SIGKILL -- but unlike + the thread it replaced, an abandoned child is a real process in its own session, so + the containment sweep finds it and the report names it.""" + + def setUp(self): + from helper import hil_util + self.hil_util = hil_util + + def test_an_error_in_work_is_not_swallowed(self): + """A `return` inside the reap's `finally` discarded it: an assert in the CDC + write half vanished and the caller went on to compare data it never sent.""" + def boom(): + raise AssertionError('the write failed') + with self.assertRaises(AssertionError): + self.hil_util.run_alongside(['sh', '-c', 'printf X'], boom, 5) + + def test_the_child_is_reaped_even_when_work_raises(self): + seen = {} + + def boom(): + raise AssertionError('x') + with self.assertRaises(AssertionError): + self.hil_util.run_alongside(['sleep', '20'], boom, 1) + # nothing of ours is left running: the reap ran on the error path too + import subprocess + out = subprocess.run(['pgrep', '-f', '^sleep 20'], capture_output=True, text=True) + seen['strays'] = [p for p in out.stdout.split() if p] + self.assertEqual(seen['strays'], [], 'work() raising leaked the child') + + def test_an_abandoned_child_is_in_its_own_session(self): + """killpg on it reaps whatever it spawned, and it cannot take our group with it.""" + import subprocess + pgids = {} + + def check(): + time.sleep(0.2) + pgids['child'] = os.getpgid(self._proc_pid) + + real_popen = subprocess.Popen + + def spy(argv, **kw): + p = real_popen(argv, **kw) + self._proc_pid = p.pid + return p + self.addCleanup(setattr, subprocess, 'Popen', real_popen) + subprocess.Popen = spy + self.hil_util.run_alongside(['sleep', '0.5'], check, 5) + subprocess.Popen = real_popen + self.assertNotEqual(pgids['child'], os.getpgid(0)) + + +class StrandedPathMemoInvalidates(unittest.TestCase): + """The memo lives in read_sysfs, so every bounded reader gets it -- call-site memos + meant each new scanner had to remember (get_printer_dev and the throughput probe did + not). And it MUST expire on re-enumeration: the key is a bus path, which does not + change when a device comes back on the same port, so a memo that never invalidates + makes a board the branch's own HUNG reflash just recovered permanently invisible.""" + + def setUp(self): + from helper import hil_util + self.hil_util = hil_util + self.td = TemporaryDirectory() + self.addCleanup(self.td.cleanup) + for name in ('_sysfs_stranded', '_sysfs_stuck'): + self.addCleanup(setattr, hil_util, name, getattr(hil_util, name)) + hil_util._sysfs_stranded = {} + hil_util._sysfs_stuck = 0 + + def test_a_stranded_path_is_not_re_read(self): + f = Path(self.td.name) / 'serial' + os.mkfifo(f) # never answers + self.assertIs(self.hil_util.read_sysfs(str(f), 0.3), self.hil_util.SYSFS_UNKNOWN) + after_first = self.hil_util._sysfs_stuck + t0 = time.monotonic() + for _ in range(3): + self.assertIs(self.hil_util.read_sysfs(str(f), 0.3), + self.hil_util.SYSFS_UNKNOWN) + self.assertLess(time.monotonic() - t0, 0.3, 'the memo did not short-circuit') + self.assertEqual(self.hil_util._sysfs_stuck, after_first, + 'repeat reads spent more of the blindness budget') + + def test_re_enumeration_clears_it(self): + """A new device on the same busport gets a fresh sysfs node, hence a fresh inode. + Without this the memo outlives the wedge it recorded.""" + f = Path(self.td.name) / 'serial' + os.mkfifo(f) + self.assertIs(self.hil_util.read_sysfs(str(f), 0.3), self.hil_util.SYSFS_UNKNOWN) + f.unlink() + f.write_text('CAFE01\n') # same path, new inode = re-enumerated + self.assertEqual(self.hil_util.read_sysfs(str(f), 0.3), 'CAFE01', + 'a recovered device stayed invisible') + + def test_a_vanished_path_is_not_remembered_as_stranded(self): + f = Path(self.td.name) / 'serial' + os.mkfifo(f) + self.assertIs(self.hil_util.read_sysfs(str(f), 0.3), self.hil_util.SYSFS_UNKNOWN) + f.unlink() + self.assertIsNone(self.hil_util.read_sysfs(str(f), 0.3)) + + +class UsbScanIsTheOneWalk(unittest.TestCase): + """Three call sites each had a different subset of the three things this must get + right; none had all three. The expensive read is `serial` -- served under the device + lock a wedged usbfs ioctl holds -- so it must come LAST, only for devices the free + descriptor fields could not rule out, and never twice for a path that stranded.""" + + def setUp(self): + from helper import hil_util + self.hil_util = hil_util + self.td = TemporaryDirectory() + self.addCleanup(self.td.cleanup) + self.root = Path(self.td.name) + self.reads = [] + real = hil_util.read_sysfs + + def counting(path, *a, **k): + self.reads.append(path) + return real(path, *a, **k) + self.addCleanup(setattr, hil_util, 'read_sysfs', real) + hil_util.read_sysfs = counting + self.addCleanup(setattr, hil_util, '_sysfs_stranded', + dict(hil_util._sysfs_stranded)) + self.addCleanup(setattr, hil_util, '_sysfs_stuck', hil_util._sysfs_stuck) + + def _dev(self, name, vid, pid, serial='S1', fifo=False): + d = self.root / name + d.mkdir() + (d / 'idVendor').write_text(vid + '\n') + (d / 'idProduct').write_text(pid + '\n') + if fifo: + os.mkfifo(d / 'serial') # a read that never answers + else: + (d / 'serial').write_text(serial + '\n') + return d + + def _scan(self, **kw): + import glob as _g + real_glob = _g.glob + self.addCleanup(setattr, self.hil_util.glob, 'glob', real_glob) + self.hil_util.glob.glob = lambda pat: [str(p) for p in self.root.iterdir()] + return self.hil_util.usb_scan(**kw) + + def test_a_mismatched_vid_pid_costs_no_serial_read(self): + self._dev('1-1', '1234', '5678') + self._dev('1-2', 'cafe', '4010', serial='UID1') + devs, unknown = self._scan(vid_pid=('cafe', '4010')) + self.assertEqual([d['serial'] for d in devs], ['UID1']) + self.assertFalse(unknown) + # the ruled-out device's locked attribute was never touched + self.assertNotIn(str(self.root / '1-1' / 'serial'), self.reads) + + def test_a_wedged_device_stays_unproven_on_every_scan(self): + """The memo lives in read_sysfs now, so usb_scan still CALLS it each pass -- what + must not repeat is the cost. StrandedPathMemoInvalidates covers the short-circuit; + here the invariant is that the device stays out of the results and absence stays + unproven, however many times we look.""" + from helper import hil_util + self._dev('1-1', 'cafe', '4010', fifo=True) + first = None + t0 = time.monotonic() + for _ in range(3): + devs, unknown = self._scan() + self.assertTrue(unknown, 'a stranded read must leave absence unproven') + self.assertEqual(devs, []) + if first is None: + first = hil_util._sysfs_stuck + self.assertEqual(hil_util._sysfs_stuck, first, + 'repeat scans spent more of the blindness budget') + self.assertLess(time.monotonic() - t0, 3.0, 'repeat scans re-paid the grace') + + +class BoundedOpenTellsAbsentFromUnknown(unittest.TestCase): + """Same contract as read_sysfs, in the sibling function of the same file: a real + OSError is a FACT (EBUSY, ENOENT, EACCES), a blocked open is UNKNOWN. Folding both + into None made an ordinary EBUSY report as a USB wedge, sending the operator to + usb-kernel-recover for healthy hardware -- and left the stranded thread uncounted, + so the cap that exists to stop the fd/thread ceiling never saw it.""" + + def setUp(self): + from helper import hil_util + self.hil_util = hil_util + self.td = TemporaryDirectory() + self.addCleanup(self.td.cleanup) + + def test_a_real_oserror_is_a_fact(self): + missing = str(Path(self.td.name) / 'nope') + self.assertIsNone(self.hil_util.bounded_open(missing, os.O_RDONLY, 1)) + + def test_a_blocked_open_is_unknown_and_counted(self): + fifo = Path(self.td.name) / 'fifo' + os.mkfifo(fifo) # no reader: O_WRONLY blocks forever + self.addCleanup(setattr, self.hil_util, '_sysfs_stuck', + self.hil_util._sysfs_stuck) + before = self.hil_util._sysfs_stuck + got = self.hil_util.bounded_open(str(fifo), os.O_WRONLY, 0.3) + self.assertIs(got, self.hil_util.SYSFS_UNKNOWN) + self.assertEqual(self.hil_util._sysfs_stuck, before + 1, + 'a stranded open is invisible to the blindness budget') + + +class UsbtestSysfsReadIsCapped(unittest.TestCase): + """find_device re-scans every cafe:4010 peer after EVERY case, so the local twin -- + which had no SYSFS_STUCK_MAX -- stranded a thread and an fd per wedged peer per case. + Delegating to hil_util gets the cap, and the deferred import keeps usbtest.py + importable standalone.""" + + def test_a_stranded_read_counts_against_the_shared_cap(self): + import usbtest + from helper import hil_util + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + wedged = Path(td.name) / 'serial' + os.mkfifo(wedged) # no writer: open() never returns + self.addCleanup(setattr, hil_util, '_sysfs_stuck', hil_util._sysfs_stuck) + before = hil_util._sysfs_stuck + # UNKNOWN, not None: folding them made a blinded scan read as "device dropped + # off the bus", which aborts past the HUNG reflash + self.assertIs(usbtest._read_sysfs_bounded(wedged, grace=0.5), + hil_util.SYSFS_UNKNOWN) + self.assertEqual(hil_util._sysfs_stuck, before + 1, + 'usbtest reads are invisible to the blindness budget') + + +class AbandonExitSurvivesAFailedFork(unittest.TestCase): + """Pool() forks, and after a convoy -- every stranded read holding a thread and an fd -- + that fork is what hits EAGAIN/ENOMEM. It now runs inside the try, so the finally can + reach _abandon_exit with pool and mgr still None.""" + + def test_none_pool_and_manager_still_write_the_banner(self): + # a subprocess, because _abandon_exit ends in os._exit: in-process it would take + # the test runner with it, before any assertion could run + import subprocess + with TemporaryDirectory() as td: + report = Path(td) / 'hil_report.md' + report.write_text('| board | test |\n|---|---|\n', encoding='utf-8') + src = ( + 'import sys, types\n' + f'sys.path.insert(0, {str(Path(TEST_DIR).parents[0])!r})\n' + 'st = types.ModuleType("serial")\n' + 'st.Serial = type("Serial", (), {})\n' + 'st.SerialException = type("SerialException", (Exception,), {})\n' + 'st.SerialTimeoutException = type("E2", (Exception,), {})\n' + 'sys.modules.setdefault("serial", st)\n' + 'import hil_test\n' + f'hil_test._abandon_exit(None, None, True, 1, __import__("pathlib")' + f'.Path({str(report)!r}))\n') + r = subprocess.run([sys.executable, '-c', src], capture_output=True, + text=True, timeout=120) + self.assertEqual(r.returncode, 1, r.stderr) + self.assertTrue(report.read_text().startswith('**HIL run abandoned'), + 'the abandon banner never reached the report') + + def test_kill_pool_children_tolerates_a_pool_that_never_existed(self): + from helper import hil_health + self.assertEqual(hil_health.kill_pool_children(None), 0) + self.assertEqual(hil_health.kill_pool_children(None, None), 0) + + +class UsbtestOuterBoundIsOneValue(unittest.TestCase): + """The bound usbtest is TOLD and the bound run_cmd ENFORCES must be the same number. + Three separate expressions disagreed: --skip-flash appended no --outer-timeout at all + (usbtest reads 0 as no limit), and the no-recovery branch narrowed only the CHILD's + view while run_cmd still waited for a recovery reserve nothing on that path can + spend -- a pool worker and its battery permit idle for the difference.""" + + def _invoke(self, flasher, skip_flash=False): + from contextlib import contextmanager + from helper import hil_lock, hil_util + + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + dev = Path(td.name) / 'dev1' + dev.mkdir() + for attr, val in (('serial', 'UID1'), ('idVendor', 'cafe'), ('idProduct', '4010')): + (dev / attr).write_text(val + '\n') + + def patch(obj, name, value): + self.addCleanup(setattr, obj, name, getattr(obj, name)) + setattr(obj, name, value) + + def _permit(uid): + yield + + seen = {} + + def fake_run(cmd, **kw): + import subprocess + seen['cmd'], seen['timeout'] = cmd, kw.get('timeout') + return subprocess.CompletedProcess(cmd, 1, stdout=b'', stderr=b'stub') + + from helper import hil_util as _hu + patch(_hu, 'glob', types.SimpleNamespace(glob=lambda p: [str(dev)])) + # the blindness latch and the stranded memo are process-global: another class's + # wedged-FIFO test would otherwise make every read here answer SYSFS_UNKNOWN + patch(_hu, '_sysfs_stuck', 0) + patch(_hu, '_sysfs_stranded', {}) + patch(hil_lock, 'usbtest_permit', contextmanager(_permit)) + patch(hil_test, 'skip_flash', skip_flash) + patch(hil_test, '_current_fw', '/tmp/fw.elf') + patch(hil_util, 'run_cmd', fake_run) + with self.assertRaises(hil_test.TestFail): + hil_test.test_device_usbtest({'name': 'b', 'uid': 'UID1', 'flasher': flasher}) + return seen + + def _outer_flag(self, cmd): + toks = cmd.split() + self.assertIn('--outer-timeout', toks, 'usbtest reads a missing bound as UNLIMITED') + return int(toks[toks.index('--outer-timeout') + 1]) + + def test_a_recoverable_board_reserves_the_recovery_budget(self): + seen = self._invoke({'name': 'openocd', 'vid_pid': '0x1366 0x1024'}) + want = hil_test.USBTEST_BATTERY_BUDGET + hil_test.USBTEST_RECOVERY_BUDGET + self.assertEqual(self._outer_flag(seen['cmd']), want) + self.assertEqual(seen['timeout'], want) + + def test_a_board_with_no_recovery_does_not_pay_for_one(self): + seen = self._invoke({'name': 'stlink', 'uid': 'X'}) # never convoy_safe + outer = self._outer_flag(seen['cmd']) + self.assertEqual(seen['timeout'], outer, 'the two bounds disagree') + # It does not carry the RECOVERY reserve it cannot spend... + self.assertLess(outer, hil_test.USBTEST_BATTERY_BUDGET + + hil_test.USBTEST_RECOVERY_BUDGET) + # ...but it MUST still exceed the child's own --budget. The battery checks the + # budget before dispatching, so it can overshoot by one already-started case; an + # equal bound SIGKILLs it just as it goes to print, turning ~29 real per-case + # verdicts into "usbtest did not run" and re-paying the whole battery on retry. + toks = seen['cmd'].split() + budget = int(toks[toks.index('--budget') + 1]) + case_timeout = int(toks[toks.index('--timeout') + 1]) + self.assertGreaterEqual(outer - budget, case_timeout, + 'the outer kill can land mid-case, before the JSON') + + def test_skip_flash_still_bounds_the_child(self): + seen = self._invoke({'name': 'openocd', 'vid_pid': '0x1366 0x1024'}, skip_flash=True) + self.assertEqual(self._outer_flag(seen['cmd']), seen['timeout']) + + +class UsbtestRetryPolicy(unittest.TestCase): + """The pool guard bounds ONE battery; the retry loop multiplies it by max_retry. + So the loop must retry only what a retry can fix.""" + + def _patch(self, obj, name, value): + # addCleanup, not a finally: a failing assert must not leave the real module + # patched for whatever test runs next (max_retry only exists once main() ran, + # so restoring it means DELETING it again) + if hasattr(obj, name): + self.addCleanup(setattr, obj, name, getattr(obj, name)) + else: + self.addCleanup(delattr, obj, name) + setattr(obj, name, value) + + def _attempts(self, exc): + """How many times test_example runs the test fn before giving up.""" + import hil_flash + calls = [] + + def fake_test(board): + calls.append(1) + raise exc + + self._patch(hil_flash, 'find_firmware', lambda *a, **k: Path('/nonexistent/fw.elf')) + self._patch(hil_test, 'skip_flash', True) # no probe, no hardware + self._patch(hil_test, 'max_retry', 3) + self._patch(hil_test, 'log_line', lambda *a, **k: None) + hil_test.test_fake_example = fake_test + self.addCleanup(delattr, hil_test, 'test_fake_example') + hil_test.test_example({'name': 'b', 'uid': 'u', 'flasher': {'name': 'openocd'}}, + 'v', 'fake/example') + return len(calls) + + def test_a_per_case_verdict_is_not_retried(self): + # re-running the battery only re-observes a number the JSON already reported + self.assertEqual(self._attempts(hil_test.TestFail('29/30', parsed=True)), 1) + + def test_a_transient_failure_is_retried(self): + self.assertEqual(self._attempts(hil_test.TestFail('usbtest did not run')), 3) + + +class UsbtestOuterKillStaysRetryable(unittest.TestCase): + """rc 124 is run_cmd's timer expiring, NOT proof the DUT is wedged -- a healthy + battery can hit it under load. Suppressing the retry to save the budget also + suppresses the reflash test_example does before each attempt, which is the only + thing left to unpoison the DUT where usbtest's in-band recovery is off.""" + + def setUp(self): + from contextlib import contextmanager + from helper import hil_lock + self.td = TemporaryDirectory() + self.addCleanup(self.td.cleanup) + dev = Path(self.td.name) / 'dev1' + dev.mkdir() + # a real (readable) fake sysfs node, so the bounded reads run unmodified + for attr, val in (('serial', 'UID1'), ('idVendor', 'cafe'), ('idProduct', '4010')): + (dev / attr).write_text(val + '\n') + self.dev = dev + + def patch(obj, name, value): + saved = getattr(obj, name) + self.addCleanup(setattr, obj, name, saved) + setattr(obj, name, value) + + from helper import hil_util as _hu + patch(_hu, 'glob', types.SimpleNamespace(glob=lambda p: [str(dev)])) + def _permit(uid): # a real generator: a lambda returning an iterator has + yield # no .throw(), so any raise inside the `with` would + # surface as an AttributeError from contextlib instead + patch(hil_lock, 'usbtest_permit', contextmanager(_permit)) + patch(hil_test, 'skip_flash', True) + + def test_rc_124_stays_retryable(self): + import subprocess + from helper import hil_util + saved = hil_util.run_cmd + self.addCleanup(setattr, hil_util, 'run_cmd', saved) + hil_util.run_cmd = lambda *a, **k: subprocess.CompletedProcess( + 'usbtest', 124, stdout=b'', stderr=b'killed on the outer bound') + with self.assertRaises(hil_test.TestFail) as cm: + hil_test.test_device_usbtest({'name': 'b', 'uid': 'UID1', + 'flasher': {'name': 'openocd'}}) + self.assertFalse(cm.exception.parsed, + 'the retry is the last reflash a poisoned DUT gets') + + def test_a_crashed_tool_stays_retryable(self): + import subprocess + from helper import hil_util + saved = hil_util.run_cmd + self.addCleanup(setattr, hil_util, 'run_cmd', saved) + hil_util.run_cmd = lambda *a, **k: subprocess.CompletedProcess( + 'usbtest', 1, stdout=b'', stderr=b'ImportError: no module named usbtest') + with self.assertRaises(hil_test.TestFail) as cm: + hil_test.test_device_usbtest({'name': 'b', 'uid': 'UID1', + 'flasher': {'name': 'openocd'}}) + self.assertFalse(cm.exception.parsed) + + +class RemoteDirIsScreened(unittest.TestCase): + """REMOTE_DIR reaches the rig through `rm -rf`, an scp remote path and an rsync + remote path -- all re-split and expanded by the REMOTE shell, none of them + protectable by quoting the local variable. So the script screens the value once + instead: it must survive that re-split unchanged, and `~` must keep working.""" + + def _run(self, remote_dir, *args, keep_going=False): + import subprocess + with TemporaryDirectory() as td: + # real ssh/scp/rsync would reach the rig; these just record the argv. Exit 77 + # unless the caller needs the script to run on to the second ssh. + rc = 0 if keep_going else 77 + for tool in ('ssh', 'scp', 'rsync'): + write_script(Path(td) / tool, f'echo "stub-{tool} $*" >&2; exit {rc}') + env = {**os.environ, 'REMOTE_DIR': remote_dir, 'REMOTE': 'stub', + 'PATH': td + os.pathsep + os.environ['PATH']} + return subprocess.run( + ['bash', str(Path(TEST_DIR).parents[0] / 'hil_ci.sh'), *args], + capture_output=True, text=True, timeout=60, env=env) + + def test_whitespace_is_refused(self): + # unscreened, the remote `rm -rf -- "$1"` gets a TRUNCATED path and deletes + # the wrong tree + r = self._run('/tmp/hil dir') + self.assertNotEqual(r.returncode, 0) + self.assertIn('REMOTE_DIR', r.stderr) + + def test_command_substitution_is_refused(self): + r = self._run('/tmp/$(touch pwned)') + self.assertNotEqual(r.returncode, 0) + self.assertIn('REMOTE_DIR', r.stderr) + + def test_bare_root_is_refused(self): + r = self._run('/') + self.assertNotEqual(r.returncode, 0) + self.assertIn('REMOTE_DIR', r.stderr) + + def test_a_tilde_path_is_accepted(self): + """The one override %q broke: `~` must reach the remote shell UNESCAPED or it + creates a literal '~' directory in the login dir.""" + r = self._run('~/tinyusb-hil') + self.assertIn('~/tinyusb-hil', r.stderr) # got as far as the first ssh + self.assertNotIn('\\~', r.stderr) # %q escapes it; the remote shell won't + + def test_paths_that_would_rm_rf_something_huge_are_refused(self): + """Passing the tilde through UNESCAPED is what makes this dangerous: the remote + shell expands `~/` to the login dir, so `rm -rf -- "$1"` takes out $HOME -- one + typo away from the documented REMOTE_DIR=~/dir override. A bare root, a + no-component path and a foreign ~user are the same class.""" + for bad in ('~/', '~root/x', '~-', '//', '/.', '/tmp/hil/'): + with self.subTest(remote_dir=bad): + r = self._run(bad) + self.assertNotEqual(r.returncode, 0, f'{bad!r} was accepted') + self.assertIn('REMOTE_DIR', r.stderr) + + def test_an_arg_containing_a_space_survives_the_remote_resplit(self): + """ssh joins its argv into ONE string the remote shell re-splits, so an unquoted + `-t 'host/cdc msc'` arrives as two arguments and hil_test.py sees a stray word + where it expects the config path.""" + r = self._run('/tmp/tinyusb-hil', '-t', 'host/cdc msc', keep_going=True) + run_line = [l for l in r.stderr.splitlines() if 'bash -s --' in l][-1] + self.assertIn(r'host/cdc\ msc', run_line) + + +class CaveatSurvivesAccumulate(unittest.TestCase): + """CI reruns with --accumulate: the sidecar keeps every earlier attempt's cells, but the + banner was recomputed per attempt. A first attempt on a degraded rig and a clean rerun + therefore published the degraded attempt's PASSES with no caveat on them -- and the + generated .failed spec reruns only failures, so those cells are never re-earned.""" + + def _rows(self, board, cell): + return [(board, 0, 0, [(board, {cell: 'OK'}, '1s')], 0)] + + def test_an_earlier_attempts_caveat_is_still_on_the_report(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + banner = '> **Rig note.** 2 process(es) in D state at start.\n' + + hil_test.accumulate_report(self._rows('boardA', 'cdc_msc'), rd, True, '', banner) + self.assertIn('Rig note', (rd / hil_test.REPORT_MD).read_text()) + + # the rerun: clean rig, so this attempt contributes no banner of its own + md = hil_test.accumulate_report(self._rows('boardB', 'cdc_msc'), rd, False, '', '') + self.assertIn('boardA', md) # the earlier cells are kept ... + self.assertIn('Rig note', md, + 'the caveat the earlier cells were collected under was dropped') + + def test_the_same_caveat_twice_is_not_stacked(self): + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + banner = '> **Rig note.** 2 process(es) in D state at start.\n' + hil_test.accumulate_report(self._rows('boardA', 'cdc_msc'), rd, True, '', banner) + md = hil_test.accumulate_report(self._rows('boardB', 'cdc_msc'), rd, False, '', banner) + self.assertEqual(md.count('Rig note'), 1) + + +class BlindWorkerReachesTheReport(unittest.TestCase): + """A worker that exhausts its bounded-read budget answers SYSFS_UNKNOWN for every + attribute, so its "device not found" means "could not tell". That reached the log and + the per-cell failure text but NOT the table -- and the table is what gets pasted into + the PR. Seen live: run 31794359407 went blind in 4 workers and published 26 red cells + with no mention of it, several of them caused by the blindness rather than the board.""" + + def test_no_note_when_every_worker_could_see(self): + mret = [('boardA', 0, [], [], 1.0, False), ('boardB', 0, [], [], 1.0, False)] + self.assertEqual(hil_test._blind_note(mret), '') + + def test_the_note_names_the_boards_whose_verdicts_are_not_evidence(self): + mret = [('boardA', 0, [], [], 1.0, True), ('boardB', 0, [], [], 1.0, False), + ('boardC', 1, [], [], 1.0, True)] + note = hil_test._blind_note(mret) + self.assertIn('boardA', note) + self.assertIn('boardC', note) + self.assertNotIn('boardB', note) # it could see; do not smear its result + self.assertTrue(note.endswith('\n'), 'banners are line-oriented') + + def test_both_row_widths_survive_the_report_writers(self): + """The blindness flag widened the worker's result tuple to 6, but the pool-timeout + path still synthesises 5-field rows for boards that never reported and feeds them + to the same two writers. A fixed-width unpack in either one raises INSIDE the + containment path, which is where a raise costs every board's results.""" + td = TemporaryDirectory() + self.addCleanup(td.cleanup) + rd = Path(td.name) + wide = ('boardA', 1, ['device/cdc_msc'], [('boardA', {'cdc_msc': '❌'}, '2s')], 2.0, True) + narrow = ('stuck', 1, [], None, 0) # what the timeout path builds + hil_test._write_failed_spec(rd / 'x.failed', rd, [wide, narrow]) + md = hil_test.accumulate_report([wide], rd, True, '', hil_test._blind_note([wide])) + self.assertIn('boardA', md) + self.assertIn('not all verdicts are evidence', md.lower()) + + def test_the_stray_note_names_the_board_and_survives_narrow_rows(self): + """Survivors ride back on the result tuple because main()'s own sweep runs after + the report is written on both abort paths -- the banner appended there was + computed and discarded.""" + wide = ('boardA', 0, [], [], 1.0, False, 2) + clean = ('boardB', 0, [], [], 1.0, False, 0) + note = hil_test._stray_note([wide, clean]) + self.assertIn('boardA', note) + self.assertNotIn('boardB', note) + self.assertIn('2', note) + self.assertEqual(hil_test._stray_note([clean]), '') + self.assertEqual(hil_test._stray_note([('stuck', 1, [], None, 0)]), '') + + def test_the_timeout_paths_synthetic_rows_do_not_crash_it(self): + """The pool-timeout path builds (name, 1, [], None, 0) for boards that never + reported -- five fields, no blindness to report -- and hands those around.""" + self.assertEqual(hil_test._blind_note([('stuck', 1, [], None, 0)]), '') + + +class PoolGuardKeepsWhatFinished(unittest.TestCase): + """The guard's 30-minute predecessor fired on 5 of the last 8 HIL jobs, so this is the + common failure, not an edge case: map_async discarded every board that had finished and + left the re-run spec unwritten, so CI re-tested all ~26 to find the one that wedged. + + Calls hil_test.drain_pool -- the loop main() actually runs. The predecessor of this test + built its own ThreadPool and its own drain loop and asserted on those, so deleting the + production drain outright left it green.""" + + class _It: + """Stands in for imap_unordered: yields, then blocks past any deadline.""" + + def __init__(self, ready): + self.ready, self.i = ready, 0 + + def next(self, timeout=None): + if self.i < len(self.ready): + self.i += 1 + return self.ready[self.i - 1] + raise MpTimeoutError + + def test_finished_rows_survive_a_guard_expiry(self): + boards = [{'name': 'fast1'}, {'name': 'fast2'}, {'name': 'wedged'}] + rows = [('fast1', 0, [], [], 1.0, False), ('fast2', 0, [], [], 1.0, False)] + with self.assertRaises(hil_test.PoolDrainTimeout) as cm: + hil_test.drain_pool(self._It(rows), boards, time.monotonic() + 5) + self.assertEqual([r[0] for r in cm.exception.finished], ['fast1', 'fast2']) + + def test_an_expired_deadline_stops_before_asking_for_more(self): + """Left <= 0 must not be handed to it.next() as a zero/negative timeout.""" + boards = [{'name': 'a'}, {'name': 'b'}] + it = self._It([('a', 0, [], [], 1.0, False)]) + with self.assertRaises(hil_test.PoolDrainTimeout) as cm: + hil_test.drain_pool(it, boards, time.monotonic() - 1) # already past + self.assertEqual(cm.exception.finished, []) + self.assertEqual(it.i, 0, 'asked the pool for a result after the deadline') + + def test_rows_collected_before_the_deadline_expires_are_kept_too(self): + """The OTHER raise site: boards finish, then the clock runs out between results. + Both sites must carry the rows -- a bare raise here loses a worker-width of rig + time just as map_async did, and the it.next() path alone does not prove it.""" + class Slow(self._It): + def next(self, timeout=None): + time.sleep(0.2) # each result eats into the deadline + return super().next(timeout) + + boards = [{'name': n} for n in ('a', 'b', 'c', 'd')] + rows = [(n, 0, [], [], 1.0, False) for n in ('a', 'b', 'c', 'd')] + with self.assertRaises(hil_test.PoolDrainTimeout) as cm: + hil_test.drain_pool(Slow(rows), boards, time.monotonic() + 0.3) + self.assertTrue(cm.exception.finished, 'rows collected before the expiry were lost') + + def test_every_board_finishing_returns_them_all(self): + boards = [{'name': 'a'}, {'name': 'b'}] + rows = [('a', 0, [], [], 1.0, False), ('b', 1, [], [], 2.0, False)] + got = hil_test.drain_pool(self._It(rows), boards, time.monotonic() + 5) + self.assertEqual(got, rows) + + +class WedgedBoardCosts(unittest.TestCase): + """Two decisions the containment latch makes, tested as decisions rather than through + test_board's loop -- the loop-level predecessor of these tests reimplemented that loop + and asserted on its own copy, which is how both defects survived it.""" + + def setUp(self): + self.addCleanup(setattr, hil_test, 'board_wedged', hil_test.board_wedged) + + def test_a_board_that_wedged_still_counts_as_an_error(self): + """It rendered a red cell but returned err_count 0, so main()'s sys.exit(err_count) + reported success and _write_failed_spec (`if err > 0`) left the board out of the + re-run entirely: a rig holding a D-state process published as a clean pass.""" + hil_test.board_wedged = 'usbtest HUNG' + # no real flasher: skip_flash isolates the accounting from hil_flash + self.addCleanup(setattr, hil_test, 'skip_flash', hil_test.skip_flash) + hil_test.skip_flash = True + # a firmware path must resolve or test_example returns 'skip (no binary)' before + # ever reaching the retry loop this is about + self.addCleanup(setattr, hil_flash, 'find_firmware', hil_flash.find_firmware) + hil_flash.find_firmware = lambda *a, **k: Path('fw.elf') + + def boom(*a, **k): + raise hil_test.TestFail('usbtest did not run') # unparsed: retryable + + self.addCleanup(setattr, hil_test, 'test_device_usbtest', hil_test.test_device_usbtest) + hil_test.test_device_usbtest = boom + board = {'name': 'b', 'uid': 'U', 'flasher': {'name': 'openocd'}, 'tests': []} + err, _status, _metric = hil_test.test_example(board, 'b', 'device/usbtest') + self.assertEqual(err, 1, 'a wedged board contributed nothing to the exit status') + + def test_the_teardown_park_does_not_flash_a_wedged_board(self): + """The park is a flash like any other: on a D-state-held node it blocks, survives + SIGKILL and leaves a stray -- added by the path that just declared the board wedged + and skipped every test for exactly that reason.""" + hil_test.board_wedged = '' + self.assertTrue(hil_test._should_park(False), 'a healthy board must still park') + hil_test.board_wedged = 'usbtest HUNG' + self.assertFalse(hil_test._should_park(False), + 'the teardown park would flash through the poisoned node') + self.assertFalse(hil_test._should_park(True), '--skip-flash must still suppress it') + + +class WedgeVerdictReachesTheLatch(unittest.TestCase): + """usbtest computes `unrecovered_hang` but never reported it, so hil_test inferred the + latch from `not recovery and 'HUNG' in out` and missed three cases: recovery ran and + FAILED (convoy-safe boards -- max32666fthr HUNG in the 08-14 run), the `inconclusive` + abort (which sets the flag but leaves no case at status HUNG), and an unparsable JSON, + which is the outer-timeout kill and the case where a wedge is most likely.""" + + def setUp(self): + self.addCleanup(setattr, hil_test, 'board_wedged', hil_test.board_wedged) + hil_test.board_wedged = '' + + def _run(self, stdout, rc=0): + from helper import hil_lock, hil_util + class R: + returncode = rc + stderr = b'' + R.stdout = stdout.encode() + self.addCleanup(setattr, hil_util, 'run_cmd', hil_util.run_cmd) + hil_util.run_cmd = lambda *a, **k: R() + # usbtest_enumerated is nested in test_device_usbtest, so stub what it calls + self.addCleanup(setattr, hil_util, 'usb_scan', hil_util.usb_scan) + hil_util.usb_scan = lambda **k: ([{'busport': '1-1', 'dir': '/x', 'vid': 'cafe', + 'pid': '4010', 'serial': 'U'}], False) + self.addCleanup(setattr, hil_lock, 'usbtest_permit', hil_lock.usbtest_permit) + from contextlib import contextmanager + hil_lock.usbtest_permit = contextmanager(lambda uid: iter([None])) + board = {'name': 'b', 'uid': 'U', 'flasher': {'name': 'openocd', 'vid_pid': '0x1 0x2'}} + try: + hil_test.test_device_usbtest(board) + except Exception: + pass + return hil_test.board_wedged + + def test_a_reported_wedge_latches_even_when_recovery_ran(self): + """`recovery` True means the flags were PASSED, not that they worked.""" + js = '{"serial":"U","speed":"480","tier":1,"passed":1,"failed":1,"notrun":0,' '"wedged":true,"cases":[{"num":1,"status":"FAIL"}]}' + self.assertTrue(self._run(js), 'a reported wedge did not latch') + + def test_no_wedge_reported_does_not_latch(self): + js = '{"serial":"U","speed":"480","tier":1,"passed":2,"failed":0,"notrun":0,' '"wedged":false,"cases":[]}' + self.assertFalse(self._run(js)) + + def test_an_unparseable_battery_that_mentions_HUNG_still_latches(self): + """rc 124 mid-print: no JSON to read, and this is the likeliest real wedge.""" + self.assertTrue(self._run('TEST 10 HUNG: device wedged mid-transfer', rc=124)) + + +class WedgedBoardCannotReportAPass(unittest.TestCase): + """The latch alone is not enough: it is set BEFORE the pass return, so an all-green + battery that still wedged returned `PASS 30/30`. That board then contributes 0 to + err_count, is omitted from the .failed re-run spec (which keys on err > 0), and the job + exits 0 with a D-state holder on the rig -- the exact silence this branch exists to end. + usbtest's `inconclusive` and `ambiguous` aborts fire AFTER the last case, so nothing + back-fills a BUDGET entry to make failed/notrun non-zero.""" + + def setUp(self): + self.addCleanup(setattr, hil_test, 'board_wedged', hil_test.board_wedged) + hil_test.board_wedged = '' + + def _cell(self, js): + """Returns ('pass', cell) or ('fail', message).""" + from helper import hil_lock, hil_util + class R: + returncode = 0 + stderr = b'' + R.stdout = js.encode() + self.addCleanup(setattr, hil_util, 'run_cmd', hil_util.run_cmd) + hil_util.run_cmd = lambda *a, **k: R() + self.addCleanup(setattr, hil_util, 'usb_scan', hil_util.usb_scan) + hil_util.usb_scan = lambda **k: ([{'busport': '1-1', 'dir': '/x', 'vid': 'cafe', + 'pid': '4010', 'serial': 'U'}], False) + self.addCleanup(setattr, hil_lock, 'usbtest_permit', hil_lock.usbtest_permit) + from contextlib import contextmanager + hil_lock.usbtest_permit = contextmanager(lambda uid: iter([None])) + board = {'name': 'b', 'uid': 'U', 'flasher': {'name': 'openocd', 'vid_pid': '0x1 0x2'}} + try: + return ('pass', hil_test.test_device_usbtest(board)) + except hil_test.TestFail as e: + return ('fail', str(e)) + + def test_an_all_pass_battery_that_wedged_is_not_a_pass(self): + kind, detail = self._cell('{"serial":"U","speed":"480","tier":1,"passed":30,' + '"failed":0,"notrun":0,"wedged":true,"cases":[]}') + self.assertEqual(kind, 'fail', f'a wedged board reported a green cell: {detail}') + self.assertIn('wedged', detail) + + def test_an_all_pass_battery_that_did_not_wedge_is_still_a_pass(self): + """The guard must key on the latch, not merely on having parsed a battery.""" + kind, cell = self._cell('{"serial":"U","speed":"480","tier":1,"passed":30,' + '"failed":0,"notrun":0,"wedged":false,"cases":[]}') + self.assertEqual(kind, 'pass', f'a healthy board was failed: {cell}') + self.assertIn('30/30', cell) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/hil/test/test_hil_health.py b/test/hil/test/test_hil_health.py new file mode 100644 index 000000000..5695cad6d --- /dev/null +++ b/test/hil/test/test_hil_health.py @@ -0,0 +1,636 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests for hil_health.py — pure logic against a synthetic /proc, no hardware. A real +# wedge cannot be manufactured on demand, so the detectors are exercised against fabricated +# inputs. hil_health is stdlib-only on purpose, so all of this runs on a bare CI runner +# with nothing skipped. Run directly: +# python3 test/hil/test/test_hil_health.py +import os +import signal +import sys +import threading +import time +import subprocess +import unittest +from multiprocessing import Pool +from pathlib import Path +from tempfile import TemporaryDirectory + +# the module under test lives in the parent dir (test/hil), not here +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from helper import hil_health + +REAL_PROC = hil_health.PROC + + +def make_proc(root: Path, procs: dict, with_pid1: bool = True) -> None: + """Build a synthetic /proc. `procs` maps pid -> (comm, state, cmdline); a None comm or + cmdline omits that file. `with_pid1=False` simulates a restricted /proc (hidepid=2), + where an empty scan must not be read as an all-clear.""" + for pid, (comm, state, cmdline) in procs.items(): + d = root / str(pid) + d.mkdir() + if comm is not None: + (d / 'comm').write_text(comm + '\n') + if cmdline is not None: + (d / 'cmdline').write_bytes(cmdline) + # field 2 is comm in parens; the state letter follows it. Deliberately use a comm + # containing ')' so a naive split() would pick the wrong field. + (d / 'stat').write_text(f'{pid} (we)ird) {state} 1 1 0 0 -1 0 0\n') + if with_pid1 and 1 not in procs: + d = root / '1' + d.mkdir() + (d / 'comm').write_text('systemd\n') + (d / 'cmdline').write_bytes(b'/sbin/init\0') + (d / 'stat').write_text('1 (systemd) S 0 1 0 0 -1 0 0\n') + (root / 'not-a-pid').mkdir() + + +class PatchCase(unittest.TestCase): + """For classes that patch PROCESS-GLOBAL state (os.kill, time.sleep, subprocess.Popen). + + addCleanup, never tearDown: tearDown does NOT run when setUp raises, so a no-op + os.kill or time.sleep would survive into every later test in this blocking pre-commit + suite -- turning one setUp failure into a cascade of nonsense results.""" + + def patch(self, obj, name, value): + self.addCleanup(setattr, obj, name, getattr(obj, name)) + setattr(obj, name, value) + + def restore(self, obj, name): + """Same guarantee for state a TEST BODY assigns directly: register the restore + from setUp so it holds even when the assert between fails.""" + self.addCleanup(setattr, obj, name, getattr(obj, name)) + + +class ProcCase(unittest.TestCase): + """Every subclass repoints hil_health.PROC at a temp tree; restore it so a later test + cannot silently keep scanning a deleted directory.""" + + def tearDown(self): + hil_health.PROC = REAL_PROC + + +class ShutdownPool(unittest.TestCase): + def test_returns_true_when_the_pool_terminates(self): + pool = Pool(processes=1) + try: + self.assertTrue(hil_health.shutdown_pool(pool, grace=30)) + finally: + pool.terminate() + + def test_returns_false_instead_of_blocking_forever(self): + """The real failure is a worker in uninterruptible sleep, which cannot be created + from userspace. What matters is that shutdown_pool gives up on the deadline rather + than hanging, because the caller must then abandon the pool to free the job slot.""" + # Cancellable, not time.sleep(3600): shutdown_pool returns while its daemon thread + # is still inside terminate(), and an uninterruptible sleep there outlives the test. + # The next test alphabetically forks a real Pool, so the leaked thread made it + # fork-from-multithreaded ('DeprecationWarning: ... may lead to deadlocks in the + # child') and its result order-dependent. addCleanup releases it either way. + release = threading.Event() + self.addCleanup(release.set) + + class NeverDies: + def terminate(self): + release.wait(3600) + + start = time.monotonic() + self.assertFalse(hil_health.shutdown_pool(NeverDies(), grace=0.5)) + self.assertLess(time.monotonic() - start, 10) + + def test_a_raising_terminate_counts_as_failure(self): + """The thread dies on the exception, so is_alive() goes False -- which would report + success for a pool that is just as alive as if terminate() had hung.""" + class Explodes: + def terminate(self): + raise RuntimeError('boom') + + self.assertFalse(hil_health.shutdown_pool(Explodes(), grace=5)) + + +class ChildProcs(ProcCase): + """A pool worker's own group is OUR group (multiprocessing never setpgid's), so its + children can only be found by walking ppid -> pgrp in /proc.""" + + def test_grandchildren_are_swept_too(self): + """usbtest.py (child, own session) spawns its recovery reflash via run_cmd (own + session again): the flasher is a GRANDCHILD no direct-child walk covers, and a + pool-guard kill mid-recovery would orphan it on the probe.""" + got = self.scan([100], { + 100: ('worker', 1, 4242), + 200: ('usbtest.py', 100, 200), # child, own session + 300: ('openocd', 200, 300), # grandchild flasher, own session + 999: ('unrelated', 1, 999), + }) + self.assertEqual(sorted(got.get(100, [])), [(200, 200), (300, 300)]) + + def scan(self, pids, procs): + """`procs` maps pid -> (comm, ppid, pgrp); a None comm omits the stat file.""" + with TemporaryDirectory() as td: + root = Path(td) + for p, (comm, ppid, pgrp) in procs.items(): + d = root / str(p) + d.mkdir() + if comm is not None: + (d / 'stat').write_text(f'{p} ({comm}) S {ppid} {pgrp} 0 0 -1 0 0\n') + (root / 'not-a-pid').mkdir() + hil_health.PROC = root + return hil_health.child_procs(pids) + + def test_finds_direct_children_only(self): + got = self.scan([100], { + 100: ('python3', 1, 4242), # the worker itself + 201: ('openocd', 100, 201), # its detached flasher + 202: ('usbtest.py', 100, 202), # a second detached session + 303: ('unrelated', 7, 303), # someone else's child + }) + self.assertEqual({100: [(201, 201), (202, 202)]}, + {k: sorted(v) for k, v in got.items()}) + + def test_covers_every_parent_in_one_walk(self): + """One pass for all workers, not one pass each: this runs on the free-the-runner + path, and per-parent walks would also see different snapshots.""" + got = self.scan([100, 101], { + 201: ('openocd', 100, 201), + 202: ('JLinkExe', 101, 202), + }) + self.assertEqual(got, {100: [(201, 201)], 101: [(202, 202)]}) + + def test_parses_a_comm_containing_spaces_and_parens(self): + """A naive split() on the whole line would read the wrong fields.""" + got = self.scan([100], {500: ('we ) ird', 100, 500)}) + self.assertEqual(got, {100: [(500, 500)]}) + + def test_reports_a_child_that_shares_our_group(self): + """subprocess.run children (arecord, iperf) get no new session, so they land in + our group. They must still be REPORTED -- kill_pool_children signals them by pid, + since killpg on that group would take down the run itself.""" + got = self.scan([100], {201: ('arecord', 100, 4242)}) + self.assertEqual(got, {100: [(201, 4242)]}) + + def test_tolerates_unreadable_and_truncated_entries(self): + got = self.scan([100], { + 201: (None, 0, 0), # stat missing (exited mid-scan) + 202: ('openocd', 100, 202), # still found + }) + self.assertEqual(got, {100: [(202, 202)]}) + + def test_returns_empty_when_proc_is_unreadable(self): + hil_health.PROC = Path('/nonexistent-proc-for-test') + self.assertEqual(hil_health.child_procs([100]), {}) + + +class FakeProc: + """Stands in for a multiprocessing worker: kill_pool_children goes through + is_alive() and Process.kill(), whose internal returncode guard is what protects + against signalling a recycled pid.""" + + def __init__(self, pid, alive=True, wedged=False): + self.pid = pid + self._alive = alive + self._wedged = wedged # D state: ignores SIGKILL, so is_alive() stays True + self.killed = False + + def is_alive(self): + return self._alive + + def kill(self): + self.killed = True + # A signalled worker DIES unless it is wedged. Modelling every worker as an + # unkillable survivor sent all of them down the confirm/sudo ladder, which is + # what let literal pids reach the real os.kill. + if not self._wedged: + self._alive = False + + +class KillWorkerChildren(PatchCase): + # os.getpgid/killpg are stubbed for the whole class: FakeProc pids are literals like + # 101, which are live pids on a real machine, so an unstubbed killpg SIGKILLs a real + # process GROUP. That happened while writing this and killed the test run itself. + """What the workers spawned, killed while their parents are still alive. + + Verified premise: Pool.terminate() reaps a worker that is merely waiting in + communicate() on a wedged flasher, reparenting that flasher to init -- so this must + run BEFORE shutdown_pool(), or the ppid link is gone and a successful terminate() + skips the cleanup entirely.""" + + OWN_PGID = 4242 + + def setUp(self): + # _kill_and_confirm's grace poll must not touch the real /proc: fake pid + # 900 can be a live process on the host, which stalls the poll for the full grace + # and prints a false survivor warning into the blocking pre-commit hook. + self.proc_tmp = TemporaryDirectory() + self.addCleanup(self.proc_tmp.cleanup) + self.patch(hil_health, 'PROC', Path(self.proc_tmp.name)) # empty: unreadable -> gone + self.patch(hil_health, 'CONFIRM_KILL_GRACE', 0.05) + # the sweep runs two passes with a real gap; the fakes never respawn, so + # stub the wait rather than pay it in every test + self.patch(hil_health.time, 'sleep', lambda _s: None) + self.groups, self.pids = [], [] + self.children = {} # worker pid -> [(pid, pgid), ...] + self.patch(hil_health, 'child_procs', lambda pids: self.children) + self.patch(os, 'killpg', lambda pgid, sig: self.groups.append((pgid, sig))) + self.patch(os, 'kill', lambda pid, sig: self.pids.append((pid, sig))) + self.patch(os, 'getpgid', lambda pid: self.OWN_PGID) + # overwritten directly by some test bodies below (eperm/boom fakes) + self.restore(hil_health, '_kill_and_confirm') + + def test_kills_a_detached_child_by_group(self): + """Flashers are spawned with start_new_session=True, so one killpg also reaps + whatever they spawned; a plain kill would leave them holding the probe with no + timeout enforcer left alive.""" + w = FakeProc(101) + self.children = {101: [(900, 900)]} + + class FakePool: + _pool = [w] + self.assertEqual(hil_health.kill_worker_children(FakePool()), 0) # none survived + # by GROUP, so whatever the flasher spawned dies with it + self.assertEqual(self.groups, [(900, signal.SIGKILL)]) + # and then confirmed by pid: killpg reports success when it reached ANY member, + # so the group kill alone is not evidence this one died + self.assertIn((900, 0), self.pids) + self.assertFalse(w.killed) # the WORKER is not this one's job + + def test_a_root_owned_group_is_still_confirmed_and_reported(self): + """killpg on an all-root session raises EPERM: the sudo wrapper died and only its + root members remain. That is the one case this handler exists for, so it must + still reach the confirm step -- otherwise the holder that strands the NEXT job is + the one holder the report never names.""" + w = FakeProc(101) + self.children = {101: [(900, 900)]} + + def eperm(pgid, sig): + raise PermissionError + os.killpg = eperm + + class FakePool: + _pool = [w] + hil_health.kill_worker_children(FakePool()) + # confirmed by pid: a liveness probe on the member killpg could not touch + self.assertIn((900, 0), self.pids) + + def test_kills_a_same_group_child_by_pid(self): + """arecord/iperf/gio go through plain subprocess.run and stay in OUR group, where + killpg would take down the run itself -- but they must still die, or a blocked + arecord keeps holding the wedged device.""" + w = FakeProc(101) + self.children = {101: [(900, self.OWN_PGID)]} + + class FakePool: + _pool = [w] + hil_health.kill_worker_children(FakePool()) + self.assertEqual(self.groups, []) # never our own group + # SIGKILL, then a (pid, 0) probe: signalling is not dying, so the kill is always + # confirmed -- see _kill_and_confirm. + self.assertIn((900, signal.SIGKILL), self.pids) + self.assertIn((900, 0), self.pids) + + + def test_signals_pids_only_when_our_group_is_unknown(self): + """If getpgid(0) fails we cannot tell our group from a detached one, so killpg is + never safe -- fall back to per-pid signals rather than guessing.""" + def boom(pid): + raise OSError('no pgid') + os.getpgid = boom + w = FakeProc(101) + self.children = {101: [(900, 900)]} + + class FakePool: + _pool = [w] + hil_health.kill_worker_children(FakePool()) + self.assertEqual(self.groups, []) + self.assertIn((900, signal.SIGKILL), self.pids) + + def test_covers_a_dead_workers_orphans(self): + """A worker reaped between the snapshot and now leaves its flasher running. The + children are keyed off the snapshot, not off is_alive(), so they still die.""" + w = FakeProc(101, alive=False) + self.children = {101: [(900, 900)]} + + class FakePool: + _pool = [w] + self.assertEqual(hil_health.kill_worker_children(FakePool()), 0) # none survived + self.assertEqual(self.groups, [(900, signal.SIGKILL)]) + + def test_a_survivor_is_returned_so_the_report_can_say_the_rig_is_dirty(self): + """A stray that ignores SIGKILL is in D state on a usbfs node or holds a probe, and + it persists into the NEXT job. The count used to be discarded by the caller (the + return was the signalled-child count, which nothing read), so the only trace was a + line in the log -- and the run still published a table that looks clean.""" + w = FakeProc(101) + # TWO strays, only ONE unkillable: signalled=2, survivors=1, so this cannot pass + # by accident on the old return value + self.children = {101: [(900, 900), (901, 901)]} + self.patch(hil_health, '_kill_and_confirm', lambda pids: [p for p in pids if p == 901]) + + class FakePool: + _pool = [w] + self.assertEqual(hil_health.kill_worker_children(FakePool()), 1) + + def test_no_signal_when_the_workers_spawned_nothing(self): + w = FakeProc(101) # no self.children entry + + class FakePool: + _pool = [w] + self.assertEqual(hil_health.kill_worker_children(FakePool()), 0) + self.assertEqual((self.groups, self.pids), ([], [])) + + def test_includes_the_managers_children(self): + mgr_proc = FakeProc(402) + self.children = {402: [(900, 900)]} + + class FakePool: + _pool = [] + + class FakeManager: + _process = mgr_proc + self.assertEqual(hil_health.kill_worker_children(FakePool(), FakeManager()), 0) + + +class ConfirmTailIsOneGrace(PatchCase): + """The grace is ONE window for the whole set, not one per pid. Paid serially it + scaled with stray count: 30 strays x 16 workers spent ~154s inside the path whose + only job is to free the runner's single job slot -- which is exactly the + 'multi-stray convoy tail is minutes' the CI ceilings budget +30 min for.""" + + def setUp(self): + self.proc_tmp = TemporaryDirectory() + self.addCleanup(self.proc_tmp.cleanup) + root = Path(self.proc_tmp.name) + # every fake pid is alive and NOT a zombie, so all of them outlast the grace + make_proc(root, {900 + i: ('flasher', 'D', b'openocd\x00') for i in range(20)}) + self.patch(hil_health, 'PROC', root) + self.patch(hil_health, 'CONFIRM_KILL_GRACE', 0.3) + self.patch(os, 'kill', lambda pid, sig: None) # never signal a real pid + + def test_twenty_survivors_cost_one_grace_not_twenty(self): + pids = [900 + i for i in range(20)] + t0 = time.monotonic() + still = hil_health._kill_and_confirm(pids) + elapsed = time.monotonic() - t0 + self.assertEqual(sorted(still), pids) # all reported, none lost + self.assertLess(elapsed, 0.3 * 4, + f'the grace is paid per pid ({elapsed:.2f}s for 20)') + + +class KillPoolChildren(PatchCase): + """The worker processes themselves. + + Verified premise: an orphaned pool worker keeps the CI runner's stdout pipe open, so a + reader never sees EOF even after the parent exits. + + Fakes throughout: FakeProc.kill() only sets a flag, so nothing here can signal a real + process. That matters historically -- an earlier revision drove this through + os.pidfd_open with literal pids (101, 102), which exist on a real machine, so the suite + was asking the kernel to signal unrelated system processes and was saved only by EPERM. + Keep the fake in charge of kill(); never let a test reach os.kill/os.killpg with a + live pid. FakeProc.kill() alone is NOT enough for that: it leaves is_alive() True, so + the pid reaches the confirm/sudo ladder, which signals for real. Stub that too.""" + + def setUp(self): + # Pids 101/102/201 are ordinary user processes on a container or a fresh runner -- + # and pre-commit.yml runs this suite on GitHub's. Unstubbed, the ladder ran + # os.kill(101, SIGKILL) and forked `sudo -n kill -9 101` on an account with + # passwordless sudo, and the assertions passed only because those pids happen to + # be unkillable kernel threads here. + self.signals = [] + self.patch(os, 'kill', lambda pid, sig: self.signals.append((pid, sig))) + self.patch(os, 'killpg', lambda pgid, sig: self.signals.append((pgid, sig))) + self.proc_tmp = TemporaryDirectory() + self.addCleanup(self.proc_tmp.cleanup) + self.patch(hil_health, 'PROC', Path(self.proc_tmp.name)) # empty: unreadable -> gone + self.patch(hil_health, 'CONFIRM_KILL_GRACE', 0.05) + + def test_a_healthy_worker_never_reaches_the_signalling_ladder(self): + """The premise every assertion below rests on. Process.kill() is the fake's job; + only a worker that SURVIVES it goes on to raw os.kill/sudo, and these pids are + literals that belong to somebody else.""" + a, b = FakeProc(101), FakeProc(102) + + class FakePool: + _pool = [a, b] + hil_health.kill_pool_children(FakePool()) + self.assertEqual(self.signals, [], 'a literal pid reached the raw-signal ladder') + + def test_signals_every_live_worker(self): + a, b = FakeProc(101), FakeProc(102) + + class FakePool: + _pool = [a, b] + # 0, not 2: the RETURN is confirmed survivors, and workers that die to SIGKILL are + # not survivors. The operator verdict ("power-cycle the host") hangs off this. + self.assertEqual(hil_health.kill_pool_children(FakePool()), 0) + self.assertTrue(a.killed and b.killed) + + def test_a_wedged_worker_is_reported_as_a_survivor(self): + """The number the power-cycle verdict is worded on.""" + make_proc(Path(self.proc_tmp.name), {301: ('python3', 'D', b'python3 hil_test.py\x00')}) + wedged = FakeProc(301, wedged=True) + + class FakePool: + _pool = [wedged] + self.assertEqual(hil_health.kill_pool_children(FakePool()), 1) + + def test_skips_a_reaped_worker(self): + """Process.kill() re-checks returncode internally, but skipping a dead child keeps + the harness from signalling a pid the OS may have recycled.""" + live, dead = FakeProc(201), FakeProc(202, alive=False) + + class FakePool: + _pool = [live, dead] + self.assertEqual(hil_health.kill_pool_children(FakePool()), 0) + self.assertTrue(live.killed) + self.assertFalse(dead.killed) + + def test_also_kills_the_manager(self): + """Manager() is a separate child holding the same descriptors, and os._exit skips + its finalizer, so leaving it behind defeats the whole purpose. The RETURN is the + confirmed-survivor count (the caller words a power-cycle verdict on it), so a + clean kill of both reports 0.""" + worker, mgr_proc = FakeProc(401), FakeProc(402) + + class FakePool: + _pool = [worker] + + class FakeManager: + _process = mgr_proc + self.assertEqual(hil_health.kill_pool_children(FakePool(), FakeManager()), 0) + self.assertTrue(worker.killed and mgr_proc.killed) + self.assertTrue(mgr_proc.killed) + + def test_tolerates_a_pool_without_workers(self): + class NoPool: + _pool = None + self.assertEqual(hil_health.kill_pool_children(NoPool()), 0) + + +class WriteTimeoutReport(unittest.TestCase): + def test_prefix_carries_the_preflight_diagnosis(self): + """The timeout aborts before accumulate_report, so without the prefix the artifact + and the PR comment lose the one line saying WHY the pool never finished.""" + with TemporaryDirectory() as td: + d = Path(td) + hil_health.write_timeout_report(d, [{'name': 'b1'}], 4200, 'r.md', + prefix='> **wedged usb_hub_wq worker.**\n') + out = (d / 'r.md').read_text() + self.assertTrue(out.startswith('> **wedged usb_hub_wq worker.**')) + self.assertIn('timed out after 4200s', out) + self.assertIn('- b1', out) + + def test_writes_a_report_where_there_would_be_none(self): + with TemporaryDirectory() as td: + hil_health.write_timeout_report(Path(td), [{'name': 'ra6m5_ek'}], 4200, + 'hil_report.md') + md = (Path(td) / 'hil_report.md').read_text() + self.assertIn('4200s', md) + self.assertIn('ra6m5_ek', md) + + def test_keeps_a_previous_attempts_table(self): + with TemporaryDirectory() as td: + path = Path(td) / 'hil_report.md' + path.write_text('| board | cdc_msc |\n') + hil_health.write_timeout_report(Path(td), [{'name': 'b1'}], 4200, 'hil_report.md') + md = path.read_text() + self.assertIn('abandoned', md) + self.assertIn('| board | cdc_msc |', md) + self.assertLess(md.index('abandoned'), md.index('| board |')) + + def test_custom_banner_is_used(self): + with TemporaryDirectory() as td: + hil_health.write_timeout_report(Path(td), [], 0, 'hil_report.md', + banner='**refused to start.**\n') + self.assertIn('refused to start', (Path(td) / 'hil_report.md').read_text()) + + def test_unwritable_dir_does_not_raise(self): + """The caller may be about to os._exit; losing the report must not also lose the + exit path.""" + hil_health.write_timeout_report(Path('/proc/nonexistent/nope'), [], 0, 'x.md') + + +class WorkerSweepsItsOwnChildren(unittest.TestCase): + """maxtasksperchild=1 makes a worker exit the moment its task returns, so by the time + main()'s finally sweeps, the strays have been reparented to init and are off the pool's + ppid tree entirely. Measured over 4 tasks: pool._pool held two FRESH workers with zero + overlap with the four that ran, child_procs() returned {}, the sweep reported 0, and all + four strays were alive. Inside the worker the ppid link is still there.""" + + def test_a_detached_child_is_killed_and_confirmed(self): + kid = subprocess.Popen(['sleep', '120'], start_new_session=True) + self.addCleanup(lambda: kid.poll() is None and kid.kill()) + time.sleep(0.3) # let it appear in /proc + + stray = hil_health.kill_own_children() + + self.assertEqual(stray, 0, 'a killable stray was reported as a survivor') + kid.wait(timeout=5) # TimeoutExpired here means it outlived us + self.assertIsNotNone(kid.poll()) + + def test_no_children_is_not_an_error(self): + self.assertEqual(hil_health.kill_own_children(), 0) + + +class PermitReleasesOnlyWhatItTook(unittest.TestCase): + """The bounded acquire skips a slot it could not get ('proceeding over-subscribed') and + deliberately leaves it out of `taken`, but __exit__ released every slot in self.slots. + multiprocessing.Semaphore is unbounded, so each timeout permanently widened that + controller's permit -- the throttle this branch NARROWED (FLASH_PARALLEL 8->4, + USBTEST_PARALLEL 4->2) for xHCI bandwidth margin.""" + + def test_a_timed_out_slot_is_not_released_on_exit(self): + from helper import hil_lock + import multiprocessing + + sems = [multiprocessing.Semaphore(1)] + sems[0].acquire() # width 1, already held: the next wait times out + self.addCleanup(setattr, hil_lock, 'PERMIT_TIMEOUT', hil_lock.PERMIT_TIMEOUT) + hil_lock.PERMIT_TIMEOUT = 0.1 + + permit = hil_lock.controller_permit(sems, 'UID') + permit.slots = [0] + with permit: + pass + + # one holder still holds it, so a correct exit leaves it unavailable + self.assertFalse(sems[0].acquire(timeout=0.1), + 'the permit released a slot it never acquired: width grew') + + +class RecoveryPrefersResetOverReflash(unittest.TestCase): + """Probe reset is the preferred cure: non-destructive (the wedged firmware survives for + autopsy), no flash wear, no risk of a bad park image (a wfe/wfi park has bricked SWD on + mimxrt1064_evk and max32666fthr through a power cycle), and measured at 128-129 ms + against a full erase+program. It also fits in budgets a reflash does not.""" + + def setUp(self): + import usbtest # test/hil is already on sys.path (see top of file) + self.u = usbtest + + def test_reset_is_attempted_before_the_reflash(self): + steps = self.u.recovery_steps('openocd', time_left=600) + self.assertEqual([s[0] for s in steps], ['reset', 'flash']) + + def test_a_budget_too_small_to_reflash_still_gets_the_reset(self): + """The old gate skipped recovery whole when a reflash did not fit, leaving the + holder in place; a reset needs a fraction of the budget.""" + steps = self.u.recovery_steps('openocd', time_left=self.u.RECOVER_FLASH_TIMEOUT - 1) + self.assertEqual([s[0] for s in steps], ['reset']) + + def test_no_budget_at_all_yields_nothing(self): + self.assertEqual(self.u.recovery_steps('openocd', time_left=1), []) + + def test_a_flasher_with_no_reset_primitive_goes_straight_to_reflash(self): + steps = self.u.recovery_steps('nosuchflasher', time_left=600) + self.assertEqual([s[0] for s in steps], ['flash']) + + +class RecoveryDoesNotClaimAResetItDidNotDo(unittest.TestCase): + """reset_esptool and reset_lm4flash return rc 0 without resetting anything, so a plan + that includes them makes the log say "resetting via " for a step that + did nothing. wedged_pids() arbitrates, so behaviour was already right -- the record was + not, and a false record is what this branch keeps having to unpick.""" + + def setUp(self): + import usbtest + self.u = usbtest + + def test_a_no_op_reset_primitive_is_not_scheduled(self): + self.assertEqual([k for k, _ in self.u.recovery_steps('esptool', 600)], ['flash']) + self.assertEqual([k for k, _ in self.u.recovery_steps('lm4flash', 600)], ['flash']) + + def test_a_real_reset_primitive_still_is(self): + self.assertEqual([k for k, _ in self.u.recovery_steps('openocd', 600)], + ['reset', 'flash']) + + +class SudoSoftNeverRaises(unittest.TestCase): + """Two of its four call sites are inside run_case's timeout handler, where ANY raise + costs the HUNG verdict, the recovery and the JSON report -- and sudo() sys.exit()s on + 'a password is required', which is a raise like any other.""" + + def setUp(self): + import usbtest + self.u = usbtest + self.addCleanup(setattr, usbtest, 'sudo', usbtest.sudo) + + def _check(self, exc): + def boom(*a, **k): + raise exc + self.u.sudo = boom + r = self.u._sudo_soft(['dmesg']) # must not propagate + self.assertEqual(r.returncode, 1) + + def test_systemexit_from_a_password_prompt_is_contained(self): + self._check(SystemExit('sudo needs a password')) + + def test_oserror_is_contained(self): + self._check(OSError('no such binary')) + + def test_subprocess_error_is_contained(self): + self._check(subprocess.SubprocessError('timed out')) + + +if __name__ == '__main__': + unittest.main(verbosity=1) diff --git a/test/hil/test/test_hil_select.py b/test/hil/test/test_hil_select.py new file mode 100644 index 000000000..9a1261878 --- /dev/null +++ b/test/hil/test/test_hil_select.py @@ -0,0 +1,689 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests for hil_select.py — pure logic, no hardware, no git. Run directly: +# python3 test/hil/test/test_hil_select.py +# +# Imports stay stdlib + hil_select/hil_util/hil_flash ONLY: the pre-commit hil-test +# hook runs this suite, on GitHub's bare runner in the pre-commit workflow as well as +# locally, and that runner has no pyserial/pymtp. hil_flash is admissible because it +# is stdlib + hil_util only (test_hil_util.BottomLayer enforces the stdlib closure of +# both) and the roster-dispatch tests need its flash_* table; never import hil_test, +# which pulls pyserial. +import glob +import json +import os +import sys +import unittest + +# the modules under test live in the parent dir (test/hil), not here +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import hil_flash +from helper import hil_select +from helper.hil_util import device_tests, dual_tests + +REPO = os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.abspath(__file__))))) + + +def real_rosters(): + """The actual rig rosters, for regression tests that need real-world data + (a specific board/family/only-list) rather than the synthetic ROSTER above.""" + rosters = [] + for name in ('tinyusb.json', 'hfp.json'): + path = os.path.join(REPO, 'test/hil', name) + with open(path) as f: + rosters.append((f'test/hil/{name}', json.load(f)['boards'])) + return rosters + + +def roster_flashers(): + """(roster path, board) for every board in the live rosters, `boards-skip` + included: a parked board's flasher name must still dispatch, so that unparking it + is not what discovers the name went stale.""" + for name in ('tinyusb.json', 'hfp.json'): + path = os.path.join(REPO, 'test/hil', name) + with open(path) as f: + cfg = json.load(f) + for key in ('boards', 'boards-skip'): + for b in cfg.get(key, []): + yield f'test/hil/{name}', b + + +def on_roster(tc, *names): + """The subset of `names` currently in the live rig rosters, skipping the test + when none are, because parking/unparking a board is routine rig maintenance. + + That skip now matters MORE than it used to, not less: this suite is a blocking + pre-commit hook AND build.yml's selector steps gate on it (a failing suite falls + open to the full matrix), so an assertion that depends on a specific board being + present goes red on every PR -- including src/-only ones that never touched the + rig -- until someone fixes the roster. Keep roster-dependent assertions behind + on_roster.""" + have = {b['name'] for _, boards in real_rosters() for b in boards} + got = [n for n in names if n in have] + if not got: + tc.skipTest(f'not in the rig roster: {", ".join(names)}') + return got + + +ROSTER = [ + # device-only, rp2040 family + {'name': 'raspberry_pi_pico', 'uid': 'u1', 'flasher': {'name': 'openocd'}, + 'tests': {'device': True, 'host': True, 'dual': True}}, + # device-only, stm32f4 family + {'name': 'stm32f407disco', 'uid': 'u2', 'flasher': {'name': 'jlink'}, + 'tests': {'device': True, 'host': False, 'dual': False}}, + # host-only board + {'name': 'raspberry_pi_pico2', 'uid': 'u3', 'flasher': {'name': 'openocd'}, + 'tests': {'device': False, 'host': True, 'dual': False}}, + # only-list board (espressif-style), flashed by the CI leg that splits on esptool + {'name': 'espressif_s3_devkitm', 'uid': 'u4', 'flasher': {'name': 'esptool'}, + 'tests': {'only': ['device/cdc_msc_freertos', 'host/device_info']}}, +] +ROSTERS = [('test/hil/tinyusb.json', ROSTER)] + + +def sel(files): + return hil_select.classify(files, REPO, ROSTERS) + + +class TestPortRule(unittest.TestCase): + def test_dcd_rp2040_selects_pico_family_only(self): + s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico', s['boards']) + self.assertNotIn('stm32f407disco', s['boards']) + self.assertNotIn('espressif_s3_devkitm', s['boards']) + # device role: no host tests in pico's list + self.assertTrue(all(not t.startswith('host/') for t in s['boards']['raspberry_pi_pico'])) + # host-only boards drop out entirely on a device-role change + self.assertNotIn('raspberry_pi_pico2', s['boards']) + + def test_shared_port_file_is_both_roles(self): + s = sel(['src/portable/synopsys/dwc2/dwc2_common.c']) + self.assertFalse(s['full']) + self.assertNotIn('raspberry_pi_pico', s['boards']) # rp2040 is not a dwc2 family + self.assertIn('stm32f407disco', s['boards']) # stm32f4 is + + +class TestCoreRoleRule(unittest.TestCase): + def test_usbd_selects_all_device_tests_everywhere(self): + s = sel(['src/device/usbd.c']) + self.assertFalse(s['full']) + self.assertNotIn('raspberry_pi_pico2', s['boards']) # host-only board dropped + pico = s['boards']['raspberry_pi_pico'] + self.assertTrue(set(device_tests).issubset(set(pico))) + self.assertTrue(set(dual_tests).issubset(set(pico))) # dual survives device role + self.assertTrue(all(not t.startswith('host/') for t in pico)) + # only-list board: selection intersects its only-list + esp = s['boards']['espressif_s3_devkitm'] + self.assertEqual(esp, ['device/cdc_msc_freertos']) + + def test_host_change_drops_device(self): + s = sel(['src/host/usbh.c']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico2', s['boards']) + self.assertNotIn('stm32f407disco', s['boards']) # device-only board dropped + + +class TestClassRule(unittest.TestCase): + def test_cdc_device_selects_cdc_examples_only(self): + s = sel(['src/class/cdc/cdc_device.c']) + self.assertFalse(s['full']) + pico = s['boards']['raspberry_pi_pico'] + self.assertIn('device/cdc_msc', pico) + self.assertIn('device/cdc_dual_ports', pico) + self.assertNotIn('device/msc_dual_lun', pico) # CFG_TUD_CDC 0 there + self.assertNotIn('device/usbtest', pico) # CFG_TUD_CDC 0 there + self.assertTrue(all(not t.startswith('host/') for t in pico)) + + def test_msc_host_selects_host_side(self): + s = sel(['src/class/msc/msc_host.c']) + self.assertFalse(s['full']) + self.assertNotIn('stm32f407disco', s['boards']) # device-only board + pico2 = s['boards']['raspberry_pi_pico2'] + self.assertIn('host/msc_file_explorer', pico2) + self.assertTrue(all(not t.startswith('device/') for t in pico2)) + + +class TestClassIncludeEdges(unittest.TestCase): + """A class header another class includes reaches that class's examples too. + src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, so + midi_test's firmware contains audio.h - but the class rule derives macros from + the directory name alone, so an audio.h change used to select only + device/audio_test_freertos. On boards that skip that example the per-board + intersection emptied and an audio.h-only PR ran ZERO HIL on them.""" + def test_edges_derived_from_includes(self): + edges = hil_select.class_include_edges(REPO) + self.assertEqual(edges.get('audio/audio.h'), {'midi'}) + self.assertEqual(edges.get('cdc/cdc.h'), {'net'}) + + def test_audio_header_selects_midi_example(self): + s = hil_select.classify(['src/class/audio/audio.h'], REPO, real_rosters()) + self.assertFalse(s['full']) + # every board that runs device/midi_test at all must run it here (boards with + # a tests.only list, e.g. espressif, run the freertos examples instead) + by_name = {b['name']: b for _, bs in real_rosters() for b in bs} + checked = 0 + for name, tests in s['boards'].items(): + if 'device/midi_test' in hil_select.board_tests(by_name[name]): + self.assertIn('device/midi_test', tests, name) + checked += 1 + self.assertTrue(checked) + + def test_audio_header_reaches_boards_that_skip_audio(self): + # both skip device/audio_test_freertos: without the midi edge their + # intersection is empty and they drop out of the selection entirely + boards = on_roster(self, 'metro_m4_express', 'nrf54lm20dk') + s = hil_select.classify(['src/class/audio/audio.h'], REPO, real_rosters()) + for board in boards: + self.assertEqual(s['boards'].get(board), ['device/midi_test'], board) + + def test_edge_is_per_header_not_per_class(self): + # midi includes audio.h, not audio_device.h: an audio_device change must + # not drag midi's examples in + s = hil_select.classify(['src/class/audio/audio_device.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for tests in s['boards'].values(): + if tests != 'all': + self.assertNotIn('device/midi_test', tests) + + +class TestFallbackRules(unittest.TestCase): + def test_unknown_tool_is_full(self): + s = sel(['tools/random_new_script.py']) + self.assertTrue(s['full']) + + def test_docs_only_is_empty_not_full(self): + s = sel(['docs/info/contributing.rst', 'README.rst']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + def test_bsp_family_selects_family_boards(self): + s = sel(['hw/bsp/rp2040/family.cmake']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico', s['boards']) + self.assertEqual(s['boards']['raspberry_pi_pico'], 'all') + self.assertNotIn('stm32f407disco', s['boards']) + + def test_bsp_board_narrows_to_board(self): + s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) + self.assertFalse(s['full']) + self.assertEqual(list(s['boards'].keys()), ['raspberry_pi_pico']) + + def test_example_change_selects_that_example(self): + s = sel(['examples/device/cdc_msc/src/main.c']) + self.assertFalse(s['full']) + self.assertEqual(s['boards']['raspberry_pi_pico'], ['device/cdc_msc']) + + def test_core_common_is_full(self): + for f in ['src/tusb.c', 'src/common/tusb_fifo.c', 'src/osal/osal_freertos.h']: + self.assertTrue(sel([f])['full'], f) + + def test_board_test_example_is_full(self): + # board_test is the park/teardown firmware hil_test.py flashes on every board, + # not an unlisted example: a regression there must not skip the whole rig + for f in ['examples/device/board_test/src/main.c', + 'examples/device/board_test/CMakeLists.txt']: + self.assertTrue(sel([f])['full'], f) + + def test_harness_is_full(self): + for f in ['test/hil/hil_test.py', '.github/workflows/build.yml', 'hw/mcu/nxp/x.c', 'lib/foo/x.c']: + self.assertTrue(sel([f])['full'], f) + + def test_mixed_roles_no_pruning(self): + s = sel(['src/device/usbd.c', 'src/host/usbh.c']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico2', s['boards']) + self.assertIn('stm32f407disco', s['boards']) + + def test_cmakelists_and_requirements_are_full(self): + for f in ['src/CMakeLists.txt', 'examples/CMakeLists.txt', + 'examples/device/CMakeLists.txt', 'test/hil/requirements.txt']: + self.assertTrue(sel([f])['full'], f) + + def test_docs_txt_is_noncode(self): + s = sel(['docs/info/changelog.txt']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + +class TestArgsEmission(unittest.TestCase): + def test_args_for_scoped_selection(self): + s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + args = hil_select.selection_args(s, ROSTERS) + a = args['tinyusb.json'] + self.assertIn('-b raspberry_pi_pico', a) + self.assertNotIn('stm32f407disco', a) + self.assertIn('-bt raspberry_pi_pico:', a) # device-only subset of a device+host board + + def test_args_full_is_empty(self): + s = sel(['tools/random_new_script.py']) + self.assertEqual(hil_select.selection_args(s, ROSTERS), {'tinyusb.json': ''}) + + def test_args_all_board_gets_bare_b(self): + s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) + a = hil_select.selection_args(s, ROSTERS)['tinyusb.json'] + self.assertIn('-b raspberry_pi_pico', a) + self.assertNotIn('-bt', a) + + def test_args_by_flasher_splits_esp_from_the_rest(self): + s = sel(['src/device/usbd.c']) + per = hil_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json'] + self.assertIn('espressif_s3_devkitm', per['esptool']) + self.assertIn('raspberry_pi_pico', per['openocd']) + self.assertNotIn('espressif_s3_devkitm', per.get('openocd', '') + per.get('jlink', '')) + + def test_args_by_flasher_omits_a_flasher_with_no_selected_board(self): + # the esp CI leg must see no args at all here, not a filter matching zero boards + s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) + per = hil_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json'] + self.assertEqual(per, {'openocd': '-b raspberry_pi_pico'}) + + def test_args_by_flasher_full_is_empty(self): + s = sel(['tools/random_new_script.py']) + self.assertEqual(hil_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}}) + + def test_cli_diff_file(self): + import subprocess, tempfile, json as j + with tempfile.NamedTemporaryFile('w', suffix='.txt', delete=False) as f: + f.write('src/class/cdc/cdc_device.c\n') + path = f.name + r = subprocess.run([sys.executable, os.path.join(REPO, 'test/hil/helper/hil_select.py'), + '--diff-file', path, os.path.join(REPO, 'test/hil/tinyusb.json')], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + out = j.loads(r.stdout) + self.assertFalse(out['full']) + self.assertIn('tinyusb.json', out['args']) + self.assertTrue(any('cdc_device' in line for line in out['reasons'])) + # A core-class diff must select boards THROUGH THE CLI: the in-process tests + # inject their own repo root, so only this subprocess path catches a broken + # repo_root derivation -- which once made every repo-relative glob match + # nothing and turned this exact diff into a silent full-HIL skip. + self.assertTrue(out['boards'], + 'CLI selected zero boards for a src/class change: repo_root broken?') + os.unlink(path) + + +class TestRealRosterPortFamilies(unittest.TestCase): + """Regression for port_families() missing espressif's dwc2 reference, which + lives in a component CMakeLists.txt rather than family.cmake/family.mk.""" + def test_dwc2_change_selects_espressif_boards(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = hil_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertIn(board, s['boards']) + + +class TestOptionGatedPort(unittest.TestCase): + """Regression: family_support.cmake compiles some ports from a build option + (MAX3421_HOST=1 -> hcd_max3421.c), so a board's family file never names them.""" + # host-side option board (max3421 as host controller), off any max3421 family + OPT_ROSTER = [('test/hil/opt.json', [ + {'name': 'fake_dual_board', 'uid': 'o1', 'flasher': {'name': 'jlink'}, + 'build': {'args': ['MAX3421_HOST=1']}, + 'tests': {'device': True, 'host': False, 'dual': True}}, + {'name': 'fake_host_board', 'uid': 'o2', 'flasher': {'name': 'jlink'}, + 'variant': [{'name': 'fake_host_board', 'flags': '-DMAX3421_HOST=1'}], + 'tests': {'device': False, 'host': True, 'dual': False}}, + {'name': 'fake_off_board', 'uid': 'o3', 'flasher': {'name': 'jlink'}, + 'variant': [{'name': 'fake_off_board', 'defines': ['MAX3421_HOST=0']}], + 'tests': {'device': True, 'host': True, 'dual': True}}, + ])] + + def test_real_roster_max3421_selects_option_board(self): + boards = on_roster(self, 'metro_m4_express') + s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertIn(board, s['boards']) + + def test_option_selects_via_args_defines_and_flags(self): + s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, self.OPT_ROSTER) + self.assertFalse(s['full']) + self.assertIn('fake_dual_board', s['boards']) # build.args + self.assertIn('fake_host_board', s['boards']) # variant flags + self.assertNotIn('fake_off_board', s['boards']) # variant defines, but =0 + + def test_device_role_port_does_not_pull_host_only_option_board(self): + s = hil_select.classify(['src/portable/analog/max3421/dcd_max3421.c'], REPO, self.OPT_ROSTER) + self.assertFalse(s['full']) + self.assertNotIn('fake_host_board', s['boards']) # host-only board, device change + self.assertIn('fake_dual_board', s['boards']) # device-capable option board + + def test_gates_parsed_from_family_support(self): + self.assertEqual(hil_select.port_option_gates(REPO).get('analog/max3421'), + {'MAX3421_HOST'}) + + def test_board_cmake_option_counts(self): + """A board can enable a gated port in its own BSP rather than via the roster + (hw/bsp/espressif/boards/*/board.cmake -> set(MAX3421_HOST 1)); board_options() + must see those too, or such a board joining the roster is silently dropped.""" + self.assertIn('MAX3421_HOST', + hil_select.bsp_board_options('adafruit_feather_esp32s3', REPO)) + self.assertIn('CFG_TUH_RPI_PIO_USB', + hil_select.bsp_board_options('adafruit_fruit_jam', REPO)) + # commented-out `# set(MAX3421_HOST 1)` must not count + self.assertNotIn('MAX3421_HOST', + hil_select.bsp_board_options('feather_nrf52840_express', REPO)) + + def test_board_cmake_option_selects_off_family_board(self): + # adafruit_feather_esp32s3 is not on any rig roster; stand it in as one to + # prove the BSP-sourced option alone pulls a max3421 change onto the board + roster = [('test/hil/opt.json', [ + {'name': 'adafruit_feather_esp32s3', 'uid': 'o1', 'flasher': {'name': 'esptool'}, + 'tests': {'device': False, 'host': True, 'dual': False}}])] + s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster) + self.assertFalse(s['full']) + self.assertIn('adafruit_feather_esp32s3', s['boards']) + + def test_board_mk_option_is_ignored(self): + """Make-only options must not select: HIL CI builds with CMake exclusively, so + hw/bsp/nrf/boards/nrf5340dk/board.mk's MAX3421_HOST compiles nothing here.""" + roster = [('test/hil/opt.json', [ + {'name': 'nrf5340dk', 'uid': 'o1', 'flasher': {'name': 'jlink'}, + 'tests': {'device': False, 'host': True, 'dual': False}}])] + s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + +class TestPortFamiliesCmakeOnly(unittest.TestCase): + """port_families() is CMake-only (HIL CI never builds with Make) and matches on + 'port_dir/' so a port dir is not a prefix of a sibling.""" + def test_make_only_family_is_not_a_family(self): + # hw/bsp/pic32mz has family.mk but no family.cmake + self.assertEqual(hil_select.port_families('microchip/pic32mz', REPO), set()) + + def test_prefix_port_does_not_inherit_sibling_families(self): + # bare-substring matching let 'microchip/pic' match '.../microchip/pic32mz/...' + self.assertEqual(hil_select.port_families('microchip/pic', REPO), set()) + + def test_make_only_port_forces_full(self): + s = sel(['src/portable/microchip/pic32mz/dcd_pic32mz.c']) + self.assertTrue(s['full']) + self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons']) + + def test_cmake_families_still_found(self): + self.assertEqual(hil_select.port_families('raspberrypi/rp2040', REPO), {'rp2040'}) + self.assertIn('stm32f4', hil_select.port_families('synopsys/dwc2', REPO)) + + +class TestPortFamiliesCoverage(unittest.TestCase): + """Systematic guard: every real dcd_*/hcd_* port directory should map to at + least one board family, so a future family.cmake/CMakeLists.txt layout that + port_families() doesn't scan fails loudly instead of silently dropping boards + (as espressif's dwc2 reference did - see TestRealRosterPortFamilies).""" + # Ports with no board family: not a bug, just not wired into any rig board. + # Add here (with a reason) only if port_families() legitimately can't find one. + # A port listed here force-fulls (fail-open), so it is never under-selected. + NO_FAMILY = { + 'template', # reference/example port, not built by any board + # hw/bsp/pic32mz has family.mk only (no family.cmake), and port_families() + # is CMake-only because HIL CI builds every board with CMake - so this port + # is compiled for no HIL board. + 'microchip/pic32mz', + 'microchip/pic', # same: only ever referenced from pic32mz's family.mk + } + + @staticmethod + def _dcd_hcd_ports(): + portable_root = os.path.join(REPO, 'src/portable') + ports = [] + for entry in sorted(os.listdir(portable_root)): + d = os.path.join(portable_root, entry) + if not os.path.isdir(d): + continue + if glob.glob(os.path.join(d, 'dcd_*.c')) or glob.glob(os.path.join(d, 'hcd_*.c')): + ports.append(entry) + continue + for sub in sorted(os.listdir(d)): + sd = os.path.join(d, sub) + if os.path.isdir(sd) and (glob.glob(os.path.join(sd, 'dcd_*.c')) or + glob.glob(os.path.join(sd, 'hcd_*.c'))): + ports.append(f'{entry}/{sub}') + return ports + + def test_every_port_maps_to_a_family(self): + ports = self._dcd_hcd_ports() + self.assertTrue(ports) # sanity: the scan itself found something + for port in ports: + if port in self.NO_FAMILY: + continue + fams = hil_select.port_families(port, REPO) + self.assertTrue(fams, f'{port}: no family references this port ' + f'(port_families() scan gap, or add to NO_FAMILY)') + + +class TestRealRosterOnlyListTests(unittest.TestCase): + """Regression for roster-only-list tests (e.g. espressif's hid_composite_freertos) + being invisible to the selector because it only knew the shared hil_util lists.""" + def test_only_list_example_change_selects_it(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = hil_select.classify(['examples/device/hid_composite_freertos/src/main.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertEqual(s['boards'][board], ['device/hid_composite_freertos']) + + def test_class_change_includes_only_list_boards(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = hil_select.classify(['src/class/hid/hid_device.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertIn(board, s['boards']) + + +class TestPortAndCoreRoleUseExtras(unittest.TestCase): + """Regression: the port rule and core-role rule must thread the roster-only + test universe (extras) the same way the class rule already does, so a DCD + or device-stack change doesn't silently drop espressif's only-list tests + (e.g. hid_composite_freertos) that aren't in the shared device_tests list.""" + def test_dcd_change_includes_only_list_test(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = hil_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + tests = s['boards'][board] + self.assertIn('device/hid_composite_freertos', tests) + self.assertIn('device/cdc_msc_freertos', tests) + self.assertIn('device/audio_test_freertos', tests) + self.assertIn('device/usbtest', tests) + + def test_core_device_change_includes_only_list_test(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = hil_select.classify(['src/device/usbd.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + tests = s['boards'][board] + self.assertIn('device/hid_composite_freertos', tests) + self.assertIn('device/cdc_msc_freertos', tests) + self.assertIn('device/audio_test_freertos', tests) + self.assertIn('device/usbtest', tests) + + def test_host_change_does_not_leak_device_only_list_test(self): + s = hil_select.classify(['src/host/usbh.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board, tests in s['boards'].items(): + if tests == 'all': + continue + self.assertNotIn('device/hid_composite_freertos', tests, board) + + +class TestFamilies(unittest.TestCase): + """`families` exists for consumers that build (not just test) the diff: most + families have no rig board, so `boards` alone would compile nothing for them.""" + def test_off_rig_port_still_reports_family(self): + s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) # no same7x board on the rig + self.assertEqual(s['families'], ['same7x']) + + def test_port_families_are_reported(self): + s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + self.assertIn('rp2040', s['families']) + + def test_bsp_family_and_board_report_family(self): + self.assertEqual(sel(['hw/bsp/rp2040/family.cmake'])['families'], ['rp2040']) + self.assertEqual(sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h'])['families'], + ['rp2040']) + + def test_docs_only_has_no_families(self): + self.assertEqual(sel(['docs/info/contributing.rst'])['families'], []) + + def test_full_selection_still_reports_families(self): + """A full-matrix file must not hide the families of the other changed files: + consumers that build from `families` (e.g. /pre-pr) ignore `boards` when full.""" + s = sel(['src/common/tusb_fifo.c', 'src/portable/microchip/samx7x/dcd_samx7x.c']) + self.assertTrue(s['full']) + self.assertIn('same7x', s['families']) + # full stays full: every roster board, and no args to narrow the run + self.assertEqual(set(s['boards']), {b['name'] for b in ROSTER}) + self.assertTrue(all(v == 'all' for v in s['boards'].values())) + self.assertEqual(hil_select.selection_args(s, ROSTERS), {'tinyusb.json': ''}) + self.assertEqual(hil_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}}) + + def test_family_order_does_not_matter(self): + # same as above with the full-matrix file last (was the only order that worked) + s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c', 'src/common/tusb_fifo.c']) + self.assertTrue(s['full']) + self.assertIn('same7x', s['families']) + + +class TestGitDiffArgv(unittest.TestCase): + def test_diff_disables_rename_detection(self): + """Without --no-renames git reports only a rename's destination, so moving an + HIL-relevant file to a non-code path would be classified as non-code only.""" + self.assertIn('--no-renames', hil_select.GIT_DIFF_ARGV) + + +class TestPortWithoutFamilyIsFull(unittest.TestCase): + """A port dir no family file references must widen (full matrix), not silently + contribute zero boards — the fail-open contract.""" + def test_unreferenced_port_forces_full(self): + orig = hil_select.port_families + hil_select.port_families = lambda port_dir, repo_root: set() + try: + s = sel(['src/portable/vendor/newip/dcd_newip.c']) + finally: + hil_select.port_families = orig + self.assertTrue(s['full']) + self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons']) + + +class TestOpenocdVidPid(unittest.TestCase): + """The roster's optional flasher `vid_pid` field (openocd-verbatim, e.g. + "0x1a86 0x8010", more pairs appended) pins openocd's probe discovery so it + never opens foreign usbfs nodes. It must be emitted BEFORE the args: the + rescue cfgs run `init` internally (rp2350-rescue.cfg errors on any + config-stage command after its init; rp2040.cfg under RESCUE scans before a + trailing flag is even parsed), and no rig cfg sets a competing list + (the 2026-08-10 convoy mechanism).""" + + def test_vid_pid_flag_precedes_args(self): + cmd = hil_flash._openocd_cmd_base( + {'uid': 'S1', 'args': '-f target/wch-riscv.cfg', 'vid_pid': '0x1a86 0x8010'}) + self.assertIn('-c "adapter usb vid_pid 0x1a86 0x8010" -f target/wch-riscv.cfg', cmd) + self.assertTrue(cmd.endswith('-f target/wch-riscv.cfg'), cmd) + + def test_rescue_cfg_command_keeps_vid_pid_before_init(self): + """rescue_openocd swaps the target cfg for one that runs `init` internally; + a vid_pid flag after the args would error there (rp2350) or be skipped + (rp2040) -- in exactly the wedged-rig scenario the pin exists for.""" + flasher = {'name': 'openocd', 'uid': 'S1', 'vid_pid': '0x2e8a 0x000c', + 'args': '-c "set RESCUE 1" -f target/rp2040.cfg'} + cmd = hil_flash._openocd_cmd_base(flasher) + self.assertLess(cmd.index('adapter usb vid_pid'), cmd.index('-f target/'), cmd) + + def test_vid_pid_multiple_pairs(self): + cmd = hil_flash._openocd_cmd_base( + {'uid': 'S1', 'args': '-f i.cfg', 'vid_pid': '0x2e8a 0x000c 0x2e8a 0x000d'}) + self.assertIn('-c "adapter usb vid_pid 0x2e8a 0x000c 0x2e8a 0x000d"', cmd) + + def test_no_field_no_flag_but_warns(self): + # the roster lint only covers the committed rosters; a dev PC's local.json entry + # without the field must at least say what it is giving up -- on STDERR, since + # hil_test captures stdout per test and would swallow it on a passing run + import io + from contextlib import redirect_stderr + hil_flash._VID_PID_WARNED.discard('S-warn') + cap = io.StringIO() + with redirect_stderr(cap): + cmd = hil_flash._openocd_cmd_base({'uid': 'S-warn', 'args': '-f i.cfg'}) + self.assertNotIn('vid_pid', cmd) + self.assertIn('vid_pid', cap.getvalue()) + + def test_roster_openocd_entries_all_pin_vid_pid(self): + # every openocd probe on the rig has a known VID/PID; a new entry without the + # pin silently reintroduces open-everything discovery + for path, board in roster_flashers(): + f = board['flasher'] + # tinyusb.json only: hfp.json is the hifiphile rig owner's file, and a + # blocking repo-wide lint over someone else's roster would red every PR the + # moment they add an openocd board (hil_flash treats the field as optional) + if f['name'] == 'openocd' and path.endswith('tinyusb.json'): + self.assertIn('vid_pid', f, + f"{path}: {board['name']} openocd flasher lacks vid_pid") + self.assertNotIn('vid_pid', f.get('args', ''), + f"{path}: {board['name']} packs vid_pid into args; use the field") + + +class TestRosterFlashersDispatch(unittest.TestCase): + """hil_test and hil_pool_check resolve a board's flasher with a bare + getattr(hil_flash, f'flash_{name}'), and hil_test does it inside a redirect_stdout — + so a renamed or typo'd roster name raises an AttributeError whose output is swallowed, + with nothing pointing at the roster as the thing to edit. Renaming a flash_*/reset_* + pair without updating every roster must fail here instead.""" + + def test_flash_and_reset_exist_for_every_roster_flasher(self): + for path, board in roster_flashers(): + name = board['flasher']['name'].lower() + for fn in (f'flash_{name}', f'reset_{name}'): + self.assertTrue(callable(getattr(hil_flash, fn, None)), + f'{path}: {board["name"]} uses flasher "{name}" ' + f'but hil_flash.{fn} does not exist') + + def test_firmware_suffix_known_for_every_roster_flasher(self): + """find_firmware falls back to accepting .elf-or-.bin when a flasher is missing + from FLASHER_SUFFIX, silently restoring the mismatch that map exists to catch.""" + for path, board in roster_flashers(): + name = board['flasher']['name'].lower() + self.assertIn(name, hil_flash.FLASHER_SUFFIX, + f'{path}: {board["name"]} uses flasher "{name}" ' + f'with no hil_flash.FLASHER_SUFFIX entry') + + +class FlasherRecoverEntry(unittest.TestCase): + """Optional roster key: a SECOND flasher used only to deliver recovery while a usbfs + node is poisoned. Boards whose primary flasher cannot get past a convoy (jlink, + stlink, lm4flash) name an openocd entry here instead of changing how they are + normally flashed.""" + + def test_recover_flasher_prefers_the_optional_entry(self): + prim = {'name': 'jlink', 'uid': 'X', 'args': '-device MIMXRT1064xxx6A'} + rec = {'name': 'openocd', 'uid': 'X', 'args': '-f interface/jlink.cfg -f target/foo.cfg'} + self.assertEqual(hil_flash.recover_flasher({'flasher': prim, 'flasher_recover': rec}), rec) + self.assertEqual(hil_flash.recover_flasher({'flasher': prim}), prim) + + def test_openocd_over_jlink_is_convoy_safe_without_a_pin(self): + """libjaylink discovery returns early unless idVendor == 0x1366 (SEGGER) and the PID + is in its table, and only THEN calls libusb_open (discovery_usb.c) -- it never opens + a foreign node. `adapter usb vid_pid` is a no-op for this driver: jlink.c reads + adapter_serial / usb address / usb location, never the vid/pid.""" + self.assertTrue(hil_flash.convoy_safe( + {'name': 'openocd', 'args': '-f interface/jlink.cfg -f target/stm32f4x.cfg'})) + + def test_openocd_with_neither_a_pin_nor_jlink_is_not_safe(self): + self.assertFalse(hil_flash.convoy_safe( + {'name': 'openocd', 'args': '-f interface/stlink.cfg -f target/stm32h7x.cfg'})) + + def test_the_existing_rules_are_unchanged(self): + self.assertTrue(hil_flash.convoy_safe( + {'name': 'openocd', 'vid_pid': '0x2e8a 0x000c', 'args': '-f interface/cmsis-dap.cfg'})) + self.assertFalse(hil_flash.convoy_safe({'name': 'jlink', 'uid': 'X'})) + self.assertTrue(hil_flash.convoy_safe({'name': 'esptool'})) + + +if __name__ == '__main__': + unittest.main(verbosity=1) diff --git a/test/hil/test/test_hil_util.py b/test/hil/test/test_hil_util.py new file mode 100644 index 000000000..9c3d5edef --- /dev/null +++ b/test/hil/test/test_hil_util.py @@ -0,0 +1,230 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests for hil_util.run_cmd's binary/split_stderr/quiet modes — real subprocesses, no +# hardware. Stdlib + hil_util only (hil_util is stdlib-only), so the pre-commit hil-test +# hook can run this on GitHub's bare runner. Run directly: +# python3 test/hil/test/test_hil_util.py +import io +import os +import shutil +import tempfile +import sys +import time +import unittest +from contextlib import redirect_stdout +from pathlib import Path + +# the module under test lives in the parent dir's helper/ package +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +from helper import hil_util + + +@unittest.skipIf(os.name == 'nt', 'POSIX shell commands') +class RunCmdModes(unittest.TestCase): + def test_default_mode_unchanged(self): + r = hil_util.run_cmd('printf out; printf err >&2') + self.assertEqual(r.returncode, 0) + self.assertIsInstance(r.stdout, str) + # stderr merged into stdout, as every existing caller expects + self.assertIn('out', r.stdout) + self.assertIn('err', r.stdout) + + def test_binary_stdout_is_exact_bytes(self): + # \xff is not valid UTF-8: text mode would mangle it via errors='replace' + r = hil_util.run_cmd(r"printf 'a\377\000b'", binary=True) + self.assertEqual(r.returncode, 0) + self.assertEqual(r.stdout, b'a\xff\x00b') + + def test_split_stderr_keeps_stdout_clean(self): + r = hil_util.run_cmd('printf out; printf err >&2', split_stderr=True) + self.assertEqual(r.returncode, 0) + self.assertEqual(r.stdout, 'out') + self.assertEqual(r.stderr, 'err') + + def test_binary_split_stderr_timeout_returns_124(self): + t0 = time.monotonic() + r = hil_util.run_cmd(r"printf 'p\377re'; printf warn >&2; sleep 30", + binary=True, split_stderr=True, timeout=1) + self.assertEqual(r.returncode, 124) + # killpg + bounded communicate: well under sleep 30 + self.assertLess(time.monotonic() - t0, 15) + self.assertIn(b'p\xffre', r.stdout or b'') + # stderr collected before the timeout must survive the kill + self.assertIn(b'warn', r.stderr or b'') + + def test_text_mode_timeout_stdout_stays_str(self): + r = hil_util.run_cmd('sleep 30', timeout=1) + self.assertEqual(r.returncode, 124) + # a text-mode caller must never get bytes back, even empty + self.assertIsInstance(r.stdout, str) + + def test_failed_banner_includes_split_stderr(self): + # with split_stderr the diagnostic is in .stderr; the banner must not go blank. + # The text travels via env, not the command string — the banner title echoes the + # command, which would make a literal assertion pass vacuously. + os.environ['RUN_CMD_TEST_ERR'] = 'diagnostic-xyzzy' + self.addCleanup(os.environ.pop, 'RUN_CMD_TEST_ERR', None) + cap = io.StringIO() + with redirect_stdout(cap): + r = hil_util.run_cmd('printf "$RUN_CMD_TEST_ERR" >&2; exit 3', split_stderr=True) + self.assertEqual(r.returncode, 3) + self.assertIn('COMMAND FAILED', cap.getvalue()) + self.assertIn('diagnostic-xyzzy', cap.getvalue()) + + def test_no_group_markers_when_stdout_is_captured(self): + # GitHub folds ::group:: only at line start of the JOB's real stdout. Pool + # workers run tests under redirect_stdout and compact the capture into one + # row line, where the markers land mid-line and render as literal noise. + saved_ci = os.environ.get('CI') # pre-exists on GitHub runners: restore, not pop + os.environ['CI'] = '1' + self.addCleanup(lambda: os.environ.update({'CI': saved_ci}) if saved_ci is not None + else os.environ.pop('CI', None)) + cap = io.StringIO() + with redirect_stdout(cap): + r = hil_util.run_cmd('printf boom; exit 3') + self.assertEqual(r.returncode, 3) + self.assertIn('COMMAND FAILED', cap.getvalue()) + self.assertNotIn('::group::', cap.getvalue()) + self.assertNotIn('::endgroup::', cap.getvalue()) + + def test_quiet_suppresses_failed_banner(self): + # retry-loop callers report failures themselves; per-poll banners are noise + cap = io.StringIO() + with redirect_stdout(cap): + r = hil_util.run_cmd('printf boom >&2; exit 3', quiet=True) + self.assertEqual(r.returncode, 3) + self.assertNotIn('COMMAND FAILED', cap.getvalue()) + + +class BottomLayer(unittest.TestCase): + def test_bad_timeout_env_falls_back(self): + # hil_select (the PR-diff selector) imports hil_util for the example rosters; + # a malformed HIL_CMD_TIMEOUT must not crash the selector at import and knock + # CI back to the full-matrix fallback + import subprocess + r = subprocess.run( + [sys.executable, '-c', 'from helper import hil_util; print(hil_util.CMD_TIMEOUT)'], + cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + env={**os.environ, 'HIL_CMD_TIMEOUT': 'bogus'}, + capture_output=True, text=True, timeout=30) + self.assertEqual(r.returncode, 0, r.stderr) + # the warning must NOT be on stdout: hil_select's stdout is machine-read JSON + self.assertEqual(r.stdout.strip(), '180') + self.assertIn('warning', r.stderr) # but a silent fallback hides the misconfiguration + + def test_tinyusb_root_is_the_repo_root(self): + # the constant is derived from __file__ parents[N]; moving hil_util.py without + # adjusting N silently re-points every firmware/build path (it happened) + self.assertTrue((hil_util.TINYUSB_ROOT / 'examples').is_dir(), hil_util.TINYUSB_ROOT) + self.assertTrue((hil_util.TINYUSB_ROOT / 'test' / 'hil').is_dir(), hil_util.TINYUSB_ROOT) + + def test_hil_util_is_a_single_module_instance(self): + # helper modules must be imported via the helper package everywhere: a plain + # `import hil_util` from inside helper/ creates a SECOND module object, and + # state like `verbose` set on one copy never reaches the other + import hil_flash + from helper import hil_pool_check + self.assertIs(hil_flash.hil_util, hil_util) + self.assertIs(hil_pool_check.hil_util, hil_util) + self.assertIs(hil_pool_check.hil_flash, hil_flash) + + def test_bare_runner_modules_stay_stdlib_only(self): + # hil_examples.py used to make this structural (a list of strings cannot grow a + # dependency); with the rosters folded into hil_util the invariant needs teeth: + # everything the bare GitHub runner imports (selector + this suite) must stay + # stdlib + local. Adding pyserial/pymtp here breaks hil_select on CI. + import ast + hil_dir = Path(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + # ONLY the modules the bare runner can import -- not every stem in the tree. + # Globbing the directory allowed `import pymtp` (and hil_test, usbtest, + # mtp_test) through, so the pymtp case this test names could never fail: that + # module runs ctypes.CDLL(find_library('mtp')) at import and raises where there + # is no libmtp, taking hil_select down with it. + local = {'helper', 'hil_util', 'hil_select', 'hil_flash', + 'hil_health', 'hil_lock', 'hil_pool_check'} + allowed = set(sys.stdlib_module_names) | local + # hil_pool_check included: test_hil_util_is_a_single_module_instance imports it + # on the bare runner, and its `import serial` is function-local for exactly + # this reason -- hoisting it must fail HERE, not on every PR's pre-commit CI + for mod in ('helper/hil_util', 'hil_flash', 'helper/hil_select', + 'helper/hil_health', 'helper/hil_lock', 'helper/hil_pool_check'): + tree = ast.parse((hil_dir / f'{mod}.py').read_text()) + # module level only: a deferred import inside a function cannot break + # importability (hil_pool_check keeps `import serial` function-local + # for exactly that reason) + for node in tree.body: + roots = [] + if isinstance(node, ast.Import): + roots = [a.name.split('.')[0] for a in node.names] + elif isinstance(node, ast.ImportFrom) and node.module: + roots = [node.module.split('.')[0]] + for root in roots: + self.assertIn(root, allowed, + f'{mod}.py imports {root}, not stdlib/local - breaks the bare CI runner') + + +class BoundedReadBookkeeping(unittest.TestCase): + """Two ways the strand accounting lied, both of which cost a blindness credit -- and + the process goes blind after four.""" + + def test_a_value_that_arrived_at_the_deadline_is_not_a_strand(self): + """join() returns, is_alive() is still True, but the reader HAS deposited its + value. read_sysfs booked a strand from is_alive() alone, so a merely-slow healthy + read was memoised as unreadable forever. bounded_open already gets this right.""" + import threading, time as _t + before = hil_util._sysfs_stuck + self.addCleanup(setattr, hil_util, '_sysfs_stuck', before) + real_thread = threading.Thread + + class Lingering(real_thread): + """Deposits the value, then outlives the join by a hair.""" + def run(self): + super().run() + _t.sleep(0.6) # still alive when join(grace) returns + + self.addCleanup(setattr, threading, 'Thread', real_thread) + threading.Thread = Lingering + with tempfile.NamedTemporaryFile('w', suffix='_attr', delete=False) as fh: + fh.write('cafe\n') + path = fh.name + self.addCleanup(os.unlink, path) + hil_util.read_sysfs(path, grace=0.2) + self.assertEqual(hil_util._sysfs_stuck, before, + 'a value that arrived was still counted as a strand') + + def test_bounded_open_does_not_re_strand_a_known_path(self): + """Same rule read_sysfs has: re-opening a path known to hang costs another thread, + another fd and another blindness credit to learn what we already know. The printer + test re-opens ONE lp node on every retry.""" + d = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, d, True) + fifo = os.path.join(d, 'lp0') + os.mkfifo(fifo) # open() blocks: no writer, ever + self.addCleanup(setattr, hil_util, '_sysfs_stuck', hil_util._sysfs_stuck) + self.addCleanup(setattr, hil_util, '_sysfs_stranded', dict(hil_util._sysfs_stranded)) + before = hil_util._sysfs_stuck + for _ in range(3): + hil_util.bounded_open(fifo, os.O_WRONLY, 0.3) + self.assertLessEqual(hil_util._sysfs_stuck - before, 1, + 'each retry spent another blindness credit on the same path') + + +class RunAlongsideKeepsStderrOffThePayload(unittest.TestCase): + """test_device_printer_to_cdc byte-compares run_alongside's stdout against the payload + it wrote. Merging stderr into that stream turns any stray child stderr byte -- a + PYTHONWARNINGS chirp, a sitecustomize print, a venv .pth deprecation -- into + 'CDC->Printer wrong data', sending a maintainer after the printer class driver for an + interpreter warning. hil_ci.sh runs python3 with no isolating flags.""" + + def test_child_stderr_does_not_contaminate_stdout(self): + from helper import hil_util + argv = [sys.executable, '-c', + 'import sys; sys.stderr.write("noise\\n"); sys.stdout.write("PAYLOAD")'] + r = hil_util.run_alongside(argv, lambda: time.sleep(0.2), timeout=20) + self.assertEqual(r.returncode, 0) + self.assertEqual(r.stdout, b'PAYLOAD', + 'child stderr leaked into the payload stream') + + +if __name__ == '__main__': + unittest.main() diff --git a/test/hil/test_hil_select.py b/test/hil/test_hil_select.py deleted file mode 100644 index 5e6b16759..000000000 --- a/test/hil/test_hil_select.py +++ /dev/null @@ -1,581 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MIT -# Unit tests for hil_select.py — pure logic, no hardware, no git. Run directly: -# python3 test/hil/test_hil_select.py -import glob -import json -import os -import sys -import unittest - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -import hil_flash -import hil_select -from hil_examples import device_tests, dual_tests, host_test - -REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - - -def real_rosters(): - """The actual rig rosters, for regression tests that need real-world data - (a specific board/family/only-list) rather than the synthetic ROSTER above.""" - rosters = [] - for name in ('tinyusb.json', 'hfp.json'): - path = os.path.join(REPO, 'test/hil', name) - with open(path) as f: - rosters.append((f'test/hil/{name}', json.load(f)['boards'])) - return rosters - - -def roster_flashers(): - """(roster path, board) for every board in the live rosters, `boards-skip` - included: a parked board's flasher name must still dispatch, so that unparking it - is not what discovers the name went stale.""" - for name in ('tinyusb.json', 'hfp.json'): - path = os.path.join(REPO, 'test/hil', name) - with open(path) as f: - cfg = json.load(f) - for key in ('boards', 'boards-skip'): - for b in cfg.get(key, []): - yield f'test/hil/{name}', b - - -def on_roster(tc, *names): - """The subset of `names` currently in the live rig rosters, skipping the test - when none are. Parking/unparking a board is routine rig maintenance and must not - fail this suite: CI runs it right before the selector and treats a failure as - 'selector unusable', dropping PR scoping and annotating the run.""" - have = {b['name'] for _, boards in real_rosters() for b in boards} - got = [n for n in names if n in have] - if not got: - tc.skipTest(f'not in the rig roster: {", ".join(names)}') - return got - - -ROSTER = [ - # device-only, rp2040 family - {'name': 'raspberry_pi_pico', 'uid': 'u1', 'flasher': {'name': 'openocd'}, - 'tests': {'device': True, 'host': True, 'dual': True}}, - # device-only, stm32f4 family - {'name': 'stm32f407disco', 'uid': 'u2', 'flasher': {'name': 'jlink'}, - 'tests': {'device': True, 'host': False, 'dual': False}}, - # host-only board - {'name': 'raspberry_pi_pico2', 'uid': 'u3', 'flasher': {'name': 'openocd'}, - 'tests': {'device': False, 'host': True, 'dual': False}}, - # only-list board (espressif-style), flashed by the CI leg that splits on esptool - {'name': 'espressif_s3_devkitm', 'uid': 'u4', 'flasher': {'name': 'esptool'}, - 'tests': {'only': ['device/cdc_msc_freertos', 'host/device_info']}}, -] -ROSTERS = [('test/hil/tinyusb.json', ROSTER)] - - -def sel(files): - return hil_select.classify(files, REPO, ROSTERS) - - -class TestPortRule(unittest.TestCase): - def test_dcd_rp2040_selects_pico_family_only(self): - s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) - self.assertFalse(s['full']) - self.assertIn('raspberry_pi_pico', s['boards']) - self.assertNotIn('stm32f407disco', s['boards']) - self.assertNotIn('espressif_s3_devkitm', s['boards']) - # device role: no host tests in pico's list - self.assertTrue(all(not t.startswith('host/') for t in s['boards']['raspberry_pi_pico'])) - # host-only boards drop out entirely on a device-role change - self.assertNotIn('raspberry_pi_pico2', s['boards']) - - def test_shared_port_file_is_both_roles(self): - s = sel(['src/portable/synopsys/dwc2/dwc2_common.c']) - self.assertFalse(s['full']) - self.assertNotIn('raspberry_pi_pico', s['boards']) # rp2040 is not a dwc2 family - self.assertIn('stm32f407disco', s['boards']) # stm32f4 is - - -class TestCoreRoleRule(unittest.TestCase): - def test_usbd_selects_all_device_tests_everywhere(self): - s = sel(['src/device/usbd.c']) - self.assertFalse(s['full']) - self.assertNotIn('raspberry_pi_pico2', s['boards']) # host-only board dropped - pico = s['boards']['raspberry_pi_pico'] - self.assertTrue(set(device_tests).issubset(set(pico))) - self.assertTrue(set(dual_tests).issubset(set(pico))) # dual survives device role - self.assertTrue(all(not t.startswith('host/') for t in pico)) - # only-list board: selection intersects its only-list - esp = s['boards']['espressif_s3_devkitm'] - self.assertEqual(esp, ['device/cdc_msc_freertos']) - - def test_host_change_drops_device(self): - s = sel(['src/host/usbh.c']) - self.assertFalse(s['full']) - self.assertIn('raspberry_pi_pico2', s['boards']) - self.assertNotIn('stm32f407disco', s['boards']) # device-only board dropped - - -class TestClassRule(unittest.TestCase): - def test_cdc_device_selects_cdc_examples_only(self): - s = sel(['src/class/cdc/cdc_device.c']) - self.assertFalse(s['full']) - pico = s['boards']['raspberry_pi_pico'] - self.assertIn('device/cdc_msc', pico) - self.assertIn('device/cdc_dual_ports', pico) - self.assertNotIn('device/msc_dual_lun', pico) # CFG_TUD_CDC 0 there - self.assertNotIn('device/usbtest', pico) # CFG_TUD_CDC 0 there - self.assertTrue(all(not t.startswith('host/') for t in pico)) - - def test_msc_host_selects_host_side(self): - s = sel(['src/class/msc/msc_host.c']) - self.assertFalse(s['full']) - self.assertNotIn('stm32f407disco', s['boards']) # device-only board - pico2 = s['boards']['raspberry_pi_pico2'] - self.assertIn('host/msc_file_explorer', pico2) - self.assertTrue(all(not t.startswith('device/') for t in pico2)) - - -class TestClassIncludeEdges(unittest.TestCase): - """A class header another class includes reaches that class's examples too. - src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, so - midi_test's firmware contains audio.h - but the class rule derives macros from - the directory name alone, so an audio.h change used to select only - device/audio_test_freertos. On boards that skip that example the per-board - intersection emptied and an audio.h-only PR ran ZERO HIL on them.""" - def test_edges_derived_from_includes(self): - edges = hil_select.class_include_edges(REPO) - self.assertEqual(edges.get('audio/audio.h'), {'midi'}) - self.assertEqual(edges.get('cdc/cdc.h'), {'net'}) - - def test_audio_header_selects_midi_example(self): - s = hil_select.classify(['src/class/audio/audio.h'], REPO, real_rosters()) - self.assertFalse(s['full']) - # every board that runs device/midi_test at all must run it here (boards with - # a tests.only list, e.g. espressif, run the freertos examples instead) - by_name = {b['name']: b for _, bs in real_rosters() for b in bs} - checked = 0 - for name, tests in s['boards'].items(): - if 'device/midi_test' in hil_select.board_tests(by_name[name]): - self.assertIn('device/midi_test', tests, name) - checked += 1 - self.assertTrue(checked) - - def test_audio_header_reaches_boards_that_skip_audio(self): - # both skip device/audio_test_freertos: without the midi edge their - # intersection is empty and they drop out of the selection entirely - boards = on_roster(self, 'metro_m4_express', 'nrf54lm20dk') - s = hil_select.classify(['src/class/audio/audio.h'], REPO, real_rosters()) - for board in boards: - self.assertEqual(s['boards'].get(board), ['device/midi_test'], board) - - def test_edge_is_per_header_not_per_class(self): - # midi includes audio.h, not audio_device.h: an audio_device change must - # not drag midi's examples in - s = hil_select.classify(['src/class/audio/audio_device.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for tests in s['boards'].values(): - if tests != 'all': - self.assertNotIn('device/midi_test', tests) - - -class TestFallbackRules(unittest.TestCase): - def test_unknown_tool_is_full(self): - s = sel(['tools/random_new_script.py']) - self.assertTrue(s['full']) - - def test_docs_only_is_empty_not_full(self): - s = sel(['docs/info/contributing.rst', 'README.rst']) - self.assertFalse(s['full']) - self.assertEqual(s['boards'], {}) - - def test_bsp_family_selects_family_boards(self): - s = sel(['hw/bsp/rp2040/family.cmake']) - self.assertFalse(s['full']) - self.assertIn('raspberry_pi_pico', s['boards']) - self.assertEqual(s['boards']['raspberry_pi_pico'], 'all') - self.assertNotIn('stm32f407disco', s['boards']) - - def test_bsp_board_narrows_to_board(self): - s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) - self.assertFalse(s['full']) - self.assertEqual(list(s['boards'].keys()), ['raspberry_pi_pico']) - - def test_example_change_selects_that_example(self): - s = sel(['examples/device/cdc_msc/src/main.c']) - self.assertFalse(s['full']) - self.assertEqual(s['boards']['raspberry_pi_pico'], ['device/cdc_msc']) - - def test_core_common_is_full(self): - for f in ['src/tusb.c', 'src/common/tusb_fifo.c', 'src/osal/osal_freertos.h']: - self.assertTrue(sel([f])['full'], f) - - def test_board_test_example_is_full(self): - # board_test is the park/teardown firmware hil_test.py flashes on every board, - # not an unlisted example: a regression there must not skip the whole rig - for f in ['examples/device/board_test/src/main.c', - 'examples/device/board_test/CMakeLists.txt']: - self.assertTrue(sel([f])['full'], f) - - def test_harness_is_full(self): - for f in ['test/hil/hil_test.py', '.github/workflows/build.yml', 'hw/mcu/nxp/x.c', 'lib/foo/x.c']: - self.assertTrue(sel([f])['full'], f) - - def test_mixed_roles_no_pruning(self): - s = sel(['src/device/usbd.c', 'src/host/usbh.c']) - self.assertFalse(s['full']) - self.assertIn('raspberry_pi_pico2', s['boards']) - self.assertIn('stm32f407disco', s['boards']) - - def test_cmakelists_and_requirements_are_full(self): - for f in ['src/CMakeLists.txt', 'examples/CMakeLists.txt', - 'examples/device/CMakeLists.txt', 'test/hil/requirements.txt']: - self.assertTrue(sel([f])['full'], f) - - def test_docs_txt_is_noncode(self): - s = sel(['docs/info/changelog.txt']) - self.assertFalse(s['full']) - self.assertEqual(s['boards'], {}) - - -class TestArgsEmission(unittest.TestCase): - def test_args_for_scoped_selection(self): - s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) - args = hil_select.selection_args(s, ROSTERS) - a = args['tinyusb.json'] - self.assertIn('-b raspberry_pi_pico', a) - self.assertNotIn('stm32f407disco', a) - self.assertIn('-bt raspberry_pi_pico:', a) # device-only subset of a device+host board - - def test_args_full_is_empty(self): - s = sel(['tools/random_new_script.py']) - self.assertEqual(hil_select.selection_args(s, ROSTERS), {'tinyusb.json': ''}) - - def test_args_all_board_gets_bare_b(self): - s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) - a = hil_select.selection_args(s, ROSTERS)['tinyusb.json'] - self.assertIn('-b raspberry_pi_pico', a) - self.assertNotIn('-bt', a) - - def test_args_by_flasher_splits_esp_from_the_rest(self): - s = sel(['src/device/usbd.c']) - per = hil_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json'] - self.assertIn('espressif_s3_devkitm', per['esptool']) - self.assertIn('raspberry_pi_pico', per['openocd']) - self.assertNotIn('espressif_s3_devkitm', per.get('openocd', '') + per.get('jlink', '')) - - def test_args_by_flasher_omits_a_flasher_with_no_selected_board(self): - # the esp CI leg must see no args at all here, not a filter matching zero boards - s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) - per = hil_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json'] - self.assertEqual(per, {'openocd': '-b raspberry_pi_pico'}) - - def test_args_by_flasher_full_is_empty(self): - s = sel(['tools/random_new_script.py']) - self.assertEqual(hil_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}}) - - def test_cli_diff_file(self): - import subprocess, tempfile, json as j - with tempfile.NamedTemporaryFile('w', suffix='.txt', delete=False) as f: - f.write('src/class/cdc/cdc_device.c\n') - path = f.name - r = subprocess.run([sys.executable, os.path.join(REPO, 'test/hil/hil_select.py'), - '--diff-file', path, os.path.join(REPO, 'test/hil/tinyusb.json')], - capture_output=True, text=True) - self.assertEqual(r.returncode, 0, r.stderr) - out = j.loads(r.stdout) - self.assertFalse(out['full']) - self.assertIn('tinyusb.json', out['args']) - self.assertTrue(any('cdc_device' in line for line in out['reasons'])) - os.unlink(path) - - -class TestRealRosterPortFamilies(unittest.TestCase): - """Regression for port_families() missing espressif's dwc2 reference, which - lives in a component CMakeLists.txt rather than family.cmake/family.mk.""" - def test_dwc2_change_selects_espressif_boards(self): - boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') - s = hil_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - self.assertIn(board, s['boards']) - - -class TestOptionGatedPort(unittest.TestCase): - """Regression: family_support.cmake compiles some ports from a build option - (MAX3421_HOST=1 -> hcd_max3421.c), so a board's family file never names them.""" - # host-side option board (max3421 as host controller), off any max3421 family - OPT_ROSTER = [('test/hil/opt.json', [ - {'name': 'fake_dual_board', 'uid': 'o1', 'flasher': {'name': 'jlink'}, - 'build': {'args': ['MAX3421_HOST=1']}, - 'tests': {'device': True, 'host': False, 'dual': True}}, - {'name': 'fake_host_board', 'uid': 'o2', 'flasher': {'name': 'jlink'}, - 'variant': [{'name': 'fake_host_board', 'flags': '-DMAX3421_HOST=1'}], - 'tests': {'device': False, 'host': True, 'dual': False}}, - {'name': 'fake_off_board', 'uid': 'o3', 'flasher': {'name': 'jlink'}, - 'variant': [{'name': 'fake_off_board', 'defines': ['MAX3421_HOST=0']}], - 'tests': {'device': True, 'host': True, 'dual': True}}, - ])] - - def test_real_roster_max3421_selects_option_board(self): - boards = on_roster(self, 'metro_m4_express') - s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - self.assertIn(board, s['boards']) - - def test_option_selects_via_args_defines_and_flags(self): - s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, self.OPT_ROSTER) - self.assertFalse(s['full']) - self.assertIn('fake_dual_board', s['boards']) # build.args - self.assertIn('fake_host_board', s['boards']) # variant flags - self.assertNotIn('fake_off_board', s['boards']) # variant defines, but =0 - - def test_device_role_port_does_not_pull_host_only_option_board(self): - s = hil_select.classify(['src/portable/analog/max3421/dcd_max3421.c'], REPO, self.OPT_ROSTER) - self.assertFalse(s['full']) - self.assertNotIn('fake_host_board', s['boards']) # host-only board, device change - self.assertIn('fake_dual_board', s['boards']) # device-capable option board - - def test_gates_parsed_from_family_support(self): - self.assertEqual(hil_select.port_option_gates(REPO).get('analog/max3421'), - {'MAX3421_HOST'}) - - def test_board_cmake_option_counts(self): - """A board can enable a gated port in its own BSP rather than via the roster - (hw/bsp/espressif/boards/*/board.cmake -> set(MAX3421_HOST 1)); board_options() - must see those too, or such a board joining the roster is silently dropped.""" - self.assertIn('MAX3421_HOST', - hil_select.bsp_board_options('adafruit_feather_esp32s3', REPO)) - self.assertIn('CFG_TUH_RPI_PIO_USB', - hil_select.bsp_board_options('adafruit_fruit_jam', REPO)) - # commented-out `# set(MAX3421_HOST 1)` must not count - self.assertNotIn('MAX3421_HOST', - hil_select.bsp_board_options('feather_nrf52840_express', REPO)) - - def test_board_cmake_option_selects_off_family_board(self): - # adafruit_feather_esp32s3 is not on any rig roster; stand it in as one to - # prove the BSP-sourced option alone pulls a max3421 change onto the board - roster = [('test/hil/opt.json', [ - {'name': 'adafruit_feather_esp32s3', 'uid': 'o1', 'flasher': {'name': 'esptool'}, - 'tests': {'device': False, 'host': True, 'dual': False}}])] - s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster) - self.assertFalse(s['full']) - self.assertIn('adafruit_feather_esp32s3', s['boards']) - - def test_board_mk_option_is_ignored(self): - """Make-only options must not select: HIL CI builds with CMake exclusively, so - hw/bsp/nrf/boards/nrf5340dk/board.mk's MAX3421_HOST compiles nothing here.""" - roster = [('test/hil/opt.json', [ - {'name': 'nrf5340dk', 'uid': 'o1', 'flasher': {'name': 'jlink'}, - 'tests': {'device': False, 'host': True, 'dual': False}}])] - s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster) - self.assertFalse(s['full']) - self.assertEqual(s['boards'], {}) - - -class TestPortFamiliesCmakeOnly(unittest.TestCase): - """port_families() is CMake-only (HIL CI never builds with Make) and matches on - 'port_dir/' so a port dir is not a prefix of a sibling.""" - def test_make_only_family_is_not_a_family(self): - # hw/bsp/pic32mz has family.mk but no family.cmake - self.assertEqual(hil_select.port_families('microchip/pic32mz', REPO), set()) - - def test_prefix_port_does_not_inherit_sibling_families(self): - # bare-substring matching let 'microchip/pic' match '.../microchip/pic32mz/...' - self.assertEqual(hil_select.port_families('microchip/pic', REPO), set()) - - def test_make_only_port_forces_full(self): - s = sel(['src/portable/microchip/pic32mz/dcd_pic32mz.c']) - self.assertTrue(s['full']) - self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons']) - - def test_cmake_families_still_found(self): - self.assertEqual(hil_select.port_families('raspberrypi/rp2040', REPO), {'rp2040'}) - self.assertIn('stm32f4', hil_select.port_families('synopsys/dwc2', REPO)) - - -class TestPortFamiliesCoverage(unittest.TestCase): - """Systematic guard: every real dcd_*/hcd_* port directory should map to at - least one board family, so a future family.cmake/CMakeLists.txt layout that - port_families() doesn't scan fails loudly instead of silently dropping boards - (as espressif's dwc2 reference did - see TestRealRosterPortFamilies).""" - # Ports with no board family: not a bug, just not wired into any rig board. - # Add here (with a reason) only if port_families() legitimately can't find one. - # A port listed here force-fulls (fail-open), so it is never under-selected. - NO_FAMILY = { - 'template', # reference/example port, not built by any board - # hw/bsp/pic32mz has family.mk only (no family.cmake), and port_families() - # is CMake-only because HIL CI builds every board with CMake - so this port - # is compiled for no HIL board. - 'microchip/pic32mz', - 'microchip/pic', # same: only ever referenced from pic32mz's family.mk - } - - @staticmethod - def _dcd_hcd_ports(): - portable_root = os.path.join(REPO, 'src/portable') - ports = [] - for entry in sorted(os.listdir(portable_root)): - d = os.path.join(portable_root, entry) - if not os.path.isdir(d): - continue - if glob.glob(os.path.join(d, 'dcd_*.c')) or glob.glob(os.path.join(d, 'hcd_*.c')): - ports.append(entry) - continue - for sub in sorted(os.listdir(d)): - sd = os.path.join(d, sub) - if os.path.isdir(sd) and (glob.glob(os.path.join(sd, 'dcd_*.c')) or - glob.glob(os.path.join(sd, 'hcd_*.c'))): - ports.append(f'{entry}/{sub}') - return ports - - def test_every_port_maps_to_a_family(self): - ports = self._dcd_hcd_ports() - self.assertTrue(ports) # sanity: the scan itself found something - for port in ports: - if port in self.NO_FAMILY: - continue - fams = hil_select.port_families(port, REPO) - self.assertTrue(fams, f'{port}: no family references this port ' - f'(port_families() scan gap, or add to NO_FAMILY)') - - -class TestRealRosterOnlyListTests(unittest.TestCase): - """Regression for roster-only-list tests (e.g. espressif's hid_composite_freertos) - being invisible to the selector because it only knew the shared hil_examples lists.""" - def test_only_list_example_change_selects_it(self): - boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') - s = hil_select.classify(['examples/device/hid_composite_freertos/src/main.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - self.assertEqual(s['boards'][board], ['device/hid_composite_freertos']) - - def test_class_change_includes_only_list_boards(self): - boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') - s = hil_select.classify(['src/class/hid/hid_device.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - self.assertIn(board, s['boards']) - - -class TestPortAndCoreRoleUseExtras(unittest.TestCase): - """Regression: the port rule and core-role rule must thread the roster-only - test universe (extras) the same way the class rule already does, so a DCD - or device-stack change doesn't silently drop espressif's only-list tests - (e.g. hid_composite_freertos) that aren't in the shared device_tests list.""" - def test_dcd_change_includes_only_list_test(self): - boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') - s = hil_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - tests = s['boards'][board] - self.assertIn('device/hid_composite_freertos', tests) - self.assertIn('device/cdc_msc_freertos', tests) - self.assertIn('device/audio_test_freertos', tests) - self.assertIn('device/usbtest', tests) - - def test_core_device_change_includes_only_list_test(self): - boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') - s = hil_select.classify(['src/device/usbd.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - tests = s['boards'][board] - self.assertIn('device/hid_composite_freertos', tests) - self.assertIn('device/cdc_msc_freertos', tests) - self.assertIn('device/audio_test_freertos', tests) - self.assertIn('device/usbtest', tests) - - def test_host_change_does_not_leak_device_only_list_test(self): - s = hil_select.classify(['src/host/usbh.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board, tests in s['boards'].items(): - if tests == 'all': - continue - self.assertNotIn('device/hid_composite_freertos', tests, board) - - -class TestFamilies(unittest.TestCase): - """`families` exists for consumers that build (not just test) the diff: most - families have no rig board, so `boards` alone would compile nothing for them.""" - def test_off_rig_port_still_reports_family(self): - s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c']) - self.assertFalse(s['full']) - self.assertEqual(s['boards'], {}) # no same7x board on the rig - self.assertEqual(s['families'], ['same7x']) - - def test_port_families_are_reported(self): - s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) - self.assertIn('rp2040', s['families']) - - def test_bsp_family_and_board_report_family(self): - self.assertEqual(sel(['hw/bsp/rp2040/family.cmake'])['families'], ['rp2040']) - self.assertEqual(sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h'])['families'], - ['rp2040']) - - def test_docs_only_has_no_families(self): - self.assertEqual(sel(['docs/info/contributing.rst'])['families'], []) - - def test_full_selection_still_reports_families(self): - """A full-matrix file must not hide the families of the other changed files: - consumers that build from `families` (e.g. /pre-pr) ignore `boards` when full.""" - s = sel(['src/common/tusb_fifo.c', 'src/portable/microchip/samx7x/dcd_samx7x.c']) - self.assertTrue(s['full']) - self.assertIn('same7x', s['families']) - # full stays full: every roster board, and no args to narrow the run - self.assertEqual(set(s['boards']), {b['name'] for b in ROSTER}) - self.assertTrue(all(v == 'all' for v in s['boards'].values())) - self.assertEqual(hil_select.selection_args(s, ROSTERS), {'tinyusb.json': ''}) - self.assertEqual(hil_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}}) - - def test_family_order_does_not_matter(self): - # same as above with the full-matrix file last (was the only order that worked) - s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c', 'src/common/tusb_fifo.c']) - self.assertTrue(s['full']) - self.assertIn('same7x', s['families']) - - -class TestGitDiffArgv(unittest.TestCase): - def test_diff_disables_rename_detection(self): - """Without --no-renames git reports only a rename's destination, so moving an - HIL-relevant file to a non-code path would be classified as non-code only.""" - self.assertIn('--no-renames', hil_select.GIT_DIFF_ARGV) - - -class TestPortWithoutFamilyIsFull(unittest.TestCase): - """A port dir no family file references must widen (full matrix), not silently - contribute zero boards — the fail-open contract.""" - def test_unreferenced_port_forces_full(self): - orig = hil_select.port_families - hil_select.port_families = lambda port_dir, repo_root: set() - try: - s = sel(['src/portable/vendor/newip/dcd_newip.c']) - finally: - hil_select.port_families = orig - self.assertTrue(s['full']) - self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons']) - - -class TestRosterFlashersDispatch(unittest.TestCase): - """hil_test and hil_pool_check resolve a board's flasher with a bare - getattr(hil_flash, f'flash_{name}'), and hil_test does it inside a redirect_stdout — - so a renamed or typo'd roster name raises an AttributeError whose output is swallowed, - with nothing pointing at the roster as the thing to edit. Renaming a flash_*/reset_* - pair without updating every roster must fail here instead.""" - - def test_flash_and_reset_exist_for_every_roster_flasher(self): - for path, board in roster_flashers(): - name = board['flasher']['name'].lower() - for fn in (f'flash_{name}', f'reset_{name}'): - self.assertTrue(callable(getattr(hil_flash, fn, None)), - f'{path}: {board["name"]} uses flasher "{name}" ' - f'but hil_flash.{fn} does not exist') - - def test_firmware_suffix_known_for_every_roster_flasher(self): - """find_firmware falls back to accepting .elf-or-.bin when a flasher is missing - from FLASHER_SUFFIX, silently restoring the mismatch that map exists to catch.""" - for path, board in roster_flashers(): - name = board['flasher']['name'].lower() - self.assertIn(name, hil_flash.FLASHER_SUFFIX, - f'{path}: {board["name"]} uses flasher "{name}" ' - f'with no hil_flash.FLASHER_SUFFIX entry') - - -if __name__ == '__main__': - unittest.main(verbosity=1) diff --git a/test/hil/tinyusb.json b/test/hil/tinyusb.json index c9b38992c..549a17cd0 100644 --- a/test/hil/tinyusb.json +++ b/test/hil/tinyusb.json @@ -149,6 +149,7 @@ "flasher": { "name": "openocd", "uid": "E6614C311B597D32", + "vid_pid": "0x2e8a 0x000c", "args": "-f interface/cmsis-dap.cfg -f target/max32665.cfg", "verify": true } @@ -240,6 +241,7 @@ "flasher": { "name": "openocd", "uid": "E6614103E72C1D2F", + "vid_pid": "0x2e8a 0x000c", "args": "-f interface/cmsis-dap.cfg -f target/rp2040.cfg -c \"adapter speed 5000\"", "verify": true } @@ -270,6 +272,7 @@ "flasher": { "name": "openocd", "uid": "E6633861A3819D38", + "vid_pid": "0x2e8a 0x000c", "args": "-f interface/cmsis-dap.cfg -f target/rp2040.cfg -c \"adapter speed 5000\"", "verify": true }, @@ -296,6 +299,7 @@ "flasher": { "name": "openocd", "uid": "E6633861A3978538", + "vid_pid": "0x2e8a 0x000c", "args": "-f interface/cmsis-dap.cfg -f target/rp2350.cfg -c \"adapter speed 5000\"", "verify": true } @@ -326,6 +330,7 @@ "flasher": { "name": "openocd", "uid": "E663AC91D3359B38", + "vid_pid": "0x2e8a 0x000c", "args": "-f interface/cmsis-dap.cfg -f target/rp2350.cfg -c \"adapter speed 5000\"", "verify": true } @@ -407,9 +412,8 @@ "dual": false }, "flasher": { - "name": "openocd", + "name": "stlink", "uid": "004C00343137510F39383538", - "args": "-f interface/stlink.cfg -f target/stm32h7x.cfg", "verify": true } }, @@ -422,9 +426,8 @@ "dual": false }, "flasher": { - "name": "openocd", + "name": "stlink", "uid": "066FFF495087534867063844", - "args": "-f interface/stlink.cfg -f target/stm32g0x.cfg", "verify": true }, "comment": "32-bit scheme, 2KB USB SRAM" @@ -472,6 +475,7 @@ "flasher": { "name": "openocd", "uid": "A76D8F062C2A", + "vid_pid": "0x1a86 0x8010", "args": "-f target/wch-riscv.cfg", "verify": false } @@ -488,6 +492,7 @@ "flasher": { "name": "openocd", "uid": "BC4954081051", + "vid_pid": "0x1a86 0x8010", "args": "-f target/wch-riscv.cfg", "verify": false } @@ -508,6 +513,7 @@ "flasher": { "name": "openocd", "uid": "BC5DA47360D0", + "vid_pid": "0x1a86 0x8010", "args": "-f target/wch-riscv.cfg", "verify": false } @@ -524,6 +530,7 @@ "flasher": { "name": "openocd", "uid": "57468F06DC03", + "vid_pid": "0x1a86 0x8010", "args": "-f target/wch-riscv.cfg", "verify": false } diff --git a/test/hil/usbtest.py b/test/hil/usbtest.py index 83ea3e24c..485e9e0e4 100755 --- a/test/hil/usbtest.py +++ b/test/hil/usbtest.py @@ -25,7 +25,9 @@ capability flags only unlock cases, they don't require the endpoints to exist. import argparse import json +from contextlib import redirect_stdout import os +import pathlib import re import shutil import subprocess @@ -33,16 +35,47 @@ import sys import time from pathlib import Path +sys.path.append(os.path.dirname(os.path.abspath(__file__))) # PYTHONSAFEPATH drops it + VID = 'cafe' PID = '4010' GZ_REF = '0525 a4a0' # copy Gadget Zero's capability profile (ctrl_out+iso+intr) SYS_USB = Path('/sys/bus/usb/devices') DRIVER = Path('/sys/bus/usb/drivers/usbtest') -USB_RECOVER = Path(__file__).resolve().parents[2] / '.claude/skills/usb-kernel-recover/scripts/usb_recover.sh' PATTERN_PARAM = Path('/sys/module/usbtest/parameters/pattern') - -# Battery per tier, in run order: control sanity first, then simple bulk, -# queued, unaligned, unlink, halt/toggle, throughput last. +RECOVER_FLASH_TIMEOUT = 90 # bound on the post-hang reflash; typical flash is 10-20s +RECOVER_RESET_TIMEOUT = 30 # bound on the post-hang probe reset; ResetTarget measures ~130ms + + +def recovery_steps(flasher_name: str, time_left: float) -> list: + """Ordered (kind, bound) recovery attempts that fit in `time_left`. + + RESET FIRST, reflash second. A probe reset fails the in-flight URB at the source just + as a park-flash does, but it is non-destructive -- the firmware under test survives, so + the wedge can still be autopsied -- writes no flash, and cannot brick SWD the way a bad + park image has on mimxrt1064_evk and max32666fthr (survived a power cycle). Measured + 128-129 ms against a full erase+program, and it works on i.MX RT and on DWC2 alike + (stm32f407disco, 2026-08-16: `r; g` -> USB disconnect, re-enumerated 325 ms later). + + The reset also fits budgets a reflash does not: the old gate skipped recovery entirely + when RECOVER_FLASH_TIMEOUT did not fit, which left the holder in place for the next + job. Whether either worked is decided by wedged_pids(), never by the exit code -- a + clean flash only proves the probe wrote the MCU. + """ + import hil_flash + steps = [] + reset_fn = getattr(hil_flash, f'reset_{flasher_name.lower()}', None) + if getattr(reset_fn, 'no_op', False): + reset_fn = None # a stub that returns rc 0 without resetting: do not claim it + if reset_fn and time_left >= RECOVER_RESET_TIMEOUT: + steps.append(('reset', RECOVER_RESET_TIMEOUT)) + if time_left >= RECOVER_FLASH_TIMEOUT: + steps.append(('flash', RECOVER_FLASH_TIMEOUT)) + return steps +HELPER_TIMEOUT = 30 # default bound for sudo helpers (dmesg/modprobe/setpci/tee) + +# Battery per tier, in run order: control sanity, simple bulk, queued, unaligned, unlink, +# halt/toggle, throughput last. TIER_CASES = { 1: [0, 9, 10, 1, 2, 3, 4, 5, 6, 7, 8, 17, 18, 19, 20, 11, 12, 24, 13, 29, 27, 28], 2: [14, 21], @@ -50,10 +83,10 @@ TIER_CASES = { 4: [15, 16, 22, 23], } -# Per-case testusb parameters (full speed / high speed). All -s/-v values are -# multiples of 512 so transfers stay packet-aligned at both speeds: the device -# streams whole max-size packets and a non-aligned IN length would babble. -# 14/21 must never run with defaults (vary >= length is -EINVAL in the kernel). +# Per-case testusb parameters (full speed / high speed). All -s/-v values are multiples +# of 512 so transfers stay packet-aligned at both speeds: the device streams whole max-size +# packets and a non-aligned IN length would babble. 14/21 must never run with defaults +# (vary >= length is -EINVAL in the kernel). PARAMS = { 0: ('-c 1', '-c 1'), 9: ('-c 256', '-c 1000'), @@ -88,9 +121,40 @@ RE_FAIL = re.compile(r'test (\d+) --> (\d+) \((.*)\)') def run(cmd, **kw): - kw.setdefault('capture_output', True) + # NOT subprocess.run(timeout=): CPython's post-timeout path is an UNBOUNDED wait() that + # never returns on a D-state child -- the hang sysfs_write's timeout exists to catch. + timeout = kw.pop('timeout', None) + data = kw.pop('input', None) # subprocess.run-only kwarg; Popen takes stdin + kw.pop('capture_output', None) # ditto: expressed by the PIPEs below kw.setdefault('text', True) - return subprocess.run(cmd, **kw) + kw.setdefault('encoding', 'utf-8') + kw.setdefault('errors', 'replace') # strict decode would raise out of _sudo_soft + # NO start_new_session: these helpers (dmesg, modprobe, setpci, tee) must stay in our + # process group so hil_test's outer killpg reaps them with us. + timeout = timeout if timeout is not None else HELPER_TIMEOUT + proc = subprocess.Popen(cmd, stdin=subprocess.PIPE if data is not None else None, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kw) + try: + out, err = proc.communicate(input=data, timeout=timeout) + return subprocess.CompletedProcess(cmd, proc.returncode, out, err) + except subprocess.TimeoutExpired: + # Under sudo our child is only the wrapper; the root grandchild survives this and + # is left for the report and hil_pool_check to name. Close our pipe ends so an + # abandoned child costs no fds. + try: + proc.kill() # same group as us: never killpg, that would kill us too + except OSError: + pass + try: + proc.communicate(timeout=5) + except subprocess.TimeoutExpired: + for pipe in (proc.stdout, proc.stderr, proc.stdin): + try: + if pipe is not None: + pipe.close() + except OSError: + pass # unkillable: abandon it, the caller reports the timeout + raise def sudo(cmd, **kw): @@ -104,29 +168,106 @@ def sudo(cmd, **kw): def sysfs_write(path, data, check=True): - # A driver-registry write (new_id/remove_id/bind) blocks in D state when a wedged device - # holds its lock (driver_attach walks the bus): fail fast and loud instead of piling up - # unkillable writers and hanging the whole run -- the rig needs USB recovery first. + # A driver-registry write (new_id/remove_id/bind) blocks in D state when a wedged + # device holds its lock: fail fast instead of piling up unkillable writers -- the rig + # needs USB recovery first. + # + # Verified in v6.12.96: unbind_store -> device_driver_detach -> + # device_release_driver_internal -> __device_driver_lock (drivers/base/dd.c), which + # takes device_lock() -- the UNINTERRUPTIBLE variant, unlike the sysfs read path -- and + # ALSO device_lock(parent), because usb_bus_type sets need_parent_lock = true + # (drivers/usb/core/driver.c:2048). So one such write against a wedged device blocks + # unkillably while holding the HUB's lock: that is the mechanism by which a single + # wedged port takes its whole bus down, and why this fails fast instead. try: r = sudo(['tee', str(path)], input=data, timeout=15) except subprocess.TimeoutExpired: sys.exit(f'write "{data}" > {path} blocked >15s: USB subsystem is wedged ' - '(a D-state device lock exists). Recover the rig (usb_recover.sh) ' + '(a D-state device lock exists). Recover the rig (usb-kernel-recover skill) ' 'before running batteries.') if check and r.returncode != 0: sys.exit(f'write "{data}" > {path} failed: {r.stderr.strip()}') return r.returncode == 0 +def _read_sysfs_bounded(path, grace=1.0): + """Bounded sysfs attribute read. The value, or None, or hil_util.SYSFS_UNKNOWN. + + Delegates to hil_util.read_sysfs (imported here, like every helper import in this + file) so both properties hold: the strand cap -- find_device re-scans after EVERY + case, so a 30-case battery against a wedged peer would otherwise strand dozens of + threads and fds -- and UNKNOWN kept distinct from None. Folding UNKNOWN into None made + a blinded scan read as "device dropped off the bus", which aborts down a path that + skips the HUNG recovery entirely. + """ + from helper import hil_util + return hil_util.read_sysfs(str(path), grace) + + +_DEV_CACHE: dict = {} # serial -> sysname, see find_device + + +def _reread(sysname, serial): + """Re-describe an already-resolved device, CONFIRMING its serial. + + idVendor/idProduct/busnum/devnum/speed are lock-free (sysfs.c:688-705), so they cannot + block on a wedged peer -- but every identical board answers them the same, so they + prove nothing about identity. `serial` does, at one bounded read: a sysname is a + topology path, and after a renumber (controller reset, reboot) it can name a DIFFERENT + cafe:4010 board whose verdicts would be filed under this one. Returns None when the + serial is gone, mismatched or unconfirmed -- caller falls back to a full scan. + """ + d = SYS_USB / sysname + try: + if ((d / 'idVendor').read_text().strip() != VID + or (d / 'idProduct').read_text().strip() != PID): + return None + dev_serial = _read_sysfs_bounded(d / 'serial') + if not isinstance(dev_serial, str) or dev_serial.lower() != serial.lower(): + return None # gone, mismatched, or unconfirmable -> full scan decides + return { + 'sysname': sysname, + 'serial': dev_serial, + 'node': '/dev/bus/usb/%03d/%03d' % (int((d / 'busnum').read_text()), + int((d / 'devnum').read_text())), + 'speed': (d / 'speed').read_text().strip(), + 'tier': int((d / 'bcdDevice').read_text().strip()[-2:], 16), + } + except (OSError, ValueError): + return None + + def find_device(serial, first=False): - """Locate the usbtest device in sysfs, return info dict or None.""" - matches = [] + """Locate the usbtest device in sysfs, return info dict or None. + + Cached by serial: this is called after EVERY case, and a full scan pays a bounded + but real `serial` read for every cafe:4010 peer on the rig. With another board + wedged that cost lands on a HEALTHY battery ~30 times over, truncating it into + BUDGET entries. The fast path pays ONE bounded read -- our own device's serial, the + only attribute that tells identical boards apart (see _reread). + """ + if serial: + sysname = _DEV_CACHE.get(serial.lower()) + if sysname: + hit = _reread(sysname, serial) + if hit: + return hit + _DEV_CACHE.pop(serial.lower(), None) + matches, inconclusive = [], [] for dev in SYS_USB.iterdir(): try: if (dev / 'idVendor').read_text().strip() != VID or \ (dev / 'idProduct').read_text().strip() != PID: continue - dev_serial = (dev / 'serial').read_text().strip() + # BOUNDED: idVendor/idProduct are cached descriptors, but `serial` is served + # under device_lock(), so an unbounded read blocks us in D state on exactly the + # DUT whose hang we are here to report, losing every verdict collected so far. + dev_serial = _read_sysfs_bounded(dev / 'serial') + if dev_serial is not None and not isinstance(dev_serial, str): + inconclusive.append(dev.name) # unknown: NOT proof it is not ours + continue + if dev_serial is None: + continue if serial and dev_serial.lower() != serial.lower(): continue matches.append({ @@ -140,12 +281,17 @@ def find_device(serial, first=False): except (OSError, ValueError): continue if not matches: - return None + # "could not tell" is not "gone". The caller aborts the battery on a falsy return + # and that path skips the HUNG reflash, so a blinded scan would report the wedge + # we exist to recover from as a physical disconnect. + return {'inconclusive': inconclusive} if inconclusive else None + if serial and len(matches) == 1: + _DEV_CACHE[serial.lower()] = matches[0]['sysname'] if len(matches) > 1 and not first: if serial: - # Dual-port parts (nanoch32v203 fsdev/usbfs, ch32v307 usbhs/usbfs) briefly enumerate - # BOTH ports with the same serial around a variant reflash; picking one arbitrarily - # could bind the stale port. Report ambiguity so the caller retries until it drops. + # Dual-port parts (nanoch32v203, ch32v307) briefly enumerate BOTH ports with + # one serial around a variant reflash, and picking one could bind the stale + # port -- report ambiguity so the caller retries until it drops. return {'ambiguous': sorted(m['sysname'] for m in matches)} sys.exit(f'multiple {VID}:{PID} devices found, use --serial: ' + ', '.join(m["serial"] for m in matches)) @@ -165,8 +311,8 @@ def check_host_compat(dev): vid_did = ((pci / 'vendor').read_text().strip(), (pci / 'device').read_text().strip()) break except (OSError, ValueError): - # transient sysfs error (e.g. racing a re-enumeration): retry so a blip doesn't - # silently pass an incompatible host; if the probe truly fails, fail open but say so + # transient sysfs error (racing a re-enumeration): retry so a blip does not + # silently pass an incompatible host, then fail open but say so if attempt == 2: print('warning: cannot probe the upstream host controller; ' 'skipping the host compatibility check', file=sys.stderr) @@ -178,19 +324,16 @@ def check_host_compat(dev): 'placed in the EHCI periodic schedule and unlinked reads complete as short ' 'transfers (EREMOTEIO). Move the DUT to an xHCI port.') if drv.startswith('xhci') and vid_did in (('0x1912', '0x0014'), ('0x1912', '0x0015')): - # The Renesas uPD720201/uPD720202 must run its latest firmware (>= 2.0.2.6, - # K2026090.mem; RAM-uploaded, so it reverts to ROM on every power cycle unless - # re-loaded). On the ROM firmware its command ring intermittently dies under unlink - # stress: a Configure Endpoint command stops completing, the hub worker deadlocks - # holding the device lock (needs a host power cycle). Three separate boards killed - # it this way (ch32v307 2026-07-10; ra6m5 test 24, mimxrt1015 2026-07-11). Both - # parts expose the FW version register at PCI config offset 0x6c. NOTE this check - # is necessary, not sufficient: board-specific batteries have killed the controller - # on current firmware too (mimxrt1015, stop-endpoint timeout) - those are handled - # by per-board skips in the rig config. + # The Renesas uPD720201/uPD720202 must run firmware >= 2.0.2.6 (K2026090.mem; + # RAM-uploaded, so it reverts to ROM on every power cycle): on ROM firmware its + # command ring dies under unlink stress and the hub worker deadlocks holding the + # device lock, needing a host power cycle (ch32v307 2026-07-10; ra6m5 test 24, + # mimxrt1015 2026-07-11). Both parts expose the FW version at PCI config 0x6c. + # Necessary, not sufficient -- batteries have killed the controller on current + # firmware too, which per-board skips in the rig config handle. fw = None try: - r = sudo(['setpci', '-s', pci.name, '0x6c.l'], capture_output=True, text=True) + r = _sudo_soft(['setpci', '-s', pci.name, '0x6c.l'], capture_output=True, text=True) if r.returncode == 0: fw = int(r.stdout.strip(), 16) except (OSError, ValueError): @@ -211,7 +354,7 @@ def check_host_compat(dev): def bind_usbtest(dev): """Bind the device's interface 0 to the usbtest driver.""" if not DRIVER.exists(): - r = sudo(['modprobe', 'usbtest']) + r = _sudo_soft(['modprobe', 'usbtest']) if r.returncode != 0 or not DRIVER.exists(): sys.exit(f'cannot load usbtest module: {r.stderr.strip()}') @@ -222,8 +365,8 @@ def bind_usbtest(dev): sysfs_write(DRIVER / 'remove_id', f'{VID} {PID}', check=False) sysfs_write(DRIVER / 'new_id', f'{VID} {PID} 0 {GZ_REF}') if stale_binding: - # bound before the re-registration: that probe captured the OLD dynamic id's capability - # profile; unbind once (device is idle here) so the loop below reprobes the fresh one + # it probed against the OLD dynamic id's capability profile; unbind once (the + # device is idle here) so the loop below reprobes the fresh one sysfs_write(drv / 'unbind', intf, check=False) deadline = time.monotonic() + 3 @@ -248,51 +391,64 @@ def set_pattern(value): 'the "pattern" param, or it is not readable') +def _sudo_soft(cmd, **kw): + """sudo() for calls whose failure must never abort the battery: run() re-raises + TimeoutExpired, and two of these are evaluated inside run_case's own timeout handler + -- a raise there loses the HUNG verdict, the recovery and the JSON report.""" + try: + return sudo(cmd, **kw) + except (OSError, ValueError, subprocess.SubprocessError, SystemExit) as e: + # SystemExit too: sudo() sys.exit()s on 'a password is required', unwinding out of + # run_case's timeout handler before the HUNG verdict is recorded -- which leaves + # unrecovered_hang False and lets the finally run the remove_id/unbind that must + # never happen while a D-state device lock is held + print(f'{cmd[0]}: {type(e).__name__}: {e}', file=sys.stderr) + return subprocess.CompletedProcess(cmd, 1, '', '') + + def dmesg_tail(): - r = sudo(['dmesg']) + r = _sudo_soft(['dmesg']) lines = [l for l in r.stdout.splitlines() if 'usbtest' in l] return '\n'.join(lines[-8:]) -def wedged_pids(devnode): - """Return (pids, complete): PIDs in uninterruptible sleep whose cmdline names devnode, i.e. - still holding its usbfs device lock, and whether every /proc entry could actually be read. - Matched by device node rather than by our child's pid because run_case() may wrap testusb in - sudo, in which case the Popen pid is the wrapper and the blocked process is its child -- - killing the wrapper would make a pid-based check look clean while the real holder is stuck. - complete is False when a PermissionError hid an entry (a hidepid/ProtectProc mount, or the - root-owned child of that same sudo). An entry we could not read might be the holder, so the - caller must treat that as unrecovered rather than as an all-clear.""" +def wedged_pids(devnode): + """(pids, complete): pids still in D state on `devnode` after a recovery reflash. + + Matched by device node rather than by our child's pid because run_case() may wrap + testusb in sudo: the Popen pid is then the wrapper and the blocked process is its + child. A clean flash only proves the probe wrote the MCU, not that the D-state holder + let go -- this is what tells the two apart. + + FAIL CLOSED. `complete` is False when an entry could be HIDDEN from us, and the caller + must then keep treating the hang as unrecovered: the holder is root-owned (run_case + uses `sudo -n` whenever the node is not writable) and a hidepid/ProtectProc mount + hides exactly that entry. Reporting "no holder" from a scan that could not see it + clears unrecovered_hang and lets cleanup run remove_id/unbind against a device whose + usbfs lock is still held -- which deadlocks the bus, not just this board. + + Self-contained: /proc is plain text and this is one pass over it, so importing a + helper to do it would only add a failure mode on the recovery path. + """ stuck, complete = [], True - # hidepid=2 and systemd's ProtectProc=invisible omit other users' processes from iterdir() - # entirely -- no entry at all, so no PermissionError to catch -- and testusb runs under sudo - # whenever the device node is not writable. The scan would then look clean while hiding the - # very holder it exists to find. pid 1 is always root-owned, so being unable to read it means - # enumeration is restricted and no result from this scan can be trusted as complete. + # A restricted /proc hides other users' entries ENTIRELY -- no entry, so no + # PermissionError to catch -- and testusb runs under sudo, so the holder is exactly + # what is hidden. Detect the restriction itself rather than its symptom. if os.geteuid() != 0 and not os.access('/proc/1/cmdline', os.R_OK): complete = False - for entry in Path('/proc').iterdir(): - if not entry.name.isdigit(): - continue + for d in pathlib.Path('/proc').glob('[0-9]*'): try: - cmdline = (entry / 'cmdline').read_bytes() - except PermissionError: - complete = False # cannot rule this pid out - continue - except OSError: - continue # raced with process exit: genuinely gone, not hidden - if devnode.encode() not in cmdline: - continue - try: - stat = (entry / 'stat').read_text() - if stat[stat.rindex(')') + 2] == 'D': # comm may contain ')', so scan from the right - stuck.append(int(entry.name)) + st = (d / 'stat').read_bytes() + if st[st.rindex(b')') + 2:st.rindex(b')') + 3] != b'D': + continue + if devnode.encode() in (d / 'cmdline').read_bytes(): + stuck.append(int(d.name)) except PermissionError: - complete = False - except (OSError, ValueError, IndexError): - continue + complete = False # cannot rule this pid out + except (OSError, ValueError): + continue # raced with exit return stuck, complete @@ -306,17 +462,29 @@ def run_case(num, dev, testusb, quick, timeout): cmd = ['sudo', '-n'] + cmd result = {'num': num, 'name': CASE_NAMES[num], 'params': fs_hs} - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + # NO start_new_session: testusb must stay in OUR process group so the caller's outer + # killpg still reaps it; a sudo-wrapped child is escalated through sudo below instead. + # errors='replace': testusb output is not guaranteed UTF-8, and a strict decode would + # raise out of here and out of main(), printing no JSON at all (battery '0/30'). + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + text=True, encoding='utf-8', errors='replace') try: out, _ = p.communicate(timeout=timeout) except subprocess.TimeoutExpired: - p.kill() + # Under sudo we only kill the wrapper; its root-owned testusb keeps the inherited + # stdout pipe, so the reap below times out and the overrun is reported as HUNG. + # Accepted rather than escalated: the rig's udev rules make the device node + # writable, so sudo is the exception, and the harness must never sudo-kill a pid + # it cannot prove is its own. + try: + p.kill() + except OSError: + pass try: out, _ = p.communicate(timeout=5) except subprocess.TimeoutExpired: - # SIGKILL had no effect: the child is in uninterruptible sleep on an - # in-kernel usbfs ioctl (device stopped responding mid-transfer). - # Abandon it — waiting or re-signalling can never succeed. + # SIGKILL had no effect: the child is in uninterruptible sleep on an in-kernel + # usbfs ioctl. Abandon it — waiting or re-signalling can never succeed. result.update(status='HUNG', detail=f'testusb stuck in D state after {timeout}s', dmesg=dmesg_tail()) return result @@ -362,7 +530,18 @@ def main(): p.add_argument('--keep-binding', action='store_true', help='leave usbtest dynamic id registered') p.add_argument('--testusb', default=None, help='path to testusb binary') p.add_argument('--timeout', type=int, default=120, help='per-case timeout in seconds') + p.add_argument('--recover-board', help='board JSON (name + flasher) for the post-hang ' + 'reflash recovery; without it a HUNG case leaves the device wedged') + p.add_argument('--recover-fw', help='firmware path reflashed by the post-hang recovery') + p.add_argument('--outer-timeout', type=int, default=0, + help='the caller\'s total bound on this process; a reflash that cannot ' + 'finish before it is skipped rather than orphaned mid-flash') + p.add_argument('--budget', type=int, default=0, + help='stop starting new cases after this many seconds (0 = no limit). ' + 'Callers that impose their own outer timeout set this to reserve ' + 'the remainder for the post-hang recovery path') args = p.parse_args() + t_start = time.monotonic() sys.stdout.reconfigure(line_buffering=True) # per-case results visible when piped/logged testusb = args.testusb or shutil.which('testusb') or os.path.expanduser('~/testusb') @@ -370,22 +549,32 @@ def main(): sys.exit('testusb binary not found: build kernel tools/usb/testusb.c ' 'and install it, or pass --testusb') - # retry briefly: right after a flash the enumeration may still be settling, and on dual-port - # parts the other port's stale same-serial node takes a moment to drop off (see find_device) + # retry briefly: after a flash the enumeration may still be settling, and a dual-port + # part's stale same-serial node takes a moment to drop off (see find_device) deadline = time.monotonic() + 8 while True: dev = find_device(args.serial) - if dev and 'ambiguous' not in dev: + # find_device is THREE-valued: a device, {'ambiguous': [...]}, or + # {'inconclusive': [...]} when bounded reads could not rule a device out. Screening + # only for 'ambiguous' let the inconclusive marker through as if it were a device, + # and the next statement subscripts dev['tier'] -> KeyError, no JSON on stdout, and + # hil_test reports "usbtest did not run / 0-30" for a merely-unreadable bus. + if dev and not ({'ambiguous', 'inconclusive'} & dev.keys()): break if time.monotonic() > deadline: - if dev: + if dev and 'ambiguous' in dev: sys.exit(f"multiple devices with serial {args.serial}: {', '.join(dev['ambiguous'])} " '— stale enumeration from another port? replug or retry') + if dev and 'inconclusive' in dev: + from helper import hil_util as _hu + sys.exit(f"cannot tell whether {VID}:{PID} is present: bounded sysfs reads " + f"did not answer for {', '.join(dev['inconclusive'])}" + f"{_hu.sysfs_blind_note()}") sys.exit(f'no {VID}:{PID} device' + (f' with serial {args.serial}' if args.serial else '')) time.sleep(0.5) - # tier drives which cases run; a stale/foreign device advertising an out-of-range tier - # must not silently run an empty battery ('0/0 passed' would read as green in CI) + # a stale/foreign device advertising an out-of-range tier must not silently run an + # empty battery ('0/0 passed' would read as green in CI) tier = args.tier or dev['tier'] if not 1 <= tier <= max(TIER_CASES): sys.exit(f"device advertises tier {tier} (bcdDevice ...{tier:02x}); reflash a usbtest build " @@ -404,8 +593,7 @@ def main(): if not args.json: print(info) - # probe the upstream controller before touching the device: an incompatible host - # (MosChip MCS9990, or uPD720201 on pre-2.0.2.6 firmware) exits here, before any bind + # before touching the device: an incompatible host exits here, before any bind check_host_compat(dev) results = [] @@ -414,7 +602,15 @@ def main(): bind_usbtest(dev) set_pattern(0) # tier 1 firmware sources zeros; also required by perf cases 27/28 - for num in cases: + abort_reason = None # set on any early exit; drives the BUDGET back-fill below + for idx, num in enumerate(cases): + # Only a HUNG case aborts the battery; an ordinary case timeout is a FAIL and + # the loop continues, each burning --timeout+5s, so without this the run can + # still be in the case loop when the outer timeout SIGKILLs it before it emits + # JSON. Checked before dispatch: worst overshoot is one case. + if args.budget and time.monotonic() - t_start > args.budget: + abort_reason = f'battery budget {args.budget}s exhausted' + break results.append(run_case(num, dev, testusb, args.quick, args.timeout)) r = results[-1] if not args.json: @@ -422,112 +618,255 @@ def main(): extra += f" {r['mbps']} MB/s" if 'mbps' in r else '' print(f"test {num:2d} {r['name']:22s} {r['status']:6s}{extra}") if r['status'] == 'HUNG': - print(f'aborting battery: kernel-side hang, device wedged mid-transfer.\n' - f'auto-recovering: {USB_RECOVER.name} root-cycle {dev["sysname"]} ' - f'(see .claude/skills/usb-kernel-recover)', file=sys.stderr) - # Cutting VBUS at the root port fails the in-flight URB so the usbfs ioctl returns. - # Must run BEFORE any unbind/remove_id, which would take the device lock the stuck - # ioctl holds and deadlock the bus. + abort_reason = 'battery aborted on a kernel-side hang' + # Reflash, NEVER a root-port cycle: resetting the MCU through the DUT's own + # debug probe fails the in-flight URB at the source, so the ioctl returns, + # the queued kill lands and the cleanup below is lock-safe -- and it reaches + # exactly one board, where a root-port cycle bounces every fixture under the + # port (and could never remove power anyway; see usb-kernel-recover). + # Deliberately not gated on a hub-worker check: our own stuck testusb is + # what drives a hub worker into usb_lock_device(), so a pre-check reads + # wedged by construction. # - # Assume unrecovered until proven otherwise, so that any early exit from this block - # -- an OSError spawning the helper, a KeyboardInterrupt, a sudo prompt killing the - # run -- still reaches the finally cleanup with the flag set, instead of running - # the remove_id/unbind the comments there forbid while a device lock is held. + # Assume unrecovered until proven otherwise, so any early exit from this + # block reaches the finally with the flag set instead of running the + # remove_id/unbind that must not happen while a device lock is held. unrecovered_hang = True - # Pass the serial so the helper refuses a stale busport rather than cutting power - # to whatever else now occupies that path. Popen rather than sudo()/subprocess.run: - # run() would kill() then wait() unbounded on timeout, which never returns if - # uhubctl is itself in D state -- the case the timeout exists for. Merge stderr - # into stdout so the helper's target-identity and action lines are not lost. - # Only pass the serial when we actually have one: an empty third argument reads as - # "no expectation" and would silently disable the helper's stale-busport guard. - cmd = [str(USB_RECOVER), 'root-cycle', dev['sysname']] - if dev['serial']: - cmd.append(dev['serial']) - if os.geteuid() != 0: - cmd = ['sudo', '-n'] + cmd - try: - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - text=True) - except OSError as e: - # helper missing or not executable, or sudo unavailable. unrecovered_hang is - # already True so the finally block still skips the unsafe cleanup -- this only - # replaces a traceback with a message that says what to fix. - print(f'cannot run {USB_RECOVER}: {e}', file=sys.stderr) + print('aborting battery: kernel-side hang, device wedged mid-transfer', + file=sys.stderr) + if not (args.recover_board and args.recover_fw): + print('no --recover-board/--recover-fw: the device stays wedged and ' + 'cleanup is skipped', file=sys.stderr) + break + # The reflash is bounded to RECOVER_FLASH_TIMEOUT and skipped when the + # caller's outer bound cannot contain it: the flasher runs in its own + # session, so an outer killpg mid-flash would ORPHAN it on the probe. Gate + # each step on the time actually LEFT -- reserving for the worst case up + # front skipped recovery for nearly every real hang, since the hang-prone + # cases run late in the tier order. + def _time_left(): + if not args.outer_timeout: + return float('inf') + # what still runs after a step: run_cmd's post-kill reap (10s), + # the settle (5s), the sudo-escalated descendant reap run_case may + # have just paid (up to 7s) and the JSON write + return args.outer_timeout - (time.monotonic() - t_start) - 35 + + if _time_left() < RECOVER_RESET_TIMEOUT: + print('insufficient time before the outer bound for even a bounded ' + 'reset; the device stays wedged and cleanup is skipped', + file=sys.stderr) break - rc = None try: - out, _ = p.communicate(timeout=60) # normal run is ~8s - rc = p.returncode - except subprocess.TimeoutExpired: - p.kill() + board = json.loads(args.recover_board) + bname, fname = board['name'], board['flasher']['name'] + import hil_flash # deferred: stdlib-only unless recovery actually runs + flash_fn = getattr(hil_flash, f'flash_{fname.lower()}') + reset_fn = getattr(hil_flash, f'reset_{fname.lower()}', None) + except Exception as e: # malformed/short json, import failure, unknown flasher + print(f'reflash recovery unavailable ({e})', file=sys.stderr) + break + # DELIVERY must be convoy-safe or the recovery makes things worse: our own + # testusb is D-state on this DUT's node, so a flasher that enumerates by + # OPENING usbfs nodes blocks on it, survives SIGKILL and is abandoned -- + # a SECOND stray, the budget spent, the device still wedged. On 2026-08-12 + # a vid_pid-pinned openocd was the only flasher that still reached its + # probe; JLinkExe's ShowEmuList returned zero. See hil_flash.convoy_safe. + if not hil_flash.convoy_safe(board['flasher']): + print(f'{fname} is not convoy-safe for delivery (it enumerates by ' + f'opening usbfs nodes, and this DUT has a D-state holder on ' + f'its own node): skipping the reflash rather than adding a ' + f'second stray. Pin the roster entry with vid_pid on an ' + f'openocd flasher to enable recovery for this board.', + file=sys.stderr) + break + # RESET FIRST (see recovery_steps). Non-destructive, ~130 ms, and it + # clears the wedge by the same mechanism as the reflash. wedged_pids is the + # arbiter: reset_esptool is a stub that returns rc 0 without resetting + # anything, so an exit code here proves nothing. + steps = recovery_steps(fname, _time_left()) + if reset_fn and any(k == 'reset' for k, _ in steps): + print(f'auto-recovering: resetting {bname} via {fname} probe ' + f'(non-destructive; reflash only if this does not clear it)', + file=sys.stderr) try: - out, _ = p.communicate(timeout=5) - rc = p.returncode - except subprocess.TimeoutExpired: - out = ('root-cycle abandoned after 60s: uhubctl did not die to SIGKILL, so ' - 'it is wedged too and the convoy has spread beyond this device') - if out: - print(out.strip(), file=sys.stderr) - if rc is not None: - time.sleep(5) # let the bus settle and the freed ioctl unwind - # Authoritative either way. A non-zero exit only means the device did not come - # back within the poll (a slow bootloader will do that) -- if nothing still - # holds the lock, the bus is usable and cleanup is safe. Conversely a zero exit - # only proves re-enumeration, not that the D-state holder let go. + with redirect_stdout(sys.stderr): + reset_fn(board, timeout=RECOVER_RESET_TIMEOUT) + except TypeError: + with redirect_stdout(sys.stderr): + reset_fn(board) # older primitives take no bound + except Exception as e: + print(f'probe reset raised: {e}; falling through to the reflash', + file=sys.stderr) + time.sleep(5) # let the freed ioctl unwind stuck, complete = wedged_pids(dev['node']) - if stuck: - print(f'{dev["sysname"]}: pid(s) {stuck} still in D state on ' - f'{dev["node"]} — the device lock was never released', file=sys.stderr) - elif not complete: - print('cannot confirm recovery: /proc is only partly readable, so a ' - 'hidden D-state holder cannot be ruled out', file=sys.stderr) - else: + if complete and not stuck: + print('probe reset cleared the wedge; skipping the reflash ' + '(firmware under test left intact for autopsy)', + file=sys.stderr) unrecovered_hang = False + break + if _time_left() < RECOVER_FLASH_TIMEOUT: + print('reset did not clear it and no budget left for a reflash; ' + 'the device stays wedged', file=sys.stderr) + break + print(f'auto-recovering: reflashing {bname} via ' + f'{fname} (see .claude/skills/usb-kernel-recover). ' + f'Unbudgeted by flash_permit, like the root-cycle it replaced: the ' + f'per-controller semaphores live in hil_test\'s process.', + file=sys.stderr) + # run_cmd bounds the flash; its banners go to stdout, which in --json mode + # carries the result object -- keep them off it. A raising flasher (missing + # serial node, unwritable CWD) must not cost the battery its JSON report. + try: + with redirect_stdout(sys.stderr): + ret = flash_fn(board, args.recover_fw, timeout=RECOVER_FLASH_TIMEOUT) + except Exception as e: + print(f'reflash raised: {e}; the device may still be wedged', file=sys.stderr) + break + if ret.returncode != 0: + # a wedged RP DAP answers nothing and the probe has no reset line; + # POR it via the Rescue DP and retry once, exactly as the normal + # flash path does (no-op for every other board/failure) + out_txt = ret.stdout if isinstance(ret.stdout, str) else '' + # inside the redirect like its siblings (hil_test slices the result + # object from the first '{' on stdout), and only if a POR + retry + # still fits before the outer kill + rescued = False + try: + if _time_left() >= 2 * RECOVER_FLASH_TIMEOUT: + with redirect_stdout(sys.stderr): + rescued = hil_flash.rescue_openocd( + board, out_txt, timeout=RECOVER_FLASH_TIMEOUT) + if rescued: + print('DAP wedged; rescued via Rescue DP, retrying reflash', + file=sys.stderr) + with redirect_stdout(sys.stderr): + ret = flash_fn(board, args.recover_fw, + timeout=RECOVER_FLASH_TIMEOUT) + except Exception as e: + # guarded like the first flash: a raise here would unwind past the + # BUDGET back-fill and the JSON print + print(f'rescue/retry raised: {e}', file=sys.stderr) + if ret.returncode != 0: + print(f'reflash failed (rc {ret.returncode}); the device may still ' + f'be wedged', file=sys.stderr) + # settle even on a non-zero exit: the reset may have landed before the + # flasher failed, and the freed ioctl needs a moment to unwind before + # wedged_pids samples + time.sleep(5) + # Authoritative either way: a clean flash only proves the probe wrote the + # MCU, not that the D-state holder let go. + stuck, complete = wedged_pids(dev['node']) + if stuck: + print(f'{dev["sysname"]}: pid(s) {stuck} still in D state on ' + f'{dev["node"]} — the device lock was never released', file=sys.stderr) + # No hub-worker verdict here: our own testusb still holds the DUT's + # device lock, which is what drives a hub worker into usb_lock_device() + # -- any verdict from here is confounded by construction. + elif not complete: + print('cannot confirm recovery: /proc is only partly readable, so a ' + 'hidden D-state holder cannot be ruled out', file=sys.stderr) + else: + unrecovered_hang = False + break + # re-resolve: a mid-battery re-enumeration changes the devnum and so the node + # path. Match on the concrete serial (not args.serial, which may be None) so + # this can never retarget to another device sharing the VID:PID. + # first=False: the ambiguity guard exists because ONE serial can match two + # sysfs nodes on the dual-port WCH parts, and `dev = live` below makes any + # mistake stick for the rest of the battery -- including wedged_pids() then + # scanning the wrong node and clearing unrecovered_hang on a device it never + # checked. Ambiguous comes back as {'ambiguous': [...]}, handled below. + live = find_device(dev['serial']) + if live and live.get('ambiguous'): + # two nodes now answer to one serial (the dual-port WCH parts do this + # around a re-enumeration). Picking either would file the rest of the + # battery's verdicts under a device we cannot identify, so stop here and + # keep the recovery in play rather than guess. + abort_reason = (f'serial {dev["serial"]} matches more than one device ' + f'({", ".join(live["ambiguous"])}) after case {num}') + unrecovered_hang = True + break + if live and live.get('inconclusive'): + # bounded reads stopped answering, so we cannot say the device left -- + # treat it as the wedge it probably is, which keeps the HUNG reflash and + # the lock-safe cleanup in play + from helper import hil_util as _hu + abort_reason = ('cannot tell whether the device is still present: bounded ' + 'sysfs reads stopped answering' + _hu.sysfs_blind_note()) + unrecovered_hang = True break - # re-resolve: after a mid-battery re-enumeration the devnum (and thus the node - # path) changes; keep testing the live node instead of the stale one. Match on the - # concrete serial (not args.serial, which may be None) so this can never retarget to - # a different device that happens to share the VID:PID. - live = find_device(dev['serial'], first=True) if not live: - results.append({'num': num, 'status': 'FAIL', - 'detail': f'device dropped off the bus after case {num}'}) + # no second entry for `num`: run_case already recorded it, and a duplicate + # inflates the denominator (31/30) and reports a PASSing case as failed + abort_reason = f'device dropped off the bus after case {num}' break dev = live + if abort_reason and all(c in {r['num'] for r in results} for c in cases) \ + and 'dropped off the bus' in abort_reason and results: + # nothing left to back-fill (the drop happened during/after the LAST case), + # so the run would report a clean pass; the case it died on is not a pass + if results[-1].get('status') == 'PASS': + # only a PASS: a real FAIL/NOTRUN verdict names the actual regression + # (errno, dmesg) and must not be overwritten by the drop message + results[-1] = dict(results[-1], status='FAIL', detail=abort_reason) + if abort_reason: + # One BUDGET entry per case never dispatched, on EVERY abort path: a shrunken + # denominator (4/5 instead of 4/30) hides that most of the battery never + # executed and makes a regression in the skipped range read as "not the + # problem". + ran = {r['num'] for r in results} + results += [{'num': n, 'status': 'BUDGET', 'detail': f'not run: {abort_reason}'} + for n in cases if n not in ran] finally: # best-effort cleanup: a sudo/sysfs failure here (sudo() may sys.exit) must not replace # an exception propagating out of the try body with a less useful one try: if unrecovered_hang: - # testusb is still stuck in a usbfs ioctl holding the device lock; remove_id/unbind - # would join the convoy and deadlock the bus (see usb-kernel-recover skill) — leave it be + # testusb still holds the device lock in a usbfs ioctl: remove_id/unbind + # would join the convoy and deadlock the bus (see usb-kernel-recover) print('skipping cleanup after unrecovered hang: ask the operator for a full PVE host ' 'power cycle (a VM reboot is not reliable — hubs latch up across the PCIe reset)', file=sys.stderr) elif not args.keep_binding: sysfs_write(DRIVER / 'remove_id', f'{VID} {PID}', check=False) - # release every claimed interface: other devices sharing the VID:PID (stale example - # firmware on a test rig) may have been grabbed on probe and would otherwise stay + # release every claimed interface: another device sharing the VID:PID + # (stale example firmware) may have been grabbed on probe and would stay # bound to usbtest until re-plugged, hijacking the next test's device for intf in DRIVER.glob('*:*'): sysfs_write(DRIVER / 'unbind', intf.name, check=False) except SystemExit: pass - failed = [r for r in results if r['status'] != 'PASS'] + # BUDGET, not NOTRUN: NOTRUN is taken, for a case the KERNEL gated off (-EOPNOTSUPP, + # see run_case) -- a real result that must stay in `failed` and keep its case number. + # BUDGET keeps the denominator honest without lying about the numerator: naming cases + # that never executed as failures sends a maintainer bisecting one of them. + notrun = [r for r in results if r['status'] == 'BUDGET'] + failed = [r for r in results if r['status'] not in ('PASS', 'BUDGET')] ran = len(results) if args.json: + # `wedged` is the verdict this process ALREADY computed; without it the caller had + # to infer one from 'HUNG' in our stdout, which misses a recovery that ran and + # failed, the inconclusive abort (no case reaches status HUNG), and any battery + # killed before it printed. print(json.dumps({'serial': dev['serial'], 'speed': dev['speed'], 'tier': tier, - 'passed': ran - len(failed), 'failed': len(failed), + 'passed': ran - len(failed) - len(notrun), + 'failed': len(failed), 'notrun': len(notrun), + 'wedged': bool(unrecovered_hang), 'cases': results}, indent=2)) else: - print(f"{ran - len(failed)}/{ran} passed") + print(f"{ran - len(failed) - len(notrun)}/{ran} passed" + + (f", {len(notrun)} not run" if notrun else "")) for r in failed: print(f" FAILED test {r['num']}: {r.get('detail', '')}") if r.get('dmesg'): print(' ' + r['dmesg'].replace('\n', '\n ')) - return len(failed) + # NOTRUN counts toward the exit status even though it is reported separately: a + # standalone run whose cases were all skipped has NOT passed, and returning 0 hands a + # false success to any script driving this directly. + return len(failed) + len(notrun) if __name__ == '__main__': diff --git a/tools/metrics_compare_base.py b/tools/metrics_compare_base.py index 799a96800..844130097 100644 --- a/tools/metrics_compare_base.py +++ b/tools/metrics_compare_base.py @@ -81,7 +81,7 @@ def symlink_deps(main_root, worktree_dir): def ci_first_boards(): """Return the first board (alphabetical) of each arm-gcc CI family.""" - matrix_py = os.path.join(TINYUSB_ROOT, '.github', 'workflows', 'ci_set_matrix.py') + matrix_py = os.path.join(TINYUSB_ROOT, '.github', 'scripts', 'ci_set_matrix.py') if not os.path.isfile(matrix_py): return [] ret = run([sys.executable, matrix_py]) @@ -188,7 +188,7 @@ def main(): args.combined = True ci_boards = ci_first_boards() if not ci_boards: - parser.error('--ci: failed to derive boards from .github/workflows/ci_set_matrix.py') + parser.error('--ci: failed to derive boards from .github/scripts/ci_set_matrix.py') # Append, dedup, preserve order seen = set(args.board) for b in ci_boards: -- cgit v1.3.1 From af5354349156d3d1bb0f2533ad802f1e1c5a6ffb Mon Sep 17 00:00:00 2001 From: hathach Date: Mon, 17 Aug 2026 01:02:38 +0700 Subject: bsp(lpc11u37): move the main stack to the USB SRAM bank The 8 KB main bank is packed tightly enough that only ~280 bytes remained above .bss, and interrupt frames overflowed into the topmost task stack - a hard fault in cdc_msc_freertos. Put the MSP at the top of the 2 KB USB SRAM bank, which nothing else uses in either build system, so the stack no longer shrinks as .bss grows. The Make build's CFG_TUSB_MEM_SECTION placement of endpoint buffers into that bank is dropped so both build systems agree on the layout. The headroom assert is written as an addition rather than a subtraction, since linker script arithmetic is unsigned and an overflowing bank would underflow the difference into a huge positive value and pass silently. --- hw/bsp/lpc11/boards/lpcxpresso11u37/board.mk | 3 +-- hw/bsp/lpc11/boards/lpcxpresso11u37/lpc11u37.ld | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) (limited to 'hw/bsp') diff --git a/hw/bsp/lpc11/boards/lpcxpresso11u37/board.mk b/hw/bsp/lpc11/boards/lpcxpresso11u37/board.mk index fdc17374b..718c46bbf 100644 --- a/hw/bsp/lpc11/boards/lpcxpresso11u37/board.mk +++ b/hw/bsp/lpc11/boards/lpcxpresso11u37/board.mk @@ -4,8 +4,7 @@ MCU_DRV = 11xx CFLAGS += \ -DCORE_M0 \ -DCFG_EXAMPLE_MSC_READONLY \ - -DCFG_EXAMPLE_VIDEO_READONLY \ - -DCFG_TUSB_MEM_SECTION='__attribute__((section(".data.$$RAM2")))' + -DCFG_EXAMPLE_VIDEO_READONLY # mcu driver cause following warnings CFLAGS += \ diff --git a/hw/bsp/lpc11/boards/lpcxpresso11u37/lpc11u37.ld b/hw/bsp/lpc11/boards/lpcxpresso11u37/lpc11u37.ld index 8e0a4e4c6..b7237a3ec 100644 --- a/hw/bsp/lpc11/boards/lpcxpresso11u37/lpc11u37.ld +++ b/hw/bsp/lpc11/boards/lpcxpresso11u37/lpc11u37.ld @@ -172,6 +172,22 @@ SECTIONS . = ALIGN(4) ; _end_noinit = .; } > RamLoc8 + /* Main (MSP/ISR) stack lives at the top of the USB SRAM bank: the 8K main bank is packed so + tight that only ~280 B remained above .bss, and ISR frames overflowed into the topmost task + stack (cdc_msc_freertos hard fault). Nothing else is placed in this bank in either build + system, so the stack owns all 2 KB; the ASSERT is future-proofing in case USB buffers are + ever mapped here again. + + This bank is clocked by SYSAHBCLKCTRL[27] (USBRAM enable), and the stack is used from the + first instruction of the reset handler - long before any TinyUSB or BSP code could turn a + clock on. It works because the boot ROM hands over with that bit already set. Anything that + gates the USB RAM clock to save power will hard fault at reset, not at USB init. */ + __user_stack_top = ORIGIN(RamUsb2) + LENGTH(RamUsb2); + /* Stated as an addition, not a subtraction: ld arithmetic is unsigned, so an overflowing + bank would underflow the difference into a huge positive value and pass silently. */ + ASSERT(ADDR(.noinit_RAM2) + SIZEOF(.noinit_RAM2) + 0x200 <= __user_stack_top, + "main stack headroom in RamUsb2 below 512 bytes") + PROVIDE(_pvHeapStart = DEFINED(__user_heap_base) ? __user_heap_base : .); PROVIDE(_vStackTop = DEFINED(__user_stack_top) ? __user_stack_top : __top_RamLoc8 - 0); -- cgit v1.3.1 From b925231216eabf277938607ba50f1f4b78c0ce7d Mon Sep 17 00:00:00 2001 From: hathach Date: Mon, 17 Aug 2026 01:02:39 +0700 Subject: bsp(lpc55): run lpcxpresso55s28 as a high-speed device, add it to the ci pool Flip the board to device-highspeed/host-fullspeed, matching lpcxpresso55s69 and the way it is cabled on the test rig, and add it to the rig pool with the unique id read from its flash PFR. This is the first hardware coverage the ip3511 high-speed device path has ever had, and it immediately exposed the clear-stall type-bit bug fixed separately. The port swap also exposed a build gap: family.mk only linked a host controller for port 1, so make host builds on port 0 failed with undefined references - mirror family.cmake and link the OHCI driver there. The board's rhport defaults now come from family.cmake's guarded ones rather than a duplicate copy, so a -D override on the command line wins. --- hw/bsp/lpc55/boards/lpcxpresso55s28/board.cmake | 4 ---- hw/bsp/lpc55/boards/lpcxpresso55s28/board.mk | 6 +++--- hw/bsp/lpc55/family.mk | 2 ++ test/hil/tinyusb.json | 14 ++++++++++++++ 4 files changed, 19 insertions(+), 7 deletions(-) (limited to 'hw/bsp') diff --git a/hw/bsp/lpc55/boards/lpcxpresso55s28/board.cmake b/hw/bsp/lpc55/boards/lpcxpresso55s28/board.cmake index b3d6ec722..d7992eec6 100644 --- a/hw/bsp/lpc55/boards/lpcxpresso55s28/board.cmake +++ b/hw/bsp/lpc55/boards/lpcxpresso55s28/board.cmake @@ -8,10 +8,6 @@ set(JLINK_OPTION "-USB 000727031389") set(PYOCD_TARGET LPC55S28) set(NXPLINK_DEVICE LPC55S28:LPCXpresso55S28) -# device fullspeed, host highspeed -set(RHPORT_DEVICE 0) -set(RHPORT_HOST 1) - function(update_board TARGET) target_compile_definitions(${TARGET} PUBLIC CPU_LPC55S28JBD100 diff --git a/hw/bsp/lpc55/boards/lpcxpresso55s28/board.mk b/hw/bsp/lpc55/boards/lpcxpresso55s28/board.mk index db2e11fd7..aecb5a100 100644 --- a/hw/bsp/lpc55/boards/lpcxpresso55s28/board.mk +++ b/hw/bsp/lpc55/boards/lpcxpresso55s28/board.mk @@ -2,9 +2,9 @@ MCU_VARIANT = LPC55S28 MCU_CORE = LPC55S28 MCU_DRIVER_VARIANT = LPC55S69 -# device fullspeed, host highspeed -RHPORT_DEVICE ?= 0 -RHPORT_HOST ?= 1 +# device highspeed, host fullspeed +RHPORT_DEVICE ?= 1 +RHPORT_HOST ?= 0 CFLAGS += -DCPU_LPC55S28JBD100 diff --git a/hw/bsp/lpc55/family.mk b/hw/bsp/lpc55/family.mk index a9b6f6af1..a640cc793 100644 --- a/hw/bsp/lpc55/family.mk +++ b/hw/bsp/lpc55/family.mk @@ -36,6 +36,8 @@ ifeq ($(RHPORT_HOST), 1) SRC_C += $(TOP)/src/portable/nxp/lpc_ip3516/hcd_lpc_ip3516.c else CFLAGS += -DBOARD_TUH_MAX_SPEED=OPT_MODE_FULL_SPEED + # host on port 0 uses the OHCI controller (mirrors family.cmake) + SRC_C += $(TOP)/src/portable/ohci/ohci.c endif # mcu driver cause following warnings diff --git a/test/hil/tinyusb.json b/test/hil/tinyusb.json index 549a17cd0..6f552f126 100644 --- a/test/hil/tinyusb.json +++ b/test/hil/tinyusb.json @@ -196,6 +196,20 @@ "args": "-device LPC11U37/401" } }, + { + "name": "lpcxpresso55s28", + "uid": "2BF1839A7D51F553A15AB03FD08F70AB", + "tests": { + "device": true, + "host": false, + "dual": false + }, + "flasher": { + "name": "jlink", + "uid": "000727031389", + "args": "-device LPC55S28" + } + }, { "name": "ra4m1_ek", "uid": "152E163038303131393346E46F26574B", -- cgit v1.3.1