diff options
Diffstat (limited to 'src/portable')
| -rw-r--r-- | src/portable/chipidea/ci_hs/ci_hs_type.h | 16 | ||||
| -rw-r--r-- | src/portable/chipidea/ci_hs/dcd_ci_hs.c | 246 | ||||
| -rw-r--r-- | src/portable/nxp/lpc_ip3511/dcd_lpc_ip3511.c | 106 | ||||
| -rw-r--r-- | src/portable/ohci/ohci.c | 83 | ||||
| -rw-r--r-- | src/portable/ohci/ohci.h | 4 | ||||
| -rw-r--r-- | src/portable/synopsys/dwc2/hcd_dwc2.c | 559 |
6 files changed, 782 insertions, 232 deletions
diff --git a/src/portable/chipidea/ci_hs/ci_hs_type.h b/src/portable/chipidea/ci_hs/ci_hs_type.h index b209c7545..b3ef3b6af 100644 --- a/src/portable/chipidea/ci_hs/ci_hs_type.h +++ b/src/portable/chipidea/ci_hs/ci_hs_type.h @@ -29,6 +29,14 @@ enum { USBCMD_INTR_THRESHOLD_MASK = 0x00FF0000u, // Interrupt Threshold bit 23:16 }; +// DEVICEADDR +#define DEVICEADDR_USBADR_POS 25 + +enum { + DEVICEADDR_USBADRA = TU_BIT(24), ///< Device Address Advance: stage USBADR until the next EP0 IN is ACKed + DEVICEADDR_USBADR_MASK = 0xFE000000u, ///< Device Address bit 31:25 +}; + // PORTSC1 #define PORTSC1_PORT_SPEED_POS 26 @@ -36,10 +44,18 @@ enum { PORTSC1_CURRENT_CONNECT_STATUS = TU_BIT(0), PORTSC1_FORCE_PORT_RESUME = TU_BIT(6), PORTSC1_SUSPEND = TU_BIT(7), + PORTSC1_PORT_RESET = TU_BIT(8), // read-only in device mode: a reset is being driven PORTSC1_FORCE_FULL_SPEED = TU_BIT(24), PORTSC1_PORT_SPEED = TU_BIT(26) | TU_BIT(27) }; +// PORTSC1 PSPD field values, once shifted down by PORTSC1_PORT_SPEED_POS. 3 is undefined. +enum { + PORTSC1_PORT_SPEED_FULL = 0, + PORTSC1_PORT_SPEED_LOW = 1, + PORTSC1_PORT_SPEED_HIGH = 2, +}; + // OTGSC enum { OTGSC_VBUS_DISCHARGE = TU_BIT(0), diff --git a/src/portable/chipidea/ci_hs/dcd_ci_hs.c b/src/portable/chipidea/ci_hs/dcd_ci_hs.c index 8c08c6bd5..f1c333280 100644 --- a/src/portable/chipidea/ci_hs/dcd_ci_hs.c +++ b/src/portable/chipidea/ci_hs/dcd_ci_hs.c @@ -154,6 +154,14 @@ TU_VERIFY_STATIC(sizeof(dcd_qhd_t) == 64, "size is not correct"); #define QTD_NEXT_INVALID 0x01 +// Bounded spin for register waits. The longest legitimate wait is a flush held off by a packet +// already in progress: ~50 us for a full-speed 64-byte packet, a low thousands of dependent +// register reads, so healthy hardware never approaches this bound. Exceeding it means the +// controller has stopped responding, and the spin then only serves to keep an ISR (or an +// IRQ-masked caller) from hanging outright - the 3 ms reset-cleanup window of IMXRT1060RM 42.5.6.2.1 (p.2394) +// is already unreachable in that state, and the manual's remedy there is a controller reset. +#define CI_HS_BUSY_SPIN 10000u + typedef struct { // Must be at 2K alignment // Each endpoint with direction (IN/OUT) occupies a queue head @@ -164,6 +172,17 @@ typedef struct { CFG_TUD_MEM_SECTION TU_ATTR_ALIGNED(2048) static dcd_data_t _dcd_data; +// What the next Port Change Detect will be. Each one is preceded by the interrupt that causes it: +// a reset interrupt for the end of a bus reset - where the speed first becomes final - or a +// suspend interrupt for the resume that ends the suspend. A suspend itself raises no port change, +// which is why there is no such value here. Indexed by rhport, which is 0 or 1 on every ci_hs +// variant (NOT the controller count: mcx/rw61x map rhport 1 to controller 0). +enum { + PORT_CHANGE_REASON_RESET = 0, + PORT_CHANGE_REASON_RESUME = 1, +}; +static volatile uint8_t _port_change_reason[2]; + //--------------------------------------------------------------------+ // Prototypes and Helper Functions //--------------------------------------------------------------------+ @@ -172,12 +191,37 @@ TU_ATTR_ALWAYS_INLINE static inline uint8_t ci_ep_count(const ci_hs_regs_t *dcd_ return dcd_reg->DCCPARAMS & DCCPARAMS_DEN_MASK; } +static bool controller_reset(uint8_t rhport); + //--------------------------------------------------------------------+ // Controller API //--------------------------------------------------------------------+ -/// follows LPC43xx User Manual 23.10.3 -static void bus_reset(uint8_t rhport) { +// Flush endpoint buffers, following IMXRT1060RM 42.5.6.6.5 Flushing/De-priming an Endpoint +// (p.2413): write ENDPTFLUSH, wait for the controller +// to acknowledge, then confirm ENDPTSTAT went to zero. The controller refuses the flush when a +// packet is in progress, and the manual requires the procedure be repeated until it takes. +// Callers proceed regardless of the result; the bound only prevents an ISR-context hang on dead +// hardware. +static bool flush_endpoints(ci_hs_regs_t *dcd_reg, uint32_t mask) { + uint32_t guard = CI_HS_BUSY_SPIN; + do { + dcd_reg->ENDPTFLUSH = mask; + while (dcd_reg->ENDPTFLUSH & mask) { + if (!guard--) { + return false; + } + } + } while ((dcd_reg->ENDPTSTAT & mask) && guard--); + + return !(dcd_reg->ENDPTSTAT & mask); +} + +/// Everything the manual asks of the DCD when a reset is detected, in its order: clear the setup +/// and completion semaphores, cancel every prime, check the reset is still being driven, and free +/// the dTDs. All of it belongs inside the reset window (IMXRT1060RM 42.5.6.2.1, p.2394); nothing +/// is left for the port change that ends the reset, which only reports the negotiated speed. +static void bus_reset_begin(uint8_t rhport) { ci_hs_regs_t *dcd_reg = CI_HS_REG(rhport); // The reset value for all endpoint types is the control endpoint. If one endpoint @@ -193,17 +237,24 @@ static void bus_reset(uint8_t rhport) { //------------- Clear All Registers -------------// dcd_reg->ENDPTNAK = dcd_reg->ENDPTNAK; dcd_reg->ENDPTNAKEN = 0; - dcd_reg->USBSTS = dcd_reg->USBSTS; dcd_reg->ENDPTSETUPSTAT = dcd_reg->ENDPTSETUPSTAT; dcd_reg->ENDPTCOMPLETE = dcd_reg->ENDPTCOMPLETE; - while (dcd_reg->ENDPTPRIME) {} - dcd_reg->ENDPTFLUSH = 0xFFFFFFFF; - while (dcd_reg->ENDPTFLUSH) {} + uint32_t guard = CI_HS_BUSY_SPIN; + while (dcd_reg->ENDPTPRIME && guard--) {} + dcd_reg->ENDPTFLUSH = 0xFFFFFFFFUL; - // read reset bit in portsc + // All of the above must land while the reset is still being driven - it lasts at least 3 ms. + // Arriving late leaves the controller in an undefined state, and the manual's remedy is to + // hardware-reset it. That clears Run/Stop, so the device detaches and the host will drive a + // fresh reset and enumeration - which is why nothing below this point is worth doing here. + if (!(dcd_reg->PORTSC1 & PORTSC1_PORT_RESET)) { + TU_LOG1("ci_hs: reset cleanup ran past the end of the reset, resetting controller\r\n"); + controller_reset(rhport); + return; // the controller detached; the host's next reset redoes everything below + } - //------------- Queue Head & Queue TD -------------// + //------------- Free all allocated dTDs: the controller will not execute them again -------------// tu_memclr(&_dcd_data, sizeof(dcd_data_t)); //------------- Set up Control Endpoints (0 OUT, 1 IN) -------------// @@ -216,21 +267,19 @@ static void bus_reset(uint8_t rhport) { dcd_dcache_clean_invalidate(&_dcd_data, sizeof(dcd_data_t)); } -bool dcd_init(uint8_t rhport, const tusb_rhport_init_t *rh_init) { - (void)rh_init; - tu_memclr(&_dcd_data, sizeof(dcd_data_t)); - +/// Reset the controller and bring it back up in device mode. Also the manual's remedy when the +/// reset cleanup misses its window: the controller reset clears Run/Stop and detaches the device, +/// so it must be re-initialised completely afterwards (IMXRT1060RM 42.5.6.2.1, p.2394). +static bool controller_reset(uint8_t rhport) { ci_hs_regs_t *dcd_reg = CI_HS_REG(rhport); - TU_ASSERT(ci_ep_count(dcd_reg) <= TUP_DCD_ENDPOINT_MAX); - - #if TU_CHECK_MCU(OPT_MCU_HPM) - usb_phy_init((USB_Type *)dcd_reg, false); - #endif + tu_memclr(&_dcd_data, sizeof(dcd_data_t)); // Reset controller dcd_reg->USBCMD |= USBCMD_RESET; - while (dcd_reg->USBCMD & USBCMD_RESET) {} + uint32_t guard = CI_HS_BUSY_SPIN; + while ((dcd_reg->USBCMD & USBCMD_RESET) && guard--) {} + TU_VERIFY(!(dcd_reg->USBCMD & USBCMD_RESET)); // reached from the ISR too, so never halt here // Set mode to device, must be set immediately after reset uint32_t usbmode = dcd_reg->USBMODE & ~USBMOD_CM_MASK; @@ -257,9 +306,11 @@ bool dcd_init(uint8_t rhport, const tusb_rhport_init_t *rh_init) { dcd_dcache_clean_invalidate(&_dcd_data, sizeof(dcd_data_t)); + _port_change_reason[rhport] = PORT_CHANGE_REASON_RESET; + dcd_reg->ENDPTLISTADDR = (uint32_t)_dcd_data.qhd; // Endpoint List Address has to be 2K alignment dcd_reg->USBSTS = dcd_reg->USBSTS; - dcd_reg->USBINTR = INTR_USB | INTR_ERROR | INTR_PORT_CHANGE | INTR_SUSPEND; + dcd_reg->USBINTR = INTR_USB | INTR_ERROR | INTR_PORT_CHANGE | INTR_RESET | INTR_SUSPEND; uint32_t usbcmd = dcd_reg->USBCMD; usbcmd &= ~USBCMD_INTR_THRESHOLD_MASK; // Interrupt Threshold Interval = 0 @@ -270,8 +321,22 @@ bool dcd_init(uint8_t rhport, const tusb_rhport_init_t *rh_init) { return true; } +bool dcd_init(uint8_t rhport, const tusb_rhport_init_t *rh_init) { + (void)rh_init; + ci_hs_regs_t *dcd_reg = CI_HS_REG(rhport); + + TU_ASSERT(ci_ep_count(dcd_reg) <= TUP_DCD_ENDPOINT_MAX); + + #if TU_CHECK_MCU(OPT_MCU_HPM) + usb_phy_init((USB_Type *)dcd_reg, false); + #endif + + return controller_reset(rhport); +} + bool dcd_deinit(uint8_t rhport) { ci_hs_regs_t* dcd_reg = CI_HS_REG(rhport); + _port_change_reason[rhport] = PORT_CHANGE_REASON_RESET; // disable all interrupt dcd_reg->USBINTR = 0; @@ -280,9 +345,9 @@ bool dcd_deinit(uint8_t rhport) { dcd_reg->USBCMD &= ~USBCMD_RUN_STOP; // flush all endpoints - while (dcd_reg->ENDPTPRIME) {} - dcd_reg->ENDPTFLUSH = 0xFFFFFFFF; - while (dcd_reg->ENDPTFLUSH) {} + uint32_t guard = CI_HS_BUSY_SPIN; + while (dcd_reg->ENDPTPRIME && guard--) {} + flush_endpoints(dcd_reg, 0xFFFFFFFF); return true; } @@ -296,11 +361,18 @@ void dcd_int_disable(uint8_t rhport) { } void dcd_set_address(uint8_t rhport, uint8_t dev_addr) { - // Response with status first before changing device address - dcd_edpt_xfer(rhport, tu_edpt_addr(0, TUSB_DIR_IN), NULL, 0, false); + ci_hs_regs_t *dcd_reg = CI_HS_REG(rhport); + const uint32_t prev = dcd_reg->DEVICEADDR & DEVICEADDR_USBADR_MASK; - ci_hs_regs_t *dcd_reg = CI_HS_REG(rhport); - dcd_reg->DEVICEADDR = (dev_addr << 25) | TU_BIT(24); + // IMXRT1060RM 42.7.23 / UM10503 Table 478: stage the address before priming the status stage so + // hardware loads USBADR at the status ACK. Priming first races that ACK against this write. + dcd_reg->DEVICEADDR = ((uint32_t)dev_addr << DEVICEADDR_USBADR_POS) | DEVICEADDR_USBADRA; + + if (!dcd_edpt_xfer(rhport, tu_edpt_addr(0, TUSB_DIR_IN), NULL, 0, false)) { + // USB 2.0 9.4.6: the address changes only after the status stage completes successfully. The + // status never went out, so drop the stage - USBADRA=0 takes effect instantly. + dcd_reg->DEVICEADDR = prev; + } } void dcd_remote_wakeup(uint8_t rhport) { @@ -468,9 +540,7 @@ bool dcd_edpt_iso_activate(uint8_t rhport, const tusb_desc_endpoint_t *desc_ep) // dcd_dcache_clean_invalidate(&_dcd_data, sizeof(dcd_data_t)); // Flush EP - const uint32_t flush_mask = TU_BIT(epnum + (dir ? 16 : 0)); - dcd_reg->ENDPTFLUSH = flush_mask; - while (dcd_reg->ENDPTFLUSH & flush_mask) {} + flush_endpoints(dcd_reg, TU_BIT(epnum + (dir ? 16 : 0))); // disable to change max packet size ep_ctrl_clear(endptctrl, dir, ENDPTCTRL_ENABLE); @@ -496,7 +566,7 @@ void dcd_edpt_close_all(uint8_t rhport) { } } -static void qhd_start_xfer(uint8_t rhport, uint8_t epnum, uint8_t dir) { +static bool qhd_start_xfer(uint8_t rhport, uint8_t epnum, uint8_t dir) { ci_hs_regs_t *dcd_reg = CI_HS_REG(rhport); dcd_qhd_t *p_qhd = &_dcd_data.qhd[epnum][dir]; dcd_qtd_t *p_qtd = &_dcd_data.qtd[epnum][dir]; @@ -509,13 +579,22 @@ static void qhd_start_xfer(uint8_t rhport, uint8_t epnum, uint8_t dir) { dcd_dcache_clean_invalidate(&_dcd_data, sizeof(dcd_data_t)); if (epnum == 0) { - // follows UM 24.10.8.1.1 Setup packet handling using setup lockout mechanism - // wait until ENDPTSETUPSTAT before priming data/status in response TODO add time out - while (dcd_reg->ENDPTSETUPSTAT & TU_BIT(0)) {} + // Setup lockout (IMXRT1060RM 42.5.6.4.2.1 Setup Phase, p.2403): never prime EP0 while a new + // SETUP is pending. The ISR + // normally consumes ENDPTSETUPSTAT quickly; if the guard trips, fail the transfer so usbd + // releases the endpoint (a pending SETUP supersedes this response anyway; without one, usbd + // stalls EP0 and the host recovers with a fresh control transfer). + uint32_t guard = CI_HS_BUSY_SPIN; + while (dcd_reg->ENDPTSETUPSTAT & TU_BIT(0)) { + if (!guard--) { + return false; + } + } } // start transfer dcd_reg->ENDPTPRIME = TU_BIT(epnum + (dir ? 16 : 0)); + return true; } bool dcd_edpt_xfer(uint8_t rhport, uint8_t ep_addr, uint8_t *buffer, uint16_t total_bytes, bool is_isr) { @@ -531,9 +610,7 @@ bool dcd_edpt_xfer(uint8_t rhport, uint8_t ep_addr, uint8_t *buffer, uint16_t to // Start qhd transfer p_qhd->ff = NULL; - qhd_start_xfer(rhport, epnum, dir); - - return true; + return qhd_start_xfer(rhport, epnum, dir); } #if !CFG_TUD_MEM_DCACHE_ENABLE @@ -584,9 +661,7 @@ bool dcd_edpt_xfer_fifo(uint8_t rhport, uint8_t ep_addr, tu_fifo_t *ff, uint16_t // Start qhd transfer p_qhd->ff = ff; - qhd_start_xfer(rhport, epnum, dir); - - return true; + return qhd_start_xfer(rhport, epnum, dir); } #endif @@ -634,43 +709,43 @@ void dcd_int_handler(uint8_t rhport) { return; } - // Set if the port controller enters the full or high-speed operational state. - // either from Bus Reset or Suspended state - if (int_status & INTR_PORT_CHANGE) { - // TU_LOG2("PortChange %08lx\r\n", dcd_reg->PORTSC1); - - // Reset interrupt is not enabled, we manually check if Port Change is due - // to connection / disconnection - if (dcd_reg->USBSTS & INTR_RESET) { - dcd_reg->USBSTS = INTR_RESET; + const uint8_t pci_reason = _port_change_reason[rhport]; // save current pci_reason - if (dcd_reg->PORTSC1 & PORTSC1_CURRENT_CONNECT_STATUS) { - const uint32_t speed = (dcd_reg->PORTSC1 & PORTSC1_PORT_SPEED) >> PORTSC1_PORT_SPEED_POS; - bus_reset(rhport); - dcd_event_bus_reset(rhport, (tusb_speed_t)speed, true); - } else { - dcd_event_bus_signal(rhport, DCD_EVENT_UNPLUGGED, true); - } - } else { - // Triggered by resuming from suspended state - if (!(dcd_reg->PORTSC1 & PORTSC1_SUSPEND)) { - dcd_event_bus_signal(rhport, DCD_EVENT_RESUME, true); - } - } + if (int_status & INTR_SUSPEND) { + _port_change_reason[rhport] = PORT_CHANGE_REASON_RESUME; // next PCI is resume + dcd_event_bus_signal(rhport, DCD_EVENT_SUSPEND, true); } - if (int_status & INTR_SUSPEND) { - // TU_LOG2("Suspend %08lx\r\n", dcd_reg->PORTSC1); + // USB Reset Received: register cleanup runs here within the reset window (IMXRT1060RM 42.5.6.2.1, p.2394) + // and BUS_RESET_START fires now; BUS_RESET_END, with the final speed, is triggered later by PCI. + if (int_status & INTR_RESET) { + _port_change_reason[rhport] = PORT_CHANGE_REASON_RESET; + bus_reset_begin(rhport); + dcd_event_bus_signal(rhport, DCD_EVENT_BUS_RESET_START, true); + } - if (dcd_reg->PORTSC1 & PORTSC1_SUSPEND) { - // Note: Host may delay more than 3 ms before and/or after bus reset before doing enumeration. - // Skip suspend event if we are not addressed - if ((dcd_reg->DEVICEADDR >> 25) & 0x0f) { - dcd_event_bus_signal(rhport, DCD_EVENT_SUSPEND, true); - } + // Port entered the full/high-speed operational state: the end of a bus reset, or a resume. + if (int_status & INTR_PORT_CHANGE) { + if (pci_reason == PORT_CHANGE_REASON_RESUME) { + dcd_event_bus_signal(rhport, DCD_EVENT_RESUME, true); + } else { + // the undefined encoding falls back to full speed + const uint32_t pspd = (dcd_reg->PORTSC1 & PORTSC1_PORT_SPEED) >> PORTSC1_PORT_SPEED_POS; + const tusb_speed_t speed = (pspd == PORTSC1_PORT_SPEED_LOW) ? TUSB_SPEED_LOW : + (pspd == PORTSC1_PORT_SPEED_HIGH) ? TUSB_SPEED_HIGH : TUSB_SPEED_FULL; + dcd_event_bus_reset(rhport, speed, true); + // This reset is over, so the next port change is a resume. Leaving it at RESET instead would + // dispatch every later resume as another end-of-reset, clearing the queue heads mid-session. + _port_change_reason[rhport] = PORT_CHANGE_REASON_RESUME; } } + // No unplug detection yet, by the manual rather than by omission: IMXRT1060RM 42.7.31 (p.2470) says a zero + // Current Connect Status means the device "did not attach successfully or was forcibly + // disconnected by the software writing a zero to the Run bit ... It does not state the device + // being disconnected or suspended", so a cable pull raises no port change at all. VBUS via + // OTGSC BSV is the manual's disconnect indicator, and it is board dependent. + if (int_status & INTR_USB) { // Make sure we read the latest version of _dcd_data. dcd_dcache_clean_invalidate(&_dcd_data, sizeof(dcd_data_t)); @@ -678,7 +753,7 @@ void dcd_int_handler(uint8_t rhport) { const uint32_t edpt_complete = dcd_reg->ENDPTCOMPLETE; dcd_reg->ENDPTCOMPLETE = edpt_complete; // acknowledge - // 23.10.12.3 Failed QTD also get ENDPTCOMPLETE set + // 42.5.6.6.4 Transfer Completion (p.2413): a failed dTD also sets ENDPTCOMPLETE // nothing to do, we will submit xfer as error to usbd // if (int_status & INTR_ERROR) { } @@ -694,12 +769,39 @@ void dcd_int_handler(uint8_t rhport) { } // Set up Received - // 23.10.10.2 Operational model for setup transfers + // 42.5.6.4.2 Control Endpoint Operation Model (p.2403) // Must be after normal transfer complete since it is possible to have both previous control status + new setup // in the same frame and we should handle previous status first. if (dcd_reg->ENDPTSETUPSTAT) { + // 42.5.6.4.2.1 Setup Phase (p.2403) steps 1-2: duplicate the setup payload BEFORE clearing + // ENDPTSETUPSTAT - + // the clear releases the setup lockout and a back-to-back SETUP (usbtest case 10) can + // overwrite the queue-head buffer immediately after. The copy is read through the volatile + // qualifier rather than memcpy'd because C orders volatile accesses only against each + // other: a plain copy may legally be sunk past the lockout-releasing store below. + union { + tusb_control_request_t request; + uint8_t byte[8]; + } setup; + const volatile uint8_t *setup_src = (const volatile uint8_t *)&_dcd_data.qhd[0][0].setup_request; + for (uint8_t i = 0; i < sizeof(setup.request); i++) { + setup.byte[i] = setup_src[i]; + } dcd_reg->ENDPTSETUPSTAT = dcd_reg->ENDPTSETUPSTAT; - dcd_event_setup_received(rhport, (uint8_t *)(uintptr_t)&_dcd_data.qhd[0][0].setup_request, true); + + // Retire a status/handshake phase left primed by the previous control sequence + // (IMXRT1060RM 42.5.6.4.2.1, p.2403), which would otherwise retire the response the task is about to + // prime for this setup. Skipped when EP0 has nothing primed or priming, since the manual + // does not want the flush wait in an interrupt handler when it has nothing to do. + // One volatile read per statement: C leaves their order unspecified within a single + // expression, which IAR rejects outright (Pa082). + const uint32_t ep0_mask = TU_BIT(0) | TU_BIT(16); + const uint32_t ep0_stat = dcd_reg->ENDPTSTAT; + const uint32_t ep0_prime = dcd_reg->ENDPTPRIME; + if ((ep0_stat | ep0_prime) & ep0_mask) { + flush_endpoints(dcd_reg, ep0_mask); + } + dcd_event_setup_received(rhport, setup.byte, true); } } diff --git a/src/portable/nxp/lpc_ip3511/dcd_lpc_ip3511.c b/src/portable/nxp/lpc_ip3511/dcd_lpc_ip3511.c index d5b03e4b1..42f6750b1 100644 --- a/src/portable/nxp/lpc_ip3511/dcd_lpc_ip3511.c +++ b/src/portable/nxp/lpc_ip3511/dcd_lpc_ip3511.c @@ -87,6 +87,10 @@ enum { DEVCMDSTAT_SUSPEND_CHANGE_MASK = TU_BIT(25), DEVCMDSTAT_RESET_CHANGE_MASK = TU_BIT(26), DEVCMDSTAT_VBUS_DEBOUNCED_MASK = TU_BIT(28), + + // write-1-to-clear latches + DEVCMDSTAT_W1C_MASK = DEVCMDSTAT_SETUP_RECEIVED_MASK | DEVCMDSTAT_CONNECT_CHANGE_MASK | + DEVCMDSTAT_SUSPEND_CHANGE_MASK | DEVCMDSTAT_RESET_CHANGE_MASK, }; enum { @@ -171,7 +175,9 @@ typedef struct ep_cmd_sts_t ep[2*MAX_EP_PAIRS][2]; xfer_dma_t dma[2*MAX_EP_PAIRS]; - TU_ATTR_ALIGNED(64) uint8_t setup_packet[8]; + // volatile: the controller DMAs a new setup packet into this buffer as soon as the SETUP + // latch is cleared, so reads of it must stay ordered against the register accesses around them + TU_ATTR_ALIGNED(64) volatile uint8_t setup_packet[8]; }dcd_data_t; // EP list must be 256-byte aligned @@ -180,8 +186,12 @@ typedef struct // Use CFG_TUD_MEM_SECTION to place it accordingly. CFG_TUD_MEM_SECTION TU_ATTR_ALIGNED(256) static dcd_data_t _dcd; -// Dummy buffer to fix ZLPs overwriting the buffer (probably an USB/DMA controller bug) -// TODO find way to save memory +// Dummy buffer to fix ZLPs overwriting the buffer: Errata LPC55S6x USB.5 / LPC55S2x USB.4 - the +// HS device controller always DMA-writes OUT data in 8-byte units, so up to 7 bytes land past the +// received length. This redirects the ZLP case; the general short-OUT case is unhandled here +// (TinyUSB's own endpoint buffers are sized/aligned so the spill stays inside them, but a tight +// caller buffer can be overrun by up to 7 bytes - the SDK's documented workaround is a bounce +// buffer). TODO find way to save memory CFG_TUD_MEM_SECTION TU_ATTR_ALIGNED(64) static uint8_t dummy[8]; //--------------------------------------------------------------------+ @@ -221,7 +231,7 @@ static const dcd_controller_t _dcd_controller[] = { // INTERNAL OBJECT & FUNCTION DECLARATION //--------------------------------------------------------------------+ -TU_ATTR_ALWAYS_INLINE static inline uint16_t get_buf_offset(void const * buffer) { +TU_ATTR_ALWAYS_INLINE static inline uint16_t get_buf_offset(void const volatile * buffer) { uint32_t addr = (uint32_t) buffer; TU_ASSERT( (addr & 0x3f) == 0, 0 ); return ( (addr >> 6) & 0xFFFFUL ) ; @@ -247,6 +257,16 @@ TU_ATTR_ALWAYS_INLINE static inline bool rhport_is_highspeed(uint8_t rhport) { return _dcd_controller[rhport].is_highspeed; } + +// DEVCMDSTAT mixes RW fields with write-1-to-clear latches (SETUP + the 3 change bits): a blind +// RMW writes a pending latch back as 1 and silently clears it (a SETUP eaten this way strands +// EP0). Mask the latches on every update; pass one in set_mask only to clear it. +TU_ATTR_ALWAYS_INLINE static inline void devcmdstat_update(dcd_registers_t* dcd_reg, + uint32_t clear_mask, uint32_t set_mask) { + const uint32_t v = dcd_reg->DEVCMDSTAT & ~(DEVCMDSTAT_W1C_MASK | clear_mask); + dcd_reg->DEVCMDSTAT = v | set_mask; +} + //--------------------------------------------------------------------+ // CONTROLLER API //--------------------------------------------------------------------+ @@ -284,8 +304,10 @@ bool dcd_init(uint8_t rhport, const tusb_rhport_init_t* rh_init) { dcd_reg->DATABUFSTART = tu_align((uint32_t) &_dcd, TU_BIT(22)); // 22-bit alignment dcd_reg->INTSTAT = dcd_reg->INTSTAT; // clear all pending interrupt dcd_reg->INTEN = INT_DEVICE_STATUS_MASK; - dcd_reg->DEVCMDSTAT |= DEVCMDSTAT_DEVICE_ENABLE_MASK | DEVCMDSTAT_DEVICE_CONNECT_MASK | - DEVCMDSTAT_RESET_CHANGE_MASK | DEVCMDSTAT_CONNECT_CHANGE_MASK | DEVCMDSTAT_SUSPEND_CHANGE_MASK; + // deliberately clear every latch (incl. a SETUP left by a bootloader/warm start) for a + // deterministic init state + devcmdstat_update(dcd_reg, 0, DEVCMDSTAT_DEVICE_ENABLE_MASK | DEVCMDSTAT_DEVICE_CONNECT_MASK | + DEVCMDSTAT_W1C_MASK); NVIC_ClearPendingIRQ(_dcd_controller[rhport].irqnum); @@ -309,8 +331,7 @@ void dcd_set_address(uint8_t rhport, uint8_t dev_addr) // Response with status first before changing device address dcd_edpt_xfer(rhport, tu_edpt_addr(0, TUSB_DIR_IN), NULL, 0, false); - dcd_reg->DEVCMDSTAT &= ~DEVCMDSTAT_DEVICE_ADDR_MASK; - dcd_reg->DEVCMDSTAT |= dev_addr; + devcmdstat_update(dcd_reg, DEVCMDSTAT_DEVICE_ADDR_MASK, dev_addr); } void dcd_remote_wakeup(uint8_t rhport) @@ -321,13 +342,13 @@ void dcd_remote_wakeup(uint8_t rhport) void dcd_connect(uint8_t rhport) { dcd_registers_t* dcd_reg = _dcd_controller[rhport].regs; - dcd_reg->DEVCMDSTAT |= DEVCMDSTAT_DEVICE_CONNECT_MASK; + devcmdstat_update(dcd_reg, 0, DEVCMDSTAT_DEVICE_CONNECT_MASK); } void dcd_disconnect(uint8_t rhport) { dcd_registers_t* dcd_reg = _dcd_controller[rhport].regs; - dcd_reg->DEVCMDSTAT &= ~DEVCMDSTAT_DEVICE_CONNECT_MASK; + devcmdstat_update(dcd_reg, DEVCMDSTAT_DEVICE_CONNECT_MASK, 0); } void dcd_sof_enable(uint8_t rhport, bool en) @@ -380,9 +401,17 @@ void dcd_edpt_clear_stall(uint8_t rhport, uint8_t ep_addr) uint8_t const ep_id = ep_addr2id(ep_addr); + // Preserve rf_tv: for non-control endpoints it is a TYPE bit, not the toggle value (UM11126: + // T=1 + RF 1/0 = interrupt/iso). Zeroing it here turned HS periodic interrupt endpoints into + // isochronous - no handshake on OUT, dead IN (usbtest cases 25/26 on lpc55 HS port). + // TODO implement the Errata LPC546xx USB.13 work-around (same semantics in UM11126): with RF/TV preserved at 1, TR + // loads the toggle from TV, so an HS interrupt endpoint restarts on DATA1 after clear-halt and + // the host discards one packet as a retransmission. The documented workaround needs an + // interrupt-on-NAK state machine (park as generic TR=1/TV=0, wait for a NAKed token to latch + // toggle 0 via EPTOGGLE, restore the type) - deferred; one lost packet beats the fully broken + // endpoint the old rf_tv clear caused. _dcd.ep[ep_id][0].cmd_sts.stall = 0; _dcd.ep[ep_id][0].cmd_sts.toggle_reset = 1; - _dcd.ep[ep_id][0].cmd_sts.rf_tv = 0; } bool dcd_edpt_open(uint8_t rhport, tusb_desc_endpoint_t const * p_endpoint_desc) @@ -432,7 +461,7 @@ void dcd_edpt_close_all (uint8_t rhport) { for (uint8_t ep_id = 0; ep_id < 2*_dcd_controller[rhport].ep_pairs; ++ep_id) { - _dcd.ep[ep_id][0].cmd_sts.active = _dcd.ep[ep_id][0].cmd_sts.active = 0; // TODO proper way is to EPSKIP then wait ep[][].active then write ep[][].disable (see table 778 in LPC55S69 Use Manual) + _dcd.ep[ep_id][0].cmd_sts.active = _dcd.ep[ep_id][1].cmd_sts.active = 0; // TODO proper way is to EPSKIP then wait ep[][].active then write ep[][].disable (see table 778 in LPC55S69 Use Manual) _dcd.ep[ep_id][0].cmd_sts.disable = _dcd.ep[ep_id][1].cmd_sts.disable = 1; } } @@ -538,7 +567,7 @@ static void bus_reset(uint8_t rhport) dcd_reg->EPSKIP = 0xFFFFFFFF; dcd_reg->INTSTAT = dcd_reg->INTSTAT; // clear all pending interrupt - dcd_reg->DEVCMDSTAT |= DEVCMDSTAT_SETUP_RECEIVED_MASK; // clear setup received interrupt + devcmdstat_update(dcd_reg, 0, DEVCMDSTAT_SETUP_RECEIVED_MASK); // clear setup received interrupt dcd_reg->INTEN = INT_DEVICE_STATUS_MASK | TU_BIT(0) | TU_BIT(1); // enable device status & control endpoints } @@ -597,18 +626,25 @@ void dcd_int_handler(uint8_t rhport) { dcd_registers_t* dcd_reg = _dcd_controller[rhport].regs; - uint32_t const cmd_stat = dcd_reg->DEVCMDSTAT; - uint32_t int_status = dcd_reg->INTSTAT; - int_status &= dcd_reg->INTEN; + int_status &= dcd_reg->INTEN; dcd_reg->INTSTAT = int_status; // Acknowledge handled interrupt if (int_status == 0) return; + // Snapshot after the INTSTAT ack: latch bits persist (RWC) so nothing is lost, while the reverse + // order could consume INTSTAT bit0 for a SETUP not yet visible in the snapshot - stranding the + // SETUP (INTSTAT is edge-latched) and feeding bit0 to process_xfer_isr as a bogus completion. + uint32_t const cmd_stat = dcd_reg->DEVCMDSTAT; + //------------- Device Status -------------// if ( int_status & INT_DEVICE_STATUS_MASK ) { - dcd_reg->DEVCMDSTAT |= DEVCMDSTAT_RESET_CHANGE_MASK | DEVCMDSTAT_CONNECT_CHANGE_MASK | DEVCMDSTAT_SUSPEND_CHANGE_MASK; + // clear only the change latches observed in the snapshot: one latched by hardware between the + // snapshot and this write would be acknowledged unseen (its DEV_INT re-latches and dispatches + // next pass instead) + devcmdstat_update(dcd_reg, 0, cmd_stat & + (DEVCMDSTAT_RESET_CHANGE_MASK | DEVCMDSTAT_CONNECT_CHANGE_MASK | DEVCMDSTAT_SUSPEND_CHANGE_MASK)); if ( cmd_stat & DEVCMDSTAT_RESET_CHANGE_MASK) // bus reset { @@ -653,15 +689,43 @@ void dcd_int_handler(uint8_t rhport) _dcd.ep[0][0].cmd_sts.active = _dcd.ep[1][0].cmd_sts.active = 0; _dcd.ep[0][0].cmd_sts.stall = _dcd.ep[1][0].cmd_sts.stall = 0; - dcd_reg->DEVCMDSTAT |= DEVCMDSTAT_SETUP_RECEIVED_MASK; + // UM flow: ack the latch FIRST, then read the payload. This IP has no setup lockout, so a + // back-to-back SETUP can overwrite _dcd.setup_packet at any time - but with the latch already + // released, any such overwrite re-latches SETUP_RECEIVED and is redelivered (worst case a + // superseded duplicate, absorbed by usbd's queued-setup counter). The reverse order can + // consume the newer SETUP's latch unseen and lose it. + devcmdstat_update(dcd_reg, 0, DEVCMDSTAT_SETUP_RECEIVED_MASK); + + // UM11126 Fig 163 (control EP0 flowchart) requires clearing the EP0IN interrupt here: a + // control IN completion latched before this SETUP must not reach usbd after it, where it + // would be applied to the new request and arm its status stage early. EP0OUT goes with it - + // bit0 is set by SETUP reception too, and left set it would replay next pass as a phantom + // completion. Neither can discard live work: the SETUP latch NAKs all EP0 traffic until the + // update above, and both EP0 Active bits were cleared a few lines up. + dcd_reg->INTSTAT = TU_BIT(0) | TU_BIT(1); - dcd_event_setup_received(rhport, _dcd.setup_packet, true); + // Copied a byte at a time rather than with memcpy: C orders volatile accesses only against + // each other, so a non-volatile copy of this buffer may be sunk below the guard read that + // follows - gcc does exactly that at -O2 and -O3, leaving only -Os correct. + uint8_t setup_copy[8]; + for (uint8_t i = 0; i < sizeof(setup_copy); i++) { + setup_copy[i] = _dcd.setup_packet[i]; + } + + // a SETUP that raced in after the acks (its bit0 consumed above) makes this copy suspect: + // its latch is visible again, so re-raise the endpoint interrupt and let the next pass + // deliver the newer payload rather than passing up bytes that may be torn between the two + if (dcd_reg->DEVCMDSTAT & DEVCMDSTAT_SETUP_RECEIVED_MASK) { + dcd_reg->INTSETSTAT = TU_BIT(0); + } else { + dcd_event_setup_received(rhport, setup_copy, true); + } // keep waiting for next setup prepare_setup_packet(rhport); - // clear bit0 - int_status = tu_bit_clear(int_status, 0); + // drop both EP0 bits: acked above, and neither belongs to the request this SETUP starts + int_status &= ~(TU_BIT(0) | TU_BIT(1)); } // Endpoint transfer complete interrupt diff --git a/src/portable/ohci/ohci.c b/src/portable/ohci/ohci.c index e2c5956b3..2a174e46c 100644 --- a/src/portable/ohci/ohci.c +++ b/src/portable/ohci/ohci.c @@ -378,7 +378,7 @@ static void ed_list_remove_by_addr(ohci_ed_t * p_head, uint8_t dev_addr) { ohci_ed_t* p_prev = p_head; while (p_prev->next) { - ohci_ed_t* ed = (ohci_ed_t*)_virt_addr((void*)p_prev->next); + ohci_ed_t* ed = hcd_dcache_uncached((ohci_ed_t*)_virt_addr((void*)p_prev->next)); if (ed->w0.dev_addr == dev_addr) { // Prevent Host Controller from processing this ED while we remove it @@ -387,12 +387,28 @@ static void ed_list_remove_by_addr(ohci_ed_t * p_head, uint8_t dev_addr) { // unlink ed, will also move up p_prev p_prev->next = ed->next; - // point the removed ED's next pointer to list head to make sure HC can always safely move away from this ED - ed->next = (uint32_t)_phys_addr(p_head); - ed->w0.used = 0; - ed->w0.skip = 0; + // Control endpoints (EP number 0) are statically allocated with the device which are only reused + // after connection of another device long after HC has finished with them now, these can be freed immediately. + if (ed->w0.ep_number != 0) { + // Wait until the next frame before reclaiming the ED and its TDs. Set the deadline before + // publishing is_reclaiming so a pending SOF IRQ cannot use an older deadline for this ED. + ohci_data.reclaim_frame = (uint16_t)(OHCI_REG->frame_number + 1); + ed->w0.is_reclaiming = 1; + + // 5.2.7.1.2 Removing. Disable list processing for bulk + if (p_head == p_ed_head[TUSB_XFER_BULK]) { + OHCI_REG->control &= ~OHCI_CONTROL_LIST_BULK_ENABLE_MASK; + } + + // Temporarily enable SOF IRQ. Clear any pending SOF first to wait for the next frame. + OHCI_REG->interrupt_status = OHCI_INT_SOF_MASK; + OHCI_REG->interrupt_enable = OHCI_INT_SOF_MASK; + } else { + ed->w0.used = 0; + ed->w0.skip = 0; + } } else { - p_prev = (ohci_ed_t*)_virt_addr((void*)p_prev->next); + p_prev = ed; } } } @@ -400,6 +416,7 @@ static void ed_list_remove_by_addr(ohci_ed_t * p_head, uint8_t dev_addr) { static ohci_gtd_t* gtd_find_free(void) { for (uint8_t i = 0; i < GTD_MAX; i++) { if (!ohci_data.gtd_pool[i].used) { + ohci_data.gtd_pool[i].used = 1; return &ohci_data.gtd_pool[i]; } } @@ -652,6 +669,60 @@ void hcd_int_handler(uint8_t hostid, bool in_isr) { // Disable MIE as per OHCI spec 5.3 OHCI_REG->interrupt_disable = OHCI_INT_MASTER_ENABLE_MASK; + // Start of frame (SOF). Signed subtraction handles frame number rollover and delayed interrupts. + if ((int_status & OHCI_INT_SOF_MASK) && + ((int16_t)((uint16_t)OHCI_REG->frame_number - ohci_data.reclaim_frame) >= 0)) { + OHCI_REG->interrupt_disable = OHCI_INT_SOF_MASK; + + bool re_enable_lists = false; + + for (size_t i = 0; i < ED_MAX; i++) { + ohci_ed_t* ed = hcd_dcache_uncached(&ohci_data.ed_pool[i]); + if (ed->w0.used && ed->w0.is_reclaiming) { + TU_ASSERT(ed->w0.skip == 1, ); + TU_ASSERT(ed->w0.ep_number != 0, ); + + // Reclaim orphaned TDs + uint32_t td_addr = ed->td_head.address & ~0x0F; + while (td_addr) { + if (!ed->w0.is_iso) { + ohci_gtd_t *gtd = (ohci_gtd_t*)_virt_addr((void*)(uintptr_t)td_addr); + gtd->used = 0; + } else { + // TODO: Free ITD once implemented + } + + if (td_addr == ed->td_tail) { + break; + } + td_addr = ((ohci_td_item_t*)_virt_addr((void*)(uintptr_t)td_addr))->next; + } + + ed->w0.is_reclaiming = 0; + ed->w0.used = 0; + ed->w0.skip = 0; + + re_enable_lists = true; + } + } + + if (re_enable_lists) { + // 5.2.7.1.2 Removing + // Reset current ED pointers and re-enable lists + // Once the next frame has started, the HcControlCurrentED or HcBulkCurrentED register should be adjusted so + // that it does not point to the Endpoint Descriptor being removed (for simplicity you may just write + // a zero to the register); + if (!(OHCI_REG->control & OHCI_CONTROL_LIST_CONTROL_ENABLE_MASK)) { + OHCI_REG->control_current_ed = 0; + OHCI_REG->control |= OHCI_CONTROL_LIST_CONTROL_ENABLE_MASK; + } + if (!(OHCI_REG->control & OHCI_CONTROL_LIST_BULK_ENABLE_MASK)) { + OHCI_REG->bulk_current_ed = 0; + OHCI_REG->control |= OHCI_CONTROL_LIST_BULK_ENABLE_MASK; + } + } + } + // Frame number overflow if (int_status & OHCI_INT_FRAME_OVERFLOW_MASK) { ohci_data.frame_number_hi++; diff --git a/src/portable/ohci/ohci.h b/src/portable/ohci/ohci.h index 84ae04b0f..e28c6404f 100644 --- a/src/portable/ohci/ohci.h +++ b/src/portable/ohci/ohci.h @@ -107,7 +107,8 @@ typedef union { // HCD: make use of 5 reserved bits uint32_t used : 1; uint32_t is_interrupt_xfer : 1; - uint32_t : 3; + uint32_t is_reclaiming : 1; + uint32_t : 2; }; uint32_t value; } ohci_ed_word0_t; @@ -182,6 +183,7 @@ typedef struct TU_ATTR_ALIGNED(256) { gtd_extra_data_t gtd_extra[GTD_MAX]; volatile uint16_t frame_number_hi; + volatile uint16_t reclaim_frame; } ohci_data_t; //--------------------------------------------------------------------+ diff --git a/src/portable/synopsys/dwc2/hcd_dwc2.c b/src/portable/synopsys/dwc2/hcd_dwc2.c index 089b839ae..5a171f80e 100644 --- a/src/portable/synopsys/dwc2/hcd_dwc2.c +++ b/src/portable/synopsys/dwc2/hcd_dwc2.c @@ -26,6 +26,12 @@ #endif #define DWC2_CHANNEL_COUNT_MAX 16u // absolute max channel count + + // Conservative time budget for enabling a slave-mode periodic OUT channel and writing its first packet before the + // current (micro)frame ends. HFNUM.FrRem is measured in PHY clocks; 1024 clocks are 17.1 us at 60 MHz, 21.3 us at + // 48 MHz, or 34.1 us at 30 MHz. Defer to SOF when less time remains. + #define DWC2_PERIODIC_OUT_MIN_FRREM 1024u + TU_VERIFY_STATIC(CFG_TUH_DWC2_ENDPOINT_MAX <= 255, "currently only use 8-bit for index"); enum { @@ -37,7 +43,9 @@ enum { }; enum { - HCD_XFER_PERIOD_SPLIT_NYET_MAX = 3 + HCD_XFER_PERIOD_SPLIT_NYET_MAX = 3, + HCD_FRAME_NUMBER_MASK = 0x3fff, + HCD_FRAME_COUNT = HCD_FRAME_NUMBER_MASK + 1 }; //-------------------------------------------------------------------- @@ -56,18 +64,22 @@ typedef struct { }; struct TU_ATTR_PACKED { - uint32_t uframe_interval : 18; // micro-frame interval + uint32_t uframe_interval : 19; // micro-frame interval uint32_t speed : 2; uint32_t next_pid : 2; // PID for next transfer uint32_t next_do_ping : 1; // Do PING for next transfer if possible (highspeed OUT) uint32_t closing : 1; // endpoint is closing - // uint32_t : 8; + uint32_t aborting : 1; // periodic DMA channel is waiting for its automatic halt + uint32_t periodic_phase : 1; // periodic transfer phase is established + uint32_t xfer_pending : 1; // periodic transfer waiting for its service interval + // uint32_t : 4; }; - uint32_t uframe_countdown; // micro-frame count down to transfer for periodic, only need 18-bit + uint32_t uframe_countdown; // micro-frame count down to transfer for periodic, only need 19-bit uint8_t* buffer; uint16_t buflen; + uint16_t periodic_frame; // frame/microframe number of the last scheduled periodic transaction } hcd_endpoint_t; // Additional info for each channel when it is active @@ -86,6 +98,7 @@ typedef struct { // be composed of multiple channel_xfer_start() (retry with NAK/NYET) uint16_t fifo_bytes; // bytes written/read from/to FIFO (may not be transferred on USB bus). uint8_t retry_disabled; // 1: channel was disabled to throttle a split retry (NAK in / XactErr out); re-arm on its halt + volatile bool aborting; // periodic DMA abort waiting for the channel's automatic halt } hcd_xfer_t; typedef struct { @@ -187,7 +200,7 @@ TU_ATTR_ALWAYS_INLINE static inline bool channel_disable(const dwc2_regs_t* dwc2 // the worst case), the controller generates a channel halted and disables the channel automatically. // - For split enabled channels (both non-periodic and periodic), channel disable must not be programmed randomly. // However, channel disable can be programmed for specific scenarios such as NAK and FrmOvrn. - if (is_period && (channel->hcsplt & HCSPLT_SPLITEN)) { + if (is_period) { return true; } } else { @@ -200,13 +213,86 @@ TU_ATTR_ALWAYS_INLINE static inline bool channel_disable(const dwc2_regs_t* dwc2 return true; } -// attempt to send IN token to receive data -TU_ATTR_ALWAYS_INLINE static inline bool channel_send_in_token(const dwc2_regs_t* dwc2, dwc2_channel_t* channel) { +// Retire all active host channels on root-port disconnect without waiting for +// Channel Halted interrupts. +// stop new channel/FIFO interrupts, flush queued slave requests, request a +// halt for enabled channels, then clear their interrupt and software state. +static void channel_cleanup_on_disconnect(dwc2_regs_t *dwc2) { + const uint32_t xfer_ints = GINTSTS_NPTX_FIFO_EMPTY | GINTSTS_PTX_FIFO_EMPTY | GINTSTS_HCINT; + dwc2->gintmsk &= ~xfer_ints; + dwc2->gintsts = xfer_ints; + dwc2->haintmsk = 0; + + const uint8_t max_channel = dwc2_channel_count(dwc2); + #if CFG_TUH_DWC2_SLAVE_ENABLE + if (!dma_host_enabled(dwc2)) { + // With CHENA clear, CHDIS flushes a posted request without consuming + // request-queue space. Clear EPDIR as required for this flush operation. + for (uint8_t ch_id = 0; ch_id < max_channel; ch_id++) { + if (_hcd_data.xfer[ch_id].allocated) { + dwc2_channel_t *channel = &dwc2->channel[ch_id]; + const uint32_t hcchar = channel->hcchar; + if (hcchar & HCCHAR_CHENA) { + channel->hcchar = (hcchar & ~(HCCHAR_CHENA | HCCHAR_EPDIR)) | HCCHAR_CHDIS; + } + } + } + } + #endif + + for (uint8_t ch_id = 0; ch_id < max_channel; ch_id++) { + if (_hcd_data.xfer[ch_id].allocated) { + dwc2_channel_t *channel = &dwc2->channel[ch_id]; + const uint32_t hcchar = channel->hcchar; + if (hcchar & HCCHAR_CHENA) { + channel->hcchar = hcchar | HCCHAR_CHDIS; + } + channel->hcintmsk = 0; + channel->hcint = 0xFFFFFFFFU; + } + } + + tu_memclr(_hcd_data.xfer, sizeof(_hcd_data.xfer)); + for (uint8_t ep_id = 0; ep_id < CFG_TUH_DWC2_ENDPOINT_MAX; ep_id++) { + hcd_endpoint_t *edpt = &_hcd_data.edpt[ep_id]; + if (edpt->hcchar_bm.enable) { + edpt->closing = 1; + edpt->xfer_pending = 0; + } + } +} + +// Enable a channel, selecting the following frame for a new periodic transfer. +// Return that frame from the same HFNUM sample used for ODDFRM selection. +// Clear CHDIS explicitly: a halted channel may retain it in HCCHAR. +TU_ATTR_ALWAYS_INLINE static inline uint16_t channel_enable(dwc2_regs_t* dwc2, dwc2_channel_t* channel, + bool next_periodic_frame) { + uint32_t hcchar = channel->hcchar & ~HCCHAR_CHDIS; + uint16_t periodic_frame = 0; + if (next_periodic_frame) { + // Prevent the USB interrupt from consuming the selected frame before + // HCCHAR.CHENA is written. Queue-space waits happen before this helper. + const uint32_t gahbcfg = dwc2->gahbcfg; + dwc2->gahbcfg = gahbcfg & ~GAHBCFG_GINT; + const uint32_t hfnum = dwc2->hfnum; + hcchar = (hcchar & ~HCCHAR_ODDFRM) | (((hfnum & 1u) ^ 1u) << HCCHAR_ODDFRM_Pos); + channel->hcchar = hcchar | HCCHAR_CHENA; + periodic_frame = (uint16_t) ((hfnum + 1u) & HCD_FRAME_NUMBER_MASK); + dwc2->gahbcfg = gahbcfg; + } else { + channel->hcchar = hcchar | HCCHAR_CHENA; + } + return periodic_frame; +} + +// Attempt to send an IN token to receive data. For a new periodic transfer, +// select its frame only after request-queue space is available. +TU_ATTR_ALWAYS_INLINE static inline uint16_t channel_send_in_token(dwc2_regs_t* dwc2, dwc2_channel_t* channel, + bool next_periodic_frame) { while (0 == req_queue_avail(dwc2, channel_is_periodic(channel->hcchar))) { // blocking wait for request queue available } - channel->hcchar |= HCCHAR_CHENA; - return true; + return channel_enable(dwc2, channel, next_periodic_frame); } // Find currently enabled channel. Note: EP0 is bidirectional @@ -262,11 +348,13 @@ static void edpt_close(dwc2_regs_t *dwc2, uint8_t ep_id) { // Find an endpoint that is opened previously with hcd_edpt_open() // Note: EP0 is bidirectional -TU_ATTR_ALWAYS_INLINE static inline uint8_t edpt_find_opened(uint8_t dev_addr, uint8_t ep_num, uint8_t ep_dir) { +TU_ATTR_ALWAYS_INLINE static inline uint8_t edpt_find_opened(uint8_t dev_addr, uint8_t ep_num, uint8_t ep_dir, + bool include_closing) { for (uint8_t i = 0; i < (uint8_t)CFG_TUH_DWC2_ENDPOINT_MAX; i++) { const hcd_endpoint_t *edpt = &_hcd_data.edpt[i]; const dwc2_channel_char_t hcchar_bm = edpt->hcchar_bm; - if (hcchar_bm.enable && hcchar_bm.dev_addr == dev_addr && hcchar_bm.ep_num == ep_num && + if (hcchar_bm.enable && (include_closing || !edpt->closing) && hcchar_bm.dev_addr == dev_addr && + hcchar_bm.ep_num == ep_num && (ep_num == 0 || hcchar_bm.ep_dir == ep_dir)) { return i; } @@ -336,13 +424,13 @@ TU_ATTR_ALWAYS_INLINE static inline uint8_t cal_next_pid(uint8_t pid, uint8_t pa static void dfifo_host_init(uint8_t rhport, bool is_hs_phy) { const dwc2_controller_t* dwc2_controller = &_dwc2_controller[rhport]; dwc2_regs_t* dwc2 = DWC2_REG(rhport); - const dwc2_ghwcfg2_t ghwcfg2 = {.value = dwc2->ghwcfg2}; + const uint8_t channel_count = dwc2_channel_count(dwc2); // Scatter/Gather DMA mode is not yet supported. Buffer DMA only need 1 words per channel const bool is_dma = dma_host_enabled(dwc2); uint16_t dfifo_top = dwc2_controller->otg_dfifo_depth; if (is_dma) { - dfifo_top -= ghwcfg2.num_host_ch; + dfifo_top -= channel_count; } // fixed allocation for now, improve later: @@ -358,13 +446,12 @@ static void dfifo_host_init(uint8_t rhport, bool is_hs_phy) { } uint16_t nptxfsiz = 2 * nptx_largest; - uint16_t rxfsiz = 2 * (ptx_largest + 2) + ghwcfg2.num_host_ch; + uint16_t rxfsiz = 2 * (ptx_largest + 2) + channel_count; TU_ASSERT(dfifo_top >= (nptxfsiz + rxfsiz),); uint16_t ptxfsiz = dfifo_top - (nptxfsiz + rxfsiz); dwc2->gdfifocfg = (dfifo_top << GDFIFOCFG_EPINFOBASE_SHIFT) | dfifo_top; - dfifo_top -= rxfsiz; dwc2->grxfsiz = rxfsiz; dfifo_top -= nptxfsiz; @@ -548,7 +635,7 @@ bool hcd_edpt_open(uint8_t rhport, uint8_t dev_addr, const tusb_desc_endpoint_t* edpt->next_pid = HCTSIZ_PID_DATA0; switch (desc_ep->bmAttributes.xfer) { case TUSB_XFER_ISOCHRONOUS: - edpt->uframe_interval = 1 << (desc_ep->bInterval - 1); + edpt->uframe_interval = 1u << (desc_ep->bInterval - 1); if (bus_info.speed == TUSB_SPEED_FULL) { edpt->uframe_interval <<= 3; } @@ -556,7 +643,7 @@ bool hcd_edpt_open(uint8_t rhport, uint8_t dev_addr, const tusb_desc_endpoint_t* case TUSB_XFER_INTERRUPT: if (bus_info.speed == TUSB_SPEED_HIGH) { - edpt->uframe_interval = 1 << (desc_ep->bInterval - 1); + edpt->uframe_interval = 1u << (desc_ep->bInterval - 1); } else { edpt->uframe_interval = desc_ep->bInterval << 3; } @@ -566,6 +653,13 @@ bool hcd_edpt_open(uint8_t rhport, uint8_t dev_addr, const tusb_desc_endpoint_t* break; } + if (channel_is_periodic(edpt->hcchar)) { + // HFNUM cannot distinguish elapsed periods longer than one counter cycle. USB permits the host to provide a + // shorter period, so bound the selected period to the history available from HFNUM. + const uint32_t ucount = (rh_speed == TUSB_SPEED_HIGH) ? 1u : 8u; + edpt->uframe_interval = tu_min32(edpt->uframe_interval, HCD_FRAME_COUNT * ucount); + } + return true; } @@ -573,7 +667,7 @@ bool hcd_edpt_close(uint8_t rhport, uint8_t daddr, uint8_t ep_addr) { dwc2_regs_t *dwc2 = DWC2_REG(rhport); const uint8_t ep_num = tu_edpt_number(ep_addr); const uint8_t ep_dir = tu_edpt_dir(ep_addr); - const uint8_t ep_id = edpt_find_opened(daddr, ep_num, ep_dir); + const uint8_t ep_id = edpt_find_opened(daddr, ep_num, ep_dir, true); TU_ASSERT(ep_id < CFG_TUH_DWC2_ENDPOINT_MAX); edpt_close(dwc2, ep_id); @@ -588,7 +682,10 @@ static void channel_xfer_out_wrapup(dwc2_regs_t* dwc2, uint8_t ch_id) { hcd_endpoint_t* edpt = &_hcd_data.edpt[xfer->ep_id]; const dwc2_channel_tsize_t hctsiz = {.value = channel->hctsiz}; - edpt->next_pid = hctsiz.pid; // save PID + const dwc2_channel_char_t hcchar = {.value = channel->hcchar}; + if (hcchar.ep_type != HCCHAR_EPTYPE_ISOCHRONOUS) { + edpt->next_pid = hctsiz.pid; // save PID + } /* Since hctsiz.xfersize field reflects the number of bytes transferred via the AHB, not the USB) * For IN: we can use hctsiz.xfersize as remaining bytes. @@ -597,7 +694,6 @@ static void channel_xfer_out_wrapup(dwc2_regs_t* dwc2, uint8_t ch_id) { * transfer was halted before its normal completion. */ const uint16_t remain_packets = hctsiz.packet_count; - const dwc2_channel_char_t hcchar = {.value = channel->hcchar}; const uint16_t total_packets = cal_packet_count(edpt->buflen, hcchar.ep_size); const uint16_t actual_bytes = (total_packets - remain_packets) * hcchar.ep_size; @@ -607,20 +703,26 @@ static void channel_xfer_out_wrapup(dwc2_regs_t* dwc2, uint8_t ch_id) { edpt->buflen -= actual_bytes; } -static bool channel_xfer_start(dwc2_regs_t* dwc2, uint8_t ch_id) { +#if CFG_TUH_DWC2_SLAVE_ENABLE +static bool channel_txfifo_write(dwc2_regs_t* dwc2, uint8_t ch_id, bool is_periodic); +#endif +static void periodic_xfer_defer(dwc2_regs_t* dwc2, hcd_endpoint_t* edpt, uint32_t uframe_countdown); + +static bool channel_xfer_start(dwc2_regs_t* dwc2, uint8_t ch_id, bool defer_periodic_out) { hcd_xfer_t* xfer = &_hcd_data.xfer[ch_id]; hcd_endpoint_t* edpt = &_hcd_data.edpt[xfer->ep_id]; dwc2_channel_char_t* hcchar_bm = &edpt->hcchar_bm; dwc2_channel_t* channel = &dwc2->channel[ch_id]; bool const is_period = channel_is_periodic(edpt->hcchar); - +#if CFG_TUH_DWC2_SLAVE_ENABLE + const uint8_t saved_pid = edpt->next_pid; + const uint8_t saved_do_ping = edpt->next_do_ping; +#endif + uint16_t periodic_frame = 0; // clear previous state xfer->fifo_bytes = 0; // hchar: restore but don't enable yet - if (is_period) { - hcchar_bm->odd_frame = 1 - (dwc2->hfnum & 1); // transfer on next frame - } channel->hcchar = (edpt->hcchar & ~HCCHAR_CHENA); // hctsiz: zero length packet still count as 1 @@ -636,15 +738,17 @@ static bool channel_xfer_start(dwc2_regs_t* dwc2, uint8_t ch_id) { channel->hctsiz = hctsiz.value; edpt->next_do_ping = 0; - // pre-calculate next PID based on packet count, adjusted in transfer complete interrupt if short packet + // Single-transaction isochronous endpoints always use DATA0. Pre-calculate the next PID for other endpoints, + // adjusted in the transfer-complete interrupt if a short packet is received. if (hcchar_bm->ep_num == 0) { edpt->next_pid = HCTSIZ_PID_DATA1; // control data and status stage always start with DATA1 - } else { + } else if (hcchar_bm->ep_type != HCCHAR_EPTYPE_ISOCHRONOUS) { edpt->next_pid = cal_next_pid(edpt->next_pid, packet_count); } channel->hcsplt = edpt->hcsplt; channel->hcint = 0xFFFFFFFFU; // clear all channel interrupts + dwc2->gintmsk |= GINTSTS_HCINT; if (dma_host_enabled(dwc2)) { channel->hcintmsk = HCINT_HALTED; @@ -653,13 +757,19 @@ static bool channel_xfer_start(dwc2_regs_t* dwc2, uint8_t ch_id) { channel->hcdma = (uint32_t) edpt->buffer; if (hcchar_bm->ep_dir == TUSB_DIR_IN) { - channel_send_in_token(dwc2, channel); + periodic_frame = channel_send_in_token(dwc2, channel, is_period); } else { hcd_dcache_clean(edpt->buffer, edpt->buflen); - channel->hcchar |= HCCHAR_CHENA; + periodic_frame = channel_enable(dwc2, channel, is_period); + } + } +#if CFG_TUH_DWC2_SLAVE_ENABLE + else { + uint32_t hcintmsk = HCINT_NAK | HCINT_XACT_ERR | HCINT_STALL | + HCINT_XFER_COMPLETE | HCINT_DATATOGGLE_ERR; + if (is_period) { + hcintmsk |= HCINT_FARME_OVERRUN; } - } else { - uint32_t hcintmsk = HCINT_NAK | HCINT_XACT_ERR | HCINT_STALL | HCINT_XFER_COMPLETE | HCINT_DATATOGGLE_ERR; if (hcchar_bm->ep_dir == TUSB_DIR_IN) { hcintmsk |= HCINT_BABBLE_ERR | HCINT_DATATOGGLE_ERR | HCINT_ACK; } else { @@ -677,16 +787,36 @@ static bool channel_xfer_start(dwc2_regs_t* dwc2, uint8_t ch_id) { // IN Token. If we got NAK, we have to re-enable the channel again in the interrupt. Due to the way usbh stack only // call hcd_edpt_xfer() once, we will need to manage de-allocate/re-allocate IN channel dynamically. if (hcchar_bm->ep_dir == TUSB_DIR_IN) { - channel_send_in_token(dwc2, channel); + periodic_frame = channel_send_in_token(dwc2, channel, is_period); } else { - channel->hcchar |= HCCHAR_CHENA; - if (edpt->buflen > 0) { - // To prevent conflict with other channel, we will enable periodic/non-periodic FIFO empty interrupt accordingly - // And write packet in the interrupt handler + // The final FIFO word creates the OUT request. Keep CHENA and that write + // atomic with respect to this controller's ISR. + // This region never waits for FIFO or queue space. + const uint32_t gahbcfg = dwc2->gahbcfg; + dwc2->gahbcfg = gahbcfg & ~GAHBCFG_GINT; + if (defer_periodic_out && is_period) { + const dwc2_hfnum_t hfnum = {.value = dwc2->hfnum}; + if (hfnum.remainning < DWC2_PERIODIC_OUT_MIN_FRREM) { + edpt->next_pid = saved_pid; + edpt->next_do_ping = saved_do_ping; + dwc2->gahbcfg = gahbcfg; + return false; + } + } + periodic_frame = channel_enable(dwc2, channel, is_period); + if (edpt->buflen > 0 && channel_txfifo_write(dwc2, ch_id, is_period)) { + // The FIFO-empty interrupt handles only work that did not fit in the + // initial synchronous write. dwc2->gintmsk |= (is_period ? GINTSTS_PTX_FIFO_EMPTY : GINTSTS_NPTX_FIFO_EMPTY); } + dwc2->gahbcfg = gahbcfg; } } +#endif + + if (is_period && defer_periodic_out) { + edpt->periodic_frame = periodic_frame; + } return true; } @@ -698,8 +828,48 @@ static bool edpt_xfer_kickoff(dwc2_regs_t* dwc2, uint8_t ep_id) { hcd_xfer_t* xfer = &_hcd_data.xfer[ch_id]; xfer->ep_id = ep_id; xfer->result = XFER_RESULT_INVALID; + hcd_endpoint_t* edpt = &_hcd_data.edpt[ep_id]; + const bool result = channel_xfer_start(dwc2, ch_id, true); + if (!result) { + channel_dealloc(dwc2, ch_id); + periodic_xfer_defer(dwc2, edpt, 0); + return true; + } + if (channel_is_periodic(_hcd_data.edpt[ep_id].hcchar)) { + edpt->periodic_phase = 1; + edpt->xfer_pending = 0; + } + return result; +} + +static uint32_t periodic_xfer_countdown(dwc2_regs_t* dwc2, hcd_endpoint_t const* edpt) { + const uint32_t ucount = (hprt_speed_get(dwc2) == TUSB_SPEED_HIGH) ? 1u : 8u; + const uint16_t frame = (uint16_t) (dwc2->hfnum & HCD_FRAME_NUMBER_MASK); + const uint16_t elapsed_frames = (uint16_t) (frame - edpt->periodic_frame) & HCD_FRAME_NUMBER_MASK; + const uint32_t elapsed_uframes = (uint32_t) elapsed_frames * ucount; + + if (elapsed_uframes < edpt->uframe_interval) { + return edpt->uframe_interval - elapsed_uframes - ucount; + } + + // The service opportunity was missed. Keep the established phase and use + // the next interval rather than starting a new interval from this request. + return edpt->uframe_interval - (elapsed_uframes % edpt->uframe_interval) - ucount; +} + +static void periodic_xfer_defer(dwc2_regs_t* dwc2, hcd_endpoint_t* edpt, uint32_t uframe_countdown) { + const uint32_t gahbcfg = dwc2->gahbcfg; + dwc2->gahbcfg = gahbcfg & ~GAHBCFG_GINT; - return channel_xfer_start(dwc2, ch_id); + edpt->uframe_countdown = uframe_countdown; + edpt->xfer_pending = 1; + + if (0 == (dwc2->gintmsk & GINTMSK_SOFM)) { + dwc2->gintsts = GINTSTS_SOF; + dwc2->gintmsk |= GINTMSK_SOFM; + } + + dwc2->gahbcfg = gahbcfg; } bool hcd_edpt_xfer(uint8_t rhport, uint8_t dev_addr, uint8_t ep_addr, uint8_t * buffer, uint16_t buflen) { @@ -707,10 +877,10 @@ bool hcd_edpt_xfer(uint8_t rhport, uint8_t dev_addr, uint8_t ep_addr, uint8_t * const uint8_t ep_num = tu_edpt_number(ep_addr); const uint8_t ep_dir = tu_edpt_dir(ep_addr); - uint8_t ep_id = edpt_find_opened(dev_addr, ep_num, ep_dir); - TU_ASSERT(ep_id < CFG_TUH_DWC2_ENDPOINT_MAX); + uint8_t ep_id = edpt_find_opened(dev_addr, ep_num, ep_dir, false); + TU_VERIFY(ep_id < CFG_TUH_DWC2_ENDPOINT_MAX); hcd_endpoint_t *edpt = &_hcd_data.edpt[ep_id]; - TU_VERIFY(edpt->closing == 0); // skip if endpoint is closing + TU_VERIFY(edpt->closing == 0 && edpt->aborting == 0); // skip if endpoint is closing or aborting edpt->buffer = buffer; edpt->buflen = buflen; @@ -720,6 +890,26 @@ bool hcd_edpt_xfer(uint8_t rhport, uint8_t dev_addr, uint8_t ep_addr, uint8_t * edpt->hcchar_bm.ep_dir = ep_dir; } + if (channel_is_periodic(edpt->hcchar)) { + const uint32_t ucount = (hprt_speed_get(dwc2) == TUSB_SPEED_HIGH) ? 1u : 8u; +#if CFG_TUH_DWC2_SLAVE_ENABLE + // Establish a slower slave-mode OUT schedule from SOF. bInterval=1 must be queued immediately to avoid + // losing every other service opportunity. + if (!dma_host_enabled(dwc2) && ep_dir == TUSB_DIR_OUT && !edpt->periodic_phase && + edpt->uframe_interval > ucount) { + periodic_xfer_defer(dwc2, edpt, 0); + return true; + } +#endif + if (edpt->periodic_phase && edpt->uframe_interval > ucount) { + const uint32_t countdown = periodic_xfer_countdown(dwc2, edpt); + if (countdown > 0) { + periodic_xfer_defer(dwc2, edpt, countdown); + return true; + } + } + } + return edpt_xfer_kickoff(dwc2, ep_id); } @@ -729,11 +919,39 @@ bool hcd_edpt_abort_xfer(uint8_t rhport, uint8_t dev_addr, uint8_t ep_addr) { dwc2_regs_t* dwc2 = DWC2_REG(rhport); const uint8_t ep_num = tu_edpt_number(ep_addr); const uint8_t ep_dir = tu_edpt_dir(ep_addr); - const uint8_t ep_id = edpt_find_opened(dev_addr, ep_num, ep_dir); + const uint8_t ep_id = edpt_find_opened(dev_addr, ep_num, ep_dir, false); TU_VERIFY(ep_id < CFG_TUH_DWC2_ENDPOINT_MAX); + hcd_endpoint_t* edpt = &_hcd_data.edpt[ep_id]; + + hcd_int_disable(rhport); + + const bool xfer_pending = edpt->xfer_pending; + if (xfer_pending) { + edpt->xfer_pending = 0; + edpt->uframe_countdown = 0; + } + + if (xfer_pending) { + hcd_int_enable(rhport); + return true; + } + + // A periodic DMA channel must halt naturally at the next service boundary. Prevent a replacement transfer until the + // halt ISR retires the channel, and suppress completion for the aborted transfer. + if (dma_host_enabled(dwc2) && channel_is_periodic(edpt->hcchar)) { + const uint8_t ch_id = channel_find_enabled(dwc2, dev_addr, ep_num, ep_dir); + if (ch_id < 16) { + hcd_xfer_t* xfer = &_hcd_data.xfer[ch_id]; + edpt->aborting = 1; + xfer->aborting = true; + hcd_int_enable(rhport); + return true; + } + } - // hcd_int_disable(rhport); + hcd_int_enable(rhport); + // Channel disable may wait for request-queue space in slave mode. // Find enabled channeled and disable it, channel will be de-allocated in the interrupt handler const uint8_t ch_id = channel_find_enabled(dwc2, dev_addr, ep_num, ep_dir); if (ch_id < 16) { @@ -741,15 +959,13 @@ bool hcd_edpt_abort_xfer(uint8_t rhport, uint8_t dev_addr, uint8_t ep_addr) { channel_disable(dwc2, channel); } - // hcd_int_enable(rhport); - return true; } // Submit a special transfer to send 8-byte Setup Packet, when complete hcd_event_xfer_complete() must be invoked bool hcd_setup_send(uint8_t rhport, uint8_t dev_addr, const uint8_t setup_packet[8]) { - uint8_t ep_id = edpt_find_opened(dev_addr, 0, TUSB_DIR_OUT); - TU_ASSERT(ep_id < CFG_TUH_DWC2_ENDPOINT_MAX); // no opened endpoint + uint8_t ep_id = edpt_find_opened(dev_addr, 0, TUSB_DIR_OUT, false); + TU_VERIFY(ep_id < CFG_TUH_DWC2_ENDPOINT_MAX); // endpoint can close asynchronously on disconnect hcd_endpoint_t* edpt = &_hcd_data.edpt[ep_id]; edpt->next_pid = HCTSIZ_PID_SETUP; @@ -761,7 +977,7 @@ bool hcd_edpt_clear_stall(uint8_t rhport, uint8_t dev_addr, uint8_t ep_addr) { (void) rhport; const uint8_t ep_num = tu_edpt_number(ep_addr); const uint8_t ep_dir = tu_edpt_dir(ep_addr); - const uint8_t ep_id = edpt_find_opened(dev_addr, ep_num, ep_dir); + const uint8_t ep_id = edpt_find_opened(dev_addr, ep_num, ep_dir, false); TU_VERIFY(ep_id < CFG_TUH_DWC2_ENDPOINT_MAX); hcd_endpoint_t* edpt = &_hcd_data.edpt[ep_id]; @@ -790,7 +1006,7 @@ static void channel_xfer_in_retry(dwc2_regs_t* dwc2, uint8_t ch_id, uint32_t hci if (xfer->period_split_nyet_count < HCD_XFER_PERIOD_SPLIT_NYET_MAX) { hcchar.odd_frame = 1 - (dwc2->hfnum & 1); // transfer on next frame channel->hcchar = hcchar.value; - channel_send_in_token(dwc2, channel); + channel_send_in_token(dwc2, channel, false); return; } else { // too many NYET, de-allocate channel with below code @@ -803,23 +1019,20 @@ static void channel_xfer_in_retry(dwc2_regs_t* dwc2, uint8_t ch_id, uint32_t hci // retry on next frame if bInterval is 1 hcchar.odd_frame = 1 - (dwc2->hfnum & 1); channel->hcchar = hcchar.value; - channel_send_in_token(dwc2, channel); + channel_send_in_token(dwc2, channel, false); } else { // otherwise, de-allocate channel, enable SOF set frame counter for later transfer const dwc2_channel_tsize_t hctsiz = {.value = channel->hctsiz}; - edpt->next_pid = hctsiz.pid; // save PID - edpt->uframe_countdown = edpt->uframe_interval - ucount; - // enable SOF interrupt if not already enabled - if (0 == (dwc2->gintmsk & GINTMSK_SOFM)) { - dwc2->gintsts = GINTSTS_SOF; - dwc2->gintmsk |= GINTMSK_SOFM; + if (hcchar.ep_type != HCCHAR_EPTYPE_ISOCHRONOUS) { + edpt->next_pid = hctsiz.pid; // save PID } + periodic_xfer_defer(dwc2, edpt, periodic_xfer_countdown(dwc2, edpt)); // already halted, de-allocate channel (called from DMA isr) channel_dealloc(dwc2, ch_id); } } else { // for control/bulk: retry immediately - channel_send_in_token(dwc2, channel); + channel_send_in_token(dwc2, channel, false); } } @@ -854,6 +1067,13 @@ static void handle_rxflvl_irq(uint8_t rhport) { // In packet received, pop this entry --> ACK interrupt const uint16_t byte_count = grxstsp.byte_count; hcd_xfer_t* xfer = &_hcd_data.xfer[ch_id]; + if (!xfer->allocated) { + // Discard data for a channel retired by disconnect. + for (uint16_t count = 0; count < byte_count; count += sizeof(uint32_t)) { + (void) dwc2->fifo[0][0]; + } + break; + } TU_ASSERT(xfer->ep_id < CFG_TUH_DWC2_ENDPOINT_MAX,); hcd_endpoint_t* edpt = &_hcd_data.edpt[xfer->ep_id]; @@ -883,38 +1103,50 @@ static void handle_rxflvl_irq(uint8_t rhport) { } } -// return true if there is still pending data and need more ISR +// Return true if data remains for a later FIFO-empty interrupt. +static bool channel_txfifo_write(dwc2_regs_t* dwc2, uint8_t ch_id, bool is_periodic) { + hcd_xfer_t* xfer = &_hcd_data.xfer[ch_id]; + dwc2_channel_t* channel = &dwc2->channel[ch_id]; + const dwc2_channel_char_t hcchar = {.value = channel->hcchar}; + TU_ASSERT(xfer->ep_id < CFG_TUH_DWC2_ENDPOINT_MAX); + hcd_endpoint_t* edpt = &_hcd_data.edpt[xfer->ep_id]; + const dwc2_channel_tsize_t hctsiz = {.value = channel->hctsiz}; + const uint16_t remain_packets = hctsiz.packet_count; + + for (uint16_t i = 0; i < remain_packets; i++) { + const uint16_t remain_bytes = edpt->buflen - xfer->fifo_bytes; + const uint16_t xact_bytes = tu_min16(remain_bytes, hcchar.ep_size); + + // The packet's last FIFO word creates its request-queue entry. + // HNPTXSTS differs by one request-queue bit, which is outside these fields. + const dwc2_hptxsts_t txsts = {.value = (is_periodic ? dwc2->hptxsts : dwc2->hnptxsts)}; + if ((xact_bytes > (txsts.fifo_available << 2)) || (txsts.req_queue_available == 0)) { + return true; + } + + tu_hwfifo_write(dwc2->fifo[ch_id], edpt->buffer + xfer->fifo_bytes, xact_bytes, NULL); + xfer->fifo_bytes += xact_bytes; + } + + return false; +} + +// Return true if at least one matching channel needs another interrupt. static bool handle_txfifo_empty(dwc2_regs_t* dwc2, bool is_periodic) { const uint8_t max_channel = dwc2_channel_count(dwc2); for (uint8_t ch_id = 0; ch_id < max_channel; ch_id++) { + hcd_xfer_t* xfer = &_hcd_data.xfer[ch_id]; dwc2_channel_t* channel = &dwc2->channel[ch_id]; const dwc2_channel_char_t hcchar = {.value = channel->hcchar}; - // skip writing to FIFO if channel is expecting halted. - if (0 == (channel->hcintmsk & HCINT_HALTED) && (hcchar.ep_dir == TUSB_DIR_OUT)) { - hcd_xfer_t *xfer = &_hcd_data.xfer[ch_id]; - TU_ASSERT(xfer->ep_id < CFG_TUH_DWC2_ENDPOINT_MAX); - hcd_endpoint_t* edpt = &_hcd_data.edpt[xfer->ep_id]; - const dwc2_channel_tsize_t hctsiz = {.value = channel->hctsiz}; - const uint16_t remain_packets = hctsiz.packet_count; - for (uint16_t i = 0; i < remain_packets; i++) { - const uint16_t remain_bytes = edpt->buflen - xfer->fifo_bytes; - const uint16_t xact_bytes = tu_min16(remain_bytes, hcchar.ep_size); - - // skip if there is not enough space in FIFO and RequestQueue. - // Packet's last word written to FIFO will trigger a request queue - // Use period txsts for both p/np to get request queue space available (1-bit difference, it is small enough) - const dwc2_hptxsts_t txsts = {.value = (is_periodic ? dwc2->hptxsts : dwc2->hnptxsts)}; - if ((xact_bytes > (txsts.fifo_available << 2)) || (txsts.req_queue_available == 0)) { - return true; - } - - tu_hwfifo_write(dwc2->fifo[ch_id], edpt->buffer + xfer->fifo_bytes, xact_bytes, NULL); - xfer->fifo_bytes += xact_bytes; + if (xfer->allocated && channel_is_periodic(hcchar.value) == is_periodic && + 0 == (channel->hcintmsk & HCINT_HALTED) && hcchar.ep_dir == TUSB_DIR_OUT) { + if (channel_txfifo_write(dwc2, ch_id, is_periodic)) { + return true; } } } - return false; // no channel has pending data + return false; } static bool handle_channel_in_slave(dwc2_regs_t* dwc2, uint8_t ch_id, uint32_t hcint) { @@ -932,7 +1164,8 @@ static bool handle_channel_in_slave(dwc2_regs_t* dwc2, uint8_t ch_id, uint32_t h // } if (hcint & HCINT_XFER_COMPLETE) { - if (edpt->hcchar_bm.ep_num != 0) { + if (edpt->hcchar_bm.ep_num != 0 && + edpt->hcchar_bm.ep_type != HCCHAR_EPTYPE_ISOCHRONOUS) { edpt->next_pid = hctsiz.pid; // save pid (already toggled) } @@ -945,6 +1178,17 @@ static bool handle_channel_in_slave(dwc2_regs_t* dwc2, uint8_t ch_id, uint32_t h xfer->result = XFER_RESULT_SUCCESS; } + if (channel_is_periodic(channel->hcchar) && remain_packets == 0) { + // The core has already halted a completed periodic IN channel. Complete + // it now so the next interval can be submitted without another halt IRQ. + is_done = true; + } else { + channel_disable(dwc2, channel); + } + } else if (hcint & HCINT_FARME_OVERRUN) { + if (edpt->hcchar_bm.ep_type == HCCHAR_EPTYPE_ISOCHRONOUS) { + xfer->result = XFER_RESULT_FAILED; + } channel_disable(dwc2, channel); } else if (hcint & (HCINT_XACT_ERR | HCINT_BABBLE_ERR | HCINT_STALL)) { if (hcint & HCINT_STALL) { @@ -982,7 +1226,7 @@ static bool handle_channel_in_slave(dwc2_regs_t* dwc2, uint8_t ch_id, uint32_t h channel->hcintmsk |= HCINT_NYET; hcsplt.split_compl = 1; channel->hcsplt = hcsplt.value; - channel_send_in_token(dwc2, channel); + channel_send_in_token(dwc2, channel, false); } else { // do nothing for complete split with DATA, this will trigger XferComplete and handled there } @@ -993,7 +1237,7 @@ static bool handle_channel_in_slave(dwc2_regs_t* dwc2, uint8_t ch_id, uint32_t h // still more packet to receive, also reset to start split hcsplt.split_compl = 0; channel->hcsplt = hcsplt.value; - channel_send_in_token(dwc2, channel); + channel_send_in_token(dwc2, channel, false); } } } else if (hcint & HCINT_HALTED) { @@ -1039,6 +1283,12 @@ static bool handle_channel_out_slave(dwc2_regs_t* dwc2, uint8_t ch_id, uint32_t } else if (hcint & HCINT_STALL) { xfer->result = XFER_RESULT_STALLED; channel_disable(dwc2, channel); + } else if (hcint & HCINT_FARME_OVERRUN) { + channel_xfer_out_wrapup(dwc2, ch_id); + if (edpt->hcchar_bm.ep_type == HCCHAR_EPTYPE_ISOCHRONOUS) { + xfer->result = XFER_RESULT_FAILED; + } + channel_disable(dwc2, channel); } else if (hcint & HCINT_NYET) { xfer->err_count = 0; if (hcsplt.split_en == 1u) { @@ -1074,7 +1324,7 @@ static bool handle_channel_out_slave(dwc2_regs_t* dwc2, uint8_t ch_id, uint32_t is_done = true; } else { // Got here due to NAK or NYET - TU_ASSERT(channel_xfer_start(dwc2, ch_id)); + TU_ASSERT(channel_xfer_start(dwc2, ch_id, false)); } } else if (hcint & HCINT_ACK) { xfer->err_count = 0; @@ -1126,9 +1376,13 @@ static bool handle_channel_in_dma(dwc2_regs_t* dwc2, uint8_t ch_id, uint32_t hci if (xfer->closing) { is_done = true; } else { - channel_send_in_token(dwc2, channel); + channel_send_in_token(dwc2, channel, false); } } else if (hcint & (HCINT_XFER_COMPLETE | HCINT_STALL | HCINT_BABBLE_ERR)) { + if (edpt->hcchar_bm.ep_num != 0 && (hcint & HCINT_XFER_COMPLETE)) { + edpt->next_pid = hctsiz.pid; // save pid (already toggled) + } + const uint16_t remain_bytes = (uint16_t) hctsiz.xfer_size; const uint16_t remain_packets = hctsiz.packet_count; const uint16_t actual_len = edpt->buflen - remain_bytes; @@ -1187,7 +1441,7 @@ static bool handle_channel_in_dma(dwc2_regs_t* dwc2, uint8_t ch_id, uint32_t hci hcchar.odd_frame = 1 - (dwc2->hfnum & 1); // transfer on next frame channel->hcchar = hcchar.value; } - channel_send_in_token(dwc2, channel); + channel_send_in_token(dwc2, channel, false); } } else if (hcint & (HCINT_NAK | HCINT_DATATOGGLE_ERR)) { xfer->err_count = 0; @@ -1204,8 +1458,12 @@ static bool handle_channel_in_dma(dwc2_regs_t* dwc2, uint8_t ch_id, uint32_t hci channel_xfer_in_retry(dwc2, ch_id, hcint); } } else if (hcint & HCINT_FARME_OVERRUN) { - // retry start-split in next binterval - channel_xfer_in_retry(dwc2, ch_id, hcint); + if (hcchar.ep_type == HCCHAR_EPTYPE_ISOCHRONOUS) { + xfer->result = XFER_RESULT_FAILED; + is_done = true; + } else { + channel_xfer_in_retry(dwc2, ch_id, hcint); + } } if (xfer->closing == 1) { @@ -1234,7 +1492,7 @@ static bool handle_channel_out_dma(dwc2_regs_t* dwc2, uint8_t ch_id, uint32_t hc if (xfer->closing) { is_done = true; } else { - channel_xfer_start(dwc2, ch_id); + channel_xfer_start(dwc2, ch_id, false); } } else if (hcint & (HCINT_XFER_COMPLETE | HCINT_STALL)) { is_done = true; @@ -1248,30 +1506,38 @@ static bool handle_channel_out_dma(dwc2_regs_t* dwc2, uint8_t ch_id, uint32_t hc } channel->hcintmsk &= ~HCINT_ACK; } else if (hcint & HCINT_XACT_ERR) { - if (hcint & (HCINT_NAK | HCINT_NYET | HCINT_ACK)) { - xfer->err_count = 0; - // clean up transfer so far and start again - channel_xfer_out_wrapup(dwc2, ch_id); - channel_xfer_start(dwc2, ch_id); - } else { - xfer->err_count++; - if (xfer->err_count >= HCD_XFER_ERROR_MAX) { - xfer->result = XFER_RESULT_FAILED; - is_done = true; - } else { - // Rewind, then retry the start-split. Non-periodic SPLIT throttles via channel_disable + re-arm on - // the halt (immediate re-fire exhausts the retry budget; the disable gives the hub TT a recovery - // gap, like slave). Periodic split is excluded: channel_disable() is a no-op for it, so the halt - // never fires and the channel would wedge. Non-split re-inits immediately (Programming Guide 5.1.2.3). - channel_xfer_out_wrapup(dwc2, ch_id); - if (hcsplt.split_en && !channel_is_periodic(channel->hcchar)) { - xfer->retry_disabled = 1; - channel_disable(dwc2, channel); - } else { - channel_xfer_start(dwc2, ch_id); - } - } - } + if (hcint & (HCINT_NAK | HCINT_NYET | HCINT_ACK)) { + xfer->err_count = 0; + // clean up transfer so far and start again + channel_xfer_out_wrapup(dwc2, ch_id); + channel_xfer_start(dwc2, ch_id, false); + } else { + xfer->err_count++; + if (xfer->err_count >= HCD_XFER_ERROR_MAX) { + xfer->result = XFER_RESULT_FAILED; + is_done = true; + } else { + // Rewind, then retry the start-split. Non-periodic SPLIT throttles via channel_disable + re-arm on + // the halt (immediate re-fire exhausts the retry budget; the disable gives the hub TT a recovery + // gap, like slave). Periodic split is excluded: channel_disable() is a no-op for it, so the halt + // never fires and the channel would wedge. Non-split re-inits immediately (Programming Guide 5.1.2.3). + channel_xfer_out_wrapup(dwc2, ch_id); + if (hcsplt.split_en && !channel_is_periodic(channel->hcchar)) { + xfer->retry_disabled = 1; + channel_disable(dwc2, channel); + } else { + channel_xfer_start(dwc2, ch_id, false); + } + } + } + } else if (hcint & HCINT_FARME_OVERRUN) { + channel_xfer_out_wrapup(dwc2, ch_id); + if (edpt->hcchar_bm.ep_type == HCCHAR_EPTYPE_ISOCHRONOUS) { + xfer->result = XFER_RESULT_FAILED; + is_done = true; + } else { + channel_xfer_start(dwc2, ch_id, false); + } } else if (hcint & HCINT_NYET) { if (hcsplt.split_en && hcsplt.split_compl) { // split not yet mean hub has no data, retry complete split @@ -1292,7 +1558,7 @@ static bool handle_channel_out_dma(dwc2_regs_t* dwc2, uint8_t ch_id, uint32_t hc // Non-split OUT NAK is core-handled (5.1.2.2), so this is split-only. xfer->err_count = 0; channel_xfer_out_wrapup(dwc2, ch_id); - channel_xfer_start(dwc2, ch_id); + channel_xfer_start(dwc2, ch_id, false); } if (xfer->closing == 1) { @@ -1320,7 +1586,29 @@ static void handle_channel_irq(uint8_t rhport, bool in_isr) { dwc2_channel_char_t hcchar = {.value = channel->hcchar}; const uint32_t hcint = channel->hcint; - channel->hcint = hcint; // clear interrupt + // Slave handlers process one cause per pass. If ChHltd arrived with + // another cause, leave it pending so the next pass retires the halt. + const uint32_t hcint_clear = (!is_dma && (hcint & ~HCINT_HALTED)) ? (hcint & ~HCINT_HALTED) : hcint; + channel->hcint = hcint_clear; + + if (is_dma && xfer->aborting && (hcint & HCINT_HALTED)) { + hcd_endpoint_t* edpt = &_hcd_data.edpt[xfer->ep_id]; + const bool closing = xfer->closing; + // channel_xfer_start() predicts the PID after all requested packets; + // an aborted transfer may have completed fewer. + if (hcchar.ep_type != HCCHAR_EPTYPE_ISOCHRONOUS) { + const dwc2_channel_tsize_t hctsiz = {.value = channel->hctsiz}; + edpt->next_pid = hctsiz.pid; + } + xfer->aborting = false; + channel_dealloc(dwc2, ch_id); + if (closing) { + edpt_dealloc(edpt); + } else { + edpt->aborting = 0; + } + continue; + } bool is_done = false; if (is_dma) { @@ -1373,15 +1661,17 @@ static bool handle_sof_irq(uint8_t rhport, bool in_isr) { for(uint8_t ep_id = 0; ep_id < CFG_TUH_DWC2_ENDPOINT_MAX; ep_id++) { hcd_endpoint_t *edpt = &_hcd_data.edpt[ep_id]; if (edpt->closing == 0) { - if (edpt->hcchar_bm.enable && channel_is_periodic(edpt->hcchar) && edpt->uframe_countdown > 0) { - edpt->uframe_countdown -= tu_min32(ucount, edpt->uframe_countdown); + if (edpt->hcchar_bm.enable && channel_is_periodic(edpt->hcchar) && edpt->xfer_pending) { + if (edpt->uframe_countdown > 0) { + edpt->uframe_countdown -= tu_min32(ucount, edpt->uframe_countdown); + } if (edpt->uframe_countdown == 0) { if (!edpt_xfer_kickoff(dwc2, ep_id)) { edpt->uframe_countdown = ucount; // failed to start, try again next frame } } - more_isr = true; + more_isr = more_isr || edpt->xfer_pending; } } } @@ -1501,25 +1791,23 @@ void hcd_int_handler(uint8_t rhport, bool in_isr) { } } - if (gintsts & GINTSTS_HPRTINT) { - // Host port interrupt: source is cleared in HPRT register - // TU_LOG1_HEX(dwc2->hprt); - handle_hprt_irq(rhport, in_isr); - } - - if (gintsts & GINTSTS_HCINT) { - // Host Channel interrupt: source is cleared in HCINT register - // must be handled after TX FIFO empty - handle_channel_irq(rhport, in_isr); - } - if (gintsts & GINTSTS_DISCINT) { - // Device disconnected dwc2->gintsts = GINTSTS_DISCINT; + channel_cleanup_on_disconnect(dwc2); + hcd_event_device_remove(rhport, in_isr); - if (0 == (dwc2->hprt & HPRT_CONN_STATUS)) { - hcd_event_device_remove(rhport, in_isr); + // A fast replug can be visible without a pending connect-detect interrupt. + const uint32_t hprt = dwc2->hprt; + if (!(hprt & HPRT_CONN_DETECT) && (hprt & HPRT_CONN_STATUS)) { + hcd_event_device_attach(rhport, in_isr); } + return; + } + + if (gintsts & GINTSTS_HPRTINT) { + // Host port interrupt: source is cleared in HPRT register + // TU_LOG1_HEX(dwc2->hprt); + handle_hprt_irq(rhport, in_isr); } #if CFG_TUH_DWC2_SLAVE_ENABLE @@ -1553,6 +1841,13 @@ void hcd_int_handler(uint8_t rhport, bool in_isr) { } } #endif + + // Draining the RxFIFO completion status can assert HCINT.XferCompl. Read + // the live status here so the completion is handled in this ISR invocation. + if ((dwc2->gintsts & dwc2->gintmsk) & GINTSTS_HCINT) { + handle_channel_irq(rhport, in_isr); + } + } #endif |