path: root/src/portable/raspberrypi/rp2040/dcd_rp2040.c

/*
 * SPDX-FileCopyrightText: Copyright (c) 2020 Raspberry Pi (Trading) Ltd.
 * SPDX-FileCopyrightText: Copyright (c) 2020 Ha Thach (tinyusb.org)
 * SPDX-License-Identifier: MIT
 *
 * This file is part of the TinyUSB stack.
 */

#include "tusb_option.h"

#if CFG_TUD_ENABLED && (CFG_TUSB_MCU == OPT_MCU_RP2040) && !CFG_TUD_RPI_PIO_USB

#include "pico.h"
#include "hardware/sync.h"
#include "rp2040_usb.h"

#if TUD_OPT_RP2040_USB_DEVICE_ENUMERATION_FIX
#include "pico/fix/rp2040_usb_device_enumeration.h"
#endif

#include "device/dcd.h"

// Current implementation force vbus detection as always present, causing device think it is always plugged into host.
// Therefore, it cannot detect disconnect event, mistaken it as suspend.
// Note: won't work if change to 0 (for now)
  #define FORCE_VBUS_DETECT 1

  #define USB_INTS_ERROR_BITS                                                                 \
    (USB_INTS_ERROR_DATA_SEQ_BITS | USB_INTS_ERROR_BIT_STUFF_BITS | USB_INTS_ERROR_CRC_BITS | \
     USB_INTS_ERROR_RX_OVERFLOW_BITS | USB_INTS_ERROR_RX_TIMEOUT_BITS)

/*------------------------------------------------------------------*/
/* Low level controller
 *------------------------------------------------------------------*/
// HW buffer pointer from USB buffer space (max 3840 bytes)
static uint8_t *hw_buffer_ptr;

// USB_MAX_ENDPOINTS Endpoints, direction TUSB_DIR_OUT for out and TUSB_DIR_IN for in.
static struct hw_endpoint hw_endpoints[USB_MAX_ENDPOINTS][2];

// SOF may be used by remote wakeup as RESUME, this indicates whether SOF is actually used by usbd
static bool _sof_enable = false;

TU_ATTR_ALWAYS_INLINE static inline hw_endpoint_t *hw_endpoint_get(uint8_t epnum, tusb_dir_t dir) {
  return &hw_endpoints[epnum][dir];
}

TU_ATTR_ALWAYS_INLINE static inline hw_endpoint_t *hw_endpoint_get_by_addr(uint8_t ep_addr) {
  const uint8_t    num = tu_edpt_number(ep_addr);
  const tusb_dir_t dir = tu_edpt_dir(ep_addr);
  return hw_endpoint_get(num, dir);
}

TU_ATTR_ALWAYS_INLINE static inline io_rw_32 *get_ep_ctrl(uint8_t epnum, tusb_dir_t dir) {
  if (epnum == 0) {
    // EP0 has no endpoint control register because the buffer offsets are fixed and always enabled
    return NULL;
  }
  struct usb_device_dpram_ep_ctrl *ep_ctrl = &usb_dpram->ep_ctrl[epnum - 1];
  return (dir == TUSB_DIR_IN) ? &ep_ctrl->in : &ep_ctrl->out;
}

TU_ATTR_ALWAYS_INLINE static inline io_rw_32 *get_buf_ctrl(uint8_t epnum, tusb_dir_t dir) {
  struct usb_device_dpram_ep_buf_ctrl *buf_ctrl = &usb_dpram->ep_buf_ctrl[epnum];
  return (dir == TUSB_DIR_IN) ? &buf_ctrl->in : &buf_ctrl->out;
}

// Init and enable endpoint
static void hw_endpoint_open(uint8_t ep_addr, uint16_t wMaxPacketSize, uint8_t transfer_type, bool ep_enabled) {
  const uint8_t    epnum = tu_edpt_number(ep_addr);
  const tusb_dir_t dir   = tu_edpt_dir(ep_addr);

  hw_endpoint_t *ep   = hw_endpoint_get(epnum, dir);
  ep->ep_addr         = ep_addr;
  ep->next_pid        = 0u;
  ep->max_packet_size = wMaxPacketSize;

  // Clear existing buffer control state
  io_rw_32 *buf_reg = get_buf_ctrl(epnum, dir);
  *buf_reg          = 0;

  // allocated hw buffer
  if (epnum == 0) {
    // Buffer offset is fixed (2 buffer allocated).
    // Note: Only single buffer for EP since Double buffered RX can be troublesome with future data.
    ep->dpram_buf = (uint8_t *)&usb_dpram->ep0_buf_a[0];
  } else {
    uint32_t ep_ctrl = EP_CTRL_INTERRUPT_PER_BUFFER | ((uint32_t)transfer_type << EP_CTRL_BUFFER_TYPE_LSB);

    // round up size to multiple of 64
    uint16_t size = (uint16_t)tu_round_up(wMaxPacketSize, 64);

    // double buffered Bulk endpoint
    if (transfer_type == TUSB_XFER_BULK) {
      size *= 2u;
  #if CFG_TUSB_RP2_ERRATA_E15
      if (dir == TUSB_DIR_IN) {
        ep->e15_bulk_in = true;
      }
  #endif
    }

    // assign buffer
    ep->dpram_buf = hw_buffer_ptr;
    hw_buffer_ptr += size;

    ep_ctrl |= hw_data_offset(ep->dpram_buf);
    if (ep_enabled) {
      ep_ctrl |= EP_CTRL_ENABLE_BITS;
    }

    *get_ep_ctrl(epnum, dir) = ep_ctrl;

    hard_assert(hw_buffer_ptr < usb_dpram->epx_data + sizeof(usb_dpram->epx_data));
    pico_info("  Allocated %d bytes (0x%p)\r\n", size, ep->dpram_buf);
  }
}

static void hw_endpoint_abort_xfer(struct hw_endpoint* ep) {
  // Abort any pending transfer
  const uint8_t  dir        = (uint8_t)tu_edpt_dir(ep->ep_addr);
  const uint8_t  epnum      = tu_edpt_number(ep->ep_addr);
  const uint32_t abort_mask = TU_BIT((epnum << 1) | (dir ? 0 : 1));

  // Due to Errata RP2040-E2: ABORT flag is only applicable for B2 and later (unusable for B0, B1).
  // Which means we are not guaranteed to safely abort pending transfer on B0 and B1.
  if (rp2040_chip_version() >= 2) {
    usb_hw_set->abort = abort_mask;
    while ((usb_hw->abort_done & abort_mask) != abort_mask) {}
  }

  io_rw_32 *buf_reg = get_buf_ctrl(epnum, dir);
  *buf_reg          = 0; // clear buffer control
  rp2usb_reset_transfer(ep);

  if (rp2040_chip_version() >= 2) {
    usb_hw_clear->abort_done = abort_mask;
    usb_hw_clear->abort = abort_mask;
  }
}

static void __tusb_irq_path_func(handle_hw_buff_status)(void) {
  uint32_t buf_status = usb_hw->buf_status;
  pico_trace("buf_status = 0x%08lx\r\n", buf_status);
  while (buf_status) {
    // ctz/clz is faster than loop which has only a few bit set in general
    const uint8_t i = (uint8_t) __builtin_ctz(buf_status);
    const uint32_t bit = TU_BIT(i);

    // IN transfer for even i, OUT transfer for odd i
    const uint8_t    epnum   = i >> 1u;
    const tusb_dir_t dir     = (i & 1u) ? TUSB_DIR_OUT : TUSB_DIR_IN;
    hw_endpoint_t   *ep      = hw_endpoint_get(epnum, dir);
    io_rw_32        *ep_reg  = get_ep_ctrl(epnum, dir);
    io_rw_32        *buf_reg = get_buf_ctrl(epnum, dir);

    // Double-buffered: if both buffers completed at once, buf_status re-sets
    // immediately after clearing (datasheet Table 406). Process the second buffer too.
    while (usb_hw->buf_status & bit) {
      const uint8_t buf_id     = (usb_hw->buf_cpu_should_handle & bit) ? 1 : 0; // before clear buf_status
      usb_hw_clear->buf_status = bit;
      buf_status &= ~bit;

      if (rp2usb_xfer_continue(ep, ep_reg, buf_reg, buf_id, dir == TUSB_DIR_OUT)) {
        const uint16_t xferred_len = ep->xferred_len;
        rp2usb_reset_transfer(ep);
        dcd_event_xfer_complete(0, ep->ep_addr, xferred_len, XFER_RESULT_SUCCESS, true);
      }
    }
  }
}

TU_ATTR_ALWAYS_INLINE static inline void reset_ep0(void) {
  // If we have finished this transfer on EP0 set pid back to 1 for next
  // setup transfer. Also clear a stall in case
  for (uint8_t dir = 0; dir < 2; dir++) {
    struct hw_endpoint *ep = hw_endpoint_get(0, dir);
    ep->next_pid = 1u;
    if (ep->state == EPSTATE_ACTIVE) {
      hw_endpoint_abort_xfer(ep); // Abort any pending transfer per USB specs
    }
  }
}

static void __tusb_irq_path_func(reset_non_control_endpoints)(void) {
  // Disable all non-control
  for (uint8_t i = 0; i < USB_MAX_ENDPOINTS - 1; i++) {
    usb_dpram->ep_ctrl[i].in = 0;
    usb_dpram->ep_ctrl[i].out = 0;
  }

  // clear non-control hw endpoints
  tu_memclr(hw_endpoints[1], sizeof(hw_endpoints) - 2 * sizeof(hw_endpoint_t));

  // reclaim buffer space
  hw_buffer_ptr = &usb_dpram->epx_data[0];
}

static void __tusb_irq_path_func(dcd_rp2040_irq)(void) {
  const uint32_t status  = usb_hw->ints;

  if (status & USB_INTF_DEV_SOF_BITS) {
    uint32_t sof_count = usb_hw->sof_rd & USB_SOF_RD_BITS; // clear interrupt by reading SOF_RD

  #if CFG_TUSB_RP2_ERRATA_E15
    e15_last_sof = time_us_32();                           // timing critical
  #endif

    dcd_event_sof(0, sof_count, true);
  }

  // xfer events are handled before setup req. So if a transfer completes immediately
  // before closing the EP, the events will be delivered in same order.
  if (status & USB_INTS_BUFF_STATUS_BITS) {
    handle_hw_buff_status();
  }

  if (status & USB_INTS_SETUP_REQ_BITS) {
    const uint8_t *setup = remove_volatile_cast(const uint8_t *, &usb_dpram->setup_packet);

    // reset pid to both 1 (data and ack)
    reset_ep0();

    // Pass setup packet to tiny usb
    dcd_event_setup_received(0, setup, true);
    usb_hw_clear->sie_status = USB_SIE_STATUS_SETUP_REC_BITS;
  }

  // Errata 15 workaround for Device Bulk-In endpoint, must be after BUF_STATUS interrupt to sync buf control first
  if (status & USB_INTF_DEV_SOF_BITS) {
    bool keep_sof_alive = false;

  #if CFG_TUSB_RP2_ERRATA_E15
    for (uint8_t i = 0; i < USB_MAX_ENDPOINTS; i++) {
      struct hw_endpoint *ep = hw_endpoint_get(i, TUSB_DIR_IN);

      // Active Bulk IN endpoint requires SOF
      if (ep->e15_bulk_in && ep->state >= EPSTATE_ACTIVE) {
        keep_sof_alive = true;
        hw_endpoint_lock_update(ep, 1);

        if (ep->state == EPSTATE_PENDING) {
          ep->state = EPSTATE_ACTIVE;

          io_rw_32 *buf_reg32 = get_buf_ctrl(i, TUSB_DIR_IN);
          io_rw_16 *buf_reg16 = (io_rw_16 *)buf_reg32;

          // Check each buffer half: idle when both FULL and AVAIL are clear.
          // Use 16-bit writes to avoid clobbering the other half (DPSRAM concurrent access).
          enum {
            BUSY_MASK = USB_BUF_CTRL_FULL | USB_BUF_CTRL_AVAIL
          };

          const bool buf0_idle = !(buf_reg16[0] & BUSY_MASK);
          const bool buf1_idle = (ep->remaining_len > 0) && !(buf_reg16[1] & BUSY_MASK);

          if (buf0_idle && buf1_idle) {
            // both are idle, start fresh
            io_rw_32 *ep_reg = get_ep_ctrl(i, TUSB_DIR_IN);
            rp2usb_buffer_start(ep, ep_reg, buf_reg32, false);
          } else if (buf0_idle) {
            uint16_t buf0 = bufctrl_prepare16(ep, ep->dpram_buf, false);
            bufctrl_write16(buf_reg16, buf0);
          } else if (buf1_idle) {
            uint16_t buf1 = bufctrl_prepare16(ep, ep->dpram_buf + 64, false);
            bufctrl_write16(buf_reg16 + 1, buf1);
          }
        }

        hw_endpoint_lock_update(ep, -1);
      }
    }
  #endif

    // disable SOF interrupt if it is used for RESUME in remote wakeup
    if (!keep_sof_alive && !_sof_enable) {
      usb_hw_clear->inte = USB_INTS_DEV_SOF_BITS;
    }
  }

  #if FORCE_VBUS_DETECT == 0
  // Since we force VBUS detect On, device will always think it is connected and
  // couldn't distinguish between disconnect and suspend
  if (status & USB_INTS_DEV_CONN_DIS_BITS) {
    if (usb_hw->sie_status & USB_SIE_STATUS_CONNECTED_BITS) {
      // Connected: nothing to do
    } else {
      // Disconnected
      dcd_event_bus_signal(0, DCD_EVENT_UNPLUGGED, true);
    }
    usb_hw_clear->sie_status = USB_SIE_STATUS_CONNECTED_BITS;
  }
#endif

  // SE0 for 2.5 us or more (will last at least 10ms)
  if (status & USB_INTS_BUS_RESET_BITS) {
    pico_trace("BUS RESET\r\n");
    usb_hw->dev_addr_ctrl = 0;
    reset_non_control_endpoints();
    dcd_event_bus_reset(0, TUSB_SPEED_FULL, true);
    usb_hw_clear->sie_status = USB_SIE_STATUS_BUS_RESET_BITS;

  #if TUD_OPT_RP2040_USB_DEVICE_ENUMERATION_FIX
    // Only run enumeration workaround if pull up is enabled
    if (usb_hw->sie_ctrl & USB_SIE_CTRL_PULLUP_EN_BITS) {
      rp2040_usb_device_enumeration_fix();
    }
  #endif
  }

  /* Note from pico datasheet 4.1.2.6.4 (v1.2)
   * If you enable the suspend interrupt, it is likely you will see a suspend interrupt when
   * the device is first connected but the bus is idle. The bus can be idle for a few ms before
   * the host begins sending start of frame packets. You will also see a suspend interrupt
   * when the device is disconnected if you do not have a VBUS detect circuit connected. This is
   * because without VBUS detection, it is impossible to tell the difference between
   * being disconnected and suspended.
   */
  if (status & USB_INTS_DEV_SUSPEND_BITS) {
    dcd_event_bus_signal(0, DCD_EVENT_SUSPEND, true);
    usb_hw_clear->sie_status = USB_SIE_STATUS_SUSPENDED_BITS;
  }

  if (status & USB_INTS_DEV_RESUME_FROM_HOST_BITS) {
    dcd_event_bus_signal(0, DCD_EVENT_RESUME, true);
    usb_hw_clear->sie_status = USB_SIE_STATUS_RESUME_BITS;
  }

}

/*------------------------------------------------------------------*/
/* Controller API
 *------------------------------------------------------------------*/

// older SDK
#ifndef PICO_SHARED_IRQ_HANDLER_HIGHEST_ORDER_PRIORITY
#define PICO_SHARED_IRQ_HANDLER_HIGHEST_ORDER_PRIORITY 0xff
#endif

bool dcd_init(uint8_t rhport, const tusb_rhport_init_t* rh_init) {
  (void) rh_init;
  assert(rhport == 0);

  // TU_LOG(1, "Chip Version B%u\r\n", rp2040_chip_version());

  // Reset hardware to default state
  rp2usb_init();

  #if FORCE_VBUS_DETECT
  // Force VBUS detect so the device thinks it is plugged into a host
  usb_hw->pwr = USB_USB_PWR_VBUS_DETECT_BITS | USB_USB_PWR_VBUS_DETECT_OVERRIDE_EN_BITS;
#endif

  irq_add_shared_handler(USBCTRL_IRQ, dcd_rp2040_irq, PICO_SHARED_IRQ_HANDLER_HIGHEST_ORDER_PRIORITY);

  // Init control endpoints
  tu_memclr(hw_endpoints[0], 2 * sizeof(hw_endpoint_t));
  hw_endpoint_open(0x0, 64, TUSB_XFER_CONTROL, false);
  hw_endpoint_open(0x80, 64, TUSB_XFER_CONTROL, false);

  // Init non-control endpoints
  reset_non_control_endpoints();

  // Initializes the USB peripheral for device mode and enables it.
  // Don't need to enable the pull up here. Force VBUS
  usb_hw->main_ctrl = USB_MAIN_CTRL_CONTROLLER_EN_BITS;

  // Enable individual controller IRQS here. Processor interrupt enable will be used
  // for the global interrupt enable...
  // Note: Force VBUS detect cause disconnection not detectable
  usb_hw->sie_ctrl = USB_SIE_CTRL_EP0_INT_1BUF_BITS;
  usb_hw->inte = USB_INTS_BUFF_STATUS_BITS | USB_INTS_BUS_RESET_BITS | USB_INTS_SETUP_REQ_BITS |
                 USB_INTS_DEV_SUSPEND_BITS | USB_INTS_DEV_RESUME_FROM_HOST_BITS |
                 (FORCE_VBUS_DETECT ? 0 : USB_INTS_DEV_CONN_DIS_BITS);

  dcd_connect(rhport);
  return true;
}

bool dcd_deinit(uint8_t rhport) {
  (void) rhport;

  reset_non_control_endpoints();
  irq_remove_handler(USBCTRL_IRQ, dcd_rp2040_irq);

  // reset usb hardware into initial state
  reset_block(RESETS_RESET_USBCTRL_BITS);
  unreset_block_wait(RESETS_RESET_USBCTRL_BITS);

  return true;
}

void dcd_int_enable(__unused uint8_t rhport) {
  assert(rhport == 0);
  irq_set_enabled(USBCTRL_IRQ, true);
}

void dcd_int_disable(__unused uint8_t rhport) {
  assert(rhport == 0);
  irq_set_enabled(USBCTRL_IRQ, false);
}

void dcd_set_address(uint8_t rhport, uint8_t dev_addr) {
  (void)dev_addr;
  // Can't set device address in hardware until status xfer has complete
  // Send 0len complete response on EP0 IN
  dcd_edpt_xfer(rhport, 0x80, NULL, 0, false);
}

void dcd_remote_wakeup(__unused uint8_t rhport) {
  pico_info("dcd_remote_wakeup %d\n", rhport);
  assert(rhport == 0);

  // since RESUME interrupt is not triggered if we are the one initiate
  // briefly enable SOF to notify usbd when bus is ready
  usb_hw_set->inte = USB_INTS_DEV_SOF_BITS;
  usb_hw_set->sie_ctrl = USB_SIE_CTRL_RESUME_BITS;
}

// disconnect by disabling internal pull-up resistor on D+/D-
void dcd_disconnect(__unused uint8_t rhport) {
  (void) rhport;
  usb_hw_clear->sie_ctrl = USB_SIE_CTRL_PULLUP_EN_BITS;
}

// connect by enabling internal pull-up resistor on D+/D-
void dcd_connect(__unused uint8_t rhport) {
  (void) rhport;
  usb_hw_set->sie_ctrl = USB_SIE_CTRL_PULLUP_EN_BITS;
}

void dcd_sof_enable(uint8_t rhport, bool en) {
  (void) rhport;

  _sof_enable = en;

  if (en) {
    usb_hw_set->inte = USB_INTS_DEV_SOF_BITS;
  }
  #if !CFG_TUSB_RP2_ERRATA_E15
  else {
    // Don't clear immediately if the SOF workaround is in use.
    // The SOF handler will conditionally disable the interrupt.
    usb_hw_clear->inte = USB_INTS_DEV_SOF_BITS;
  }
#endif
}

/*------------------------------------------------------------------*/
/* DCD Endpoint port
 *------------------------------------------------------------------*/
void dcd_edpt0_status_complete(uint8_t rhport, tusb_control_request_t const* request) {
  (void) rhport;

  if (request->bmRequestType_bit.recipient == TUSB_REQ_RCPT_DEVICE &&
      request->bmRequestType_bit.type == TUSB_REQ_TYPE_STANDARD &&
      request->bRequest == TUSB_REQ_SET_ADDRESS) {
    usb_hw->dev_addr_ctrl = (uint8_t) request->wValue;
  }
}

bool dcd_edpt_open(uint8_t rhport, tusb_desc_endpoint_t const* desc_edpt) {
  (void) rhport;
  const uint8_t xfer_type = desc_edpt->bmAttributes.xfer;
  hw_endpoint_open(desc_edpt->bEndpointAddress, tu_edpt_packet_size(desc_edpt), xfer_type, true);
  return true;
}

// New API: Allocate packet buffer used by ISO endpoints
// Some MCU need manual packet buffer allocation, we allocate the largest size to avoid clustering
bool dcd_edpt_iso_alloc(uint8_t rhport, uint8_t ep_addr, uint16_t largest_packet_size) {
  (void)rhport;
  hw_endpoint_open(ep_addr, largest_packet_size, TUSB_XFER_ISOCHRONOUS, false);
  return true;
}

// New API: Configure and enable an ISO endpoint according to descriptor
bool dcd_edpt_iso_activate(uint8_t rhport, const tusb_desc_endpoint_t *ep_desc) {
  (void)rhport;
  const uint8_t       epnum = tu_edpt_number(ep_desc->bEndpointAddress);
  const tusb_dir_t    dir   = tu_edpt_dir(ep_desc->bEndpointAddress);
  struct hw_endpoint *ep    = hw_endpoint_get(epnum, dir);
  TU_ASSERT(ep->dpram_buf != NULL); // must be inited and allocated previously

  if (ep->state == EPSTATE_ACTIVE) {
    hw_endpoint_abort_xfer(ep); // abort any pending transfer
  }
  ep->max_packet_size = ep_desc->wMaxPacketSize;

  // enable endpoint
  io_rw_32 *ep_reg = get_ep_ctrl(epnum, dir);
  if (ep_reg != NULL) {
    *ep_reg |= EP_CTRL_ENABLE_BITS;
  }
  return true;
}

void dcd_edpt_close_all(uint8_t rhport) {
  (void) rhport;
  // may need to use EP Abort
  reset_non_control_endpoints();
}

bool dcd_edpt_xfer(uint8_t rhport, uint8_t ep_addr, uint8_t *buffer, uint16_t total_bytes, bool is_isr) {
  (void)rhport;
  (void)is_isr;
  const uint8_t    epnum = tu_edpt_number(ep_addr);
  const tusb_dir_t dir = tu_edpt_dir(ep_addr);

  hw_endpoint_t *ep = hw_endpoint_get(epnum, dir);
  io_rw_32      *ep_reg  = get_ep_ctrl(epnum, dir);
  io_rw_32      *buf_reg = get_buf_ctrl(epnum, dir);
  rp2usb_xfer_start(ep, ep_reg, buf_reg, buffer, NULL, total_bytes);
  return true;
}

#if CFG_TUD_EDPT_DEDICATED_HWFIFO
bool dcd_edpt_xfer_fifo(uint8_t rhport, uint8_t ep_addr, tu_fifo_t *ff, uint16_t total_bytes, bool is_isr) {
  (void)rhport;
  (void)is_isr;
  hw_endpoint_t *ep = hw_endpoint_get(epnum, dir);
  io_rw_32      *ep_reg  = get_ep_ctrl(epnum, dir);
  io_rw_32      *buf_reg = get_buf_ctrl(epnum, dir);
  rp2usb_xfer_start(ep, ep_reg, buf_reg, NULL, ff, total_bytes);
  return true;
}
#endif

void dcd_edpt_stall(uint8_t rhport, uint8_t ep_addr) {
  (void)rhport;
  const uint8_t    epnum = tu_edpt_number(ep_addr);
  const tusb_dir_t dir   = tu_edpt_dir(ep_addr);
  hw_endpoint_t   *ep    = hw_endpoint_get(epnum, dir);

  if (epnum == 0) {
    // A stall on EP0 has to be armed so it can be cleared on the next setup packet
    usb_hw_set->ep_stall_arm = (dir == TUSB_DIR_IN) ? USB_EP_STALL_ARM_EP0_IN_BITS : USB_EP_STALL_ARM_EP0_OUT_BITS;
  }

  // abort first then stall and clear current pending buffer
  hw_endpoint_abort_xfer(ep);
  io_rw_32 *buf_reg = get_buf_ctrl(epnum, dir);
  *buf_reg          = USB_BUF_CTRL_STALL;
}

void dcd_edpt_clear_stall(uint8_t rhport, uint8_t ep_addr) {
  (void) rhport;
  const uint8_t    epnum = tu_edpt_number(ep_addr);
  const tusb_dir_t dir   = tu_edpt_dir(ep_addr);

  if (epnum != 0) {
    struct hw_endpoint* ep = hw_endpoint_get(epnum, dir);

    if (ep->state == EPSTATE_ACTIVE) {
      // Clear-halt on an endpoint with an in-flight transfer is used as a data-toggle reset
      // (e.g. usbtest case 29) rather than to recover from a real stall (a stall aborts the
      // transfer, leaving the endpoint IDLE). Abort and re-issue the transfer with the toggle
      // reset to DATA0 so it still completes and releases the usbd claim, instead of silently
      // dropping it and starving the endpoint. Save the buffer/length before the abort clears them.
      uint8_t*       user_buf  = ep->user_buf;
      uint16_t       remaining = ep->remaining_len;
      const uint16_t xferred   = ep->xferred_len; // bytes already moved on this submission
      io_rw_32 *ep_reg  = get_ep_ctrl(epnum, dir);
      io_rw_32 *buf_reg = get_buf_ctrl(epnum, dir);
      // bufctrl_prepare16() subtracts each armed buffer's length from remaining_len when arming,
      // for BOTH directions, before the host has drained (IN) or filled (OUT) it. The abort below
      // discards those still-armed buffers, so rewind remaining_len by their lengths or the re-issue
      // is short by 1-2 packets. IN additionally advances user_buf as packets are copied into DPRAM,
      // so its pointer must rewind too; OUT copies out only on completion, so its pointer is intact.
      const uint32_t bc = *buf_reg;
      uint16_t staged = 0;
      if (bc & USB_BUF_CTRL_AVAIL) {
        staged = (uint16_t)(bc & USB_BUF_CTRL_LEN_MASK);
      }
      if ((bc >> 16) & USB_BUF_CTRL_AVAIL) {
        staged = (uint16_t)(staged + ((bc >> 16) & USB_BUF_CTRL_LEN_MASK));
      }
      remaining = (uint16_t)(remaining + staged);
      if (dir == TUSB_DIR_IN) {
        user_buf -= staged;
      }
      hw_endpoint_abort_xfer(ep); // safe abort (handles RP2040-E2), resets ep transfer state
      ep->next_pid = 0;           // DATA0
      rp2usb_xfer_start(ep, ep_reg, buf_reg, user_buf, NULL, remaining);
      // rp2usb_xfer_start() zeroes xferred_len; add back what the aborted transfer already moved so
      // the eventual completion reports the full length, not just the post-clear-halt remainder.
      ep->xferred_len += xferred;
    } else {
      ep->next_pid = 0; // reset data toggle
      io_rw_32 *buf_reg = get_buf_ctrl(epnum, dir);
      *buf_reg          = 0; // clear the stall response
    }
  }
}

void __tusb_irq_path_func(dcd_int_handler)(uint8_t rhport) {
  (void) rhport;
  dcd_rp2040_irq();
}

#endif