Merge pull request #3657 from hathach/usbh-add-control-queue

Add control transfer fifo for host stack

4 files changed, 267 insertions, 82 deletions

diff --git a/test/hil/hil_ci.sh b/test/hil/hil_ci.sh
index 96872e2e1..4c7ba2936 100644
--- a/test/hil/hil_ci.sh
+++ b/test/hil/hil_ci.sh
@@ -54,11 +54,14 @@ scp -q "$ROOT_DIR/test/hil/hil_test.py" \
        "$CONFIG" \
        "$REMOTE:$REMOTE_DIR/test/hil/"
 
-# Copy only firmware binaries (elf/bin/hex), preserving directory structure
+# Copy only firmware binaries (elf/bin/hex) plus esptool metadata
+# (config.env + flash_args needed by the esptool flasher), preserving structure
 copy_board_binaries() {
   local src="$1"
   rsync -a --prune-empty-dirs \
-    --include='*/' --include='*.elf' --include='*.bin' --include='*.hex' --exclude='*' \
+    --include='*/' --include='*.elf' --include='*.bin' --include='*.hex' \
+    --include='config.env' --include='flash_args' \
+    --exclude='*' \
     "$src" "$REMOTE:$REMOTE_DIR/examples/"
 }
 
@@ -73,8 +76,11 @@ if [ -n "$BOARD" ]; then
   copy_board_binaries "$BUILD_DIR"
 else
   echo "==> Copying all built binaries"
+  # Use `%/` parameter expansion to strip the trailing slash from the glob —
+  # rsync needs the bare dir name so the per-board cmake-build-<BOARD>/ subdir
+  # is preserved on the remote (hil_test.py looks up binaries by that path).
   for dir in "$ROOT_DIR"/examples/cmake-build-*/; do
-    [ -d "$dir" ] && copy_board_binaries "$dir"
+    [ -d "$dir" ] && copy_board_binaries "${dir%/}"
   done
 fi
 
@@ -85,5 +91,8 @@ echo "==> Running HIL test on $REMOTE"
 ssh "$REMOTE" bash -s -- "$REMOTE_DIR" "${ARGS[@]}" "test/hil/$CONFIG_BASENAME" <<'REMOTE'
 cd -- "$1"
 shift
+# esptool/idf tools live in ~/.local/bin on ci.lan; the non-interactive shell
+# subprocess used for flashing doesn't pick that up otherwise.
+export PATH="$HOME/.local/bin:$PATH"
 exec python3 -u test/hil/hil_test.py -B examples "$@"
 REMOTE
diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py
index ed9ebbf1a..b0b3fc17e 100755
--- a/test/hil/hil_test.py
+++ b/test/hil/hil_test.py
@@ -22,6 +22,14 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 # THE SOFTWARE.
 
+# Host setup:
+#   - System packages: sudo apt install mtools libmtp9 alsa-utils iperf
+#       mtools     - read_disk_file (device/cdc_msc, device/msc_dual_lun)
+#       libmtp9    - pymtp ctypes load (device/mtp); Debian 13 uses libmtp9t64
+#       alsa-utils - arecord (device/audio_test_freertos)
+#       iperf      - throughput tests (device/net_lwip_*)
+#   - Python packages: pip install -r requirements.txt
+#
 # udev rules :
 # ACTION=="add", SUBSYSTEM=="tty", SUBSYSTEMS=="usb", MODE="0666", PROGRAM="/bin/sh -c 'echo $$ID_SERIAL_SHORT | rev | cut -c -8 | rev'", SYMLINK+="ttyUSB_%c.%s{bInterfaceNumber}"
 # ACTION=="add", SUBSYSTEM=="block", SUBSYSTEMS=="usb", ENV{ID_FS_USAGE}=="filesystem", MODE="0666", PROGRAM="/bin/sh -c 'echo $$ID_SERIAL_SHORT | rev | cut -c -8 | rev'", RUN{program}+="/usr/bin/systemd-mount --no-block --automount=yes --collect $devnode /media/blkUSB_%c.%s{bInterfaceNumber}"
@@ -34,17 +42,11 @@ import re
 import select
 import sys
 import time
-import warnings
 import signal
 from contextlib import redirect_stdout
 from pathlib import Path
 from typing import Any, TypedDict, NotRequired, cast
 
-# Suppress pkg_resources deprecation warning from fs module
-warnings.filterwarnings("ignore", message="pkg_resources is deprecated")
-# Suppress pyfatfs unclean unmount warning
-warnings.filterwarnings("ignore", message="Filesystem was not cleanly unmounted")
-
 import serial
 import subprocess
 import json
@@ -52,7 +54,6 @@ import glob
 import shutil
 from multiprocessing import Pool, Lock
 from multiprocessing import TimeoutError as MpTimeoutError
-import fs
 import hashlib
 import ctypes
 from pymtp import MTP
@@ -232,23 +233,24 @@ def open_serial_dev(port: str):
 
 
 def read_disk_file(uid: str, lun: int, fname: str) -> bytes:
-    # open_fs("fat://{dev}) require 'pip install pyfatfs'
+    # Reads a file from a FAT volume on a block device without mounting it.
+    # Requires mtools: `apt install mtools` (no pip dependency).
     dev = get_disk_dev(uid, 'TinyUSB', lun)
     timeout = ENUM_TIMEOUT
+    last_err = None
     while timeout > 0:
         if os.path.exists(dev):
-            fat = fs.open_fs(f'fat://{dev}?read_only=true')
             try:
-                with fat.open(fname, 'rb') as f:
-                    data = f.read()
-            finally:
-                fat.close()
-            assert data, f'Cannot read file {fname} from {dev}'
-            return data
+                data = subprocess.check_output(
+                    ['mtype', '-i', dev, f'::/{fname}'], stderr=subprocess.PIPE)
+                assert data, f'Cannot read file {fname} from {dev}'
+                return data
+            except subprocess.CalledProcessError as e:
+                last_err = e.stderr.decode(errors='replace').strip()
         time.sleep(1)
         timeout -= 1
 
-    raise AssertionError(f'Storage {dev} not existed')
+    raise AssertionError(f'mtype failed on {dev}: {last_err}' if last_err else f'Storage {dev} not existed')
 
 
 def open_mtp_dev(uid):
@@ -796,6 +798,10 @@ def test_host_msc_file_explorer(board):
     ser.close()
 
 
+def test_host_msc_file_explorer_freertos(board):
+    return test_host_msc_file_explorer(board)
+
+
 # -------------------------------------------------------------
 # Tests: device
 # -------------------------------------------------------------
@@ -1402,7 +1408,7 @@ def test_device_audio_test_freertos(board):
 
 def test_device_hid_generic_inout(board):
     uid = board['uid']
-    import hid
+    import hid  # cython-hidapi (pip: hidapi, apt: python3-hid)
 
     # Find HID device by UID (VID=0xCafe)
     timeout = ENUM_TIMEOUT
@@ -1418,22 +1424,23 @@ def test_device_hid_generic_inout(board):
         timeout -= 1
     assert dev is not None, f'HID device not found for {uid}'
 
-    h = hid.Device(vid=dev['vendor_id'], pid=dev['product_id'], serial=uid)
-
-    # Echo test: send random data and verify echo
-    for size in [8, 32, 63]:
-        # Report ID (0) + payload, padded to 64 bytes
-        payload = bytes([random.randint(1, 255) for _ in range(size)])
-        report = bytes([0]) + payload + bytes(64 - size)
-        h.write(report)
-        echo = h.read(64, timeout=2000)
-        assert echo is not None and len(echo) >= size, (
-            f'HID echo timeout or short read ({size} bytes)')
-        assert bytes(echo[:size]) == payload, (
-            f'HID echo wrong data ({size} bytes):\n'
-            f'  expected: {payload.hex()}\n  received: {bytes(echo[:size]).hex()}')
-
-    h.close()
+    h = hid.device()
+    h.open(dev['vendor_id'], dev['product_id'], uid)
+    try:
+        # Echo test: send random data and verify echo
+        for size in [8, 32, 63]:
+            # Report ID (0) + payload, padded to 64 bytes
+            payload = bytes([random.randint(1, 255) for _ in range(size)])
+            report = bytes([0]) + payload + bytes(64 - size)
+            h.write(report)
+            echo = h.read(64, 2000)
+            assert echo and len(echo) >= size, (
+                f'HID echo timeout or short read ({size} bytes)')
+            assert bytes(echo[:size]) == payload, (
+                f'HID echo wrong data ({size} bytes):\n'
+                f'  expected: {payload.hex()}\n  received: {bytes(echo[:size]).hex()}')
+    finally:
+        h.close()
 
 
 # -------------------------------------------------------------
@@ -1465,6 +1472,7 @@ dual_tests = [
 host_test = [
     'host/cdc_msc_hid',
     'host/msc_file_explorer',
+    'host/msc_file_explorer_freertos',
     'host/device_info',
 ]
 
@@ -1596,7 +1604,18 @@ def test_board(board: Board) -> tuple[str, int, list[str]]:
     if name in board_test:
         test_list = board_test[name]
     elif len(test_only) > 0:
-        test_list = test_only
+        # Explicit -t: filter against the board's capabilities so a device-only
+        # board doesn't try to run host/dual tests (the test functions need a
+        # `dev_attached` entry in the board config that won't exist).
+        board_tests = board.get('tests', {})
+        if 'only' in board_tests:
+            allowed = set(board_tests['only'])
+            test_list = [t for t in test_only if t in allowed]
+        else:
+            for t in test_only:
+                category = t.split('/', 1)[0]
+                if board_tests.get(category) is True:
+                    test_list.append(t)
     else:
         if 'tests' in board:
             board_tests = board['tests']
diff --git a/test/hil/requirements.txt b/test/hil/requirements.txt
index ef2fecebe..ef1cf575b 100644
--- a/test/hil/requirements.txt
+++ b/test/hil/requirements.txt
@@ -1,4 +1,9 @@
-fs
-hid
-pyfatfs
+# System packages (install separately):
+#   sudo apt install mtools libmtp9 alsa-utils iperf
+#     mtools     - read_disk_file (device/cdc_msc, device/msc_dual_lun)
+#     libmtp9    - pymtp ctypes load (device/mtp); Debian 13 uses libmtp9t64
+#     alsa-utils - arecord (device/audio_test_freertos)
+#     iperf      - throughput tests (device/net_lwip_*)
+hidapi
 pyserial
+esptool
diff --git a/test/hil/tinyusb.json b/test/hil/tinyusb.json
index cba7677cf..dc28df7b9 100644
--- a/test/hil/tinyusb.json
+++ b/test/hil/tinyusb.json
@@ -3,12 +3,35 @@
         {
             "name": "espressif_p4_function_ev",
             "uid": "6055F9F98715",
-            "build" : {
-                "flags_on": ["", "CFG_TUD_DWC2_DMA_ENABLE CFG_TUH_DWC2_DMA_ENABLE"]
+            "build": {
+                "flags_on": [
+                    "",
+                    "CFG_TUD_DWC2_DMA_ENABLE CFG_TUH_DWC2_DMA_ENABLE"
+                ]
             },
             "tests": {
-                "only": ["device/cdc_msc_freertos", "device/hid_composite_freertos", "device/audio_test_freertos", "host/device_info"],
-                "dev_attached": [{"vid_pid": "1a86_55d4", "serial": "52D2002427", "is_cdc": true}]
+                "only": [
+                    "device/cdc_msc_freertos",
+                    "device/hid_composite_freertos",
+                    "device/audio_test_freertos",
+                    "host/device_info",
+                    "host/msc_file_explorer_freertos"
+                ],
+                "dev_attached": [
+                    {
+                        "vid_pid": "1a86_55d4",
+                        "serial": "52D2002427",
+                        "is_cdc": true
+                    },
+                    {
+                        "vid_pid": "21c4_0cc7",
+                        "serial": "900058944CB80A53",
+                        "is_msc": true,
+                        "block_size": 512,
+                        "block_count": 60620800,
+                        "msc_inquiry": "Lexar    USB Flash Drive  PMAP"
+                    }
+                ]
             },
             "flasher": {
                 "name": "esptool",
@@ -21,12 +44,36 @@
         {
             "name": "espressif_s3_devkitm",
             "uid": "84F703C084E4",
-            "build" : {
-                "flags_on": ["", "CFG_TUD_DWC2_DMA_ENABLE CFG_TUH_DWC2_DMA_ENABLE"]
+            "build": {
+                "flags_on": [
+                    "",
+                    "CFG_TUD_DWC2_DMA_ENABLE CFG_TUH_DWC2_DMA_ENABLE"
+                ]
             },
             "tests": {
-                "only": ["device/cdc_msc_freertos", "device/hid_composite_freertos", "device/audio_test_freertos", "host/device_info"],
-                "dev_attached": [{"vid_pid": "1a86_55d4", "serial": "52D2005402", "is_cdc": true}]
+                "only": [
+                    "device/cdc_msc_freertos",
+                    "device/hid_composite_freertos",
+                    "device/audio_test_freertos",
+                    "host/device_info",
+                    "host/msc_file_explorer_freertos"
+                ],
+                "dev_attached": [
+                    {
+                        "vid_pid": "1a86_55d4",
+                        "serial": "52D2005402",
+                        "is_cdc": true
+                    },
+                    {
+                        "vid_pid": "048d_04d2",
+                        "serial": "\u0409",
+                        "is_msc": true,
+                        "block_size": 512,
+                        "block_count": 30720000,
+                        "msc_inquiry": "General  UDisk            5.00",
+                        "comment": "General UDisk reports iSerialNumber=U+0409"
+                    }
+                ]
             },
             "flasher": {
                 "name": "esptool",
@@ -39,7 +86,9 @@
             "name": "feather_nrf52840_express",
             "uid": "1F0479CD0F764471",
             "tests": {
-                "device": true, "host": false, "dual": false
+                "device": true,
+                "host": false,
+                "dual": false
             },
             "flasher": {
                 "name": "jlink",
@@ -51,7 +100,9 @@
             "name": "max32666fthr",
             "uid": "0C81464124010B20FF0A08CC2C",
             "tests": {
-                "device": true, "host": false, "dual": false
+                "device": true,
+                "host": false,
+                "dual": false
             },
             "flasher": {
                 "name": "openocd_adi",
@@ -71,7 +122,13 @@
                 "device": true,
                 "host": false,
                 "dual": true,
-                "dev_attached": [{"vid_pid": "067b_2303", "serial": "0", "is_cdc": true}],
+                "dev_attached": [
+                    {
+                        "vid_pid": "067b_2303",
+                        "serial": "0",
+                        "is_cdc": true
+                    }
+                ],
                 "comment": "pl23x"
             },
             "flasher": {
@@ -84,7 +141,9 @@
             "name": "mimxrt1015_evk",
             "uid": "DC28F865D2111D228D00B0543A70463C",
             "tests": {
-                "device": true, "host": false, "dual": false
+                "device": true,
+                "host": false,
+                "dual": false
             },
             "flasher": {
                 "name": "jlink",
@@ -96,9 +155,25 @@
             "name": "mimxrt1064_evk",
             "uid": "BAE96FB95AFA6DBB8F00005002001200",
             "tests": {
-                "device": true, "host": true, "dual": true,
-                "dev_attached": [{"vid_pid": "10c4_ea60", "serial": "0001", "is_cdc": true}],
-                "comment": "cp2102"
+                "device": true,
+                "host": true,
+                "dual": true,
+                "dev_attached": [
+                    {
+                        "vid_pid": "10c4_ea60",
+                        "serial": "0001",
+                        "is_cdc": true,
+                        "comment": "cp2102"
+                    },
+                    {
+                        "vid_pid": "21c4_0cc7",
+                        "serial": "900058874D871F66",
+                        "is_msc": true,
+                        "block_size": 512,
+                        "block_count": 60620800,
+                        "msc_inquiry": "Lexar    USB Flash Drive  PMAP"
+                    }
+                ]
             },
             "flasher": {
                 "name": "jlink",
@@ -110,7 +185,9 @@
             "name": "lpcxpresso11u37",
             "uid": "17121919",
             "tests": {
-                "device": true, "host": false, "dual": false
+                "device": true,
+                "host": false,
+                "dual": false
             },
             "flasher": {
                 "name": "jlink",
@@ -135,13 +212,32 @@
         {
             "name": "raspberry_pi_pico",
             "uid": "E6614C311B764A37",
-            "build" : {
-                "flags_on": ["CFG_TUH_RPI_PIO_USB"]
+            "build": {
+                "flags_on": [
+                    "CFG_TUH_RPI_PIO_USB"
+                ]
             },
             "tests": {
-                "device": true, "host": true, "dual": true,
-                "dev_attached": [{"vid_pid": "1a86_7523", "serial": "0", "is_cdc": true}],
-                "comment": "ch34x"
+                "device": true,
+                "host": true,
+                "dual": true,
+                "dev_attached": [
+                    {
+                        "vid_pid": "1a86_7523",
+                        "serial": "0",
+                        "is_cdc": true,
+                        "comment": "ch34x"
+                    },
+                    {
+                        "vid_pid": "048d_04d2",
+                        "serial": "\u0409",
+                        "is_msc": true,
+                        "block_size": 512,
+                        "block_count": 30720000,
+                        "msc_inquiry": "General  UDisk            5.00",
+                        "comment": "General UDisk reports iSerialNumber=U+0409"
+                    }
+                ]
             },
             "flasher": {
                 "name": "openocd",
@@ -153,9 +249,16 @@
             "name": "raspberry_pi_pico_w",
             "uid": "E6614864D35DAE36",
             "tests": {
-                "device": false, "host": true, "dual": false,
+                "device": false,
+                "host": true,
+                "dual": false,
                 "dev_attached": [
                     {
+                        "vid_pid": "1a86_55d4",
+                        "serial": "52D2002694",
+                        "is_cdc": true
+                    },
+                    {
                         "vid_pid": "2008_2018",
                         "serial": "O20070925A002746",
                         "is_msc": true,
@@ -176,7 +279,9 @@
             "name": "raspberry_pi_pico2",
             "uid": "560AE75E1C7152C9",
             "tests": {
-                "device": false, "host": true, "dual": false,
+                "device": false,
+                "host": true,
+                "dual": false,
                 "dev_attached": [
                     {
                         "vid_pid": "0951_1603",
@@ -184,7 +289,7 @@
                         "is_msc": true,
                         "block_size": 512,
                         "block_count": 3987456,
-                        "msc_inquiry": "Kingston DataTraveler 2.0 1.0"
+                        "msc_inquiry": "Kingston DataTraveler 2.0 1.00"
                     }
                 ]
             },
@@ -202,14 +307,24 @@
                 "host": true,
                 "dual": true,
                 "dev_attached": [
-                    {"vid_pid": "0403_6001", "serial": "0", "is_cdc": true},
-                    {"vid_pid": "058f_6387", "serial": "A8BEE062633D", "is_msc": true,
-                        "block_size": 512, "block_count": 7639040, "msc_inquiry": "Generic  Flash Disk       8.07"}
+                    {
+                        "vid_pid": "0403_6001",
+                        "serial": "0",
+                        "is_cdc": true
+                    },
+                    {
+                        "vid_pid": "058f_6387",
+                        "serial": "A8BEE062633D",
+                        "is_msc": true,
+                        "block_size": 512,
+                        "block_count": 7639040,
+                        "msc_inquiry": "Generic  Flash Disk       8.07"
+                    }
                 ]
             },
             "flasher": {
                 "name": "openocd",
-                "uid": "E6614103E78E8324",
+                "uid": "E663AC91D3359B38",
                 "args": "-f interface/cmsis-dap.cfg -f target/rp2350.cfg -c \"adapter speed 5000\""
             }
         },
@@ -217,7 +332,9 @@
             "name": "stm32f072disco",
             "uid": "3A001A001357364230353532",
             "tests": {
-                "device": true, "host": false, "dual": false
+                "device": true,
+                "host": false,
+                "dual": false
             },
             "flasher": {
                 "name": "jlink",
@@ -229,12 +346,31 @@
         {
             "name": "stm32f723disco",
             "uid": "460029001951373031313335",
-            "build" : {
-                "flags_on": ["", "CFG_TUH_DWC2_DMA_ENABLE"]
+            "build": {
+                "flags_on": [
+                    "",
+                    "CFG_TUH_DWC2_DMA_ENABLE"
+                ]
             },
             "tests": {
-                "device": true, "host": true, "dual": false,
-                "dev_attached": [{"vid_pid": "1a86_55d4", "serial": "52D2003414", "is_cdc": true}]
+                "device": true,
+                "host": true,
+                "dual": false,
+                "dev_attached": [
+                    {
+                        "vid_pid": "1a86_55d4",
+                        "serial": "52D2003414",
+                        "is_cdc": true
+                    },
+                    {
+                        "vid_pid": "21c4_0cc7",
+                        "serial": "90005893730A1A63",
+                        "is_msc": true,
+                        "block_size": 512,
+                        "block_count": 60620800,
+                        "msc_inquiry": "Lexar    USB Flash Drive  PMAP"
+                    }
+                ]
             },
             "flasher": {
                 "name": "jlink",
@@ -246,11 +382,16 @@
         {
             "name": "stm32h743nucleo",
             "uid": "110018000951383432343236",
-            "build" : {
-                "flags_on": ["", "CFG_TUD_DWC2_DMA_ENABLE"]
+            "build": {
+                "flags_on": [
+                    "",
+                    "CFG_TUD_DWC2_DMA_ENABLE"
+                ]
             },
             "tests": {
-                "device": true, "host": false, "dual": false
+                "device": true,
+                "host": false,
+                "dual": false
             },
             "flasher": {
                 "name": "openocd",
@@ -262,7 +403,9 @@
             "name": "stm32g0b1nucleo",
             "uid": "4D0038000450434E37343120",
             "tests": {
-                "device": true, "host": false, "dual": false
+                "device": true,
+                "host": false,
+                "dual": false
             },
             "flasher": {
                 "name": "openocd",
@@ -276,11 +419,16 @@
         {
             "name": "stm32f769disco",
             "uid": "21002F000F51363531383437",
-            "build" : {
-                "flags_on": ["", "CFG_TUD_DWC2_DMA_ENABLE"]
+            "build": {
+                "flags_on": [
+                    "",
+                    "CFG_TUD_DWC2_DMA_ENABLE"
+                ]
             },
             "tests": {
-                "device": true, "host": false, "dual": false
+                "device": true,
+                "host": false,
+                "dual": false
             },
             "flasher": {
                 "name": "jlink",
@@ -292,7 +440,9 @@
             "name": "nanoch32v203",
             "uid": "CDAB277B0FBC03E339E339E3",
             "tests": {
-                "device": true, "host": false, "dual": false
+                "device": true,
+                "host": false,
+                "dual": false
             },
             "flasher": {
                 "name": "openocd_wch",
@@ -304,7 +454,9 @@
             "name": "stm32f407disco",
             "uid": "30001A000647313332353735",
             "tests": {
-                "device": true, "host": false, "dual": false
+                "device": true,
+                "host": false,
+                "dual": false
             },
             "flasher": {
                 "name": "jlink",