Merge upstream master into net descriptor-based ep_size branch

- Resolve .gitignore conflict: incorporate upstream's .worktrees entry and
  expand dependency path patterns to cover all tools/get_deps.py fetched dirs
  (lib/, tools/linkermap, tools/uf2, hw/mcu/*) instead of listing only a few

- Auto-merged upstream changes: build system cleanups, BSP updates,
  portability fixes, new boards (nrf54lm20dk, stm32h743_weact),
  fatfs relocation, and many other upstream improvements

- Net driver changes (ecm_rndis_device.c, ncm_device.c, net_device.h,
  usbd.h, usb_descriptors.c) retain our PR's descriptor-based ep_size
  approach as our branch takes precedence

Co-authored-by: HiFiPhile <[email protected]>

4 files changed, 255 insertions, 57 deletions

diff --git a/test/fuzz/dcd_fuzz.cc b/test/fuzz/dcd_fuzz.cc
index 7a5d51623..3e73f0acf 100644
--- a/test/fuzz/dcd_fuzz.cc
+++ b/test/fuzz/dcd_fuzz.cc
@@ -61,14 +61,22 @@ void dcd_int_handler(uint8_t rhport) {
 
   // Choose if we want to generate a signal based on the fuzzed data.
   if (_fuzz_data_provider->ConsumeBool()) {
-    dcd_event_bus_signal(
-        rhport,
-        // Choose a random event based on the fuzz data.
-        (dcd_eventid_t)_fuzz_data_provider->ConsumeIntegralInRange<uint8_t>(
-            DCD_EVENT_INVALID + 1, DCD_EVENT_COUNT - 1),
-        // Identify trigger as either an interrupt or a syncrhonous call
-        // depending on fuzz data.
-        _fuzz_data_provider->ConsumeBool());
+    // Only generate bus signal events that don't carry additional union data.
+    // DCD_EVENT_XFER_COMPLETE, DCD_EVENT_SOF, and DCD_EVENT_BUS_RESET need
+    // properly initialized union fields; USBD_EVENT_FUNC_CALL is internal only.
+    // Valid bus-signal-only events: UNPLUGGED(2), SUSPEND(4), RESUME(5).
+    static const dcd_eventid_t bus_signal_events[] = {
+        DCD_EVENT_UNPLUGGED, DCD_EVENT_SUSPEND, DCD_EVENT_RESUME};
+    uint8_t idx = _fuzz_data_provider->ConsumeIntegralInRange<uint8_t>(0, 2);
+    dcd_event_bus_signal(rhport, bus_signal_events[idx],
+                         _fuzz_data_provider->ConsumeBool());
+  }
+
+  // Optionally generate a BUS_RESET event with a valid speed value.
+  if (_fuzz_data_provider->ConsumeBool()) {
+    tusb_speed_t speed = (tusb_speed_t)_fuzz_data_provider->ConsumeIntegralInRange<uint8_t>(
+        TUSB_SPEED_FULL, TUSB_SPEED_HIGH);
+    dcd_event_bus_reset(rhport, speed, _fuzz_data_provider->ConsumeBool());
   }
 
   if (_fuzz_data_provider->ConsumeBool()) {
diff --git a/test/hil/hil_ci.sh b/test/hil/hil_ci.sh
new file mode 100644
index 000000000..fa8bb0245
--- /dev/null
+++ b/test/hil/hil_ci.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+# Run HIL test remotely on ci.lan
+# Usage: test/hil/hil_ci.sh [-b BOARD] [-t TEST] [extra hil_test.py args...]
+# Example:
+#   test/hil/hil_ci.sh -b stm32f723disco
+#   test/hil/hil_ci.sh -b stm32f723disco -t host/cdc_msc_hid -r 1
+
+set -e
+
+REMOTE=ci.lan
+REMOTE_DIR=/tmp/tinyusb-hil
+SCRIPT_DIR="$(cd "$(dirname "$0")/../.." && pwd)"
+
+# Parse -b BOARD from arguments to know which build to copy
+BOARD=""
+ARGS=()
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    -b)
+      BOARD="$2"
+      ARGS+=("$1" "$2")
+      shift 2
+      ;;
+    *)
+      ARGS+=("$1")
+      shift
+      ;;
+  esac
+done
+
+# Setup remote directory
+echo "==> Setting up remote $REMOTE:$REMOTE_DIR"
+ssh "$REMOTE" "rm -rf $REMOTE_DIR && mkdir -p $REMOTE_DIR/test/hil $REMOTE_DIR/examples"
+
+# Copy HIL test script and config
+echo "==> Copying test scripts"
+scp -q "$SCRIPT_DIR/test/hil/hil_test.py" \
+       "$SCRIPT_DIR/test/hil/pymtp.py" \
+       "$SCRIPT_DIR/test/hil/tinyusb.json" \
+       "$REMOTE:$REMOTE_DIR/test/hil/"
+
+# Copy only firmware binaries (elf/bin/hex), preserving directory structure
+copy_board_binaries() {
+  local src="$1"
+  local board_name
+  board_name=$(basename "$src")
+  rsync -a --include='*/' --include='*.elf' --include='*.bin' --include='*.hex' --exclude='*' \
+    "$src" "$REMOTE:$REMOTE_DIR/examples/"
+}
+
+if [ -n "$BOARD" ]; then
+  BUILD_DIR="$SCRIPT_DIR/examples/cmake-build-$BOARD"
+  if [ ! -d "$BUILD_DIR" ]; then
+    echo "Error: build directory not found: $BUILD_DIR"
+    echo "Build first with: cd examples && cmake -DBOARD=$BOARD -G Ninja -B cmake-build-$BOARD .. && cmake --build cmake-build-$BOARD"
+    exit 1
+  fi
+  echo "==> Copying binaries for $BOARD"
+  copy_board_binaries "$BUILD_DIR"
+else
+  echo "==> Copying all built binaries"
+  for dir in "$SCRIPT_DIR"/examples/cmake-build-*/; do
+    [ -d "$dir" ] && copy_board_binaries "$dir"
+  done
+fi
+
+# Run test
+echo "==> Running HIL test on $REMOTE"
+ssh -t "$REMOTE" "cd $REMOTE_DIR && python3 -u test/hil/hil_test.py -B examples ${ARGS[*]} tinyusb.json"
diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py
index 41e9fad88..d50a60894 100755
--- a/test/hil/hil_test.py
+++ b/test/hil/hil_test.py
@@ -50,7 +50,7 @@ import ctypes
 from pymtp import MTP
 import string
 
-ENUM_TIMEOUT = 30
+ENUM_TIMEOUT = 15
 
 STATUS_OK = "\033[32mOK\033[0m"
 STATUS_FAILED = "\033[31mFailed\033[0m"
@@ -102,6 +102,8 @@ def get_serial_dev(id, vendor_str, product_str, ifnum):
         # just use id: mostly for cp210x/ftdi flasher
         pattern = f'/dev/serial/by-id/usb-*_{id}-if*'
         port_list = glob.glob(pattern)
+        if len(port_list) == 0:
+            raise RuntimeError(f'No serial device found for {pattern}')
         return port_list[0]
 
 
@@ -155,8 +157,7 @@ def read_disk_file(uid, lun, fname):
 
 def open_mtp_dev(uid):
     mtp = MTP()
-    # MTP seems to take a while to enumerate
-    timeout = 2 * ENUM_TIMEOUT
+    timeout = ENUM_TIMEOUT
     while timeout > 0:
         # unmount gio/gvfs MTP mount which blocks libmtp from accessing the device
         subprocess.run(f"gio mount -u mtp://TinyUsb_TinyUsb_Device_{uid}/",
@@ -456,17 +457,30 @@ def test_host_device_info(board):
     return 0
 
 
-def print_msc_info(lines):
-    """Print MSC inquiry and disk size on a single line"""
+def check_msc_info(lines, msc_devs):
+    """Print MSC info and verify block_count/block_size against config"""
     inquiry = ''
     disk_size = ''
     for l in lines:
-        if re.match(r'^[A-Za-z].*\s+rev\s+', l):
+        if re.match(r'^[A-Za-z].*\s+(rev\s+|[0-9])', l) and 'Disk Size' not in l:
             inquiry = l.strip()
         if 'Disk Size' in l:
             disk_size = l.strip()
     if inquiry or disk_size:
         print(f'\r\n  {inquiry} {disk_size} ', end='')
+    # Verify block_count and block_size from "Disk Size: COUNT SIZE-byte blocks: N MB"
+    if disk_size and msc_devs:
+        m = re.match(r'Disk Size:\s+(\d+)\s+(\d+)-byte blocks', disk_size)
+        if m:
+            actual_count = int(m.group(1))
+            actual_size = int(m.group(2))
+            for dev in msc_devs:
+                exp_count = dev.get('block_count')
+                exp_size = dev.get('block_size')
+                if exp_count and actual_count == exp_count:
+                    assert actual_size == exp_size, (
+                        f'MSC block_size mismatch: expected {exp_size}, got {actual_size}')
+                    break
 
 
 def test_host_cdc_msc_hid(board):
@@ -475,7 +489,7 @@ def test_host_cdc_msc_hid(board):
     cdc_devs = [d for d in dev_attached if d.get('is_cdc')]
     msc_devs = [d for d in dev_attached if d.get('is_msc')]
     if not cdc_devs and not msc_devs:
-        return
+        return 'skipped'
 
     port = get_serial_dev(flasher["uid"], None, None, 0)
     ser = open_serial_dev(port)
@@ -525,7 +539,7 @@ def test_host_cdc_msc_hid(board):
     if msc_devs:
         assert b'MassStorage device is mounted' in data, 'MSC device not mounted on host'
         assert b'Disk Size' in data, 'MSC Disk Size not reported'
-        print_msc_info(lines)
+        check_msc_info(lines, msc_devs)
 
     # CDC echo test via flasher serial
     if not cdc_devs:
@@ -533,33 +547,34 @@ def test_host_cdc_msc_hid(board):
         return
 
     time.sleep(2)
+    ser.read(ser.in_waiting)
     ser.reset_input_buffer()
 
     def rand_ascii(length):
         return "".join(random.choices(string.ascii_letters + string.digits, k=length)).encode("ascii")
 
-    sizes = [8, 32, 64, 128]
-    for size in sizes:
-        test_data = rand_ascii(size)
-        ser.reset_input_buffer()
-
-        # Write byte-by-byte with delay to avoid UART overrun
-        for b in test_data:
-            ser.write(bytes([b]))
-            ser.flush()
-            time.sleep(0.001)
+    packet_size = 64
 
-        # Read echo back with timeout
+    # Echo test: write random 1-packet_size chunks, wait for echo before sending next
+    echo_len = 1024
+    echo_data = rand_ascii(echo_len)
+    ser.reset_input_buffer()
+    offset = 0
+    while offset < echo_len:
+        chunk_size = min(random.randint(1, packet_size), echo_len - offset)
+        ser.write(echo_data[offset:offset + chunk_size])
+        ser.flush()
+        # wait until this chunk is echoed back
         echo = b''
-        t = 5.0
-        while t > 0 and len(echo) < size:
-            rd = ser.read(max(1, ser.in_waiting))
+        t_end = time.monotonic() + 1.0
+        while time.monotonic() < t_end and len(echo) < chunk_size:
+            rd = ser.read(chunk_size - len(echo))
             if rd:
                 echo += rd
-            time.sleep(0.05)
-            t -= 0.05
-        assert echo == test_data, (f'CDC echo wrong data ({size} bytes):\n'
-                                   f'  expected: {test_data}\n  received: {echo}')
+        expected = echo_data[offset:offset + chunk_size]
+        assert echo == expected, (f'CDC echo mismatch at offset {offset} ({chunk_size} bytes):\n'
+                                  f'  expected: {expected}\n  received: {echo}')
+        offset += chunk_size
 
     ser.close()
 
@@ -568,7 +583,7 @@ def test_host_msc_file_explorer(board):
     flasher = board['flasher']
     msc_devs = [d for d in board['tests'].get('dev_attached', []) if d.get('is_msc')]
     if not msc_devs:
-        return
+        return 'skipped'
 
     port = get_serial_dev(flasher["uid"], None, None, 0)
     ser = open_serial_dev(port)
@@ -591,9 +606,9 @@ def test_host_msc_file_explorer(board):
         timeout -= 0.1
     assert b'Disk Size' in data, 'MSC device not mounted'
     lines = data.decode('utf-8', errors='ignore').splitlines()
-    print_msc_info(lines)
+    check_msc_info(lines, msc_devs)
 
-    # Send "cat README.TXT" and read response
+    # Send "cat README.TXT" and check response (optional — file may not exist on all drives)
     time.sleep(1)
     ser.reset_input_buffer()
     for ch in 'cat README.TXT\r':
@@ -601,24 +616,46 @@ def test_host_msc_file_explorer(board):
         ser.flush()
         time.sleep(0.002)
 
-    # Read response
     resp = b''
     t = 10.0
     while t > 0:
         rd = ser.read(max(1, ser.in_waiting))
         if rd:
             resp += rd
-        # wait for prompt after command output
         if b'>' in resp and resp.rstrip().endswith(b'>'):
             break
         time.sleep(0.05)
         t -= 0.05
 
-    # Verify response contains README content
     resp_text = resp.decode('utf-8', errors='ignore')
-    assert MSC_README_TXT.decode() in resp_text, (f'MSC README.TXT not found in response:\n'
-                                                   f'  received: {resp_text}')
-    print('README.TXT matched ', end='')
+    if MSC_README_TXT.decode() in resp_text:
+        print('README.TXT matched ', end='')
+
+    # MSC throughput test: send dd command to read sectors
+    time.sleep(0.5)
+    ser.reset_input_buffer()
+    for ch in 'dd 1024\r':
+        ser.write(ch.encode())
+        ser.flush()
+        time.sleep(0.002)
+
+    # Read dd output until prompt
+    resp = b''
+    t = 30.0
+    while t > 0:
+        rd = ser.read(max(1, ser.in_waiting))
+        if rd:
+            resp += rd
+        if b'KB/s' in resp and b'>' in resp:
+            break
+        time.sleep(0.05)
+        t -= 0.05
+
+    resp_text = resp.decode('utf-8', errors='ignore')
+    for line in resp_text.splitlines():
+        if 'KB/s' in line:
+            print(f'{line.strip()} ', end='')
+            break
 
     ser.close()
 
@@ -700,6 +737,53 @@ def test_device_cdc_msc(board):
     data = read_disk_file(uid, 0, 'README.TXT')
     assert data == MSC_README_TXT, f'MSC wrong data in README.TXT\n expected: {MSC_README_TXT.decode()}\n received: {data.decode()}'
 
+    # MSC dd throughput test: read all sectors then write back same data
+    dev = get_disk_dev(uid, 'TinyUSB', 0)
+    timeout = ENUM_TIMEOUT
+    while timeout > 0:
+        if os.path.exists(dev):
+            break
+        time.sleep(1)
+        timeout -= 1
+    assert timeout > 0, f'Disk {dev} not found for dd test'
+
+    block_count = 16
+    block_size = 512
+    tmp_file = f'/tmp/msc_dd_{uid}.bin'
+
+    # dd reports speed based on payload only. Each block also transfers 31-byte CBW + 13-byte CSW on USB.
+    scsi_ratio = (block_size + 31 + 13) / block_size
+
+    def parse_dd_speed(dd_output):
+        """Parse dd output, return USB-adjusted speed string"""
+        for line in dd_output.splitlines():
+            m = re.search(r'([\d.]+)\s+([kMG]?B/s)', line)
+            if m:
+                speed_val = float(m.group(1)) * scsi_ratio
+                return f'{speed_val:.1f} {m.group(2)}'
+        return ''
+
+    # Read: dd from device to file
+    ret = run_cmd(f'dd if={dev} of={tmp_file} bs={block_size} count={block_count} iflag=direct 2>&1')
+    assert ret.returncode == 0, f'dd read failed: {ret.stdout.decode()}'
+    read_speed = parse_dd_speed(ret.stdout.decode())
+
+    # Write back the same data to avoid corrupting the disk (skip if read-only)
+    ret = run_cmd(f'dd if={tmp_file} of={dev} bs={block_size} count={block_count} oflag=direct 2>&1')
+    if ret.returncode != 0 and 'Read-only' in ret.stdout.decode():
+        write_speed = 'skip (read-only)'
+    else:
+        assert ret.returncode == 0, f'dd write failed: {ret.stdout.decode()}'
+        write_speed = parse_dd_speed(ret.stdout.decode())
+
+    try:
+        os.remove(tmp_file)
+    except OSError:
+        pass
+
+    if read_speed and write_speed:
+        print(f'  dd read: {read_speed}, write: {write_speed}', end='')
+
 
 def test_device_cdc_msc_freertos(board):
     test_device_cdc_msc(board)
@@ -1107,24 +1191,26 @@ def test_example(board, f1, example):
         print(f'Flashing {fw_name}.elf')
 
     # flash firmware. It may fail randomly, retry a few times
-    max_rety = 3
     start_s = time.time()
-    for i in range(max_rety):
+    for i in range(max_retry):
         ret = globals()[f'flash_{board["flasher"]["name"].lower()}'](board, fw_name)
         if ret.returncode == 0:
             try:
-                globals()[f'test_{example.replace("/", "_")}'](board)
-                print('  OK', end='')
+                tret = globals()[f'test_{example.replace("/", "_")}'](board)
+                if tret == 'skipped':
+                    print(f'  {STATUS_SKIPPED}', end='')
+                else:
+                    print('  OK', end='')
                 break
             except Exception as e:
-                if i == max_rety - 1:
+                if i == max_retry - 1:
                     err_count += 1
                     print(f'{STATUS_FAILED}: {e}')
                 else:
-                    print(f'\n  Test failed: {e}, retry {i+2}/{max_rety}', end='')
+                    print(f'\n  Test failed: {e}, retry {i+2}/{max_retry}', end='')
                     time.sleep(0.5)
         else:
-            print(f'\n  Flash failed, retry {i+2}/{max_rety}', end='')
+            print(f'\n  Flash failed, retry {i+2}/{max_retry}', end='')
             time.sleep(0.5)
 
     if ret.returncode != 0:
@@ -1184,6 +1270,7 @@ def main():
     global verbose
     global test_only
     global build_dir
+    global max_retry
 
     duration = time.time()
 
@@ -1193,6 +1280,7 @@ def main():
     parser.add_argument('-s', '--skip', action='append', default=[], help='Skip boards from test')
     parser.add_argument('-t', '--test-only', action='append', default=[], help='Tests to run, all if not specified')
     parser.add_argument('-B', '--build', default='cmake-build', help='Build folder name (default: cmake-build)')
+    parser.add_argument('-r', '--retry', type=int, default=3, help='Retry count for failed tests (default: 3)')
     parser.add_argument('-v', '--verbose', action='store_true', help='Verbose output')
     args = parser.parse_args()
 
@@ -1202,6 +1290,7 @@ def main():
     verbose = args.verbose
     test_only = args.test_only
     build_dir = args.build
+    max_retry = args.retry
 
     # if config file is not found, try to find it in the same directory as this script
     if not os.path.exists(config_file):
diff --git a/test/hil/tinyusb.json b/test/hil/tinyusb.json
index b4c6aaefe..92b7b21b0 100644
--- a/test/hil/tinyusb.json
+++ b/test/hil/tinyusb.json
@@ -62,11 +62,15 @@
         {
             "name": "metro_m4_express",
             "uid": "9995AD485337433231202020FF100A34",
-            "build" : {
-                "args": ["MAX3421_HOST=1"]
+            "build": {
+                "args": [
+                    "MAX3421_HOST=1"
+                ]
             },
             "tests": {
-                "device": true, "host": false, "dual": true,
+                "device": true,
+                "host": false,
+                "dual": true,
                 "dev_attached": [{"vid_pid": "067b_2303", "serial": "0", "is_cdc": true}],
                 "comment": "pl23x"
             },
@@ -147,10 +151,24 @@
         },
         {
             "name": "raspberry_pi_pico_w",
-            "uid": "E6614C311B764A37",
+            "uid": "E6614864D35DAE36",
             "tests": {
                 "device": false, "host": true, "dual": false,
-                "dev_attached": [{"vid_pid": "1a86_55d4", "serial": "52D2023934", "is_cdc": true}]
+                "dev_attached": [
+                    {
+                        "vid_pid": "1a86_55d4",
+                        "serial": "52D2023934",
+                        "is_cdc": true
+                    },
+                    {
+                        "vid_pid": "2008_2018",
+                        "serial": "O20070925A002746",
+                        "is_msc": true,
+                        "block_size": 512,
+                        "block_count": 4124152,
+                        "msc_inquiry": "USB2.0   Flash Disk       2.10"
+                    }
+                ]
             },
             "flasher": {
                 "name": "openocd",
@@ -164,7 +182,21 @@
             "uid": "560AE75E1C7152C9",
             "tests": {
                 "device": false, "host": true, "dual": false,
-                "dev_attached": [{"vid_pid": "1a86_55d4", "serial": "52D2002694", "is_cdc": true}]
+                "dev_attached": [
+                    {
+                        "vid_pid": "1a86_55d4",
+                        "serial": "52D2002694",
+                        "is_cdc": true
+                    },
+                    {
+                        "vid_pid": "0951_1603",
+                        "serial": "820000000000000045B46338",
+                        "is_msc": true,
+                        "block_size": 512,
+                        "block_count": 3987456,
+                        "msc_inquiry": "Kingston DataTraveler 2.0 1.0"
+                    }
+                ]
             },
             "flasher": {
                 "name": "openocd",
@@ -182,7 +214,7 @@
                 "dev_attached": [
                     {"vid_pid": "0403_6001", "serial": "0", "is_cdc": true},
                     {"vid_pid": "058f_6387", "serial": "A8BEE062633D", "is_msc": true,
-                        "msc_disk_size": 3730, "msc_inquiry": "Generic  Flash Disk       rev 8.07"}
+                        "block_size": 512, "block_count": 7639040, "msc_inquiry": "Generic  Flash Disk       8.07"}
                 ]
             },
             "flasher": {