Merge branch 'master' into dcd_notif

Signed-off-by: HiFiPhile <[email protected]>

13 files changed, 491 insertions, 230 deletions

diff --git a/test/fuzz/dcd_fuzz.cc b/test/fuzz/dcd_fuzz.cc
index 06ddddc4a..046a90555 100644
--- a/test/fuzz/dcd_fuzz.cc
+++ b/test/fuzz/dcd_fuzz.cc
@@ -46,9 +46,10 @@ tu_static State state = {false, 0, 0};
 // All no-ops as we are fuzzing.
 //--------------------------------------------------------------------+
 extern "C" {
-void dcd_init(uint8_t rhport) {
+bool dcd_init(uint8_t rhport, const tusb_rhport_init_t* rh_init) {
   UNUSED(rhport);
-  return;
+  UNUSED(rh_init);
+  return true;
 }
 
 void dcd_int_handler(uint8_t rhport) {
diff --git a/test/fuzz/net_fuzz.cc b/test/fuzz/net_fuzz.cc
index 63cd1ac98..468437b0c 100644
--- a/test/fuzz/net_fuzz.cc
+++ b/test/fuzz/net_fuzz.cc
@@ -69,14 +69,6 @@ void tud_network_init_cb(void) {
 // TODO removed later since it is not part of tinyusb stack
 uint8_t tud_network_mac_address[6] = {0};
 
-//------------- NCM -------------//
-
-// callback to client providing optional indication of internal state of network
-// driver
-void tud_network_link_state_cb(bool state) {
-  (void)state;
-  // NoOp.
-}
-}
+}   // extern "C"
 
 #endif
diff --git a/test/hil/hil_hfp.json b/test/hil/hfp.json
index e79a3cfc9..e79a3cfc9 100644
--- a/test/hil/hil_hfp.json
+++ b/test/hil/hfp.json
diff --git a/test/hil/hil_ci_set_matrix.py b/test/hil/hil_ci_set_matrix.py
new file mode 100644
index 000000000..192174e16
--- /dev/null
+++ b/test/hil/hil_ci_set_matrix.py
@@ -0,0 +1,43 @@
+import argparse
+import json
+import os
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('config_file', help='Configuration JSON file')
+    args = parser.parse_args()
+
+    config_file = args.config_file
+
+    # if config file is not found, try to find it in the same directory as this script
+    if not os.path.exists(config_file):
+        config_file = os.path.join(os.path.dirname(__file__), config_file)
+    with open(config_file) as f:
+        config = json.load(f)
+
+    matrix = {
+        'arm-gcc': [],
+        'esp-idf': []
+    }
+    for board in config['boards']:
+        name = board['name']
+        if board['flasher'] == 'esptool':
+            toolchain = 'esp-idf'
+        else:
+            toolchain = 'arm-gcc'
+
+        if 'build' in board and 'flags_on' in board['build']:
+            for f in board['build']['flags_on']:
+                if f == '':
+                    matrix[toolchain].append(f'-b {name}')
+                else:
+                    matrix[toolchain].append(f'-b {name} -f1 {f.replace(" ", " -f1 ")}')
+        else:
+            matrix[toolchain].append(f'-b {name}')
+
+    print(json.dumps(matrix))
+
+
+if __name__ == '__main__':
+    main()
diff --git a/test/hil/hil_pi4.json b/test/hil/hil_pi4.json
deleted file mode 100644
index 8aff81910..000000000
--- a/test/hil/hil_pi4.json
+++ /dev/null
@@ -1,37 +0,0 @@
-{
-    "boards": [
-        {
-            "name": "raspberry_pi_pico",
-            "uid": "E6614C311B764A37",
-            "flasher": "openocd",
-            "flasher_sn": "E6614103E72C1D2F",
-            "flasher_args": "-f interface/cmsis-dap.cfg -f target/rp2040.cfg -c \"adapter speed 5000\""
-        },
-        {
-            "name": "espressif_s3_devkitm",
-            "uid": "84F703C084E4",
-            "tests": [
-                "cdc_msc_freertos", "hid_composite_freertos"
-            ],
-            "flasher": "esptool",
-            "flasher_sn": "3ea619acd1cdeb11a0a0b806e93fd3f1",
-            "flasher_args": "-b 1500000"
-        },
-        {
-            "name": "feather_nrf52840_express",
-            "uid": "1F0479CD0F764471",
-            "flasher": "jlink",
-            "flasher_sn": "000682804350",
-            "flasher_args": "-device nrf52840_xxaa"
-        },
-        {
-            "name": "itsybitsy_m4",
-            "uid": "D784B28C5338533335202020FF044726",
-            "flasher": "bossac",
-            "flashser_vendor": "Adafruit Industries",
-            "flasher_product": "ItsyBitsy M4 Express",
-            "flasher_reset_pin": "2",
-            "flasher_args": "--offset 0x4000"
-        }
-    ]
-}
diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py
index 7f03ce43f..0d7cae77e 100644..100755
--- a/test/hil/hil_test.py
+++ b/test/hil/hil_test.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 #
 # The MIT License (MIT)
 #
@@ -25,24 +26,30 @@
 # ACTION=="add", SUBSYSTEM=="tty", SUBSYSTEMS=="usb", MODE="0666", PROGRAM="/bin/sh -c 'echo $$ID_SERIAL_SHORT | rev | cut -c -8 | rev'", SYMLINK+="ttyUSB_%c.%s{bInterfaceNumber}"
 # ACTION=="add", SUBSYSTEM=="block", SUBSYSTEMS=="usb", ENV{ID_FS_USAGE}=="filesystem", MODE="0666", PROGRAM="/bin/sh -c 'echo $$ID_SERIAL_SHORT | rev | cut -c -8 | rev'", RUN{program}+="/usr/bin/systemd-mount --no-block --automount=yes --collect $devnode /media/blkUSB_%c.%s{bInterfaceNumber}"
 
+import argparse
 import os
+import re
 import sys
 import time
-import click
 import serial
 import subprocess
 import json
 import glob
+from multiprocessing import Pool
+import fs
 
-# for RPI double reset
-try:
-    import gpiozero
-except ImportError:
-    pass
+ENUM_TIMEOUT = 30
 
+STATUS_OK = "\033[32mOK\033[0m"
+STATUS_FAILED = "\033[31mFailed\033[0m"
+STATUS_SKIPPED = "\033[33mSkipped\033[0m"
 
-ENUM_TIMEOUT = 10
+verbose = False
 
+# -------------------------------------------------------------
+# Path
+# -------------------------------------------------------------
+OPENCOD_ADI_PATH = f'{os.getenv("HOME")}/app/openocd_adi'
 
 # get usb serial by id
 def get_serial_dev(id, vendor_str, product_str, ifnum):
@@ -58,9 +65,8 @@ def get_serial_dev(id, vendor_str, product_str, ifnum):
         return port_list[0]
 
 
-# Currently not used, left as reference
+# get usb disk by id
 def get_disk_dev(id, vendor_str, lun):
-    # get usb disk by id
     return f'/dev/disk/by-id/usb-{vendor_str}_Mass_Storage_{id}-0:{lun}'
 
 
@@ -77,113 +83,186 @@ def open_serial_dev(port):
                 # slight delay since kernel may occupy the port briefly
                 time.sleep(0.5)
                 timeout = timeout - 0.5
-                ser = serial.Serial(port, timeout=1)
+                ser = serial.Serial(port, timeout=5)
                 break
             except serial.SerialException:
                 pass
         time.sleep(0.5)
         timeout = timeout - 0.5
-    assert timeout, 'Device not available or Cannot open port'
+
+    assert timeout, f'Cannot open port f{port}' if os.path.exists(port) else f'Port {port} not existed'
     return ser
 
 
-def read_disk_file(id, fname):
-    # on different self-hosted, the mount point is different
-    file_list = [
-        f'/media/blkUSB_{id[-8:]}.02/{fname}',
-        f'/media/{os.getenv("USER")}/TinyUSB MSC/{fname}'
-    ]
+def read_disk_file(uid, lun, fname):
+    # open_fs("fat://{dev}) require 'pip install pyfatfs'
+    dev = get_disk_dev(uid, 'TinyUSB', lun)
     timeout = ENUM_TIMEOUT
     while timeout:
-        for file in file_list:
-            if os.path.isfile(file):
-                with open(file, 'rb') as f:
+        if os.path.exists(dev):
+            fat = fs.open_fs(f'fat://{dev}?read_only=true')
+            try:
+                with fat.open(fname, 'rb') as f:
                     data = f.read()
-                    return data
-
+            finally:
+                fat.close()
+            assert data, f'Cannot read file {fname} from {dev}'
+            return data
         time.sleep(1)
-        timeout = timeout - 1
+        timeout -= 1
 
-    assert timeout, 'Device not available'
+    assert timeout, f'Storage {dev} not existed'
     return None
 
 
 # -------------------------------------------------------------
 # Flashing firmware
 # -------------------------------------------------------------
+def run_cmd(cmd, cwd=None):
+    r = subprocess.run(cmd, cwd=cwd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    if r.returncode != 0:
+        title = f'COMMAND FAILED: {cmd}'
+        print()
+        if os.getenv('CI'):
+            print(f"::group::{title}")
+            print(r.stdout.decode("utf-8"))
+            print(f"::endgroup::")
+        else:
+            print(title)
+            print(r.stdout.decode("utf-8"))
+    elif verbose:
+        print(cmd)
+        print(r.stdout.decode("utf-8"))
+    return r
+
+
 def flash_jlink(board, firmware):
-    script = ['halt', 'r', f'loadfile {firmware}', 'r', 'go', 'exit']
-    with open('flash.jlink', 'w') as f:
+    script = ['halt', 'r', f'loadfile {firmware}.elf', 'r', 'go', 'exit']
+    f_jlink = f'{board["name"]}_{os.path.basename(firmware)}.jlink'
+    with open(f_jlink, 'w') as f:
         f.writelines(f'{s}\n' for s in script)
-    ret = subprocess.run(
-        f'JLinkExe -USB {board["flasher_sn"]} {board["flasher_args"]} -if swd -JTAGConf -1,-1 -speed auto -NoGui 1 -ExitOnError 1 -CommandFile flash.jlink',
-        shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    os.remove('flash.jlink')
+    ret = run_cmd(f'JLinkExe -USB {board["flasher_sn"]} {board["flasher_args"]} -if swd -JTAGConf -1,-1 -speed auto -NoGui 1 -ExitOnError 1 -CommandFile {f_jlink}')
+    os.remove(f_jlink)
+    return ret
+
+
+def flash_stlink(board, firmware):
+    ret = run_cmd(f'STM32_Programmer_CLI --connect port=swd sn={board["flasher_sn"]} --write {firmware}.elf --go')
+    return ret
+
+
+def flash_stflash(board, firmware):
+    ret = run_cmd(f'st-flash --serial {board["flasher_sn"]} write {firmware}.bin 0x8000000')
     return ret
 
 
 def flash_openocd(board, firmware):
-    ret = subprocess.run(
-        f'openocd -c "adapter serial {board["flasher_sn"]}" {board["flasher_args"]} -c "program {firmware} reset exit"',
-        shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    ret = run_cmd(f'openocd -c "adapter serial {board["flasher_sn"]}" {board["flasher_args"]} -c "program {firmware}.elf reset exit"')
+    return ret
+
+
+def flash_openocd_wch(board, firmware):
+    # Content of the wch-riscv.cfg file
+    cfg_content = """
+adapter driver wlinke
+adapter speed 6000
+transport select sdi
+
+wlink_set_address 0x00000000
+set _CHIPNAME wch_riscv
+sdi newtap $_CHIPNAME cpu -irlen 5 -expected-id 0x00001
+
+set _TARGETNAME $_CHIPNAME.cpu
+
+target create $_TARGETNAME.0 wch_riscv -chain-position $_TARGETNAME
+$_TARGETNAME.0 configure  -work-area-phys 0x20000000 -work-area-size 10000 -work-area-backup 1
+set _FLASHNAME $_CHIPNAME.flash
+
+flash bank $_FLASHNAME wch_riscv 0x00000000 0 0 0 $_TARGETNAME.0
+
+echo "Ready for Remote Connections"
+"""
+    f_wch = f"wch-riscv_{board['uid']}.cfg"
+    if not os.path.exists(f_wch):
+        with open(f_wch, 'w') as file:
+            file.write(cfg_content)
+
+    ret = run_cmd(f'openocd_wch -c "adapter serial {board["flasher_sn"]}" -f {f_wch} '
+                  f'-c "program {firmware}.elf reset exit"')
+    return ret
+
+
+def flash_openocd_adi(board, firmware):
+    ret = run_cmd(f'{OPENCOD_ADI_PATH}/src/openocd -c "adapter serial {board["flasher_sn"]}" -s {OPENCOD_ADI_PATH}/tcl '
+                  f'{board["flasher_args"]} -c "program {firmware}.elf reset exit"')
+    return ret
+
+def flash_wlink_rs(board, firmware):
+    # wlink use index for probe selection and lacking usb serial support
+    ret = run_cmd(f'wlink flash {firmware}.elf')
     return ret
 
 
 def flash_esptool(board, firmware):
     port = get_serial_dev(board["flasher_sn"], None, None, 0)
-    dir = os.path.dirname(firmware)
-    with open(f'{dir}/config.env') as f:
-        IDF_TARGET = json.load(f)['IDF_TARGET']
-    with open(f'{dir}/flash_args') as f:
+    fw_dir = os.path.dirname(f'{firmware}.bin')
+    with open(f'{fw_dir}/config.env') as f:
+        idf_target = json.load(f)['IDF_TARGET']
+    with open(f'{fw_dir}/flash_args') as f:
         flash_args = f.read().strip().replace('\n', ' ')
-    command = (f'esptool.py --chip {IDF_TARGET} -p {port} {board["flasher_args"]} '
+    command = (f'esptool.py --chip {idf_target} -p {port} {board["flasher_args"]} '
                f'--before=default_reset --after=hard_reset write_flash {flash_args}')
-    ret = subprocess.run(command, shell=True, cwd=dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    ret = run_cmd(command, cwd=fw_dir)
     return ret
 
 
-def doublereset_with_rpi_gpio(board):
-    # Off = 0 = Reset
-    led = gpiozero.LED(board["flasher_reset_pin"])
+def flash_uniflash(board, firmware):
+    ret = run_cmd(f'dslite.sh {board["flasher_args"]} -f {firmware}.hex')
+    return ret
 
-    led.off()
-    time.sleep(0.1)
-    led.on()
-    time.sleep(0.1)
-    led.off()
-    time.sleep(0.1)
-    led.on()
 
-def flash_bossac(board, firmware):
-    # double reset to enter bootloader
-    doublereset_with_rpi_gpio(board)
+# -------------------------------------------------------------
+# Tests: dual
+# -------------------------------------------------------------
 
-    port = get_serial_dev(board["uid"], board["flashser_vendor"], board["flasher_product"], 0)
-    timeout = ENUM_TIMEOUT
-    while timeout:
-        if os.path.exists(port):
-            break
+def test_dual_host_info_to_device_cdc(board):
+    uid = board['uid']
+    declared_devs = [f'{d["vid_pid"]}_{d["serial"]}' for d in board['tests']['dual_attached']]
+
+    port = get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0)
+    ser = open_serial_dev(port)
+    # read from cdc, first line should contain vid/pid and serial
+    data = ser.read(1000)
+    lines = data.decode('utf-8').splitlines()
+    enum_dev_sn = []
+    for l in lines:
+        vid_pid_sn = re.search(r'ID ([0-9a-fA-F]+):([0-9a-fA-F]+) SN (\w+)', l)
+        if vid_pid_sn:
+            print(f'\r\n  {l} ', end='')
+            enum_dev_sn.append(f'{vid_pid_sn.group(1)}_{vid_pid_sn.group(2)}_{vid_pid_sn.group(3)}')
+
+    if set(declared_devs) != set(enum_dev_sn):
+        # for pico/pico2 make this test optional
+        failed_msg = f'Enumerated devices {enum_dev_sn} not match with declared {declared_devs}'
+        if 'raspberry_pi_pico' in board['name']:
+            print(f'\r\n  {failed_msg} {STATUS_FAILED} ', end='')
         else:
-            time.sleep(0.5)
-            timeout = timeout - 0.5
-    assert timeout, 'bossac bootloader is not available'
-    # sleep a bit more for bootloader to be ready
-    time.sleep(0.5)
-    ret = subprocess.run(f'bossac --port {port} {board["flasher_args"]} -U -i -R -e -w {firmware}', shell=True, stdout=subprocess.PIPE,
-                         stderr=subprocess.STDOUT)
-    return ret
+            assert False, failed_msg
+    return 0
+
 
 # -------------------------------------------------------------
-# Tests
+# Tests: device
 # -------------------------------------------------------------
-def test_board_test(id):
+def test_device_board_test(board):
     # Dummy test
     pass
 
 
-def test_cdc_dual_ports(id):
-    port1 = get_serial_dev(id, 'TinyUSB', "TinyUSB_Device", 0)
-    port2 = get_serial_dev(id, 'TinyUSB', "TinyUSB_Device", 2)
+def test_device_cdc_dual_ports(board):
+    uid = board['uid']
+    port1 = get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0)
+    port2 = get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 2)
 
     ser1 = open_serial_dev(port1)
     ser2 = open_serial_dev(port2)
@@ -192,28 +271,29 @@ def test_cdc_dual_ports(id):
     str1 = b"test_no1"
     ser1.write(str1)
     ser1.flush()
-    assert ser1.read(100) == str1.lower(), 'Port1 wrong data'
-    assert ser2.read(100) == str1.upper(), 'Port2 wrong data'
+    assert ser1.read(len(str1)) == str1.lower(), 'Port1 wrong data'
+    assert ser2.read(len(str1)) == str1.upper(), 'Port2 wrong data'
 
     str2 = b"test_no2"
     ser2.write(str2)
     ser2.flush()
-    assert ser1.read(100) == str2.lower(), 'Port1 wrong data'
-    assert ser2.read(100) == str2.upper(), 'Port2 wrong data'
+    assert ser1.read(len(str2)) == str2.lower(), 'Port1 wrong data'
+    assert ser2.read(len(str2)) == str2.upper(), 'Port2 wrong data'
 
 
-def test_cdc_msc(id):
+def test_device_cdc_msc(board):
+    uid = board['uid']
     # Echo test
-    port = get_serial_dev(id, 'TinyUSB', "TinyUSB_Device", 0)
+    port = get_serial_dev(uid, 'TinyUSB', "TinyUSB_Device", 0)
     ser = open_serial_dev(port)
 
     str = b"test_str"
     ser.write(str)
     ser.flush()
-    assert ser.read(100) == str, 'CDC wrong data'
+    assert ser.read(len(str)) == str, 'CDC wrong data'
 
     # Block test
-    data = read_disk_file(id, 'README.TXT')
+    data = read_disk_file(uid,0,'README.TXT')
     readme = \
     b"This is tinyusb's MassStorage Class demo.\r\n\r\n\
 If you find any bugs or get any questions, feel free to file an\r\n\
@@ -222,57 +302,60 @@ issue at github.com/hathach/tinyusb"
     assert data == readme, 'MSC wrong data'
 
 
-def test_cdc_msc_freertos(id):
-    test_cdc_msc(id)
+def test_device_cdc_msc_freertos(board):
+    test_device_cdc_msc(board)
+
 
+def test_device_dfu(board):
+    uid = board['uid']
 
-def test_dfu(id):
     # Wait device enum
     timeout = ENUM_TIMEOUT
     while timeout:
-        ret = subprocess.run(f'dfu-util -l',
-                             shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        ret = run_cmd(f'dfu-util -l')
         stdout = ret.stdout.decode()
-        if f'serial="{id}"' in stdout and 'Found DFU: [cafe:4000]' in stdout:
+        if f'serial="{uid}"' in stdout and 'Found DFU: [cafe:4000]' in stdout:
             break
         time.sleep(1)
         timeout = timeout - 1
 
     assert timeout, 'Device not available'
 
+    f_dfu0 = f'dfu0_{uid}'
+    f_dfu1 = f'dfu1_{uid}'
+
     # Test upload
     try:
-        os.remove('dfu0')
-        os.remove('dfu1')
+        os.remove(f_dfu0)
+        os.remove(f_dfu1)
     except OSError:
         pass
 
-    ret = subprocess.run(f'dfu-util -S {id} -a 0 -U dfu0',
-                         shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    ret = run_cmd(f'dfu-util -S {uid} -a 0 -U {f_dfu0}')
     assert ret.returncode == 0, 'Upload failed'
 
-    ret = subprocess.run(f'dfu-util -S {id} -a 1 -U dfu1',
-                         shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    ret = run_cmd(f'dfu-util -S {uid} -a 1 -U {f_dfu1}')
     assert ret.returncode == 0, 'Upload failed'
 
-    with open('dfu0') as f:
+    with open(f_dfu0) as f:
         assert 'Hello world from TinyUSB DFU! - Partition 0' in f.read(), 'Wrong uploaded data'
 
-    with open('dfu1') as f:
+    with open(f_dfu1) as f:
         assert 'Hello world from TinyUSB DFU! - Partition 1' in f.read(), 'Wrong uploaded data'
 
-    os.remove('dfu0')
-    os.remove('dfu1')
+    os.remove(f_dfu0)
+    os.remove(f_dfu1)
+
 
+def test_device_dfu_runtime(board):
+    uid = board['uid']
 
-def test_dfu_runtime(id):
     # Wait device enum
     timeout = ENUM_TIMEOUT
     while timeout:
-        ret = subprocess.run(f'dfu-util -l',
-                             shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        ret = run_cmd(f'dfu-util -l')
         stdout = ret.stdout.decode()
-        if f'serial="{id}"' in stdout and 'Found Runtime: [cafe:4000]' in stdout:
+        if f'serial="{uid}"' in stdout and 'Found Runtime: [cafe:4000]' in stdout:
             break
         time.sleep(1)
         timeout = timeout - 1
@@ -280,10 +363,11 @@ def test_dfu_runtime(id):
     assert timeout, 'Device not available'
 
 
-def test_hid_boot_interface(id):
-    kbd = get_hid_dev(id, 'TinyUSB', 'TinyUSB_Device', 'event-kbd')
-    mouse1 = get_hid_dev(id, 'TinyUSB', 'TinyUSB_Device', 'if01-event-mouse')
-    mouse2 = get_hid_dev(id, 'TinyUSB', 'TinyUSB_Device', 'if01-mouse')
+def test_device_hid_boot_interface(board):
+    uid = board['uid']
+    kbd = get_hid_dev(uid, 'TinyUSB', 'TinyUSB_Device', 'event-kbd')
+    mouse1 = get_hid_dev(uid, 'TinyUSB', 'TinyUSB_Device', 'if01-event-mouse')
+    mouse2 = get_hid_dev(uid, 'TinyUSB', 'TinyUSB_Device', 'if01-mouse')
     # Wait device enum
     timeout = ENUM_TIMEOUT
     while timeout:
@@ -292,10 +376,10 @@ def test_hid_boot_interface(id):
         time.sleep(1)
         timeout = timeout - 1
 
-    assert timeout, 'Device not available'
+    assert timeout, 'HID device not available'
 
 
-def test_hid_composite_freertos(id):
+def test_device_hid_composite_freertos(id):
     # TODO implement later
     pass
 
@@ -303,82 +387,124 @@ def test_hid_composite_freertos(id):
 # -------------------------------------------------------------
 # Main
 # -------------------------------------------------------------
[email protected]()
[email protected]('config_file')
[email protected]('-b', '--board', multiple=True, default=None, help='Boards to test, all if not specified')
-def main(config_file, board):
-    """
-    Hardware test on specified boards
-    """
-    config_file = os.path.join(os.path.dirname(__file__), config_file)
-    with open(config_file) as f:
-        config = json.load(f)
-
-    # all possible tests
-    all_tests = [
-        'cdc_dual_ports', 'cdc_msc', 'dfu', 'dfu_runtime', 'hid_boot_interface',
-    ]
+# device tests
+device_tests = [
+    'device/cdc_dual_ports',
+    'device/cdc_msc',
+    'device/dfu',
+    'device/cdc_msc_freertos',  # don't test 2 cdc_msc next to each other
+    'device/dfu_runtime',
+    'device/hid_boot_interface',
+]
 
-    if len(board) == 0:
-        config_boards = config['boards']
-    else:
-        config_boards = [e for e in config['boards'] if e['name'] in board]
+dual_tests = [
+    'dual/host_info_to_device_cdc',
+]
 
-    for item in config_boards:
-        print(f'Testing board:{item["name"]}')
-        flasher = item['flasher'].lower()
 
-        # default to all tests
-        if 'tests' in item:
-            test_list = item['tests']
-        else:
-            test_list = all_tests
+def test_board(board):
+    name = board['name']
+    flasher = board['flasher'].lower()
 
-        # board_test is added last to disable board's usb
-        test_list.append('board_test')
+    # default to all tests
+    test_list = list(device_tests)
 
-        # remove skip_tests
-        if 'tests_skip' in item:
-            for skip in item['tests_skip']:
+    if 'tests' in board:
+        board_tests = board['tests']
+        if 'dual_attached' in board_tests:
+            test_list += dual_tests
+        if 'only' in board_tests:
+            test_list = board_tests['only']
+        if 'skip' in board_tests:
+            for skip in board_tests['skip']:
                 if skip in test_list:
                     test_list.remove(skip)
+                    print(f'{name:25} {skip:30} ... Skip')
 
-        for test in test_list:
-            fw_list = [
-                # cmake: esp32 & samd51 use .bin file
-                f'cmake-build/cmake-build-{item["name"]}/device/{test}/{test}.elf',
-                f'cmake-build/cmake-build-{item["name"]}/device/{test}/{test}.bin',
-                # make
-                f'examples/device/{test}/_build/{item["name"]}/{test}.elf'
-            ]
+    # board_test is added last to disable board's usb
+    test_list.append('device/board_test')
 
-            fw = None
-            for f in fw_list:
-                if os.path.isfile(f):
-                    fw = f
-                    break
+    err_count = 0
+    flags_on_list = [""]
+    if 'build' in board and 'flags_on' in board['build']:
+        flags_on_list = board['build']['flags_on']
 
-            if fw is None:
-                print(f'Cannot find binary file for {test}')
-                sys.exit(-1)
+    for f1 in flags_on_list:
+        f1_str = ""
+        if f1 != "":
+            f1_str = '-' + f1.replace(' ', '-')
+        for test in test_list:
+            fw_dir = f'cmake-build/cmake-build-{name}{f1_str}/{test}'
+            if not os.path.exists(fw_dir):
+                fw_dir = f'examples/cmake-build-{name}{f1_str}/{test}'
+            fw_name = f'{fw_dir}/{os.path.basename(test)}'
+            print(f'{name+f1_str:40} {test:30} ... ', end='')
 
-            print(f'  {test} ...', end='')
+            if not os.path.exists(fw_dir):
+                print('Skip (no binary)')
+                continue
 
             # flash firmware. It may fail randomly, retry a few times
             for i in range(3):
-                ret = globals()[f'flash_{flasher}'](item, fw)
+                ret = globals()[f'flash_{flasher}'](board, fw_name)
                 if ret.returncode == 0:
                     break
                 else:
                     print(f'Flashing failed, retry {i+1}')
                     time.sleep(1)
 
-            assert ret.returncode == 0, 'Flash failed\n' + ret.stdout.decode()
+            if ret.returncode == 0:
+                try:
+                    ret = globals()[f'test_{test.replace("/", "_")}'](board)
+                    print('OK')
+                except Exception as e:
+                    err_count += 1
+                    print(STATUS_FAILED)
+                    print(f'  {e}')
+            else:
+                err_count += 1
+                print(f'Flash {STATUS_FAILED}')
+    return err_count
+
+
+def main():
+    """
+    Hardware test on specified boards
+    """
+    global verbose
+
+    duration = time.time()
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('config_file', help='Configuration JSON file')
+    parser.add_argument('-b', '--board', action='append', default=[], help='Boards to test, all if not specified')
+    parser.add_argument('-v', '--verbose', action='store_true', help='Verbose output')
+    args = parser.parse_args()
+
+    config_file = args.config_file
+    boards = args.board
+    verbose = args.verbose
+
+    # if config file is not found, try to find it in the same directory as this script
+    if not os.path.exists(config_file):
+        config_file = os.path.join(os.path.dirname(__file__), config_file)
+    with open(config_file) as f:
+        config = json.load(f)
+
+    if len(boards) == 0:
+        config_boards = config['boards']
+    else:
+        config_boards = [e for e in config['boards'] if e['name'] in boards]
 
-            # run test
-            globals()[f'test_{test}'](item['uid'])
+    with Pool(processes=os.cpu_count()) as pool:
+        err_count = sum(pool.map(test_board, config_boards))
 
-            print('OK')
+    duration = time.time() - duration
+    print()
+    print("-" * 30)
+    print(f'Total failed: {err_count} in {duration:.1f}s')
+    print("-" * 30)
+    sys.exit(err_count)
 
 
 if __name__ == '__main__':
diff --git a/test/hil/requirements.txt b/test/hil/requirements.txt
new file mode 100644
index 000000000..c33980c9d
--- /dev/null
+++ b/test/hil/requirements.txt
@@ -0,0 +1,2 @@
+fs
+pyfatfs
diff --git a/test/hil/tinyusb.json b/test/hil/tinyusb.json
new file mode 100644
index 000000000..2313f5d13
--- /dev/null
+++ b/test/hil/tinyusb.json
@@ -0,0 +1,129 @@
+{
+    "boards": [
+        {
+            "name": "espressif_s3_devkitm",
+            "uid": "84F703C084E4",
+            "build" : {
+                "flags_on": ["", "CFG_TUD_DWC2_DMA"]
+            },
+            "tests": {
+                "only": ["device/cdc_msc_freertos", "device/hid_composite_freertos"]
+            },
+            "flasher": "esptool",
+            "flasher_sn": "3ea619acd1cdeb11a0a0b806e93fd3f1",
+            "flasher_args": "-b 1500000"
+        },
+        {
+            "name": "feather_nrf52840_express",
+            "uid": "1F0479CD0F764471",
+            "flasher": "jlink",
+            "flasher_sn": "000682804350",
+            "flasher_args": "-device nrf52840_xxaa"
+        },
+        {
+            "name": "max32666fthr",
+            "uid": "0C81464124010B20FF0A08CC2C",
+            "flasher": "openocd_adi",
+            "flasher_sn": "E6614C311B597D32",
+            "flasher_args": "-f interface/cmsis-dap.cfg -f target/max32665.cfg"
+        },
+        {
+            "name": "metro_m4_express",
+            "uid": "9995AD485337433231202020FF100A34",
+            "flasher": "jlink",
+            "flasher_sn": "123456",
+            "flasher_args": "-device ATSAMD51J19",
+            "tests": {
+                "dual_attached": [{"vid_pid": "1a86_55d4", "serial": "52D2002130"}]
+            }
+        },
+        {
+            "name": "lpcxpresso11u37",
+            "uid": "17121919",
+            "flasher": "jlink",
+            "flasher_sn": "000724441579",
+            "flasher_args": "-device LPC11U37/401"
+        },
+        {
+            "name": "ra4m1_ek",
+            "uid": "152E163038303131393346E46F26574B",
+            "tests": {
+                "skip": ["device/cdc_msc", "device/cdc_msc_freertos"]
+            },
+            "comment": "MSC is slow to enumerated #2602",
+            "flasher": "jlink",
+            "flasher_sn": "000831174392",
+            "flasher_args": "-device R7FA4M1AB"
+        },
+        {
+            "name": "raspberry_pi_pico",
+            "uid": "E6614C311B764A37",
+            "flasher": "openocd",
+            "flasher_sn": "E6614103E72C1D2F",
+            "flasher_args": "-f interface/cmsis-dap.cfg -f target/rp2040.cfg -c \"adapter speed 5000\"",
+            "tests": {
+                "dual_attached": [{"vid_pid": "1a86_55d4", "serial": "52D2002470"}]
+            }
+        },
+        {
+            "name": "raspberry_pi_pico2",
+            "uid": "560AE75E1C7152C9",
+            "flasher": "openocd",
+            "flasher_sn": "E6633861A3978538",
+            "flasher_args": "-f interface/cmsis-dap.cfg -f target/rp2350.cfg -c \"adapter speed 5000\"",
+            "tests": {
+                "dual_attached": [{"vid_pid": "1a86_55d4", "serial": "533D004242"}]
+            }
+        },
+        {
+            "name": "stm32f072disco",
+            "uid": "3A001A001357364230353532",
+            "flasher": "jlink",
+            "flasher_sn": "779541626",
+            "flasher_args": "-device stm32f072rb"
+        },
+        {
+            "name": "stm32f407disco",
+            "uid": "30001A000647313332353735",
+            "build" : {
+                "flags_on": ["", "CFG_TUD_DWC2_DMA"]
+            },
+            "flasher": "jlink",
+            "flasher_sn": "000773661813",
+            "flasher_args": "-device stm32f407vg"
+        },
+        {
+            "name": "stm32g0b1nucleo",
+            "uid": "4D0038000450434E37343120",
+            "flasher": "openocd",
+            "flasher_sn": "066FFF495087534867063844",
+            "flasher_args": "-f interface/stlink.cfg -f target/stm32g0x.cfg"
+        }
+    ],
+    "boards-skip": [
+        {
+            "name": "stm32f769disco",
+            "uid": "21002F000F51363531383437",
+            "build" : {
+                "flags_on": ["", "CFG_TUD_DWC2_DMA"]
+            },
+            "flasher": "jlink",
+            "flasher_sn": "000778170924",
+            "flasher_args": "-device stm32f769ni"
+        },
+        {
+            "name": "mimxrt1015_evk",
+            "uid": "DC28F865D2111D228D00B0543A70463C",
+            "flasher": "jlink",
+            "flasher_sn": "000726284213",
+            "flasher_args": "-device MIMXRT1015DAF5A"
+        },
+        {
+            "name": "nanoch32v203",
+            "uid": "CDAB277B0FBC03E339E339E3",
+            "flasher": "openocd_wch",
+            "flasher_sn": "EBCA8F0670AF",
+            "flasher_args": ""
+        }
+    ]
+}
diff --git a/test/unit-test/test/device/msc/test_msc_device.c b/test/unit-test/test/device/msc/test_msc_device.c
index a62418524..f2a4fa295 100644
--- a/test/unit-test/test/device/msc/test_msc_device.c
+++ b/test/unit-test/test/device/msc/test_msc_device.c
@@ -197,10 +197,14 @@ void setUp(void)
   dcd_int_disable_Ignore();
   dcd_int_enable_Ignore();
 
-  if ( !tud_inited() )
-  {
-    dcd_init_Expect(rhport);
-    tusb_init();
+  if ( !tud_inited() ) {
+    tusb_rhport_init_t dev_init = {
+      .role = TUSB_ROLE_DEVICE,
+      .speed = TUSB_SPEED_AUTO
+    };
+
+    dcd_init_ExpectAndReturn(0, &dev_init, true);
+    tusb_init(0, &dev_init);
   }
 
   dcd_event_bus_reset(rhport, TUSB_SPEED_HIGH, false);
diff --git a/test/unit-test/test/device/usbd/test_usbd.c b/test/unit-test/test/device/usbd/test_usbd.c
index 00950c5a9..9879cd4ba 100644
--- a/test/unit-test/test/device/usbd/test_usbd.c
+++ b/test/unit-test/test/device/usbd/test_usbd.c
@@ -102,38 +102,38 @@ uint8_t const* desc_configuration;
 //--------------------------------------------------------------------+
 //
 //--------------------------------------------------------------------+
-uint8_t const * tud_descriptor_device_cb(void)
-{
+uint8_t const * tud_descriptor_device_cb(void) {
   return desc_device;
 }
 
-uint8_t const * tud_descriptor_configuration_cb(uint8_t index)
-{
+uint8_t const * tud_descriptor_configuration_cb(uint8_t index) {
   return desc_configuration;
 }
 
-uint16_t const* tud_descriptor_string_cb(uint8_t index, uint16_t langid)
-{
+uint16_t const* tud_descriptor_string_cb(uint8_t index, uint16_t langid) {
   (void) langid;
 
   return NULL;
 }
 
-void setUp(void)
-{
+void setUp(void) {
   dcd_int_disable_Ignore();
   dcd_int_enable_Ignore();
 
-  if ( !tud_inited() )
-  {
+  if ( !tud_inited() ) {
+    tusb_rhport_init_t dev_init = {
+      .role = TUSB_ROLE_DEVICE,
+      .speed = TUSB_SPEED_AUTO
+    };
+
     mscd_init_Expect();
-    dcd_init_Expect(rhport);
-    tusb_init();
+    dcd_init_ExpectAndReturn(0, &dev_init, true);
+
+    tusb_init(0, &dev_init);
   }
 }
 
-void tearDown(void)
-{
+void tearDown(void) {
 }
 
 //--------------------------------------------------------------------+
diff --git a/test/unit-test/vendor/ceedling/vendor/unity/auto/unity_test_summary.py b/test/unit-test/vendor/ceedling/vendor/unity/auto/unity_test_summary.py
index 00c0da8cc..a8e5b2726 100644..100755
--- a/test/unit-test/vendor/ceedling/vendor/unity/auto/unity_test_summary.py
+++ b/test/unit-test/vendor/ceedling/vendor/unity/auto/unity_test_summary.py
@@ -1,4 +1,4 @@
-#! python3
+#!/usr/bin/env python3
 # ==========================================
 #   Unity Project - A Test Framework for C
 #   Copyright (c) 2015 Alexander Mueller / [email protected]
diff --git a/test/unit-test/vendor/ceedling/vendor/unity/auto/unity_to_junit.py b/test/unit-test/vendor/ceedling/vendor/unity/auto/unity_to_junit.py
index 71dd56888..2379ad49a 100644..100755
--- a/test/unit-test/vendor/ceedling/vendor/unity/auto/unity_to_junit.py
+++ b/test/unit-test/vendor/ceedling/vendor/unity/auto/unity_to_junit.py
@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 import sys
 import os
 from glob import glob
diff --git a/test/unit-test/vendor/ceedling/vendor/unity/src/unity_internals.h b/test/unit-test/vendor/ceedling/vendor/unity/src/unity_internals.h
index 2c91b6db1..d66309f96 100644
--- a/test/unit-test/vendor/ceedling/vendor/unity/src/unity_internals.h
+++ b/test/unit-test/vendor/ceedling/vendor/unity/src/unity_internals.h
@@ -632,14 +632,14 @@ void UnityAssertNumbersArrayWithin(const UNITY_UINT delta,
                                    const UNITY_FLAGS_T flags);
 
 #ifndef UNITY_EXCLUDE_SETJMP_H
-UNITY_NORETURN void UnityFail(const char* message, const UNITY_LINE_TYPE line);
-UNITY_NORETURN void UnityIgnore(const char* message, const UNITY_LINE_TYPE line);
+UNITY_NORETURN void UnityFail(const char* msg, const UNITY_LINE_TYPE line);
+UNITY_NORETURN void UnityIgnore(const char* msg, const UNITY_LINE_TYPE line);
 #else
-void UnityFail(const char* message, const UNITY_LINE_TYPE line);
-void UnityIgnore(const char* message, const UNITY_LINE_TYPE line);
+void UnityFail(const char* msg, const UNITY_LINE_TYPE line);
+void UnityIgnore(const char* msg, const UNITY_LINE_TYPE line);
 #endif
 
-void UnityMessage(const char* message, const UNITY_LINE_TYPE line);
+void UnityMessage(const char* msg, const UNITY_LINE_TYPE line);
 
 #ifndef UNITY_EXCLUDE_FLOAT
 void UnityAssertFloatsWithin(const UNITY_FLOAT delta,