Merge pull request #2356 from hathach/hil-esp32s3

update esp32 to also be supported by hil test

18 files changed, 103 insertions, 36 deletions

diff --git a/.github/workflows/build_esp.yml b/.github/workflows/build_esp.yml
index 897616f09..4568695c2 100644
--- a/.github/workflows/build_esp.yml
+++ b/.github/workflows/build_esp.yml
@@ -32,7 +32,7 @@ jobs:
         # ESP32-S2
         - 'espressif_kaluga_1'
         # ESP32-S3
-        - 'espressif_s3_devkitm'
+        - 'espressif_s3_devkitc'
 
     steps:
     - name: Setup Python
diff --git a/examples/device/audio_4_channel_mic/skip.txt b/examples/device/audio_4_channel_mic/skip.txt
index 3c42a96d9..157605df1 100644
--- a/examples/device/audio_4_channel_mic/skip.txt
+++ b/examples/device/audio_4_channel_mic/skip.txt
@@ -2,3 +2,4 @@ mcu:SAMD11
 mcu:SAME5X
 mcu:SAMG
 family:broadcom_64bit
+family:espressif
diff --git a/examples/device/audio_test/skip.txt b/examples/device/audio_test/skip.txt
index 1ee86a485..65b137814 100644
--- a/examples/device/audio_test/skip.txt
+++ b/examples/device/audio_test/skip.txt
@@ -1,3 +1,4 @@
 mcu:SAMD11
 mcu:SAME5X
 mcu:SAMG
+family:espressif
diff --git a/examples/device/audio_test_multi_rate/skip.txt b/examples/device/audio_test_multi_rate/skip.txt
index 1ee86a485..65b137814 100644
--- a/examples/device/audio_test_multi_rate/skip.txt
+++ b/examples/device/audio_test_multi_rate/skip.txt
@@ -1,3 +1,4 @@
 mcu:SAMD11
 mcu:SAME5X
 mcu:SAMG
+family:espressif
diff --git a/examples/device/board_test/src/tusb_config.h b/examples/device/board_test/src/tusb_config.h
index 2c2eb5280..1a27cc87f 100644
--- a/examples/device/board_test/src/tusb_config.h
+++ b/examples/device/board_test/src/tusb_config.h
@@ -48,6 +48,11 @@
   #define CFG_TUSB_OS               OPT_OS_NONE
 #endif
 
+// Espressif IDF requires "freertos/" prefix in include path
+#if TU_CHECK_MCU(OPT_MCU_ESP32S2, OPT_MCU_ESP32S3)
+#define CFG_TUSB_OS_INC_PATH  freertos/
+#endif
+
 // This example only test LED & GPIO, disable both device and host stack
 #define CFG_TUD_ENABLED   0
 #define CFG_TUH_ENABLED   0
diff --git a/examples/device/cdc_msc/skip.txt b/examples/device/cdc_msc/skip.txt
index eadb6e74a..833fd072c 100644
--- a/examples/device/cdc_msc/skip.txt
+++ b/examples/device/cdc_msc/skip.txt
@@ -1 +1,2 @@
 mcu:SAMD11
+family:espressif
diff --git a/examples/device/cdc_msc_freertos/src/msc_disk.c b/examples/device/cdc_msc_freertos/src/msc_disk.c
index 9520dfec1..c8a04bb74 100644
--- a/examples/device/cdc_msc_freertos/src/msc_disk.c
+++ b/examples/device/cdc_msc_freertos/src/msc_disk.c
@@ -184,7 +184,7 @@ int32_t tud_msc_read10_cb(uint8_t lun, uint32_t lba, uint32_t offset, void* buff
   uint8_t const* addr = msc_disk[lba] + offset;
   memcpy(buffer, addr, bufsize);
 
-  return bufsize;
+  return (int32_t) bufsize;
 }
 
 // Callback invoked when received WRITE10 command.
@@ -203,7 +203,7 @@ int32_t tud_msc_write10_cb(uint8_t lun, uint32_t lba, uint32_t offset, uint8_t*
   (void) lba; (void) offset; (void) buffer;
 #endif
 
-  return bufsize;
+  return (int32_t) bufsize;
 }
 
 // Callback invoked when received an SCSI command not in built-in list below
@@ -237,14 +237,14 @@ int32_t tud_msc_scsi_cb (uint8_t lun, uint8_t const scsi_cmd[16], void* buffer,
   {
     if(in_xfer)
     {
-      memcpy(buffer, response, resplen);
+      memcpy(buffer, response, (size_t) resplen);
     }else
     {
       // SCSI output
     }
   }
 
-  return resplen;
+  return (int32_t) resplen;
 }
 
 #endif
diff --git a/examples/device/dfu/skip.txt b/examples/device/dfu/skip.txt
index 9ac346bad..9dde06c30 100644
--- a/examples/device/dfu/skip.txt
+++ b/examples/device/dfu/skip.txt
@@ -1,2 +1,3 @@
 mcu:TM4C123
 mcu:BCM2835
+family:espressif
diff --git a/examples/device/dynamic_configuration/skip.txt b/examples/device/dynamic_configuration/skip.txt
index eadb6e74a..833fd072c 100644
--- a/examples/device/dynamic_configuration/skip.txt
+++ b/examples/device/dynamic_configuration/skip.txt
@@ -1 +1,2 @@
 mcu:SAMD11
+family:espressif
diff --git a/examples/device/msc_dual_lun/skip.txt b/examples/device/msc_dual_lun/skip.txt
index 47e561cf0..a9e3a99b1 100644
--- a/examples/device/msc_dual_lun/skip.txt
+++ b/examples/device/msc_dual_lun/skip.txt
@@ -1,2 +1,3 @@
 mcu:SAMD11
 mcu:MKL25ZXX
+family:espressif
diff --git a/examples/device/net_lwip_webserver/skip.txt b/examples/device/net_lwip_webserver/skip.txt
index e3a726a2b..75aa2ef14 100644
--- a/examples/device/net_lwip_webserver/skip.txt
+++ b/examples/device/net_lwip_webserver/skip.txt
@@ -9,3 +9,4 @@ family:broadcom_64bit
 family:broadcom_32bit
 board:curiosity_nano
 board:frdm_kl25z
+family:espressif
diff --git a/examples/device/uac2_headset/skip.txt b/examples/device/uac2_headset/skip.txt
index 70d8e8838..a2a76af0e 100644
--- a/examples/device/uac2_headset/skip.txt
+++ b/examples/device/uac2_headset/skip.txt
@@ -5,3 +5,4 @@ mcu:SAMD11
 mcu:SAME5X
 mcu:SAMG
 board:stm32l052dap52
+family:espressif
diff --git a/examples/device/usbtmc/skip.txt b/examples/device/usbtmc/skip.txt
index a43106cf0..ccff857ac 100644
--- a/examples/device/usbtmc/skip.txt
+++ b/examples/device/usbtmc/skip.txt
@@ -1 +1,2 @@
 mcu:BCM2835
+family:espressif
diff --git a/examples/device/video_capture/skip.txt b/examples/device/video_capture/skip.txt
index 5898664a2..db64cc639 100644
--- a/examples/device/video_capture/skip.txt
+++ b/examples/device/video_capture/skip.txt
@@ -1,3 +1,4 @@
 mcu:MSP430x5xx
 mcu:NUC121
 mcu:SAMD11
+family:espressif
diff --git a/hw/bsp/espressif/boards/espressif_s3_devkitc/board.h b/hw/bsp/espressif/boards/espressif_s3_devkitc/board.h
index fe33b5c43..a319fbc61 100644
--- a/hw/bsp/espressif/boards/espressif_s3_devkitc/board.h
+++ b/hw/bsp/espressif/boards/espressif_s3_devkitc/board.h
@@ -36,6 +36,14 @@
 #define BUTTON_PIN            0
 #define BUTTON_STATE_ACTIVE   0
 
+// SPI for USB host shield
+#define MAX3421_SPI_HOST  SPI2_HOST
+#define MAX3421_SCK_PIN  39
+#define MAX3421_MOSI_PIN 42
+#define MAX3421_MISO_PIN 21
+#define MAX3421_CS_PIN   15
+#define MAX3421_INTR_PIN 14
+
 #ifdef __cplusplus
  }
 #endif
diff --git a/hw/bsp/espressif/boards/family.c b/hw/bsp/espressif/boards/family.c
index 2029295dc..c32ccbc84 100644
--- a/hw/bsp/espressif/boards/family.c
+++ b/hw/bsp/espressif/boards/family.c
@@ -28,6 +28,7 @@
 #include "board.h"
 
 #include "esp_rom_gpio.h"
+#include "esp_mac.h"
 #include "hal/gpio_ll.h"
 #include "hal/usb_hal.h"
 #include "soc/usb_periph.h"
@@ -149,6 +150,12 @@ static void configure_pins(usb_hal_context_t* usb) {
 // Board porting API
 //--------------------------------------------------------------------+
 
+size_t board_get_unique_id(uint8_t id[], size_t max_len) {
+  // use factory default MAC as serial ID
+  esp_efuse_mac_get_default(id);
+  return 6;
+}
+
 void board_led_write(bool state) {
 #ifdef NEOPIXEL_PIN
   strip->set_pixel(strip, 0, (state ? 0x88 : 0x00), 0x00, 0x00);
diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py
index 5f8425549..b307a45f6 100644
--- a/test/hil/hil_test.py
+++ b/test/hil/hil_test.py
@@ -31,13 +31,21 @@ import time
 import serial
 import subprocess
 import json
+import glob
 
 ENUM_TIMEOUT = 10
 
 
+# get usb serial by id
 def get_serial_dev(id, vendor_str, product_str, ifnum):
-    # get usb serial by id
-    return f'/dev/serial/by-id/usb-{vendor_str}_{product_str}_{id}-if{ifnum:02d}'
+    if vendor_str and product_str:
+        # known vendor and product
+        return f'/dev/serial/by-id/usb-{vendor_str}_{product_str}_{id}-if{ifnum:02d}'
+    else:
+        # just use id: mostly for cp210x/ftdi debugger
+        pattern = f'/dev/serial/by-id/usb-*_{id}-if{ifnum:02d}*'
+        port_list = glob.glob(pattern)
+        return port_list[0]
 
 
 # Currently not used, left as reference
@@ -57,13 +65,14 @@ def open_serial_dev(port):
         if os.path.exists(port):
             try:
                 # slight delay since kernel may occupy the port briefly
-                time.sleep(0.2)
+                time.sleep(0.5)
+                timeout = timeout - 0.5
                 ser = serial.Serial(port, timeout=1)
                 break
             except serial.SerialException:
                 pass
-        time.sleep(0.8)
-        timeout = timeout - 1
+        time.sleep(0.5)
+        timeout = timeout - 0.5
     assert timeout, 'Device not available or Cannot open port'
     return ser
 
@@ -92,23 +101,35 @@ def read_disk_file(id, fname):
 # -------------------------------------------------------------
 # Flash with debugger
 # -------------------------------------------------------------
-def flash_jlink(sn, dev, firmware):
+def flash_jlink(board, firmware):
     script = ['halt', 'r', f'loadfile {firmware}', 'r', 'go', 'exit']
-    f = open('flash.jlink', 'w')
-    f.writelines(f'{s}\n' for s in script)
-    f.close()
-    ret = subprocess.run(f'JLinkExe -USB {sn} -device {dev} -if swd -JTAGConf -1,-1 -speed auto -NoGui 1 -ExitOnError 1 -CommandFile flash.jlink',
-                         shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    stdout = ret.stdout.decode()
+    with open('flash.jlink', 'w') as f:
+        f.writelines(f'{s}\n' for s in script)
+    ret = subprocess.run(
+        f'JLinkExe -USB {board["debugger_sn"]} -device {board["cpu"]} -if swd -JTAGConf -1,-1 -speed auto -NoGui 1 -ExitOnError 1 -CommandFile flash.jlink',
+        shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
     os.remove('flash.jlink')
-    assert ret.returncode == 0, 'Flash failed\n' + stdout
+    return ret
 
 
-def flash_openocd(sn, args, firmware):
-    ret = subprocess.run(f'openocd {args} -c "program {firmware} reset exit"',
-                         shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    stdout = ret.stdout.decode()
-    assert ret.returncode == 0, 'Flash failed\n' + stdout
+def flash_openocd(board, firmware):
+    ret = subprocess.run(
+        f'openocd -c "adapter serial {board["debugger_sn"]}" {board["debugger_args"]} -c "program {firmware} reset exit"',
+        shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    return ret
+
+
+def flash_esptool(board, firmware):
+    port = get_serial_dev(board["debugger_sn"], None, None, 0)
+    dir = os.path.dirname(firmware)
+    with open(f'{dir}/config.env') as f:
+        IDF_TARGET = json.load(f)['IDF_TARGET']
+    with open(f'{dir}/flash_args') as f:
+        flash_args = f.read().strip().replace('\n', ' ')
+    command = (f'esptool.py --chip {IDF_TARGET} -p {port} {board["debugger_args"]} '
+               f'--before=default_reset --after=hard_reset write_flash {flash_args}')
+    ret = subprocess.run(command, shell=True, cwd=dir, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    return ret
 
 
 # -------------------------------------------------------------
@@ -160,12 +181,16 @@ issue at github.com/hathach/tinyusb"
     assert data == readme, 'MSC wrong data'
 
 
+def test_cdc_msc_freertos(id):
+    test_cdc_msc(id)
+
+
 def test_dfu(id):
     # Wait device enum
     timeout = ENUM_TIMEOUT
     while timeout:
         ret = subprocess.run(f'dfu-util -l',
-                         shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+                             shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
         stdout = ret.stdout.decode()
         if f'serial="{id}"' in stdout and 'Found DFU: [cafe:4000]' in stdout:
             break
@@ -190,10 +215,10 @@ def test_dfu(id):
     assert ret.returncode == 0, 'Upload failed'
 
     with open('dfu0') as f:
-        assert 'Hello world from TinyUSB DFU! - Partition 0' in f.read(),  'Wrong uploaded data'
+        assert 'Hello world from TinyUSB DFU! - Partition 0' in f.read(), 'Wrong uploaded data'
 
     with open('dfu1') as f:
-        assert 'Hello world from TinyUSB DFU! - Partition 1' in f.read(),  'Wrong uploaded data'
+        assert 'Hello world from TinyUSB DFU! - Partition 1' in f.read(), 'Wrong uploaded data'
 
     os.remove('dfu0')
     os.remove('dfu1')
@@ -204,7 +229,7 @@ def test_dfu_runtime(id):
     timeout = ENUM_TIMEOUT
     while timeout:
         ret = subprocess.run(f'dfu-util -l',
-                         shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+                             shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
         stdout = ret.stdout.decode()
         if f'serial="{id}"' in stdout and 'Found Runtime: [cafe:4000]' in stdout:
             break
@@ -229,6 +254,14 @@ def test_hid_boot_interface(id):
     assert timeout, 'Device not available'
 
 
+def test_hid_composite_freertos(id):
+    # TODO implement later
+    pass
+
+
+# -------------------------------------------------------------
+# Main
+# -------------------------------------------------------------
 if __name__ == '__main__':
     if len(sys.argv) != 2:
         print('Usage:')
@@ -238,10 +271,9 @@ if __name__ == '__main__':
     with open(f'{os.path.dirname(__file__)}/{sys.argv[1]}') as f:
         config = json.load(f)
 
-    # all possible tests, board_test is last to disable board's usb
+    # all possible tests
     all_tests = [
         'cdc_dual_ports', 'cdc_msc', 'dfu', 'dfu_runtime', 'hid_boot_interface',
-        'board_test'
     ]
 
     for board in config['boards']:
@@ -254,6 +286,9 @@ if __name__ == '__main__':
         else:
             test_list = all_tests
 
+        # board_test is added last to disable board's usb
+        test_list.append('board_test')
+
         # remove skip_tests
         if 'tests_skip' in board:
             for skip in board['tests_skip']:
@@ -278,13 +313,13 @@ if __name__ == '__main__':
                 print(f'Cannot find firmware file for {test}')
                 sys.exit(-1)
 
-            if debugger == 'jlink':
-                flash_jlink(board['debugger_sn'], board['cpu'], elf)
-            elif debugger == 'openocd':
-                flash_openocd(board['debugger_sn'], board['debugger_args'], elf)
-            else:
-                # ToDo
-                pass
             print(f'  {test} ...', end='')
+
+            # flash firmware
+            ret = locals()[f'flash_{debugger}'](board, elf)
+            assert ret.returncode == 0, 'Flash failed\n' + ret.stdout.decode()
+
+            # run test
             locals()[f'test_{test}'](board['uid'])
+
             print('OK')
diff --git a/tools/build_esp32.py b/tools/build_esp32.py
index 1f73d3b22..951467c23 100644
--- a/tools/build_esp32.py
+++ b/tools/build_esp32.py
@@ -30,6 +30,7 @@ def filter_with_input(mylist):
 # Build all examples if not specified
 all_examples = [entry.replace('examples/', '') for entry in glob.glob("examples/*/*_freertos")]
 filter_with_input(all_examples)
+all_examples.append('device/board_test')
 all_examples.sort()
 
 # Build all boards if not specified