More ci tweak (#2636)

* change concurrency group to ${{ github.workflow }}-${{ github.ref }}
* use argparse for build.py hil_test.py, remove the need to install click
* move ci win/mac to build_cmake.yml
* rename build_family.yml to build_util.yml
* build_util.yml support esp32
* integrate build-espressif into build.yml
* build.py support make with --board option
* add get_deps action
* update hil test to reuse action

1 files changed, 89 insertions, 32 deletions

diff --git a/.github/workflows/hil_test.yml b/.github/workflows/hil_test.yml
index 024ab969d..796ff32dc 100644
--- a/.github/workflows/hil_test.yml
+++ b/.github/workflows/hil_test.yml
@@ -24,15 +24,18 @@ on:
       - '.github/actions/**'
       - '.github/workflows/hil_test.yml'
 concurrency:
-  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
+  # ---------------------------------------
+  # Build Non Espressif
+  # ---------------------------------------
   build:
     if: github.repository_owner == 'hathach'
     runs-on: ubuntu-latest
     outputs:
-      BOARD_LIST: ${{ steps.parse_hil_json.outputs.BOARD_LIST }}
+      BOARDS_LIST: ${{ steps.parse_hil_json.outputs.BOARDS_LIST }}
     steps:
       - name: Checkout TinyUSB
         uses: actions/checkout@v4
@@ -42,38 +45,29 @@ jobs:
         with:
           python-version: '3.x'
 
-      - name: Install ARM GCC
-        uses: carlosperate/arm-none-eabi-gcc-action@v1
-        with:
-          release: '12.3.Rel1'
-
       - name: Parse HIL json
         id: parse_hil_json
         run: |
           sudo apt install -y jq
-          BOARD_LIST=$(jq -r '.boards[] | "-b " + .name' test/hil/pi4.json | tr '\n' ' ')
-          echo "BOARD_LIST=$BOARD_LIST"
-          echo >> $GITHUB_ENV "BOARD_LIST=$BOARD_LIST"
-          echo >> $GITHUB_OUTPUT "BOARD_LIST=$BOARD_LIST"
 
-      - name: Checkout pico-sdk for rp2040
-        uses: actions/checkout@v4
+          # Non-Espresif boards
+          BOARDS_LIST=$(jq -r '.boards[] | select(.flasher != "esptool") | "-b " + .name' test/hil/pi4.json | tr '\n' ' ')
+          echo "BOARDS_LIST=$BOARDS_LIST"
+          echo "BOARDS_LIST=$BOARDS_LIST" >> $GITHUB_ENV
+          echo "BOARDS_LIST=$BOARDS_LIST" >> $GITHUB_OUTPUT
+
+      - name: Setup ARM Toolchain
+        uses: ./.github/actions/setup_toolchain
         with:
-          repository: raspberrypi/pico-sdk
-          ref: develop
-          path: pico-sdk
+          toolchain: 'arm-gcc'
 
       - name: Get Dependencies
-        run: |
-          pip install click
-          sudo apt install -y ninja-build
-          python3 tools/get_deps.py $BOARD_LIST
+        uses: ./.github/actions/get_deps
+        with:
+          arg: ${{ env.BOARDS_LIST }}
 
       - name: Build
-        run: |
-          python tools/build.py $BOARD_LIST
-        env:
-          PICO_SDK_PATH: ${{ github.workspace }}/pico-sdk
+        run: python tools/build.py $BOARDS_LIST
 
       - name: Upload Artifacts for Hardware Testing
         uses: actions/upload-artifact@v4
@@ -84,15 +78,70 @@ jobs:
             cmake-build/cmake-build-*/*/*/*.bin
 
   # ---------------------------------------
+  # Build Espressif
+  # ---------------------------------------
+  build-esp:
+    runs-on: ubuntu-latest
+    outputs:
+      BOARDS_LIST: ${{ steps.parse_hil_json.outputs.BOARDS_LIST }}
+    steps:
+      - name: Checkout TinyUSB
+        uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+
+      - name: Parse HIL json
+        id: parse_hil_json
+        run: |
+          sudo apt install -y jq
+          # Espressif boards
+          BOARDS_LIST=$(jq -r '.boards[] | select(.flasher == "esptool") | "-b " + .name' test/hil/pi4.json | tr '\n' ' ')
+          echo "BOARDS_LIST=$BOARDS_LIST"
+          echo "BOARDS_LIST=$BOARDS_LIST" >> $GITHUB_ENV
+          echo "BOARDS_LIST=$BOARDS_LIST" >> $GITHUB_OUTPUT
+
+      - name: Setup ESP-IDF
+        if: env.BOARDS_LIST != ''
+        uses: ./.github/actions/setup_toolchain
+        with:
+          toolchain: 'esp-idf'
+          toolchain_url: 'v5.1.1'
+
+      - name: Get Dependencies
+        uses: ./.github/actions/get_deps
+        with:
+          arg: ${{ env.BOARDS_LIST }}
+
+      - name: Build Espressif
+        if: env.BOARDS_LIST != ''
+        run: docker run --rm -v $PWD:/project -w /project espressif/idf:v5.1.1 python3 tools/build.py $BOARDS_LIST
+
+      - name: Upload Artifacts for Hardware Testing
+        uses: actions/upload-artifact@v4
+        with:
+          name: hil_pi4_esp
+          path: |
+            cmake-build/cmake-build-*/*/*/*.bin
+            cmake-build/cmake-build-*/*/*/bootloader/bootloader.bin
+            cmake-build/cmake-build-*/*/*/partition_table/partition-table.bin
+            cmake-build/cmake-build-*/*/*/config.env
+            cmake-build/cmake-build-*/*/*/flash_args
+
+  # ---------------------------------------
   # Hardware in the loop (HIL)
-  # Current self-hosted instance is running on an RPI4. For attached hardware checkout hil_pi4.json
+  # Current self-hosted instance is running on an RPI4. For attached hardware checkout test/hil/pi4.json
   # ---------------------------------------
   hil-pi4:
     if: github.repository_owner == 'hathach'
-    needs: build
-    runs-on: [self-hosted, rp2040, nrf52840, hardware-in-the-loop]
+    needs:
+      - build
+      - build-esp
+    runs-on: [self-hosted, rp2040, nrf52840, esp32s3, hardware-in-the-loop]
     env:
-        BOARD_LIST: ${{ needs.build.outputs.BOARD_LIST }}
+        BOARDS_LIST: "${{ needs.build.outputs.BOARDS_LIST }} ${{ needs.build-esp.outputs.BOARDS_LIST }}"
     steps:
       - name: Clean workspace
         run: |
@@ -103,8 +152,7 @@ jobs:
         # USB bus on rpi4 is not stable, reset it before testing
       - name: Reset USB bus
         run: |
-          lsusb
-          lsusb -t
+          # lsusb -t
           # reset VIA Labs 2.0 hub
           sudo usbreset 001/002
 
@@ -119,7 +167,16 @@ jobs:
           name: hil_pi4
           path: cmake-build
 
+      - name: Download Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: hil_pi4_esp
+          path: cmake-build
+
       - name: Test on actual hardware
         run: |
-          echo "BOARD_LIST=$BOARD_LIST"
-          python3 test/hil/hil_test.py $BOARD_LIST pi4.json
+          echo "BOARDS_LIST=$BOARDS_LIST"
+          echo "::group::{cmake-build contents}"
+          tree cmake-build
+          echo "::endgroup::"
+          python3 test/hil/hil_test.py $BOARDS_LIST pi4.json