From ba6babf570752a28379aab51d9c27e7b9efe872d Mon Sep 17 00:00:00 2001 From: Ha Thach Date: Thu, 9 May 2024 20:43:46 +0700 Subject: Rework ci (#2631) * add name field to usbd_class_driver_t * ci: use set matrix py script * add riscv32 and cmake support for ch32v307, fomu, gd32vf103 * update build_cmake.py to take --family --board --toolchain * separate hil test to its own workflow * move esp32 board into separated hil json * add make build to ci * remov build_make.py * build.py support esp32 board * setup toolchain support esp-idf * fix missing click * merge family in matrix build to reduce jobs * skip cifuzz since it still has issue with get_deps and click --- .github/workflows/pre-commit.yml | 1 + 1 file changed, 1 insertion(+) (limited to '.github/workflows/pre-commit.yml') diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index e36259daa..6b3151702 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -38,6 +38,7 @@ jobs: - name: Build Fuzzer run: | + pip install click export CC=clang export CXX=clang++ fuzz_harness=$(ls -d test/fuzz/device/*/) -- cgit v1.3.1 From 3e2ea7750681b27238043c7158ff5508f01914a3 Mon Sep 17 00:00:00 2001 From: Ha Thach Date: Fri, 10 May 2024 18:55:43 +0700 Subject: More ci tweak (#2636) * change concurrency group to ${{ github.workflow }}-${{ github.ref }} * use argparse for build.py hil_test.py, remove the need to install click * move ci win/mac to build_cmake.yml * rename build_family.yml to build_util.yml * build_util.yml support esp32 * integrate build-espressif into build.yml * build.py support make with --board option * add get_deps action * update hil test to reuse action --- .github/actions/get_deps/action.yml | 29 +++++++ .github/actions/prepare_build/action.yml | 30 ------- .github/actions/setup_toolchain/action.yml | 18 ++++- .github/workflows/build_cmake.yml | 46 +++++++++-- .github/workflows/build_esp.yml | 109 ------------------------- .github/workflows/build_family.yml | 64 --------------- .github/workflows/build_iar.yml | 2 +- .github/workflows/build_renesas.yml | 3 +- .github/workflows/build_util.yml | 59 ++++++++++++++ .github/workflows/build_win_mac.yml | 56 ------------- .github/workflows/ci_set_matrix.py | 2 +- .github/workflows/hil_test.yml | 121 ++++++++++++++++++++-------- .github/workflows/pre-commit.yml | 3 +- examples/device/net_lwip_webserver/skip.txt | 4 +- examples/device/video_capture_2ch/skip.txt | 2 + examples/host/cdc_msc_hid_freertos/only.txt | 1 - hw/bsp/lpc17/family.mk | 2 +- hw/bsp/lpc40/family.mk | 2 + src/class/net/ncm_device.c | 16 ++-- test/hil/hil_test.py | 19 +++-- test/hil/pi4.json | 10 +++ test/hil/pi4_esp32.json | 14 ---- tools/build.py | 46 +++++++---- tools/build_esp32.py | 106 ------------------------ tools/get_deps.py | 11 ++- 25 files changed, 314 insertions(+), 461 deletions(-) create mode 100644 .github/actions/get_deps/action.yml delete mode 100644 .github/actions/prepare_build/action.yml delete mode 100644 .github/workflows/build_esp.yml delete mode 100644 .github/workflows/build_family.yml create mode 100644 .github/workflows/build_util.yml delete mode 100644 .github/workflows/build_win_mac.yml delete mode 100644 test/hil/pi4_esp32.json delete mode 100644 tools/build_esp32.py (limited to '.github/workflows/pre-commit.yml') diff --git a/.github/actions/get_deps/action.yml b/.github/actions/get_deps/action.yml new file mode 100644 index 000000000..38b44a70e --- /dev/null +++ b/.github/actions/get_deps/action.yml @@ -0,0 +1,29 @@ +name: Get dependencies + +inputs: + arg: + required: true + type: string + +runs: + using: "composite" + steps: + - name: Checkout pico-sdk for rp2040 + if: contains(inputs.arg, 'rp2040') || contains(inputs.arg, 'raspberry_pi_pico') + uses: actions/checkout@v4 + with: + repository: raspberrypi/pico-sdk + ref: develop + path: pico-sdk + + - name: Linux dependencies + if: runner.os == 'Linux' + run: | + sudo apt install -y ninja-build + shell: bash + + - name: Get Dependencies + run: | + python3 tools/get_deps.py ${{ inputs.arg }} + echo "PICO_SDK_PATH=${{ github.workspace }}/pico-sdk" >> $GITHUB_ENV + shell: bash diff --git a/.github/actions/prepare_build/action.yml b/.github/actions/prepare_build/action.yml deleted file mode 100644 index 5f2c544c1..000000000 --- a/.github/actions/prepare_build/action.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Prepare to build - -inputs: - family: - required: true - type: string - -runs: - using: "composite" - steps: - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - - name: Checkout pico-sdk for rp2040 - if: contains(inputs.family, 'rp2040') - uses: actions/checkout@v4 - with: - repository: raspberrypi/pico-sdk - ref: develop - path: pico-sdk - - - name: Get Dependencies - run: | - sudo apt install -y ninja-build - pip install click - python3 tools/get_deps.py ${{ inputs.family }} - echo >> $GITHUB_ENV "PICO_SDK_PATH=$GITHUB_WORKSPACE/pico-sdk" - shell: bash diff --git a/.github/actions/setup_toolchain/action.yml b/.github/actions/setup_toolchain/action.yml index b59ece116..e6c79e7dd 100644 --- a/.github/actions/setup_toolchain/action.yml +++ b/.github/actions/setup_toolchain/action.yml @@ -8,6 +8,11 @@ inputs: required: false type: string +outputs: + build_option: + description: 'Build option for the toolchain e.g --toolchain clang' + value: ${{ steps.set-toolchain-option.outputs.build_option }} + runs: using: "composite" steps: @@ -19,7 +24,7 @@ runs: - name: Pull ESP-IDF docker if: inputs.toolchain == 'esp-idf' - run: docker pull espressif/idf:latest + run: docker pull espressif/idf:${{ inputs.toolchain_url }} shell: bash - name: Download Toolchain @@ -29,3 +34,14 @@ runs: uses: ./.github/actions/setup_toolchain/download with: toolchain_url: ${{ inputs.toolchain_url }} + + - name: Set toolchain option + id: set-toolchain-option + run: | + BUILD_OPTION="" + if [[ "${{ inputs.toolchain }}" == *"clang"* ]]; then + BUILD_OPTION="--toolchain clang" + fi + echo "build_option=$BUILD_OPTION" + echo "build_option=$BUILD_OPTION" >> $GITHUB_OUTPUT + shell: bash diff --git a/.github/workflows/build_cmake.yml b/.github/workflows/build_cmake.yml index b62a0e9bd..4723cf8d9 100644 --- a/.github/workflows/build_cmake.yml +++ b/.github/workflows/build_cmake.yml @@ -12,6 +12,7 @@ on: - 'tools/build.py' - '.github/actions/**' - '.github/workflows/build_cmake.yml' + - '.github/workflows/build_util.yml' - '.github/workflows/ci_set_matrix.py' pull_request: branches: [ master ] @@ -24,9 +25,10 @@ on: - 'tools/build.py' - '.github/actions/**' - '.github/workflows/build_cmake.yml' + - '.github/workflows/build_util.yml' - '.github/workflows/ci_set_matrix.py' concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: @@ -55,7 +57,7 @@ jobs: # --------------------------------------- cmake: needs: set-matrix - uses: ./.github/workflows/build_family.yml + uses: ./.github/workflows/build_util.yml strategy: fail-fast: false matrix: @@ -69,14 +71,14 @@ jobs: build-system: 'cmake' toolchain: ${{ matrix.toolchain }} toolchain_url: ${{ fromJSON(needs.set-matrix.outputs.json)[matrix.toolchain].toolchain_url }} - build-family: ${{ toJSON(fromJSON(needs.set-matrix.outputs.json)[matrix.toolchain].family) }} + build-args: ${{ toJSON(fromJSON(needs.set-matrix.outputs.json)[matrix.toolchain].family) }} # --------------------------------------- # Build Make # --------------------------------------- make: needs: set-matrix - uses: ./.github/workflows/build_family.yml + uses: ./.github/workflows/build_util.yml strategy: fail-fast: false matrix: @@ -90,4 +92,38 @@ jobs: build-system: 'make' toolchain: ${{ matrix.toolchain }} toolchain_url: ${{ fromJSON(needs.set-matrix.outputs.json)[matrix.toolchain].toolchain_url }} - build-family: ${{ toJSON(fromJSON(needs.set-matrix.outputs.json)[matrix.toolchain].family) }} + build-args: ${{ toJSON(fromJSON(needs.set-matrix.outputs.json)[matrix.toolchain].family) }} + + # --------------------------------------- + # Build Make on Windows/MacOS + # --------------------------------------- + make-os: + uses: ./.github/workflows/build_util.yml + strategy: + fail-fast: false + matrix: + os: [windows-latest, macos-latest] + with: + os: ${{ matrix.os }} + build-system: 'make' + toolchain: 'arm-gcc' + build-args: '["-bstm32f411disco"]' + + # --------------------------------------- + # Build Espressif + # --------------------------------------- + espressif: + uses: ./.github/workflows/build_util.yml + strategy: + fail-fast: false + matrix: + board: + # ESP32-S2 + - 'espressif_kaluga_1' + # ESP32-S3 skip since devkitm is also compiled in hil-test workflow + #- 'espressif_s3_devkitm' + with: + build-system: 'cmake' + toolchain: 'esp-idf' + toolchain_url: 'v5.1.1' + build-args: '["-b${{ matrix.board }}"]' diff --git a/.github/workflows/build_esp.yml b/.github/workflows/build_esp.yml deleted file mode 100644 index 66fa8548e..000000000 --- a/.github/workflows/build_esp.yml +++ /dev/null @@ -1,109 +0,0 @@ -name: Build ESP - -on: - workflow_dispatch: - push: - paths: - - 'src/**' - - 'examples/**' - - 'lib/**' - - 'hw/**' - - 'test/hil/**' - - '.github/workflows/build_esp.yml' - pull_request: - branches: [ master ] - paths: - - 'src/**' - - 'examples/**' - - 'lib/**' - - 'hw/**' - - 'test/hil/**' - - '.github/workflows/build_esp.yml' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - build-esp: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - board: - # ESP32-S2 - - 'espressif_kaluga_1' - # ESP32-S3 - - 'espressif_s3_devkitm' - steps: - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - - name: Checkout TinyUSB - uses: actions/checkout@v4 - - - name: Setup Toolchain - uses: ./.github/actions/setup_toolchain - with: - toolchain: 'esp-idf' - - - name: Build - run: docker run --rm -v $PWD:/project -w /project espressif/idf:v5.1.1 python3 tools/build.py -b ${{ matrix.board }} - - - name: Upload Artifacts for Hardware Testing - if: matrix.board == 'espressif_s3_devkitm' && github.repository_owner == 'hathach' - uses: actions/upload-artifact@v4 - with: - name: ${{ matrix.board }} - path: | - cmake-build/cmake-build-${{ matrix.board }}/*/*/bootloader/bootloader.bin - cmake-build/cmake-build-${{ matrix.board }}/*/*/*.bin - cmake-build/cmake-build-${{ matrix.board }}/*/*/partition_table/partition-table.bin - cmake-build/cmake-build-${{ matrix.board }}/*/*/config.env - cmake-build/cmake-build-${{ matrix.board }}/*/*/flash_args - - # --------------------------------------- - # Hardware in the loop (HIL) - # Current self-hosted instance is running on an RPI4. For attached hardware checkout hil_pi4.json - # --------------------------------------- - hil-test: - # run only with hathach's commit due to limited resource on RPI4 - if: github.repository_owner == 'hathach' - needs: build-esp - runs-on: [self-hosted, esp32s3, hardware-in-the-loop] - strategy: - fail-fast: false - matrix: - board: - - 'espressif_s3_devkitm' - steps: - - name: Clean workspace - run: | - echo "Cleaning up previous run" - rm -rf "${{ github.workspace }}" - mkdir -p "${{ github.workspace }}" - - # USB bus on rpi4 is not stable, reset it before testing - - name: Reset USB bus - run: | - lsusb - lsusb -t - # reset VIA Labs 2.0 hub - sudo usbreset 001/002 - - - name: Checkout test/hil - uses: actions/checkout@v4 - with: - sparse-checkout: test/hil - - - name: Download Artifacts - uses: actions/download-artifact@v4 - with: - name: ${{ matrix.board }} - path: cmake-build/cmake-build-${{ matrix.board }} - - - name: Test on actual hardware - run: | - python3 test/hil/hil_test.py --board ${{ matrix.board }} pi4_esp32.json diff --git a/.github/workflows/build_family.yml b/.github/workflows/build_family.yml deleted file mode 100644 index 2e89267a3..000000000 --- a/.github/workflows/build_family.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: Build family - -on: - workflow_call: - inputs: - build-system: - required: true - type: string - toolchain: - required: true - type: string - toolchain_url: - required: true - type: string - build-family: - required: true - type: string - -jobs: - family: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - family: ${{ fromJSON(inputs.build-family) }} - steps: - - name: Checkout TinyUSB - uses: actions/checkout@v4 - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - - name: Setup Toolchain - uses: ./.github/actions/setup_toolchain - with: - toolchain: ${{ inputs.toolchain }} - toolchain_url: ${{ inputs.toolchain_url }} - - - name: Checkout pico-sdk for rp2040 - if: contains(matrix.family, 'rp2040') - uses: actions/checkout@v4 - with: - repository: raspberrypi/pico-sdk - ref: develop - path: pico-sdk - - - name: Get Dependencies - run: | - sudo apt install -y ninja-build - pip install click - python3 tools/get_deps.py ${{ matrix.family }} - - - name: Build - run: | - OPTION="" - if [[ "${{ inputs.toolchain }}" == *"clang"* ]]; then - OPTION="--toolchain clang" - fi - echo "OPTION=$OPTION" - python tools/build.py -s ${{ inputs.build-system }} $OPTION ${{ matrix.family }} - env: - PICO_SDK_PATH: ${{ github.workspace }}/pico-sdk diff --git a/.github/workflows/build_iar.yml b/.github/workflows/build_iar.yml index c4a7f96b7..34dbda192 100644 --- a/.github/workflows/build_iar.yml +++ b/.github/workflows/build_iar.yml @@ -23,7 +23,7 @@ on: - '.github/workflows/build_iar.yml' concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: diff --git a/.github/workflows/build_renesas.yml b/.github/workflows/build_renesas.yml index 8c83bdbbf..1be49344f 100644 --- a/.github/workflows/build_renesas.yml +++ b/.github/workflows/build_renesas.yml @@ -21,7 +21,7 @@ on: - '.github/workflows/build_renesas.yml' concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: @@ -65,7 +65,6 @@ jobs: - name: Get Dependencies run: | - pip install click python3 tools/get_deps.py ${{ matrix.family }} - name: Build diff --git a/.github/workflows/build_util.yml b/.github/workflows/build_util.yml new file mode 100644 index 000000000..f8ad1900c --- /dev/null +++ b/.github/workflows/build_util.yml @@ -0,0 +1,59 @@ +name: Reusable build util + +on: + workflow_call: + inputs: + build-system: + required: true + type: string + toolchain: + required: true + type: string + toolchain_url: + required: false + type: string + build-args: + required: true + type: string + os: + required: false + type: string + default: 'ubuntu-latest' + +jobs: + family: + runs-on: ${{ inputs.os }} + strategy: + fail-fast: false + matrix: + arg: ${{ fromJSON(inputs.build-args) }} + steps: + - name: Checkout TinyUSB + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Setup Toolchain + id: setup-toolchain + uses: ./.github/actions/setup_toolchain + with: + toolchain: ${{ inputs.toolchain }} + toolchain_url: ${{ inputs.toolchain_url }} + + - name: Get Dependencies + uses: ./.github/actions/get_deps + with: + arg: ${{ matrix.arg }} + + - name: Build + if: inputs.toolchain != 'esp-idf' + run: | + python tools/build.py -s ${{ inputs.build-system }} ${{ steps.setup-toolchain.outputs.build_option }} ${{ matrix.arg }} + + - name: Build using ESP-IDF docker + if: inputs.toolchain == 'esp-idf' + run: | + docker run --rm -v $PWD:/project -w /project espressif/idf:${{ inputs.toolchain_url }} python3 tools/build.py ${{ matrix.arg }} diff --git a/.github/workflows/build_win_mac.yml b/.github/workflows/build_win_mac.yml deleted file mode 100644 index 35328aa32..000000000 --- a/.github/workflows/build_win_mac.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: Build Windows/MacOS - -on: - workflow_dispatch: - push: - paths: - - 'src/**' - - 'examples/**' - - 'lib/**' - - 'hw/**' - - '.github/workflows/build_win_mac.yml' - pull_request: - branches: [ master ] - paths: - - 'src/**' - - 'examples/**' - - 'lib/**' - - 'hw/**' - - '.github/workflows/build_win_mac.yml' - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - # --------------------------------------- - # Build ARM family - # --------------------------------------- - build-arm: - strategy: - fail-fast: false - matrix: - os: [windows-latest, macos-latest] - runs-on: ${{ matrix.os }} - - steps: - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - - name: Install ARM GCC - uses: carlosperate/arm-none-eabi-gcc-action@v1 - with: - release: '10.3-2021.10' - - - name: Checkout TinyUSB - uses: actions/checkout@v4 - - - name: Get Dependencies - run: | - pip install click - python3 tools/get_deps.py stm32f4 - - - name: Build - run: python3 tools/build.py -s make stm32f2 diff --git a/.github/workflows/ci_set_matrix.py b/.github/workflows/ci_set_matrix.py index 2a17be6a1..ea758d917 100644 --- a/.github/workflows/ci_set_matrix.py +++ b/.github/workflows/ci_set_matrix.py @@ -16,7 +16,7 @@ family_list = { "ch32v307 fomu gd32vf103": ["riscv-gcc"], "imxrt": ["arm-gcc", "arm-clang"], "kinetis_k kinetis_kl kinetis_k32l2": ["arm-gcc", "arm-clang"], - "lpc11 lpc13 lpc15": ["arm-gcc"], + "lpc11 lpc13 lpc15": ["arm-gcc", "arm-clang"], "lpc17 lpc18 lpc40 lpc43": ["arm-gcc", "arm-clang"], "lpc51 lpc54 lpc55": ["arm-gcc", "arm-clang"], "mcx": ["arm-gcc"], diff --git a/.github/workflows/hil_test.yml b/.github/workflows/hil_test.yml index 024ab969d..796ff32dc 100644 --- a/.github/workflows/hil_test.yml +++ b/.github/workflows/hil_test.yml @@ -24,15 +24,18 @@ on: - '.github/actions/**' - '.github/workflows/hil_test.yml' concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: + # --------------------------------------- + # Build Non Espressif + # --------------------------------------- build: if: github.repository_owner == 'hathach' runs-on: ubuntu-latest outputs: - BOARD_LIST: ${{ steps.parse_hil_json.outputs.BOARD_LIST }} + BOARDS_LIST: ${{ steps.parse_hil_json.outputs.BOARDS_LIST }} steps: - name: Checkout TinyUSB uses: actions/checkout@v4 @@ -42,38 +45,29 @@ jobs: with: python-version: '3.x' - - name: Install ARM GCC - uses: carlosperate/arm-none-eabi-gcc-action@v1 - with: - release: '12.3.Rel1' - - name: Parse HIL json id: parse_hil_json run: | sudo apt install -y jq - BOARD_LIST=$(jq -r '.boards[] | "-b " + .name' test/hil/pi4.json | tr '\n' ' ') - echo "BOARD_LIST=$BOARD_LIST" - echo >> $GITHUB_ENV "BOARD_LIST=$BOARD_LIST" - echo >> $GITHUB_OUTPUT "BOARD_LIST=$BOARD_LIST" - - name: Checkout pico-sdk for rp2040 - uses: actions/checkout@v4 + # Non-Espresif boards + BOARDS_LIST=$(jq -r '.boards[] | select(.flasher != "esptool") | "-b " + .name' test/hil/pi4.json | tr '\n' ' ') + echo "BOARDS_LIST=$BOARDS_LIST" + echo "BOARDS_LIST=$BOARDS_LIST" >> $GITHUB_ENV + echo "BOARDS_LIST=$BOARDS_LIST" >> $GITHUB_OUTPUT + + - name: Setup ARM Toolchain + uses: ./.github/actions/setup_toolchain with: - repository: raspberrypi/pico-sdk - ref: develop - path: pico-sdk + toolchain: 'arm-gcc' - name: Get Dependencies - run: | - pip install click - sudo apt install -y ninja-build - python3 tools/get_deps.py $BOARD_LIST + uses: ./.github/actions/get_deps + with: + arg: ${{ env.BOARDS_LIST }} - name: Build - run: | - python tools/build.py $BOARD_LIST - env: - PICO_SDK_PATH: ${{ github.workspace }}/pico-sdk + run: python tools/build.py $BOARDS_LIST - name: Upload Artifacts for Hardware Testing uses: actions/upload-artifact@v4 @@ -83,16 +77,71 @@ jobs: cmake-build/cmake-build-*/*/*/*.elf cmake-build/cmake-build-*/*/*/*.bin + # --------------------------------------- + # Build Espressif + # --------------------------------------- + build-esp: + runs-on: ubuntu-latest + outputs: + BOARDS_LIST: ${{ steps.parse_hil_json.outputs.BOARDS_LIST }} + steps: + - name: Checkout TinyUSB + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Parse HIL json + id: parse_hil_json + run: | + sudo apt install -y jq + # Espressif boards + BOARDS_LIST=$(jq -r '.boards[] | select(.flasher == "esptool") | "-b " + .name' test/hil/pi4.json | tr '\n' ' ') + echo "BOARDS_LIST=$BOARDS_LIST" + echo "BOARDS_LIST=$BOARDS_LIST" >> $GITHUB_ENV + echo "BOARDS_LIST=$BOARDS_LIST" >> $GITHUB_OUTPUT + + - name: Setup ESP-IDF + if: env.BOARDS_LIST != '' + uses: ./.github/actions/setup_toolchain + with: + toolchain: 'esp-idf' + toolchain_url: 'v5.1.1' + + - name: Get Dependencies + uses: ./.github/actions/get_deps + with: + arg: ${{ env.BOARDS_LIST }} + + - name: Build Espressif + if: env.BOARDS_LIST != '' + run: docker run --rm -v $PWD:/project -w /project espressif/idf:v5.1.1 python3 tools/build.py $BOARDS_LIST + + - name: Upload Artifacts for Hardware Testing + uses: actions/upload-artifact@v4 + with: + name: hil_pi4_esp + path: | + cmake-build/cmake-build-*/*/*/*.bin + cmake-build/cmake-build-*/*/*/bootloader/bootloader.bin + cmake-build/cmake-build-*/*/*/partition_table/partition-table.bin + cmake-build/cmake-build-*/*/*/config.env + cmake-build/cmake-build-*/*/*/flash_args + # --------------------------------------- # Hardware in the loop (HIL) - # Current self-hosted instance is running on an RPI4. For attached hardware checkout hil_pi4.json + # Current self-hosted instance is running on an RPI4. For attached hardware checkout test/hil/pi4.json # --------------------------------------- hil-pi4: if: github.repository_owner == 'hathach' - needs: build - runs-on: [self-hosted, rp2040, nrf52840, hardware-in-the-loop] + needs: + - build + - build-esp + runs-on: [self-hosted, rp2040, nrf52840, esp32s3, hardware-in-the-loop] env: - BOARD_LIST: ${{ needs.build.outputs.BOARD_LIST }} + BOARDS_LIST: "${{ needs.build.outputs.BOARDS_LIST }} ${{ needs.build-esp.outputs.BOARDS_LIST }}" steps: - name: Clean workspace run: | @@ -103,8 +152,7 @@ jobs: # USB bus on rpi4 is not stable, reset it before testing - name: Reset USB bus run: | - lsusb - lsusb -t + # lsusb -t # reset VIA Labs 2.0 hub sudo usbreset 001/002 @@ -119,7 +167,16 @@ jobs: name: hil_pi4 path: cmake-build + - name: Download Artifacts + uses: actions/download-artifact@v4 + with: + name: hil_pi4_esp + path: cmake-build + - name: Test on actual hardware run: | - echo "BOARD_LIST=$BOARD_LIST" - python3 test/hil/hil_test.py $BOARD_LIST pi4.json + echo "BOARDS_LIST=$BOARDS_LIST" + echo "::group::{cmake-build contents}" + tree cmake-build + echo "::endgroup::" + python3 test/hil/hil_test.py $BOARDS_LIST pi4.json diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 6b3151702..d1ffe6ca1 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -7,7 +7,7 @@ on: branches: [ master ] concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: @@ -38,7 +38,6 @@ jobs: - name: Build Fuzzer run: | - pip install click export CC=clang export CXX=clang++ fuzz_harness=$(ls -d test/fuzz/device/*/) diff --git a/examples/device/net_lwip_webserver/skip.txt b/examples/device/net_lwip_webserver/skip.txt index bb3ff7885..43cdab71a 100644 --- a/examples/device/net_lwip_webserver/skip.txt +++ b/examples/device/net_lwip_webserver/skip.txt @@ -9,6 +9,8 @@ mcu:STM32F0 mcu:KINETIS_KL family:broadcom_64bit family:broadcom_32bit +family:espressif board:curiosity_nano board:frdm_kl25z -family:espressif +# lpc55 has weird error 'ncm_interface' causes a section type conflict with 'ntb_parameters' +family:lpc55 diff --git a/examples/device/video_capture_2ch/skip.txt b/examples/device/video_capture_2ch/skip.txt index 15c176a2a..86697899b 100644 --- a/examples/device/video_capture_2ch/skip.txt +++ b/examples/device/video_capture_2ch/skip.txt @@ -12,3 +12,5 @@ board:lpcxpresso11u68 board:stm32f303disco board:stm32l412nucleo board:ek_tm4c123gxl +board:uno_r4 +board:ra4m1_ek diff --git a/examples/host/cdc_msc_hid_freertos/only.txt b/examples/host/cdc_msc_hid_freertos/only.txt index 81d993ffa..1e0e60075 100644 --- a/examples/host/cdc_msc_hid_freertos/only.txt +++ b/examples/host/cdc_msc_hid_freertos/only.txt @@ -8,5 +8,4 @@ mcu:MIMXRT10XX mcu:MIMXRT11XX mcu:MSP432E4 mcu:RX65X -mcu:RAXXX mcu:MAX3421 diff --git a/hw/bsp/lpc17/family.mk b/hw/bsp/lpc17/family.mk index 84ed95648..d719a47b7 100644 --- a/hw/bsp/lpc17/family.mk +++ b/hw/bsp/lpc17/family.mk @@ -22,8 +22,8 @@ LDFLAGS_GCC += --specs=nosys.specs --specs=nano.specs SRC_C += \ src/portable/nxp/lpc17_40/dcd_lpc17_40.c \ - src/portable/ohci/ohci.c \ src/portable/nxp/lpc17_40/hcd_lpc17_40.c \ + src/portable/ohci/ohci.c \ $(MCU_DIR)/../gcc/cr_startup_lpc175x_6x.c \ $(MCU_DIR)/src/chip_17xx_40xx.c \ $(MCU_DIR)/src/clock_17xx_40xx.c \ diff --git a/hw/bsp/lpc40/family.mk b/hw/bsp/lpc40/family.mk index 79d868f35..ef9fe57b2 100644 --- a/hw/bsp/lpc40/family.mk +++ b/hw/bsp/lpc40/family.mk @@ -20,6 +20,8 @@ LDFLAGS_GCC += --specs=nosys.specs --specs=nano.specs # All source paths should be relative to the top level. SRC_C += \ src/portable/nxp/lpc17_40/dcd_lpc17_40.c \ + src/portable/nxp/lpc17_40/hcd_lpc17_40.c \ + src/portable/ohci/ohci.c \ $(MCU_DIR)/../gcc/cr_startup_lpc40xx.c \ $(MCU_DIR)/src/chip_17xx_40xx.c \ $(MCU_DIR)/src/clock_17xx_40xx.c \ diff --git a/src/class/net/ncm_device.c b/src/class/net/ncm_device.c index 4b237e4cf..64aba011a 100644 --- a/src/class/net/ncm_device.c +++ b/src/class/net/ncm_device.c @@ -112,7 +112,7 @@ typedef struct { bool notification_xmit_is_running; // notification is currently transmitted } ncm_interface_t; -CFG_TUSB_MEM_SECTION CFG_TUSB_MEM_ALIGN tu_static ncm_interface_t ncm_interface; +CFG_TUD_MEM_SECTION CFG_TUD_MEM_ALIGN tu_static ncm_interface_t ncm_interface; /** * This is the NTB parameter structure @@ -120,7 +120,7 @@ CFG_TUSB_MEM_SECTION CFG_TUSB_MEM_ALIGN tu_static ncm_interface_t ncm_interface; * \attention * We are lucky, that byte order is correct */ -CFG_TUSB_MEM_SECTION CFG_TUSB_MEM_ALIGN tu_static const ntb_parameters_t ntb_parameters = { +CFG_TUD_MEM_SECTION CFG_TUD_MEM_ALIGN tu_static const ntb_parameters_t ntb_parameters = { .wLength = sizeof(ntb_parameters_t), .bmNtbFormatsSupported = 0x01,// 16-bit NTB supported .dwNtbInMaxSize = CFG_TUD_NCM_IN_NTB_MAX_SIZE, @@ -285,7 +285,7 @@ static xmit_ntb_t *xmit_get_next_ready_ntb(void) { * This must be called from netd_xfer_cb() so that ep_in is ready */ static bool xmit_insert_required_zlp(uint8_t rhport, uint32_t xferred_bytes) { - TU_LOG_DRV("xmit_insert_required_zlp(%d,%d)\n", rhport, xferred_bytes); + TU_LOG_DRV("xmit_insert_required_zlp(%d,%ld)\n", rhport, xferred_bytes); if (xferred_bytes == 0 || xferred_bytes % CFG_TUD_NET_ENDPOINT_SIZE != 0) { return false; @@ -521,11 +521,11 @@ static bool recv_validate_datagram(const recv_ntb_t *ntb, uint32_t len) { return false; } if (len < sizeof(nth16_t) + sizeof(ndp16_t) + 2 * sizeof(ndp16_datagram_t)) { - TU_LOG_DRV("(EE) ill min len: %d\n", len); + TU_LOG_DRV("(EE) ill min len: %lu\n", len); return false; } if (nth16->wBlockLength > len) { - TU_LOG_DRV("(EE) ill block length: %d > %d\n", nth16->wBlockLength, len); + TU_LOG_DRV("(EE) ill block length: %d > %lu\n", nth16->wBlockLength, len); return false; } if (nth16->wBlockLength > CFG_TUD_NCM_OUT_NTB_MAX_SIZE) { @@ -533,7 +533,7 @@ static bool recv_validate_datagram(const recv_ntb_t *ntb, uint32_t len) { return false; } if (nth16->wNdpIndex < sizeof(nth16) || nth16->wNdpIndex > len - (sizeof(ndp16_t) + 2 * sizeof(ndp16_datagram_t))) { - TU_LOG_DRV("(EE) ill position of first ndp: %d (%d)\n", nth16->wNdpIndex, len); + TU_LOG_DRV("(EE) ill position of first ndp: %d (%lu)\n", nth16->wNdpIndex, len); return false; } @@ -567,11 +567,11 @@ static bool recv_validate_datagram(const recv_ntb_t *ntb, uint32_t len) { while (ndp16_datagram[ndx].wDatagramIndex != 0 && ndp16_datagram[ndx].wDatagramLength != 0) { TU_LOG_DRV(" << %d %d\n", ndp16_datagram[ndx].wDatagramIndex, ndp16_datagram[ndx].wDatagramLength); if (ndp16_datagram[ndx].wDatagramIndex > len) { - TU_LOG_DRV("(EE) ill start of datagram[%d]: %d (%d)\n", ndx, ndp16_datagram[ndx].wDatagramIndex, len); + TU_LOG_DRV("(EE) ill start of datagram[%d]: %d (%lu)\n", ndx, ndp16_datagram[ndx].wDatagramIndex, len); return false; } if (ndp16_datagram[ndx].wDatagramIndex + ndp16_datagram[ndx].wDatagramLength > len) { - TU_LOG_DRV("(EE) ill end of datagram[%d]: %d (%d)\n", ndx, ndp16_datagram[ndx].wDatagramIndex + ndp16_datagram[ndx].wDatagramLength, len); + TU_LOG_DRV("(EE) ill end of datagram[%d]: %d (%lu)\n", ndx, ndp16_datagram[ndx].wDatagramIndex + ndp16_datagram[ndx].wDatagramLength, len); return false; } ++ndx; diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py index dffb81765..c2699cc6f 100644 --- a/test/hil/hil_test.py +++ b/test/hil/hil_test.py @@ -25,10 +25,10 @@ # ACTION=="add", SUBSYSTEM=="tty", SUBSYSTEMS=="usb", MODE="0666", PROGRAM="/bin/sh -c 'echo $$ID_SERIAL_SHORT | rev | cut -c -8 | rev'", SYMLINK+="ttyUSB_%c.%s{bInterfaceNumber}" # ACTION=="add", SUBSYSTEM=="block", SUBSYSTEMS=="usb", ENV{ID_FS_USAGE}=="filesystem", MODE="0666", PROGRAM="/bin/sh -c 'echo $$ID_SERIAL_SHORT | rev | cut -c -8 | rev'", RUN{program}+="/usr/bin/systemd-mount --no-block --automount=yes --collect $devnode /media/blkUSB_%c.%s{bInterfaceNumber}" +import argparse import os import sys import time -import click import serial import subprocess import json @@ -318,13 +318,18 @@ def test_hid_composite_freertos(id): # ------------------------------------------------------------- # Main # ------------------------------------------------------------- -@click.command() -@click.argument('config_file') -@click.option('-b', '--board', multiple=True, default=None, help='Boards to test, all if not specified') -def main(config_file, board): +def main(): """ Hardware test on specified boards """ + parser = argparse.ArgumentParser() + parser.add_argument('config_file', help='Configuration JSON file') + parser.add_argument('-b', '--board', action='append', default=[], help='Boards to test, all if not specified') + args = parser.parse_args() + + config_file = args.config_file + boards = args.board + config_file = os.path.join(os.path.dirname(__file__), config_file) with open(config_file) as f: config = json.load(f) @@ -334,10 +339,10 @@ def main(config_file, board): 'cdc_dual_ports', 'cdc_msc', 'dfu', 'dfu_runtime', 'hid_boot_interface', ] - if len(board) == 0: + if len(boards) == 0: config_boards = config['boards'] else: - config_boards = [e for e in config['boards'] if e['name'] in board] + config_boards = [e for e in config['boards'] if e['name'] in boards] for item in config_boards: name = item['name'] diff --git a/test/hil/pi4.json b/test/hil/pi4.json index 04329bb64..bdb8a5fa5 100644 --- a/test/hil/pi4.json +++ b/test/hil/pi4.json @@ -22,6 +22,16 @@ "flasher_product": "ItsyBitsy M4 Express", "flasher_reset_pin": "2", "flasher_args": "--offset 0x4000" + }, + { + "name": "espressif_s3_devkitm", + "uid": "84F703C084E4", + "tests": [ + "cdc_msc_freertos", "hid_composite_freertos" + ], + "flasher": "esptool", + "flasher_sn": "3ea619acd1cdeb11a0a0b806e93fd3f1", + "flasher_args": "-b 1500000" } ] } diff --git a/test/hil/pi4_esp32.json b/test/hil/pi4_esp32.json deleted file mode 100644 index c95dd2d4e..000000000 --- a/test/hil/pi4_esp32.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "boards": [ - { - "name": "espressif_s3_devkitm", - "uid": "84F703C084E4", - "tests": [ - "cdc_msc_freertos", "hid_composite_freertos" - ], - "flasher": "esptool", - "flasher_sn": "3ea619acd1cdeb11a0a0b806e93fd3f1", - "flasher_args": "-b 1500000" - } - ] -} diff --git a/tools/build.py b/tools/build.py index 884cd56f4..967f7c95e 100644 --- a/tools/build.py +++ b/tools/build.py @@ -1,8 +1,8 @@ +import argparse import os import sys import time import subprocess -import click from pathlib import Path from multiprocessing import Pool @@ -125,13 +125,20 @@ def build_family(family, toolchain, build_system): return ret -@click.command() -@click.argument('families', nargs=-1, required=False) -@click.option('-b', '--board', multiple=True, default=None, help='Boards to build') -@click.option('-t', '--toolchain', default='gcc', help='Toolchain to use, default is gcc') -@click.option('-s', '--build-system', default='cmake', help='Build system to use, default is cmake') -def main(families, board, toolchain, build_system): - if len(families) == 0 and len(board) == 0: +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('families', nargs='*', default=[], help='Families to build') + parser.add_argument('-b', '--board', action='append', default=[], help='Boards to build') + parser.add_argument('-t', '--toolchain', default='gcc', help='Toolchain to use, default is gcc') + parser.add_argument('-s', '--build-system', default='cmake', help='Build system to use, default is cmake') + args = parser.parse_args() + + families = args.families + boards = args.board + toolchain = args.toolchain + build_system = args.build_system + + if len(families) == 0 and len(boards) == 0: print("Please specify families or board to build") return 1 @@ -159,12 +166,23 @@ def main(families, board, toolchain, build_system): total_result[2] += fret[2] # build board (only cmake) - if board is not None: - for b in board: - r = build_board_cmake(b, toolchain) - total_result[0] += r[0] - total_result[1] += r[1] - total_result[2] += r[2] + if boards is not None: + for b in boards: + if build_system == 'cmake': + r = build_board_cmake(b, toolchain) + total_result[0] += r[0] + total_result[1] += r[1] + total_result[2] += r[2] + elif build_system == 'make': + all_examples = get_examples(find_family(b)) + with Pool(processes=os.cpu_count()) as pool: + pool_args = list((map(lambda e, bb=b, o=f"TOOLCHAIN={toolchain}": [e, bb, o], all_examples))) + r = pool.starmap(build_utils.build_example, pool_args) + # sum all element of same index (column sum) + rsum = list(map(sum, list(zip(*r)))) + total_result[0] += rsum[0] + total_result[1] += rsum[1] + total_result[2] += rsum[2] total_time = time.monotonic() - total_time print(build_separator) diff --git a/tools/build_esp32.py b/tools/build_esp32.py deleted file mode 100644 index 951467c23..000000000 --- a/tools/build_esp32.py +++ /dev/null @@ -1,106 +0,0 @@ -import os -import glob -import sys -import subprocess -import time - -import build_utils - -SUCCEEDED = "\033[32msucceeded\033[0m" -FAILED = "\033[31mfailed\033[0m" -SKIPPED = "\033[33mskipped\033[0m" - -success_count = 0 -fail_count = 0 -skip_count = 0 -exit_status = 0 - -total_time = time.monotonic() - -build_format = '| {:30} | {:30} | {:18} | {:7} | {:6} | {:6} |' -build_separator = '-' * 107 - -def filter_with_input(mylist): - if len(sys.argv) > 1: - input_args = list(set(mylist).intersection(sys.argv)) - if len(input_args) > 0: - mylist[:] = input_args - - -# Build all examples if not specified -all_examples = [entry.replace('examples/', '') for entry in glob.glob("examples/*/*_freertos")] -filter_with_input(all_examples) -all_examples.append('device/board_test') -all_examples.sort() - -# Build all boards if not specified -all_boards = [] -for entry in os.scandir("hw/bsp/espressif/boards"): - if entry.is_dir(): - all_boards.append(entry.name) -filter_with_input(all_boards) -all_boards.sort() - -def build_board(example, board): - global success_count, fail_count, skip_count, exit_status - start_time = time.monotonic() - - # Check if board is skipped - build_dir = f"cmake-build/cmake-build-{board}/{example}" - - # Generate and build - r = subprocess.run(f"cmake examples/{example} -B {build_dir} -G \"Ninja\" -DBOARD={board} -DMAX3421_HOST=1", - shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - if r.returncode == 0: - r = subprocess.run(f"cmake --build {build_dir}", shell=True, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - build_duration = time.monotonic() - start_time - flash_size = "-" - sram_size = "-" - - if r.returncode == 0: - success = SUCCEEDED - success_count += 1 - #(flash_size, sram_size) = build_size(example, board) - else: - exit_status = r.returncode - success = FAILED - fail_count += 1 - - title = build_format.format(example, board, success, "{:.2f}s".format(build_duration), flash_size, sram_size) - if os.getenv('CI'): - # always print build output if in CI - print(f"::group::{title}") - print(r.stdout.decode("utf-8")) - print(f"::endgroup::") - else: - # print build output if failed - print(title) - if r.returncode != 0: - print(r.stdout.decode("utf-8")) - - -def build_size(example, board): - #elf_file = 'examples/device/{}/_build/{}/{}-firmware.elf'.format(example, board, board) - elf_file = 'examples/device/{}/_build/{}/*.elf'.format(example, board) - size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8") - size_list = size_output.split('\n')[1].split('\t') - flash_size = int(size_list[0]) - sram_size = int(size_list[1]) + int(size_list[2]) - return (flash_size, sram_size) - - -print(build_separator) -print(build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM')) -print(build_separator) - -for example in all_examples: - for board in all_boards: - build_board(example, board) - -total_time = time.monotonic() - total_time -print(build_separator) -print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(success_count, SUCCEEDED, fail_count, FAILED, skip_count, SKIPPED, total_time)) -print(build_separator) - -sys.exit(exit_status) diff --git a/tools/get_deps.py b/tools/get_deps.py index 2359b4bd0..20cbe64c7 100644 --- a/tools/get_deps.py +++ b/tools/get_deps.py @@ -245,11 +245,10 @@ def main(): args = parser.parse_args() families = args.families - board = args.board + boards = args.board - if len(families) == 0 and len(board) == 0: - print("Please specify family or board to fetch") - return + if len(families) == 0 and len(boards) == 0: + print("Warning: family and board are not specified, only fetching mandatory dependencies.") status = 0 deps = list(deps_mandatory.keys()) @@ -258,8 +257,8 @@ def main(): deps += deps_optional.keys() else: families = list(families) - if board is not None: - for b in board: + if boards is not None: + for b in boards: f = find_family(b) if f is not None: families.append(f) -- cgit v1.3.1 From 7cf1bdd2848d6c36e9659acaf77c83a1fa99d04f Mon Sep 17 00:00:00 2001 From: Ha Thach Date: Mon, 13 May 2024 20:27:49 +0700 Subject: Ci tweak3 (#2643) - enable --one-per-family to build 1 board per family, also skip family if board specified in -b also present - minimize ci run for push event - only build one board per family - skip hil test on both pi4 and hfp - full build will be runn for PR event - IAR always build 1 board per family regardless of event - update build.py to optimize make - remove all setup python since we don't really need it --- .github/actions/setup_toolchain/action.yml | 3 + .github/workflows/build.yml | 50 +++++++--- .github/workflows/build_iar.yml | 50 ---------- .github/workflows/build_renesas.yml | 5 - .github/workflows/build_util.yml | 21 +++-- .github/workflows/ci_set_matrix.py | 22 +++-- .github/workflows/hil_test.yml | 12 +-- .github/workflows/pre-commit.yml | 5 - hw/bsp/stm32f0/family.c | 2 +- hw/bsp/stm32f1/family.c | 2 +- hw/bsp/stm32f2/family.c | 3 +- hw/bsp/stm32f3/family.c | 5 +- hw/bsp/stm32u5/family.c | 3 +- hw/bsp/stm32wb/family.c | 5 +- tools/build.py | 142 +++++++++++++++++------------ 15 files changed, 166 insertions(+), 164 deletions(-) delete mode 100644 .github/workflows/build_iar.yml (limited to '.github/workflows/pre-commit.yml') diff --git a/.github/actions/setup_toolchain/action.yml b/.github/actions/setup_toolchain/action.yml index 34bcbc68b..19fe28b0c 100644 --- a/.github/actions/setup_toolchain/action.yml +++ b/.github/actions/setup_toolchain/action.yml @@ -32,6 +32,7 @@ runs: - name: Download Toolchain if: >- inputs.toolchain != 'arm-gcc' && + inputs.toolchain != 'arm-iar' && inputs.toolchain != 'esp-idf' uses: ./.github/actions/setup_toolchain/download with: @@ -44,6 +45,8 @@ runs: BUILD_OPTION="" if [[ "${{ inputs.toolchain }}" == *"clang"* ]]; then BUILD_OPTION="--toolchain clang" + elif [[ "${{ inputs.toolchain }}" == "arm-iar" ]]; then + BUILD_OPTION="--toolchain iar" fi echo "build_option=$BUILD_OPTION" echo "build_option=$BUILD_OPTION" >> $GITHUB_OUTPUT diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d51e1c92a..2cd348847 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,11 +37,6 @@ jobs: outputs: json: ${{ steps.set-matrix-json.outputs.matrix }} steps: - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - name: Checkout TinyUSB uses: actions/checkout@v4 @@ -63,27 +58,22 @@ jobs: matrix: toolchain: - 'aarch64-gcc' - - 'arm-clang' +# - 'arm-clang' - 'arm-gcc' - 'msp430-gcc' - 'riscv-gcc' - if: >- - matrix.toolchain != 'arm-clang' || - github.event_name == 'pull_request' || - (github.event_name == 'push' && github.ref == 'refs/heads/master') with: build-system: 'cmake' toolchain: ${{ matrix.toolchain }} toolchain_url: ${{ fromJSON(needs.set-matrix.outputs.json)[matrix.toolchain].toolchain_url }} build-args: ${{ toJSON(fromJSON(needs.set-matrix.outputs.json)[matrix.toolchain].family) }} + one-per-family: ${{ github.event_name != 'pull_request' }} # --------------------------------------- # Build Make # --------------------------------------- make: - if: >- - github.event_name == 'pull_request' || - (github.event_name == 'push' && github.ref == 'refs/heads/master') + #if: github.event_name == 'pull_request' needs: set-matrix uses: ./.github/workflows/build_util.yml strategy: @@ -100,6 +90,7 @@ jobs: toolchain: ${{ matrix.toolchain }} toolchain_url: ${{ fromJSON(needs.set-matrix.outputs.json)[matrix.toolchain].toolchain_url }} build-args: ${{ toJSON(fromJSON(needs.set-matrix.outputs.json)[matrix.toolchain].family) }} + one-per-family: ${{ github.event_name != 'pull_request' }} # --------------------------------------- # Build Make on Windows/MacOS @@ -114,7 +105,8 @@ jobs: os: ${{ matrix.os }} build-system: 'make' toolchain: 'arm-gcc' - build-args: '["-bstm32f411disco"]' + build-args: '["stm32h7"]' + one-per-family: true # --------------------------------------- # Build Espressif @@ -134,3 +126,33 @@ jobs: toolchain: 'esp-idf' toolchain_url: 'v5.1.1' build-args: '["-b${{ matrix.board }}"]' + + # --------------------------------------- + # Build IAR on HFP self-hosted + # --------------------------------------- + arm-iar: + if: github.repository_owner == 'hathach' + needs: set-matrix + runs-on: [self-hosted, Linux, X64, hifiphile] + env: + BUILD_ARGS: ${{ join(fromJSON(needs.set-matrix.outputs.json)['arm-iar'].family, ' ') }} + steps: + - name: Clean workspace + run: | + echo "Cleaning up previous run" + rm -rf "${{ github.workspace }}" + mkdir -p "${{ github.workspace }}" + + - name: Checkout TinyUSB + uses: actions/checkout@v4 + + - name: Get Dependencies + run: python3 tools/get_deps.py $BUILD_ARGS + + - name: Build + run: python3 tools/build.py --one-per-family --toolchain iar $BUILD_ARGS + + - name: Test on actual hardware (hardware in the loop) + if: github.event_name == 'pull_request' + run: | + python3 test/hil/hil_test.py hfp.json diff --git a/.github/workflows/build_iar.yml b/.github/workflows/build_iar.yml deleted file mode 100644 index c8a095c18..000000000 --- a/.github/workflows/build_iar.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: Build IAR - -on: - workflow_dispatch: - pull_request: - branches: [ master ] - paths: - - 'src/**' - - 'examples/**' - - 'lib/**' - - 'hw/**' - - 'tools/get_deps.py' - - 'test/hil/**' - - '.github/workflows/build_iar.yml' - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - cmake: - if: github.repository_owner == 'hathach' - runs-on: [self-hosted, Linux, X64, hifiphile] - strategy: - fail-fast: false - matrix: - family: - # Alphabetical order - # Note: bundle multiple families into a matrix since there is only one self-hosted instance can - # run IAR build. Too many matrix can hurt due to setup/teardown overhead. - - 'lpc43 stm32f0 stm32f1 stm32f7 stm32g0 stm32g4 stm32l4' - steps: - - name: Clean workspace - run: | - echo "Cleaning up previous run" - rm -rf "${{ github.workspace }}" - mkdir -p "${{ github.workspace }}" - - - name: Checkout TinyUSB - uses: actions/checkout@v4 - - - name: Get Dependencies - run: python3 tools/get_deps.py ${{ matrix.family }} - - - name: Build - run: python3 tools/build.py --toolchain iar ${{ matrix.family }} - - - name: Test on actual hardware (hardware in the loop) - run: | - python3 test/hil/hil_test.py hfp.json diff --git a/.github/workflows/build_renesas.yml b/.github/workflows/build_renesas.yml index 1be49344f..3d2cbab28 100644 --- a/.github/workflows/build_renesas.yml +++ b/.github/workflows/build_renesas.yml @@ -34,11 +34,6 @@ jobs: # Alphabetical order - 'rx' steps: - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - name: Checkout TinyUSB uses: actions/checkout@v4 diff --git a/.github/workflows/build_util.yml b/.github/workflows/build_util.yml index f8ad1900c..49a9feabd 100644 --- a/.github/workflows/build_util.yml +++ b/.github/workflows/build_util.yml @@ -15,6 +15,10 @@ on: build-args: required: true type: string + one-per-family: + required: false + default: false + type: boolean os: required: false type: string @@ -31,11 +35,6 @@ jobs: - name: Checkout TinyUSB uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - name: Setup Toolchain id: setup-toolchain uses: ./.github/actions/setup_toolchain @@ -48,10 +47,20 @@ jobs: with: arg: ${{ matrix.arg }} + - name: Set build one-per-family option + id: set-one-per-family + run: | + if [[ "${{ inputs.one-per-family }}" == "true" ]]; then + BUILD_OPTION="--one-per-family" + fi + echo "build_option=$BUILD_OPTION" + echo "build_option=$BUILD_OPTION" >> $GITHUB_OUTPUT + shell: bash + - name: Build if: inputs.toolchain != 'esp-idf' run: | - python tools/build.py -s ${{ inputs.build-system }} ${{ steps.setup-toolchain.outputs.build_option }} ${{ matrix.arg }} + python tools/build.py -s ${{ inputs.build-system }} ${{ steps.setup-toolchain.outputs.build_option }} ${{ steps.set-one-per-family.outputs.build_option }} ${{ matrix.arg }} - name: Build using ESP-IDF docker if: inputs.toolchain == 'esp-idf' diff --git a/.github/workflows/ci_set_matrix.py b/.github/workflows/ci_set_matrix.py index ea758d917..62239c9ad 100644 --- a/.github/workflows/ci_set_matrix.py +++ b/.github/workflows/ci_set_matrix.py @@ -4,6 +4,7 @@ import json toolchain_list = { "aarch64-gcc": "https://developer.arm.com/-/media/Files/downloads/gnu-a/10.3-2021.07/binrel/gcc-arm-10.3-2021.07-x86_64-aarch64-none-elf.tar.xz", "arm-clang": "https://github.com/ARM-software/LLVM-embedded-toolchain-for-Arm/releases/download/release-17.0.1/LLVMEmbeddedToolchainForArm-17.0.1-Linux-x86_64.tar.xz", + "arm-iar": "", "arm-gcc": "", "msp430-gcc": "http://software-dl.ti.com/msp430/msp430_public_sw/mcu/msp430/MSPGCC/9_2_0_0/export/msp430-gcc-9.2.0.50_linux64.tar.bz2", "riscv-gcc": "https://github.com/xpack-dev-tools/riscv-none-embed-gcc-xpack/releases/download/v10.1.0-1.1/xpack-riscv-none-embed-gcc-10.1.0-1.1-linux-x64.tar.gz", @@ -28,12 +29,12 @@ family_list = { "rp2040": ["arm-gcc"], "samd11 samd21 saml2x": ["arm-gcc", "arm-clang"], "samd5x_e5x samg": ["arm-gcc", "arm-clang"], - "stm32f0 stm32f1 stm32f2 stm32f3": ["arm-gcc", "arm-clang"], - "stm32f4": ["arm-gcc", "arm-clang"], - "stm32f7": ["arm-gcc", "arm-clang"], - "stm32g0 stm32g4 stm32h5": ["arm-gcc", "arm-clang"], - "stm32h7": ["arm-gcc", "arm-clang"], - "stm32l4 stm32u5 stm32wb": ["arm-gcc", "arm-clang"], + "stm32f0 stm32f1 stm32f2 stm32f3": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32f4": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32f7": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32g0 stm32g4 stm32h5": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32h7": ["arm-gcc", "arm-clang", "arm-iar"], + "stm32l4 stm32u5 stm32wb": ["arm-gcc", "arm-clang", "arm-iar"], "xmc4000": ["arm-gcc"], } @@ -43,7 +44,16 @@ def set_matrix_json(): for toolchain in toolchain_list.keys(): filtered_families = [family for family, supported_toolchain in family_list.items() if toolchain in supported_toolchain] + + # always add board in hfp.json for arm-iar + if toolchain == 'arm-iar': + with open('test/hil/hfp.json') as f: + hfp_data = json.load(f) + hfp_boards = [f"-b{board['name']}" for board in hfp_data['boards']] + filtered_families = filtered_families + hfp_boards + matrix[toolchain] = {"family": filtered_families, "toolchain_url": toolchain_list[toolchain]} + print(json.dumps(matrix)) diff --git a/.github/workflows/hil_test.yml b/.github/workflows/hil_test.yml index eb3f2cbab..adb5bf00e 100644 --- a/.github/workflows/hil_test.yml +++ b/.github/workflows/hil_test.yml @@ -30,11 +30,6 @@ jobs: - name: Checkout TinyUSB uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - name: Parse HIL json id: parse_hil_json run: | @@ -78,11 +73,6 @@ jobs: - name: Checkout TinyUSB uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - name: Parse HIL json id: parse_hil_json run: | @@ -131,7 +121,7 @@ jobs: - build-esp runs-on: [self-hosted, rp2040, nrf52840, esp32s3, hardware-in-the-loop] env: - BOARDS_LIST: "${{ needs.build.outputs.BOARDS_LIST }} ${{ needs.build-esp.outputs.BOARDS_LIST }}" + BOARDS_LIST: "${{ needs.build.outputs.BOARDS_LIST }} ${{ needs.build-esp.outputs.BOARDS_LIST }}" steps: - name: Clean workspace run: | diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index d1ffe6ca1..379a22ee2 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -14,11 +14,6 @@ jobs: pre-commit: runs-on: ubuntu-latest steps: - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: '3.x' - - name: Setup Ruby uses: ruby/setup-ruby@v1 with: diff --git a/hw/bsp/stm32f0/family.c b/hw/bsp/stm32f0/family.c index 7ef126ae6..3079a1ed3 100644 --- a/hw/bsp/stm32f0/family.c +++ b/hw/bsp/stm32f0/family.c @@ -115,7 +115,7 @@ void board_init(void) { //--------------------------------------------------------------------+ void board_led_write(bool state) { - GPIO_PinState pin_state = (GPIO_PinState)(state ? LED_STATE_ON : (1 - LED_STATE_ON)); + GPIO_PinState pin_state = (GPIO_PinState) (state ? LED_STATE_ON : (1 - LED_STATE_ON)); HAL_GPIO_WritePin(LED_PORT, LED_PIN, pin_state); } diff --git a/hw/bsp/stm32f1/family.c b/hw/bsp/stm32f1/family.c index 0c1b362ab..70f81e7fc 100644 --- a/hw/bsp/stm32f1/family.c +++ b/hw/bsp/stm32f1/family.c @@ -124,7 +124,7 @@ void board_init(void) { //--------------------------------------------------------------------+ void board_led_write(bool state) { - GPIO_PinState pin_state = (GPIO_PinState)(state ? LED_STATE_ON : (1 - LED_STATE_ON)); + GPIO_PinState pin_state = (GPIO_PinState) (state ? LED_STATE_ON : (1 - LED_STATE_ON)); HAL_GPIO_WritePin(LED_PORT, LED_PIN, pin_state); } diff --git a/hw/bsp/stm32f2/family.c b/hw/bsp/stm32f2/family.c index 8b1c56423..62cca327b 100644 --- a/hw/bsp/stm32f2/family.c +++ b/hw/bsp/stm32f2/family.c @@ -108,7 +108,8 @@ void board_init(void) { //--------------------------------------------------------------------+ void board_led_write(bool state) { - HAL_GPIO_WritePin(LED_PORT, LED_PIN, state ? LED_STATE_ON : (1 - LED_STATE_ON)); + GPIO_PinState pin_state = (GPIO_PinState) (state ? LED_STATE_ON : (1 - LED_STATE_ON)); + HAL_GPIO_WritePin(LED_PORT, LED_PIN, pin_state); } uint32_t board_button_read(void) { diff --git a/hw/bsp/stm32f3/family.c b/hw/bsp/stm32f3/family.c index 7c194a694..e7488ba84 100644 --- a/hw/bsp/stm32f3/family.c +++ b/hw/bsp/stm32f3/family.c @@ -108,7 +108,8 @@ void board_init(void) { //--------------------------------------------------------------------+ void board_led_write(bool state) { - HAL_GPIO_WritePin(LED_PORT, LED_PIN, state ? LED_STATE_ON : (1 - LED_STATE_ON)); + GPIO_PinState pin_state = (GPIO_PinState) (state ? LED_STATE_ON : (1 - LED_STATE_ON)); + HAL_GPIO_WritePin(LED_PORT, LED_PIN, pin_state); } uint32_t board_button_read(void) { @@ -142,7 +143,7 @@ uint32_t board_millis(void) { #endif void HardFault_Handler(void) { - asm("bkpt"); + asm("bkpt 0"); } // Required by __libc_init_array in startup code if we are compiling using diff --git a/hw/bsp/stm32u5/family.c b/hw/bsp/stm32u5/family.c index ec64f7622..d779b5c96 100644 --- a/hw/bsp/stm32u5/family.c +++ b/hw/bsp/stm32u5/family.c @@ -203,7 +203,8 @@ void board_init(void) { //--------------------------------------------------------------------+ void board_led_write(bool state) { - HAL_GPIO_WritePin(LED_PORT, LED_PIN, state ? LED_STATE_ON : (1 - LED_STATE_ON)); + GPIO_PinState pin_state = (GPIO_PinState) (state ? LED_STATE_ON : (1 - LED_STATE_ON)); + HAL_GPIO_WritePin(LED_PORT, LED_PIN, pin_state); } uint32_t board_button_read(void) { diff --git a/hw/bsp/stm32wb/family.c b/hw/bsp/stm32wb/family.c index 1dc789407..6051388a7 100644 --- a/hw/bsp/stm32wb/family.c +++ b/hw/bsp/stm32wb/family.c @@ -136,7 +136,8 @@ void board_init(void) { //--------------------------------------------------------------------+ void board_led_write(bool state) { - HAL_GPIO_WritePin(LED_PORT, LED_PIN, state ? LED_STATE_ON : (1 - LED_STATE_ON)); + GPIO_PinState pin_state = (GPIO_PinState) (state ? LED_STATE_ON : (1 - LED_STATE_ON)); + HAL_GPIO_WritePin(LED_PORT, LED_PIN, pin_state); } uint32_t board_button_read(void) { @@ -174,7 +175,7 @@ uint32_t board_millis(void) { #endif void HardFault_Handler(void) { - asm("bkpt"); + asm("bkpt 1"); } // Required by __libc_init_array in startup code if we are compiling using diff --git a/tools/build.py b/tools/build.py index 967f7c95e..b937a7342 100644 --- a/tools/build.py +++ b/tools/build.py @@ -1,4 +1,5 @@ import argparse +import random import os import sys import time @@ -96,32 +97,61 @@ def build_board_cmake(board, toolchain): return ret -def build_family(family, toolchain, build_system): +def build_board_make_all_examples(board, toolchain, all_examples): + start_time = time.monotonic() + ret = [0, 0, 0] + + with Pool(processes=os.cpu_count()) as pool: + pool_args = list((map(lambda e, b=board, o=f"TOOLCHAIN={toolchain}": [e, b, o], all_examples))) + r = pool.starmap(build_utils.build_example, pool_args) + # sum all element of same index (column sum) + rsum = list(map(sum, list(zip(*r)))) + ret[0] += rsum[0] + ret[1] += rsum[1] + ret[2] += rsum[2] + duration = time.monotonic() - start_time + if ret[1] == 0: + status = SUCCEEDED + else: + status = FAILED + + flash_size = "-" + sram_size = "-" + example = 'all' + title = build_utils.build_format.format(example, board, status, "{:.2f}s".format(duration), flash_size, sram_size) + print(title) + return ret + + +def build_family(family, toolchain, build_system, one_per_family, boards): all_boards = [] for entry in os.scandir(f"hw/bsp/{family}/boards"): if entry.is_dir() and entry.name != 'pico_sdk': all_boards.append(entry.name) all_boards.sort() - # success, failed, skipped ret = [0, 0, 0] - if build_system == 'cmake': - for board in all_boards: - if build_board_cmake(board, toolchain): - ret[0] += 1 - else: - ret[1] += 1 - elif build_system == 'make': - all_examples = get_examples(family) - for example in all_examples: - with Pool(processes=os.cpu_count()) as pool: - pool_args = list((map(lambda b, e=example, o=f"TOOLCHAIN={toolchain}": [e, b, o], all_boards))) - r = pool.starmap(build_utils.build_example, pool_args) - # sum all element of same index (column sum) - rsum = list(map(sum, list(zip(*r)))) - ret[0] += rsum[0] - ret[1] += rsum[1] - ret[2] += rsum[2] + + # If only-one flag is set, select one random board + if one_per_family: + for b in boards: + # skip if -b already specify one in this family + if find_family(b) == family: + return ret + all_boards = [random.choice(all_boards)] + + # success, failed, skipped + all_examples = get_examples(family) + for board in all_boards: + r = [0, 0, 0] + if build_system == 'cmake': + r = build_board_cmake(board, toolchain) + elif build_system == 'make': + r = build_board_make_all_examples(board, toolchain, all_examples) + ret[0] += r[0] + ret[1] += r[1] + ret[2] += r[2] + return ret @@ -131,12 +161,14 @@ def main(): parser.add_argument('-b', '--board', action='append', default=[], help='Boards to build') parser.add_argument('-t', '--toolchain', default='gcc', help='Toolchain to use, default is gcc') parser.add_argument('-s', '--build-system', default='cmake', help='Build system to use, default is cmake') + parser.add_argument('-1', '--one-per-family', action='store_true', default=False, help='Build only one random board inside a family') args = parser.parse_args() families = args.families boards = args.board toolchain = args.toolchain build_system = args.build_system + one_per_family = args.one_per_family if len(families) == 0 and len(boards) == 0: print("Please specify families or board to build") @@ -145,50 +177,42 @@ def main(): print(build_separator) print(build_utils.build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM')) total_time = time.monotonic() - total_result = [0, 0, 0] - - # build families: cmake, make - if families is not None: - all_families = [] - if 'all' in families: - for entry in os.scandir("hw/bsp"): - if entry.is_dir() and entry.name != 'espressif' and os.path.isfile(entry.path + "/family.cmake"): - all_families.append(entry.name) - else: - all_families = list(families) - all_families.sort() - - # succeeded, failed - for f in all_families: - fret = build_family(f, toolchain, build_system) - total_result[0] += fret[0] - total_result[1] += fret[1] - total_result[2] += fret[2] - - # build board (only cmake) - if boards is not None: - for b in boards: - if build_system == 'cmake': - r = build_board_cmake(b, toolchain) - total_result[0] += r[0] - total_result[1] += r[1] - total_result[2] += r[2] - elif build_system == 'make': - all_examples = get_examples(find_family(b)) - with Pool(processes=os.cpu_count()) as pool: - pool_args = list((map(lambda e, bb=b, o=f"TOOLCHAIN={toolchain}": [e, bb, o], all_examples))) - r = pool.starmap(build_utils.build_example, pool_args) - # sum all element of same index (column sum) - rsum = list(map(sum, list(zip(*r)))) - total_result[0] += rsum[0] - total_result[1] += rsum[1] - total_result[2] += rsum[2] + result = [0, 0, 0] + + # build families + all_families = [] + if 'all' in families: + for entry in os.scandir("hw/bsp"): + if entry.is_dir() and entry.name != 'espressif' and os.path.isfile(entry.path + "/family.cmake"): + all_families.append(entry.name) + else: + all_families = list(families) + all_families.sort() + + # succeeded, failed + for f in all_families: + fret = build_family(f, toolchain, build_system, one_per_family, boards) + result[0] += fret[0] + result[1] += fret[1] + result[2] += fret[2] + + # build boards + for b in boards: + r = [0, 0, 0] + if build_system == 'cmake': + r = build_board_cmake(b, toolchain) + elif build_system == 'make': + all_examples = get_examples(find_family(b)) + r = build_board_make_all_examples(b, toolchain, all_examples) + result[0] += r[0] + result[1] += r[1] + result[2] += r[2] total_time = time.monotonic() - total_time print(build_separator) - print(f"Build Summary: {total_result[0]} {SUCCEEDED}, {total_result[1]} {FAILED} and took {total_time:.2f}s") + print(f"Build Summary: {result[0]} {SUCCEEDED}, {result[1]} {FAILED} and took {total_time:.2f}s") print(build_separator) - return total_result[1] + return result[1] if __name__ == '__main__': -- cgit v1.3.1