Merge remote-tracking branch 'remotes/hathach/master' into HostLogsFixes

11 files changed, 193 insertions, 45 deletions

diff --git a/.github/workflows/build_aarch64.yml b/.github/workflows/build_aarch64.yml
index 6ac7ad015..8a91ee630 100644
--- a/.github/workflows/build_aarch64.yml
+++ b/.github/workflows/build_aarch64.yml
@@ -73,7 +73,7 @@ jobs:
       run: python3 tools/get_deps.py ${{ matrix.family }}
 
     - name: Build
-      run: python3 tools/build_family.py ${{ matrix.family }}
+      run: python3 tools/build_make.py ${{ matrix.family }}
 
     - name: Linker Map
       run: |
diff --git a/.github/workflows/build_arm.yml b/.github/workflows/build_arm.yml
index 171c1fec3..402b89e0a 100644
--- a/.github/workflows/build_arm.yml
+++ b/.github/workflows/build_arm.yml
@@ -66,7 +66,7 @@ jobs:
       run: python3 tools/get_deps.py ${{ matrix.family }}
 
     - name: Build
-      run: python3 tools/build_family.py ${{ matrix.family }}
+      run: python3 tools/build_make.py ${{ matrix.family }}
 
     - name: Linker Map
       run: |
diff --git a/.github/workflows/build_esp.yml b/.github/workflows/build_esp.yml
index 897616f09..4568695c2 100644
--- a/.github/workflows/build_esp.yml
+++ b/.github/workflows/build_esp.yml
@@ -32,7 +32,7 @@ jobs:
         # ESP32-S2
         - 'espressif_kaluga_1'
         # ESP32-S3
-        - 'espressif_s3_devkitm'
+        - 'espressif_s3_devkitc'
 
     steps:
     - name: Setup Python
diff --git a/.github/workflows/build_msp430.yml b/.github/workflows/build_msp430.yml
index c62056940..60c5feef3 100644
--- a/.github/workflows/build_msp430.yml
+++ b/.github/workflows/build_msp430.yml
@@ -71,7 +71,7 @@ jobs:
       run: python3 tools/get_deps.py ${{ matrix.family }}
 
     - name: Build
-      run: python3 tools/build_family.py ${{ matrix.family }}
+      run: python3 tools/build_make.py ${{ matrix.family }}
 
     - name: Linker Map
       run: |
diff --git a/.github/workflows/build_renesas.yml b/.github/workflows/build_renesas.yml
index 66b98a71b..1cc4f8132 100644
--- a/.github/workflows/build_renesas.yml
+++ b/.github/workflows/build_renesas.yml
@@ -71,7 +71,7 @@ jobs:
       run: python3 tools/get_deps.py ${{ matrix.family }}
 
     - name: Build
-      run: python3 tools/build_family.py ${{ matrix.family }}
+      run: python3 tools/build_make.py ${{ matrix.family }}
 
     - name: Linker Map
       run: |
diff --git a/.github/workflows/build_riscv.yml b/.github/workflows/build_riscv.yml
index 8ec549072..dfc6b672f 100644
--- a/.github/workflows/build_riscv.yml
+++ b/.github/workflows/build_riscv.yml
@@ -72,7 +72,7 @@ jobs:
       run: python3 tools/get_deps.py ${{ matrix.family }}
 
     - name: Build
-      run: python3 tools/build_family.py ${{ matrix.family }}
+      run: python3 tools/build_make.py ${{ matrix.family }}
 
     - name: Linker Map
       run: |
diff --git a/.github/workflows/build_win_mac.yml b/.github/workflows/build_win_mac.yml
index cb879a705..f6a42cf0a 100644
--- a/.github/workflows/build_win_mac.yml
+++ b/.github/workflows/build_win_mac.yml
@@ -51,4 +51,4 @@ jobs:
       run: python3 tools/get_deps.py stm32f4
 
     - name: Build
-      run: python3 tools/build_family.py stm32f4 stm32f411disco
+      run: python3 tools/build_make.py stm32f4 stm32f411disco
diff --git a/.github/workflows/cmake_arm.yml b/.github/workflows/cmake_arm.yml
index e57c297d7..f4b93cec7 100644
--- a/.github/workflows/cmake_arm.yml
+++ b/.github/workflows/cmake_arm.yml
@@ -105,15 +105,14 @@ jobs:
 
   # ---------------------------------------
   # Hardware in the loop (HIL)
-  # Current self-hosted instance is running on an RPI4 with
-  # - pico + pico-probe connected via USB
-  # - pico-probe is /dev/ttyACM0
+  # Current self-hosted instance is running on an RPI4.
+  # For attached hardware checkout hil_pi4.json
   # ---------------------------------------
   hw-rp2040-test:
     # run only with hathach's commit due to limited resource on RPI4
     if: github.repository_owner == 'hathach'
     needs: build-arm
-    runs-on: [self-hosted, Linux, ARM64, rp2040]
+    runs-on: [self-hosted, rp2040, hardware-in-the-loop]
 
     steps:
       - name: Clean workspace
@@ -122,43 +121,16 @@ jobs:
           rm -rf "${{ github.workspace }}"
           mkdir -p "${{ github.workspace }}"
 
+      - name: Checkout test/hil
+        uses: actions/checkout@v3
+        with:
+          sparse-checkout: test/hil
+
       - name: Download rp2040 Artifacts
         uses: actions/download-artifact@v3
         with:
           name: rp2040
 
-      - name: Create flash.sh
-        run: |
-          echo > flash.sh 'cmdout=$(openocd -f "interface/cmsis-dap.cfg" -f "target/rp2040.cfg" -c "adapter speed 5000" -c "program $1 reset exit")'
-          echo >> flash.sh 'if (( $? )) ; then echo $cmdout ; fi'
-          chmod +x flash.sh
-
-      - name: Test cdc_dual_ports
-        run: |
-          ./flash.sh cdc_dual_ports.elf
-          while (! ([ -e /dev/ttyACM1 ] && [ -e /dev/ttyACM2 ])) && [ $SECONDS -le 10 ]; do :; done
-          test -e /dev/ttyACM1 && echo "ttyACM1 exists"
-          test -e /dev/ttyACM2 && echo "ttyACM2 exists"
-
-      - name: Test cdc_msc
-        run: |
-          ./flash.sh cdc_msc.elf
-          readme='/media/pi/TinyUSB MSC/README.TXT'
-          while (! ([ -e /dev/ttyACM1 ] && [ -f "$readme" ])) && [ $SECONDS -le 10 ]; do :; done
-          test -e /dev/ttyACM1 && echo "ttyACM1 exists"
-          test -f "$readme" && echo "$readme exists"
-          cat "$readme"
-
-      - name: Test dfu
-        run: |
-          ./flash.sh dfu.elf
-          while (! (dfu-util -l | grep "Found DFU")) && [ $SECONDS -le 10 ]; do :; done
-          dfu-util -d cafe -a 0 -U dfu0
-          dfu-util -d cafe -a 1 -U dfu1
-          grep "TinyUSB DFU! - Partition 0" dfu0
-          grep "TinyUSB DFU! - Partition 1" dfu1
-
-      - name: Test dfu_runtime
+      - name: Test on actual hardware (hardware in the loop)
         run: |
-          ./flash.sh dfu_runtime.elf
-          while (! (dfu-util -l | grep "Found Runtime")) && [ $SECONDS -le 10 ]; do :; done
+          python3 test/hil/hil_test.py hil_pi4.json
diff --git a/.github/workflows/codeql-buildscript.sh b/.github/workflows/codeql-buildscript.sh
new file mode 100644
index 000000000..35e029922
--- /dev/null
+++ b/.github/workflows/codeql-buildscript.sh
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+FAMILY=stm32l4
+python3 tools/get_deps.py $FAMILY
+python3 tools/build_make.py $FAMILY
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
new file mode 100644
index 000000000..dec071753
--- /dev/null
+++ b/.github/workflows/codeql.yml
@@ -0,0 +1,137 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ 'master' ]
+    paths:
+      - 'src/**'
+      - 'examples/**'
+      - 'lib/**'
+      - 'hw/**'
+      - '.github/workflows/codeql.yml'
+  pull_request:
+    branches: [ 'master' ]
+    paths:
+      - 'src/**'
+      - 'examples/**'
+      - 'lib/**'
+      - 'hw/**'
+      - '.github/workflows/codeql.yml'
+  schedule:
+    - cron: '0 0 * * *'
+
+jobs:
+  analyze:
+    name: Analyze
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners
+    # Consider using larger runners for possible analysis time improvements.
+    runs-on: ubuntu-latest
+    timeout-minutes: 360
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'c-cpp' ]
+        # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ]
+        # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+
+    - name: Install ARM GCC
+      uses: carlosperate/arm-none-eabi-gcc-action@v1
+      with:
+        release: '11.2-2022.02'
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+        queries: security-and-quality
+
+
+    # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    #- name: Autobuild
+    #  uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    - run: |
+        ./.github/workflows/codeql-buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v2
+      with:
+        category: "/language:${{matrix.language}}"
+        upload: false
+      id: step1
+
+    # Filter out rules with low severity or high false positive rate
+    # Also filter out warnings in third-party code
+    - name: Filter out unwanted errors and warnings
+      uses: advanced-security/filter-sarif@v1
+      with:
+        patterns: |
+          -**:cpp/path-injection
+          -**:cpp/world-writable-file-creation
+          -**:cpp/poorly-documented-function
+          -**:cpp/potentially-dangerous-function
+          -**:cpp/use-of-goto
+          -**:cpp/integer-multiplication-cast-to-long
+          -**:cpp/comparison-with-wider-type
+          -**:cpp/leap-year/*
+          -**:cpp/ambiguously-signed-bit-field
+          -**:cpp/suspicious-pointer-scaling
+          -**:cpp/suspicious-pointer-scaling-void
+          -**:cpp/unsigned-comparison-zero
+          -**/third*party/**
+          -**/3rd*party/**
+          -**/external/**
+        input: ${{ steps.step1.outputs.sarif-output }}/cpp.sarif
+        output: ${{ steps.step1.outputs.sarif-output }}/cpp.sarif
+
+    - name: Upload SARIF
+      uses: github/codeql-action/upload-sarif@v2
+      with:
+        sarif_file: ${{ steps.step1.outputs.sarif-output }}
+        category: "/language:${{matrix.language}}"
+
+    - name: Archive CodeQL results
+      uses: actions/upload-artifact@v3
+      with:
+        name: codeql-results
+        path: ${{ steps.step1.outputs.sarif-output }}
+        retention-days: 5
diff --git a/.github/workflows/fail_on_error.py b/.github/workflows/fail_on_error.py
new file mode 100755
index 000000000..29791742b
--- /dev/null
+++ b/.github/workflows/fail_on_error.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+
+import json
+import sys
+
+# Return whether SARIF file contains error-level results
+def codeql_sarif_contain_error(filename):
+    with open(filename, 'r') as f:
+        s = json.load(f)
+
+    for run in s.get('runs', []):
+        rules_metadata = run['tool']['driver']['rules']
+        if not rules_metadata:
+            rules_metadata = run['tool']['extensions'][0]['rules']
+
+        for res in run.get('results', []):
+            if 'ruleIndex' in res:
+                rule_index = res['ruleIndex']
+            elif 'rule' in res and 'index' in res['rule']:
+                rule_index = res['rule']['index']
+            else:
+                continue
+            try:
+                rule_level = rules_metadata[rule_index]['defaultConfiguration']['level']
+            except IndexError as e:
+                print(e, rule_index, len(rules_metadata))
+            else:
+                if rule_level == 'error':
+                    return True
+    return False
+
+if __name__ == "__main__":
+    if codeql_sarif_contain_error(sys.argv[1]):
+        sys.exit(1)