diff options
Diffstat (limited to '.github/workflows')
| -rw-r--r-- | .github/workflows/build.yml | 432 | ||||
| -rw-r--r-- | .github/workflows/build_util.yml | 77 | ||||
| -rwxr-xr-x | .github/workflows/ci_set_matrix.py | 110 | ||||
| -rw-r--r-- | .github/workflows/pr_comment.yml | 9 | ||||
| -rw-r--r-- | .github/workflows/pre-commit.yml | 1 |
5 files changed, 494 insertions, 135 deletions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 76e19ee02..70555b111 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,31 +37,203 @@ jobs: - 'hw/**' - 'test/hil/**' - 'tools/build.py' + - 'tools/build_utils.py' + - 'tools/ci_select.py' - 'tools/get_deps.py' + - 'tools/metrics.py' + - 'tools/rtt.py' - '.github/actions/**' - '.github/workflows/build.yml' - '.github/workflows/build_util.yml' - - '.github/workflows/ci_set_matrix.py' + - '.github/scripts/**' set-matrix: runs-on: ubuntu-latest outputs: json: ${{ steps.set-matrix-json.outputs.matrix }} hil_json: ${{ steps.set-matrix-json.outputs.hil_matrix }} + example_map: ${{ steps.set-matrix-json.outputs.example_map }} + build_filtered: ${{ steps.set-matrix-json.outputs.build_filtered }} + build_families_regex: ${{ steps.set-matrix-json.outputs.build_families_regex }} + # one pair per rig job: hil-tinyusb (tinyusb.json minus esptool boards), + # hil-tinyusb-esp (esptool boards only), hil-tinyusb (hfp.json) + hil_args_tinyusb: ${{ steps.hil-select.outputs.args_tinyusb }} + hil_run_tinyusb: ${{ steps.hil-select.outputs.run_tinyusb }} + hil_args_tinyusb_esp: ${{ steps.hil-select.outputs.args_tinyusb_esp }} + hil_run_tinyusb_esp: ${{ steps.hil-select.outputs.run_tinyusb_esp }} + hil_args_hfp: ${{ steps.hil-select.outputs.args_hfp }} + hil_run_hfp: ${{ steps.hil-select.outputs.run_hfp }} steps: - name: Checkout TinyUSB uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: CI selection (PR only) + id: hil-select + if: github.event_name == 'pull_request' + env: + BASE_REF: ${{ github.base_ref }} + run: | + # Best-effort by design: set-matrix gates cmake, hil-build and every rig job, + # so a missing origin/<base>, a shallow-clone hiccup or a selector traceback + # must fall back to the FULL matrix (no --select, run=true, no args) instead + # of failing the job. Same fail-open shape as pr_comment.yml's `|| true`. + # + # The selector's own unit suite gates it (stdlib-only, seconds): a selector + # whose tests fail can still exit 0 with valid-but-WRONG JSON -- fail-open alone + # never catches that class, and the pre-commit hil-test hook is a separate, + # advisory workflow that nothing here can `needs:`. Test-failing selector => + # full matrix, same as a crashing one. + SELECT_JSON='' + if ! python3 test/hil/test/test_ci_select.py; then + echo "::warning::ci_select unit suite failed - falling back to the full HIL matrix" + elif ! SELECT_JSON=$(python3 tools/ci_select.py --base "origin/$BASE_REF" test/hil/tinyusb.json test/hil/hfp.json); then + echo "::warning::ci_select failed - falling back to the full HIL matrix" + SELECT_JSON='' + fi + + # The selection is handed on as a FILE in the workspace, never as a step + # output/env var: it is ~KBs normally but a mass-sweep PR reaches hundreds of + # KB, and an env var that big makes the consuming exec fail with E2BIG BEFORE + # any fallback in it can run. Written here, ahead of its first reader. + # No file (non-PR event, or any fallback) = full matrix. + rm -f ci_select_out.json + if [ -n "$SELECT_JSON" ]; then + printf '%s' "$SELECT_JSON" > ci_select_out.json + fi + + # One args/run pair per rig job, split by flasher: a job whose own subset is + # empty skips explicitly instead of running a board filter that matches zero + # boards ("No tests were run." exits 0 and would read as a green HIL run). + OUT='' + if [ -s ci_select_out.json ]; then + OUT=$(python3 -c ' + import json, re, sys + s = json.load(open("ci_select_out.json")) + # the same reading hil_ci_set_matrix.py applies: full false with no usable + # boards map is an UNUSABLE selection, not "nothing selected". Both must agree + # - one falling open to the whole roster while the other computes run=false + # buys a full 37-leg build and still zero hardware coverage. + if not s.get("full") and not isinstance(s.get("boards"), dict): + sys.exit("selection has full false but no usable boards map") + tin = s.get("args_flasher", {}).get("tinyusb.json", {}) + legs = (("tinyusb", " ".join(a for f, a in sorted(tin.items()) if f != "esptool" and a)), + ("tinyusb_esp", tin.get("esptool", "")), + ("hfp", s.get("args", {}).get("hfp.json", ""))) + for key, a in legs: + # roster board names reach $GITHUB_OUTPUT as bare NAME=VALUE lines; a + # newline in one would inject extra run_* lines and flip which rig jobs run. + # ":" and "," are part of the normal shape - a partial filter is + # `-bt <board>:<test>,<test>` (ci_select._board_args) + if not re.fullmatch(r"[-A-Za-z0-9_/ .=+:,]*", a): + sys.exit("unexpected characters in the " + key + " board filter") + print("args_" + key + "=" + a) + print("run_" + key + "=" + ("true" if (s.get("full") or a) else "false")) + ') || OUT='' + if [ -z "$OUT" ]; then + echo "::warning::ci_select output unusable - falling back to the full HIL matrix" + # the same unusable selection must not stay behind for the build axis + rm -f ci_select_out.json + fi + fi + if [ -z "$OUT" ]; then + OUT=$(for k in tinyusb tinyusb_esp hfp; do printf 'args_%s=\nrun_%s=true\n' "$k" "$k"; done) + fi + echo "$OUT" + echo "$OUT" >> $GITHUB_OUTPUT - name: Generate matrix json id: set-matrix-json run: | - # build matrix - MATRIX_JSON=$(python .github/workflows/ci_set_matrix.py) + # Build matrix, scoped by the PR selection when one exists. Best-effort: + # ci_set_matrix falls back to the full matrix itself on unusable JSON, + # and a missing file (non-PR event, selector fallback) means no flags. + SELECT_FILE=ci_select_out.json + [ -s "$SELECT_FILE" ] || SELECT_FILE='' + BUILD_SELECT_FILE="$SELECT_FILE" + MATRIX_JSON='' + if [ -n "$SELECT_FILE" ]; then + # ci_set_matrix falls open on a selection it cannot use with rc 0 - it prints + # the full matrix and says UNSCOPED on stderr. The build extras below must + # not stay scoped when it did, or a nominally full build compiles 1 of 44 + # examples per family and code-metrics compares that partial run against a + # full baseline. Only the BUILD axis is dropped: build.families being + # unusable says nothing about the boards map the HIL matrix reads. + MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py --select-file "$SELECT_FILE" 2>ci_set_matrix.err) || MATRIX_JSON='' + cat ci_set_matrix.err >&2 + if [ -z "$MATRIX_JSON" ] || grep -q 'ci_set_matrix: UNSCOPED' ci_set_matrix.err; then + BUILD_SELECT_FILE='' + fi + fi + [ -z "$MATRIX_JSON" ] && MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py) + + # Build-axis extras: the per-family example map rides as a side channel + # (a value inside matrix entries would break CircleCI's family parameter + # and multiply GHA matrix legs). These stay step outputs - they are small + # derived values, unlike the selection they are read from. NOTE jq's // + # treats false like null, so .build.full is compared explicitly. + EXAMPLE_MAP='{}' + BUILD_FILTERED='false' + FAMILY_REGEX='' + if [ -n "$BUILD_SELECT_FILE" ]; then + EXAMPLE_MAP=$(jq -c '.build.family_examples // {}' "$BUILD_SELECT_FILE") || EXAMPLE_MAP='{}' + BUILD_FILTERED=$(jq -r 'if (.build? | type) == "object" and .build.full == false then "true" else "false" end' "$BUILD_SELECT_FILE") || BUILD_FILTERED='false' + if [ "$BUILD_FILTERED" = "true" ]; then + FAMILY_COUNT=$(jq -r '.build.families | length' "$BUILD_SELECT_FILE") || FAMILY_COUNT=0 + FAMILY_REGEX=$(jq -r '.build.families | join("|")' "$BUILD_SELECT_FILE") || FAMILY_REGEX='' + # family names come from hw/bsp dir names, which rule 6 reads straight out + # of the PR's diff path - and this is interpolated raw into a + # `name_is_regexp` artifact pattern, so a regex metacharacter there would + # silently match another family's baseline + FAMILY_REJECTED=0 + case "$FAMILY_REGEX" in + *[!-A-Za-z0-9_\|]*) + echo "::warning::unexpected characters in the family list - dropping the scoping" + FAMILY_REGEX=''; FAMILY_REJECTED=1 ;; + esac + # An EMPTY families list and a REJECTED one both leave FAMILY_REGEX empty and + # mean opposite things, so branch on which happened. Testing `-z` alone sent + # every nothing-selected PR down the fall-open path: a docs/.gitignore diff + # (#3842) and a test/hil-only diff (#3840) each rebuilt all 74 cmake legs + # after the selector had correctly chosen none. + if [ "$FAMILY_REJECTED" = "1" ]; then + # unusable: fall open, and all three drop together. Resetting only + # build_filtered leaves the build scoped while code-metrics takes the + # UNSCOPED branch, diffing a 1-family run against the full averaged + # baseline and publishing that as the PR's code-size impact. + BUILD_FILTERED='false' + EXAMPLE_MAP='{}' + MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py) + elif [ "$FAMILY_COUNT" = "0" ]; then + # legitimate nothing-selected. MATRIX_JSON already holds the all-empty + # matrix ci_set_matrix produced from this selection - keep it, so every + # leg skips. Nothing is built, so there is nothing to compare a baseline + # against: build_filtered goes false to keep code-metrics off the scoped + # path, and EXAMPLE_MAP stays '{}' (family_examples is empty anyway). + BUILD_FILTERED='false' + fi + fi + fi + # emitted once, after every path that can still change it echo "matrix=$MATRIX_JSON" echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT + echo "example_map=$EXAMPLE_MAP" >> $GITHUB_OUTPUT + echo "build_filtered=$BUILD_FILTERED" >> $GITHUB_OUTPUT + echo "build_families_regex=$FAMILY_REGEX" >> $GITHUB_OUTPUT - # HIL matrix (merged from tinyusb + hifiphile configs) - HIL_MATRIX_JSON=$(python test/hil/hil_ci_set_matrix.py test/hil/tinyusb.json test/hil/hfp.json) + # HIL matrix (merged from tinyusb + hifiphile configs), scoped on PRs. + # Scoping is best-effort too: fall back to the unscoped (full) matrix. + HIL_MATRIX_JSON='' + if [ -n "$SELECT_FILE" ]; then + HIL_MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py --select-file "$SELECT_FILE" test/hil/tinyusb.json test/hil/hfp.json) || HIL_MATRIX_JSON='' + if [ -z "$HIL_MATRIX_JSON" ]; then + echo "::warning::scoped HIL matrix failed - falling back to the full HIL matrix" + fi + fi + if [ -z "$HIL_MATRIX_JSON" ]; then + HIL_MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py test/hil/tinyusb.json test/hil/hfp.json) + fi echo "hil_matrix=$HIL_MATRIX_JSON" echo "hil_matrix=$HIL_MATRIX_JSON" >> $GITHUB_OUTPUT @@ -89,6 +261,7 @@ jobs: toolchain: ${{ matrix.toolchain }} build-args: ${{ toJSON(fromJSON(needs.set-matrix.outputs.json)[matrix.toolchain]) }} build-options: '--one-first' + example-map: ${{ needs.set-matrix.outputs.example_map }} upload-metrics: true upload-artifacts: false upload-membrowse: true @@ -96,8 +269,17 @@ jobs: secrets: inherit code-metrics: - needs: [ check-paths, cmake ] - if: needs.check-paths.outputs.code_changed == 'true' + needs: [ check-paths, cmake, set-matrix ] + # A scoped selection can empty every cmake toolchain (a test/hil-only PR). This + # job must still run then: skipping it leaves the sticky comment showing the + # PREVIOUS push's size table as if it were current. set-matrix must have + # SUCCEEDED though: !cancelled() alone let a failed set-matrix through, and this + # job would then overwrite the sticky comment with a wrong "built no families" + # diagnosis while reporting itself green. + if: | + !cancelled() && needs.check-paths.outputs.code_changed == 'true' && + needs.set-matrix.result == 'success' && + (needs.cmake.result == 'success' || needs.cmake.result == 'skipped') runs-on: ubuntu-latest permissions: pull-requests: write @@ -114,8 +296,21 @@ jobs: pattern: metrics-* path: cmake-build merge-multiple: true + # download-artifact does not fail on a pattern that matches nothing, so a + # scoped PR that built no family simply lands here with an empty dir + + - name: Detect empty metrics set + run: | + # No metrics at all => nothing to aggregate or compare. Write the marker the + # sticky comment will carry, so the size section says "skipped" for THIS push + # instead of silently keeping the previous push's table. + if ! ls cmake-build/*/metrics.json >/dev/null 2>&1; then + echo "_Code-size comparison skipped: PR selection built no families on this push._" > metrics_compare.md + echo "NO_METRICS=true" >> $GITHUB_ENV + fi - name: Aggregate Code Metrics + if: env.NO_METRICS != 'true' run: | python tools/get_deps.py python tools/metrics.py combine -j -m -f tinyusb/src cmake-build/*/metrics.json @@ -128,7 +323,7 @@ jobs: path: metrics.json - name: Download Base Branch Metrics - if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + if: env.NO_METRICS != 'true' && (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch') && needs.set-matrix.outputs.build_filtered != 'true' uses: dawidd6/action-download-artifact@v11 with: workflow: build.yml @@ -138,6 +333,29 @@ jobs: path: base-metrics continue-on-error: true + - name: Download base per-family metrics (scoped PR) + if: env.NO_METRICS != 'true' && github.event_name == 'pull_request' && needs.set-matrix.outputs.build_filtered == 'true' + uses: dawidd6/action-download-artifact@v11 + with: + workflow: build.yml + workflow_conclusion: '' + search_artifacts: true # a docs-only master push uploads no per-family artifacts + branch: ${{ github.base_ref }} + name: ^metrics-(${{ needs.set-matrix.outputs.build_families_regex }})$ + name_is_regexp: true + path: base-family-metrics + continue-on-error: true + + - name: Compare with Base Branch (scoped) + if: env.NO_METRICS != 'true' && github.event_name == 'pull_request' && needs.set-matrix.outputs.build_filtered == 'true' + run: | + # never fall back to the averaged metrics-tinyusb here: a scoped PR vs the + # 64-family/46-example average is exactly the mismatch this path prevents + python .github/scripts/metrics_pair_compare.py \ + --base-dir base-family-metrics --new-dir cmake-build --out metrics_compare || \ + echo "_Code-size comparison failed on the scoped path - see the code-metrics job log._" > metrics_compare.md + cat metrics_compare.md + - name: Download Previous Release Asset if: github.event_name == 'release' env: @@ -151,7 +369,7 @@ jobs: gh release download $PREV_TAG -p metrics.json -D base-metrics || echo "No metrics.json found in $PREV_TAG release" - name: Compare with Base Branch - if: github.event_name != 'push' + if: env.NO_METRICS != 'true' && github.event_name != 'push' && needs.set-matrix.outputs.build_filtered != 'true' run: | if [ -f base-metrics/metrics.json ]; then python tools/metrics.py compare -m -f tinyusb/src base-metrics/metrics.json metrics.json @@ -179,6 +397,9 @@ jobs: path: | metrics_compare.md metrics.json + # metrics.json is absent when the selection built no family; the marker + # in metrics_compare.md is still what the sticky comment needs + if-no-files-found: ignore - name: Post Code Metrics as PR Comment if: (github.event_name == 'workflow_dispatch') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) @@ -271,6 +492,18 @@ jobs: strategy: fail-fast: false matrix: + # These names are the bucket keys of .github/scripts/hil_ci_set_matrix.py: every + # non-esptool roster board must land in one of them (esptool boards go to + # 'esp-idf', built by hil-build-esp below). hil_ci_set_matrix.py rejects a + # board whose "toolchain" is not a bucket, so a new bucket must be added in + # both places. + # + # INVARIANT the PR-scoped skip cascade rests on: hil_run_tinyusb / hil_run_hfp + # true => hil-build has at least one non-empty leg. It holds because those + # flags count only non-esptool boards and every such board builds here. It + # would break if an esptool board were added to hfp.json, because + # hil_args_hfp is NOT flasher-split: the hfp leg of hil-tinyusb would want to + # run while hil-build (and therefore that leg) skipped. toolchain: - 'arm-gcc' - 'riscv-gcc' @@ -298,8 +531,16 @@ jobs: # self-hosted on local VM, for attached hardware checkout HIL_JSON # --------------------------------------- hil-tinyusb: - needs: hil-build + needs: [ hil-build, set-matrix ] name: hil-tinyusb (${{ matrix.display }}) + # Above hil_test.py's pool guard (HIL_POOL_TIMEOUT, 60 min) so the guard fires first + # and still gets to write its report. The 30 min on top is what the job pays OUTSIDE + # the guard clock: workspace cleanup, checkout, the multi-board artifact merge and + # the D-state note before it; kill_worker_children, shutdown_pool's 30 s grace, the + # report write and the upload after it. On a multi-stray convoy that tail alone is + # minutes, and a ceiling below guard+tail cancels the job before hil_report.md exists + # -- the inversion this branch removes. Both legs share the script and the guard. + timeout-minutes: 90 strategy: fail-fast: false matrix: @@ -316,6 +557,9 @@ jobs: test_args: '' runs-on: ${{ matrix.runner }} env: + # HIL_POOL_TIMEOUT deliberately unset: hil_test.py's 60 min default is below every + # ceiling here, so ceiling > guard holds by construction. Pin it to SHORTEN a run + # only -- pinning it above a ceiling re-inverts the two. HIL_JSON: ${{ matrix.hil_json }} steps: - name: Set HIL report dir (per run+job; persists across run attempts) @@ -357,7 +601,31 @@ jobs: - name: Test on actual hardware # Single attempt per test (--retry 1), no in-run second pass: a broken fixture # fails fast instead of holding the runner (and other PRs' HIL jobs) for hours. - run: python3 test/hil/hil_test.py --retry 1 ${{ matrix.test_args }} ${{ env.HIL_JSON }} $RERUN_ARGS + env: + # tinyusb.json minus the esptool boards (they run in hil-tinyusb-esp): each + # job gates on its own flasher subset, never on a rig-wide flag + SEL_ARGS_TINYUSB: ${{ needs.set-matrix.outputs.hil_args_tinyusb }} + SEL_RUN_TINYUSB: ${{ needs.set-matrix.outputs.hil_run_tinyusb }} + SEL_ARGS_HFP: ${{ needs.set-matrix.outputs.hil_args_hfp }} + SEL_RUN_HFP: ${{ needs.set-matrix.outputs.hil_run_hfp }} + run: | + case "$HIL_JSON" in + *tinyusb.json) SEL_ARGS="$SEL_ARGS_TINYUSB"; SEL_RUN="$SEL_RUN_TINYUSB" ;; + *hfp.json) SEL_ARGS="$SEL_ARGS_HFP"; SEL_RUN="$SEL_RUN_HFP" ;; + esac + if [ "$SEL_RUN" = "false" ]; then + echo "HIL skipped by PR selection (no affected boards on this rig)" + # leave a marker so the combined PR comment says so instead of dropping the + # section (and leaving a stale table from an earlier push in its place) + mkdir -p "$HIL_REPORT_DIR" + echo "_Skipped by PR selection: no affected boards on this rig._" > "$HIL_REPORT_DIR/hil_report.md" + exit 0 + fi + # a re-run spec is already a subset of the selection (only the boards/tests + # that failed); -b/-bt accumulate, so keeping SEL_ARGS here would re-run the + # entire original selection instead of just what failed + if [ -n "$RERUN_ARGS" ]; then SEL_ARGS=''; fi + python3 test/hil/hil_test.py --retry 1 ${{ matrix.test_args }} $SEL_ARGS ${{ env.HIL_JSON }} $RERUN_ARGS - name: Upload HIL report if: always() && github.event_name == 'pull_request' @@ -376,9 +644,14 @@ jobs: # second slot would double the per-controller flash/usbtest budgets. # --------------------------------------- hil-tinyusb-esp: - needs: hil-build-esp + needs: [ hil-build-esp, set-matrix ] name: hil-tinyusb (tinyusb-esp.json) runs-on: [ self-hosted, X64, hathach, hardware-in-the-loop ] + # above hil_test.py's pool guard (60 min) with room for the pre-pool checkout + # and the post-guard sweep + report upload, so its own guard still writes a report; + # only a job wedged past that (unkillable D-state worker) hits this ceiling, which + # must exist because the runner has one job slot and holds every queued job hostage + timeout-minutes: 90 env: HIL_JSON: test/hil/tinyusb.json TEST_ARGS: '--flasher esptool' @@ -420,7 +693,25 @@ jobs: merge-multiple: true - name: Test on actual hardware - run: python3 test/hil/hil_test.py --retry 1 $TEST_ARGS ${{ env.HIL_JSON }} $RERUN_ARGS + env: + # esptool subset of tinyusb.json: this job must gate on its own boards, not + # on the rig-wide flag (which would run a filter matching zero boards) + SEL_ARGS: ${{ needs.set-matrix.outputs.hil_args_tinyusb_esp }} + SEL_RUN: ${{ needs.set-matrix.outputs.hil_run_tinyusb_esp }} + run: | + if [ "$SEL_RUN" = "false" ]; then + echo "HIL skipped by PR selection (no affected esptool boards)" + # leave a marker so the combined PR comment says so instead of dropping the + # section (and leaving a stale table from an earlier push in its place) + mkdir -p "$HIL_REPORT_DIR" + echo "_Skipped by PR selection: no affected esptool boards._" > "$HIL_REPORT_DIR/hil_report.md" + exit 0 + fi + # a re-run spec is already a subset of the selection (only the boards/tests + # that failed); -b/-bt accumulate, so keeping SEL_ARGS here would re-run the + # entire original selection instead of just what failed + if [ -n "$RERUN_ARGS" ]; then SEL_ARGS=''; fi + python3 test/hil/hil_test.py --retry 1 $TEST_ARGS $SEL_ARGS ${{ env.HIL_JSON }} $RERUN_ARGS - name: Upload HIL report if: always() && github.event_name == 'pull_request' @@ -443,7 +734,13 @@ jobs: github.repository_owner == 'hathach' && !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == true) runs-on: [ self-hosted, Linux, X64, hifiphile ] - timeout-minutes: 30 + # Unlike the hil-tinyusb jobs, this one BUILDS with IAR in the same job before running + # hil_test.py -- hfp.json's 3 boards, 4 variant entries, "up to 30 minutes" (see the + # comment above the selection step). The ceiling has to cover build + the 60 min pool + # guard + overhead, or GitHub cancels before the guard can write its report -- the + # inversion this branch removes. 30 + 60 = 90; the remaining 30 is the full-history + # checkout, get_deps, the post-guard sweep and the report upload. + timeout-minutes: 120 env: IAR_LMS_BEARER_TOKEN: ${{ secrets.IAR_LMS_BEARER_TOKEN }} PYTHONUNBUFFERED: '1' @@ -460,23 +757,102 @@ jobs: - name: Checkout TinyUSB uses: actions/checkout@v6 + with: + # full history: the "HIL selection" step below needs + # merge-base(HEAD, origin/<base_ref>) for PR-scoped selection + fetch-depth: 0 + + # Computed BEFORE the build: the IAR build is four boards and up to 30 minutes on + # a runner that hil-tinyusb (hfp.json) also needs, so an unaffected PR must release + # it immediately instead of building everything and then skipping. The selection + # also narrows what gets built. + # This job has no needs: on set-matrix (it must run even if that unrelated job + # fails), so it computes its own selection instead of reading set-matrix's outputs. + - name: HIL selection (PR only) + if: github.event_name == 'pull_request' + env: + BASE_REF: ${{ github.base_ref }} + run: | + # Best-effort: this job is deliberately decoupled from set-matrix so unrelated + # failures cannot kill hfp coverage - a selector failure here must likewise + # fall back to the full hfp matrix (no ci_select.json, no SEL_* vars), never + # fail the job. + if ! python3 test/hil/test/test_ci_select.py; then + echo "::warning::ci_select unit suite failed - running the full hfp matrix" + rm -f ci_select.json + exit 0 + fi + if ! python3 tools/ci_select.py --base "origin/$BASE_REF" test/hil/hfp.json > ci_select.json; then + echo "::warning::ci_select failed - running the full hfp matrix" + rm -f ci_select.json + exit 0 + fi + # ci_select.json is passed to hil_ci_set_matrix.py --select below to scope the + # build; it already honours full=true by ignoring the board list. + # The hil_test.py args go to a file, never to $GITHUB_ENV: they are derived + # from roster board names, which a PR can edit. Only SEL_RUN (a literal + # true/false computed here, needed by the step-level `if:`) goes to the env. + if ! SEL_RUN=$(python3 -c ' + import json + s = json.load(open("ci_select.json")) + a = s["args"]["hfp.json"] + open("hil_sel_args.txt", "w").write(a) + print("true" if (s["full"] or a) else "false") + '); then + echo "::warning::ci_select output unusable - running the full hfp matrix" + rm -f ci_select.json hil_sel_args.txt + exit 0 + fi + echo "SEL_RUN=$SEL_RUN" + echo "SEL_RUN=$SEL_RUN" >> $GITHUB_ENV - name: Get build boards + if: env.SEL_RUN != 'false' run: | - MATRIX_JSON=$(python test/hil/hil_ci_set_matrix.py test/hil/hfp.json) - BUILD_ARGS=$(echo $MATRIX_JSON | jq -r '.["arm-gcc"] | join(" ")') + # --select-file, never --select "$(cat ...)": a whole selection as one argv + # can exceed MAX_ARG_STRLEN on a big diff, and this job's design is to fall + # back to the full hfp matrix on any selector trouble, not to fail the step. + MATRIX_JSON='' + if [ -f ci_select.json ]; then + MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py --select-file ci_select.json test/hil/hfp.json) || MATRIX_JSON='' + if [ -z "$MATRIX_JSON" ]; then + echo "::warning::scoped hfp matrix failed - building the full hfp matrix" + fi + fi + if [ -z "$MATRIX_JSON" ]; then + MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py test/hil/hfp.json) + fi + # Each variant carries its own --build-name/--cflag, which are global to a + # single build.py invocation — so keep one matrix entry per line and build + # them one at a time (joining would leak a variant's flags onto every board). + echo "$MATRIX_JSON" | jq -r '.["arm-gcc"][]' > hil_build_entries.txt + cat hil_build_entries.txt + BUILD_ARGS=$(echo "$MATRIX_JSON" | jq -r '.["arm-gcc"] | join(" ")') + # board and example names are roster data a PR can edit, and jq -r un-escapes + # them: a newline here writes extra NAME=VALUE lines into GITHUB_ENV for every + # later step of a job that holds the IAR token. Refuse rather than guess. + case "$BUILD_ARGS" in + *[!-A-Za-z0-9_/\ .=+]*) + echo "::error::unexpected characters in the hfp build args"; exit 1 ;; + esac echo "BUILD_ARGS=$BUILD_ARGS" echo "BUILD_ARGS=$BUILD_ARGS" >> $GITHUB_ENV - name: Get Dependencies + if: env.SEL_RUN != 'false' run: python3 tools/get_deps.py $BUILD_ARGS - name: Build + if: env.SEL_RUN != 'false' + # Bounded SEPARATELY from the job. This is the only HIL job that builds inline + # (hil-tinyusb downloads artifacts), and the job ceiling went 30 -> 120 to give the + # HIL step room -- which would hand a stalled IAR build the whole two hours on the + # shared self-hosted runner, never reaching hil_test.py or the report upload. That + # is the stranded-runner-with-no-report failure this branch exists to prevent. + # Typical full build here is a few minutes; 30 leaves generous headroom. + timeout-minutes: 30 run: | - # Each variant carries its own --build-name/--cflag, which are global to a - # single build.py invocation — so build one matrix entry at a time rather - # than joining them (joining would leak a variant's flags onto every board). - readarray -t ENTRIES < <(python test/hil/hil_ci_set_matrix.py test/hil/hfp.json | jq -r '.["arm-gcc"][]') + readarray -t ENTRIES < hil_build_entries.txt for entry in "${ENTRIES[@]}"; do echo "+ tools/build.py --toolchain iar $entry" python3 tools/build.py --toolchain iar $entry @@ -484,7 +860,21 @@ jobs: - name: Test on actual hardware (hardware in the loop) run: | - python3 test/hil/hil_test.py hfp.json + if [ "$SEL_RUN" = "false" ]; then + echo "HIL skipped by PR selection (no affected boards on this rig)" + # leave a marker so the combined PR comment says so instead of dropping + # the section (and leaving a stale table from an earlier push) + echo "_Skipped by PR selection: no affected boards on this rig._" > hil_report.md + exit 0 + fi + # empty/absent on a non-PR event or a selector fallback -> full hfp matrix + SEL_ARGS=$(cat hil_sel_args.txt 2>/dev/null || true) + # --retry 1, like the other two HIL legs. The pool guard is a FLAT 3600s and + # does NOT scale with max_retry, so argparse's default of 3 would multiply the + # serialized usbtest tail (hfp.json runs four batteries) by three against an + # unchanged guard -- on a runner with a single job slot that queues every other + # job behind it. + python3 test/hil/hil_test.py --retry 1 $SEL_ARGS hfp.json - name: Upload HIL report if: always() && github.event_name == 'pull_request' diff --git a/.github/workflows/build_util.yml b/.github/workflows/build_util.yml index 90115862b..407ed1e71 100644 --- a/.github/workflows/build_util.yml +++ b/.github/workflows/build_util.yml @@ -20,6 +20,10 @@ on: required: false default: '' type: string + example-map: + required: false + default: '' + type: string upload-artifacts: required: false default: false @@ -39,6 +43,21 @@ on: jobs: family: + # PR-scoped HIL selection can produce an empty build-args list for a toolchain + # (e.g. a dwc2-only change with no riscv boards affected); an empty matrix + # vector fails the job outright ("Matrix vector 'arg' does not contain any + # values"), so skip cleanly instead. + # + # Why that is safe for callers: GitHub SKIPS the dependents of a skipped + # `needs:` job, so this only works because the caller (hil-build) is itself a + # *matrix* job - a matrix with one skipped leg and one successful leg + # aggregates to success, and its dependents run. + # + # NOT covered: if every leg is empty the whole caller job skips, and so does + # everything that needs it. That is fine only because an all-empty selection + # means no board was selected for those rigs, so the rig jobs would have had + # nothing to run anyway (see the invariant on hil-build in build.yml). + if: inputs.build-args != '[]' runs-on: ${{ inputs.os }} strategy: fail-fast: false @@ -61,19 +80,42 @@ jobs: with: arg: ${{ matrix.arg }} + - name: Resolve PR example filter + if: inputs.example-map != '' && inputs.example-map != '{}' + env: + # values are PR-derived - keep them out of ${{ }} script interpolation + # (env expansion word-splits but never re-parses shell metacharacters) + EXAMPLE_MAP: ${{ inputs.example-map }} + FAMILY: ${{ matrix.arg }} + run: | + # -e flags for this family; a family absent from the map builds everything + EX_ARGS=$(printf '%s' "$EXAMPLE_MAP" | jq -r --arg fam "$FAMILY" '(.[$fam] // []) | map("-e " + .) | join(" ")') || EX_ARGS='' + # the map's values are example dir names from the PR checkout, and `jq -r` + # un-escapes them: a path with a newline (git allows it) would otherwise write + # extra NAME=VALUE lines into GITHUB_ENV for every later step of this job. + # Anything outside the example-name alphabet drops the filter (= build all), + # which is the safe direction. + case "$EX_ARGS" in + *[!-A-Za-z0-9_/\ ]*) + echo "::warning::unexpected characters in the example filter - building all examples" + EX_ARGS='' ;; + esac + echo "EX_ARGS=$EX_ARGS" + echo "EX_ARGS=$EX_ARGS" >> $GITHUB_ENV + - name: Build if: ${{ inputs.code-changed }} env: IAR_LMS_BEARER_TOKEN: ${{ secrets.IAR_LMS_BEARER_TOKEN }} run: | if [ "${{ inputs.toolchain }}" == "esp-idf" ]; then - docker run --rm -e MEMBROWSE_API_KEY="$MEMBROWSE_API_KEY" -e CI="$CI" -v $PWD:/project -w /project espressif/idf:tinyusb python tools/build.py --target all ${{ matrix.arg }} + docker run --rm -e MEMBROWSE_API_KEY="$MEMBROWSE_API_KEY" -e CI="$CI" -v $PWD:/project -w /project espressif/idf:tinyusb python tools/build.py --target all ${{ matrix.arg }} $EX_ARGS else BUILD_PY_ARGS="-s ${{ inputs.build-system }} ${{ steps.setup-toolchain.outputs.build_option }} ${{ inputs.build-options }} --target all" if [ "${{ inputs.upload-metrics }}" = "true" ]; then BUILD_PY_ARGS="$BUILD_PY_ARGS --target tinyusb_metrics" fi - python tools/build.py $BUILD_PY_ARGS ${{ matrix.arg }} + python tools/build.py $BUILD_PY_ARGS ${{ matrix.arg }} $EX_ARGS fi shell: bash @@ -84,6 +126,9 @@ jobs: MEMBROWSE_API_KEY: ${{ secrets.MEMBROWSE_API_KEY }} run: | # if code-changed is false --> there is no elf -> membrowse target upload with --identical flag + # deliberately unscoped by $EX_ARGS: keeps the size history on a stable board + # per family, at the cost of an --identical-only upload where that board is not + # the one the Build step picked (test_ci_metrics pins which families those are) BUILD_PY_ARGS="-s ${{ inputs.build-system }} ${{ steps.setup-toolchain.outputs.build_option }} ${{ inputs.build-options }}" python tools/build.py $BUILD_PY_ARGS --target examples-membrowse-upload -j 1 ${{ matrix.arg }} shell: bash @@ -93,13 +138,37 @@ jobs: uses: actions/upload-artifact@v7 with: name: metrics-${{ matrix.arg }} - path: cmake-build/cmake-build-*/metrics.json + path: | + cmake-build/cmake-build-*/metrics.json + cmake-build/cmake-build-*/metrics_by_example.json + + - name: Artifact name + if: inputs.upload-artifacts == true + env: + ARG: ${{ matrix.arg }} + run: | + # -e example filters carry '/', which upload-artifact forbids in artifact + # names; strip them from the NAME only (the build already consumed them). + # Names without -e stay byte-identical to before. Two entries differing + # only in their -e list cannot exist - the -e list is a function of + # (board), and variant suffixes (--build-name/-D/--cflag) survive the + # strip - so the stripped name is still unique per matrix entry. + TAG=$(printf '%s' "$ARG" | sed -E 's/ -e [^ ]+//g') + # board and example names come from the roster, which a PR can edit; a newline + # in one would write extra NAME=VALUE lines into GITHUB_ENV for every later + # step. There is no safe fallback name here - a wrong one mislabels the + # firmware the rig then flashes - so refuse instead. + case "$TAG" in + *[!-A-Za-z0-9_/\ .=+]*) + echo "::error::refusing to build an artifact name from '$ARG'"; exit 1 ;; + esac + echo "ARTIFACT_TAG=$TAG" >> $GITHUB_ENV - name: Upload Artifacts for Hardware Testing if: inputs.upload-artifacts == true && inputs.code-changed == true uses: actions/upload-artifact@v7 with: - name: binaries-${{ inputs.toolchain }}-${{ matrix.arg }} + name: binaries-${{ inputs.toolchain }}-${{ env.ARTIFACT_TAG }} path: | cmake-build/cmake-build-*/*/*/*.elf cmake-build/cmake-build-*/*/*/*.bin diff --git a/.github/workflows/ci_set_matrix.py b/.github/workflows/ci_set_matrix.py deleted file mode 100755 index dc0d3871f..000000000 --- a/.github/workflows/ci_set_matrix.py +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env python3 -import json - -# toolchain, url -toolchain_list = [ - "aarch64-gcc", - "arm-clang", - "arm-iar", - "arm-gcc", - "esp-idf", - "ft9xx-gcc", - "msp430-gcc", - "riscv-gcc", - "rx-gcc" -] - -# family: [supported toolchain] -family_list = { - "at32f402_405": ["arm-gcc"], - "at32f403a_407": ["arm-gcc"], - "at32f413": ["arm-gcc"], - "at32f415": ["arm-gcc"], - "at32f423": ["arm-gcc"], - "at32f425": ["arm-gcc"], - "at32f435_437": ["arm-gcc"], - "at32f45x": ["arm-gcc"], - "broadcom_32bit": ["arm-gcc"], - "broadcom_64bit": ["aarch64-gcc"], - "ch32f20x": ["arm-gcc"], - "ch32v10x": ["riscv-gcc"], - "ch32v20x": ["riscv-gcc"], - "ch32v30x": ["riscv-gcc"], - "ch583": ["riscv-gcc"], - "da1469x": ["arm-gcc"], - "fomu": ["riscv-gcc"], - "ft9xx": ["ft9xx-gcc"], - "gd32vf103": ["riscv-gcc"], - "hpmicro": ["riscv-gcc"], - "imxrt": ["arm-gcc", "arm-clang"], - "kinetis_k": ["arm-gcc"], - "kinetis_k32l": ["arm-gcc"], - "kinetis_kl": ["arm-gcc"], - "lpc11": ["arm-gcc", "arm-clang"], - "lpc13": ["arm-gcc", "arm-clang"], - "lpc15": ["arm-gcc", "arm-clang"], - "lpc17": ["arm-gcc", "arm-clang"], - "lpc18": ["arm-gcc", "arm-clang"], - "lpc40": ["arm-gcc", "arm-clang"], - "lpc43": ["arm-gcc", "arm-clang"], - "lpc51": ["arm-gcc", "arm-clang"], - "lpc54": ["arm-gcc", "arm-clang"], - "lpc55": ["arm-gcc", "arm-clang"], - "maxim": ["arm-gcc"], - "mcx": ["arm-gcc"], - "mm32": ["arm-gcc"], - "msp430": ["msp430-gcc"], - "msp432e4": ["arm-gcc"], - "nrf": ["arm-gcc", "arm-clang"], - "nuc100_120": ["arm-gcc"], - "nuc121_125": ["arm-gcc"], - "nuc126": ["arm-gcc"], - "nuc505": ["arm-gcc"], - "ra": ["arm-gcc"], - "rp2040": ["arm-gcc"], - "rw61x": ["arm-gcc"], - "rx": ["rx-gcc"], - "samd11": ["arm-gcc", "arm-clang"], - "samd2x_l2x": ["arm-gcc", "arm-clang"], - "samd5x_e5x": ["arm-gcc", "arm-clang"], - "samg": ["arm-gcc", "arm-clang"], - "stm32c0": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32c5": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32f0": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32f1": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32f2": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32f3": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32f4": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32f7": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32g0": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32g4": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32h5": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32h7": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32h7rs": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32l0": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32l4": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32n6": ["arm-gcc"], - "stm32u0": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32u5": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32wb": ["arm-gcc", "arm-clang", "arm-iar"], - "stm32wba": ["arm-gcc", "arm-clang", "arm-iar"], - "tm4c": ["arm-gcc"], - "xmc4000": ["arm-gcc"], - # S3, P4 will be built by hil test - # "-bespressif_s3_devkitm": ["esp-idf"], - # "-bespressif_p4_function_ev": ["esp-idf"], -} - - -def set_matrix_json(): - matrix = {} - for toolchain in toolchain_list: - filtered_families = [family for family, supported_toolchain in family_list.items() if - toolchain in supported_toolchain] - matrix[toolchain] = filtered_families - - print(json.dumps(matrix)) - - -if __name__ == '__main__': - set_matrix_json() diff --git a/.github/workflows/pr_comment.yml b/.github/workflows/pr_comment.yml index 4d50817b4..868e56405 100644 --- a/.github/workflows/pr_comment.yml +++ b/.github/workflows/pr_comment.yml @@ -101,7 +101,16 @@ jobs: shopt -s nullglob dirs=(hil-reports/hil-report-*) if [ ${#dirs[@]} -eq 0 ]; then + # No rig produced a report: PR selection matched no board anywhere, or the + # HIL jobs did not run at all. Post it rather than exiting, so a table from + # an earlier push is replaced instead of being left to look current. echo "No HIL reports found" + { + echo "## Hardware-in-the-loop (HIL) Test Report" + echo + echo "_No HIL run for this push (no affected boards, or hardware testing did not run)._" + } > hil_combined.md + echo "found=true" >> "$GITHUB_OUTPUT" exit 0 fi { diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 70dd3894d..09f912bd3 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -30,6 +30,7 @@ jobs: #cd test/unit-test #ceedling test:all + # runs --all-files, so the hil-test hook fires here regardless of its `files:` scope - name: Run pre-commit uses: pre-commit/[email protected] |