diff options
Diffstat (limited to '.github')
| -rwxr-xr-x | .github/scripts/ci_set_matrix.py | 88 | ||||
| -rw-r--r-- | .github/scripts/hil_ci_set_matrix.py | 49 | ||||
| -rwxr-xr-x | .github/scripts/metrics_pair_compare.py | 130 | ||||
| -rw-r--r-- | .github/workflows/build.yml | 227 | ||||
| -rw-r--r-- | .github/workflows/build_util.yml | 69 |
5 files changed, 510 insertions, 53 deletions
diff --git a/.github/scripts/ci_set_matrix.py b/.github/scripts/ci_set_matrix.py index 50ada5964..79f466893 100755 --- a/.github/scripts/ci_set_matrix.py +++ b/.github/scripts/ci_set_matrix.py @@ -1,5 +1,9 @@ #!/usr/bin/env python3 +import argparse import json +import os +import subprocess +import sys # toolchain, url toolchain_list = [ @@ -97,15 +101,87 @@ family_list = { } -def set_matrix_json(): +def set_matrix_json(select=None): + sel_fams = None + if select: + # every shape check is explicit: this runs AFTER main()'s fail-open handler, so + # an AttributeError on e.g. {"build": ["stm32f4"]} would red the step instead + # of falling back to the full matrix - the outcome that handler exists to prevent + b = select.get('build') if isinstance(select, dict) else None + if not isinstance(b, dict): + b = {} + if b.get('full') is False: + fams = b.get('families') + if not (isinstance(fams, list) and all(isinstance(f, str) for f in fams)): + # key ABSENT (or not a list of names) is an unusable selection, not + # "nothing selected": scoping every toolchain to [] would build zero + # families and report a vacuous green. An explicit families: [] stays a + # legitimate nothing-selected. + print('ci_set_matrix: UNSCOPED - build.full is false but the families ' + 'list is unusable, emitting the full matrix', file=sys.stderr) + else: + sel_fams = set(fams) matrix = {} for toolchain in toolchain_list: - filtered_families = [family for family, supported_toolchain in family_list.items() if - toolchain in supported_toolchain] - matrix[toolchain] = filtered_families - + fams = [family for family, tc in family_list.items() if toolchain in tc] + if sel_fams is not None: + fams = [f for f in fams if f in sel_fams] + matrix[toolchain] = fams + if sel_fams: + # a family this file does not list builds on no toolchain, so it contributes no + # leg. hw/bsp holds several CI has never built (efm32, py32f0, same7x, ...) plus + # espressif, whose boards hil-build-esp builds by name. + unbuilt = sorted(f for f in sel_fams if f not in family_list) + if unbuilt and not any(matrix.values()): + # NONE of the selected families is buildable here, so every leg would skip + # and the PR would go green from a build job that ran no compiler. That is + # an unusable selection, not "nothing selected": say UNSCOPED - which + # build.yml and .circleci/config.yml both grep for - and emit the full + # matrix. An explicit families: [] is still a legitimate nothing-selected, + # and a PARTIAL miss still scopes to the families that do build. + print(f'ci_set_matrix: UNSCOPED - no selected family is built by any ' + f'toolchain here ({", ".join(unbuilt)}), emitting the full matrix', + file=sys.stderr) + return set_matrix_json(None) + if unbuilt: + print(f'ci_set_matrix: selected families built by no toolchain here: ' + f'{", ".join(unbuilt)}', file=sys.stderr) print(json.dumps(matrix)) +def main(): + parser = argparse.ArgumentParser() + group = parser.add_mutually_exclusive_group() + group.add_argument('--select', help='tools/ci_select.py JSON; scopes families when build.full is false') + # a whole selection as one argv/env value can exceed the exec limits on a big + # diff, which fails the calling step BEFORE it can fall open; callers that + # already have the selection on disk pass the path instead + group.add_argument('--select-file', help='file holding the same JSON as --select') + group.add_argument('--base', help='git ref: run tools/ci_select.py --base REF and scope from it') + args = parser.parse_args() + + select = None + try: + if args.select: + select = json.loads(args.select) + elif args.select_file: + with open(args.select_file) as f: + select = json.load(f) + elif args.base: + root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + r = subprocess.run([sys.executable, os.path.join(root, 'tools', 'ci_select.py'), + '--base', args.base], + capture_output=True, text=True, cwd=root, check=True) + select = json.loads(r.stdout) + except Exception as e: # fail-open: an unusable selection must never turn into a red job + # UNSCOPED is the marker build.yml greps for: it must then drop the build extras + # (example map, family regex) too, or a full build gets labelled and filtered as + # a scoped one. Keep the token on every fall-open path. + print(f'ci_set_matrix: UNSCOPED - selection unusable ({e}), emitting the full ' + f'matrix', file=sys.stderr) + select = None + set_matrix_json(select) + + if __name__ == '__main__': - set_matrix_json() + main() diff --git a/.github/scripts/hil_ci_set_matrix.py b/.github/scripts/hil_ci_set_matrix.py index 65f50788e..bf50061dd 100644 --- a/.github/scripts/hil_ci_set_matrix.py +++ b/.github/scripts/hil_ci_set_matrix.py @@ -1,6 +1,7 @@ import argparse import json import os +import sys def _resolve_config_path(config_file): @@ -19,13 +20,47 @@ def _resolve_config_path(config_file): def main(): parser = argparse.ArgumentParser() parser.add_argument('config_files', nargs='+', help='Configuration JSON file(s)') - parser.add_argument('--select', help='hil_select.py JSON; scopes boards when full=false') + g = parser.add_mutually_exclusive_group() + g.add_argument('--select', help='ci_select.py JSON; scopes boards when full=false') + # a whole selection as one argv can exceed MAX_ARG_STRLEN on a big diff, which + # would fail the step instead of falling open; callers that already have the + # selection on disk pass the path instead + g.add_argument('--select-file', help='file holding the same JSON as --select') args = parser.parse_args() + raw = args.select + sel = None + try: + if args.select_file: + with open(args.select_file) as f: + raw = f.read() + if raw: + sel = json.loads(raw) + if sel is not None and not isinstance(sel, dict): + raise ValueError(f'selection is {type(sel).__name__}, not an object') + except Exception as e: # fail-open: an unusable selection must never red the job + print(f'hil_ci_set_matrix: selection unusable ({e}) - full roster', + file=sys.stderr) + sel = None + selected = None - sel = json.loads(args.select) if args.select else None if sel and not sel.get('full'): - selected = set(sel.get('boards', {})) + # key ABSENT is an unusable selection, not "nothing selected" - same reading as + # ci_set_matrix.py. Filtering every board out would skip every hil-build leg and, + # through needs:, both rig jobs: an all-green PR with zero hardware coverage. + # An explicit boards: {} stays a legitimate nothing-selected. + if not isinstance(sel.get('boards'), dict): + print('hil_ci_set_matrix: selection has full false but no usable boards ' + 'map - full roster', file=sys.stderr) + sel = None # ALL of it is unusable, hil_examples included: keeping + # the -e lists would build a few examples per board + # while the rig, unfiltered, runs that board's whole + # test list - flash failures on the fail-open path + else: + selected = set(sel['boards']) + ex_map = (sel or {}).get('hil_examples') or {} + if not isinstance(ex_map, dict): + ex_map = {} # Toolchain buckets must match the toolchains instantiated by the hil-build # job in .github/workflows/build.yml. Keep all keys present (even if empty) @@ -68,8 +103,12 @@ def main(): f'hil-build-esp jobs in .github/workflows/build.yml') build_board = f'-b {name}' - if 'build' in board and 'args' in board['build']: - build_board += ' ' + ' '.join(f'-D{a}' for a in board['build']['args']) + + # PR selection: build only the examples this board will run (its test + # list plus device/board_test, the parking firmware) - tools/build.py -e. + # Absent key (hand runs, full non-PR builds) keeps --target all. + for ex in ex_map.get(name, []): + build_board += f' -e {ex}' # Each variant builds into cmake-build-<variant.name> with its own cmake # -D defines and raw CFLAGS. No 'variant' -> a single build named after diff --git a/.github/scripts/metrics_pair_compare.py b/.github/scripts/metrics_pair_compare.py new file mode 100755 index 000000000..50107cf72 --- /dev/null +++ b/.github/scripts/metrics_pair_compare.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Board+example-matched code-size compare for PR-scoped builds. + +The averaged metrics baseline (metrics-tinyusb) spans every family and example; +a scoped PR builds a subset, so comparing against it is apples-to-oranges. This +compares the intersection of (board, example) pairs present on BOTH sides, +averaged over exactly those pairs, and names what was dropped. See +docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md #code-metrics. +""" +import argparse +import glob +import json +import os +import sys +import tempfile + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'tools')) +import metrics + +# dropped (board, example) pairs named in the PR comment before it truncates +DROPPED_SHOWN = 20 + + +def board_family(board, repo_root): + hits = glob.glob(os.path.join(repo_root, 'hw/bsp/*/boards', board)) + return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None + + +def collect(root, repo_root): + """{(board, 'role/example'): [file entries]} from every + **/cmake-build-<board>/metrics_by_example.json under root. + + Keyed on the BOARD, not its family. The two sides are built by + `--one-first`, which returns all_boards[0] for a family with no + ci_preferred_boards entry - so a PR that adds hw/bsp/<family>/boards/a_new_board + shifts which board is built, and a family key would file the base run's sizes and + the PR run's sizes under the same name and publish the difference between two + unrelated MCUs as this PR's code-size impact. On the board key that mismatch lands + in `dropped` (reported as not compared), which is the truth.""" + pairs = {} + pat = os.path.join(root, '**', 'metrics_by_example.json') + for f in sorted(glob.glob(pat, recursive=True)): + board = os.path.basename(os.path.dirname(f)) + if not board.startswith('cmake-build-'): + print(f'pair_compare: {f} not under a cmake-build-<board> dir, skipping', file=sys.stderr) + continue + board = board[len('cmake-build-'):] + if not board_family(board, repo_root): + # unknown board: the name is still a usable key, but say so - it means the + # artifact came from a tree whose hw/bsp does not match this checkout + print(f'pair_compare: no family for board {board}', file=sys.stderr) + # parse into a LOCAL dict and merge only once the whole file came out clean: + # a file that blows up half way through must drop WHOLE, or the entries read + # before the malformation stay in the comparison while stderr says the file + # was skipped, and a silently truncated table gets published as the verdict + try: + one = {} + for ex, ent in json.load(open(f)).items(): + one.setdefault((board, ex), []).extend(ent.get('files', [])) + except (OSError, ValueError, AttributeError, TypeError) as e: + print(f'pair_compare: unreadable {f} ({e}), skipping', file=sys.stderr) + continue + for k, v in one.items(): + pairs.setdefault(k, []).extend(v) + return pairs + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument('--base-dir', required=True) + ap.add_argument('--new-dir', required=True) + ap.add_argument('--out', default='metrics_compare') + a = ap.parse_args() + repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + base = collect(a.base_dir, repo_root) + new = collect(a.new_dir, repo_root) + common = sorted(set(base) & set(new)) + dropped = sorted(set(base) ^ set(new)) + + if not common: + with open(a.out + '.md', 'w') as f: + if new and not base: + # interim state: master has not uploaded a per-example baseline yet. + # Blaming the PR's scoping for that sends people hunting the wrong bug + f.write('_No per-example baseline from the base branch yet (the first ' + 'master push after this feature merges uploads it); comparison ' + 'will appear on the next push._\n') + else: + f.write('_Code-size comparison skipped: no (board, example) pair was ' + 'built on both the base branch and this PR._\n') + return + + def synth(pairs, path): + with open(path, 'w') as f: + json.dump({'files': [e for k in common for e in pairs[k]]}, f) + + with tempfile.TemporaryDirectory() as td: + b, n = os.path.join(td, 'base.json'), os.path.join(td, 'new.json') + synth(base, b) + synth(new, n) + comparison = metrics.compare_files(b, n, ['tinyusb/src']) + if comparison is None: + with open(a.out + '.md', 'w') as f: + f.write('_Code-size comparison failed to produce data._\n') + return + metrics.write_compare_markdown(comparison, a.out + '.md', 'name+') + + with open(a.out + '.md', 'a') as f: + boards = sorted({k[0] for k in common}) + f.write(f'\n_Scoped compare: {len(common)} (board, example) pairs across ' + f'{", ".join(boards)}._\n') + if dropped: + # GitHub caps a comment at 65,536 chars and this footer rides inside the + # sticky code-metrics comment: a broad scoped PR drops hundreds of pairs, + # and the raw list alone reached ~65KB and reddened the whole job. Only a + # summary goes in the comment; the full list goes to the job log. + names = [f'{board}:{ex}' for board, ex in dropped] + print('pair_compare: not compared (missing on one side): ' + + ', '.join(names), file=sys.stderr) + more = len(names) - DROPPED_SHOWN + f.write(f'_Not compared (missing on one side): {len(names)} pairs - ' + + ', '.join(names[:DROPPED_SHOWN]) + + (f', ... and {more} more (see the code-metrics job log)' + if more > 0 else '') + + '._\n') + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8f6014f48..39a4e7afd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,7 +37,10 @@ jobs: - 'hw/**' - 'test/hil/**' - 'tools/build.py' + - 'tools/build_utils.py' + - 'tools/ci_select.py' - 'tools/get_deps.py' + - 'tools/metrics.py' - '.github/actions/**' - '.github/workflows/build.yml' - '.github/workflows/build_util.yml' @@ -48,6 +51,9 @@ jobs: outputs: json: ${{ steps.set-matrix-json.outputs.matrix }} hil_json: ${{ steps.set-matrix-json.outputs.hil_matrix }} + example_map: ${{ steps.set-matrix-json.outputs.example_map }} + build_filtered: ${{ steps.set-matrix-json.outputs.build_filtered }} + build_families_regex: ${{ steps.set-matrix-json.outputs.build_families_regex }} # one pair per rig job: hil-tinyusb (tinyusb.json minus esptool boards), # hil-tinyusb-esp (esptool boards only), hil-tinyusb (hfp.json) hil_args_tinyusb: ${{ steps.hil-select.outputs.args_tinyusb }} @@ -62,9 +68,14 @@ jobs: with: fetch-depth: 0 - - name: HIL selection (PR only) + # The `ci-full` PR label turns the scoping off for one PR: no selection file is + # written, so both matrices and every rig job fall back to the unscoped behaviour. + # An escape hatch is the point - a selector bug under-selects SILENTLY, and without + # a label the only routes back to a full matrix are accidental (touch an + # unclassified path, or break the selector badly enough that it falls open). + - name: CI selection (PR only) id: hil-select - if: github.event_name == 'pull_request' + if: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ci-full') env: BASE_REF: ${{ github.base_ref }} run: | @@ -79,55 +90,133 @@ jobs: # advisory workflow that nothing here can `needs:`. Test-failing selector => # full matrix, same as a crashing one. SELECT_JSON='' - if ! python3 test/hil/test/test_hil_select.py; then - echo "::warning::hil_select unit suite failed - falling back to the full HIL matrix" - elif ! SELECT_JSON=$(python3 test/hil/helper/hil_select.py --base "origin/$BASE_REF" test/hil/tinyusb.json test/hil/hfp.json); then - echo "::warning::hil_select failed - falling back to the full HIL matrix" + if ! python3 test/hil/test/test_ci_select.py; then + echo "::warning::ci_select unit suite failed - falling back to the full HIL matrix" + elif ! SELECT_JSON=$(python3 tools/ci_select.py --base "origin/$BASE_REF" test/hil/tinyusb.json test/hil/hfp.json); then + echo "::warning::ci_select failed - falling back to the full HIL matrix" SELECT_JSON='' fi + # The selection is handed on as a FILE in the workspace, never as a step + # output/env var: it is ~KBs normally but a mass-sweep PR reaches hundreds of + # KB, and an env var that big makes the consuming exec fail with E2BIG BEFORE + # any fallback in it can run. Written here, ahead of its first reader. + # No file (non-PR event, or any fallback) = full matrix. + rm -f ci_select_out.json + if [ -n "$SELECT_JSON" ]; then + printf '%s' "$SELECT_JSON" > ci_select_out.json + fi + # One args/run pair per rig job, split by flasher: a job whose own subset is # empty skips explicitly instead of running a board filter that matches zero # boards ("No tests were run." exits 0 and would read as a green HIL run). OUT='' - if [ -n "$SELECT_JSON" ]; then - OUT=$(SELECT_JSON="$SELECT_JSON" python3 -c ' - import json, os - s = json.loads(os.environ["SELECT_JSON"]) + if [ -s ci_select_out.json ]; then + OUT=$(python3 -c ' + import json, re, sys + s = json.load(open("ci_select_out.json")) + # the same reading hil_ci_set_matrix.py applies: full false with no usable + # boards map is an UNUSABLE selection, not "nothing selected". Both must agree + # - one falling open to the whole roster while the other computes run=false + # buys a full 37-leg build and still zero hardware coverage. + if not s.get("full") and not isinstance(s.get("boards"), dict): + sys.exit("selection has full false but no usable boards map") tin = s.get("args_flasher", {}).get("tinyusb.json", {}) legs = (("tinyusb", " ".join(a for f, a in sorted(tin.items()) if f != "esptool" and a)), ("tinyusb_esp", tin.get("esptool", "")), ("hfp", s.get("args", {}).get("hfp.json", ""))) for key, a in legs: + # roster board names reach $GITHUB_OUTPUT as bare NAME=VALUE lines; a + # newline in one would inject extra run_* lines and flip which rig jobs run. + # ":" and "," are part of the normal shape - a partial filter is + # `-bt <board>:<test>,<test>` (ci_select._board_args) + if not re.fullmatch(r"[-A-Za-z0-9_/ .=+:,]*", a): + sys.exit("unexpected characters in the " + key + " board filter") print("args_" + key + "=" + a) print("run_" + key + "=" + ("true" if (s.get("full") or a) else "false")) ') || OUT='' if [ -z "$OUT" ]; then - echo "::warning::hil_select output unusable - falling back to the full HIL matrix" - SELECT_JSON='' + echo "::warning::ci_select output unusable - falling back to the full HIL matrix" + # the same unusable selection must not stay behind for the build axis + rm -f ci_select_out.json fi fi if [ -z "$OUT" ]; then OUT=$(for k in tinyusb tinyusb_esp hfp; do printf 'args_%s=\nrun_%s=true\n' "$k" "$k"; done) fi echo "$OUT" - { echo "select=$SELECT_JSON"; echo "$OUT"; } >> $GITHUB_OUTPUT + echo "$OUT" >> $GITHUB_OUTPUT - name: Generate matrix json id: set-matrix-json - env: - SELECT: ${{ steps.hil-select.outputs.select }} run: | - # build matrix - MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py) + # Build matrix, scoped by the PR selection when one exists. Best-effort: + # ci_set_matrix falls back to the full matrix itself on unusable JSON, + # and a missing file (non-PR event, selector fallback) means no flags. + SELECT_FILE=ci_select_out.json + [ -s "$SELECT_FILE" ] || SELECT_FILE='' + BUILD_SELECT_FILE="$SELECT_FILE" + MATRIX_JSON='' + if [ -n "$SELECT_FILE" ]; then + # ci_set_matrix falls open on a selection it cannot use with rc 0 - it prints + # the full matrix and says UNSCOPED on stderr. The build extras below must + # not stay scoped when it did, or a nominally full build compiles 1 of 44 + # examples per family and code-metrics compares that partial run against a + # full baseline. Only the BUILD axis is dropped: build.families being + # unusable says nothing about the boards map the HIL matrix reads. + MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py --select-file "$SELECT_FILE" 2>ci_set_matrix.err) || MATRIX_JSON='' + cat ci_set_matrix.err >&2 + if [ -z "$MATRIX_JSON" ] || grep -q 'ci_set_matrix: UNSCOPED' ci_set_matrix.err; then + BUILD_SELECT_FILE='' + fi + fi + [ -z "$MATRIX_JSON" ] && MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py) + + # Build-axis extras: the per-family example map rides as a side channel + # (a value inside matrix entries would break CircleCI's family parameter + # and multiply GHA matrix legs). These stay step outputs - they are small + # derived values, unlike the selection they are read from. NOTE jq's // + # treats false like null, so .build.full is compared explicitly. + EXAMPLE_MAP='{}' + BUILD_FILTERED='false' + FAM_REGEX='' + if [ -n "$BUILD_SELECT_FILE" ]; then + EXAMPLE_MAP=$(jq -c '.build.family_examples // {}' "$BUILD_SELECT_FILE") || EXAMPLE_MAP='{}' + BUILD_FILTERED=$(jq -r 'if (.build? | type) == "object" and .build.full == false then "true" else "false" end' "$BUILD_SELECT_FILE") || BUILD_FILTERED='false' + if [ "$BUILD_FILTERED" = "true" ]; then + FAM_REGEX=$(jq -r '.build.families | join("|")' "$BUILD_SELECT_FILE") || FAM_REGEX='' + # family names come from hw/bsp dir names, which rule 6 reads straight out + # of the PR's diff path - and this is interpolated raw into a + # `name_is_regexp` artifact pattern, so a regex metacharacter there would + # silently match another family's baseline + case "$FAM_REGEX" in + *[!-A-Za-z0-9_\|]*) + echo "::warning::unexpected characters in the family list - dropping the scoping" + FAM_REGEX='' ;; + esac + if [ -z "$FAM_REGEX" ]; then + # all three drop together, as CircleCI's fall-open does. Resetting only + # build_filtered leaves the build scoped while code-metrics takes the + # UNSCOPED branch, diffing a 1-family run against the full averaged + # baseline and publishing that as the PR's code-size impact. + BUILD_FILTERED='false' + EXAMPLE_MAP='{}' + MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py) + fi + fi + fi + # emitted once, after every path that can still change it echo "matrix=$MATRIX_JSON" echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT + echo "example_map=$EXAMPLE_MAP" >> $GITHUB_OUTPUT + echo "build_filtered=$BUILD_FILTERED" >> $GITHUB_OUTPUT + echo "build_families_regex=$FAM_REGEX" >> $GITHUB_OUTPUT # HIL matrix (merged from tinyusb + hifiphile configs), scoped on PRs. # Scoping is best-effort too: fall back to the unscoped (full) matrix. HIL_MATRIX_JSON='' - if [ -n "$SELECT" ]; then - HIL_MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py --select "$SELECT" test/hil/tinyusb.json test/hil/hfp.json) || HIL_MATRIX_JSON='' + if [ -n "$SELECT_FILE" ]; then + HIL_MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py --select-file "$SELECT_FILE" test/hil/tinyusb.json test/hil/hfp.json) || HIL_MATRIX_JSON='' if [ -z "$HIL_MATRIX_JSON" ]; then echo "::warning::scoped HIL matrix failed - falling back to the full HIL matrix" fi @@ -162,6 +251,7 @@ jobs: toolchain: ${{ matrix.toolchain }} build-args: ${{ toJSON(fromJSON(needs.set-matrix.outputs.json)[matrix.toolchain]) }} build-options: '--one-first' + example-map: ${{ needs.set-matrix.outputs.example_map }} upload-metrics: true upload-artifacts: false upload-membrowse: true @@ -169,8 +259,17 @@ jobs: secrets: inherit code-metrics: - needs: [ check-paths, cmake ] - if: needs.check-paths.outputs.code_changed == 'true' + needs: [ check-paths, cmake, set-matrix ] + # A scoped selection can empty every cmake toolchain (a test/hil-only PR). This + # job must still run then: skipping it leaves the sticky comment showing the + # PREVIOUS push's size table as if it were current. set-matrix must have + # SUCCEEDED though: !cancelled() alone let a failed set-matrix through, and this + # job would then overwrite the sticky comment with a wrong "built no families" + # diagnosis while reporting itself green. + if: | + !cancelled() && needs.check-paths.outputs.code_changed == 'true' && + needs.set-matrix.result == 'success' && + (needs.cmake.result == 'success' || needs.cmake.result == 'skipped') runs-on: ubuntu-latest permissions: pull-requests: write @@ -187,8 +286,21 @@ jobs: pattern: metrics-* path: cmake-build merge-multiple: true + # download-artifact does not fail on a pattern that matches nothing, so a + # scoped PR that built no family simply lands here with an empty dir + + - name: Detect empty metrics set + run: | + # No metrics at all => nothing to aggregate or compare. Write the marker the + # sticky comment will carry, so the size section says "skipped" for THIS push + # instead of silently keeping the previous push's table. + if ! ls cmake-build/*/metrics.json >/dev/null 2>&1; then + echo "_Code-size comparison skipped: PR selection built no families on this push._" > metrics_compare.md + echo "NO_METRICS=true" >> $GITHUB_ENV + fi - name: Aggregate Code Metrics + if: env.NO_METRICS != 'true' run: | python tools/get_deps.py python tools/metrics.py combine -j -m -f tinyusb/src cmake-build/*/metrics.json @@ -201,7 +313,7 @@ jobs: path: metrics.json - name: Download Base Branch Metrics - if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + if: env.NO_METRICS != 'true' && (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch') && needs.set-matrix.outputs.build_filtered != 'true' uses: dawidd6/action-download-artifact@v11 with: workflow: build.yml @@ -211,6 +323,29 @@ jobs: path: base-metrics continue-on-error: true + - name: Download base per-family metrics (scoped PR) + if: env.NO_METRICS != 'true' && github.event_name == 'pull_request' && needs.set-matrix.outputs.build_filtered == 'true' + uses: dawidd6/action-download-artifact@v11 + with: + workflow: build.yml + workflow_conclusion: '' + search_artifacts: true # a docs-only master push uploads no per-family artifacts + branch: ${{ github.base_ref }} + name: ^metrics-(${{ needs.set-matrix.outputs.build_families_regex }})$ + name_is_regexp: true + path: base-family-metrics + continue-on-error: true + + - name: Compare with Base Branch (scoped) + if: env.NO_METRICS != 'true' && github.event_name == 'pull_request' && needs.set-matrix.outputs.build_filtered == 'true' + run: | + # never fall back to the averaged metrics-tinyusb here: a scoped PR vs the + # 64-family/46-example average is exactly the mismatch this path prevents + python .github/scripts/metrics_pair_compare.py \ + --base-dir base-family-metrics --new-dir cmake-build --out metrics_compare || \ + echo "_Code-size comparison failed on the scoped path - see the code-metrics job log._" > metrics_compare.md + cat metrics_compare.md + - name: Download Previous Release Asset if: github.event_name == 'release' env: @@ -224,7 +359,7 @@ jobs: gh release download $PREV_TAG -p metrics.json -D base-metrics || echo "No metrics.json found in $PREV_TAG release" - name: Compare with Base Branch - if: github.event_name != 'push' + if: env.NO_METRICS != 'true' && github.event_name != 'push' && needs.set-matrix.outputs.build_filtered != 'true' run: | if [ -f base-metrics/metrics.json ]; then python tools/metrics.py compare -m -f tinyusb/src base-metrics/metrics.json metrics.json @@ -252,6 +387,9 @@ jobs: path: | metrics_compare.md metrics.json + # metrics.json is absent when the selection built no family; the marker + # in metrics_compare.md is still what the sticky comment needs + if-no-files-found: ignore - name: Post Code Metrics as PR Comment if: (github.event_name == 'workflow_dispatch') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) @@ -627,32 +765,32 @@ jobs: run: | # Best-effort: this job is deliberately decoupled from set-matrix so unrelated # failures cannot kill hfp coverage - a selector failure here must likewise - # fall back to the full hfp matrix (no hil_select.json, no SEL_* vars), never + # fall back to the full hfp matrix (no ci_select.json, no SEL_* vars), never # fail the job. - if ! python3 test/hil/test/test_hil_select.py; then - echo "::warning::hil_select unit suite failed - running the full hfp matrix" - rm -f hil_select.json + if ! python3 test/hil/test/test_ci_select.py; then + echo "::warning::ci_select unit suite failed - running the full hfp matrix" + rm -f ci_select.json exit 0 fi - if ! python3 test/hil/helper/hil_select.py --base "origin/$BASE_REF" test/hil/hfp.json > hil_select.json; then - echo "::warning::hil_select failed - running the full hfp matrix" - rm -f hil_select.json + if ! python3 tools/ci_select.py --base "origin/$BASE_REF" test/hil/hfp.json > ci_select.json; then + echo "::warning::ci_select failed - running the full hfp matrix" + rm -f ci_select.json exit 0 fi - # hil_select.json is passed to hil_ci_set_matrix.py --select below to scope the + # ci_select.json is passed to hil_ci_set_matrix.py --select below to scope the # build; it already honours full=true by ignoring the board list. # The hil_test.py args go to a file, never to $GITHUB_ENV: they are derived # from roster board names, which a PR can edit. Only SEL_RUN (a literal # true/false computed here, needed by the step-level `if:`) goes to the env. if ! SEL_RUN=$(python3 -c ' import json - s = json.load(open("hil_select.json")) + s = json.load(open("ci_select.json")) a = s["args"]["hfp.json"] open("hil_sel_args.txt", "w").write(a) print("true" if (s["full"] or a) else "false") '); then - echo "::warning::hil_select output unusable - running the full hfp matrix" - rm -f hil_select.json hil_sel_args.txt + echo "::warning::ci_select output unusable - running the full hfp matrix" + rm -f ci_select.json hil_sel_args.txt exit 0 fi echo "SEL_RUN=$SEL_RUN" @@ -661,9 +799,17 @@ jobs: - name: Get build boards if: env.SEL_RUN != 'false' run: | - if [ -f hil_select.json ]; then - MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py --select "$(cat hil_select.json)" test/hil/hfp.json) - else + # --select-file, never --select "$(cat ...)": a whole selection as one argv + # can exceed MAX_ARG_STRLEN on a big diff, and this job's design is to fall + # back to the full hfp matrix on any selector trouble, not to fail the step. + MATRIX_JSON='' + if [ -f ci_select.json ]; then + MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py --select-file ci_select.json test/hil/hfp.json) || MATRIX_JSON='' + if [ -z "$MATRIX_JSON" ]; then + echo "::warning::scoped hfp matrix failed - building the full hfp matrix" + fi + fi + if [ -z "$MATRIX_JSON" ]; then MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py test/hil/hfp.json) fi # Each variant carries its own --build-name/--cflag, which are global to a @@ -672,6 +818,13 @@ jobs: echo "$MATRIX_JSON" | jq -r '.["arm-gcc"][]' > hil_build_entries.txt cat hil_build_entries.txt BUILD_ARGS=$(echo "$MATRIX_JSON" | jq -r '.["arm-gcc"] | join(" ")') + # board and example names are roster data a PR can edit, and jq -r un-escapes + # them: a newline here writes extra NAME=VALUE lines into GITHUB_ENV for every + # later step of a job that holds the IAR token. Refuse rather than guess. + case "$BUILD_ARGS" in + *[!-A-Za-z0-9_/\ .=+]*) + echo "::error::unexpected characters in the hfp build args"; exit 1 ;; + esac echo "BUILD_ARGS=$BUILD_ARGS" echo "BUILD_ARGS=$BUILD_ARGS" >> $GITHUB_ENV diff --git a/.github/workflows/build_util.yml b/.github/workflows/build_util.yml index 02f16488a..52999616d 100644 --- a/.github/workflows/build_util.yml +++ b/.github/workflows/build_util.yml @@ -20,6 +20,10 @@ on: required: false default: '' type: string + example-map: + required: false + default: '' + type: string upload-artifacts: required: false default: false @@ -76,19 +80,42 @@ jobs: with: arg: ${{ matrix.arg }} + - name: Resolve PR example filter + if: inputs.example-map != '' && inputs.example-map != '{}' + env: + # values are PR-derived - keep them out of ${{ }} script interpolation + # (env expansion word-splits but never re-parses shell metacharacters) + EXAMPLE_MAP: ${{ inputs.example-map }} + FAMILY: ${{ matrix.arg }} + run: | + # -e flags for this family; a family absent from the map builds everything + EX_ARGS=$(printf '%s' "$EXAMPLE_MAP" | jq -r --arg fam "$FAMILY" '(.[$fam] // []) | map("-e " + .) | join(" ")') || EX_ARGS='' + # the map's values are example dir names from the PR checkout, and `jq -r` + # un-escapes them: a path with a newline (git allows it) would otherwise write + # extra NAME=VALUE lines into GITHUB_ENV for every later step of this job. + # Anything outside the example-name alphabet drops the filter (= build all), + # which is the safe direction. + case "$EX_ARGS" in + *[!-A-Za-z0-9_/\ ]*) + echo "::warning::unexpected characters in the example filter - building all examples" + EX_ARGS='' ;; + esac + echo "EX_ARGS=$EX_ARGS" + echo "EX_ARGS=$EX_ARGS" >> $GITHUB_ENV + - name: Build if: ${{ inputs.code-changed }} env: IAR_LMS_BEARER_TOKEN: ${{ secrets.IAR_LMS_BEARER_TOKEN }} run: | if [ "${{ inputs.toolchain }}" == "esp-idf" ]; then - docker run --rm -e MEMBROWSE_API_KEY="$MEMBROWSE_API_KEY" -e CI="$CI" -v $PWD:/project -w /project espressif/idf:tinyusb python tools/build.py --target all ${{ matrix.arg }} + docker run --rm -e MEMBROWSE_API_KEY="$MEMBROWSE_API_KEY" -e CI="$CI" -v $PWD:/project -w /project espressif/idf:tinyusb python tools/build.py --target all ${{ matrix.arg }} $EX_ARGS else BUILD_PY_ARGS="-s ${{ inputs.build-system }} ${{ steps.setup-toolchain.outputs.build_option }} ${{ inputs.build-options }} --target all" if [ "${{ inputs.upload-metrics }}" = "true" ]; then BUILD_PY_ARGS="$BUILD_PY_ARGS --target tinyusb_metrics" fi - python tools/build.py $BUILD_PY_ARGS ${{ matrix.arg }} + python tools/build.py $BUILD_PY_ARGS ${{ matrix.arg }} $EX_ARGS fi shell: bash @@ -99,8 +126,16 @@ jobs: MEMBROWSE_API_KEY: ${{ secrets.MEMBROWSE_API_KEY }} run: | # if code-changed is false --> there is no elf -> membrowse target upload with --identical flag + # $EX_ARGS is passed for the BOARD it picks, not to scope the targets: + # --one-first now chooses a board that can build the -e set (tools/build.py), + # so omitting it here would configure a DIFFERENT, empty build dir and upload + # --identical for a board that was never compiled. The target list is not + # scoped by it - `examples-membrowse-upload` is not `all`, so it passes + # through as the aggregate, which has no DEPENDS (hw/bsp/family_support.cmake): + # it rebuilds nothing and still records every example, --identical for the + # ones without an elf. BUILD_PY_ARGS="-s ${{ inputs.build-system }} ${{ steps.setup-toolchain.outputs.build_option }} ${{ inputs.build-options }}" - python tools/build.py $BUILD_PY_ARGS --target examples-membrowse-upload -j 1 ${{ matrix.arg }} + python tools/build.py $BUILD_PY_ARGS --target examples-membrowse-upload -j 1 ${{ matrix.arg }} $EX_ARGS shell: bash - name: Upload Artifacts for Metrics @@ -108,13 +143,37 @@ jobs: uses: actions/upload-artifact@v7 with: name: metrics-${{ matrix.arg }} - path: cmake-build/cmake-build-*/metrics.json + path: | + cmake-build/cmake-build-*/metrics.json + cmake-build/cmake-build-*/metrics_by_example.json + + - name: Artifact name + if: inputs.upload-artifacts == true + env: + ARG: ${{ matrix.arg }} + run: | + # -e example filters carry '/', which upload-artifact forbids in artifact + # names; strip them from the NAME only (the build already consumed them). + # Names without -e stay byte-identical to before. Two entries differing + # only in their -e list cannot exist - the -e list is a function of + # (board), and variant suffixes (--build-name/-D/--cflag) survive the + # strip - so the stripped name is still unique per matrix entry. + TAG=$(printf '%s' "$ARG" | sed -E 's/ -e [^ ]+//g') + # board and example names come from the roster, which a PR can edit; a newline + # in one would write extra NAME=VALUE lines into GITHUB_ENV for every later + # step. There is no safe fallback name here - a wrong one mislabels the + # firmware the rig then flashes - so refuse instead. + case "$TAG" in + *[!-A-Za-z0-9_/\ .=+]*) + echo "::error::refusing to build an artifact name from '$ARG'"; exit 1 ;; + esac + echo "ARTIFACT_TAG=$TAG" >> $GITHUB_ENV - name: Upload Artifacts for Hardware Testing if: inputs.upload-artifacts == true && inputs.code-changed == true uses: actions/upload-artifact@v7 with: - name: binaries-${{ inputs.toolchain }}-${{ matrix.arg }} + name: binaries-${{ inputs.toolchain }}-${{ env.ARTIFACT_TAG }} path: | cmake-build/cmake-build-*/*/*/*.elf cmake-build/cmake-build-*/*/*/*.bin |