diff options
61 files changed, 7393 insertions, 1588 deletions
diff --git a/.circleci/config.yml b/.circleci/config.yml index 48fa87899..8c3f09111 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,9 +15,87 @@ jobs: - run: name: Set matrix command: | - MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py) + # PR-scoped selection (best-effort: any failure falls back to the full + # matrix). CircleCI has no base-branch var; tinyusb PRs target master. + # The selection lands in a FILE and never travels as an argv: a mass-sweep + # diff selects hundreds of KB, and E2BIG would fail the step before the + # `||` fallback could fire - leaving a full build labelled scoped, because + # EXAMPLE_MAP/BUILD_FILTERED below have no such limit and stay scoped. + SELECT_FILE=ci_select_out.json + rm -f "$SELECT_FILE" + if [ -n "${CIRCLE_PULL_REQUEST:-}" ]; then + git fetch --no-tags origin master || true + # both suites gate the selector: test_ci_select.py owns the rules, + # test_ci_metrics.py owns the config2 sentinel contract this job rewrites + if python3 test/hil/test/test_ci_select.py >/dev/null 2>&1 && + python3 test/hil/test/test_ci_metrics.py >/dev/null 2>&1; then + python3 tools/ci_select.py --base origin/master > "$SELECT_FILE" || rm -f "$SELECT_FILE" + else + echo "ci_select unit suite failed - using the full matrix" + fi + fi + [ -s "$SELECT_FILE" ] || rm -f "$SELECT_FILE" + + # computed once, up front: it is both the fallback and what the scoping is + # dropped back to further down, and a second invocation there would be an + # unguarded command under `set -e` inside the very branch that exists to + # keep the pipeline green + FULL_MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py 2>/dev/null) || FULL_MATRIX_JSON='' + MATRIX_JSON='' + if [ -f "$SELECT_FILE" ]; then + # ci_set_matrix also falls open with rc 0, saying UNSCOPED on stderr. The + # extras below must follow it, exactly as build.yml does: a full matrix + # paired with a still-scoped -e list builds a fraction of each family and + # tells code-metrics it was an unscoped run. + MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py --select-file "$SELECT_FILE" 2>ci_set_matrix.err) || MATRIX_JSON='' + cat ci_set_matrix.err + if [ -z "$MATRIX_JSON" ] || grep -q 'ci_set_matrix: UNSCOPED' ci_set_matrix.err; then + SELECT_FILE='' + fi + fi + [ -n "$MATRIX_JSON" ] || MATRIX_JSON="$FULL_MATRIX_JSON" echo "MATRIX_JSON=$MATRIX_JSON" + EXAMPLE_MAP='{}' + BUILD_FILTERED='false' + if [ -f "$SELECT_FILE" ]; then + EXAMPLE_MAP=$(jq -c '.build.family_examples // {}' < "$SELECT_FILE") || EXAMPLE_MAP='{}' + BUILD_FILTERED=$(jq -r 'if (.build? | type) == "object" and .build.full == false then "true" else "false" end' < "$SELECT_FILE") || BUILD_FILTERED='false' + fi + + # /pipeline/continue caps parameter values at 512 chars - a scoped map is + # KBs, so both values ride inside the generated config itself (config max + # is 3MB), swapped into the parameter defaults by sentinel-line match. + # Fail-open: a sentinel that drifted (renamed comment, reformatted line) + # must not red EVERY CircleCI pipeline. The rewrite is all-or-nothing + # (config2.yml is only written once both substitutions succeeded). + # + # Done BEFORE the family entries are generated, and a failure drops the + # scoping entirely: the checked-in defaults are {} / false = unfiltered, so + # a scoped FAMILY list with unfiltered defaults would build a subset of + # families while telling code-metrics it had built them all. + if ! EXAMPLE_MAP="$EXAMPLE_MAP" BUILD_FILTERED="$BUILD_FILTERED" python3 - \<<'PYEOF' + import os + p = '.circleci/config2.yml' + t = open(p).read() + def yq(s): # YAML single-quoted scalar + return "'" + s.replace("'", "''") + "'" + for env, tag in (('EXAMPLE_MAP', 'example-map-default'), + ('BUILD_FILTERED', 'build-filtered-default')): + old = [l for l in t.splitlines() if l.strip().endswith(f'# {tag}: rewritten in-place by config.yml set-matrix')] + assert len(old) == 1, f'{tag}: sentinel not found exactly once' + line = old[0] + new = line.split('default:')[0] + 'default: ' + yq(os.environ[env]) + f' # {tag}' + t = t.replace(line, new, 1) + open(p, 'w').write(t) + PYEOF + then + echo "warning: sentinel rewrite failed - dropping the scoping, full build" + MATRIX_JSON="$FULL_MATRIX_JSON" + EXAMPLE_MAP='{}' + BUILD_FILTERED='false' + fi + BUILDSYSTEM_LIST=( "cmake" "make" @@ -75,7 +153,15 @@ jobs: FAMILY=$(echo $MATRIX_JSON | jq -r ".\"$toolchain\"") echo "FAMILY_${toolchain}=$FAMILY" + + if [ "$(echo "$FAMILY" | jq 'length')" = "0" ]; then + # an empty matrix parameter is a hard CircleCI config error, not a skip + echo "skip build-${build_system}-${toolchain}: no families selected" + continue + fi + gen_build_entry "$build_system" "$toolchain" "$FAMILY" + ANY_BUILD=1 # Only add cmake builds: excluding esp-idf or build_args="--one-random" to metrics requirements if [ "$build_system" == "cmake" ] && [ "$toolchain" != "esp-idf" ] && [ "$toolchain" != "arm-iar" ]; then @@ -84,12 +170,17 @@ jobs: done done - # Add code-metrics job that requires all build jobs - echo " - code-metrics:" >> .circleci/config2.yml - echo " requires:" >> .circleci/config2.yml - for alias in "${BUILD_ALIASES[@]}"; do - echo " - $alias" >> .circleci/config2.yml - done + if [ ${#BUILD_ALIASES[@]} -gt 0 ]; then + echo " - code-metrics:" >> .circleci/config2.yml + echo " requires:" >> .circleci/config2.yml + for alias in "${BUILD_ALIASES[@]}"; do + echo " - $alias" >> .circleci/config2.yml + done + fi + if [ "${ANY_BUILD:-0}" != "1" ]; then + # a workflow with zero jobs is invalid config + echo " - no-op" >> .circleci/config2.yml + fi - continuation/continue: configuration_path: .circleci/config2.yml diff --git a/.circleci/config2.yml b/.circleci/config2.yml index e0bd917a4..2e69588ae 100644 --- a/.circleci/config2.yml +++ b/.circleci/config2.yml @@ -1,5 +1,13 @@ version: 2.1 +parameters: + example-map: + type: string + default: "{}" # example-map-default: rewritten in-place by config.yml set-matrix + build-filtered: + type: string + default: "false" # build-filtered-default: rewritten in-place by config.yml set-matrix + commands: setup-toolchain: parameters: @@ -109,9 +117,26 @@ commands: - run: name: Build no_output_timeout: 20m + environment: + EXAMPLE_MAP: << pipeline.parameters.example-map >> command: | + # PR example filter for this family ('{}' or a missing key = build all). + # The map is the PR-derived value, so it must ride via env rather than + # shell-text interpolation (unsafe characters); family is a job + # parameter with charset [a-z0-9_], safe to interpolate directly. + EX_ARGS=$(printf '%s' "$EXAMPLE_MAP" | jq -r --arg fam "<< parameters.family >>" '(.[$fam] // []) | map("-e " + .) | join(" ")' 2>/dev/null) || EX_ARGS='' + # same screen as build_util.yml's: the values are example dir names from the + # PR checkout and $EX_ARGS is used unquoted below, so a glob metacharacter + # would pathname-expand against the build cwd. Dropping the filter builds + # everything - the safe direction, and what GHA does for the same input. + case "$EX_ARGS" in + *[!-A-Za-z0-9_/\ ]*) + echo "warning: unexpected characters in the example filter - building all examples" + EX_ARGS='' ;; + esac + if [ << parameters.toolchain >> == esp-idf ]; then - docker run --rm -v $PWD:/project -w /project espressif/idf:v5.5.3 python tools/build.py << parameters.build-args >> --target all << parameters.family >> + docker run --rm -v $PWD:/project -w /project espressif/idf:v5.5.3 python tools/build.py << parameters.build-args >> --target all $EX_ARGS << parameters.family >> else # Toolchain option default is gcc if [ << parameters.toolchain >> == arm-clang ]; then @@ -129,7 +154,7 @@ commands: if [ << parameters.build-system >> == "cmake" ]; then BUILD_PY_ARGS="$BUILD_PY_ARGS --target tinyusb_metrics" fi - python tools/build.py $BUILD_PY_ARGS << parameters.family >> + python tools/build.py $BUILD_PY_ARGS $EX_ARGS << parameters.family >> fi # Only collect and persist metrics for cmake builds (excluding esp-idf and --one-random) @@ -237,8 +262,13 @@ jobs: if ls /tmp/metrics/*/*.json 1> /dev/null 2>&1; then python tools/metrics.py combine -j -m -f tinyusb/src /tmp/metrics/*/*.json else - echo "No metrics files found" - exit 1 + # A scoped PR can legitimately build no metrics leg at all (every selected + # family empty, or none of them on a metrics toolchain), so this is not an + # error any more - it was, when the matrix was always the full 64 families. + # An empty file keeps store_artifacts and the compare step below honest: + # both would otherwise act on a missing path. + echo "No metrics files found - PR selection built no metrics leg" + echo '{"files": []}' > metrics.json fi - store_artifacts: @@ -248,8 +278,10 @@ jobs: # Compare with base master metrics on PR branches - when: condition: - not: - equal: [ master, << pipeline.git.branch >> ] + and: + - not: + equal: [ master, << pipeline.git.branch >> ] + - equal: [ "false", << pipeline.parameters.build-filtered >> ] steps: - run: name: Download Base Branch Metrics @@ -276,6 +308,32 @@ jobs: - store_artifacts: path: metrics_compare.md destination: metrics_compare.md + - when: + condition: + and: + - not: + equal: [ master, << pipeline.git.branch >> ] + - equal: [ "true", << pipeline.parameters.build-filtered >> ] + steps: + - run: + name: Scoped build - comparison unavailable + command: | + # CircleCI stores only the averaged metrics.json; the per-example + # baseline lives on GHA. See the GHA code-metrics PR comment. + echo "_Code-size comparison skipped on CircleCI: this PR built a scoped example set._" > metrics_compare.md + + - store_artifacts: + path: metrics_compare.md + destination: metrics_compare.md + + no-op: + docker: + - image: cimg/base:current + resource_class: small + steps: + - run: + name: No families selected + command: echo "PR selection - no families to build on CircleCI" workflows: build: diff --git a/.claude/skills/hil/SKILL.md b/.claude/skills/hil/SKILL.md index f0c449d33..d1e4bdcd5 100644 --- a/.claude/skills/hil/SKILL.md +++ b/.claude/skills/hil/SKILL.md @@ -43,11 +43,11 @@ Use it before a HIL campaign, after rig maintenance/reboot, or when boards fail ## PR-scoped selection -`test/hil/helper/hil_select.py` maps a diff to affected boards/tests (used by CI on PRs; fail-open +`tools/ci_select.py` maps a diff to affected boards/tests (used by CI on PRs; fail-open to the full matrix). Manual use: ```bash -SEL=$(python3 test/hil/helper/hil_select.py --base master test/hil/tinyusb.json) +SEL=$(python3 tools/ci_select.py --base master test/hil/tinyusb.json) FULL=$(printf '%s' "$SEL" | python3 -c "import json,sys; print(json.load(sys.stdin)['full'])") ARGS=$(printf '%s' "$SEL" | python3 -c "import json,sys; print(json.load(sys.stdin)['args']['tinyusb.json'])") if [ "$FULL" = "True" ] || [ -n "$ARGS" ]; then @@ -60,11 +60,14 @@ fi Read `full`, never `args` alone: `args` is empty for BOTH `full: true` (run the whole matrix — a broad or unclassified change) and "nothing selected" (skip). Skip only when `full` is false AND `args` is empty. -Unit suites (no hardware), all four run by the `hil-test`/`hil-select-test` pre-commit -hooks: `test_hil_select.py` covers only board selection. The containment work --- bounded -reads, the kill ladders, the build and pool guards --- lives in `test_hil_bounded.py`, -`test_hil_health.py` and `test_hil_util.py`, so run all four when changing `test/hil`: -`for f in test/hil/test/test_*.py; do python3 "$f"; done` (~55s). +Unit suites (no hardware), all five run by the `hil-test`/`ci-select-test` pre-commit +hooks: `test_ci_select.py` covers only selection, `test_ci_metrics.py` only the code-size +plumbing. The containment work --- bounded reads, the kill ladders, the build and pool +guards --- lives in `test_hil_bounded.py`, `test_hil_health.py` and `test_hil_util.py`, so +run all five when changing `test/hil`: +`for f in test/hil/test/test_*.py; do python3 "$f"; done` (~84s, of which +`test_hil_bounded.py` is ~76s of deliberate hang/timeout simulation; the two `test_ci_*` +suites are ~4s together). ## Pre-flight rig health check diff --git a/.claude/skills/pre-pr/SKILL.md b/.claude/skills/pre-pr/SKILL.md index b96750e4f..8e5c408a6 100644 --- a/.claude/skills/pre-pr/SKILL.md +++ b/.claude/skills/pre-pr/SKILL.md @@ -15,7 +15,7 @@ Run the software + hardware gate for the current branch. The user invoking this ## 2. Map changes to boards -- `python3 test/hil/helper/hil_select.py --base $BASE test/hil/tinyusb.json` → JSON with the affected +- `python3 tools/ci_select.py --base $BASE test/hil/tinyusb.json` → JSON with the affected bsp `families`, the affected rig `boards`, and per-file `reasons`. `full: true` means a broad/infra change. - Affected families = `families` ∪ the family of every name in `boards`. Neither half is diff --git a/.github/scripts/ci_set_matrix.py b/.github/scripts/ci_set_matrix.py index 50ada5964..79f466893 100755 --- a/.github/scripts/ci_set_matrix.py +++ b/.github/scripts/ci_set_matrix.py @@ -1,5 +1,9 @@ #!/usr/bin/env python3 +import argparse import json +import os +import subprocess +import sys # toolchain, url toolchain_list = [ @@ -97,15 +101,87 @@ family_list = { } -def set_matrix_json(): +def set_matrix_json(select=None): + sel_fams = None + if select: + # every shape check is explicit: this runs AFTER main()'s fail-open handler, so + # an AttributeError on e.g. {"build": ["stm32f4"]} would red the step instead + # of falling back to the full matrix - the outcome that handler exists to prevent + b = select.get('build') if isinstance(select, dict) else None + if not isinstance(b, dict): + b = {} + if b.get('full') is False: + fams = b.get('families') + if not (isinstance(fams, list) and all(isinstance(f, str) for f in fams)): + # key ABSENT (or not a list of names) is an unusable selection, not + # "nothing selected": scoping every toolchain to [] would build zero + # families and report a vacuous green. An explicit families: [] stays a + # legitimate nothing-selected. + print('ci_set_matrix: UNSCOPED - build.full is false but the families ' + 'list is unusable, emitting the full matrix', file=sys.stderr) + else: + sel_fams = set(fams) matrix = {} for toolchain in toolchain_list: - filtered_families = [family for family, supported_toolchain in family_list.items() if - toolchain in supported_toolchain] - matrix[toolchain] = filtered_families - + fams = [family for family, tc in family_list.items() if toolchain in tc] + if sel_fams is not None: + fams = [f for f in fams if f in sel_fams] + matrix[toolchain] = fams + if sel_fams: + # a family this file does not list builds on no toolchain, so it contributes no + # leg. hw/bsp holds several CI has never built (efm32, py32f0, same7x, ...) plus + # espressif, whose boards hil-build-esp builds by name. + unbuilt = sorted(f for f in sel_fams if f not in family_list) + if unbuilt and not any(matrix.values()): + # NONE of the selected families is buildable here, so every leg would skip + # and the PR would go green from a build job that ran no compiler. That is + # an unusable selection, not "nothing selected": say UNSCOPED - which + # build.yml and .circleci/config.yml both grep for - and emit the full + # matrix. An explicit families: [] is still a legitimate nothing-selected, + # and a PARTIAL miss still scopes to the families that do build. + print(f'ci_set_matrix: UNSCOPED - no selected family is built by any ' + f'toolchain here ({", ".join(unbuilt)}), emitting the full matrix', + file=sys.stderr) + return set_matrix_json(None) + if unbuilt: + print(f'ci_set_matrix: selected families built by no toolchain here: ' + f'{", ".join(unbuilt)}', file=sys.stderr) print(json.dumps(matrix)) +def main(): + parser = argparse.ArgumentParser() + group = parser.add_mutually_exclusive_group() + group.add_argument('--select', help='tools/ci_select.py JSON; scopes families when build.full is false') + # a whole selection as one argv/env value can exceed the exec limits on a big + # diff, which fails the calling step BEFORE it can fall open; callers that + # already have the selection on disk pass the path instead + group.add_argument('--select-file', help='file holding the same JSON as --select') + group.add_argument('--base', help='git ref: run tools/ci_select.py --base REF and scope from it') + args = parser.parse_args() + + select = None + try: + if args.select: + select = json.loads(args.select) + elif args.select_file: + with open(args.select_file) as f: + select = json.load(f) + elif args.base: + root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + r = subprocess.run([sys.executable, os.path.join(root, 'tools', 'ci_select.py'), + '--base', args.base], + capture_output=True, text=True, cwd=root, check=True) + select = json.loads(r.stdout) + except Exception as e: # fail-open: an unusable selection must never turn into a red job + # UNSCOPED is the marker build.yml greps for: it must then drop the build extras + # (example map, family regex) too, or a full build gets labelled and filtered as + # a scoped one. Keep the token on every fall-open path. + print(f'ci_set_matrix: UNSCOPED - selection unusable ({e}), emitting the full ' + f'matrix', file=sys.stderr) + select = None + set_matrix_json(select) + + if __name__ == '__main__': - set_matrix_json() + main() diff --git a/.github/scripts/hil_ci_set_matrix.py b/.github/scripts/hil_ci_set_matrix.py index 65f50788e..bf50061dd 100644 --- a/.github/scripts/hil_ci_set_matrix.py +++ b/.github/scripts/hil_ci_set_matrix.py @@ -1,6 +1,7 @@ import argparse import json import os +import sys def _resolve_config_path(config_file): @@ -19,13 +20,47 @@ def _resolve_config_path(config_file): def main(): parser = argparse.ArgumentParser() parser.add_argument('config_files', nargs='+', help='Configuration JSON file(s)') - parser.add_argument('--select', help='hil_select.py JSON; scopes boards when full=false') + g = parser.add_mutually_exclusive_group() + g.add_argument('--select', help='ci_select.py JSON; scopes boards when full=false') + # a whole selection as one argv can exceed MAX_ARG_STRLEN on a big diff, which + # would fail the step instead of falling open; callers that already have the + # selection on disk pass the path instead + g.add_argument('--select-file', help='file holding the same JSON as --select') args = parser.parse_args() + raw = args.select + sel = None + try: + if args.select_file: + with open(args.select_file) as f: + raw = f.read() + if raw: + sel = json.loads(raw) + if sel is not None and not isinstance(sel, dict): + raise ValueError(f'selection is {type(sel).__name__}, not an object') + except Exception as e: # fail-open: an unusable selection must never red the job + print(f'hil_ci_set_matrix: selection unusable ({e}) - full roster', + file=sys.stderr) + sel = None + selected = None - sel = json.loads(args.select) if args.select else None if sel and not sel.get('full'): - selected = set(sel.get('boards', {})) + # key ABSENT is an unusable selection, not "nothing selected" - same reading as + # ci_set_matrix.py. Filtering every board out would skip every hil-build leg and, + # through needs:, both rig jobs: an all-green PR with zero hardware coverage. + # An explicit boards: {} stays a legitimate nothing-selected. + if not isinstance(sel.get('boards'), dict): + print('hil_ci_set_matrix: selection has full false but no usable boards ' + 'map - full roster', file=sys.stderr) + sel = None # ALL of it is unusable, hil_examples included: keeping + # the -e lists would build a few examples per board + # while the rig, unfiltered, runs that board's whole + # test list - flash failures on the fail-open path + else: + selected = set(sel['boards']) + ex_map = (sel or {}).get('hil_examples') or {} + if not isinstance(ex_map, dict): + ex_map = {} # Toolchain buckets must match the toolchains instantiated by the hil-build # job in .github/workflows/build.yml. Keep all keys present (even if empty) @@ -68,8 +103,12 @@ def main(): f'hil-build-esp jobs in .github/workflows/build.yml') build_board = f'-b {name}' - if 'build' in board and 'args' in board['build']: - build_board += ' ' + ' '.join(f'-D{a}' for a in board['build']['args']) + + # PR selection: build only the examples this board will run (its test + # list plus device/board_test, the parking firmware) - tools/build.py -e. + # Absent key (hand runs, full non-PR builds) keeps --target all. + for ex in ex_map.get(name, []): + build_board += f' -e {ex}' # Each variant builds into cmake-build-<variant.name> with its own cmake # -D defines and raw CFLAGS. No 'variant' -> a single build named after diff --git a/.github/scripts/metrics_pair_compare.py b/.github/scripts/metrics_pair_compare.py new file mode 100755 index 000000000..50107cf72 --- /dev/null +++ b/.github/scripts/metrics_pair_compare.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Board+example-matched code-size compare for PR-scoped builds. + +The averaged metrics baseline (metrics-tinyusb) spans every family and example; +a scoped PR builds a subset, so comparing against it is apples-to-oranges. This +compares the intersection of (board, example) pairs present on BOTH sides, +averaged over exactly those pairs, and names what was dropped. See +docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md #code-metrics. +""" +import argparse +import glob +import json +import os +import sys +import tempfile + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'tools')) +import metrics + +# dropped (board, example) pairs named in the PR comment before it truncates +DROPPED_SHOWN = 20 + + +def board_family(board, repo_root): + hits = glob.glob(os.path.join(repo_root, 'hw/bsp/*/boards', board)) + return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None + + +def collect(root, repo_root): + """{(board, 'role/example'): [file entries]} from every + **/cmake-build-<board>/metrics_by_example.json under root. + + Keyed on the BOARD, not its family. The two sides are built by + `--one-first`, which returns all_boards[0] for a family with no + ci_preferred_boards entry - so a PR that adds hw/bsp/<family>/boards/a_new_board + shifts which board is built, and a family key would file the base run's sizes and + the PR run's sizes under the same name and publish the difference between two + unrelated MCUs as this PR's code-size impact. On the board key that mismatch lands + in `dropped` (reported as not compared), which is the truth.""" + pairs = {} + pat = os.path.join(root, '**', 'metrics_by_example.json') + for f in sorted(glob.glob(pat, recursive=True)): + board = os.path.basename(os.path.dirname(f)) + if not board.startswith('cmake-build-'): + print(f'pair_compare: {f} not under a cmake-build-<board> dir, skipping', file=sys.stderr) + continue + board = board[len('cmake-build-'):] + if not board_family(board, repo_root): + # unknown board: the name is still a usable key, but say so - it means the + # artifact came from a tree whose hw/bsp does not match this checkout + print(f'pair_compare: no family for board {board}', file=sys.stderr) + # parse into a LOCAL dict and merge only once the whole file came out clean: + # a file that blows up half way through must drop WHOLE, or the entries read + # before the malformation stay in the comparison while stderr says the file + # was skipped, and a silently truncated table gets published as the verdict + try: + one = {} + for ex, ent in json.load(open(f)).items(): + one.setdefault((board, ex), []).extend(ent.get('files', [])) + except (OSError, ValueError, AttributeError, TypeError) as e: + print(f'pair_compare: unreadable {f} ({e}), skipping', file=sys.stderr) + continue + for k, v in one.items(): + pairs.setdefault(k, []).extend(v) + return pairs + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument('--base-dir', required=True) + ap.add_argument('--new-dir', required=True) + ap.add_argument('--out', default='metrics_compare') + a = ap.parse_args() + repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + base = collect(a.base_dir, repo_root) + new = collect(a.new_dir, repo_root) + common = sorted(set(base) & set(new)) + dropped = sorted(set(base) ^ set(new)) + + if not common: + with open(a.out + '.md', 'w') as f: + if new and not base: + # interim state: master has not uploaded a per-example baseline yet. + # Blaming the PR's scoping for that sends people hunting the wrong bug + f.write('_No per-example baseline from the base branch yet (the first ' + 'master push after this feature merges uploads it); comparison ' + 'will appear on the next push._\n') + else: + f.write('_Code-size comparison skipped: no (board, example) pair was ' + 'built on both the base branch and this PR._\n') + return + + def synth(pairs, path): + with open(path, 'w') as f: + json.dump({'files': [e for k in common for e in pairs[k]]}, f) + + with tempfile.TemporaryDirectory() as td: + b, n = os.path.join(td, 'base.json'), os.path.join(td, 'new.json') + synth(base, b) + synth(new, n) + comparison = metrics.compare_files(b, n, ['tinyusb/src']) + if comparison is None: + with open(a.out + '.md', 'w') as f: + f.write('_Code-size comparison failed to produce data._\n') + return + metrics.write_compare_markdown(comparison, a.out + '.md', 'name+') + + with open(a.out + '.md', 'a') as f: + boards = sorted({k[0] for k in common}) + f.write(f'\n_Scoped compare: {len(common)} (board, example) pairs across ' + f'{", ".join(boards)}._\n') + if dropped: + # GitHub caps a comment at 65,536 chars and this footer rides inside the + # sticky code-metrics comment: a broad scoped PR drops hundreds of pairs, + # and the raw list alone reached ~65KB and reddened the whole job. Only a + # summary goes in the comment; the full list goes to the job log. + names = [f'{board}:{ex}' for board, ex in dropped] + print('pair_compare: not compared (missing on one side): ' + + ', '.join(names), file=sys.stderr) + more = len(names) - DROPPED_SHOWN + f.write(f'_Not compared (missing on one side): {len(names)} pairs - ' + + ', '.join(names[:DROPPED_SHOWN]) + + (f', ... and {more} more (see the code-metrics job log)' + if more > 0 else '') + + '._\n') + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8f6014f48..39a4e7afd 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,7 +37,10 @@ jobs: - 'hw/**' - 'test/hil/**' - 'tools/build.py' + - 'tools/build_utils.py' + - 'tools/ci_select.py' - 'tools/get_deps.py' + - 'tools/metrics.py' - '.github/actions/**' - '.github/workflows/build.yml' - '.github/workflows/build_util.yml' @@ -48,6 +51,9 @@ jobs: outputs: json: ${{ steps.set-matrix-json.outputs.matrix }} hil_json: ${{ steps.set-matrix-json.outputs.hil_matrix }} + example_map: ${{ steps.set-matrix-json.outputs.example_map }} + build_filtered: ${{ steps.set-matrix-json.outputs.build_filtered }} + build_families_regex: ${{ steps.set-matrix-json.outputs.build_families_regex }} # one pair per rig job: hil-tinyusb (tinyusb.json minus esptool boards), # hil-tinyusb-esp (esptool boards only), hil-tinyusb (hfp.json) hil_args_tinyusb: ${{ steps.hil-select.outputs.args_tinyusb }} @@ -62,9 +68,14 @@ jobs: with: fetch-depth: 0 - - name: HIL selection (PR only) + # The `ci-full` PR label turns the scoping off for one PR: no selection file is + # written, so both matrices and every rig job fall back to the unscoped behaviour. + # An escape hatch is the point - a selector bug under-selects SILENTLY, and without + # a label the only routes back to a full matrix are accidental (touch an + # unclassified path, or break the selector badly enough that it falls open). + - name: CI selection (PR only) id: hil-select - if: github.event_name == 'pull_request' + if: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'ci-full') env: BASE_REF: ${{ github.base_ref }} run: | @@ -79,55 +90,133 @@ jobs: # advisory workflow that nothing here can `needs:`. Test-failing selector => # full matrix, same as a crashing one. SELECT_JSON='' - if ! python3 test/hil/test/test_hil_select.py; then - echo "::warning::hil_select unit suite failed - falling back to the full HIL matrix" - elif ! SELECT_JSON=$(python3 test/hil/helper/hil_select.py --base "origin/$BASE_REF" test/hil/tinyusb.json test/hil/hfp.json); then - echo "::warning::hil_select failed - falling back to the full HIL matrix" + if ! python3 test/hil/test/test_ci_select.py; then + echo "::warning::ci_select unit suite failed - falling back to the full HIL matrix" + elif ! SELECT_JSON=$(python3 tools/ci_select.py --base "origin/$BASE_REF" test/hil/tinyusb.json test/hil/hfp.json); then + echo "::warning::ci_select failed - falling back to the full HIL matrix" SELECT_JSON='' fi + # The selection is handed on as a FILE in the workspace, never as a step + # output/env var: it is ~KBs normally but a mass-sweep PR reaches hundreds of + # KB, and an env var that big makes the consuming exec fail with E2BIG BEFORE + # any fallback in it can run. Written here, ahead of its first reader. + # No file (non-PR event, or any fallback) = full matrix. + rm -f ci_select_out.json + if [ -n "$SELECT_JSON" ]; then + printf '%s' "$SELECT_JSON" > ci_select_out.json + fi + # One args/run pair per rig job, split by flasher: a job whose own subset is # empty skips explicitly instead of running a board filter that matches zero # boards ("No tests were run." exits 0 and would read as a green HIL run). OUT='' - if [ -n "$SELECT_JSON" ]; then - OUT=$(SELECT_JSON="$SELECT_JSON" python3 -c ' - import json, os - s = json.loads(os.environ["SELECT_JSON"]) + if [ -s ci_select_out.json ]; then + OUT=$(python3 -c ' + import json, re, sys + s = json.load(open("ci_select_out.json")) + # the same reading hil_ci_set_matrix.py applies: full false with no usable + # boards map is an UNUSABLE selection, not "nothing selected". Both must agree + # - one falling open to the whole roster while the other computes run=false + # buys a full 37-leg build and still zero hardware coverage. + if not s.get("full") and not isinstance(s.get("boards"), dict): + sys.exit("selection has full false but no usable boards map") tin = s.get("args_flasher", {}).get("tinyusb.json", {}) legs = (("tinyusb", " ".join(a for f, a in sorted(tin.items()) if f != "esptool" and a)), ("tinyusb_esp", tin.get("esptool", "")), ("hfp", s.get("args", {}).get("hfp.json", ""))) for key, a in legs: + # roster board names reach $GITHUB_OUTPUT as bare NAME=VALUE lines; a + # newline in one would inject extra run_* lines and flip which rig jobs run. + # ":" and "," are part of the normal shape - a partial filter is + # `-bt <board>:<test>,<test>` (ci_select._board_args) + if not re.fullmatch(r"[-A-Za-z0-9_/ .=+:,]*", a): + sys.exit("unexpected characters in the " + key + " board filter") print("args_" + key + "=" + a) print("run_" + key + "=" + ("true" if (s.get("full") or a) else "false")) ') || OUT='' if [ -z "$OUT" ]; then - echo "::warning::hil_select output unusable - falling back to the full HIL matrix" - SELECT_JSON='' + echo "::warning::ci_select output unusable - falling back to the full HIL matrix" + # the same unusable selection must not stay behind for the build axis + rm -f ci_select_out.json fi fi if [ -z "$OUT" ]; then OUT=$(for k in tinyusb tinyusb_esp hfp; do printf 'args_%s=\nrun_%s=true\n' "$k" "$k"; done) fi echo "$OUT" - { echo "select=$SELECT_JSON"; echo "$OUT"; } >> $GITHUB_OUTPUT + echo "$OUT" >> $GITHUB_OUTPUT - name: Generate matrix json id: set-matrix-json - env: - SELECT: ${{ steps.hil-select.outputs.select }} run: | - # build matrix - MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py) + # Build matrix, scoped by the PR selection when one exists. Best-effort: + # ci_set_matrix falls back to the full matrix itself on unusable JSON, + # and a missing file (non-PR event, selector fallback) means no flags. + SELECT_FILE=ci_select_out.json + [ -s "$SELECT_FILE" ] || SELECT_FILE='' + BUILD_SELECT_FILE="$SELECT_FILE" + MATRIX_JSON='' + if [ -n "$SELECT_FILE" ]; then + # ci_set_matrix falls open on a selection it cannot use with rc 0 - it prints + # the full matrix and says UNSCOPED on stderr. The build extras below must + # not stay scoped when it did, or a nominally full build compiles 1 of 44 + # examples per family and code-metrics compares that partial run against a + # full baseline. Only the BUILD axis is dropped: build.families being + # unusable says nothing about the boards map the HIL matrix reads. + MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py --select-file "$SELECT_FILE" 2>ci_set_matrix.err) || MATRIX_JSON='' + cat ci_set_matrix.err >&2 + if [ -z "$MATRIX_JSON" ] || grep -q 'ci_set_matrix: UNSCOPED' ci_set_matrix.err; then + BUILD_SELECT_FILE='' + fi + fi + [ -z "$MATRIX_JSON" ] && MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py) + + # Build-axis extras: the per-family example map rides as a side channel + # (a value inside matrix entries would break CircleCI's family parameter + # and multiply GHA matrix legs). These stay step outputs - they are small + # derived values, unlike the selection they are read from. NOTE jq's // + # treats false like null, so .build.full is compared explicitly. + EXAMPLE_MAP='{}' + BUILD_FILTERED='false' + FAM_REGEX='' + if [ -n "$BUILD_SELECT_FILE" ]; then + EXAMPLE_MAP=$(jq -c '.build.family_examples // {}' "$BUILD_SELECT_FILE") || EXAMPLE_MAP='{}' + BUILD_FILTERED=$(jq -r 'if (.build? | type) == "object" and .build.full == false then "true" else "false" end' "$BUILD_SELECT_FILE") || BUILD_FILTERED='false' + if [ "$BUILD_FILTERED" = "true" ]; then + FAM_REGEX=$(jq -r '.build.families | join("|")' "$BUILD_SELECT_FILE") || FAM_REGEX='' + # family names come from hw/bsp dir names, which rule 6 reads straight out + # of the PR's diff path - and this is interpolated raw into a + # `name_is_regexp` artifact pattern, so a regex metacharacter there would + # silently match another family's baseline + case "$FAM_REGEX" in + *[!-A-Za-z0-9_\|]*) + echo "::warning::unexpected characters in the family list - dropping the scoping" + FAM_REGEX='' ;; + esac + if [ -z "$FAM_REGEX" ]; then + # all three drop together, as CircleCI's fall-open does. Resetting only + # build_filtered leaves the build scoped while code-metrics takes the + # UNSCOPED branch, diffing a 1-family run against the full averaged + # baseline and publishing that as the PR's code-size impact. + BUILD_FILTERED='false' + EXAMPLE_MAP='{}' + MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py) + fi + fi + fi + # emitted once, after every path that can still change it echo "matrix=$MATRIX_JSON" echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT + echo "example_map=$EXAMPLE_MAP" >> $GITHUB_OUTPUT + echo "build_filtered=$BUILD_FILTERED" >> $GITHUB_OUTPUT + echo "build_families_regex=$FAM_REGEX" >> $GITHUB_OUTPUT # HIL matrix (merged from tinyusb + hifiphile configs), scoped on PRs. # Scoping is best-effort too: fall back to the unscoped (full) matrix. HIL_MATRIX_JSON='' - if [ -n "$SELECT" ]; then - HIL_MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py --select "$SELECT" test/hil/tinyusb.json test/hil/hfp.json) || HIL_MATRIX_JSON='' + if [ -n "$SELECT_FILE" ]; then + HIL_MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py --select-file "$SELECT_FILE" test/hil/tinyusb.json test/hil/hfp.json) || HIL_MATRIX_JSON='' if [ -z "$HIL_MATRIX_JSON" ]; then echo "::warning::scoped HIL matrix failed - falling back to the full HIL matrix" fi @@ -162,6 +251,7 @@ jobs: toolchain: ${{ matrix.toolchain }} build-args: ${{ toJSON(fromJSON(needs.set-matrix.outputs.json)[matrix.toolchain]) }} build-options: '--one-first' + example-map: ${{ needs.set-matrix.outputs.example_map }} upload-metrics: true upload-artifacts: false upload-membrowse: true @@ -169,8 +259,17 @@ jobs: secrets: inherit code-metrics: - needs: [ check-paths, cmake ] - if: needs.check-paths.outputs.code_changed == 'true' + needs: [ check-paths, cmake, set-matrix ] + # A scoped selection can empty every cmake toolchain (a test/hil-only PR). This + # job must still run then: skipping it leaves the sticky comment showing the + # PREVIOUS push's size table as if it were current. set-matrix must have + # SUCCEEDED though: !cancelled() alone let a failed set-matrix through, and this + # job would then overwrite the sticky comment with a wrong "built no families" + # diagnosis while reporting itself green. + if: | + !cancelled() && needs.check-paths.outputs.code_changed == 'true' && + needs.set-matrix.result == 'success' && + (needs.cmake.result == 'success' || needs.cmake.result == 'skipped') runs-on: ubuntu-latest permissions: pull-requests: write @@ -187,8 +286,21 @@ jobs: pattern: metrics-* path: cmake-build merge-multiple: true + # download-artifact does not fail on a pattern that matches nothing, so a + # scoped PR that built no family simply lands here with an empty dir + + - name: Detect empty metrics set + run: | + # No metrics at all => nothing to aggregate or compare. Write the marker the + # sticky comment will carry, so the size section says "skipped" for THIS push + # instead of silently keeping the previous push's table. + if ! ls cmake-build/*/metrics.json >/dev/null 2>&1; then + echo "_Code-size comparison skipped: PR selection built no families on this push._" > metrics_compare.md + echo "NO_METRICS=true" >> $GITHUB_ENV + fi - name: Aggregate Code Metrics + if: env.NO_METRICS != 'true' run: | python tools/get_deps.py python tools/metrics.py combine -j -m -f tinyusb/src cmake-build/*/metrics.json @@ -201,7 +313,7 @@ jobs: path: metrics.json - name: Download Base Branch Metrics - if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + if: env.NO_METRICS != 'true' && (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch') && needs.set-matrix.outputs.build_filtered != 'true' uses: dawidd6/action-download-artifact@v11 with: workflow: build.yml @@ -211,6 +323,29 @@ jobs: path: base-metrics continue-on-error: true + - name: Download base per-family metrics (scoped PR) + if: env.NO_METRICS != 'true' && github.event_name == 'pull_request' && needs.set-matrix.outputs.build_filtered == 'true' + uses: dawidd6/action-download-artifact@v11 + with: + workflow: build.yml + workflow_conclusion: '' + search_artifacts: true # a docs-only master push uploads no per-family artifacts + branch: ${{ github.base_ref }} + name: ^metrics-(${{ needs.set-matrix.outputs.build_families_regex }})$ + name_is_regexp: true + path: base-family-metrics + continue-on-error: true + + - name: Compare with Base Branch (scoped) + if: env.NO_METRICS != 'true' && github.event_name == 'pull_request' && needs.set-matrix.outputs.build_filtered == 'true' + run: | + # never fall back to the averaged metrics-tinyusb here: a scoped PR vs the + # 64-family/46-example average is exactly the mismatch this path prevents + python .github/scripts/metrics_pair_compare.py \ + --base-dir base-family-metrics --new-dir cmake-build --out metrics_compare || \ + echo "_Code-size comparison failed on the scoped path - see the code-metrics job log._" > metrics_compare.md + cat metrics_compare.md + - name: Download Previous Release Asset if: github.event_name == 'release' env: @@ -224,7 +359,7 @@ jobs: gh release download $PREV_TAG -p metrics.json -D base-metrics || echo "No metrics.json found in $PREV_TAG release" - name: Compare with Base Branch - if: github.event_name != 'push' + if: env.NO_METRICS != 'true' && github.event_name != 'push' && needs.set-matrix.outputs.build_filtered != 'true' run: | if [ -f base-metrics/metrics.json ]; then python tools/metrics.py compare -m -f tinyusb/src base-metrics/metrics.json metrics.json @@ -252,6 +387,9 @@ jobs: path: | metrics_compare.md metrics.json + # metrics.json is absent when the selection built no family; the marker + # in metrics_compare.md is still what the sticky comment needs + if-no-files-found: ignore - name: Post Code Metrics as PR Comment if: (github.event_name == 'workflow_dispatch') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) @@ -627,32 +765,32 @@ jobs: run: | # Best-effort: this job is deliberately decoupled from set-matrix so unrelated # failures cannot kill hfp coverage - a selector failure here must likewise - # fall back to the full hfp matrix (no hil_select.json, no SEL_* vars), never + # fall back to the full hfp matrix (no ci_select.json, no SEL_* vars), never # fail the job. - if ! python3 test/hil/test/test_hil_select.py; then - echo "::warning::hil_select unit suite failed - running the full hfp matrix" - rm -f hil_select.json + if ! python3 test/hil/test/test_ci_select.py; then + echo "::warning::ci_select unit suite failed - running the full hfp matrix" + rm -f ci_select.json exit 0 fi - if ! python3 test/hil/helper/hil_select.py --base "origin/$BASE_REF" test/hil/hfp.json > hil_select.json; then - echo "::warning::hil_select failed - running the full hfp matrix" - rm -f hil_select.json + if ! python3 tools/ci_select.py --base "origin/$BASE_REF" test/hil/hfp.json > ci_select.json; then + echo "::warning::ci_select failed - running the full hfp matrix" + rm -f ci_select.json exit 0 fi - # hil_select.json is passed to hil_ci_set_matrix.py --select below to scope the + # ci_select.json is passed to hil_ci_set_matrix.py --select below to scope the # build; it already honours full=true by ignoring the board list. # The hil_test.py args go to a file, never to $GITHUB_ENV: they are derived # from roster board names, which a PR can edit. Only SEL_RUN (a literal # true/false computed here, needed by the step-level `if:`) goes to the env. if ! SEL_RUN=$(python3 -c ' import json - s = json.load(open("hil_select.json")) + s = json.load(open("ci_select.json")) a = s["args"]["hfp.json"] open("hil_sel_args.txt", "w").write(a) print("true" if (s["full"] or a) else "false") '); then - echo "::warning::hil_select output unusable - running the full hfp matrix" - rm -f hil_select.json hil_sel_args.txt + echo "::warning::ci_select output unusable - running the full hfp matrix" + rm -f ci_select.json hil_sel_args.txt exit 0 fi echo "SEL_RUN=$SEL_RUN" @@ -661,9 +799,17 @@ jobs: - name: Get build boards if: env.SEL_RUN != 'false' run: | - if [ -f hil_select.json ]; then - MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py --select "$(cat hil_select.json)" test/hil/hfp.json) - else + # --select-file, never --select "$(cat ...)": a whole selection as one argv + # can exceed MAX_ARG_STRLEN on a big diff, and this job's design is to fall + # back to the full hfp matrix on any selector trouble, not to fail the step. + MATRIX_JSON='' + if [ -f ci_select.json ]; then + MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py --select-file ci_select.json test/hil/hfp.json) || MATRIX_JSON='' + if [ -z "$MATRIX_JSON" ]; then + echo "::warning::scoped hfp matrix failed - building the full hfp matrix" + fi + fi + if [ -z "$MATRIX_JSON" ]; then MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py test/hil/hfp.json) fi # Each variant carries its own --build-name/--cflag, which are global to a @@ -672,6 +818,13 @@ jobs: echo "$MATRIX_JSON" | jq -r '.["arm-gcc"][]' > hil_build_entries.txt cat hil_build_entries.txt BUILD_ARGS=$(echo "$MATRIX_JSON" | jq -r '.["arm-gcc"] | join(" ")') + # board and example names are roster data a PR can edit, and jq -r un-escapes + # them: a newline here writes extra NAME=VALUE lines into GITHUB_ENV for every + # later step of a job that holds the IAR token. Refuse rather than guess. + case "$BUILD_ARGS" in + *[!-A-Za-z0-9_/\ .=+]*) + echo "::error::unexpected characters in the hfp build args"; exit 1 ;; + esac echo "BUILD_ARGS=$BUILD_ARGS" echo "BUILD_ARGS=$BUILD_ARGS" >> $GITHUB_ENV diff --git a/.github/workflows/build_util.yml b/.github/workflows/build_util.yml index 02f16488a..52999616d 100644 --- a/.github/workflows/build_util.yml +++ b/.github/workflows/build_util.yml @@ -20,6 +20,10 @@ on: required: false default: '' type: string + example-map: + required: false + default: '' + type: string upload-artifacts: required: false default: false @@ -76,19 +80,42 @@ jobs: with: arg: ${{ matrix.arg }} + - name: Resolve PR example filter + if: inputs.example-map != '' && inputs.example-map != '{}' + env: + # values are PR-derived - keep them out of ${{ }} script interpolation + # (env expansion word-splits but never re-parses shell metacharacters) + EXAMPLE_MAP: ${{ inputs.example-map }} + FAMILY: ${{ matrix.arg }} + run: | + # -e flags for this family; a family absent from the map builds everything + EX_ARGS=$(printf '%s' "$EXAMPLE_MAP" | jq -r --arg fam "$FAMILY" '(.[$fam] // []) | map("-e " + .) | join(" ")') || EX_ARGS='' + # the map's values are example dir names from the PR checkout, and `jq -r` + # un-escapes them: a path with a newline (git allows it) would otherwise write + # extra NAME=VALUE lines into GITHUB_ENV for every later step of this job. + # Anything outside the example-name alphabet drops the filter (= build all), + # which is the safe direction. + case "$EX_ARGS" in + *[!-A-Za-z0-9_/\ ]*) + echo "::warning::unexpected characters in the example filter - building all examples" + EX_ARGS='' ;; + esac + echo "EX_ARGS=$EX_ARGS" + echo "EX_ARGS=$EX_ARGS" >> $GITHUB_ENV + - name: Build if: ${{ inputs.code-changed }} env: IAR_LMS_BEARER_TOKEN: ${{ secrets.IAR_LMS_BEARER_TOKEN }} run: | if [ "${{ inputs.toolchain }}" == "esp-idf" ]; then - docker run --rm -e MEMBROWSE_API_KEY="$MEMBROWSE_API_KEY" -e CI="$CI" -v $PWD:/project -w /project espressif/idf:tinyusb python tools/build.py --target all ${{ matrix.arg }} + docker run --rm -e MEMBROWSE_API_KEY="$MEMBROWSE_API_KEY" -e CI="$CI" -v $PWD:/project -w /project espressif/idf:tinyusb python tools/build.py --target all ${{ matrix.arg }} $EX_ARGS else BUILD_PY_ARGS="-s ${{ inputs.build-system }} ${{ steps.setup-toolchain.outputs.build_option }} ${{ inputs.build-options }} --target all" if [ "${{ inputs.upload-metrics }}" = "true" ]; then BUILD_PY_ARGS="$BUILD_PY_ARGS --target tinyusb_metrics" fi - python tools/build.py $BUILD_PY_ARGS ${{ matrix.arg }} + python tools/build.py $BUILD_PY_ARGS ${{ matrix.arg }} $EX_ARGS fi shell: bash @@ -99,8 +126,16 @@ jobs: MEMBROWSE_API_KEY: ${{ secrets.MEMBROWSE_API_KEY }} run: | # if code-changed is false --> there is no elf -> membrowse target upload with --identical flag + # $EX_ARGS is passed for the BOARD it picks, not to scope the targets: + # --one-first now chooses a board that can build the -e set (tools/build.py), + # so omitting it here would configure a DIFFERENT, empty build dir and upload + # --identical for a board that was never compiled. The target list is not + # scoped by it - `examples-membrowse-upload` is not `all`, so it passes + # through as the aggregate, which has no DEPENDS (hw/bsp/family_support.cmake): + # it rebuilds nothing and still records every example, --identical for the + # ones without an elf. BUILD_PY_ARGS="-s ${{ inputs.build-system }} ${{ steps.setup-toolchain.outputs.build_option }} ${{ inputs.build-options }}" - python tools/build.py $BUILD_PY_ARGS --target examples-membrowse-upload -j 1 ${{ matrix.arg }} + python tools/build.py $BUILD_PY_ARGS --target examples-membrowse-upload -j 1 ${{ matrix.arg }} $EX_ARGS shell: bash - name: Upload Artifacts for Metrics @@ -108,13 +143,37 @@ jobs: uses: actions/upload-artifact@v7 with: name: metrics-${{ matrix.arg }} - path: cmake-build/cmake-build-*/metrics.json + path: | + cmake-build/cmake-build-*/metrics.json + cmake-build/cmake-build-*/metrics_by_example.json + + - name: Artifact name + if: inputs.upload-artifacts == true + env: + ARG: ${{ matrix.arg }} + run: | + # -e example filters carry '/', which upload-artifact forbids in artifact + # names; strip them from the NAME only (the build already consumed them). + # Names without -e stay byte-identical to before. Two entries differing + # only in their -e list cannot exist - the -e list is a function of + # (board), and variant suffixes (--build-name/-D/--cflag) survive the + # strip - so the stripped name is still unique per matrix entry. + TAG=$(printf '%s' "$ARG" | sed -E 's/ -e [^ ]+//g') + # board and example names come from the roster, which a PR can edit; a newline + # in one would write extra NAME=VALUE lines into GITHUB_ENV for every later + # step. There is no safe fallback name here - a wrong one mislabels the + # firmware the rig then flashes - so refuse instead. + case "$TAG" in + *[!-A-Za-z0-9_/\ .=+]*) + echo "::error::refusing to build an artifact name from '$ARG'"; exit 1 ;; + esac + echo "ARTIFACT_TAG=$TAG" >> $GITHUB_ENV - name: Upload Artifacts for Hardware Testing if: inputs.upload-artifacts == true && inputs.code-changed == true uses: actions/upload-artifact@v7 with: - name: binaries-${{ inputs.toolchain }}-${{ matrix.arg }} + name: binaries-${{ inputs.toolchain }}-${{ env.ARTIFACT_TAG }} path: | cmake-build/cmake-build-*/*/*/*.elf cmake-build/cmake-build-*/*/*/*.bin diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 17170b7d6..7a29dc89a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,18 +48,29 @@ repos: types_or: [c, header] language: system - # Two hooks, split by what each suite actually reads. The full discovery run costs - # ~55s (deliberate hang/timeout simulations); only test_hil_select (~0.1s) reads - # hw/bsp (board.cmake), src (portable dirs + class include graph) and examples - # (tusb_config.h per test) -- renaming a board, port dir or example breaks it without - # touching test/hil, and catching that here beats waiting for pre-commit CI. + # Two hooks, split by what each suite RUNS, not by what it reads: discovery is + # disjoint (test_hil*.py vs the two named suites) so nothing runs twice, but the + # file patterns overlap where both suites care. hil-test runs test_hil*.py only + # (~80s: deliberate hang and timeout simulations) and is scoped to the rig harness + # that owns them. The one part of it the selector depends on - the BottomLayer + # stdlib-closure AST guard over tools/ci_select.py and its imports - is named + # explicitly by ci-select-test instead, so a tools/ or workflow edit costs 4s + # rather than 80s of hang simulations that have nothing to say about it. + # ci-select-test runs the two selector-adjacent suites (~4s together) that read + # hw/bsp (board.cmake, FAMILY_MCUS), src (portable dirs + class include graph), + # examples (tusb_config.h, skip/only.txt), hw/mcu, the rig rosters under test/hil + # (a roster edit changes what the selector emits), .circleci (the sentinel contract + # config.yml rewrites config2.yml through) and .github/workflows (build.yml's own + # file hand-off and GITHUB_ENV guards) -- renaming a board, port dir or example + # breaks them without touching test/hil, and catching that here beats waiting for + # pre-commit CI. # No types_or: the rig rosters (*.json) are inputs too. # examples/device/mtp/src is in scope: test_hil_bounded parses README_TXT_CONTENT # and md5-checks the logo header from there as its MTP fixtures. - id: hil-test name: hil-test files: ^(test/hil/|examples/device/mtp/src/) - entry: python3 -m unittest discover -s test/hil/test + entry: python3 -m unittest discover -s test/hil/test -p 'test_hil*.py' pass_filenames: false language: system # hil-validate.js decides which boards ship. Its result join has been wrong three times -- @@ -71,10 +82,10 @@ repos: entry: node .claude/workflows/test-hil-validate.mjs pass_filenames: false language: system - - id: hil-select-test - name: hil-select-test - files: ^(hw/bsp/|src/|examples/) - entry: python3 test/hil/test/test_hil_select.py + - id: ci-select-test + name: ci-select-test + files: ^(hw/bsp/|hw/mcu/|src/|examples/|test/hil/|tools/(ci_select|build|build_utils|get_deps|metrics)\.py$|\.github/(scripts|workflows)/|\.circleci/) + entry: sh -c "python3 test/hil/test/test_ci_select.py && python3 test/hil/test/test_ci_metrics.py && cd test/hil/test && python3 -m unittest -q test_hil_util.BottomLayer" pass_filenames: false language: system diff --git a/docs/reference/dependencies.rst b/docs/reference/dependencies.rst index 146192ef8..4118b94c3 100644 --- a/docs/reference/dependencies.rst +++ b/docs/reference/dependencies.rst @@ -7,7 +7,7 @@ MCU low-level peripheral drivers and external libraries for building TinyUSB exa ======================================== ================================================================ ======================================== =================================================================================================================================================================================================================================================================================================================================================== Local Path Repo Commit Required by ======================================== ================================================================ ======================================== =================================================================================================================================================================================================================================================================================================================================================== -hw/mcu/allwinner https://github.com/hathach/allwinner_driver.git 8e5e89e8e132c0fd90e72d5422e5d3d68232b756 fc100s +hw/mcu/allwinner https://github.com/hathach/allwinner_driver.git 8e5e89e8e132c0fd90e72d5422e5d3d68232b756 f1c100s hw/mcu/analog/msdk https://github.com/analogdevicesinc/msdk.git b20b398d3e5e2007594e54a74ba3d2a2e50ddd75 maxim hw/mcu/artery/at32f402_405 https://github.com/ArteryTek/AT32F402_405_Firmware_Library.git 4424515c2663e82438654e0947695295df2abdfe at32f402_405 hw/mcu/artery/at32f403a_407 https://github.com/ArteryTek/AT32F403A_407_Firmware_Library.git f2cb360c3d28fada76b374308b8c4c61d37a090b at32f403a_407 @@ -38,7 +38,7 @@ hw/mcu/raspberry_pi/Pico-PIO-USB https://github.com/sekigon-gonnoc/Pico hw/mcu/renesas/fsp https://github.com/renesas/fsp.git edcc97d684b6f716728a60d7a6fea049d9870bd6 ra hw/mcu/renesas/rx https://github.com/kkitayam/rx_device.git 706b4e0cf485605c32351e2f90f5698267996023 rx hw/mcu/silabs/cmsis-dfp-efm32gg12b https://github.com/cmsis-packs/cmsis-dfp-efm32gg12b.git f1c31b7887669cb230b3ea63f9b56769078960bc efm32 -hw/mcu/sony/cxd56/spresense-exported-sdk https://github.com/sonydevworld/spresense-exported-sdk.git 2ec2a1538362696118dc3fdf56f33dacaf8f4067 spresense +hw/mcu/sony/cxd56/spresense-exported-sdk https://github.com/sonydevworld/spresense-exported-sdk.git 2ec2a1538362696118dc3fdf56f33dacaf8f4067 cxd56 hw/mcu/st/cmsis-device-u0 https://github.com/STMicroelectronics/cmsis-device-u0.git e3a627c6a5bc4eb2388e1885a95cc155e1672253 stm32u0 hw/mcu/st/cmsis-device-wba https://github.com/STMicroelectronics/cmsis-device-wba.git 647d8522e5fd15049e9a1cc30ed19d85e5911eaf stm32wba hw/mcu/st/cmsis_device_c0 https://github.com/STMicroelectronics/cmsis_device_c0.git 517611273f835ffe95318947647bc1408f69120d stm32c0 diff --git a/docs/reference/hardware-in-the-loop.md b/docs/reference/hardware-in-the-loop.md index 48c362f4f..cf7e3fe69 100644 --- a/docs/reference/hardware-in-the-loop.md +++ b/docs/reference/hardware-in-the-loop.md @@ -281,8 +281,9 @@ Both files are the source of truth — this table is generated from them. `test/hil/hil_test.py`, which flashes each board and runs its tests. Espressif boards run in `hil-tinyusb-esp`, gated on the slower ESP-IDF build, and `hil-hfp-iar` builds with IAR inside the job. -3. On pull requests, `test/hil/helper/hil_select.py` narrows the run to the boards a diff - can affect, falling open to the full matrix when it cannot tell. +3. On pull requests, `tools/ci_select.py` narrows the run to the boards a diff can + affect — and each board's build to the examples its tests need — falling open to the + full matrix when it cannot tell. The same pass scopes the build matrix. 4. Each board is arbitrated by a kernel flock in `/tmp/tinyusb-hil-locks/`, so interactive work and CI can share the rig without colliding. 5. Each rig job uploads its report as an artifact; `pr_comment.yml` downloads them and diff --git a/docs/superpowers/followup/pr3803-flasher-recover.md b/docs/superpowers/followup/pr3803-flasher-recover.md index e9fff7480..1f71c990f 100644 --- a/docs/superpowers/followup/pr3803-flasher-recover.md +++ b/docs/superpowers/followup/pr3803-flasher-recover.md @@ -19,18 +19,18 @@ libjaylink, J-Link probes. - Roster JSON: `test/hil/tinyusb.json`. `flasher_recover` is OPTIONAL; absent means today's behaviour (`recover_flasher` returns the primary). - Never change the shape of `board['flasher']` — it is read as a dict in `hil_flash`, - `hil_test`, `usbtest`, `hil_pool_check`, `hil_select` and the roster lint, and is shipped + `hil_test`, `usbtest`, `hil_pool_check`, `ci_select` and the roster lint, and is shipped as JSON to a subprocess. - Flasher dispatch is by name: `getattr(hil_flash, f'flash_{name}')` / `reset_{name}`. - `RECOVER_FLASH_TIMEOUT = 90`, `RECOVER_RESET_TIMEOUT = 30` (`usbtest.py`). Any board whose flash cannot finish inside 90 s is not a candidate. -- Tests run offline: `cd test/hil && python3 test/test_hil_select.py`. +- Tests run offline: `cd test/hil && python3 test/test_ci_select.py`. ## What is already established **Landed on PR #3803 and inert without roster entries:** `hil_flash.recover_flasher()`, `convoy_safe()` accepting openocd-over-jlink, `hil_test` substituting the recovery flasher -into `--recover-board`, and `test_hil_select.FlasherRecoverEntry` (4 tests). +into `--recover-board`, and `test_ci_select.FlasherRecoverEntry` (4 tests). **Verified in source:** - openocd's jlink driver ignores `adapter usb vid_pid` — `jlink.c` never reads @@ -77,7 +77,7 @@ is a different scope from containing a wedge; and it needs bench time on seven b - `test/hil/hil_flash.py` — add `flash_openocd_seq` / `reset_openocd_seq`; extend `convoy_safe` to accept the new name. This is the only file that learns the command form. - `test/hil/tinyusb.json` — seven `flasher_recover` entries. -- `test/hil/test/test_hil_select.py` — extend `FlasherRecoverEntry`; add a roster lint. +- `test/hil/test/test_ci_select.py` — extend `FlasherRecoverEntry`; add a roster lint. --- @@ -85,7 +85,7 @@ is a different scope from containing a wedge; and it needs bench time on seven b **Files:** - Modify: `test/hil/hil_flash.py` (beside `flash_openocd`, ~line 100) -- Test: `test/hil/test/test_hil_select.py` +- Test: `test/hil/test/test_ci_select.py` **Interfaces:** - Consumes: `_openocd_cmd_base(flasher)`, `hil_util.run_cmd`. @@ -120,7 +120,7 @@ is a different scope from containing a wedge; and it needs bench time on seven b - [ ] **Step 2: Run test to verify it fails** -Run: `cd test/hil && python3 test/test_hil_select.py FlasherRecoverEntry -v` +Run: `cd test/hil && python3 test/test_ci_select.py FlasherRecoverEntry -v` Expected: FAIL — `module 'hil_flash' has no attribute 'flash_openocd_seq'` - [ ] **Step 3: Write minimal implementation** @@ -155,13 +155,13 @@ In `convoy_safe`, replace `if name != 'openocd':` with: - [ ] **Step 4: Run test to verify it passes** -Run: `cd test/hil && python3 test/test_hil_select.py FlasherRecoverEntry -v` +Run: `cd test/hil && python3 test/test_ci_select.py FlasherRecoverEntry -v` Expected: PASS - [ ] **Step 5: Commit** ```bash -git add test/hil/hil_flash.py test/hil/test/test_hil_select.py +git add test/hil/hil_flash.py test/hil/test/test_ci_select.py git commit -m "hil: add openocd_seq flasher for convoy-safe recovery delivery" ``` @@ -171,7 +171,7 @@ git commit -m "hil: add openocd_seq flasher for convoy-safe recovery delivery" **Files:** - Modify: `test/hil/tinyusb.json` -- Test: `test/hil/test/test_hil_select.py` +- Test: `test/hil/test/test_ci_select.py` **Interfaces:** - Consumes: `flash_openocd_seq` / `reset_openocd_seq` from Task 1. @@ -196,7 +196,7 @@ git commit -m "hil: add openocd_seq flasher for convoy-safe recovery delivery" - [ ] **Step 2: Run test to verify it fails** -Run: `cd test/hil && python3 test/test_hil_select.py FlasherRecoverEntry -v` +Run: `cd test/hil && python3 test/test_ci_select.py FlasherRecoverEntry -v` Expected: FAIL — `0 >= 7` - [ ] **Step 3: Add the entries** @@ -224,13 +224,13 @@ Add to each board below, using the SAME `uid` as its primary jlink entry: - [ ] **Step 4: Run test to verify it passes** -Run: `cd test/hil && python3 test/test_hil_select.py -v` +Run: `cd test/hil && python3 test/test_ci_select.py -v` Expected: PASS, and no other selector test regresses. - [ ] **Step 5: Commit** ```bash -git add test/hil/tinyusb.json test/hil/test/test_hil_select.py +git add test/hil/tinyusb.json test/hil/test/test_ci_select.py git commit -m "hil: give seven J-Link boards a convoy-safe recovery flasher" ``` diff --git a/docs/superpowers/plans/2026-08-19-ci-build-family-filter.md b/docs/superpowers/plans/2026-08-19-ci-build-family-filter.md new file mode 100644 index 000000000..0d8b9cfa4 --- /dev/null +++ b/docs/superpowers/plans/2026-08-19-ci-build-family-filter.md @@ -0,0 +1,1804 @@ +# PR-Scoped CI Selection Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Promote `test/hil/helper/hil_select.py` to a repo-wide `tools/ci_select.py` whose one classification of a PR diff narrows three CI axes — build families, per-family example targets, and per-board HIL examples — wired into both GitHub Actions and CircleCI. + +**Architecture:** The selector gains an independent build classifier beside the untouched HIL one (17-rule table in the spec). `ci_set_matrix.py` filters the family matrix from the selector JSON; the per-family example map travels as a side channel (GHA job output / CircleCI pipeline parameter), resolved to `-e` flags per build job by a new `tools/build.py --example` filter. `hil_ci_set_matrix.py` appends `-e` per rig board. Code metrics gain per-example artifacts and a (family, example)-intersection compare. + +**Tech Stack:** Python 3 stdlib (selector must run on bare CI runners), GitHub Actions YAML, CircleCI dynamic config (continuation orb), jq, CMake/Ninja. + +**Spec:** `docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md` — read it first; every rule number below refers to its rule table. + +## Global Constraints + +- Commit messages: imperative mood, **no** `Co-Authored-By:` or `Claude-Session:` trailers (hathach is sole author — this overrides harness defaults). +- Never stage or touch `.idea/`. Always `git add` explicit paths, never `-A`. +- Bare-runner Python modules (`tools/ci_select.py`, `tools/build.py`, `tools/build_utils.py`, everything under `test/hil/helper/`) stay stdlib-only at module level — `test_hil_util.BottomLayer` enforces this; extend its lists, never work around them. +- `ci_select.py` stdout is machine-read JSON; every diagnostic goes to stderr. +- The family reference scan is **CMake-only** (`family.cmake` + espressif component `CMakeLists.txt`, never `family.mk`): CMake is the first-class build system, Make follows it. +- Fail-open everywhere: a selector/matrix-script failure must yield the full matrix, never a red job or a silently-empty one. +- Python style: match the existing modules (4-space indent in tools/ and test/hil/, terse targeted comments explaining *why*). +- YAML: 2-space indent, match surrounding style in `.github/workflows/` and `.circleci/`. +- Run suites from the repo root. Selector suite: `python3 test/hil/test/test_ci_select.py` (after Task 1). Full HIL-side suite: `python3 -m unittest discover -s test/hil/test`. + +--- + +### Task 1: Move the selector to `tools/ci_select.py` (mechanical, no behavior change) + +**Files:** +- Move: `test/hil/helper/hil_select.py` → `tools/ci_select.py` (git mv) +- Move: `test/hil/test/test_hil_select.py` → `test/hil/test/test_ci_select.py` (git mv) +- Modify: `test/hil/test/test_hil_util.py` (BottomLayer lists), `test/hil/hil_ci.sh` (scp list), `.pre-commit-config.yaml` (both hooks), `.github/workflows/build.yml` (4 path refs), `.claude/skills/pre-pr/SKILL.md`, `test/hil/helper/hil_util.py:21` (comment), `test/hil/hil_flash.py:297` (comment) + +**Interfaces:** +- Produces: module `tools/ci_select.py` importable as `ci_select` with `tools/` on `sys.path`; module attribute `_REPO_ROOT` (absolute repo root); CLI `python3 tools/ci_select.py --base REF|--diff-file F CONFIG.json...` — output JSON byte-compatible with today's `hil_select.py`. +- Consumes: `test/hil/helper/hil_util.py` rosters (unchanged). + +- [ ] **Step 1: git mv both files** + +```bash +git mv test/hil/helper/hil_select.py tools/ci_select.py +git mv test/hil/test/test_hil_select.py test/hil/test/test_ci_select.py +``` + +- [ ] **Step 2: Fix `tools/ci_select.py` imports and repo root** + +Replace the current path setup (line 24, `sys.path.insert(0, os.path.dirname(os.path.dirname(...)))` and its comment) with: + +```python +# tools/ -> repo root is ONE level up. Guarded by TestModuleMove.test_repo_root_guard: +# a wrong parent count here silently re-points every repo-relative glob (it happened +# at the helper/ move). +_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(_REPO_ROOT, 'test', 'hil')) # for `from helper...` +from helper.hil_util import device_tests, dual_tests, host_test +``` + +In `main()`, replace the 4-level `repo_root` derivation (lines 503-505) with `repo_root = _REPO_ROOT`. Change the stderr prefix at line 519 from `hil_select:` to `ci_select:`. Update the module docstring: it now lives in `tools/`, serves HIL and (from Task 3) build selection; keep the fail-open sentence and the spec pointer, adding this spec's path. + +- [ ] **Step 3: Fix `test/hil/test/test_ci_select.py` imports** + +Replace the header import block (`from helper import hil_select`) so `REPO` is computed first, then: + +```python +REPO = os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.abspath(__file__))))) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # test/hil, for hil_flash/helper +sys.path.insert(0, os.path.join(REPO, 'tools')) +import hil_flash +import ci_select +from helper.hil_util import device_tests, dual_tests +``` + +Then `sed -i 's/\bhil_select\b/ci_select/g' test/hil/test/test_ci_select.py` and fix the header comment (file names, run command). Add the guard test: + +```python +class TestModuleMove(unittest.TestCase): + def test_repo_root_guard(self): + # __file__-derived root: moving the module without re-deriving the parent + # count re-points every scan at the wrong tree (it happened once already) + self.assertTrue(os.path.isdir(os.path.join(ci_select._REPO_ROOT, 'src'))) + self.assertTrue(os.path.isdir(os.path.join(ci_select._REPO_ROOT, 'hw', 'bsp'))) + self.assertEqual(os.path.realpath(ci_select._REPO_ROOT), os.path.realpath(REPO)) +``` + +- [ ] **Step 4: Update every reference** + +- `test/hil/test/test_hil_util.py` BottomLayer: in `test_bare_runner_modules_stay_stdlib_only`, replace `'hil_select'` with `'ci_select'` in the `local` set and replace `'helper/hil_select'` with `'../../tools/ci_select'` in the module-path tuple (the loop builds `hil_dir / f'{mod}.py'`, so a relative path out of test/hil works). Update the docstring sentence naming hil_select. +- `test/hil/hil_ci.sh`: delete the `"$ROOT_DIR/test/hil/helper/hil_select.py" \` scp line (nothing on the rig imports it). +- `.pre-commit-config.yaml`: rename hook `hil-select-test` → `ci-select-test`; `entry: python3 test/hil/test/test_ci_select.py`; `files: ^(hw/bsp/|src/|examples/|tools/ci_select\.py$)`. In the `hil-test` hook comment, s/test_hil_select/test_ci_select/. +- `.github/workflows/build.yml`: four call sites — lines ~82/84 (set-matrix) and ~632/637 (hil-hfp-iar): `test/hil/test/test_hil_select.py` → `test/hil/test/test_ci_select.py`, `test/hil/helper/hil_select.py` → `tools/ci_select.py`; s/hil_select/ci_select/ in the adjacent `::warning::` strings and comments (keep `hil_select.json` file names as `ci_select.json` for consistency — update both writers and both readers in the hfp-iar job). +- `.claude/skills/pre-pr/SKILL.md`: `python3 test/hil/helper/hil_select.py` → `python3 tools/ci_select.py`. +- Comments only: `test/hil/helper/hil_util.py:21` (hil_select → ci_select), `test/hil/hil_flash.py:297` (test_hil_select → test_ci_select). + +- [ ] **Step 5: Verify** + +```bash +python3 test/hil/test/test_ci_select.py # all pass +python3 -m unittest discover -s test/hil/test # all pass (~55 s) +python3 tools/ci_select.py --diff-file /dev/null test/hil/tinyusb.json | python3 -m json.tool >/dev/null +grep -rn "hil_select" --include='*.py' --include='*.yml' --include='*.yaml' --include='*.sh' --include='*.md' . | grep -v docs/superpowers | grep -v '\.worktrees' +``` + +Expected: suites green; last grep returns nothing (historical spec docs are the only allowed hits). + +- [ ] **Step 6: Commit** + +```bash +git add tools/ci_select.py test/hil/test/test_ci_select.py test/hil/test/test_hil_util.py \ + test/hil/hil_ci.sh .pre-commit-config.yaml .github/workflows/build.yml \ + .claude/skills/pre-pr/SKILL.md test/hil/helper/hil_util.py test/hil/hil_flash.py +git commit -m "tools: promote hil_select.py to tools/ci_select.py" +``` + +--- + +### Task 2: Generalize the family scan and re-rule `hw/mcu/**` (HIL side) + +**Files:** +- Modify: `tools/ci_select.py` (`port_families` → `path_families` + `mcu_families`, `_FULL_RE`, `_classify_one`) +- Test: `test/hil/test/test_ci_select.py` + +**Interfaces:** +- Produces: `path_families(rel_dir: str, repo_root: str) -> set[str]` — families whose `family.cmake`/espressif component CMakeLists reference `rel_dir` at a directory boundary; `mcu_families(path: str, repo_root: str) -> set[str]` — longest-resolving-prefix lookup for a changed `hw/mcu/...` path; `port_families(port_dir, repo_root)` kept as a thin wrapper (existing callers/tests unchanged). +- HIL JSON change: `hw/mcu/**` no longer forces `full: true`; it selects the resolved families' boards, all their tests (spec rule 7). + +- [ ] **Step 1: Write the failing tests** (append to `test_ci_select.py`) + +```python +class TestPathFamilies(unittest.TestCase): + def test_port_wrapper_unchanged(self): + self.assertEqual(ci_select.port_families('raspberrypi/rp2040', REPO), {'rp2040'}) + self.assertIn('stm32f4', ci_select.port_families('synopsys/dwc2', REPO)) + + def test_boundary_without_trailing_slash(self): + # hw/bsp/nrf/family.cmake writes `${TOP}/hw/mcu/nordic/nrfx` — no trailing + # slash; the match must accept a directory-boundary end-of-token + self.assertEqual(ci_select.path_families('hw/mcu/nordic/nrfx', REPO), {'nrf'}) + + def test_boundary_rejects_prefix_sibling(self): + # 'microchip/pic' must not inherit pic32mz's references (and pic32mz itself + # is family.mk-only, which the CMake-only scan never reads) + self.assertEqual(ci_select.port_families('microchip/pic', REPO), set()) + self.assertEqual(ci_select.port_families('microchip/pic32mz', REPO), set()) + + def test_mcu_families_prefix_walk(self): + self.assertEqual(ci_select.mcu_families('hw/mcu/nordic/nrf5x/nrf_clock.h', REPO), {'nrf'}) + self.assertEqual(ci_select.mcu_families('hw/mcu/dialog/da1469x/x.h', REPO), {'da1469x'}) + self.assertEqual(ci_select.mcu_families('hw/mcu/no_such_vendor/x.c', REPO), set()) + + +class TestMcuHilRule(unittest.TestCase): + def test_mcu_no_longer_forces_full(self): + s = ci_select.classify(['hw/mcu/nordic/nrf5x/nrf_clock.h'], REPO, ROSTERS) + self.assertFalse(s['full']) + self.assertIn('nrf', s['families']) # recorded even with no nrf rig board + + def test_mcu_selects_family_boards(self): + got = on_roster(self, 'feather_nrf52840_express', 'pca10056', 'pca10095') + s = ci_select.classify(['hw/mcu/nordic/nrf5x/nrf_clock.h'], REPO, real_rosters()) + self.assertFalse(s['full']) + for b in got: + self.assertIn(b, s['boards']) + + +class TestOrphanInvariant(unittest.TestCase): + ALLOW = {'microchip/pic', 'microchip/pic32mz'} # spec: known orphans, CMake builds neither + + def test_every_port_resolves_to_a_family(self): + for d in sorted(glob.glob(os.path.join(REPO, 'src/portable/*/*'))): + if not os.path.isdir(d): + continue + port = os.path.relpath(d, os.path.join(REPO, 'src/portable')).replace(os.sep, '/') + fams = ci_select.port_families(port, REPO) + if port in self.ALLOW: + self.assertEqual(fams, set(), f'{port}: no longer an orphan - drop it from ALLOW') + else: + self.assertTrue(fams, f'{port}: no family.cmake references it - wire it up or allowlist it') + + def test_tracked_mcu_vendors_resolve(self): + import subprocess as sp + r = sp.run(['git', 'ls-files', 'hw/mcu'], cwd=REPO, capture_output=True, text=True) + if r.returncode != 0: + self.skipTest('not a git checkout') + vendors = sorted({'/'.join(p.split('/')[:3]) for p in r.stdout.split()}) + for v in vendors: + self.assertTrue(ci_select.mcu_families(v + '/x.c', REPO), f'{v}: resolves to no family') +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `python3 test/hil/test/test_ci_select.py TestPathFamilies -v` +Expected: FAIL/ERROR — `path_families`/`mcu_families` not defined. + +- [ ] **Step 3: Implement** + +In `tools/ci_select.py`, replace `port_families` with: + +```python [email protected]_cache(maxsize=None) +def path_families(rel_dir: str, repo_root: str) -> set: + """Board families whose family.cmake (or espressif component CMakeLists) + references rel_dir at a directory boundary. CMake only, on every axis: CMake + is the first-class build system and Make follows it, so family.mk is never + read - a port wired up in family.mk alone (microchip/pic32mz) is built by no + CI job and resolves to nothing. Boundary = '/', whitespace, quote, paren, + brace or end: `${TOP}/hw/mcu/nordic/nrfx` has no trailing slash, while bare + 'microchip/pic' must not match '.../microchip/pic32mz/...'.""" + fams = set() + bsp_root = os.path.join(repo_root, 'hw/bsp') + pat = re.compile(re.escape(rel_dir) + r'(?=[/\s"\')}]|$)', re.M) + for f in glob.glob(os.path.join(bsp_root, '*/family.cmake')) + \ + glob.glob(os.path.join(bsp_root, '*/components/*/CMakeLists.txt')): + try: + if pat.search(open(f).read()): + fams.add(os.path.relpath(f, bsp_root).split(os.sep, 1)[0]) + except OSError: + pass + return fams + + +def port_families(port_dir: str, repo_root: str) -> set: + return path_families('src/portable/' + port_dir, repo_root) + + +def mcu_families(path: str, repo_root: str) -> set: + """Families referencing a changed hw/mcu path: longest resolving dir prefix, + hw/mcu/<vendor>/<sub>/... down to hw/mcu/<vendor>.""" + parts = path.split('/') + for n in range(len(parts) - 1, 2, -1): + fams = path_families('/'.join(parts[:n]), repo_root) + if fams: + return fams + return set() +``` + +Keep the old docstring's CMake-only rationale for HIL (folded into the new one). Remove `hw/mcu/|` from `_FULL_RE`. In `_classify_one`, insert after the `hw/bsp/` block, before the `examples/` block: + +```python + if re.match(r'hw/mcu/', path): + fams = mcu_families(path, repo_root) + s.families.update(fams) + boards = [b['name'] for b in roster_boards + if board_family(b['name'], repo_root) in fams] + s.roles.update(('device', 'host')) + s.add(boards, 'all', f'{path}: mcu dir -> families {sorted(fams)} -> boards {boards}') + return +``` + +- [ ] **Step 4: Run tests** + +Run: `python3 test/hil/test/test_ci_select.py -v 2>&1 | tail -5` +Expected: all pass (the pre-existing port tests exercise the wrapper). + +- [ ] **Step 5: Commit** + +```bash +git add tools/ci_select.py test/hil/test/test_ci_select.py +git commit -m "ci_select: generalize family scan to hw/mcu, drop hw/mcu from HIL full-matrix rule" +``` + +--- + +### Task 3: Build classifier — rules 1-17, raw two-axis selection + +**Files:** +- Modify: `tools/ci_select.py` +- Test: `test/hil/test/test_ci_select.py` + +**Interfaces:** +- Produces: `classify_build(changed_files, repo_root) -> dict` with keys `full: bool`, `families: [str]` (sorted bsp-dir names), `family_examples: {family: [example]}` (key absent ⇒ that family builds all examples; examples as `role/name`), `reasons: [str]`. Also `all_examples(repo_root) -> tuple[str]`, `role_examples(repo_root, roles) -> set[str]`, `all_bsp_families(repo_root) -> list[str]`. Buildability pruning is Task 4 — this task emits the raw rule output. +- Consumes: `path_families`, `mcu_families`, `class_macros`, `class_include_edges`, `_config_enables`, `_NONCODE_RE` (all existing). + +- [ ] **Step 1: Write the failing tests** + +```python +class TestBuildClassifier(unittest.TestCase): + def b(self, files): + return ci_select.classify_build(files, REPO) + + def test_noncode_and_test_hil_contribute_nothing(self): # rules 1, 2 + s = self.b(['docs/info/index.rst', 'README.rst', 'test/hil/hil_test.py', '.claude/skills/hil/SKILL.md']) + self.assertFalse(s['full']) + self.assertEqual(s['families'], []) + self.assertEqual(s['family_examples'], {}) + + def test_port_device_rule(self): # rule 3 + s = self.b(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + self.assertFalse(s['full']) + self.assertEqual(s['families'], ['rp2040']) + exs = s['family_examples']['rp2040'] + self.assertIn('device/cdc_msc', exs) + self.assertFalse(any(e.startswith(('host/', 'typec/')) for e in exs)) + # dual inclusion asserted on the pure role helper: whether a dual example + # survives Task 4's buildability pruning depends on the environment-gated + # CI board pick, so the classifier-output assertion must not rely on it + self.assertIn('dual/host_info_to_device_cdc', + ci_select.role_examples(REPO, ('device', 'dual'))) + self.assertNotIn('host/bare_api', ci_select.role_examples(REPO, ('device', 'dual'))) + + def test_port_host_rule(self): # rule 4 + s = self.b(['src/portable/analog/max3421/hcd_max3421.c']) + self.assertFalse(s['full']) + # max3421 is referenced only by the espressif component CMakeLists — and + # espressif is in no provider's family list, so this may prune to nothing + self.assertLessEqual(set(s['families']), {'espressif'}) + for exs in s['family_examples'].values(): + self.assertFalse(any(e.startswith(('device/', 'typec/')) for e in exs)) + + def test_port_shared_file_selects_all_examples(self): # rule 5 + s = self.b(['src/portable/synopsys/dwc2/dwc2_common.c']) + self.assertFalse(s['full']) + self.assertIn('stm32f4', s['families']) + self.assertNotIn('rp2040', s['families']) + self.assertNotIn('stm32f4', s['family_examples']) # 'all' => no map key + + def test_bsp_family_rule(self): # rule 6 + s = self.b(['hw/bsp/stm32f4/boards/stm32f407disco/board.h']) + self.assertEqual(s['families'], ['stm32f4']) + self.assertNotIn('stm32f4', s['family_examples']) + + def test_bsp_top_level_file_is_full(self): # rule 16 + self.assertTrue(self.b(['hw/bsp/board.c'])['full']) + self.assertTrue(self.b(['hw/bsp/family_support.cmake'])['full']) + + def test_mcu_rule(self): # rule 7 + s = self.b(['hw/mcu/nordic/nrf5x/nrf_clock.h']) + self.assertEqual(s['families'], ['nrf']) + s = self.b(['hw/mcu/no_such_vendor/x.c']) # empty means empty + self.assertFalse(s['full']) + self.assertEqual(s['families'], []) + + def test_class_device_rule(self): # rule 8 + s = self.b(['src/class/cdc/cdc_device.c']) + self.assertFalse(s['full']) + # near-all families (Task 4's pruning may drop a few); never equality + # against all_bsp_families — that's a tuple, and pruning shrinks the list + self.assertIn('stm32f4', s['families']) + self.assertGreater(len(s['families']), 50) + exs = s['family_examples']['stm32f4'] + self.assertIn('device/cdc_msc', exs) + self.assertNotIn('device/hid_composite', exs) + self.assertNotIn('host/cdc_msc_hid', exs) # TUH_CDC examples are rule 9's + + def test_class_host_rule(self): # rule 9 + s = self.b(['src/class/msc/msc_host.c']) + exs = s['family_examples']['stm32f4'] + self.assertIn('host/msc_file_explorer', exs) + self.assertNotIn('device/cdc_msc', exs) + + def test_class_shared_header_and_include_edge(self): # rule 10 + s = self.b(['src/class/audio/audio.h']) + exs = s['family_examples']['stm32f4'] + self.assertIn('device/audio_test', exs) + self.assertIn('device/midi_test', exs) # midi headers include audio.h + + def test_core_device_rule(self): # rule 11 + s = self.b(['src/device/usbd.c']) + exs = s['family_examples']['stm32f4'] + self.assertIn('device/cdc_msc', exs) + # no dual In-assertion: dual examples are only.txt-gated to max3421/pio-usb + # boards, so pruning legitimately drops them on a plain stm32f4 board + self.assertFalse(any(e.startswith(('host/', 'typec/')) for e in exs)) + + def test_core_host_rule(self): # rule 12 + s = self.b(['src/host/usbh.c']) + exs = s['family_examples']['stm32f4'] + self.assertFalse(any(e.startswith(('device/', 'typec/')) for e in exs)) + + def test_example_rule(self): # rules 13, 14 + s = self.b(['examples/device/cdc_msc/src/main.c']) + self.assertEqual(s['family_examples']['stm32f4'], ['device/cdc_msc']) + s = self.b(['examples/device/board_test/src/main.c']) + self.assertEqual(s['family_examples']['stm32f4'], ['device/board_test']) + s = self.b(['examples/device/no_such_example/src/main.c']) # deleted example: nothing + self.assertFalse(s['full']) + self.assertEqual(s['families'], []) + + def test_full_paths(self): # rules 15-17 + for p in ('src/common/tusb_fifo.c', 'src/osal/osal.h', 'src/tusb.c', + 'src/tusb_option.h', 'lib/SEGGER_RTT/RTT/SEGGER_RTT.c', + 'tools/build.py', 'tools/get_deps.py', 'tools/cmake/cpu/cortex-m4.cmake', + 'examples/CMakeLists.txt', 'examples/device/CMakeLists.txt', + 'examples/build_system/cmake/cpu.cmake', '.github/workflows/build.yml', + 'sonar-project.properties', 'some/unknown/path.c'): + self.assertTrue(self.b([p])['full'], p) + + def test_mixed_diff_unions_per_family(self): + s = self.b(['src/portable/raspberrypi/rp2040/dcd_rp2040.c', 'src/class/cdc/cdc_device.c']) + self.assertFalse(s['full']) + self.assertIn('stm32f4', s['families']) + self.assertGreater(len(s['families']), 50) + self.assertIn('device/hid_composite', s['family_examples']['rp2040']) # from the dcd rule + self.assertNotIn('device/hid_composite', s['family_examples']['stm32f4']) # cdc-only there + + def test_example_names_are_real_dirs(self): + for ex in ci_select.all_examples(REPO): + role, name = ex.split('/') + self.assertTrue(os.path.isdir(os.path.join(REPO, 'examples', role, name)), ex) + self.assertRegex(ex, r'^(device|dual|host|typec)/[A-Za-z0-9_]+$') +``` + +Note for `test_mixed_diff_unions_per_family`: it encodes the per-family union — rp2040 gets DEV+DUAL ∪ cdc-set, every other family only the cdc-set (spec §Two axes). Buildability pruning may later remove entries; these Task-3 tests use families/examples that survive pruning (stm32f4 and rp2040 build all the named examples), so they stay valid after Task 4. + +- [ ] **Step 2: Run to verify failure** + +Run: `python3 test/hil/test/test_ci_select.py TestBuildClassifier -v 2>&1 | tail -3` +Expected: ERROR — `classify_build` not defined. + +- [ ] **Step 3: Implement** (append to `tools/ci_select.py`, after the HIL classifier) + +```python +# ------------------------------------------------------------- +# Build-axis classifier (spec rule table, docs/superpowers/specs/ +# 2026-08-19-ci-build-family-filter-design.md). Independent of the HIL +# classifier: same diff, second walk, its own fail-open. +# ------------------------------------------------------------- +_EX_ROLES = ('device', 'dual', 'host', 'typec') + + [email protected]_cache(maxsize=None) +def all_examples(repo_root: str) -> tuple: + """Every examples/<role>/<name> with a CMakeLists.txt, as 'role/name'.""" + out = [] + for role in _EX_ROLES: + for d in sorted(glob.glob(os.path.join(repo_root, 'examples', role, '*/'))): + if os.path.isfile(os.path.join(d, 'CMakeLists.txt')): + out.append(f'{role}/{os.path.basename(d.rstrip(os.sep))}') + return tuple(out) + + +def role_examples(repo_root: str, roles) -> set: + want = set(roles) + return {e for e in all_examples(repo_root) if e.split('/', 1)[0] in want} + + [email protected]_cache(maxsize=None) +def all_bsp_families(repo_root: str) -> tuple: + return tuple(sorted(d for d in os.listdir(os.path.join(repo_root, 'hw/bsp')) + if os.path.isdir(os.path.join(repo_root, 'hw/bsp', d)))) + + +def _build_class_examples(cls: str, base: str, roles: set, repo_root: str) -> set: + """Examples (all 46, not the HIL lists) whose tusb_config.h enables the class's + macros for the given roles, plus classes that #include the changed header.""" + via = sorted(class_include_edges(repo_root).get(f'{cls}/{base}', ())) + out = set() + for prefix, role in (('TUD', 'device'), ('TUH', 'host')): + if role not in roles: + continue + macros = class_macros(cls, base, prefix) + \ + [m for c in via for m in class_macros(c, '', prefix)] + for ex in all_examples(repo_root): + cfg = os.path.join(repo_root, 'examples', ex, 'src', 'tusb_config.h') + if _config_enables(cfg, macros): + out.add(ex) + return out + + +class _BSel: + """family -> set(examples) | 'all', unioned per family.""" + def __init__(self): + self.full = False + self.fam_ex = {} + self.reasons = [] + + def add(self, fams, examples, reason): + self.reasons.append(reason) + for f in fams: + cur = self.fam_ex.get(f) + if examples == 'all' or cur == 'all': + self.fam_ex[f] = 'all' + else: + self.fam_ex[f] = (cur or set()) | set(examples) + + def force_full(self, reason): + self.full = True + self.reasons.append(reason) + + +def _classify_build_one(path, repo_root, s: _BSel): + base = os.path.basename(path) + if _NONCODE_RE.match(path): # rule 1 + return + if re.match(r'test/hil/', path): # rule 2 + s.reasons.append(f'{path}: HIL harness, no build contribution') + return + m = re.match(r'src/portable/((?:[^/]+/)?[^/]+)/', path) + if m: # rules 3-5 + port = m.group(1) + fams = port_families(port, repo_root) + if re.match(r'(dcd_|.*_device)', base): + exs = role_examples(repo_root, ('device', 'dual')) + elif re.match(r'(hcd_|.*_host)', base): + exs = role_examples(repo_root, ('host', 'dual')) + else: + exs = 'all' + s.add(fams, exs, f'{path}: port {port} -> families {sorted(fams)}') + return + if re.match(r'hw/bsp/[^/]+/', path): # rule 6 + fam = path.split('/')[2] + s.add({fam}, 'all', f'{path}: bsp family {fam}') + return + if re.match(r'hw/mcu/', path): # rule 7 + fams = mcu_families(path, repo_root) + s.add(fams, 'all', f'{path}: mcu -> families {sorted(fams)}') + return + m = re.match(r'src/class/([^/]+)/', path) + if m: # rules 8-10 + cls = m.group(1) + if re.search(r'_device\.[ch]$', base): + roles = {'device'} + elif re.search(r'_host\.[ch]$', base): + roles = {'host'} + else: + roles = {'device', 'host'} + exs = _build_class_examples(cls, base, roles, repo_root) + s.add(all_bsp_families(repo_root), exs, + f'{path}: class {cls} -> {sorted(exs)}') + return + m = re.match(r'src/(device|host)/', path) + if m: # rules 11-12 + role = m.group(1) + s.add(all_bsp_families(repo_root), role_examples(repo_root, (role, 'dual')), + f'{path}: core {role} stack') + return + m = re.match(r'examples/(device|dual|host|typec)/([^/]+)/', path) + if m: # rules 13-14 + ex = f'{m.group(1)}/{m.group(2)}' + if ex in all_examples(repo_root): + s.add(all_bsp_families(repo_root), {ex}, f'{path}: example {ex}') + else: + # a deleted example builds nothing; removing it from the role + # CMakeLists (rule 15) is what forces the full matrix + s.reasons.append(f'{path}: not an example dir, no build contribution') + return + s.force_full(f'{path}: unclassified -> full build matrix') # rules 15-17 + + +def classify_build(changed_files, repo_root): + s = _BSel() + for p in changed_files: + _classify_build_one(p, repo_root, s) + if s.full: + return {'full': True, 'families': list(all_bsp_families(repo_root)), + 'family_examples': {}, 'reasons': s.reasons} + fams, fam_ex = [], {} + for fam, exs in sorted(s.fam_ex.items()): + fams.append(fam) + if exs != 'all': + fam_ex[fam] = sorted(exs) + return {'full': False, 'families': fams, 'family_examples': fam_ex, + 'reasons': s.reasons} +``` + +Note: `examples/<role>/CMakeLists.txt` has no trailing slash after the second component, so the example regex misses it and it correctly falls through to `force_full` (rule 15) — `test_full_paths` pins this. + +- [ ] **Step 4: Run tests** + +Run: `python3 test/hil/test/test_ci_select.py TestBuildClassifier -v` +Expected: all pass. Then the full file: `python3 test/hil/test/test_ci_select.py 2>&1 | tail -3` — all pass. + +- [ ] **Step 5: Commit** + +```bash +git add tools/ci_select.py test/hil/test/test_ci_select.py +git commit -m "ci_select: add build-axis classifier (families x example targets)" +``` + +--- + +### Task 4: Buildability post-filter, `build` + `hil_examples` output keys + +**Files:** +- Modify: `tools/ci_select.py` (imports, post-filter, `main()`), `test/hil/test/test_hil_util.py` (BottomLayer lists) +- Test: `test/hil/test/test_ci_select.py` + +**Interfaces:** +- Produces: `classify_build` result is now pruned: each family's list intersected with what that family's CI board can build (`build_utils.skip_example`); family dropped when nothing survives; map key omitted when the kept set equals everything the board can build. `hil_examples(sel, rosters) -> {board: [example]}` — the board's selected tests (`sel['boards'][name]` when narrowed, else `board_tests`) plus always `device/board_test`. CLI JSON gains top-level `"build": {...}` (always) and `"hil_examples": {...}` (when rosters given; emitted even when `full` is true). +- Consumes: `tools/build_utils.skip_example(example, board)`; `tools/build.py:get_family_boards(family, one_random, one_first)` (module import — no behavior change to build.py yet). + +- [ ] **Step 1: Write the failing tests** + +```python +class TestBuildPostFilter(unittest.TestCase): + def test_kept_examples_are_buildable(self): + import build_utils, build as build_py + s = ci_select.classify_build(['src/class/msc/msc_host.c'], REPO) + self.assertFalse(s['full']) + # families that cannot build a single TUH_MSC example drop out entirely + self.assertNotIn('msp430', s['families']) + old = os.getcwd() + os.chdir(REPO) + try: + for fam, exs in s['family_examples'].items(): + board = build_py.get_family_boards(fam, False, True)[0] + for e in exs: + self.assertFalse(build_utils.skip_example(e, board), f'{fam}: {e}') + finally: + os.chdir(old) + + def test_unfiltered_family_has_no_map_key(self): + s = ci_select.classify_build(['hw/bsp/stm32f4/family.c'], REPO) + self.assertEqual(s['families'], ['stm32f4']) + self.assertEqual(s['family_examples'], {}) + + def test_no_stdout_pollution(self): + # get_family_boards prints on odd families; the selector's stdout is JSON + import io, contextlib + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + ci_select.classify_build(['src/class/msc/msc_host.c'], REPO) + self.assertEqual(buf.getvalue(), '') + + +class TestHilExamples(unittest.TestCase): + def test_board_test_always_present_and_full_emits(self): + s = ci_select.classify(['src/common/tusb_fifo.c'], REPO, ROSTERS) # full + he = ci_select.hil_examples(s, ROSTERS) + self.assertEqual(set(he), {b['name'] for b in ROSTER}) + for name, exs in he.items(): + self.assertIn('device/board_test', exs) + + def test_narrowed_board_gets_chosen_tests_only(self): + s = ci_select.classify(['examples/device/cdc_msc/src/main.c'], REPO, ROSTERS) + he = ci_select.hil_examples(s, ROSTERS) + self.assertEqual(he['stm32f407disco'], ['device/board_test', 'device/cdc_msc']) + + def test_full_board_gets_its_whole_test_list(self): + s = ci_select.classify(['hw/bsp/stm32f4/boards/stm32f407disco/board.h'], REPO, ROSTERS) + he = ci_select.hil_examples(s, ROSTERS) + want = set(ci_select.board_tests(ROSTER[1])) | {'device/board_test'} + self.assertEqual(set(he['stm32f407disco']), want) + self.assertNotIn('raspberry_pi_pico', he) # deselected board: no firmware needed + + +class TestCliJson(unittest.TestCase): + def test_build_key_without_rosters(self): + r = subprocess.run([sys.executable, os.path.join(REPO, 'tools/ci_select.py'), + '--diff-file', '/dev/null'], capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + j = json.loads(r.stdout) + self.assertIn('build', j) + self.assertNotIn('hil_examples', j) # rosters not given + + def test_build_and_hil_keys_with_rosters(self): + import tempfile + with tempfile.NamedTemporaryFile('w', suffix='.txt', delete=False) as f: + f.write('src/portable/raspberrypi/rp2040/dcd_rp2040.c\n') + df = f.name + r = subprocess.run([sys.executable, os.path.join(REPO, 'tools/ci_select.py'), + '--diff-file', df, os.path.join(REPO, 'test/hil/tinyusb.json')], + capture_output=True, text=True) + os.unlink(df) + j = json.loads(r.stdout) + self.assertEqual(j['build']['families'], ['rp2040']) + self.assertIn('hil_examples', j) + for exs in j['hil_examples'].values(): + self.assertIn('device/board_test', exs) +``` + +(`subprocess`, `sys` are already imported in the test file.) + +- [ ] **Step 2: Run to verify failure** + +Run: `python3 test/hil/test/test_ci_select.py TestBuildPostFilter TestHilExamples TestCliJson -v 2>&1 | tail -3` +Expected: FAIL — no pruning, no `hil_examples`, no `build` key. + +- [ ] **Step 3: Implement** + +In `tools/ci_select.py` module header, after the existing `helper` import, add: + +```python +import contextlib +import io + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # tools/, for build helpers +import build_utils +import build as build_py +``` + +(`contextlib`/`io` go into the stdlib import block at the top.) Add the pruning helpers and rewrite the tail of `classify_build`: + +```python +def _in_repo(repo_root): + """build_utils/build.py use repo-relative paths; scope a chdir around them. + get_family_boards also prints on an empty family - swallow stdout so the + selector's machine-read JSON stays clean (diagnostics belong on stderr).""" + old = os.getcwd() + os.chdir(repo_root) + try: + with contextlib.redirect_stdout(io.StringIO()): + yield + finally: + os.chdir(old) + + +def _prune_buildable(fams, fam_ex, repo_root): + """Intersect each family's selection with what its CI board can build + (build_utils.skip_example - the same skip.txt/only.txt data CMake's + family_filter reads). get_family_boards mirrors the build jobs' one-first + pick, CI preferred/skip lists included.""" + out_fams, out_ex = [], {} + allex = list(all_examples(repo_root)) + with _in_repo(repo_root): + for fam in fams: + boards = build_py.get_family_boards(fam, False, True) + if not boards: + out_fams.append(fam) # unknown layout: keep unfiltered + continue + board = boards[0] + buildable = [e for e in allex if not build_utils.skip_example(e, board)] + want = fam_ex.get(fam) + kept = buildable if want is None else [e for e in want if e in set(buildable)] + if not kept: + continue # this diff builds nothing for this family + out_fams.append(fam) + if set(kept) != set(buildable): + out_ex[fam] = kept + return out_fams, out_ex +``` + +Replace `classify_build`'s non-full return with: + +```python + fams = sorted(s.fam_ex) + fam_ex = {f: sorted(e) for f, e in s.fam_ex.items() if e != 'all'} + fams, fam_ex = _prune_buildable(fams, fam_ex, repo_root) + return {'full': False, 'families': fams, 'family_examples': fam_ex, + 'reasons': s.reasons} +``` + +Add `hil_examples` beside `selection_args`: + +```python +def hil_examples(sel, rosters): + """{board: examples hil-build must produce}: the board's selected tests plus + device/board_test, which hil_test.py flashes to park at every variant + boundary and at end-of-board teardown. Emitted for full selections too - the + HIL example universe is a fraction of the tree regardless of the diff.""" + by_name = {} + for _, boards in rosters: + for b in boards: + by_name.setdefault(b['name'], b) + if sel['full']: + chosen = {n: 'all' for n in by_name} + else: + chosen = sel['boards'] + out = {} + for name, tests in chosen.items(): + run = board_tests(by_name[name]) if tests == 'all' else list(tests) + out[name] = sorted(set(run) | {'device/board_test'}) + return out +``` + +In `main()`: change the configs argument to optional — `ap.add_argument('configs', nargs='*', help='rig roster JSON file(s); omit for the build view alone')` — so CircleCI (which never touches HIL) can run without rosters; with no configs, `rosters` is `[]`, the HIL keys degrade to empty, and `hil_examples` is omitted. Then after the `args_flasher` line: + +```python + if rosters: + s['hil_examples'] = hil_examples(s, rosters) + s['build'] = classify_build(files, repo_root) + for r in s['build']['reasons']: + print(f'ci_select[build]: {r}', file=sys.stderr) +``` + +Update `test/hil/test/test_hil_util.py` BottomLayer: add `'build'`, `'build_utils'` to the `local` allowed set and `'../../tools/build'`, `'../../tools/build_utils'` to the module-path tuple (ci_select now imports both on the bare runner). + +- [ ] **Step 4: Run tests + timing check** + +```bash +python3 test/hil/test/test_ci_select.py 2>&1 | tail -3 +python3 -m unittest discover -s test/hil/test 2>&1 | tail -3 +time python3 tools/ci_select.py --diff-file <(echo src/class/cdc/cdc_device.c) test/hil/tinyusb.json >/dev/null +``` + +Expected: suites pass; the timed run stays under ~5 s (skip_example over 75 families × 46 examples re-reads small files — if it exceeds that, memoize `skip_example` results per (example, board) inside `_prune_buildable`). + +- [ ] **Step 5: Commit** + +```bash +git add tools/ci_select.py test/hil/test/test_ci_select.py test/hil/test/test_hil_util.py +git commit -m "ci_select: prune build selection by example buildability, emit build + hil_examples keys" +``` + +--- + +### Task 5: `ci_set_matrix.py --select / --base` + +**Files:** +- Modify: `.github/scripts/ci_set_matrix.py` +- Test: `test/hil/test/test_ci_select.py` + +**Interfaces:** +- Produces: CLI `python .github/scripts/ci_set_matrix.py [--select JSON | --base REF]`. No flags → byte-identical to today's output. `--select`: families intersected with `select.build.families` unless `build.full`; unusable JSON → full matrix + stderr warning. `--base REF`: runs `tools/ci_select.py --base REF` itself and proceeds as `--select`. Output shape `{toolchain: [family]}` unchanged. + +- [ ] **Step 1: Write the failing tests** + +```python +SET_MATRIX = os.path.join(REPO, '.github/scripts/ci_set_matrix.py') + +class TestCiSetMatrix(unittest.TestCase): + def run_matrix(self, *args): + return subprocess.run([sys.executable, SET_MATRIX, *args], + capture_output=True, text=True) + + def test_no_flags_is_todays_output(self): + r = self.run_matrix() + self.assertEqual(r.returncode, 0, r.stderr) + self.baseline = json.loads(r.stdout) + self.assertIn('stm32f4', self.baseline['arm-gcc']) + + def test_select_full_is_identical(self): + base = json.loads(self.run_matrix().stdout) + sel = json.dumps({'build': {'full': True, 'families': [], 'family_examples': {}}}) + self.assertEqual(json.loads(self.run_matrix('--select', sel).stdout), base) + + def test_select_narrow_is_a_subset(self): + sel = json.dumps({'build': {'full': False, 'families': ['rp2040', 'stm32f4'], + 'family_examples': {}}}) + m = json.loads(self.run_matrix('--select', sel).stdout) + self.assertEqual(m['arm-gcc'], ['rp2040', 'stm32f4']) + self.assertEqual(m['riscv-gcc'], []) + self.assertEqual(set(m), set(json.loads(self.run_matrix().stdout))) # all keys kept + + def test_malformed_select_falls_open(self): + base = json.loads(self.run_matrix().stdout) + r = self.run_matrix('--select', 'not json {') + self.assertEqual(r.returncode, 0) + self.assertEqual(json.loads(r.stdout), base) + self.assertIn('full matrix', r.stderr) +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `python3 test/hil/test/test_ci_select.py TestCiSetMatrix -v 2>&1 | tail -3` +Expected: FAIL — argparse rejects `--select`. + +- [ ] **Step 3: Implement** + +In `.github/scripts/ci_set_matrix.py`, add imports `argparse, os, subprocess, sys` and replace `set_matrix_json` + the main guard: + +```python +def set_matrix_json(select=None): + sel_fams = None + if select: + b = select.get('build') or {} + if b.get('full') is False: + sel_fams = set(b.get('families') or []) + matrix = {} + for toolchain in toolchain_list: + fams = [family for family, tc in family_list.items() if toolchain in tc] + if sel_fams is not None: + fams = [f for f in fams if f in sel_fams] + matrix[toolchain] = fams + print(json.dumps(matrix)) + + +def main(): + parser = argparse.ArgumentParser() + group = parser.add_mutually_exclusive_group() + group.add_argument('--select', help='tools/ci_select.py JSON; scopes families when build.full is false') + group.add_argument('--base', help='git ref: run tools/ci_select.py --base REF and scope from it') + args = parser.parse_args() + + select = None + try: + if args.select: + select = json.loads(args.select) + elif args.base: + root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + r = subprocess.run([sys.executable, os.path.join(root, 'tools', 'ci_select.py'), + '--base', args.base], + capture_output=True, text=True, cwd=root, check=True) + select = json.loads(r.stdout) + except Exception as e: # fail-open: an unusable selection must never turn into a red job + print(f'ci_set_matrix: selection unusable ({e}) - full matrix', file=sys.stderr) + select = None + set_matrix_json(select) + + +if __name__ == '__main__': + main() +``` + +- [ ] **Step 4: Run tests** + +Run: `python3 test/hil/test/test_ci_select.py TestCiSetMatrix -v` — all pass. +Also: `python3 .github/scripts/ci_set_matrix.py | diff - <(git show HEAD:.github/scripts/ci_set_matrix.py | python3 -)` → no diff (byte-identical default output). + +- [ ] **Step 5: Extend the pre-commit hook scope and commit** + +In `.pre-commit-config.yaml`, `ci-select-test` hook: `files: ^(hw/bsp/|src/|examples/|tools/(ci_select|build|build_utils)\.py$|\.github/scripts/)`. + +```bash +git add .github/scripts/ci_set_matrix.py test/hil/test/test_ci_select.py .pre-commit-config.yaml +git commit -m "ci_set_matrix: scope the family matrix from a ci_select selection" +``` + +--- + +### Task 6: `hil_ci_set_matrix.py` emits `-e` per board + +**Files:** +- Modify: `.github/scripts/hil_ci_set_matrix.py` +- Test: `test/hil/test/test_ci_select.py` + +**Interfaces:** +- Produces: each build entry for board `B` gains ` -e <ex>` for every entry of `select.hil_examples[B]` (before variant expansion, so all of a board's variants carry the same list). No `hil_examples` key (hand runs, old selectors) → output byte-identical to today. +- Consumed by: `hil-build` / `hil-build-esp` (via `build_util.yml` → `tools/build.py`), `hil-hfp-iar`'s inline build loop — all funnel into `tools/build.py`, which learns `-e` in Task 7. + +- [ ] **Step 1: Write the failing tests** + +```python +HIL_SET_MATRIX = os.path.join(REPO, '.github/scripts/hil_ci_set_matrix.py') + +class TestHilCiSetMatrixExamples(unittest.TestCase): + def run_matrix(self, *args): + r = subprocess.run([sys.executable, HIL_SET_MATRIX, *args, + os.path.join(REPO, 'test/hil/tinyusb.json')], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + return r.stdout + + def test_no_hil_examples_is_byte_identical(self): + plain = self.run_matrix() + sel = json.dumps({'full': True, 'boards': {}}) + self.assertEqual(self.run_matrix('--select', sel), plain) + + def test_examples_appended_per_board(self): + board = on_roster(self, 'stm32f407disco')[0] + sel = json.dumps({'full': False, 'boards': {board: 'all'}, + 'hil_examples': {board: ['device/board_test', 'device/cdc_msc']}}) + m = json.loads(self.run_matrix('--select', sel)) + entries = [e for entries in m.values() for e in entries] + self.assertTrue(entries) + for e in entries: + self.assertIn(f'-b {board}', e) + self.assertIn('-e device/board_test', e) + self.assertIn('-e device/cdc_msc', e) +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `python3 test/hil/test/test_ci_select.py TestHilCiSetMatrixExamples -v` +Expected: `test_examples_appended_per_board` FAILS (no `-e` in entries). + +- [ ] **Step 3: Implement** + +In `hil_ci_set_matrix.py` `main()`, after the `selected` computation add `ex_map = (sel or {}).get('hil_examples', {})`, and in the board loop, after the `build.args` append (line ~72): + +```python + # PR selection: build only the examples this board will run (its test + # list plus device/board_test, the parking firmware) - tools/build.py -e. + # Absent key (hand runs, full non-PR builds) keeps --target all. + for ex in ex_map.get(name, []): + build_board += f' -e {ex}' +``` + +- [ ] **Step 4: Run tests** + +Run: `python3 test/hil/test/test_ci_select.py -v 2>&1 | tail -3` — all pass. + +- [ ] **Step 5: Commit** + +```bash +git add .github/scripts/hil_ci_set_matrix.py test/hil/test/test_ci_select.py +git commit -m "hil_ci_set_matrix: append per-board -e example filters from the selection" +``` + +--- + +### Task 7: `tools/build.py --example` + +**Files:** +- Modify: `tools/build.py` +- Test: `test/hil/test/test_ci_select.py` + +**Interfaces:** +- Produces: repeatable `-e/--example role/name`. Without it, behavior is exactly today's (`--target all`). With it: cmake builds one `--target <name>` per requested example the board can build (`build_utils.skip_example`), mapping `all` → example names and `examples-membrowse-upload` → `<name>-membrowse-upload` (the aggregate target `DEPENDS` every example — `hw/bsp/family_support.cmake:346-360` — and would rebuild the excluded ones); `tinyusb_metrics` and other targets pass through, order preserved. A board whose intersection is empty reports **skipped**. Make and espressif paths filter their example lists the same way. New helper `resolve_example_targets(build_targets, examples, board) -> list | None` (None = nothing buildable). + +- [ ] **Step 1: Write the failing tests** + +```python +class TestBuildPyExampleFilter(unittest.TestCase): + def setUp(self): + import build as build_py + self.build = build_py + self.old = os.getcwd() + os.chdir(REPO) # skip_example uses repo-relative paths + + def tearDown(self): + os.chdir(self.old) + + def test_all_maps_to_example_names(self): + t = self.build.resolve_example_targets(['all'], ['device/cdc_msc', 'device/dfu'], + 'stm32f407disco') + self.assertEqual(t, ['cdc_msc', 'dfu']) + + def test_membrowse_maps_per_example(self): + t = self.build.resolve_example_targets(['all', 'examples-membrowse-upload'], + ['device/cdc_msc'], 'stm32f407disco') + self.assertEqual(t, ['cdc_msc', 'cdc_msc-membrowse-upload']) + + def test_other_targets_pass_through_in_order(self): + t = self.build.resolve_example_targets(['all', 'tinyusb_metrics'], + ['device/cdc_msc'], 'stm32f407disco') + self.assertEqual(t, ['cdc_msc', 'tinyusb_metrics']) + + def test_unbuildable_examples_drop_and_empty_is_none(self): + # typec/power_delivery only builds on stm32g4-class parts, never on f4 + t = self.build.resolve_example_targets(['all'], + ['typec/power_delivery', 'device/cdc_msc'], + 'stm32f407disco') + self.assertEqual(t, ['cdc_msc']) + self.assertIsNone(self.build.resolve_example_targets(['all'], + ['typec/power_delivery'], + 'stm32f407disco')) +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `python3 test/hil/test/test_ci_select.py TestBuildPyExampleFilter -v` +Expected: ERROR — `resolve_example_targets` not defined. + +- [ ] **Step 3: Implement** + +In `tools/build.py` add near `get_examples`: + +```python +def resolve_example_targets(build_targets, examples, board): + """Map generic targets onto per-example targets for a filtered build (-e). + 'all' -> the example executables; 'examples-membrowse-upload' -> per-example + upload targets (the aggregate DEPENDS on every example and would rebuild the + excluded ones); anything else (e.g. tinyusb_metrics) passes through. + Returns None when no requested example is buildable on this board.""" + buildable = [e for e in examples if not build_utils.skip_example(e, board)] + if not buildable: + return None + names = [e.split('/', 1)[1] for e in buildable] + out = [] + for t in build_targets: + if t == 'all': + out += names + elif t == 'examples-membrowse-upload': + out += [f'{n}-membrowse-upload' for n in names] + else: + out.append(t) + return list(dict.fromkeys(out)) +``` + +Thread `examples` (a list or `None`) through `main()` → `build_boards_list` → `cmake_board`/`make_board`: + +- `main()`: `parser.add_argument('-e', '--example', action='append', default=[], help='Only build these examples (role/name, repeatable). Default: all examples')`; pass `args.example or None` as a new final parameter of `build_boards_list`. +- `build_boards_list(..., examples=None)`: forward to both branches. +- `cmake_board(..., examples=None)`: in the espressif branch, after `all_examples = get_examples(family)` insert: + +```python + if examples is not None: + all_examples = [e for e in all_examples if e in examples] +``` + + In the generic branch, replace the target loop: + +```python + if rcmd.returncode == 0: + targets = build_targets + if examples is not None: + targets = resolve_example_targets(build_targets, examples, board) + if targets is None: + print_build_result(board, 'examples (PR filter)', 2, '-') + return [0, 0, 1] + cmd = ["cmake", "--build", build_dir, '--parallel', str(parallel_jobs)] + for target in targets: + rcmd = run_cmd(cmd + ['--target', target]) + if rcmd.returncode != 0: + break +``` + +- `make_board(..., examples=None)`: after `all_examples = get_examples(family)`: + +```python + if examples is not None: + all_examples = [e for e in all_examples if e in examples] + if not all_examples: + print_build_result(board, 'examples (PR filter)', 2, '-') + return [0, 0, 1] +``` + +- [ ] **Step 4: Run tests + a real filtered build** + +```bash +python3 test/hil/test/test_ci_select.py TestBuildPyExampleFilter -v +python3 tools/build.py -e device/cdc_msc -e device/cdc_dual_ports -b stm32f407disco +ls cmake-build/cmake-build-stm32f407disco/device/cdc_msc/cdc_msc.elf \ + cmake-build/cmake-build-stm32f407disco/device/cdc_dual_ports/cdc_dual_ports.elf +python3 tools/build.py -e typec/power_delivery -b stm32f407disco # expect: Skipped row, exit 0 +``` + +Expected: tests pass; both elfs exist; the typec run prints a Skipped result and exits 0. + +- [ ] **Step 5: Commit** + +```bash +git add tools/build.py test/hil/test/test_ci_select.py +git commit -m "build.py: add -e/--example filter with per-example target mapping" +``` + +--- + +### Task 8: `metrics.py --by-example` + by-example expansion + CMake wiring + +**Files:** +- Modify: `tools/metrics.py`, `examples/CMakeLists.txt`, `.pre-commit-config.yaml` +- Create + Test: `test/hil/test/test_ci_metrics.py` + +**Interfaces:** +- Produces: `metrics.py combine --by-example` additionally writes `<out>_by_example.json` = `{"<role>/<example>": {"files": [...]}}`, the example id taken from the map.json's two parent dirs (`<build>/<role>/<example>/*.map.json`). `combine` also accepts a by-example JSON as *input*, expanding each example to one data entry, with `--only-examples a,b` filtering which. `combine_files(input_files, filters=None, only_examples=None)`. Existing outputs byte-identical when the new flags are absent. +- Consumed by: `examples/CMakeLists.txt` `tinyusb_metrics` target (adds the flag), Task 9's pair-compare, Task 10's artifact upload. + +- [ ] **Step 1: Write the failing tests** (new file `test/hil/test/test_ci_metrics.py`) + +```python +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests for the by-example half of tools/metrics.py and the (family, example) +# pair-compare script. Stdlib only; synthetic map.json fixtures, no builds. +# python3 test/hil/test/test_ci_metrics.py +import json +import os +import subprocess +import sys +import tempfile +import unittest + +REPO = os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.abspath(__file__))))) +METRICS = os.path.join(REPO, 'tools', 'metrics.py') + + +def fake_map(path, files): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'w') as f: + json.dump({'files': files}, f) + + +def entry(name, size, path_prefix='tinyusb/src'): + return {'file': name, 'path': f'{path_prefix}/{name}', 'size': size, + 'symbols': [{'name': f'{name}_fn', 'size': size}], 'sections': {'.text': size}} + + +class TestByExample(unittest.TestCase): + def build_tree(self, td): + fake_map(os.path.join(td, 'device', 'cdc_msc', 'cdc_msc.map.json'), + [entry('usbd.c', 100), entry('cdc_device.c', 50)]) + fake_map(os.path.join(td, 'host', 'bare_api', 'bare_api.map.json'), + [entry('usbh.c', 200)]) + + def test_by_example_output(self): + with tempfile.TemporaryDirectory() as td: + self.build_tree(td) + out = os.path.join(td, 'metrics') + r = subprocess.run([sys.executable, METRICS, 'combine', '-q', '-j', + '--by-example', '-o', out, + os.path.join(td, '*', '*', '*.map.json')], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + by_ex = json.load(open(out + '_by_example.json')) + self.assertEqual(set(by_ex), {'device/cdc_msc', 'host/bare_api'}) + self.assertEqual({f['file'] for f in by_ex['device/cdc_msc']['files']}, + {'usbd.c', 'cdc_device.c'}) + # the plain averaged output is unchanged by the extra flag + avg = json.load(open(out + '.json')) + self.assertIn('files', avg) + + def test_by_example_json_roundtrips_as_combine_input(self): + with tempfile.TemporaryDirectory() as td: + self.build_tree(td) + out = os.path.join(td, 'metrics') + subprocess.run([sys.executable, METRICS, 'combine', '-q', '-j', '--by-example', + '-o', out, os.path.join(td, '*', '*', '*.map.json')], check=True) + out2 = os.path.join(td, 'sub') + r = subprocess.run([sys.executable, METRICS, 'combine', '-q', '-j', + '--only-examples', 'device/cdc_msc', + '-o', out2, out + '_by_example.json'], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + sub = json.load(open(out2 + '.json')) + names = {f['file'] for f in sub['files']} + self.assertEqual(names, {'usbd.c', 'cdc_device.c'}) # bare_api filtered out + + +if __name__ == '__main__': + unittest.main() +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `python3 test/hil/test/test_ci_metrics.py -v` +Expected: FAIL — argparse rejects `--by-example`. + +- [ ] **Step 3: Implement in `tools/metrics.py`** + +`combine_files` signature → `combine_files(input_files, filters=None, only_examples=None)`. Inside the `.json` branch, after `json.load`, insert the by-example expansion before the filter logic: + +```python + if 'files' not in json_data and json_data and \ + all(isinstance(v, dict) and 'files' in v for v in json_data.values()): + # a metrics_by_example.json: one data entry per example + for ex in sorted(json_data): + if only_examples and ex not in only_examples: + continue + sub = {'files': list(json_data[ex]['files'])} + if filters: + sub['files'] = [f for f in sub['files'] + if f.get('path') and any(x in f['path'] for x in filters)] + all_json_data['file_list'].append(f'{fin}:{ex}') + all_json_data['data'].append(sub) + continue +``` + +Add a writer near `write_json_output`: + +```python +def write_by_example(input_files, filters, path): + """{<role>/<example>: {files: [...]}} from map.json inputs laid out as + <build>/<role>/<example>/<name>.map.json (examples/CMakeLists.txt's pattern).""" + out = {} + for fin in input_files: + d = os.path.dirname(os.path.abspath(fin)) + ex = f'{os.path.basename(os.path.dirname(d))}/{os.path.basename(d)}' + data = combine_files([fin], filters) + if data['data']: + out.setdefault(ex, {'files': []})['files'] += data['data'][0].get('files', []) + with open(path, 'w', encoding='utf-8') as f: + json.dump(out, f) +``` + +`cmd_combine`: pass `only_examples=set(args.only_examples.split(',')) if args.only_examples else None` into `combine_files`, and after the existing outputs: + +```python + if args.by_example: + write_by_example(input_files, args.filters, args.out + '_by_example.json') +``` + +Argparse additions on the combine subparser: + +```python + combine_parser.add_argument('--by-example', dest='by_example', action='store_true', + help='Also write <out>_by_example.json: per-example file lists keyed by role/example') + combine_parser.add_argument('--only-examples', dest='only_examples', default='', + help='Comma-separated role/example ids to keep when reading by-example JSON inputs') +``` + +- [ ] **Step 4: Wire CMake + hooks** + +`examples/CMakeLists.txt` `tinyusb_metrics` target: change the command to +`combine -f tinyusb/src -j --by-example -o ${CMAKE_BINARY_DIR}/metrics` (one added flag). +`.pre-commit-config.yaml` `hil-test` hook: `files: ^(test/hil/|examples/device/mtp/src/|tools/metrics\.py$|\.github/scripts/metrics_pair_compare\.py$)`. + +- [ ] **Step 5: Run tests** + +```bash +python3 test/hil/test/test_ci_metrics.py -v # pass +python3 -m unittest discover -s test/hil/test 2>&1 | tail -3 # discovery picks the new file up +``` + +- [ ] **Step 6: Commit** + +```bash +git add tools/metrics.py examples/CMakeLists.txt test/hil/test/test_ci_metrics.py .pre-commit-config.yaml +git commit -m "metrics: emit and consume per-example size data (--by-example, --only-examples)" +``` + +--- + +### Task 9: `(family, example)`-intersection compare script + +**Files:** +- Create: `.github/scripts/metrics_pair_compare.py` +- Test: `test/hil/test/test_ci_metrics.py` + +**Interfaces:** +- Produces: CLI `metrics_pair_compare.py --base-dir D1 --new-dir D2 [--out metrics_compare]`. Each dir is searched recursively for `cmake-build-<board>/metrics_by_example.json`; board → family via `hw/bsp/*/boards/<board>`. Writes `<out>.md`: the standard compare table over the intersection of `(family, example)` pairs, then a scope footer naming the compared families and any pairs missing on one side. Empty intersection → an explanatory one-line `.md`, exit 0. +- Consumes: `tools/metrics.py` internals `combine_files`/`compute_avg`-backed `compare_files` and `write_compare_markdown` (via `sys.path` import). + +- [ ] **Step 1: Write the failing tests** (append to `test_ci_metrics.py`) + +```python +PAIR_COMPARE = os.path.join(REPO, '.github/scripts/metrics_pair_compare.py') + + +def fake_by_example(root, board, data): + d = os.path.join(root, f'cmake-build-{board}') + os.makedirs(d, exist_ok=True) + with open(os.path.join(d, 'metrics_by_example.json'), 'w') as f: + json.dump(data, f) + + +class TestPairCompare(unittest.TestCase): + def test_intersection_compare(self): + with tempfile.TemporaryDirectory() as td: + base, new = os.path.join(td, 'base'), os.path.join(td, 'new') + # real board names so board->family resolution works against hw/bsp + fake_by_example(base, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('usbd.c', 100)]}, + 'device/dfu': {'files': [entry('dfu_device.c', 10)]}}) + fake_by_example(new, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('usbd.c', 120)]}}) + out = os.path.join(td, 'cmp') + r = subprocess.run([sys.executable, PAIR_COMPARE, '--base-dir', base, + '--new-dir', new, '--out', out], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + md = open(out + '.md').read() + self.assertIn('usbd.c', md) + self.assertNotIn('dfu_device.c', md) # not on both sides + self.assertIn('rp2040', md) # scope footer + self.assertIn('device/dfu', md) # named as dropped + + def test_empty_intersection_writes_note(self): + with tempfile.TemporaryDirectory() as td: + base, new = os.path.join(td, 'base'), os.path.join(td, 'new') + fake_by_example(base, 'raspberry_pi_pico', {'device/dfu': {'files': [entry('a.c', 1)]}}) + fake_by_example(new, 'stm32f407disco', {'device/cdc_msc': {'files': [entry('b.c', 1)]}}) + out = os.path.join(td, 'cmp') + r = subprocess.run([sys.executable, PAIR_COMPARE, '--base-dir', base, + '--new-dir', new, '--out', out], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn('skipped', open(out + '.md').read()) +``` + +- [ ] **Step 2: Run to verify failure** + +Run: `python3 test/hil/test/test_ci_metrics.py TestPairCompare -v` +Expected: FAIL — script does not exist. + +- [ ] **Step 3: Implement `.github/scripts/metrics_pair_compare.py`** + +```python +#!/usr/bin/env python3 +"""Family+example-matched code-size compare for PR-scoped builds. + +The averaged metrics baseline (metrics-tinyusb) spans every family and example; +a scoped PR builds a subset, so comparing against it is apples-to-oranges. This +compares the intersection of (family, example) pairs present on BOTH sides, +averaged over exactly those pairs, and names what was dropped. See +docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md #code-metrics. +""" +import argparse +import glob +import json +import os +import sys +import tempfile + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'tools')) +import metrics + + +def board_family(board, repo_root): + hits = glob.glob(os.path.join(repo_root, 'hw/bsp/*/boards', board)) + return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None + + +def collect(root, repo_root): + """{(family, 'role/example'): [file entries]} from every + **/cmake-build-<board>/metrics_by_example.json under root.""" + pairs = {} + pat = os.path.join(root, '**', 'metrics_by_example.json') + for f in sorted(glob.glob(pat, recursive=True)): + board = os.path.basename(os.path.dirname(f)) + if not board.startswith('cmake-build-'): + continue + fam = board_family(board[len('cmake-build-'):], repo_root) + if not fam: + print(f'pair_compare: no family for {board}, skipping', file=sys.stderr) + continue + try: + data = json.load(open(f)) + except (OSError, ValueError) as e: + print(f'pair_compare: unreadable {f} ({e}), skipping', file=sys.stderr) + continue + for ex, ent in data.items(): + pairs.setdefault((fam, ex), []).extend(ent.get('files', [])) + return pairs + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument('--base-dir', required=True) + ap.add_argument('--new-dir', required=True) + ap.add_argument('--out', default='metrics_compare') + a = ap.parse_args() + repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + base = collect(a.base_dir, repo_root) + new = collect(a.new_dir, repo_root) + common = sorted(set(base) & set(new)) + dropped = sorted(set(base) ^ set(new)) + + if not common: + with open(a.out + '.md', 'w') as f: + f.write('_Code-size comparison skipped: no (family, example) pair was built ' + 'on both the base branch and this PR._\n') + return + + def synth(pairs, path): + with open(path, 'w') as f: + json.dump({'files': [e for k in common for e in pairs[k]]}, f) + + with tempfile.TemporaryDirectory() as td: + b, n = os.path.join(td, 'base.json'), os.path.join(td, 'new.json') + synth(base, b) + synth(new, n) + comparison = metrics.compare_files(b, n, ['tinyusb/src']) + if comparison is None: + with open(a.out + '.md', 'w') as f: + f.write('_Code-size comparison failed to produce data._\n') + return + metrics.write_compare_markdown(comparison, a.out + '.md', 'name+') + + with open(a.out + '.md', 'a') as f: + fams = sorted({k[0] for k in common}) + f.write(f'\n_Scoped compare: {len(common)} (family, example) pairs across ' + f'{", ".join(fams)}._\n') + if dropped: + f.write('_Not compared (missing on one side): ' + + ', '.join(f'{fam}:{ex}' for fam, ex in dropped) + '._\n') + + +if __name__ == '__main__': + main() +``` + +- [ ] **Step 4: Run tests** + +Run: `python3 test/hil/test/test_ci_metrics.py -v` — all pass. + +- [ ] **Step 5: Commit** + +```bash +git add .github/scripts/metrics_pair_compare.py test/hil/test/test_ci_metrics.py +git commit -m "ci: add (family, example)-intersection code-size compare for scoped PRs" +``` + +--- + +### Task 10: GitHub Actions wiring (`build.yml` + `build_util.yml`) + +**Files:** +- Modify: `.github/workflows/build.yml`, `.github/workflows/build_util.yml` + +**Interfaces:** +- `set-matrix` new outputs: `example_map` (JSON `{family: [example]}`), `build_filtered` (`'true'`/`'false'`), `build_families_regex` (`fam1|fam2`, only when filtered). +- `build_util.yml` new input `example-map` (string, default `''`); when set, each leg resolves `-e` flags for its `matrix.arg` family and appends them (via env `$EX_ARGS`) to the Build and Membrowse invocations; metrics upload also grabs `metrics_by_example.json`. +- `code-metrics` gains `needs: set-matrix` and a scoped-baseline path. + +- [ ] **Step 1: Rename + thread the selection in `set-matrix`** + +Rename the step `HIL selection (PR only)` → `CI selection (PR only)` (id stays `hil-select`; renaming the id would touch every `steps.hil-select` reference — leave it). In the **Generate matrix json** step, replace the first three lines of the script (`MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py)` and the two echo lines) with: + +```bash + # Build matrix, scoped by the PR selection when one exists. Best-effort: + # ci_set_matrix falls back to the full matrix itself on unusable JSON, + # and an empty $SELECT (non-PR event, selector fallback) means no flags. + if [ -n "$SELECT" ]; then + MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py --select "$SELECT") || MATRIX_JSON='' + else + MATRIX_JSON='' + fi + [ -z "$MATRIX_JSON" ] && MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py) + echo "matrix=$MATRIX_JSON" + echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT + + # Build-axis extras: the per-family example map rides as a side channel + # (a value inside matrix entries would break CircleCI's family parameter + # and multiply GHA matrix legs). NOTE jq's // treats false like null, so + # .build.full is compared explicitly. + EXAMPLE_MAP=$(printf '%s' "${SELECT:-null}" | jq -c '.build.family_examples // {}') || EXAMPLE_MAP='{}' + BUILD_FILTERED=$(printf '%s' "${SELECT:-null}" | jq -r 'if (.build? | type) == "object" and .build.full == false then "true" else "false" end') || BUILD_FILTERED='false' + FAM_REGEX='' + if [ "$BUILD_FILTERED" = "true" ]; then + FAM_REGEX=$(printf '%s' "$SELECT" | jq -r '.build.families | join("|")') || FAM_REGEX='' + [ -z "$FAM_REGEX" ] && BUILD_FILTERED='false' + fi + echo "example_map=$EXAMPLE_MAP" >> $GITHUB_OUTPUT + echo "build_filtered=$BUILD_FILTERED" >> $GITHUB_OUTPUT + echo "build_families_regex=$FAM_REGEX" >> $GITHUB_OUTPUT +``` + +Add to the `set-matrix` job `outputs:` block: + +```yaml + example_map: ${{ steps.set-matrix-json.outputs.example_map }} + build_filtered: ${{ steps.set-matrix-json.outputs.build_filtered }} + build_families_regex: ${{ steps.set-matrix-json.outputs.build_families_regex }} +``` + +- [ ] **Step 2: `build_util.yml` — example-map input** + +Add the input: + +```yaml + example-map: + required: false + default: '' + type: string +``` + +Insert between **Get Dependencies** and **Build**: + +```yaml + - name: Resolve PR example filter + if: inputs.example-map != '' && inputs.example-map != '{}' + env: + # values are PR-derived - keep them out of ${{ }} script interpolation + # (env expansion word-splits but never re-parses shell metacharacters) + EXAMPLE_MAP: ${{ inputs.example-map }} + FAMILY: ${{ matrix.arg }} + run: | + # -e flags for this family; a family absent from the map builds everything + EX_ARGS=$(printf '%s' "$EXAMPLE_MAP" | jq -r --arg fam "$FAMILY" '(.[$fam] // []) | map("-e " + .) | join(" ")') || EX_ARGS='' + echo "EX_ARGS=$EX_ARGS" + echo "EX_ARGS=$EX_ARGS" >> $GITHUB_ENV +``` + +Append `$EX_ARGS` to all three `tools/build.py` invocations (the esp-idf docker line, the generic Build line, and the Membrowse line — build.py maps `examples-membrowse-upload` per example when `-e` is active, because the aggregate target rebuilds everything). Extend the metrics upload: + +```yaml + path: | + cmake-build/cmake-build-*/metrics.json + cmake-build/cmake-build-*/metrics_by_example.json +``` + +- [ ] **Step 3: `cmake` job passes the map** + +In the `cmake` job's `with:` block add `example-map: ${{ needs.set-matrix.outputs.example_map }}`. Do **not** add it to `hil-build`/`hil-build-esp`/`build-os` — hil legs carry `-e` inside their matrix entries; build-os keeps the full example set. + +- [ ] **Step 4: `code-metrics` scoped baseline** + +Verify the download action supports regexp names: +`curl -fsSL https://raw.githubusercontent.com/dawidd6/action-download-artifact/v11/action.yml | grep -n name_is_regexp` — expect a hit. (Fallback if absent: replace the download step below with a `gh run download`-based loop over `build_families_regex` split on `|`, using `gh api` to find the newest master run per artifact; keep the same directory layout.) + +Change `needs: [ check-paths, cmake ]` → `needs: [ check-paths, cmake, set-matrix ]`. Guard the two unscoped steps with the filtered flag: on **Download Base Branch Metrics** change the `if:` to + +```yaml + if: (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch') && needs.set-matrix.outputs.build_filtered != 'true' +``` + +and on **Compare with Base Branch** change `if: github.event_name != 'push'` to + +```yaml + if: github.event_name != 'push' && needs.set-matrix.outputs.build_filtered != 'true' +``` + +Insert after **Download Base Branch Metrics**: + +```yaml + - name: Download base per-family metrics (scoped PR) + if: github.event_name == 'pull_request' && needs.set-matrix.outputs.build_filtered == 'true' + uses: dawidd6/action-download-artifact@v11 + with: + workflow: build.yml + workflow_conclusion: '' + search_artifacts: true # a docs-only master push uploads no per-family artifacts + branch: ${{ github.base_ref }} + name: ^metrics-(${{ needs.set-matrix.outputs.build_families_regex }})$ + name_is_regexp: true + path: base-family-metrics + continue-on-error: true + + - name: Compare with Base Branch (scoped) + if: github.event_name == 'pull_request' && needs.set-matrix.outputs.build_filtered == 'true' + run: | + # never fall back to the averaged metrics-tinyusb here: a scoped PR vs the + # 64-family/46-example average is exactly the mismatch this path prevents + python .github/scripts/metrics_pair_compare.py \ + --base-dir base-family-metrics --new-dir cmake-build --out metrics_compare + cat metrics_compare.md +``` + +(The PR-side `cmake-build/` dir already holds this run's `metrics_by_example.json` files from the artifact download at the top of the job.) + +- [ ] **Step 5: Validate and commit** + +```bash +python3 -c "import yaml,sys; yaml.safe_load(open('.github/workflows/build.yml')); yaml.safe_load(open('.github/workflows/build_util.yml')); print('yaml ok')" +command -v actionlint >/dev/null && actionlint .github/workflows/build.yml .github/workflows/build_util.yml || true +git add .github/workflows/build.yml .github/workflows/build_util.yml +git commit -m "ci: scope the GHA build matrix and code-metrics baseline by PR selection" +``` + +--- + +### Task 11: CircleCI wiring + +**Files:** +- Modify: `.circleci/config.yml`, `.circleci/config2.yml` + +**Interfaces:** +- `config.yml` set-matrix: on PRs, runs the selector (gated on its own unit suite), scopes `MATRIX_JSON` via `--select`, skips empty toolchains, and forwards `example-map` + `build-filtered` to the continued workflow as pipeline parameters. +- `config2.yml`: declares those parameters; the `build` command resolves `-e` flags per family; `code-metrics` compare is bypassed with a note when filtered; a `no-op` job keeps the workflow valid when nothing is selected. + +- [ ] **Step 1: Verify the continuation orb accepts parameters** + +`curl -fsSL "https://circleci.com/developer/orbs/orb/circleci/continuation" | grep -io 'parameters' | head -1` — the `continuation/continue` command takes a `parameters` input (inline JSON or a file path). If the page is unreachable, proceed — the orb has carried this input since 0.2; the fallback is `parameters: '{"example-map": ...}'` inline via an env-composed string. + +- [ ] **Step 2: `config.yml` — selector + scoping + parameters** + +In the `Set matrix` run command, replace the first two lines (`MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py)` and its echo) with: + +```bash + # PR-scoped selection (best-effort: any failure falls back to the full + # matrix). CircleCI has no base-branch var; tinyusb PRs target master. + SELECT_JSON='' + if [ -n "${CIRCLE_PULL_REQUEST:-}" ]; then + git fetch --no-tags origin master || true + if python3 test/hil/test/test_ci_select.py >/dev/null 2>&1; then + SELECT_JSON=$(python3 tools/ci_select.py --base origin/master) || SELECT_JSON='' + else + echo "ci_select unit suite failed - using the full matrix" + fi + fi + MATRIX_JSON='' + if [ -n "$SELECT_JSON" ]; then + MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py --select "$SELECT_JSON") || MATRIX_JSON='' + fi + [ -z "$MATRIX_JSON" ] && MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py) + echo "MATRIX_JSON=$MATRIX_JSON" + + EXAMPLE_MAP=$(printf '%s' "${SELECT_JSON:-null}" | jq -c '.build.family_examples // {}') || EXAMPLE_MAP='{}' + BUILD_FILTERED=$(printf '%s' "${SELECT_JSON:-null}" | jq -r 'if (.build? | type) == "object" and .build.full == false then "true" else "false" end') || BUILD_FILTERED='false' + jq -n --arg map "$EXAMPLE_MAP" --arg filt "$BUILD_FILTERED" \ + '{"example-map": $map, "build-filtered": $filt}' > /tmp/continue_params.json +``` + +In the toolchain loop, after `FAMILY=$(echo $MATRIX_JSON | jq -r ".\"$toolchain\"")` add: + +```bash + if [ "$(echo "$FAMILY" | jq 'length')" = "0" ]; then + # an empty matrix parameter is a hard CircleCI config error, not a skip + echo "skip build-${build_system}-${toolchain}: no families selected" + continue + fi +``` + +(the `continue` also keeps the alias out of `BUILD_ALIASES`, so `code-metrics` never requires a job that was not generated). Guard the code-metrics emission and keep the workflow non-empty: + +```bash + if [ ${#BUILD_ALIASES[@]} -gt 0 ]; then + echo " - code-metrics:" >> .circleci/config2.yml + echo " requires:" >> .circleci/config2.yml + for alias in "${BUILD_ALIASES[@]}"; do + echo " - $alias" >> .circleci/config2.yml + done + else + # a workflow with zero jobs is invalid config + echo " - no-op" >> .circleci/config2.yml + fi +``` + +(replacing the current unconditional code-metrics block). Change the continuation call to: + +```yaml + - continuation/continue: + configuration_path: .circleci/config2.yml + parameters: /tmp/continue_params.json +``` + +- [ ] **Step 3: `config2.yml` — parameters, `-e` resolution, scoped-compare note, no-op job** + +At the top, after `version: 2.1`: + +```yaml +parameters: + example-map: + type: string + default: "{}" + build-filtered: + type: string + default: "false" +``` + +In the `build` command's **Build** step, before the toolchain if/else, insert: + +```bash + # PR example filter for this family ('{}' or a missing key = build all). + # The parameter is a JSON string composed by set-matrix from ci_select. + EX_ARGS=$(printf '%s' '<< pipeline.parameters.example-map >>' | jq -r --arg fam "<< parameters.family >>" '(.[$fam] // []) | map("-e " + .) | join(" ")' 2>/dev/null) || EX_ARGS='' +``` + +and append `$EX_ARGS` to both `tools/build.py` invocations (docker esp-idf and the generic one). In `code-metrics`, wrap the existing compare `when:` condition with the filter guard and add the note branch: + +```yaml + - when: + condition: + and: + - not: + equal: [ master, << pipeline.git.branch >> ] + - equal: [ "false", << pipeline.parameters.build-filtered >> ] + steps: + # ... the existing Download Base Branch Metrics + Compare + store_artifacts steps, unchanged ... + - when: + condition: + and: + - not: + equal: [ master, << pipeline.git.branch >> ] + - equal: [ "true", << pipeline.parameters.build-filtered >> ] + steps: + - run: + name: Scoped build - comparison unavailable + command: | + # CircleCI stores only the averaged metrics.json; the per-example + # baseline lives on GHA. See the GHA code-metrics PR comment. + echo "_Code-size comparison skipped on CircleCI: this PR built a scoped example set._" > metrics_compare.md + - store_artifacts: + path: metrics_compare.md + destination: metrics_compare.md +``` + +Add the no-op job beside the other job definitions: + +```yaml + no-op: + docker: + - image: cimg/base:current + resource_class: small + steps: + - run: + name: No families selected + command: echo "PR selection - no families to build on CircleCI" +``` + +- [ ] **Step 4: Validate and commit** + +```bash +python3 -c "import yaml; yaml.safe_load(open('.circleci/config.yml')); yaml.safe_load(open('.circleci/config2.yml')); print('yaml ok')" +command -v circleci >/dev/null && circleci config validate .circleci/config.yml || true +git add .circleci/config.yml .circleci/config2.yml +git commit -m "ci: scope the CircleCI build matrix and example set by PR selection" +``` + +--- + +### Task 12: End-to-end validation, review, hand-off + +**Files:** none new — verification only (fix-ups amend the relevant earlier area). + +- [ ] **Step 1: Full hooks + suites** + +```bash +pre-commit run --all-files # ~55 s; HIL hooks exercise real timeouts deliberately +``` + +Expected: all hooks pass (`ci-select-test` and `hil-test` among them). + +- [ ] **Step 2: Selector scenario table** + +```bash +for f in src/portable/raspberrypi/rp2040/dcd_rp2040.c src/class/cdc/cdc_device.c \ + src/host/usbh.c examples/device/cdc_msc/src/main.c test/hil/hil_test.py \ + src/common/tusb_fifo.c hw/mcu/nordic/nrf5x/x.h; do + echo "== $f" + python3 tools/ci_select.py --diff-file <(echo "$f") test/hil/tinyusb.json 2>/dev/null | \ + python3 -c "import json,sys; s=json.load(sys.stdin); b=s['build']; print('hil_full:', s['full'], ' build_full:', b['full'], ' fams:', len(b['families']), ' mapped:', len(b['family_examples']))" +done +``` + +Expected (spot-check against the spec's measured table): rp2040 → 1 family; cdc_device → all families, mapped lists; usbh → ~25 families; example → all families, 1-example lists; test/hil → 0 families, hil_full true; common → build_full true; hw/mcu → 1 family (`nrf`). + +- [ ] **Step 3: Matrix + build smoke** + +```bash +SEL=$(python3 tools/ci_select.py --diff-file <(echo src/portable/raspberrypi/rp2040/dcd_rp2040.c) test/hil/tinyusb.json 2>/dev/null) +python3 .github/scripts/ci_set_matrix.py --select "$SEL" | python3 -m json.tool | head +python3 .github/scripts/hil_ci_set_matrix.py --select "$SEL" test/hil/tinyusb.json | python3 -m json.tool | head +python3 tools/build.py -e device/cdc_msc -b stm32f407disco --target all --target tinyusb_metrics +python3 -c "import json; d=json.load(open('cmake-build/cmake-build-stm32f407disco/metrics_by_example.json')); print(sorted(d))" +``` + +Expected: matrix shows only rp2040 under arm-gcc; hil matrix entries carry `-e ... -e device/board_test`; the by-example JSON lists exactly `['device/cdc_msc']`. + +- [ ] **Step 4: Full example set for one board** (repo validation rule after tool changes) + +```bash +cd examples && cmake -B cmake-build-stm32f407disco -DBOARD=stm32f407disco -G Ninja -DCMAKE_BUILD_TYPE=MinSizeRel . \ + && cmake --build cmake-build-stm32f407disco && cd .. +``` + +Expected: builds green (objcopy warnings non-critical per CLAUDE.md). + +- [ ] **Step 5: Local review, then stop** + +Run the `/code-review` skill on the branch diff (user policy: every push carrying local changes gets a local review pass first) and fix what holds up, amending into the appropriate task commits. Then **stop and hand back to the user** — pushing `build-filter` and opening the PR is their call; note for the PR description that the workflow changes only fully prove out on a real PR run (first PR after merge-to-branch should be watched with `gh pr checks --watch`, and the `hil-select` step's warnings checked for silent fallbacks). + +--- + +## Self-Review Notes + +- Spec coverage: rule table (Tasks 2-4), CMake-only scan (Task 2), orphan invariant (Task 2), build/hil_examples JSON contract (Task 4), `ci_set_matrix` flags (Task 5), `hil_ci_set_matrix -e` (Task 6), `build.py -e` incl. membrowse aggregate-dependency workaround (Task 7), metrics by-example + intersection compare + never-fall-back rule (Tasks 8-10), GHA side channel + injection-safe env passing (Task 10), CircleCI empty-toolchain/alias/no-op fixes + parameters (Task 11), move fallout table (Task 1). +- Known deviation from the spec text, both directions justified inline: `hil_examples` uses the *narrowed* chosen test list when a board is narrowed (the spec's JSON example implies this; its prose says `board_tests` — the narrowed form is a strict subset and matches what the rig runs, and re-run specs are subsets of it). +- Spec's measured "hcd_max3421.c → 1 leg" is really 1 *bsp* family (`espressif`) that neither provider's family list builds → 0 CI legs; Task 3's rule-4 test therefore asserts shape, not that specific count. diff --git a/docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md b/docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md new file mode 100644 index 000000000..8f77dc50a --- /dev/null +++ b/docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md @@ -0,0 +1,501 @@ +# PR-scoped CI selection: promoting hil_select to tools/ci_select.py + +**Date:** 2026-08-19 +**Branch:** `build-filter` + +## Motivation + +Every PR builds every example on one board per family, on both CI providers: **74 legs / +2494 example-builds** on the GitHub Actions `cmake` job, and 129 family-legs per build system +on CircleCI (which runs cmake *and* make, plus clang/IAR). Most PRs touch one port, one class, +or one example, and a `hid_host.c` change cannot break an MSC device example on msp430. + +`hil-build` is worse in a different way: it builds **1702 example-builds** (37 board-builds × +46 examples, `--target all`) to run a test suite that needs at most **515**. The HIL example +universe is only 21 of the 46 examples in tree, and the median board needs 15 of them. + +`test/hil/helper/hil_select.py` already maps a PR diff to affected boards and per-board test +lists for HIL, and already owns both mappings the build matrix needs: port-to-family, and +class-macro-to-example +(`docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md`). That design listed +"scoping the non-HIL build jobs" as an explicit non-goal; this is that follow-up. + +## Goal / non-goals + +**Goal:** promote the selector to a repo-wide `tools/ci_select.py` whose single classification +of a diff drives **all three** CI axes from one rule table — which families to build, which +example targets to build on each, and which rig boards run which tests — wired into +`ci_set_matrix.py` and `hil_ci_set_matrix.py` so both providers and the rig filter from one +source. Scoping applies to `pull_request` events only; push, release and `workflow_dispatch` +keep the full matrix. + +**Non-goals:** +- Variant-level or board-level selection below one-board-per-family on the build axis (all + variants of a selected HIL board still build and run). +- Changing `hil_test.py` behaviour. The selector only *composes* existing `-b` / `-bt` args. +- Changing which tests HIL decides to run. The HIL board/test decision is preserved except for + the single rule-7 change called out below. + +## The rule table + +One classification, three outputs. Every rule yields build families, build examples, and HIL +boards/tests. Pairs are unioned **per family** (build) and **per board** (HIL), so a mixed diff +never inflates one axis with another's breadth. + +`DEV` = 33 `examples/device/*`, `HOST` = 9, `DUAL` = 3, `TYPEC` = 1, `ALL` = 46. +`FAM` = the families whose `family.cmake` references the changed path (CMake only — see below). +"roster boards" = boards on `test/hil/{tinyusb,hfp}.json`. + +| # | Changed path | Build families | Build examples | HIL boards → tests | +| --- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------- | ----------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- | +| 1 | `docs/`, `.claude/`, `*.md`, `*.rst`, `LICENSE` | — | — | — | +| 2 | `test/hil/**` | — | — | all boards → all tests | +| 2b | `tools/metrics.py`, `.github/scripts/metrics_*.py` | `ALL` (unchanged — `tinyusb_metrics` runs `metrics.py` as a build target) | `ALL` | — (nothing on the rig runs it) | +| 3 | `src/portable/<port>/dcd_*`, `*_device.[ch]` | `FAM` | `DEV`+`DUAL` | `FAM`'s device-role boards → device+dual tests | +| 4 | `src/portable/<port>/hcd_*`, `*_host.[ch]` | `FAM` | `HOST`+`DUAL` | `FAM`'s host-role boards → host+dual tests | +| 5 | `src/portable/<port>/**` (anything else) | `FAM` | `ALL` | `FAM`'s boards → all their tests | +| 5b | `src/portable/<port>/**` where `FAM` is empty | — | — | — (empty resolves to nothing on BOTH axes) | +| 6 | `hw/bsp/<family>/**` | that family | `ALL` | that family's boards → all tests (a `boards/<board>/` path narrows to that board) | +| 7 | `hw/mcu/<vendor>/**` | `FAM` — empty resolves to nothing (maintainer ruling) | `ALL` | `FAM`'s boards → all tests; empty resolves to nothing (maintainer ruling) ⚠ *see below* | +| 8 | `src/class/<cls>/*_device.[ch]` | `ALL` | examples enabling `CFG_TUD_<CLS>` | device-role boards → HIL tests enabling `CFG_TUD_<CLS>` | +| 9 | `src/class/<cls>/*_host.[ch]` | `ALL` | examples enabling `CFG_TUH_<CLS>` | host-role boards → HIL tests enabling `CFG_TUH_<CLS>` | +| 10 | `src/class/<cls>/**` (shared header) | `ALL` | either, **plus include-edge classes** | both roles → same, plus include-edge classes | +| 11 | `src/device/**` | `ALL` | `DEV`+`DUAL` | device-role boards → device+dual tests | +| 12 | `src/host/**` | `ALL` | `HOST`+`DUAL` | host-role boards → host+dual tests | +| 13 | `examples/<role>/<name>/**` | `ALL` | just `<name>` | if `<name>` is a HIL test: all boards → that test; else nothing | +| 14 | `examples/device/board_test/**` | `ALL` | just `board_test` | all boards → all tests (HIL parking firmware) | +| 15 | `examples/build_system/**`, `examples/CMakeLists.txt`, `examples/<role>/CMakeLists.txt` | `ALL` | `ALL` | all boards → all tests | +| 16 | `src/common/`, `src/osal/`, `src/tusb.[ch]`, `src/tusb_option.h`, `tools/build*.py`, `tools/cmake/**`, `hw/bsp/{family_support.cmake,board.c,board_api.h,ansi_escape.h}`, `.github/**` | `ALL` | `ALL` | all boards → all tests | +| 16a | `lib/<name>/**` | `ALL` | examples whose own `CMakeLists.txt`/`Makefile` names `lib/<name>` | those examples that are HIL tests, on all boards; empty resolves to nothing | +| 16b | `tools/get_deps.py` | families whose `deps_mandatory`/`deps_optional` entries changed | `ALL` | those families' boards → all tests; a logic change, an `'all'` entry, no base content or a changed token naming no family → full | +| 17 | anything unclassified | `ALL` | `ALL` | all boards → all tests (fail-open) | + +**Rule 2 is deliberately asymmetric.** A `test/hil/**` change is invisible to the family matrix +but is exactly what the rig exercises, so it builds nothing and runs everything. + +**Rule 7 is the one HIL-side behaviour change in this design.** Today `hw/mcu/` sits in +`hil_select`'s `_FULL_RE` and forces the full HIL matrix. Since the build axis now resolves +those paths to a family through the same scan, forcing full on the rig is inconsistent. The +path fires rarely — 4 commits in 3 years — so this is low-risk either way; if you would rather +keep the HIL view untouched, rule 7's HIL column becomes "all boards → all tests" and nothing +else in this design changes. + +Rules 8–10 reuse machinery `hil_select` already has — `class_macros`, `_config_enables`, +`class_include_edges` — applied over all 46 examples' `src/tusb_config.h` for the build axis +and over the HIL test list for the HIL axis. The include edges are why an `audio.h` change also +selects the MIDI examples (`midi{,2}_{device,host}.h` include `class/audio/audio.h`) and a +`cdc.h` change the net one. + +### Buildability post-filter (build axis) + +After the pairs are unioned, every `(family, examples)` pair is pruned with +`build_utils.skip_example(example, <family's first board>)` — the same `skip.txt` / `only.txt` +data CMake's `family_filter` uses (40 `skip.txt`, 13 `only.txt` in tree). Examples the family +cannot build are dropped; a family left with none is dropped entirely. + +This is where most of the host-side saving comes from: only 23 of 75 CI families can build +`host/bare_api` at all, and 2 can build `typec/power_delivery`. + +### Measured effect + +GHA `cmake` job, baseline **74 legs / 2494 example-builds**; `hil-build`, baseline **1702 +example-builds** across 37 board-builds. + +| PR shape | Build legs | Build ex-builds | HIL boards | hil-build ex-builds | +| ---------------------------- | ---------: | --------------: | ---------: | ------------------: | +| `dcd_rp2040.c` | 1 | 35 | 2 | 32 | +| `hcd_max3421.c` | 1 | 10 | 7 | 36 | +| `hw/bsp/stm32f4/**` | 1 | 45 | 1 | 15 | +| `dcd_dwc2.c` | 20 | 646 | 10 | 184 | +| `hid_host.c` | 24 | 68 | — | — | +| `msc_host.c` | — | — | 9 | 40 | +| `usbh.c` | 25 | 217 | 10 | 57 | +| `msc_device.c` | 74 | 350 | — | — | +| `cdc_device.c` | 74 | 588 | 27 | 192 | +| `examples/device/cdc_msc/**` | 73 | 73 | 25 | 60 | +| `usbd.c` | 74 | 2297 | 27 | 472 | +| `src/common/**` (full) | 74 | 2494 | 30 | 515 | +| `test/hil/**` only | 0 | 0 | 30 | 515 | + +The full-matrix row is the headline for `hil-build`: even with **no** PR narrowing, per-board +example selection takes it from 1702 to 515. + +### Why "empty means empty" + +Both views answer an empty `FAM` the same way (rule 5b): nothing. The HIL view used to force +the full 30-board rig there, on the theory that an empty result might be a scan miss — but the +build view answered the identical condition with zero families for the same path, so the rig +ran every board to validate a file that nothing compiled. In the build view that theory costs +74 legs, and the +evidence does not support it: of the 28 `src/portable/*/*` directories, **26 resolve to at +least one family**. The two that do not are both real orphans as far as CI is concerned: +`microchip/pic` (only `dcd_pic.c` and a README, with no `hw/bsp/pic` family at all) and +`microchip/pic32mz` (`hw/bsp/pic32mz` has only a `family.mk`, and `pic32mz` is in neither +provider's family list, so no CI job builds it today). A file no CI job compiles cannot be +validated by building anything. + +The safety this gives up is recovered structurally: a unit test asserts every +`src/portable/*/*` and every tracked `hw/mcu/<vendor>` resolves to ≥1 family, with an explicit +allowlist of known orphans (`microchip/pic`, `microchip/pic32mz`). Adding a port without +wiring a family then fails +pre-commit instead of silently building nothing on every later PR. Same enforcement style as +the existing `test_hil_util.BottomLayer` structural tests. + +Fail-open survives where it belongs: an *unclassified* path or any exception widens to `ALL` on +every axis. + +### Why `hw/mcu/**` is rule 7 and not "full" + +`hw/mcu` is overwhelmingly dependency territory — `tools/get_deps.py` has 87 entries under it, +and those paths are gitignored, so they can never appear in a diff. Only 51 files survive +in-tree, touched 4 times in 3 years, and they resolve through the same scan the ports use: + +| Tracked directory | In `get_deps`? | Resolves to | +| ----------------------------- | ------------------------------------------- | ----------- | +| `hw/mcu/dialog/` (`da1469x`) | **no** — real in-repo MCU support, 21 files | `da1469x` | +| `hw/mcu/nordic/` (`nrf5x`) | beside the `nrfx` dep | `nrf` | +| `hw/mcu/sony/` (`cxd56`) | beside the `spresense-exported-sdk` dep | `cxd56` | +| `hw/mcu/bridgetek/` (`ft9xx`) | beside the `ft90x-sdk` dep | `ft9xx` | + +Rule 7 is therefore not a mechanism of its own — it is rules 3–5's scan pointed at a second +tree, because `src/portable/<port>` and `hw/mcu/<vendor>` ask the same question. + +Unlike the port rule, an `hw/mcu` path that resolves to no family contributes *nothing* on +either axis (maintainer ruling): if no family's build references it, no build compiles it. The +table above is kept honest by `test_tracked_mcu_vendors_resolve`, which fails pre-commit if a +tracked vendor directory stops resolving. + +### Why `lib/**` is rule 16a and scanned per example + +Its tracked contents (`SEGGER_RTT`, `networking`, `rt-thread`, `embedded-cli`, 22 commits in +3 years) are wired in at `examples/build_system` and per-example `CMakeLists.txt`, not per +family — so the family scan the ports use is the wrong instrument here: it would *wrongly* +narrow `SEGGER_RTT` to the three families that name the path in their `family.cmake`, while +the path is not compiled by any of them by default (it is reached only through `LOGGER=rtt`). +That scan stays applied to `src/portable/` and `hw/mcu/` only. + +Rule 16a asks the per-example question instead (maintainer ruling: only the examples that use +the lib need building): `lib_examples()` reads each example's own `CMakeLists.txt` and +`Makefile` and keeps the ones naming `lib/<name>` at a directory boundary. Every family stays +in play — any of them can build those examples — while the example list collapses: + +| Tracked lib | Examples that build it | HIL tests among them | +| -------------- | ----------------------------------------------------------- | -------------------- | +| `embedded-cli` | `host/msc_file_explorer`, `host/msc_file_explorer_freertos` | both | +| `networking` | `device/net_lwip_webserver` | none (test disabled) | +| `SEGGER_RTT` | — | — | +| `rt-thread` | — | — | + +`SEGGER_RTT` and `rt-thread` resolve to nothing, and "empty means empty" applies: no CI build +compiles them, so there is nothing to validate by building. + +### Why `tools/get_deps.py` is rule 16b + +`deps_mandatory` / `deps_optional` are data: `path -> [url, commit, 'fam1 fam2 ...']`. A commit +bump therefore affects exactly the families listed in that entry, and building the other 70+ is +pure waste. `get_deps_changed_families()` parses both sides of the file with `ast` (never +`exec` — this is PR content), diffs the two dict literals **separately**, and unions the family +tokens of every added, removed or edited entry, from **both** sides (a removed entry has only a +base side; an edited family list must cover the families that lose the dep as well as the ones +that gain it). Separately, because merging the dicts before diffing hides a *move* between +`deps_mandatory` and `deps_optional` — the value is untouched, but mandatory deps are fetched +for every family, so demoting one stops families fetching it. + +It falls open to the full matrix whenever the entries are not the whole answer: + +* anything outside the two dict assignments differs — a logic change to `get_deps` can change + what every family fetches (compared as `ast.dump(..., annotate_fields=False)` of the module + with those two assignments removed, so comments and reformatting alone are not a logic + change); +* an `'all'` entry (every mandatory dep) changed; +* the file will not parse; +* there is no base content: `--diff-file` mode has no git, so no merge-base blob; +* a changed entry carries a family token that names no `hw/bsp/<dir>` and is not one of the + known aliases. "Changed but unmappable" is not "nothing changed": reading it as the + latter empties the whole build matrix for a dep bump. + +The six known aliases (`sam3x`, `samd21`, `samd51`, `same5x`, `stm32l1`, `stm32l5`) are +pinned in `_DEPS_ALIAS_TOKENS` and select nothing. `get_deps` matches a token against a +requested family name verbatim (`f in entry[2].split()`), so these tokens match nothing +there either — four are pre-rename spellings listed beside the current name in the same +entry, and two name no family in the tree. (`fc100s` and `spresense` were on this list +until they were corrected in `get_deps.py`; those two were the only ones that left a +real dep unreachable for its own family.) A seventh appearing fails `TestOrphanInvariant`. + +## Component: `tools/ci_select.py` + +`git mv test/hil/helper/hil_select.py tools/ci_select.py` (history preserved). The HIL +classifier is unchanged apart from rule 7; a second, independent build classifier is added +beside it. One diff read, two classifiers, one unit suite. + +``` +python3 tools/ci_select.py --base <ref> [--diff-file <path>] [CONFIG.json ...] +``` + +`configs` becomes `nargs='*'`. With rosters it emits everything it emits today plus the new +keys; with none it emits only the build view, so CircleCI never needs to know HIL exists. + +```json +{ + "full": false, + "boards": {"raspberry_pi_pico": "all"}, + "families": ["rp2040"], + "args": {"tinyusb.json": "-b raspberry_pi_pico"}, + "args_flasher": {"tinyusb.json": {"openocd": "-b raspberry_pi_pico"}}, + "hil_examples": {"raspberry_pi_pico": ["device/cdc_msc", "device/board_test"]}, + "build": { + "full": false, + "families": ["rp2040"], + "family_examples": { + "rp2040": ["device/cdc_msc", "device/hid_composite", "dual/dynamic_switch"] + } + }, + "reasons": ["src/portable/raspberrypi/rp2040/dcd_rp2040.c: port rp2040 -> ..."] +} +``` + +`build.families` is the build family axis. `build.family_examples` maps a family to its example +list; **a family absent from the map builds all its examples**, so the common "narrow families, +all examples" case carries no payload. `build.full` true means no build narrowing at all. + +`hil_examples` is the new HIL build axis: per roster board, the examples `hil-build` must +produce. It is `board_tests(board)` — which the selector already computes — **plus +`device/board_test`**, which `hil_test.py` flashes to park every board at each variant boundary +and at end-of-board teardown (`hil_test.py:1798`, `:1866`). It is emitted even when +`full: true`, because the HIL example universe is 21 of 46 examples regardless of any diff. + +All pre-existing keys keep their exact meaning, so `.github/scripts/hil_ci_set_matrix.py`, the +HIL legs in `build.yml` and `.claude/skills/pre-pr/SKILL.md` need only a path update. + +### Shared helper change + +`port_families(port_dir, repo_root)` generalizes to a path-to-families reference scan over a +second tree (`hw/mcu/`). Its existing **CMake-only** behaviour is kept unchanged and is now the +rule for every axis: it scans `hw/bsp/*/family.cmake` plus the espressif component +`CMakeLists.txt`, and never `family.mk`. + +CMake is the first-class build system; Make follows whatever CMake decides. A family that +CMake does not wire up to a port is not a consumer of that port, and the Make legs on CircleCI +build the same families CMake does. Scanning `family.mk` as well would only ever *widen* the +selection to families CMake never builds, which is coverage nobody asked for — and it would +resolve `microchip/pic32mz` to a family that appears in no CI family list. + +One consequence to keep in view: because HIL and build now share one scan, there is no +per-caller flag, no second cache key, and no way for the two axes to disagree about which +families own a port. + +**Boundary matching.** A directory reference must match at a directory boundary — a trailing +`/` *or* end-of-token — not as a bare substring. Both traps are live: `hw/bsp/nrf/family.cmake` +writes `${TOP}/hw/mcu/nordic/nrf5x` with no trailing slash, while the existing port scan +requires a trailing `/` precisely to stop `microchip/pic` matching `microchip/pic32mz`. + +### Move fallout + +All mechanical, all one-line: + +| File | Change | +| ---------------------------------- | ----------------------------------------------------------------- | +| `tools/ci_select.py` | `sys.path` walk 4 levels → 2 | +| `test/hil/hil_ci.sh` | drop from the scp list (nothing on the rig imports it) | +| `test/hil/test/test_hil_select.py` | rename to `test_ci_select.py`, import path | +| `test/hil/test/test_hil_util.py` | `BottomLayer` stdlib-closure allowlist + module list | +| `.pre-commit-config.yaml` | both hooks (`hil-select-test` → `ci-select-test`, `files:` globs) | +| `.github/workflows/build.yml` | selector path, step name | +| `.claude/skills/pre-pr/SKILL.md` | selector path | + +The test file stays in `test/hil/test/` — it still consumes the rig rosters and `hil_util`. + +The selector gains one non-stdlib-but-local import: `tools/build_utils.skip_example` for the +buildability post-filter. `build_utils` imports only `subprocess`, `pathlib` and `re`, so the +stdlib closure the bare GitHub runner depends on is preserved; `BottomLayer` must be extended +to cover it. + +Because a wrong parents-count already broke this module once (there is a comment in the source +recording it), the moved module gets a guard test asserting its derived repo root contains +`src/` and `hw/bsp/`. + +## Component: `tools/build.py --example` + +`build.py` has no example filter today. `-T/--target` exists and maps to +`cmake --build --target <name>`, but it hard-fails on a target that does not exist, and absent +targets are routine (40 `skip.txt`, 13 `only.txt`). + +New repeatable `-e/--example <role>/<name>`: + +- Default (none given) keeps today's behaviour exactly: `--target all`. +- Given, each board's list is intersected with `build_utils.skip_example(example, board)`, then + passed as one `--target <name>` per example. Example target names are the directory names and + are unique across all four roles (verified: 46 examples, zero collisions). +- A board whose intersection is empty is reported **skipped**, not failed. +- `--target tinyusb_metrics` must stay last so metrics run after the examples that feed them. +- The espressif path already builds per example via `get_examples` + `skip_example`; it takes + the same filter. + +Both the family matrix and `hil-build` use this one flag. + +## CI wiring + +### `.github/scripts/ci_set_matrix.py` + +Two mutually exclusive optional flags. **Output shape is unchanged** — `{toolchain: [family]}`, +just fewer families. With no flags the output is byte-for-byte today's, so push, release and +`workflow_dispatch` are untouched. + +| Flag | Caller | Behaviour | +| --------------- | -------- | -------------------------------------------------------- | +| `--select JSON` | GHA | consumes the selector JSON the workflow already computes | +| `--base REF` | CircleCI | runs `tools/ci_select.py` itself | + +`build.full` true, or any exception, prints the full matrix with a warning on stderr. + +### `.github/scripts/hil_ci_set_matrix.py` + +Already takes `--select` and already scopes boards. It additionally appends `-e <example>` per +board from `hil_examples`, so each `hil-build` entry builds only what its board will run plus +`board_test`. When `hil_examples` is absent (hand-runs), it falls back to today's `--target all`. + +### The example map is a side channel, not a matrix entry + +The build example list deliberately does **not** ride inside the family matrix entry string. On +CircleCI the `family` parameter is also passed to `python tools/get_deps.py +<< parameters.family >>` and tested with `if [ << parameters.family >> == "rp2040" ]` — a value +carrying `-e` flags breaks both — and CircleCI matrix parameters form a cartesian product, so a +parallel `example-args` parameter would multiply the jobs rather than zip with them. + +So `build.family_examples` travels as one JSON blob and each build job resolves its own entry: + +- **GHA:** `set-matrix` exposes it as an output; `build_util.yml` gains an optional + `example-map` input (default `''`); a step resolves `-e` flags for `matrix.arg` with `jq`. +- **CircleCI:** `set-matrix` writes `example_map.json` and `persist_to_workspace`s it; the + `build` job gains `attach_workspace` and resolves the same way. + +Consequences of keeping the matrix shape: the metrics artifact name stays `metrics-<family>`, +and CircleCI's generated `config2.yml` does not inflate to one entry per family. `hil-build` +needs none of this — its matrix entries are already compound per-board strings from +`hil_ci_set_matrix.py`, so `-e` flags go straight in. + +### `.github/workflows/build.yml` + +The existing `HIL selection (PR only)` step in `set-matrix` is already gated on +`pull_request` — exactly the gate wanted. It is renamed, repointed at `tools/ci_select.py`, and +its `select` output is threaded into `ci_set_matrix.py --select`, so the filter costs zero extra +selector invocations. + +`build_util.yml`'s `if: inputs.build-args != '[]'` already skips a toolchain leg whose list is +empty, and a partially-skipped matrix aggregating to success is the documented pattern +`hil-build` already relies on. When every leg is empty (a `test/hil`-only PR), the `cmake` job +has nothing to build. Accepted: GitHub treats a skipped job as satisfying a required status +check, and HIL is unaffected because `hil-build` is a separate matrix. `code-metrics` still +runs (`!cancelled()` plus `cmake` success-or-skipped) and posts a "built no families on this +push" marker, so the sticky size comment never shows a stale table from an earlier push. + +### `.circleci/config.yml` + +The `set-matrix` job passes `--base origin/master` when `CIRCLE_PULL_REQUEST` is set, after +`git fetch --no-tags origin master || true`; unfiltered otherwise. CircleCI does not expose the +PR base branch, so `master` is assumed — true for essentially every tinyusb PR, and any ref or +clone problem falls back to the full matrix. + +Two fixes the GHA side does not need: + +- `gen_build_entry` must **skip** a toolchain whose family list is `[]`. An empty matrix + parameter is a hard CircleCI config error, not a skipped job. +- `BUILD_ALIASES` must collect only aliases that were actually generated, or `code-metrics`' + `requires:` names a job that does not exist. + +## Code metrics + +`tools/metrics.py` averages per-file sizes across every build, and the per-family +`metrics-<family>` artifact stores only that average — over whichever examples were built. Both +build axes therefore break the comparison: a 3-family PR against master's 64-family average, +and an 11-example average against master's 46-example one. + +The fix is to make the artifact carry per-example detail and compare the intersection. + +1. **`metrics.py combine --by-example`** additionally writes `metrics_by_example.json`, + `{example: {files: [...]}}`. The example name is the map.json's parent directory + (`<build>/<role>/<example>/*.map.json`). +2. `examples/CMakeLists.txt`'s `tinyusb_metrics` target emits both files; `build_util.yml` + uploads both under the existing `metrics-<family>` artifact name. +3. `combine` learns to expand a by-example JSON into one data entry per example and an + `--only-examples` filter, so a subset can be averaged on demand. +4. `code-metrics` computes the **intersection of `(family, example)` pairs present on both + sides**, averages each side over exactly those pairs, and compares. Dropped pairs are named + in the PR comment. An empty intersection skips the compare with an explicit note. + +`search_artifacts: true` is required on the base-side download: a docs-only master push +produces no per-family artifacts — which is why `metrics-carry-forward` exists for the +aggregate — so per-family baselines may come from different master runs. That is still a valid +per-family baseline. + +Today's `metrics-tinyusb` aggregate keeps being produced for the unfiltered path, releases and +`metrics-carry-forward`. The filtered path never falls back to it — that is precisely the +mismatched compare this section exists to prevent. `hil-build` uploads no metrics, so its +narrowing does not touch any of this. + +For narrow PRs this is sharper than today: a `dcd_rp2040` PR's size delta stops being diluted +by a 64-family, 46-example average. + +**Size check the plan must run first:** the by-example JSON is ~46× the entries of today's +average. If it proves too large as an artifact, drop per-symbol detail from the by-example file +(sizes only) — symbols are only needed in the aggregate. The plan must also verify that +`dawidd6/action-download-artifact@v11` supports `name_is_regexp`; the fallback is a +`gh run download` loop. + +## Testing + +Extended in `test/hil/test/test_ci_select.py` (stdlib-only, ~0.1 s, already a pre-commit hook +and already gating CI's selector step): + +- One case per rule 1–17, asserting all three outputs. +- Per-family union: a mixed diff (`dcd_rp2040.c` + `cdc_device.c`) gives `rp2040` the device + list and every other family the CDC list — not the cross product of both. +- Include edges: an `audio.h` change selects the MIDI examples; a `cdc.h` change the net one. +- `hil_examples` always contains `device/board_test` for every selected board, including when + `full: true`, and is otherwise exactly `board_tests(board)`. +- `hil_examples` never exceeds the 21-example HIL universe. +- The scan is CMake-only: a port referenced solely from a `family.mk` (`microchip/pic32mz`) + resolves to no family, and no `family.mk` is ever read. +- Boundary matching: `microchip/pic` does not inherit `microchip/pic32mz`'s families, and + `hw/mcu/nordic/nrf5x` resolves despite having no trailing slash at its reference site. +- Buildability post-filter: `typec/power_delivery` prunes to 2 families, `host/bare_api` to 23. +- Structural invariant: every `src/portable/*/*` and every tracked `hw/mcu/<vendor>` resolves + to ≥1 family, allowlist `{microchip/pic, microchip/pic32mz}`. +- Every name in `build.families` is a real `hw/bsp/<dir>`; every example name on either axis is + a real `examples/<role>/<name>` directory. +- Repo-root guard for the moved module. +- `ci_set_matrix.py`: no flags → byte-identical to today; `--select` with `build.full` → + identical; `--select` narrow → a subset; malformed `--select` → full plus a warning. +- `hil_ci_set_matrix.py`: no `hil_examples` → today's args byte-for-byte; with it → `-e` flags + appended per board, `board_test` always present. +- `build.py`: `-e` with an example the board skips builds nothing and reports skipped, not + failed; no `-e` still passes `--target all`. + +## Known gaps + +- **CircleCI size comparison.** CircleCI stores only the combined `metrics.json`, so the + intersection compare is unavailable there; when filtered it prints a note and copies + `metrics.md`. Its `metrics_compare.md` is a stored artifact that nothing reads in review — the + PR comment comes from GHA. Making CircleCI store per-example metrics is a follow-up. +- **HIL re-run attempts.** A re-run spec is a subset of the original selection, so the + firmware `hil-build` produced already covers it. This holds only while re-run specs stay + subsets; a future "re-run with extra tests" feature would need `hil-build` re-run too. +- **Membrowse** receives rows for fewer families and fewer examples on filtered PRs. If that + service misbehaves, the escape hatch is keeping the membrowse upload leg unfiltered. +- **`typec/power_delivery`** is reached only through rules 5 and 13 (`src/portable/st/typec` + has neither a `dcd_` nor an `hcd_` prefix, so it selects `ALL` examples on its 5 families, + which the post-filter then prunes to 2). A dedicated typec rule is possible later; the + post-filter already makes it cheap. +- **A `test/hil`-only PR reports `cmake` as skipped** rather than passing. Accepted; + revertible with a one-family floor if branch protection turns out to disagree. + `code-metrics` still runs in that case: with no `cmake-build/*/metrics.json` to + aggregate it writes `_Code-size comparison skipped: PR selection built no families + on this push._` and posts that as the sticky comment, so the size section reflects + THIS push instead of keeping the previous one's table. +- **`microchip/pic32mz` builds nothing.** The scan is CMake-only and `hw/bsp/pic32mz` ships + only a `family.mk`, so a change there selects no family. That matches reality — `pic32mz` is + in neither provider's family list — but it means the port is unbuilt by CI whether or not + this design lands. Giving it a `family.cmake` is the fix, and is out of scope here. +- **Seven bsp families are in no CI toolchain today** (`espressif`, `efm32`, `same7x`, + `cxd56`, `f1c100s`, `pic32mz`, `py32f0`); the intersection drops them, matching current + behaviour. This change does not alter that. diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 7669290a8..6122b7d54 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -21,7 +21,7 @@ endforeach () find_package(Python3 REQUIRED COMPONENTS Interpreter) add_custom_target(tinyusb_metrics COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/../tools/metrics.py - combine -f tinyusb/src -j -o ${CMAKE_BINARY_DIR}/metrics + combine -f tinyusb/src -j --by-example -o ${CMAKE_BINARY_DIR}/metrics ${MAPJSON_PATTERNS} COMMENT "Generating average code size metrics" VERBATIM diff --git a/examples/host/bare_api/only.txt b/examples/host/bare_api/only.txt index a2ff93be5..3287df65a 100644 --- a/examples/host/bare_api/only.txt +++ b/examples/host/bare_api/only.txt @@ -1,5 +1,5 @@ family:hpmicro -family:samd21 +family:samd2x_l2x family:samd5x_e5x mcu:CH32V20X mcu:KINETIS_KL diff --git a/examples/host/bare_api/skip.txt b/examples/host/bare_api/skip.txt index 308796869..dc3c2981d 100644 --- a/examples/host/bare_api/skip.txt +++ b/examples/host/bare_api/skip.txt @@ -1 +1,4 @@ board:lpcxpresso54114 +board:atsaml21_xpro +board:saml22_feather +board:sensorwatch_m0 diff --git a/examples/host/cdc_msc_hid/only.txt b/examples/host/cdc_msc_hid/only.txt index a2f4f273a..c4a23328e 100644 --- a/examples/host/cdc_msc_hid/only.txt +++ b/examples/host/cdc_msc_hid/only.txt @@ -1,5 +1,5 @@ family:hpmicro -family:samd21 +family:samd2x_l2x family:samd5x_e5x mcu:CH32V20X mcu:KINETIS_KL diff --git a/examples/host/cdc_msc_hid/skip.txt b/examples/host/cdc_msc_hid/skip.txt index 308796869..dc3c2981d 100644 --- a/examples/host/cdc_msc_hid/skip.txt +++ b/examples/host/cdc_msc_hid/skip.txt @@ -1 +1,4 @@ board:lpcxpresso54114 +board:atsaml21_xpro +board:saml22_feather +board:sensorwatch_m0 diff --git a/examples/host/cdc_msc_hid_freertos/only.txt b/examples/host/cdc_msc_hid_freertos/only.txt index 4ab8a906e..09b0125b0 100644 --- a/examples/host/cdc_msc_hid_freertos/only.txt +++ b/examples/host/cdc_msc_hid_freertos/only.txt @@ -1,5 +1,5 @@ family:espressif -family:samd21 +family:samd2x_l2x family:samd5x_e5x mcu:LPC175X_6X mcu:LPC177X_8X diff --git a/examples/host/cdc_msc_hid_freertos/skip.txt b/examples/host/cdc_msc_hid_freertos/skip.txt index f0be07d25..f5fe146c8 100644 --- a/examples/host/cdc_msc_hid_freertos/skip.txt +++ b/examples/host/cdc_msc_hid_freertos/skip.txt @@ -1,3 +1,6 @@ mcu:CH32F20X board:lpcxpresso54114 mcu:FT90X +board:atsaml21_xpro +board:saml22_feather +board:sensorwatch_m0 diff --git a/examples/host/device_info/only.txt b/examples/host/device_info/only.txt index 7f30218df..2c9c8834a 100644 --- a/examples/host/device_info/only.txt +++ b/examples/host/device_info/only.txt @@ -1,6 +1,6 @@ family:espressif family:hpmicro -family:samd21 +family:samd2x_l2x family:samd5x_e5x mcu:CH32V20X mcu:KINETIS_KL diff --git a/examples/host/device_info/skip.txt b/examples/host/device_info/skip.txt index 308796869..dc3c2981d 100644 --- a/examples/host/device_info/skip.txt +++ b/examples/host/device_info/skip.txt @@ -1 +1,4 @@ board:lpcxpresso54114 +board:atsaml21_xpro +board:saml22_feather +board:sensorwatch_m0 diff --git a/examples/host/hid_controller/only.txt b/examples/host/hid_controller/only.txt index 45ca6846f..7b4f14863 100644 --- a/examples/host/hid_controller/only.txt +++ b/examples/host/hid_controller/only.txt @@ -1,5 +1,5 @@ family:hpmicro -family:samd21 +family:samd2x_l2x family:samd5x_e5x mcu:CH32V20X mcu:KINETIS_KL diff --git a/examples/host/hid_controller/skip.txt b/examples/host/hid_controller/skip.txt index 308796869..dc3c2981d 100644 --- a/examples/host/hid_controller/skip.txt +++ b/examples/host/hid_controller/skip.txt @@ -1 +1,4 @@ board:lpcxpresso54114 +board:atsaml21_xpro +board:saml22_feather +board:sensorwatch_m0 diff --git a/examples/host/midi2_host/only.txt b/examples/host/midi2_host/only.txt index c71aacd87..8ffe1cd04 100644 --- a/examples/host/midi2_host/only.txt +++ b/examples/host/midi2_host/only.txt @@ -1,5 +1,5 @@ family:hpmicro -family:samd21 +family:samd2x_l2x family:samd5x_e5x mcu:CH32V20X mcu:ESP32P4 diff --git a/examples/host/midi2_host/skip.txt b/examples/host/midi2_host/skip.txt index 308796869..dc3c2981d 100644 --- a/examples/host/midi2_host/skip.txt +++ b/examples/host/midi2_host/skip.txt @@ -1 +1,4 @@ board:lpcxpresso54114 +board:atsaml21_xpro +board:saml22_feather +board:sensorwatch_m0 diff --git a/examples/host/midi_rx/only.txt b/examples/host/midi_rx/only.txt index 65ef8fac9..b467c6e3b 100644 --- a/examples/host/midi_rx/only.txt +++ b/examples/host/midi_rx/only.txt @@ -1,5 +1,5 @@ family:hpmicro -family:samd21 +family:samd2x_l2x family:samd5x_e5x mcu:CH32V20X mcu:ESP32P4 diff --git a/examples/host/midi_rx/skip.txt b/examples/host/midi_rx/skip.txt index 308796869..dc3c2981d 100644 --- a/examples/host/midi_rx/skip.txt +++ b/examples/host/midi_rx/skip.txt @@ -1 +1,4 @@ board:lpcxpresso54114 +board:atsaml21_xpro +board:saml22_feather +board:sensorwatch_m0 diff --git a/examples/host/msc_file_explorer/only.txt b/examples/host/msc_file_explorer/only.txt index a2ff93be5..3287df65a 100644 --- a/examples/host/msc_file_explorer/only.txt +++ b/examples/host/msc_file_explorer/only.txt @@ -1,5 +1,5 @@ family:hpmicro -family:samd21 +family:samd2x_l2x family:samd5x_e5x mcu:CH32V20X mcu:KINETIS_KL diff --git a/examples/host/msc_file_explorer/skip.txt b/examples/host/msc_file_explorer/skip.txt index 308796869..dc3c2981d 100644 --- a/examples/host/msc_file_explorer/skip.txt +++ b/examples/host/msc_file_explorer/skip.txt @@ -1 +1,4 @@ board:lpcxpresso54114 +board:atsaml21_xpro +board:saml22_feather +board:sensorwatch_m0 diff --git a/examples/host/msc_file_explorer_freertos/only.txt b/examples/host/msc_file_explorer_freertos/only.txt index 519ac2ebd..4f4f8fe6b 100644 --- a/examples/host/msc_file_explorer_freertos/only.txt +++ b/examples/host/msc_file_explorer_freertos/only.txt @@ -1,5 +1,5 @@ family:espressif -family:samd21 +family:samd2x_l2x family:samd5x_e5x mcu:LPC175X_6X mcu:LPC177X_8X diff --git a/examples/host/msc_file_explorer_freertos/skip.txt b/examples/host/msc_file_explorer_freertos/skip.txt index a8c9bea2a..06afcf825 100644 --- a/examples/host/msc_file_explorer_freertos/skip.txt +++ b/examples/host/msc_file_explorer_freertos/skip.txt @@ -2,3 +2,7 @@ mcu:CH32F20X board:lpcxpresso54114 mcu:FT90X board:stm32h7s3nucleo +board:atsaml21_xpro +board:saml22_feather +board:sensorwatch_m0 +board:curiosity_nano diff --git a/hw/bsp/mcx/family.cmake b/hw/bsp/mcx/family.cmake index b2b4fd45b..00c8c4ead 100644 --- a/hw/bsp/mcx/family.cmake +++ b/hw/bsp/mcx/family.cmake @@ -95,7 +95,7 @@ function(family_configure_example TARGET RTOS) endif() # PORT is set per board (board.cmake), so pick the driver at configure time. Spelled out - # rather than $<IF:${PORT},...> so the port path stays greppable: test/hil/helper/hil_select.py + # rather than $<IF:${PORT},...> so the port path stays greppable: tools/ci_select.py # maps a portable-driver change to the families whose build file names that directory. if (PORT) set(PORT_SRC ${TOP}/src/portable/chipidea/ci_hs/dcd_ci_hs.c) diff --git a/hw/bsp/rp2040/family.cmake b/hw/bsp/rp2040/family.cmake index 43b1dc234..57be416a2 100644 --- a/hw/bsp/rp2040/family.cmake +++ b/hw/bsp/rp2040/family.cmake @@ -126,7 +126,6 @@ target_sources(tinyusb_host_base INTERFACE ${TOP}/src/class/midi/midi_host.c ${TOP}/src/class/midi/midi2_host.c ${TOP}/src/class/msc/msc_host.c - ${TOP}/src/class/vendor/vendor_host.c ) # Sometimes have to do host specific actions in mostly common functions diff --git a/hw/bsp/samd2x_l2x/family.cmake b/hw/bsp/samd2x_l2x/family.cmake index 76371ccdc..2edcea0cd 100644 --- a/hw/bsp/samd2x_l2x/family.cmake +++ b/hw/bsp/samd2x_l2x/family.cmake @@ -106,7 +106,6 @@ function(family_configure_example TARGET RTOS) ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/family.c ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/../board.c ${TOP}/src/portable/microchip/samd/dcd_samd.c - ${TOP}/src/portable/microchip/samd/hcd_samd.c ${STARTUP_FILE_${CMAKE_C_COMPILER_ID}} ) # Add HCD support for SAMD21 (has host capability) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b3e05f60f..e113f2d88 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -31,7 +31,6 @@ function(tinyusb_sources_get OUTPUT_VAR) ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/class/midi/midi_host.c ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/class/midi/midi2_host.c ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/class/msc/msc_host.c - ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/class/vendor/vendor_host.c # typec ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/typec/usbc.c PARENT_SCOPE diff --git a/src/class/vendor/vendor_host.c b/src/class/vendor/vendor_host.c deleted file mode 100644 index dd2c5ac5d..000000000 --- a/src/class/vendor/vendor_host.c +++ /dev/null @@ -1,127 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2019 Ha Thach (tinyusb.org) - * SPDX-License-Identifier: MIT - * - * This file is part of the TinyUSB stack. - */ - -#include "tusb_option.h" - -#if (CFG_TUH_ENABLED && CFG_TUH_VENDOR) - -//--------------------------------------------------------------------+ -// INCLUDE -//--------------------------------------------------------------------+ -#include "host/usbh.h" -#include "vendor_host.h" - -//--------------------------------------------------------------------+ -// MACRO CONSTANT TYPEDEF -//--------------------------------------------------------------------+ - -//--------------------------------------------------------------------+ -// INTERNAL OBJECT & FUNCTION DECLARATION -//--------------------------------------------------------------------+ -custom_interface_info_t custom_interface[CFG_TUH_DEVICE_MAX]; - -static tusb_error_t cush_validate_paras(uint8_t dev_addr, uint16_t vendor_id, uint16_t product_id, void * p_buffer, uint16_t length) -{ - if ( !tusbh_custom_is_mounted(dev_addr, vendor_id, product_id) ) - { - return TUSB_ERROR_DEVICE_NOT_READY; - } - - TU_ASSERT( p_buffer != NULL && length != 0, TUSB_ERROR_INVALID_PARA); - - return TUSB_ERROR_NONE; -} -//--------------------------------------------------------------------+ -// APPLICATION API (need to check parameters) -//--------------------------------------------------------------------+ -tusb_error_t tusbh_custom_read(uint8_t dev_addr, uint16_t vendor_id, uint16_t product_id, void * p_buffer, uint16_t length) -{ - TU_ASSERT_ERR( cush_validate_paras(dev_addr, vendor_id, product_id, p_buffer, length) ); - - if ( !hcd_pipe_is_idle(custom_interface[dev_addr-1].pipe_in) ) - { - return TUSB_ERROR_INTERFACE_IS_BUSY; - } - - (void) usbh_edpt_xfer( custom_interface[dev_addr-1].pipe_in, p_buffer, length); - - return TUSB_ERROR_NONE; -} - -tusb_error_t tusbh_custom_write(uint8_t dev_addr, uint16_t vendor_id, uint16_t product_id, void const * p_data, uint16_t length) -{ - TU_ASSERT_ERR( cush_validate_paras(dev_addr, vendor_id, product_id, p_data, length) ); - - if ( !hcd_pipe_is_idle(custom_interface[dev_addr-1].pipe_out) ) - { - return TUSB_ERROR_INTERFACE_IS_BUSY; - } - - (void) usbh_edpt_xfer( custom_interface[dev_addr-1].pipe_out, p_data, length); - - return TUSB_ERROR_NONE; -} - -//--------------------------------------------------------------------+ -// USBH-CLASS API -//--------------------------------------------------------------------+ -void cush_init(void) -{ - tu_memclr(&custom_interface, sizeof(custom_interface_info_t) * CFG_TUH_DEVICE_MAX); -} - -tusb_error_t cush_open_subtask(uint8_t dev_addr, tusb_desc_interface_t const *p_interface_desc, uint16_t *p_length) -{ - // FIXME quick hack to test lpc1k custom class with 2 bulk endpoints - uint8_t const *p_desc = (uint8_t const *) p_interface_desc; - p_desc = tu_desc_next(p_desc); - - //------------- Bulk Endpoints Descriptor -------------// - for(uint32_t i=0; i<2; i++) - { - tusb_desc_endpoint_t const *p_endpoint = (tusb_desc_endpoint_t const *) p_desc; - TU_ASSERT(TUSB_DESC_ENDPOINT == p_endpoint->bDescriptorType, TUSB_ERROR_INVALID_PARA); - - pipe_handle_t * p_pipe_hdl = ( p_endpoint->bEndpointAddress & TUSB_DIR_IN_MASK ) ? - &custom_interface[dev_addr-1].pipe_in : &custom_interface[dev_addr-1].pipe_out; - *p_pipe_hdl = usbh_edpt_open(dev_addr, p_endpoint, TUSB_CLASS_VENDOR_SPECIFIC); - TU_ASSERT ( pipehandle_is_valid(*p_pipe_hdl), TUSB_ERROR_HCD_OPEN_PIPE_FAILED ); - - p_desc = tu_desc_next(p_desc); - } - - (*p_length) = sizeof(tusb_desc_interface_t) + 2*sizeof(tusb_desc_endpoint_t); - return TUSB_ERROR_NONE; -} - -void cush_isr(pipe_handle_t pipe_hdl, xfer_result_t event) -{ - -} - -void cush_close(uint8_t dev_addr) -{ - tusb_error_t err1, err2; - custom_interface_info_t * p_interface = &custom_interface[dev_addr-1]; - - // TODO re-consider to check pipe valid before calling pipe_close - if( pipehandle_is_valid( p_interface->pipe_in ) ) - { - err1 = hcd_pipe_close( p_interface->pipe_in ); - } - - if ( pipehandle_is_valid( p_interface->pipe_out ) ) - { - err2 = hcd_pipe_close( p_interface->pipe_out ); - } - - tu_memclr(p_interface, sizeof(custom_interface_info_t)); - - TU_ASSERT(err1 == TUSB_ERROR_NONE && err2 == TUSB_ERROR_NONE, (void) 0 ); -} - -#endif diff --git a/src/class/vendor/vendor_host.h b/src/class/vendor/vendor_host.h deleted file mode 100644 index dc55663b9..000000000 --- a/src/class/vendor/vendor_host.h +++ /dev/null @@ -1,48 +0,0 @@ -/* - * SPDX-FileCopyrightText: Copyright (c) 2019 Ha Thach (tinyusb.org) - * SPDX-License-Identifier: MIT - * - * This file is part of the TinyUSB stack. - */ - -#ifndef TUSB_VENDOR_HOST_H_ -#define TUSB_VENDOR_HOST_H_ - -#include "common/tusb_common.h" - -#ifdef __cplusplus - extern "C" { -#endif - -typedef struct { - pipe_handle_t pipe_in; - pipe_handle_t pipe_out; -}custom_interface_info_t; - -//--------------------------------------------------------------------+ -// USBH-CLASS DRIVER API -//--------------------------------------------------------------------+ -static inline bool tusbh_custom_is_mounted(uint8_t dev_addr, uint16_t vendor_id, uint16_t product_id) -{ - (void) vendor_id; // TODO check this later - (void) product_id; -// return (tusbh_device_get_mounted_class_flag(dev_addr) & TU_BIT(TUSB_CLASS_MAPPED_INDEX_END-1) ) != 0; - return false; -} - -bool tusbh_custom_read(uint8_t dev_addr, uint16_t vendor_id, uint16_t product_id, void * p_buffer, uint16_t length); -bool tusbh_custom_write(uint8_t dev_addr, uint16_t vendor_id, uint16_t product_id, void const * p_data, uint16_t length); - -//--------------------------------------------------------------------+ -// Internal Class Driver API -//--------------------------------------------------------------------+ -void cush_init(void); -bool cush_open_subtask(uint8_t dev_addr, tusb_desc_interface_t const *p_interface_desc, uint16_t *p_length); -void cush_isr(pipe_handle_t pipe_hdl, xfer_result_t event); -void cush_close(uint8_t dev_addr); - -#ifdef __cplusplus - } -#endif - -#endif /* TUSB_VENDOR_HOST_H_ */ diff --git a/src/host/usbh.c b/src/host/usbh.c index e307bb5e5..44819b016 100644 --- a/src/host/usbh.c +++ b/src/host/usbh.c @@ -306,17 +306,6 @@ static usbh_class_driver_t const usbh_class_drivers[] = { }, #endif - #if CFG_TUH_VENDOR - { - .name = DRIVER_NAME("VENDOR"), - .init = cush_init, - .deinit = cush_deinit, - .open = cush_open, - .set_config = cush_set_config, - .xfer_cb = cush_isr, - .close = cush_close - } - #endif }; // Additional class drivers implemented by application diff --git a/src/tinyusb.mk b/src/tinyusb.mk index 365043927..941791670 100644 --- a/src/tinyusb.mk +++ b/src/tinyusb.mk @@ -26,4 +26,3 @@ TINYUSB_SRC_C += \ src/class/midi/midi_host.c \ src/class/midi/midi2_host.c \ src/class/msc/msc_host.c \ - src/class/vendor/vendor_host.c \ diff --git a/src/tusb.h b/src/tusb.h index 6a30f7c13..cdf6f8171 100644 --- a/src/tusb.h +++ b/src/tusb.h @@ -48,9 +48,6 @@ #include "class/midi/midi2_host.h" #endif - #if CFG_TUH_VENDOR - #include "class/vendor/vendor_host.h" - #endif #else #ifndef tuh_int_handler #define tuh_int_handler(...) diff --git a/src/tusb_option.h b/src/tusb_option.h index 24f802b73..1eb23fb00 100644 --- a/src/tusb_option.h +++ b/src/tusb_option.h @@ -894,9 +894,6 @@ #define CFG_TUH_MSC 0 #endif -#ifndef CFG_TUH_VENDOR - #define CFG_TUH_VENDOR 0 -#endif #ifndef CFG_TUH_API_EDPT_XFER #define CFG_TUH_API_EDPT_XFER 0 diff --git a/test/hil/helper/hil_pool_check.py b/test/hil/helper/hil_pool_check.py index d926bbe3d..179a417ed 100644 --- a/test/hil/helper/hil_pool_check.py +++ b/test/hil/helper/hil_pool_check.py @@ -433,7 +433,7 @@ def build_example(board: dict, variant: str, example: str) -> int: cmd = ['idf.py', '-C', f'examples/{example}', '-B', f'cmake-build/cmake-build-{vcfg["name"]}/{example}', '-G', 'Ninja', f'-DBOARD={name}', 'build'] - for d in board.get('build', {}).get('args', []) + vcfg.get('defines', []): + for d in vcfg.get('defines', []): cmd.insert(-1, f'-D{d}') if vcfg.get('flags'): cmd.insert(-1, f'-DCFLAGS_CLI={vcfg["flags"]}') @@ -446,8 +446,6 @@ def build_example(board: dict, variant: str, example: str) -> int: cmd = [sys.executable, str(hil_util.TINYUSB_ROOT / 'tools' / 'build.py'), '-b', name, '-T', Path(example).name, '-j', str(max(1, (os.cpu_count() or _jobs) // _jobs))] - for d in board.get('build', {}).get('args', []): - cmd += ['-D', d] if vcfg['name'] != name: cmd += ['--build-name', vcfg['name']] for d in vcfg.get('defines', []): diff --git a/test/hil/helper/hil_select.py b/test/hil/helper/hil_select.py deleted file mode 100755 index f0d4f0b9f..000000000 --- a/test/hil/helper/hil_select.py +++ /dev/null @@ -1,524 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MIT -"""PR-diff -> HIL selection: which rig boards and which tests a change can affect. - -Stdlib-only (runs on bare CI runners; imports hil_util for the example rosters, -never hil_test/pyserial — test_hil_util.BottomLayer enforces the stdlib closure). -Fail-open: any file no rule classifies forces the full matrix. See -docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md. - -JSON: full, boards (name -> 'all' | [tests]), families (bsp families the diff -touches, including ones with no rig board - build-only consumers such as /pre-pr -sample from these), args (hil_test.py args per config) and args_flasher (the same -args split by each board's flasher, for CI legs that split one rig by flasher). -""" -import argparse -import functools -import glob -import json -import os -import re -import subprocess -import sys - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # helper/ scripts import via the test/hil root -from helper.hil_util import device_tests, dual_tests, host_test - -ALL_TESTS = {'device': device_tests, 'dual': dual_tests, 'host': host_test} - -# class dir -> config macro suffix exceptions (rule 3); dfu is per-file, handled inline -NET_MACROS = ('ECM_RNDIS', 'NCM') - -_NONCODE_RE = re.compile( - r'^(docs/|\.claude/|.*\.(md|rst)$|LICENSE)') -_FULL_RE = re.compile( - r'^(src/common/|src/osal/|src/tusb\.c$|src/tusb\.h$|src/tusb_option\.h$|' - r'test/hil/|\.github/workflows/build.*\.yml$|\.github/actions/|\.github/scripts/|' - r'tools/build\.py$|tools/get_deps\.py$|tools/cmake/|hw/mcu/|lib/|' - r'hw/bsp/(family_support\.cmake|board_api\.h|board\.c|ansi_escape\.h)$|' - r'examples/build_system/|examples/CMakeLists\.txt$|' - # board_test is HIL infrastructure, not a test: hil_test.py flashes it to park - # every board (variant boundary + end-of-board teardown), so every board depends on it - r'examples/device/board_test/)') - -# --no-renames: with rename detection git reports only a rename's destination, so code -# moved out of an HIL-relevant path would be classified by its new path alone -GIT_DIFF_ARGV = ['git', 'diff', '--no-renames', '--name-only'] - - -def test_role(test: str) -> str: - return test.split('/', 1)[0] # 'device' | 'dual' | 'host' - - -def board_roles(board: dict) -> set: - t = board.get('tests', {}) - roles = set() - if t.get('device'): - roles.add('device') - if t.get('host'): - roles.add('host') - if t.get('dual'): - roles.update(('device', 'host')) - for only in t.get('only', []): - r = test_role(only) - roles.update(('device', 'host') if r == 'dual' else (r,)) - return roles - - -def board_tests(board: dict) -> list: - """Every test this board would run today (mirrors hil_test.test_board's default).""" - t = board.get('tests', {}) - if 'only' in t: - run = list(t['only']) - else: - run = [] - if t.get('device'): - run += device_tests - if t.get('dual'): - run += dual_tests - if t.get('host'): - run += host_test - return [x for x in run if x not in t.get('skip', [])] - - -# cached: called per changed file x roster board, and the tree doesn't change mid-run [email protected]_cache(maxsize=None) -def board_family(board_name: str, repo_root: str): - hits = glob.glob(os.path.join(repo_root, 'hw/bsp/*/boards', board_name)) - return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None - - -# `if (OPTION STREQUAL "1")` guards in family_support.cmake, and the option tokens -# a roster entry passes to the build (NAME=VALUE / -DNAME=VALUE) -_CM_IF_RE = re.compile(r'if\s*\(') -_CM_ELSE_RE = re.compile(r'else(if)?\s*\(') -_CM_ENDIF_RE = re.compile(r'endif\s*\(') -_CM_OPT_RE = re.compile(r'if\s*\(\s*\$?\{?([A-Za-z_]\w*)\}?\s+STREQUAL\s+"?1"?\s*\)') -_CM_PORT_RE = re.compile(r'src/portable/((?:[^/\s]+/)?[^/\s]+)/') -_FALSY = ('', '0', 'off', 'false', 'no') - - [email protected]_cache(maxsize=None) -def port_option_gates(repo_root: str) -> dict: - """port dir -> build options that compile it regardless of the board's family - file, e.g. {'analog/max3421': {'MAX3421_HOST'}} from family_support.cmake.""" - gates = {} - try: - text = open(os.path.join(repo_root, 'hw/bsp/family_support.cmake')).read() - except OSError: - return gates - stack = [] # one entry per open if(): its option, or None - for line in text.splitlines(): - line = line.strip() - if _CM_IF_RE.match(line): - m = _CM_OPT_RE.match(line) - stack.append(m.group(1) if m else None) - elif _CM_ELSE_RE.match(line): - if stack: - stack[-1] = None # the guard doesn't hold in this branch - elif _CM_ENDIF_RE.match(line): - if stack: - stack.pop() - opts = {o for o in stack if o} - m = _CM_PORT_RE.search(line) - if opts and m: - gates.setdefault(m.group(1), set()).update(opts) - return gates - - -_CM_SET_RE = re.compile(r'set\s*\(\s*([A-Za-z_]\w*)\s+([^)\s]+)\s*\)') - - -# cached: called per changed portable file x roster board [email protected]_cache(maxsize=None) -def bsp_board_options(board_name: str, repo_root: str) -> frozenset: - """Build options a board turns on in its own BSP: `set(<OPT> <value>)` in - hw/bsp/<family>/boards/<board>/board.cmake, e.g. MAX3421_HOST on the espressif - and rp2040 max3421 boards. CMake only - HIL CI builds nothing with Make, so a - board.mk-only option (e.g. nrf5340dk's MAX3421_HOST) compiles no port here.""" - fam = board_family(board_name, repo_root) - if not fam: - return frozenset() - path = os.path.join(repo_root, 'hw/bsp', fam, 'boards', board_name, 'board.cmake') - try: - text = open(path).read() - except OSError: - return frozenset() - out = set() - for line in text.splitlines(): - line = line.strip() - if line.startswith('#'): - continue - m = _CM_SET_RE.match(line) - if m and m.group(2).strip('"').lower() not in _FALSY: - out.add(m.group(1)) - return frozenset(out) - - -def board_options(board: dict, repo_root: str) -> set: - """Build options a board has truthy: the roster entry's build.args plus each - variant's defines (NAME=VALUE) and raw CFLAGS (-DNAME=VALUE), plus whatever its - own board.cmake sets (a board can enable a gated port without the roster saying so).""" - toks = list(board.get('build', {}).get('args', [])) - for v in board.get('variant', []): - toks += list(v.get('defines', [])) - toks += v.get('flags', '').split() - out = set(bsp_board_options(board['name'], repo_root)) - for t in toks: - name, _, val = (t[2:] if t.startswith('-D') else t).partition('=') - if name and val.strip().strip('"').lower() not in _FALSY: - out.add(name.strip()) - return out - - [email protected]_cache(maxsize=None) -def port_families(port_dir: str, repo_root: str) -> set: - """Board families that compile this src/portable dir. CMake only: HIL CI builds - every board with CMake, so a port wired up in family.mk alone is compiled for no - HIL board and must not select one. family.cmake lists portable sources directly - for most families; espressif instead references them from a nested component - CMakeLists.txt (hw/bsp/espressif/components/tinyusb_src/CMakeLists.txt).""" - fams = set() - bsp_root = os.path.join(repo_root, 'hw/bsp') - # trailing '/' so a port dir is not a prefix of a sibling: bare 'microchip/pic' - # would otherwise match '.../microchip/pic32mz/...' and inherit its families - needle = port_dir + '/' - for f in glob.glob(os.path.join(bsp_root, '*/family.cmake')) + \ - glob.glob(os.path.join(bsp_root, '*/components/*/CMakeLists.txt')): - try: - if needle in open(f).read(): - fam = os.path.relpath(f, bsp_root).split(os.sep, 1)[0] - fams.add(fam) - except OSError: - pass - return fams - - -_CLS_INC_RE = re.compile(r'#\s*include\s*[<"]class/([^/"<>]+)/([^"<>]+)[">]') - - [email protected]_cache(maxsize=None) -def class_include_edges(repo_root: str) -> dict: - """'<class>/<header>' -> the other class dirs that include it. A class header - pulled in by a second class ships in every firmware enabling that second class: - src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, and - net_device.h includes class/cdc/cdc.h. The class rule derives macros from the - directory name alone, so without this edge a change to the included header - selects only its own class's examples - and on a board that skips those (e.g. - metro_m4_express skips audio_test_freertos), nothing at all. - - Derived from the actual #include lines rather than a hand-written table so it - cannot rot when a class picks up or drops a cross-class include.""" - edges = {} - for f in sorted(glob.glob(os.path.join(repo_root, 'src/class/*/*.[ch]'))): - cls = os.path.basename(os.path.dirname(f)) - try: - text = open(f).read() - except OSError: - continue - for inc_cls, inc_hdr in _CLS_INC_RE.findall(text): - if inc_cls != cls: - edges.setdefault(f'{inc_cls}/{inc_hdr}', set()).add(cls) - return edges - - -def class_macros(cls: str, base: str, prefix: str) -> list: - """Config macros that compile a class dir's code, for role prefix TUD/TUH. - `base` refines dfu only (it splits DFU from DFU_RUNTIME per file); pass '' for - a class reached through an include edge, where the widest set is correct.""" - if cls == 'net': - return [f'CFG_{prefix}_{m}' for m in NET_MACROS] - if cls == 'dfu': - if base.startswith('dfu_rt'): - return [f'CFG_{prefix}_DFU_RUNTIME'] - if base.startswith('dfu_device') or base.startswith('dfu_host'): - return [f'CFG_{prefix}_DFU'] - return [f'CFG_{prefix}_DFU', f'CFG_{prefix}_DFU_RUNTIME'] - return [f'CFG_{prefix}_{cls.upper()}'] - - -def _config_enables(cfg_path: str, macros) -> bool: - try: - text = open(cfg_path).read() - except OSError: - return False - return any(re.search(rf'#define\s+{m}\s+\(?\s*0*[1-9]', text) for m in macros) - - -def roster_only_tests(all_boards) -> set: - """Test paths that only appear in a roster board's tests.only list (e.g. - espressif boards), not in the shared device/dual/host_test lists.""" - out = set() - for b in all_boards: - out.update(b.get('tests', {}).get('only', [])) - return out - - -def class_examples(macros, role: str, repo_root: str, extra_tests: set) -> set: - """Tests (from role's + dual lists, plus roster-only-list tests of that role) - whose example config enables any macro.""" - pool = role_tests({role}, extra_tests) - out = set() - for test in pool: - cfg = os.path.join(repo_root, 'examples', test, 'src', 'tusb_config.h') - if _config_enables(cfg, macros): - out.add(test) - return out - - -def role_tests(roles: set, extras: set) -> set: - """Every test for the given role(s): each role's own list + dual tests, - plus roster-only-list tests (extras) matching those roles or 'dual'.""" - pool = set(dual_tests) - for r in roles: - pool |= set(ALL_TESTS[r]) - pool |= {t for t in extras if test_role(t) in roles or test_role(t) == 'dual'} - return pool - - -class _Sel: - """Accumulates contributions. board->set(tests) plus 'all-board' markers.""" - def __init__(self): - self.full = False - self.by_board = {} # name -> set of tests, or 'all' - self.roles = set() # roles touched by any contribution - self.families = set() # bsp families touched (incl. off-rig ones: build-only consumers) - self.reasons = [] - - def add(self, boards, tests, reason): - """tests: 'all' or iterable of test paths.""" - self.reasons.append(reason) - for b in boards: - cur = self.by_board.get(b) - if tests == 'all' or cur == 'all': - self.by_board[b] = 'all' - else: - self.by_board[b] = (cur or set()) | set(tests) - - def force_full(self, reason): - self.full = True - self.reasons.append(reason) - - -def _classify_one(path, repo_root, roster_boards, extras: set, s: _Sel): - base = os.path.basename(path) - if _NONCODE_RE.match(path): - s.reasons.append(f'{path}: non-code, no contribution') - return - if _FULL_RE.match(path): - s.force_full(f'{path}: core/infra -> full matrix') - return - - m = re.match(r'src/portable/((?:[^/]+/)?[^/]+)/', path) - if m: - port = m.group(1) - if re.match(r'(dcd_|.*_device)', base): - roles = {'device'} - elif re.match(r'(hcd_|.*_host)', base): - roles = {'host'} - else: - roles = {'device', 'host'} - fams = port_families(port, repo_root) - if not fams: - # no family references this port: either a new/renamed port dir or a - # family.cmake layout the scan misses - widen instead of contributing nothing - s.force_full(f'{path}: port {port} maps to no board family -> full matrix') - return - s.families.update(fams) - # a board can also pull the port in through a build option (e.g. MAX3421_HOST=1 - # from the roster on metro_m4_express, or from its own board.cmake), which its - # family file never names - gates = port_option_gates(repo_root).get(port, set()) - boards = [b['name'] for b in roster_boards - if (board_family(b['name'], repo_root) in fams or - (gates and board_options(b, repo_root) & gates)) and (board_roles(b) & roles)] - tests = role_tests(roles, extras) - s.roles.update(roles) - why = f'{path}: port {port} -> families {sorted(fams)}' - if gates: - why += f' + option {sorted(gates)}' - s.add(boards, tests, f'{why} -> boards {boards} ({"/".join(sorted(roles))})') - return - - m = re.match(r'src/class/([^/]+)/', path) - if m: - cls = m.group(1) - if re.search(r'_device\.[ch]$', base): - roles = {'device'} - elif re.search(r'_host\.[ch]$', base): - roles = {'host'} - else: - roles = {'device', 'host'} - # this file's own class, plus any class whose headers include it - via = sorted(class_include_edges(repo_root).get(f'{cls}/{base}', ())) - - def macros(prefix): - return (class_macros(cls, base, prefix) + - [m2 for c in via for m2 in class_macros(c, '', prefix)]) - tests = set() - if 'device' in roles: - tests |= class_examples(macros('TUD'), 'device', repo_root, extras) - if 'host' in roles: - tests |= class_examples(macros('TUH'), 'host', repo_root, extras) - boards = [b['name'] for b in roster_boards if board_roles(b) & roles] - s.roles.update(roles) - why = f'{path}: class {cls}' + (f' (+ included by {via})' if via else '') - s.add(boards, tests, f'{why} -> {sorted(tests)} ({"/".join(sorted(roles))})') - return - - m = re.match(r'src/(device|host)/', path) - if m: - role = m.group(1) - boards = [b['name'] for b in roster_boards if role in board_roles(b)] - s.roles.add(role) - s.add(boards, role_tests({role}, extras), f'{path}: core {role} stack -> all {role} tests') - return - - m = re.match(r'hw/bsp/([^/]+)/(?:boards/([^/]+)/)?', path) - if m: - fam, brd = m.group(1), m.group(2) - s.families.add(fam) - if brd: - boards = [b['name'] for b in roster_boards if b['name'] == brd] - why = f'{path}: bsp board {brd}' - else: - boards = [b['name'] for b in roster_boards - if board_family(b['name'], repo_root) == fam] - why = f'{path}: bsp family {fam}' - s.roles.update(('device', 'host')) - s.add(boards, 'all', f'{why} -> boards {boards}') - return - - m = re.match(r'examples/(device|host|dual)/([^/]+)/', path) - if m: - test = f'{m.group(1)}/{m.group(2)}' - known = any(test in pool for pool in ALL_TESTS.values()) or test in extras - if known: - boards = [b['name'] for b in roster_boards] - role = test_role(test) - s.roles.update(('device', 'host') if role == 'dual' else (role,)) - s.add(boards, [test], f'{path}: example -> {test} on all boards') - else: - s.reasons.append(f'{path}: example not in HIL lists, no contribution') - return - - s.force_full(f'{path}: unclassified -> full matrix') - - -def classify(changed_files, repo_root, rosters): - all_boards = [] - seen = set() - for _, boards in rosters: - for b in boards: - if b['name'] not in seen: - seen.add(b['name']) - all_boards.append(b) - - extras = roster_only_tests(all_boards) - s = _Sel() - # no early exit once full: keep classifying so `families` still reports every - # family the diff touches (build-only consumers need it). Nothing after the first - # force_full can change full/boards/args - the full branch below ignores by_board. - for path in changed_files: - _classify_one(path, repo_root, all_boards, extras, s) - - if s.full: - return {'full': True, 'boards': {b['name']: 'all' for b in all_boards}, - 'families': sorted(s.families), 'reasons': s.reasons} - - # role pruning: single-role selections drop the other role's tests and boards - by_name = {b['name']: b for b in all_boards} - out = {} - for name, tests in s.by_board.items(): - allowed = board_tests(by_name[name]) - if tests == 'all': - kept = list(allowed) - else: - kept = [t for t in allowed if t in tests] - if s.roles and s.roles != {'device', 'host'}: - role = next(iter(s.roles)) - kept = [t for t in kept if test_role(t) in (role, 'dual')] - if kept: - out[name] = 'all' if set(kept) == set(allowed) else sorted(kept) - return {'full': False, 'boards': out, 'families': sorted(s.families), - 'reasons': s.reasons} - - -def _board_args(name, chosen) -> list: - parts = [f'-b {name}'] - if chosen != 'all': - parts.append(f'-bt {name}:{",".join(chosen)}') - return parts - - -def selection_args(sel, rosters): - """hil_test.py args per config. Empty means either 'full matrix' or 'nothing - selected' - callers must read sel['full'] to tell them apart.""" - args = {} - for cfg_path, boards in rosters: - parts = [] - if not sel['full']: - for b in boards: - chosen = sel['boards'].get(b['name']) - if chosen is not None: - parts += _board_args(b['name'], chosen) - args[os.path.basename(cfg_path)] = ' '.join(parts) - return args - - -def selection_args_by_flasher(sel, rosters): - """{config: {flasher name: args}}. CI runs one rig as several jobs split by - flasher (esptool vs the rest); each must gate on its own subset, otherwise the - other leg runs a filter matching zero boards and reports a vacuous green.""" - out = {} - for cfg_path, boards in rosters: - per = {} - if not sel['full']: - for b in boards: - chosen = sel['boards'].get(b['name']) - if chosen is None: - continue - per.setdefault(b.get('flasher', {}).get('name', ''), []).extend( - _board_args(b['name'], chosen)) - out[os.path.basename(cfg_path)] = {f: ' '.join(p) for f, p in per.items()} - return out - - -def changed_files_from_git(base, repo_root): - mb = subprocess.run(['git', 'merge-base', 'HEAD', base], cwd=repo_root, - capture_output=True, text=True, check=True).stdout.strip() - diff = subprocess.run(GIT_DIFF_ARGV + [f'{mb}..HEAD'], cwd=repo_root, - capture_output=True, text=True, check=True).stdout - return [l for l in diff.splitlines() if l.strip()] - - -def main(): - ap = argparse.ArgumentParser(description=__doc__) - g = ap.add_mutually_exclusive_group(required=True) - g.add_argument('--base', help='git ref to diff against (merge-base..HEAD)') - g.add_argument('--diff-file', help='newline-separated changed-file list') - ap.add_argument('configs', nargs='+', help='rig roster JSON file(s)') - a = ap.parse_args() - - # test/hil/helper/ -> repo root is FOUR levels up; three left this at <repo>/test - # after the helper/ move and every repo-relative glob silently matched nothing - repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) - rosters = [] - for c in a.configs: - with open(c) as f: - rosters.append((c, json.load(f)['boards'])) - - files = (open(a.diff_file).read().splitlines() if a.diff_file - else changed_files_from_git(a.base, repo_root)) - files = [f for f in files if f.strip()] - - s = classify(files, repo_root, rosters) - s['args'] = selection_args(s, rosters) - s['args_flasher'] = selection_args_by_flasher(s, rosters) - for r in s['reasons']: - print(f'hil_select: {r}', file=sys.stderr) - print(json.dumps(s)) - - -if __name__ == '__main__': - main() diff --git a/test/hil/helper/hil_util.py b/test/hil/helper/hil_util.py index 54984d20f..0a2a13fca 100644 --- a/test/hil/helper/hil_util.py +++ b/test/hil/helper/hil_util.py @@ -18,7 +18,7 @@ from typing import Any # ------------------------------------------------------------- -# HIL example test lists, shared by hil_test.py (runner) and hil_select.py (PR-diff +# HIL example test lists, shared by hil_test.py (runner) and ci_select.py (PR-diff # selector). Run order is shuffled per board (see test_board); every example carries a # unique hardcoded idProduct (see its usb_descriptors.c). # ------------------------------------------------------------- diff --git a/test/hil/hil_ci.sh b/test/hil/hil_ci.sh index 66f4e48d4..514b0f174 100644 --- a/test/hil/hil_ci.sh +++ b/test/hil/hil_ci.sh @@ -224,7 +224,6 @@ scp -q "$ROOT_DIR/test/hil/helper/__init__.py" \ "$ROOT_DIR/test/hil/helper/hil_health.py" \ "$ROOT_DIR/test/hil/helper/hil_lock.py" \ "$ROOT_DIR/test/hil/helper/hil_summary.py" \ - "$ROOT_DIR/test/hil/helper/hil_select.py" \ "$REMOTE:$REMOTE_DIR/test/hil/helper/" # Copy only firmware binaries (elf/bin/hex) plus esptool metadata diff --git a/test/hil/hil_flash.py b/test/hil/hil_flash.py index f4bed45a6..c4d4e6552 100755 --- a/test/hil/hil_flash.py +++ b/test/hil/hil_flash.py @@ -294,7 +294,7 @@ reset_lm4flash.no_op = True # The one place a flasher's firmware extension is decided. A flasher with no entry falls -# back to .elf-or-.bin and can be handed the wrong file — test_hil_select's +# back to .elf-or-.bin and can be handed the wrong file — test_ci_select's # TestRosterFlashersDispatch fails if a roster names one. FLASHER_SUFFIX = { 'esptool': '.bin', diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py index 174251343..fcd7c7e6f 100755 --- a/test/hil/hil_test.py +++ b/test/hil/hil_test.py @@ -194,10 +194,6 @@ class TestsCfg(TypedDict, total=False): dev_attached: list[AttachedDevCfg] -class BuildCfg(TypedDict, total=False): - args: list[str] - - class VariantCfg(TypedDict, total=False): name: str # build dir (cmake-build-<name>) and HIL report row flags: str # raw CFLAGS, e.g. "-DCFG_TUD_DWC2_DMA_ENABLE=1" @@ -209,7 +205,9 @@ class Board(TypedDict): uid: str tests: TestsCfg flasher: FlasherCfg - build: NotRequired[BuildCfg] + # every build knob lives here, including a board's always-on defines: a board that + # needs one carries a single variant named after itself (metro_m4_express / + # MAX3421_HOST=1), which is exactly what the `or [...]` default below synthesises variant: NotRequired[list[VariantCfg]] toolchain: NotRequired[str] # CI build bucket override, e.g. "riscv-gcc" (consumed by hil_ci_set_matrix.py) @@ -1670,21 +1668,17 @@ def test_example(board: Board, variant: str, example: str) -> tuple[int, str, st def build_board(board: Board) -> tuple[str, int]: """Build firmware for this board via tools/build.py. - Honors board config's variant list and build.args defines. + Honors board config's variant list (name, defines, flags). Output goes to cmake-build/cmake-build-<variant>/ (tools/build.py layout). Unbounded on purpose: --build is a local convenience (no CI workflow passes it), so the developer watching the build is the timeout.""" name = board['name'] - bcfg = cast(BuildCfg, board.get('build', {})) - extra_defs = bcfg.get('args', []) variants = board.get('variant') or [{'name': name, 'flags': ''}] failed = 0 for v in variants: cmd = [sys.executable, str(hil_util.TINYUSB_ROOT / 'tools' / 'build.py'), '-b', name] - for d in extra_defs: - cmd += ['-D', d] if v['name'] != name: cmd += ['--build-name', v['name']] for d in v.get('defines', []): diff --git a/test/hil/test/test_ci_metrics.py b/test/hil/test/test_ci_metrics.py new file mode 100644 index 000000000..a76b6e3a0 --- /dev/null +++ b/test/hil/test/test_ci_metrics.py @@ -0,0 +1,463 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests for the by-example half of tools/metrics.py and the (family, example) +# pair-compare script. Stdlib only; synthetic map.json fixtures, no builds. +# python3 test/hil/test/test_ci_metrics.py +import json +import os +import subprocess +import sys +import tempfile +import unittest + +REPO = os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.abspath(__file__))))) +METRICS = os.path.join(REPO, 'tools', 'metrics.py') + + +def fake_map(path, files): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'w') as f: + json.dump({'files': files}, f) + + +def entry(name, size, path_prefix='tinyusb/src'): + return {'file': name, 'path': f'{path_prefix}/{name}', 'size': size, + 'symbols': [{'name': f'{name}_fn', 'size': size}], 'sections': {'.text': size}} + + +class TestByExample(unittest.TestCase): + def build_tree(self, td): + fake_map(os.path.join(td, 'device', 'cdc_msc', 'cdc_msc.map.json'), + [entry('usbd.c', 100), entry('cdc_device.c', 50)]) + fake_map(os.path.join(td, 'host', 'bare_api', 'bare_api.map.json'), + [entry('usbh.c', 200)]) + + def test_by_example_output(self): + with tempfile.TemporaryDirectory() as td: + self.build_tree(td) + out = os.path.join(td, 'metrics') + r = subprocess.run([sys.executable, METRICS, 'combine', '-q', '-j', + '--by-example', '-o', out, + os.path.join(td, '*', '*', '*.map.json')], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + by_ex = json.load(open(out + '_by_example.json')) + self.assertEqual(set(by_ex), {'device/cdc_msc', 'host/bare_api'}) + self.assertEqual({f['file'] for f in by_ex['device/cdc_msc']['files']}, + {'usbd.c', 'cdc_device.c'}) + # the plain averaged output is unchanged by the extra flag + avg = json.load(open(out + '.json')) + self.assertIn('files', avg) + + def test_by_example_json_roundtrips_as_combine_input(self): + with tempfile.TemporaryDirectory() as td: + self.build_tree(td) + out = os.path.join(td, 'metrics') + subprocess.run([sys.executable, METRICS, 'combine', '-q', '-j', '--by-example', + '-o', out, os.path.join(td, '*', '*', '*.map.json')], check=True) + out2 = os.path.join(td, 'sub') + r = subprocess.run([sys.executable, METRICS, 'combine', '-q', '-j', + '-o', out2, out + '_by_example.json'], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + sub = json.load(open(out2 + '.json')) + names = {f['file'] for f in sub['files']} + # one data entry per example, not one blob: reading it as an ordinary + # metrics.json would double-count every file + self.assertIn('usbd.c', names) + self.assertIn('cdc_device.c', names) + self.assertNotIn('TOTAL', {n.upper() for n in names}) + + def test_by_example_expansion_is_keyed_on_the_filename(self): + # the '_by_example.json' suffix IS the contract (write_by_example, the CMake + # rule and metrics_pair_compare all spell it). A shape-sniff would reroute + # any coincidentally-shaped JSON into the per-example branch instead. + with tempfile.TemporaryDirectory() as td: + look_alike = os.path.join(td, 'metrics.json') + with open(look_alike, 'w') as f: + json.dump({'device/cdc_msc': {'files': [entry('usbd.c', 100)]}}, f) + out = os.path.join(td, 'combined') + r = subprocess.run([sys.executable, METRICS, 'combine', '-q', '-j', + '-o', out, look_alike], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + combined = json.load(open(out + '.json')) + self.assertNotIn('usbd.c', {f['file'] for f in combined.get('files', [])}) + + +PAIR_COMPARE = os.path.join(REPO, '.github/scripts/metrics_pair_compare.py') + + +def fake_by_example(root, board, data): + d = os.path.join(root, f'cmake-build-{board}') + os.makedirs(d, exist_ok=True) + with open(os.path.join(d, 'metrics_by_example.json'), 'w') as f: + json.dump(data, f) + + +class TestPairCompare(unittest.TestCase): + def test_intersection_compare(self): + with tempfile.TemporaryDirectory() as td: + base, new = os.path.join(td, 'base'), os.path.join(td, 'new') + # real board names so board->family resolution works against hw/bsp + fake_by_example(base, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('usbd.c', 100)]}, + 'device/dfu': {'files': [entry('dfu_device.c', 10)]}}) + fake_by_example(new, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('usbd.c', 120)]}}) + out = os.path.join(td, 'cmp') + r = subprocess.run([sys.executable, PAIR_COMPARE, '--base-dir', base, + '--new-dir', new, '--out', out], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + md = open(out + '.md').read() + self.assertIn('usbd.c', md) + self.assertNotIn('dfu_device.c', md) # not on both sides + self.assertIn('raspberry_pi_pico', md) # scope footer names the board + self.assertIn('device/dfu', md) # named as dropped + + def test_a_different_board_of_the_same_family_is_not_compared(self): + """--one-first returns all_boards[0], so adding a board can shift which one a + family builds. Keyed on the family, the base run's sizes and the PR run's sizes + would land under one key and the difference between two unrelated MCUs would be + published as this PR's code-size impact.""" + with tempfile.TemporaryDirectory() as td: + base, new = os.path.join(td, 'base'), os.path.join(td, 'new') + # both rp2040, both device/cdc_msc - only the board differs + fake_by_example(base, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('usbd.c', 100)]}}) + fake_by_example(new, 'adafruit_fruit_jam', + {'device/cdc_msc': {'files': [entry('usbd.c', 900)]}}) + out = os.path.join(td, 'cmp') + r = subprocess.run([sys.executable, PAIR_COMPARE, '--base-dir', base, + '--new-dir', new, '--out', out], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + md = open(out + '.md').read() + self.assertIn('skipped', md) + self.assertNotIn('+800', md) + + def test_empty_intersection_writes_note(self): + with tempfile.TemporaryDirectory() as td: + base, new = os.path.join(td, 'base'), os.path.join(td, 'new') + fake_by_example(base, 'raspberry_pi_pico', {'device/dfu': {'files': [entry('a.c', 1)]}}) + fake_by_example(new, 'stm32f407disco', {'device/cdc_msc': {'files': [entry('b.c', 1)]}}) + out = os.path.join(td, 'cmp') + r = subprocess.run([sys.executable, PAIR_COMPARE, '--base-dir', base, + '--new-dir', new, '--out', out], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn('skipped', open(out + '.md').read()) + + def test_malformed_files_are_skipped_with_stderr_note(self): + with tempfile.TemporaryDirectory() as td: + base, new = os.path.join(td, 'base'), os.path.join(td, 'new') + # good pair on both sides -- must survive the malformed siblings below + fake_by_example(base, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('usbd.c', 100)]}}) + fake_by_example(new, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('usbd.c', 120)]}}) + # well-formed JSON, wrong shape (a list, not a {example: {files: [...]}} dict) + wrong_shape = os.path.join(base, 'cmake-build-stm32f407disco', 'metrics_by_example.json') + os.makedirs(os.path.dirname(wrong_shape), exist_ok=True) + with open(wrong_shape, 'w') as f: + json.dump(['not', 'a', 'dict'], f) + # metrics_by_example.json not under a cmake-build-<board> dir + misplaced = os.path.join(base, 'not_a_board_dir', 'metrics_by_example.json') + os.makedirs(os.path.dirname(misplaced), exist_ok=True) + with open(misplaced, 'w') as f: + json.dump({'device/dfu': {'files': [entry('dfu_device.c', 10)]}}, f) + + out = os.path.join(td, 'cmp') + r = subprocess.run([sys.executable, PAIR_COMPARE, '--base-dir', base, + '--new-dir', new, '--out', out], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) # fail-open: never crash the job + md = open(out + '.md').read() + self.assertIn('usbd.c', md) # good pair still compared + self.assertIn(wrong_shape, r.stderr) + self.assertIn(misplaced, r.stderr) + self.assertIn('skipping', r.stderr) + + + def test_missing_base_baseline_gets_its_own_note(self): + # interim state right after this feature merges: master has not uploaded a + # per-example baseline yet, so the BASE side collects nothing. The generic + # "no pair on both sides" note misattributes that to the PR's own scoping. + with tempfile.TemporaryDirectory() as td: + base, new = os.path.join(td, 'base'), os.path.join(td, 'new') + os.makedirs(base) + fake_by_example(new, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('usbd.c', 100)]}}) + out = os.path.join(td, 'cmp') + r = subprocess.run([sys.executable, PAIR_COMPARE, '--base-dir', base, + '--new-dir', new, '--out', out], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + md = open(out + '.md').read() + self.assertIn('No per-example baseline from the base branch yet', md) + self.assertIn('next push', md) + self.assertNotIn('comparison skipped', md) + + def test_a_partially_malformed_file_contributes_nothing(self): + """A file that blows up half way through must drop WHOLE. Entries parsed + before the malformation used to stay in the comparison while stderr claimed + the file had been skipped - a silently truncated table published as the + code-size verdict. A non-list 'files' (TypeError) also has to be caught.""" + with tempfile.TemporaryDirectory() as td: + base, new = os.path.join(td, 'base'), os.path.join(td, 'new') + # good entry FIRST, malformed second: the leak is order-dependent + fake_by_example(base, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('leaked.c', 100)]}, + 'device/dfu': {'files': 42}}) + fake_by_example(new, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('leaked.c', 120)]}}) + # a sibling file that is fine on both sides must still be compared + fake_by_example(base, 'stm32f407disco', + {'device/cdc_msc': {'files': [entry('good.c', 10)]}}) + fake_by_example(new, 'stm32f407disco', + {'device/cdc_msc': {'files': [entry('good.c', 12)]}}) + out = os.path.join(td, 'cmp') + r = subprocess.run([sys.executable, PAIR_COMPARE, '--base-dir', base, + '--new-dir', new, '--out', out], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + md = open(out + '.md').read() + self.assertIn('good.c', md) + self.assertNotIn('leaked.c', md) + self.assertIn('skipping', r.stderr) + self.assertIn(os.path.join(base, 'cmake-build-raspberry_pi_pico'), r.stderr) + + def test_dropped_footer_is_summarised_not_dumped(self): + """The sticky PR comment is capped at 65,536 chars by GitHub; a broad scoped + PR drops hundreds of (family, example) pairs and the full list alone ran to + tens of KB, pushing the comment past the cap and reddening code-metrics.""" + with tempfile.TemporaryDirectory() as td: + base, new = os.path.join(td, 'base'), os.path.join(td, 'new') + common = {'device/cdc_msc': {'files': [entry('usbd.c', 100)]}} + extra = {f'device/example_{i:03d}': {'files': [entry(f'f{i}.c', i + 1)]} + for i in range(30)} + fake_by_example(base, 'raspberry_pi_pico', dict(common, **extra)) + fake_by_example(new, 'raspberry_pi_pico', common) + out = os.path.join(td, 'cmp') + r = subprocess.run([sys.executable, PAIR_COMPARE, '--base-dir', base, + '--new-dir', new, '--out', out], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + md = open(out + '.md').read() + footer = md[md.index('_Scoped compare:'):] + self.assertLess(len(footer), 2048, footer) + self.assertIn('30', footer) # the count is still reported + self.assertIn('more', footer) # truncation marker + self.assertIn('device/example_029', r.stderr) # full list on stderr + + +CIRCLECI = os.path.join(REPO, '.circleci') +SENTINELS = ('example-map-default', 'build-filtered-default') + + +class TestCircleCiSentinelContract(unittest.TestCase): + """config.yml's set-matrix rewrites config2.yml's parameter defaults by matching + a sentinel comment line — the only way past /pipeline/continue's 512-char + parameter cap. Renaming or reformatting either side is a silent full-build + fallback that no CI job reports, so pin the contract here.""" + + def setUp(self): + self.config = open(os.path.join(CIRCLECI, 'config.yml')).read() + self.config2 = open(os.path.join(CIRCLECI, 'config2.yml')).read() + + def test_each_sentinel_appears_once_on_a_default_line(self): + for tag in SENTINELS: + marker = f'# {tag}: rewritten in-place by config.yml set-matrix' + hits = [l for l in self.config2.splitlines() if l.strip().endswith(marker)] + self.assertEqual(len(hits), 1, f'{tag}: {len(hits)} sentinel lines in config2.yml') + self.assertIn('default:', hits[0], f'{tag}: sentinel is not on a default: line') + + def test_the_selection_travels_as_a_file(self): + # a mass-sweep selection runs to hundreds of KB: handed to ci_set_matrix as one + # argv it E2BIGs the step before the `||` fallback can fire, and EXAMPLE_MAP / + # BUILD_FILTERED (derived with jq, no argv limit) would then label a FULL build + # scoped -- the build and its label disagreeing is worse than either alone + self.assertIn('--select-file', self.config) + self.assertNotIn('--select "', self.config) + + def test_the_rewriter_names_the_same_sentinels(self): + for tag in SENTINELS: + self.assertIn(f"'{tag}'", self.config, + f'{tag}: config.yml rewrite block does not name this sentinel') + self.assertIn("# {tag}: rewritten in-place by config.yml set-matrix", self.config, + 'config.yml no longer builds the sentinel comment it matches on') + + def test_the_rewrite_precedes_the_scoped_entries(self): + # the scoping is all-or-nothing: config2's checked-in defaults are {} / false = + # unfiltered, so a rewrite that fails AFTER the family entries were generated + # leaves a subset of families built and code-metrics told it was a full build. + # Rewrite first, and on failure drop the scoping (back to the full matrix). + rewrite = self.config.index("p = '.circleci/config2.yml'") + entries = self.config.index('gen_build_entry() {') + self.assertLess(rewrite, entries, + 'the sentinel rewrite must run before any build entry is generated') + tail = self.config[rewrite:entries] + self.assertIn('MATRIX_JSON="$FULL_MATRIX_JSON"', tail, + 'a failed rewrite must fall back to the FULL matrix, not keep the ' + 'scoped one') + # and that fallback must be a plain assignment: a second `python ...` here is an + # unguarded command under CircleCI's `set -e`, inside the one branch whose whole + # job is to keep the pipeline green + self.assertNotIn('ci_set_matrix.py)', tail) + + def test_the_selector_gate_runs_both_suites(self): + # test_ci_select.py owns the rules; this file owns the sentinel contract the + # very same job rewrites. Gating on one of the two leaves the other unguarded. + for suite in ('test_ci_select.py', 'test_ci_metrics.py'): + self.assertIn(suite, self.config, f'{suite} does not gate the CircleCI selector') + + +class TestWorkflowSelectionHandOff(unittest.TestCase): + """build.yml's counterpart of the CircleCI contract above: same E2BIG limit, same + consequence (the scoping silently turns itself off on exactly the PRs where it + saves most), plus the GITHUB_ENV lines that carry PR-derived values.""" + + def setUp(self): + wf = os.path.join(os.path.dirname(CIRCLECI), '.github', 'workflows') + self.build = open(os.path.join(wf, 'build.yml')).read() + self.util = open(os.path.join(wf, 'build_util.yml')).read() + + def test_no_step_execs_with_the_selection_in_its_environment(self): + # SELECT_JSON="$SELECT_JSON" python3 -c ... E2BIGs at ~128KiB: measured 261KB + # for a `git ls-files hw/bsp/**` sweep. Every reader takes the file instead. + self.assertNotIn('SELECT_JSON="$SELECT_JSON"', self.build) + self.assertIn('json.load(open("ci_select_out.json"))', self.build) + + def test_the_file_is_written_before_its_first_reader(self): + self.assertLess(self.build.index("printf '%s' \"$SELECT_JSON\" > ci_select_out.json"), + self.build.index('json.load(open("ci_select_out.json"))'), + 'the selection file must exist before the step that reads it') + + def test_pr_derived_env_values_are_character_guarded(self): + # values reach GITHUB_ENV/GITHUB_OUTPUT as bare NAME=VALUE lines; a newline in + # one (git allows it in a path, and both the example map and the roster are + # PR-editable) writes extra variables into every later step of a job that runs + # with secrets - and for run_*, flips which rig jobs execute + for name in ('EX_ARGS', 'ARTIFACT_TAG'): + self.assertIn(f'echo "{name}=', self.util) + # per guard, not a sum: `count(a) + count(b) == 2` stays green when one guard is + # deleted and the other duplicated + for guard in ('case "$EX_ARGS" in', 'case "$TAG" in'): + self.assertEqual(self.util.count(guard), 1, + f'{guard}: each GITHUB_ENV write screens its value exactly once') + # CircleCI builds from the same PR-derived map and uses $EX_ARGS unquoted + cci = open(os.path.join(CIRCLECI, 'config2.yml')).read() + self.assertIn('case "$EX_ARGS" in', cci, + 'the CircleCI copy of the example filter needs the same screen') + self.assertIn('case "$BUILD_ARGS" in', self.build) + self.assertIn('unexpected characters in the " + key', self.build, + 'the args_*/run_* emitter must screen each board filter') + + def test_the_guards_accept_what_the_selector_actually_emits(self): + """A guard that rejects a NORMAL value is worse than no guard: build.yml throws + the whole selection away, warns, and both axes fall back to full - silently + turning the feature off. So run the real character classes over real selections + rather than only asserting that the guard text is present. + + The one that got away: `[-A-Za-z0-9_/ .=+]` has no ':' or ',', and every partial + board filter is `-bt <board>:<test>,<test>`.""" + import re, subprocess, sys, tempfile, json + repo = os.path.dirname(CIRCLECI) + # the character classes, lifted from the three places they are written + classes = {} + m = re.search(r're\.fullmatch\(r"\[([^"]+)\]\*"', self.build) + self.assertTrue(m, 'args_*/run_* guard not found in build.yml') + classes['args'] = m.group(1) + for name, text in (('BUILD_ARGS', self.build), ('EX_ARGS', self.util), + ('TAG', self.util)): + m = re.search(r'case "\$%s" in\s*\n\s*\*\[!([^\]]+)\]\*\)' % name, text) + self.assertTrue(m, f'{name} guard not found') + classes[name] = m.group(1).replace('\\', '') + + def ok(cls, value): + return re.fullmatch('[%s]*' % cls.replace('!', ''), value) is not None + + with tempfile.TemporaryDirectory() as d: + for path in ('src/class/cdc/cdc_device.c', 'src/device/usbd.c', + 'src/portable/synopsys/dwc2/dcd_dwc2.c', + 'examples/device/cdc_msc/src/main.c', + 'hw/bsp/stm32f4/family.cmake'): + f = os.path.join(d, 'diff.txt') + with open(f, 'w') as fh: + fh.write(path + '\n') + r = subprocess.run([sys.executable, os.path.join(repo, 'tools/ci_select.py'), + '--diff-file', f, + os.path.join(repo, 'test/hil/tinyusb.json'), + os.path.join(repo, 'test/hil/hfp.json')], + capture_output=True, text=True, cwd=repo) + self.assertEqual(r.returncode, 0, r.stderr) + s = json.loads(r.stdout) + for flasher, a in s.get('args_flasher', {}).get('tinyusb.json', {}).items(): + self.assertTrue(ok(classes['args'], a), + f'{path}/{flasher}: the args guard rejects {a!r}') + hfp = s.get('args', {}).get('hfp.json', '') + self.assertTrue(ok(classes['args'], hfp), f'{path}: hfp {hfp!r}') + # BUILD_ARGS is the hfp job's `-b <board> [-e ...]` list, not the -bt + # test filter above - screen the value that step actually builds + with open(os.path.join(d, 'sel.json'), 'w') as fh: + fh.write(r.stdout) + hm = subprocess.run( + [sys.executable, os.path.join(repo, '.github/scripts/hil_ci_set_matrix.py'), + '--select-file', os.path.join(d, 'sel.json'), + os.path.join(repo, 'test/hil/hfp.json')], + capture_output=True, text=True, cwd=repo) + self.assertEqual(hm.returncode, 0, hm.stderr) + build_args = ' '.join(json.loads(hm.stdout)['arm-gcc']) + self.assertTrue(ok(classes['BUILD_ARGS'], build_args), + f'{path}: the BUILD_ARGS guard rejects {build_args!r}') + for entry in json.loads(hm.stdout)['arm-gcc']: + tag = re.sub(r' -e [^ ]+', '', entry) + self.assertTrue(ok(classes['TAG'], tag), + f'{path}: the artifact-name guard rejects {tag!r}') + for fam, exs in (s.get('build', {}).get('family_examples') or {}).items(): + ex_args = ' '.join('-e ' + e for e in exs) + self.assertTrue(ok(classes['EX_ARGS'], ex_args), + f'{path}/{fam}: the EX_ARGS guard rejects {ex_args!r}') + + def test_an_unusable_selection_is_unusable_for_both_matrices(self): + # hil_ci_set_matrix reads "full false with no boards map" as unusable and falls + # open to the whole roster; if this emitter instead computed run_*=false, the + # rig jobs would skip while all 37 build legs ran - a full build and still zero + # hardware coverage, which is the outcome the guard exists to prevent + self.assertIn('isinstance(s.get("boards"), dict)', self.build) + + def test_the_build_extras_drop_when_the_matrix_falls_open(self): + # ci_set_matrix falls open with rc 0, so the example map and family regex must + # follow it or a nominally full build is filtered and labelled as a scoped one + self.assertIn("grep -q 'ci_set_matrix: UNSCOPED'", self.build) + self.assertIn('BUILD_SELECT_FILE', self.build) + scripts = os.path.join(os.path.dirname(CIRCLECI), '.github', 'scripts') + matrix = open(os.path.join(scripts, 'ci_set_matrix.py')).read() + # count-independent: pin the INVARIANT, not the number of fall-open paths - + # every message that emits the full matrix must carry the marker, and a purely + # informational note (a partial family miss) must not claim to have done so. + # Adjacent string literals are joined first, since these messages wrap. + import re as _re + flat = _re.sub(r"['\"]\s*\n\s*f?['\"]", '', matrix) + hits = [m.start() for m in _re.finditer('emitting the full ', flat)] + self.assertGreaterEqual(len(hits), 2, 'fall-open messages not found') + for i in hits: + self.assertIn('UNSCOPED', flat[max(0, i - 200):i], + 'a fall-open path without the marker build.yml greps for') + + def test_membrowse_upload_sees_the_same_board_as_the_build(self): + # $EX_ARGS is passed for the BOARD it selects: --one-first picks a board that can + # build the -e set, so without it membrowse configures a different, empty build + # dir and uploads --identical for a board that was never compiled. It does NOT + # scope the targets - `examples-membrowse-upload` is not `all`, so it passes + # through as the aggregate, which has no DEPENDS and still records every example. + line = [l for l in self.util.splitlines() + if '--target examples-membrowse-upload' in l][0] + self.assertIn('$EX_ARGS', line) + self.assertNotIn('-e ', line.replace('$EX_ARGS', '')) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/hil/test/test_ci_select.py b/test/hil/test/test_ci_select.py new file mode 100644 index 000000000..8f1841531 --- /dev/null +++ b/test/hil/test/test_ci_select.py @@ -0,0 +1,2202 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests for ci_select.py — pure logic, no hardware, no git. Run directly: +# python3 test/hil/test/test_ci_select.py +# +# Imports stay stdlib + ci_select/hil_util/hil_flash ONLY: the pre-commit hil-test +# hook runs this suite, on GitHub's bare runner in the pre-commit workflow as well as +# locally, and that runner has no pyserial/pymtp. hil_flash is admissible because it +# is stdlib + hil_util only (test_hil_util.BottomLayer enforces the stdlib closure of +# both) and the roster-dispatch tests need its flash_* table; never import hil_test, +# which pulls pyserial. +import contextlib +import glob +import io +import json +import os +import pathlib +import re +import subprocess +import sys +import unittest + +REPO = os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.abspath(__file__))))) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # test/hil, for hil_flash/helper +sys.path.insert(0, os.path.join(REPO, 'tools')) +import hil_flash +import ci_select +from helper.hil_util import device_tests, dual_tests + + +def real_rosters(): + """The actual rig rosters, for regression tests that need real-world data + (a specific board/family/only-list) rather than the synthetic ROSTER above.""" + rosters = [] + for name in ('tinyusb.json', 'hfp.json'): + path = os.path.join(REPO, 'test/hil', name) + with open(path) as f: + rosters.append((f'test/hil/{name}', json.load(f)['boards'])) + return rosters + + +def roster_flashers(): + """(roster path, board) for every board in the live rosters, `boards-skip` + included: a parked board's flasher name must still dispatch, so that unparking it + is not what discovers the name went stale.""" + for name in ('tinyusb.json', 'hfp.json'): + path = os.path.join(REPO, 'test/hil', name) + with open(path) as f: + cfg = json.load(f) + for key in ('boards', 'boards-skip'): + for b in cfg.get(key, []): + yield f'test/hil/{name}', b + + +def on_roster(tc, *names): + """The subset of `names` currently in the live rig rosters, skipping the test + when none are, because parking/unparking a board is routine rig maintenance. + + That skip now matters MORE than it used to, not less: this suite is a blocking + pre-commit hook AND build.yml's selector steps gate on it (a failing suite falls + open to the full matrix), so an assertion that depends on a specific board being + present goes red on every PR -- including src/-only ones that never touched the + rig -- until someone fixes the roster. Keep roster-dependent assertions behind + on_roster.""" + have = {b['name'] for _, boards in real_rosters() for b in boards} + got = [n for n in names if n in have] + if not got: + tc.skipTest(f'not in the rig roster: {", ".join(names)}') + return got + + +ROSTER = [ + # device-only, rp2040 family + {'name': 'raspberry_pi_pico', 'uid': 'u1', 'flasher': {'name': 'openocd'}, + 'tests': {'device': True, 'host': True, 'dual': True}}, + # device-only, stm32f4 family + {'name': 'stm32f407disco', 'uid': 'u2', 'flasher': {'name': 'jlink'}, + 'tests': {'device': True, 'host': False, 'dual': False}}, + # host-only board + {'name': 'raspberry_pi_pico2', 'uid': 'u3', 'flasher': {'name': 'openocd'}, + 'tests': {'device': False, 'host': True, 'dual': False}}, + # only-list board (espressif-style), flashed by the CI leg that splits on esptool + {'name': 'espressif_s3_devkitm', 'uid': 'u4', 'flasher': {'name': 'esptool'}, + 'tests': {'only': ['device/cdc_msc_freertos', 'host/device_info']}}, +] +ROSTERS = [('test/hil/tinyusb.json', ROSTER)] + + +def sel(files): + return ci_select.classify(files, REPO, ROSTERS) + + +class TestPortRule(unittest.TestCase): + def test_dcd_rp2040_selects_pico_family_only(self): + s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico', s['boards']) + self.assertNotIn('stm32f407disco', s['boards']) + self.assertNotIn('espressif_s3_devkitm', s['boards']) + # device role: no host tests in pico's list + self.assertTrue(all(not t.startswith('host/') for t in s['boards']['raspberry_pi_pico'])) + # host-only boards drop out entirely on a device-role change + self.assertNotIn('raspberry_pi_pico2', s['boards']) + + def test_shared_port_file_is_both_roles(self): + s = sel(['src/portable/synopsys/dwc2/dwc2_common.c']) + self.assertFalse(s['full']) + self.assertNotIn('raspberry_pi_pico', s['boards']) # rp2040 is not a dwc2 family + self.assertIn('stm32f407disco', s['boards']) # stm32f4 is + + +class TestCoreRoleRule(unittest.TestCase): + def test_usbd_selects_all_device_tests_everywhere(self): + s = sel(['src/device/usbd.c']) + self.assertFalse(s['full']) + self.assertNotIn('raspberry_pi_pico2', s['boards']) # host-only board dropped + pico = s['boards']['raspberry_pi_pico'] + self.assertTrue(set(device_tests).issubset(set(pico))) + self.assertTrue(set(dual_tests).issubset(set(pico))) # dual survives device role + self.assertTrue(all(not t.startswith('host/') for t in pico)) + # only-list board: selection intersects its only-list + esp = s['boards']['espressif_s3_devkitm'] + self.assertEqual(esp, ['device/cdc_msc_freertos']) + + def test_host_change_drops_device(self): + s = sel(['src/host/usbh.c']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico2', s['boards']) + self.assertNotIn('stm32f407disco', s['boards']) # device-only board dropped + + +class TestClassRule(unittest.TestCase): + def test_cdc_device_selects_cdc_examples_only(self): + s = sel(['src/class/cdc/cdc_device.c']) + self.assertFalse(s['full']) + pico = s['boards']['raspberry_pi_pico'] + self.assertIn('device/cdc_msc', pico) + self.assertIn('device/cdc_dual_ports', pico) + self.assertNotIn('device/msc_dual_lun', pico) # CFG_TUD_CDC 0 there + self.assertNotIn('device/usbtest', pico) # CFG_TUD_CDC 0 there + self.assertTrue(all(not t.startswith('host/') for t in pico)) + + def test_msc_host_selects_host_side(self): + s = sel(['src/class/msc/msc_host.c']) + self.assertFalse(s['full']) + self.assertNotIn('stm32f407disco', s['boards']) # device-only board + pico2 = s['boards']['raspberry_pi_pico2'] + self.assertIn('host/msc_file_explorer', pico2) + self.assertTrue(all(not t.startswith('device/') for t in pico2)) + + +class TestClassIncludeEdges(unittest.TestCase): + """A class header another class includes reaches that class's examples too. + src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, so + midi_test's firmware contains audio.h - but the class rule derives macros from + the directory name alone, so an audio.h change used to select only + device/audio_test_freertos. On boards that skip that example the per-board + intersection emptied and an audio.h-only PR ran ZERO HIL on them.""" + def test_edges_derived_from_includes(self): + edges = ci_select.class_include_edges(REPO) + self.assertEqual(edges.get('audio/audio.h'), {'midi'}) + self.assertEqual(edges.get('cdc/cdc.h'), {'net'}) + + def test_audio_header_selects_midi_example(self): + s = ci_select.classify(['src/class/audio/audio.h'], REPO, real_rosters()) + self.assertFalse(s['full']) + # every board that runs device/midi_test at all must run it here (boards with + # a tests.only list, e.g. espressif, run the freertos examples instead) + by_name = {b['name']: b for _, bs in real_rosters() for b in bs} + checked = 0 + for name, tests in s['boards'].items(): + if 'device/midi_test' in ci_select.board_tests(by_name[name]): + self.assertIn('device/midi_test', tests, name) + checked += 1 + self.assertTrue(checked) + + def test_audio_header_reaches_boards_that_skip_audio(self): + # both skip device/audio_test_freertos: without the midi edge their + # intersection is empty and they drop out of the selection entirely + boards = on_roster(self, 'metro_m4_express', 'nrf54lm20dk') + s = ci_select.classify(['src/class/audio/audio.h'], REPO, real_rosters()) + for board in boards: + self.assertEqual(s['boards'].get(board), ['device/midi_test'], board) + + def test_edge_is_per_header_not_per_class(self): + # midi includes audio.h, not audio_device.h: an audio_device change must + # not drag midi's examples in + s = ci_select.classify(['src/class/audio/audio_device.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for tests in s['boards'].values(): + if tests != 'all': + self.assertNotIn('device/midi_test', tests) + + +class TestFallbackRules(unittest.TestCase): + def test_unknown_tool_is_full(self): + s = sel(['tools/random_new_script.py']) + self.assertTrue(s['full']) + + def test_docs_only_is_empty_not_full(self): + s = sel(['docs/info/contributing.rst', 'README.rst']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + def test_bsp_family_selects_family_boards(self): + s = sel(['hw/bsp/rp2040/family.cmake']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico', s['boards']) + self.assertEqual(s['boards']['raspberry_pi_pico'], 'all') + self.assertNotIn('stm32f407disco', s['boards']) + + def test_bsp_board_narrows_to_board(self): + s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) + self.assertFalse(s['full']) + self.assertEqual(list(s['boards'].keys()), ['raspberry_pi_pico']) + + def test_example_change_selects_that_example(self): + s = sel(['examples/device/cdc_msc/src/main.c']) + self.assertFalse(s['full']) + self.assertEqual(s['boards']['raspberry_pi_pico'], ['device/cdc_msc']) + + def test_core_common_is_full(self): + for f in ['src/tusb.c', 'src/common/tusb_fifo.c', 'src/osal/osal_freertos.h']: + self.assertTrue(sel([f])['full'], f) + + def test_board_test_example_is_full(self): + # board_test is the park/teardown firmware hil_test.py flashes on every board, + # not an unlisted example: a regression there must not skip the whole rig + for f in ['examples/device/board_test/src/main.c', + 'examples/device/board_test/CMakeLists.txt']: + self.assertTrue(sel([f])['full'], f) + + def test_harness_is_full(self): + # hw/mcu/ is no longer here: it resolves to families/boards via mcu_families() + # instead of forcing full - see TestMcuHilRule + for f in ['test/hil/hil_test.py', '.github/workflows/build.yml']: + self.assertTrue(sel([f])['full'], f) + + def test_mixed_roles_no_pruning(self): + s = sel(['src/device/usbd.c', 'src/host/usbh.c']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico2', s['boards']) + self.assertIn('stm32f407disco', s['boards']) + + def test_cmakelists_and_requirements_are_full(self): + for f in ['src/CMakeLists.txt', 'examples/CMakeLists.txt', + 'examples/device/CMakeLists.txt', 'test/hil/requirements.txt']: + self.assertTrue(sel([f])['full'], f) + + def test_docs_txt_is_noncode(self): + s = sel(['docs/info/changelog.txt']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + +class TestArgsEmission(unittest.TestCase): + def test_args_for_scoped_selection(self): + s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + args = ci_select.selection_args(s, ROSTERS) + a = args['tinyusb.json'] + self.assertIn('-b raspberry_pi_pico', a) + self.assertNotIn('stm32f407disco', a) + self.assertIn('-bt raspberry_pi_pico:', a) # device-only subset of a device+host board + + def test_args_full_is_empty(self): + s = sel(['tools/random_new_script.py']) + self.assertEqual(ci_select.selection_args(s, ROSTERS), {'tinyusb.json': ''}) + + def test_args_all_board_gets_bare_b(self): + s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) + a = ci_select.selection_args(s, ROSTERS)['tinyusb.json'] + self.assertIn('-b raspberry_pi_pico', a) + self.assertNotIn('-bt', a) + + def test_args_by_flasher_splits_esp_from_the_rest(self): + s = sel(['src/device/usbd.c']) + per = ci_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json'] + self.assertIn('espressif_s3_devkitm', per['esptool']) + self.assertIn('raspberry_pi_pico', per['openocd']) + self.assertNotIn('espressif_s3_devkitm', per.get('openocd', '') + per.get('jlink', '')) + + def test_args_by_flasher_omits_a_flasher_with_no_selected_board(self): + # the esp CI leg must see no args at all here, not a filter matching zero boards + s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) + per = ci_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json'] + self.assertEqual(per, {'openocd': '-b raspberry_pi_pico'}) + + def test_args_by_flasher_full_is_empty(self): + s = sel(['tools/random_new_script.py']) + self.assertEqual(ci_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}}) + + def test_cli_diff_file(self): + import subprocess, tempfile, json as j + with tempfile.NamedTemporaryFile('w', suffix='.txt', delete=False) as f: + f.write('src/class/cdc/cdc_device.c\n') + path = f.name + r = subprocess.run([sys.executable, os.path.join(REPO, 'tools/ci_select.py'), + '--diff-file', path, os.path.join(REPO, 'test/hil/tinyusb.json')], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + out = j.loads(r.stdout) + self.assertFalse(out['full']) + self.assertIn('tinyusb.json', out['args']) + self.assertTrue(any('cdc_device' in line for line in out['reasons'])) + # A core-class diff must select boards THROUGH THE CLI: the in-process tests + # inject their own repo root, so only this subprocess path catches a broken + # repo_root derivation -- which once made every repo-relative glob match + # nothing and turned this exact diff into a silent full-HIL skip. + self.assertTrue(out['boards'], + 'CLI selected zero boards for a src/class change: repo_root broken?') + os.unlink(path) + + +class TestRealRosterPortFamilies(unittest.TestCase): + """Regression for port_families() missing espressif's dwc2 reference, which + lives in a component CMakeLists.txt rather than family.cmake/family.mk.""" + def test_dwc2_change_selects_espressif_boards(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = ci_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertIn(board, s['boards']) + + +class TestOptionGatedPort(unittest.TestCase): + """Regression: family_support.cmake compiles some ports from a build option + (MAX3421_HOST=1 -> hcd_max3421.c), so a board's family file never names them.""" + # host-side option board (max3421 as host controller), off any max3421 family + OPT_ROSTER = [('test/hil/opt.json', [ + {'name': 'fake_dual_board', 'uid': 'o1', 'flasher': {'name': 'jlink'}, + 'variant': [{'name': 'fake_dual_board', 'defines': ['MAX3421_HOST=1']}], + 'tests': {'device': True, 'host': False, 'dual': True}}, + {'name': 'fake_host_board', 'uid': 'o2', 'flasher': {'name': 'jlink'}, + 'variant': [{'name': 'fake_host_board', 'flags': '-DMAX3421_HOST=1'}], + 'tests': {'device': False, 'host': True, 'dual': False}}, + {'name': 'fake_off_board', 'uid': 'o3', 'flasher': {'name': 'jlink'}, + 'variant': [{'name': 'fake_off_board', 'defines': ['MAX3421_HOST=0']}], + 'tests': {'device': True, 'host': True, 'dual': True}}, + ])] + + def test_real_roster_max3421_selects_option_board(self): + boards = on_roster(self, 'metro_m4_express') + s = ci_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertIn(board, s['boards']) + + def test_option_selects_via_defines_and_flags(self): + s = ci_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, self.OPT_ROSTER) + self.assertFalse(s['full']) + self.assertIn('fake_dual_board', s['boards']) # variant defines + self.assertIn('fake_host_board', s['boards']) # variant flags + self.assertNotIn('fake_off_board', s['boards']) # variant defines, but =0 + + def test_device_role_port_does_not_pull_host_only_option_board(self): + s = ci_select.classify(['src/portable/analog/max3421/dcd_max3421.c'], REPO, self.OPT_ROSTER) + self.assertFalse(s['full']) + self.assertNotIn('fake_host_board', s['boards']) # host-only board, device change + self.assertIn('fake_dual_board', s['boards']) # device-capable option board + + def test_gates_parsed_from_family_support(self): + self.assertEqual(ci_select.port_option_gates(REPO).get('analog/max3421'), + {'MAX3421_HOST'}) + + def test_board_cmake_option_counts(self): + """A board can enable a gated port in its own BSP rather than via the roster + (hw/bsp/espressif/boards/*/board.cmake -> set(MAX3421_HOST 1)); board_options() + must see those too, or such a board joining the roster is silently dropped.""" + self.assertIn('MAX3421_HOST', + ci_select.bsp_board_options('adafruit_feather_esp32s3', REPO)) + self.assertIn('CFG_TUH_RPI_PIO_USB', + ci_select.bsp_board_options('adafruit_fruit_jam', REPO)) + # commented-out `# set(MAX3421_HOST 1)` must not count + self.assertNotIn('MAX3421_HOST', + ci_select.bsp_board_options('feather_nrf52840_express', REPO)) + + def test_board_cmake_option_selects_off_family_board(self): + # adafruit_feather_esp32s3 is not on any rig roster; stand it in as one to + # prove the BSP-sourced option alone pulls a max3421 change onto the board + roster = [('test/hil/opt.json', [ + {'name': 'adafruit_feather_esp32s3', 'uid': 'o1', 'flasher': {'name': 'esptool'}, + 'tests': {'device': False, 'host': True, 'dual': False}}])] + s = ci_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster) + self.assertFalse(s['full']) + self.assertIn('adafruit_feather_esp32s3', s['boards']) + + def test_board_mk_option_is_ignored(self): + """Make-only options must not select: HIL CI builds with CMake exclusively, so + hw/bsp/nrf/boards/nrf5340dk/board.mk's MAX3421_HOST compiles nothing here.""" + roster = [('test/hil/opt.json', [ + {'name': 'nrf5340dk', 'uid': 'o1', 'flasher': {'name': 'jlink'}, + 'tests': {'device': False, 'host': True, 'dual': False}}])] + s = ci_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + +class TestPortFamiliesCmakeOnly(unittest.TestCase): + """port_families() is CMake-only (HIL CI never builds with Make) and matches on + 'port_dir/' so a port dir is not a prefix of a sibling.""" + def test_make_only_family_is_not_a_family(self): + # hw/bsp/pic32mz has family.mk but no family.cmake + self.assertEqual(ci_select.port_families('microchip/pic32mz', REPO), set()) + + def test_prefix_port_does_not_inherit_sibling_families(self): + # bare-substring matching let 'microchip/pic' match '.../microchip/pic32mz/...' + self.assertEqual(ci_select.port_families('microchip/pic', REPO), set()) + + def test_make_only_port_contributes_nothing(self): + s = sel(['src/portable/microchip/pic32mz/dcd_pic32mz.c']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons']) + + def test_cmake_families_still_found(self): + self.assertEqual(ci_select.port_families('raspberrypi/rp2040', REPO), {'rp2040'}) + self.assertIn('stm32f4', ci_select.port_families('synopsys/dwc2', REPO)) + + +class TestPortFamiliesCoverage(unittest.TestCase): + """Systematic guard: every real dcd_*/hcd_* port directory should map to at + least one board family, so a future family.cmake/CMakeLists.txt layout that + port_families() doesn't scan fails loudly instead of silently dropping boards + (as espressif's dwc2 reference did - see TestRealRosterPortFamilies).""" + # Ports with no board family: not a bug, just not wired into any rig board. + # Add here (with a reason) only if port_families() legitimately can't find one. + # A port listed here contributes NOTHING on either axis (empty means empty), so + # this list is the tripwire: a port that stops resolving must show up as a test + # failure, not as a PR that quietly builds and tests nothing. + NO_FAMILY = { + 'template', # reference/example port, not built by any board + # hw/bsp/pic32mz has family.mk only (no family.cmake), and port_families() + # is CMake-only because HIL CI builds every board with CMake - so this port + # is compiled for no HIL board. + 'microchip/pic32mz', + 'microchip/pic', # same: only ever referenced from pic32mz's family.mk + } + + @staticmethod + def _dcd_hcd_ports(): + portable_root = os.path.join(REPO, 'src/portable') + ports = [] + for entry in sorted(os.listdir(portable_root)): + d = os.path.join(portable_root, entry) + if not os.path.isdir(d): + continue + if glob.glob(os.path.join(d, 'dcd_*.c')) or glob.glob(os.path.join(d, 'hcd_*.c')): + ports.append(entry) + continue + for sub in sorted(os.listdir(d)): + sd = os.path.join(d, sub) + if os.path.isdir(sd) and (glob.glob(os.path.join(sd, 'dcd_*.c')) or + glob.glob(os.path.join(sd, 'hcd_*.c'))): + ports.append(f'{entry}/{sub}') + return ports + + def test_every_port_maps_to_a_family(self): + ports = self._dcd_hcd_ports() + self.assertTrue(ports) # sanity: the scan itself found something + for port in ports: + if port in self.NO_FAMILY: + continue + fams = ci_select.port_families(port, REPO) + self.assertTrue(fams, f'{port}: no family references this port ' + f'(port_families() scan gap, or add to NO_FAMILY)') + + +class TestRealRosterOnlyListTests(unittest.TestCase): + """Regression for roster-only-list tests (e.g. espressif's hid_composite_freertos) + being invisible to the selector because it only knew the shared hil_util lists.""" + def test_only_list_example_change_selects_it(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = ci_select.classify(['examples/device/hid_composite_freertos/src/main.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertEqual(s['boards'][board], ['device/hid_composite_freertos']) + + def test_class_change_includes_only_list_boards(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = ci_select.classify(['src/class/hid/hid_device.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertIn(board, s['boards']) + + +class TestPortAndCoreRoleUseExtras(unittest.TestCase): + """Regression: the port rule and core-role rule must thread the roster-only + test universe (extras) the same way the class rule already does, so a DCD + or device-stack change doesn't silently drop espressif's only-list tests + (e.g. hid_composite_freertos) that aren't in the shared device_tests list.""" + def test_dcd_change_includes_only_list_test(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = ci_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + tests = s['boards'][board] + self.assertIn('device/hid_composite_freertos', tests) + self.assertIn('device/cdc_msc_freertos', tests) + self.assertIn('device/audio_test_freertos', tests) + self.assertIn('device/usbtest', tests) + + def test_core_device_change_includes_only_list_test(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = ci_select.classify(['src/device/usbd.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + tests = s['boards'][board] + self.assertIn('device/hid_composite_freertos', tests) + self.assertIn('device/cdc_msc_freertos', tests) + self.assertIn('device/audio_test_freertos', tests) + self.assertIn('device/usbtest', tests) + + def test_host_change_does_not_leak_device_only_list_test(self): + s = ci_select.classify(['src/host/usbh.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board, tests in s['boards'].items(): + if tests == 'all': + continue + self.assertNotIn('device/hid_composite_freertos', tests, board) + + +class TestFamilies(unittest.TestCase): + """`families` exists for consumers that build (not just test) the diff: most + families have no rig board, so `boards` alone would compile nothing for them.""" + def test_off_rig_port_still_reports_family(self): + s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) # no same7x board on the rig + self.assertEqual(s['families'], ['same7x']) + + def test_port_families_are_reported(self): + s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + self.assertIn('rp2040', s['families']) + + def test_bsp_family_and_board_report_family(self): + self.assertEqual(sel(['hw/bsp/rp2040/family.cmake'])['families'], ['rp2040']) + self.assertEqual(sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h'])['families'], + ['rp2040']) + + def test_docs_only_has_no_families(self): + self.assertEqual(sel(['docs/info/contributing.rst'])['families'], []) + + def test_full_selection_still_reports_families(self): + """A full-matrix file must not hide the families of the other changed files: + consumers that build from `families` (e.g. /pre-pr) ignore `boards` when full.""" + s = sel(['src/common/tusb_fifo.c', 'src/portable/microchip/samx7x/dcd_samx7x.c']) + self.assertTrue(s['full']) + self.assertIn('same7x', s['families']) + # full stays full: every roster board, and no args to narrow the run + self.assertEqual(set(s['boards']), {b['name'] for b in ROSTER}) + self.assertTrue(all(v == 'all' for v in s['boards'].values())) + self.assertEqual(ci_select.selection_args(s, ROSTERS), {'tinyusb.json': ''}) + self.assertEqual(ci_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}}) + + def test_family_order_does_not_matter(self): + # same as above with the full-matrix file last (was the only order that worked) + s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c', 'src/common/tusb_fifo.c']) + self.assertTrue(s['full']) + self.assertIn('same7x', s['families']) + + +class TestGitDiffArgv(unittest.TestCase): + def test_diff_disables_rename_detection(self): + """Without --no-renames git reports only a rename's destination, so moving an + HIL-relevant file to a non-code path would be classified as non-code only.""" + self.assertIn('--no-renames', ci_select.GIT_DIFF_ARGV) + + +class TestPortWithoutFamilyContributesNothing(unittest.TestCase): + """A port dir no family file references contributes nothing on BOTH axes (the + maintainer's empty-means-empty ruling): nothing compiles the file, so there is + nothing to run. Forcing the full 30-board rig here bought no coverage - the build + walk answered the identical condition with zero families for the same path.""" + def test_unreferenced_port_contributes_nothing(self): + orig = ci_select.port_families + ci_select.port_families = lambda port_dir, repo_root: set() + try: + s = sel(['src/portable/vendor/newip/dcd_newip.c']) + b = ci_select.classify_build(['src/portable/vendor/newip/dcd_newip.c'], REPO) + finally: + ci_select.port_families = orig + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons']) + self.assertFalse(b['full']) + self.assertEqual(b['families'], []) + + +class TestOpenocdVidPid(unittest.TestCase): + """The roster's optional flasher `vid_pid` field (openocd-verbatim, e.g. + "0x1a86 0x8010", more pairs appended) pins openocd's probe discovery so it + never opens foreign usbfs nodes. It must be emitted BEFORE the args: the + rescue cfgs run `init` internally (rp2350-rescue.cfg errors on any + config-stage command after its init; rp2040.cfg under RESCUE scans before a + trailing flag is even parsed), and no rig cfg sets a competing list + (the 2026-08-10 convoy mechanism).""" + + def test_vid_pid_flag_precedes_args(self): + cmd = hil_flash._openocd_cmd_base( + {'uid': 'S1', 'args': '-f target/wch-riscv.cfg', 'vid_pid': '0x1a86 0x8010'}) + self.assertIn('-c "adapter usb vid_pid 0x1a86 0x8010" -f target/wch-riscv.cfg', cmd) + self.assertTrue(cmd.endswith('-f target/wch-riscv.cfg'), cmd) + + def test_rescue_cfg_command_keeps_vid_pid_before_init(self): + """rescue_openocd swaps the target cfg for one that runs `init` internally; + a vid_pid flag after the args would error there (rp2350) or be skipped + (rp2040) -- in exactly the wedged-rig scenario the pin exists for.""" + flasher = {'name': 'openocd', 'uid': 'S1', 'vid_pid': '0x2e8a 0x000c', + 'args': '-c "set RESCUE 1" -f target/rp2040.cfg'} + cmd = hil_flash._openocd_cmd_base(flasher) + self.assertLess(cmd.index('adapter usb vid_pid'), cmd.index('-f target/'), cmd) + + def test_vid_pid_multiple_pairs(self): + cmd = hil_flash._openocd_cmd_base( + {'uid': 'S1', 'args': '-f i.cfg', 'vid_pid': '0x2e8a 0x000c 0x2e8a 0x000d'}) + self.assertIn('-c "adapter usb vid_pid 0x2e8a 0x000c 0x2e8a 0x000d"', cmd) + + def test_no_field_no_flag_but_warns(self): + # the roster lint only covers the committed rosters; a dev PC's local.json entry + # without the field must at least say what it is giving up -- on STDERR, since + # hil_test captures stdout per test and would swallow it on a passing run + import io + from contextlib import redirect_stderr + hil_flash._VID_PID_WARNED.discard('S-warn') + cap = io.StringIO() + with redirect_stderr(cap): + cmd = hil_flash._openocd_cmd_base({'uid': 'S-warn', 'args': '-f i.cfg'}) + self.assertNotIn('vid_pid', cmd) + self.assertIn('vid_pid', cap.getvalue()) + + def test_roster_openocd_entries_all_pin_vid_pid(self): + # every openocd probe on the rig has a known VID/PID; a new entry without the + # pin silently reintroduces open-everything discovery + for path, board in roster_flashers(): + f = board['flasher'] + # tinyusb.json only: hfp.json is the hifiphile rig owner's file, and a + # blocking repo-wide lint over someone else's roster would red every PR the + # moment they add an openocd board (hil_flash treats the field as optional) + if f['name'] == 'openocd' and path.endswith('tinyusb.json'): + self.assertIn('vid_pid', f, + f"{path}: {board['name']} openocd flasher lacks vid_pid") + self.assertNotIn('vid_pid', f.get('args', ''), + f"{path}: {board['name']} packs vid_pid into args; use the field") + + +class TestRosterFlashersDispatch(unittest.TestCase): + """hil_test and hil_pool_check resolve a board's flasher with a bare + getattr(hil_flash, f'flash_{name}'), and hil_test does it inside a redirect_stdout — + so a renamed or typo'd roster name raises an AttributeError whose output is swallowed, + with nothing pointing at the roster as the thing to edit. Renaming a flash_*/reset_* + pair without updating every roster must fail here instead.""" + + def test_flash_and_reset_exist_for_every_roster_flasher(self): + for path, board in roster_flashers(): + name = board['flasher']['name'].lower() + for fn in (f'flash_{name}', f'reset_{name}'): + self.assertTrue(callable(getattr(hil_flash, fn, None)), + f'{path}: {board["name"]} uses flasher "{name}" ' + f'but hil_flash.{fn} does not exist') + + def test_firmware_suffix_known_for_every_roster_flasher(self): + """find_firmware falls back to accepting .elf-or-.bin when a flasher is missing + from FLASHER_SUFFIX, silently restoring the mismatch that map exists to catch.""" + for path, board in roster_flashers(): + name = board['flasher']['name'].lower() + self.assertIn(name, hil_flash.FLASHER_SUFFIX, + f'{path}: {board["name"]} uses flasher "{name}" ' + f'with no hil_flash.FLASHER_SUFFIX entry') + + +class FlasherRecoverEntry(unittest.TestCase): + """Optional roster key: a SECOND flasher used only to deliver recovery while a usbfs + node is poisoned. Boards whose primary flasher cannot get past a convoy (jlink, + stlink, lm4flash) name an openocd entry here instead of changing how they are + normally flashed.""" + + def test_recover_flasher_prefers_the_optional_entry(self): + prim = {'name': 'jlink', 'uid': 'X', 'args': '-device MIMXRT1064xxx6A'} + rec = {'name': 'openocd', 'uid': 'X', 'args': '-f interface/jlink.cfg -f target/foo.cfg'} + self.assertEqual(hil_flash.recover_flasher({'flasher': prim, 'flasher_recover': rec}), rec) + self.assertEqual(hil_flash.recover_flasher({'flasher': prim}), prim) + + def test_openocd_over_jlink_is_convoy_safe_without_a_pin(self): + """libjaylink discovery returns early unless idVendor == 0x1366 (SEGGER) and the PID + is in its table, and only THEN calls libusb_open (discovery_usb.c) -- it never opens + a foreign node. `adapter usb vid_pid` is a no-op for this driver: jlink.c reads + adapter_serial / usb address / usb location, never the vid/pid.""" + self.assertTrue(hil_flash.convoy_safe( + {'name': 'openocd', 'args': '-f interface/jlink.cfg -f target/stm32f4x.cfg'})) + + def test_openocd_with_neither_a_pin_nor_jlink_is_not_safe(self): + self.assertFalse(hil_flash.convoy_safe( + {'name': 'openocd', 'args': '-f interface/stlink.cfg -f target/stm32h7x.cfg'})) + + def test_the_existing_rules_are_unchanged(self): + self.assertTrue(hil_flash.convoy_safe( + {'name': 'openocd', 'vid_pid': '0x2e8a 0x000c', 'args': '-f interface/cmsis-dap.cfg'})) + self.assertFalse(hil_flash.convoy_safe({'name': 'jlink', 'uid': 'X'})) + self.assertTrue(hil_flash.convoy_safe({'name': 'esptool'})) + + +class TestModuleMove(unittest.TestCase): + def test_repo_root_guard(self): + # __file__-derived root: moving the module without re-deriving the parent + # count re-points every scan at the wrong tree (it happened once already) + self.assertTrue(os.path.isdir(os.path.join(ci_select._REPO_ROOT, 'src'))) + self.assertTrue(os.path.isdir(os.path.join(ci_select._REPO_ROOT, 'hw', 'bsp'))) + self.assertEqual(os.path.realpath(ci_select._REPO_ROOT), os.path.realpath(REPO)) + + +class TestPathFamilies(unittest.TestCase): + def test_port_wrapper_unchanged(self): + self.assertEqual(ci_select.port_families('raspberrypi/rp2040', REPO), {'rp2040'}) + self.assertIn('stm32f4', ci_select.port_families('synopsys/dwc2', REPO)) + + def test_boundary_without_trailing_slash(self): + # hw/bsp/nrf/family.cmake writes `${TOP}/hw/mcu/nordic/nrfx` — no trailing + # slash; the match must accept a directory-boundary end-of-token + self.assertEqual(ci_select.path_families('hw/mcu/nordic/nrfx', REPO), {'nrf'}) + + def test_boundary_rejects_prefix_sibling(self): + # 'microchip/pic' must not inherit pic32mz's references (and pic32mz itself + # is family.mk-only, which the CMake-only scan never reads) + self.assertEqual(ci_select.port_families('microchip/pic', REPO), set()) + self.assertEqual(ci_select.port_families('microchip/pic32mz', REPO), set()) + + def test_mcu_families_prefix_walk(self): + self.assertEqual(ci_select.mcu_families('hw/mcu/nordic/nrf5x/nrf_clock.h', REPO), {'nrf'}) + self.assertEqual(ci_select.mcu_families('hw/mcu/dialog/da1469x/x.h', REPO), {'da1469x'}) + self.assertEqual(ci_select.mcu_families('hw/mcu/no_such_vendor/x.c', REPO), set()) + + +class TestMcuHilRule(unittest.TestCase): + def test_mcu_no_longer_forces_full(self): + s = ci_select.classify(['hw/mcu/nordic/nrf5x/nrf_clock.h'], REPO, ROSTERS) + self.assertFalse(s['full']) + self.assertIn('nrf', s['families']) # recorded even with no nrf rig board + + def test_mcu_selects_family_boards(self): + got = on_roster(self, 'feather_nrf52840_express', 'pca10056', 'pca10095') + s = ci_select.classify(['hw/mcu/nordic/nrf5x/nrf_clock.h'], REPO, real_rosters()) + self.assertFalse(s['full']) + for b in got: + self.assertIn(b, s['boards']) + + def test_unresolved_mcu_path_selects_nothing(self): + # empty means empty (maintainer ruling): if no family's build references the + # path, no build consumes the change - there is nothing to compile or run. + # test_tracked_mcu_vendors_resolve is the drift guard for a real vendor dir + s = ci_select.classify(['hw/mcu/no_such_vendor/x.c'], REPO, ROSTERS) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + self.assertEqual(s['families'], []) + + +class TestOrphanInvariant(unittest.TestCase): + ALLOW = {'microchip/pic', 'microchip/pic32mz'} # spec: known orphans, CMake builds neither + + def test_every_port_resolves_to_a_family(self): + for d in sorted(glob.glob(os.path.join(REPO, 'src/portable/*/*'))): + if not os.path.isdir(d): + continue + port = os.path.relpath(d, os.path.join(REPO, 'src/portable')).replace(os.sep, '/') + fams = ci_select.port_families(port, REPO) + if port in self.ALLOW: + self.assertEqual(fams, set(), f'{port}: no longer an orphan - drop it from ALLOW') + else: + self.assertTrue(fams, f'{port}: no family.cmake references it - wire it up or allowlist it') + + def test_tracked_mcu_vendors_resolve(self): + import subprocess as sp + r = sp.run(['git', 'ls-files', 'hw/mcu'], cwd=REPO, capture_output=True, text=True) + if r.returncode != 0: + self.skipTest('not a git checkout') + vendors = sorted({'/'.join(p.split('/')[:3]) for p in r.stdout.split()}) + for v in vendors: + self.assertTrue(ci_select.mcu_families(v + '/x.c', REPO), f'{v}: resolves to no family') + + # hw/bsp families ci_set_matrix's family_list does not map to any toolchain. Master + # gave a PR touching one of these no compile coverage either - none of the other 64 + # families compiles same7x's board.h - so this is not new. What IS new is that the + # gap used to be masked by a full matrix and is now the whole answer, which is why + # ci_set_matrix treats a selection that intersects family_list to NOTHING as + # unusable (UNSCOPED -> full matrix) rather than emitting an all-empty one. + # espressif is here because hil-build-esp builds its boards by name rather than by + # family - though only on hathach/tinyusb: that job is gated on repository_owner, + # so on a fork an espressif-only PR builds nowhere. + UNBUILT_FAMILIES = {'cxd56', 'efm32', 'espressif', 'f1c100s', 'pic32mz', 'py32f0', + 'same7x'} + + def test_every_bsp_family_is_in_the_ci_matrix(self): + sys.path.insert(0, os.path.join(REPO, '.github/scripts')) + import ci_set_matrix + fams = set(ci_select.all_bsp_families(REPO)) + self.assertEqual(fams - set(ci_set_matrix.family_list), self.UNBUILT_FAMILIES, + 'a hw/bsp family that no toolchain in ci_set_matrix.family_list ' + 'builds: a PR touching only it now selects zero build legs. Wire ' + 'it into family_list, or add it here with a reason.') + + def test_every_get_deps_family_token_resolves_or_is_a_known_alias(self): + """Same drift guard, dep side. A token naming no hw/bsp dir makes the entry + unreachable for its family in get_deps.py itself (`f in entry[2].split()`), and + makes a bump of it select nothing here. The four known ones are pinned; a fifth + appearing is a real bug in get_deps.py, not something to swallow.""" + sys.path.insert(0, os.path.join(REPO, 'tools')) + import get_deps + fams = set(ci_select.all_bsp_families(REPO)) + stale = {} + for name, d in (('deps_mandatory', get_deps.deps_mandatory), + ('deps_optional', get_deps.deps_optional)): + for path, entry in d.items(): + for tok in str(entry[2]).split(): + if tok != 'all' and tok not in fams: + stale.setdefault(tok, []).append(f'{name}[{path}]') + # subset, not equality: correcting a token in get_deps.py (fc100s -> f1c100s) + # should be a one-file change, while a NEW unmappable token - which force-fulls + # every get_deps edit that touches its entry - has to be a deliberate act + self.assertFalse(set(stale) - set(ci_select._DEPS_ALIAS_TOKENS), + f'get_deps family tokens naming no hw/bsp dir: ' + f'{ {k: v for k, v in stale.items() if k not in ci_select._DEPS_ALIAS_TOKENS} }') + + +class TestRostersDoNotOverlap(unittest.TestCase): + """sel['boards'] is one map across every roster, so a board listed in TWO rosters + with different test lists would get the union - and hil_test.py on the rig that + only runs half of them would be handed a -t it has no fixture for. No overlap + exists today; this is the tripwire for the day one is added.""" + + def test_no_board_name_is_in_two_rosters(self): + seen = {} + for name in ('tinyusb.json', 'hfp.json'): + cfg = json.load(open(os.path.join(REPO, 'test/hil', name))) + for b in cfg['boards']: + if b['name'] in seen: + self.assertEqual( + seen[b['name']], b.get('tests'), + f"{b['name']}: on two rosters with different test lists - " + f"selection_args must then filter per roster, not from the union") + seen[b['name']] = b.get('tests') + + +class TestLibRule(unittest.TestCase): + """lib/** is not a full-matrix path: only the examples that build the lib need it.""" + + def b(self, files): + return ci_select.classify_build(files, REPO) + + def test_lib_examples_ground_truth(self): + self.assertEqual(ci_select.lib_examples('embedded-cli', REPO), + {'host/msc_file_explorer', 'host/msc_file_explorer_freertos'}) + self.assertEqual(ci_select.lib_examples('networking', REPO), + {'device/net_lwip_webserver'}) + # only family_support.cmake's LOGGER=rtt plumbing names it, and no CI example + # build turns that on - the scan is per-example on purpose + self.assertEqual(ci_select.lib_examples('SEGGER_RTT', REPO), set()) + self.assertEqual(ci_select.lib_examples('rt-thread', REPO), set()) + + def test_lib_examples_matches_at_a_directory_boundary(self): + # 'lib/net' must not inherit lib/networking's example + self.assertEqual(ci_select.lib_examples('net', REPO), set()) + + def test_build_lib_selects_only_the_using_examples(self): + s = self.b(['lib/embedded-cli/embedded_cli.h']) + self.assertFalse(s['full']) + self.assertTrue(s['families']) + want = {'host/msc_file_explorer', 'host/msc_file_explorer_freertos'} + mapped = set() + for fam, exs in s['family_examples'].items(): + self.assertTrue(set(exs) <= want, f'{fam}: {exs}') + mapped |= set(exs) + self.assertEqual(mapped, want) + + def test_build_lib_nobody_builds_selects_nothing(self): + s = self.b(['lib/SEGGER_RTT/RTT/SEGGER_RTT.c']) + self.assertFalse(s['full']) + self.assertEqual(s['families'], []) + self.assertEqual(s['family_examples'], {}) + + def test_hil_lib_selects_the_using_tests(self): + s = sel(['lib/embedded-cli/embedded_cli.h']) + self.assertFalse(s['full']) + want = {'host/msc_file_explorer', 'host/msc_file_explorer_freertos'} + self.assertEqual(set(s['boards']['raspberry_pi_pico']), want) + self.assertEqual(set(s['boards']['raspberry_pi_pico2']), want) + # device-only board and the only-list board run neither test + self.assertNotIn('stm32f407disco', s['boards']) + self.assertNotIn('espressif_s3_devkitm', s['boards']) + + def test_hil_lib_used_only_by_a_disabled_test_selects_nothing(self): + # device/net_lwip_webserver is commented out of hil_util.device_tests, so the + # intersection with the HIL universe is empty + s = sel(['lib/networking/dhserver.c']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + def test_hil_lib_nobody_builds_selects_nothing(self): + s = sel(['lib/SEGGER_RTT/RTT/SEGGER_RTT.c']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + +# A miniature get_deps.py: the module shape the parser must cope with (imports, +# both dep dicts, the derived deps_all, a function) without the real 300-entry file. +_GD_BASE = """#!/usr/bin/env python3 +import argparse + +deps_mandatory = { + 'lib/fatfs': ['https://github.com/abbrev/fatfs.git', 'aaa', 'all'], +} + +deps_optional = { + 'hw/mcu/st/cmsis_device_f4': ['https://github.com/x/f4.git', 'bbb', 'stm32f4 stm32f7'], + 'hw/mcu/nordic/nrfx': ['https://github.com/x/nrfx.git', 'ccc', 'nrf'], +} + +deps_all = {**deps_mandatory, **deps_optional} + + +def main(): + return 1 +""" + + +class TestGetDepsChangedFamilies(unittest.TestCase): + """Pure text-in, families-out: no git, no exec of the parsed module.""" + + def f(self, head, base=_GD_BASE): + return ci_select.get_deps_changed_families(base, head, REPO) + + def test_no_change_selects_nothing(self): + self.assertEqual(self.f(_GD_BASE), set()) + + def test_comment_only_change_selects_nothing(self): + self.assertEqual(self.f(_GD_BASE.replace('import argparse', + 'import argparse # noqa')), set()) + + def test_optional_commit_bump_selects_its_families(self): + self.assertEqual(self.f(_GD_BASE.replace("'bbb'", "'bbb2'")), + {'stm32f4', 'stm32f7'}) + + def test_mandatory_all_entry_is_full(self): + self.assertIsNone(self.f(_GD_BASE.replace("'aaa'", "'aaa2'"))) + + def test_logic_change_is_full(self): + self.assertIsNone(self.f(_GD_BASE.replace('return 1', 'return 2'))) + + def test_unparseable_text_is_full(self): + self.assertIsNone(self.f('def broken(:\n')) + + def test_unresolvable_token_is_full(self): + # a changed entry we cannot map to a family is NOT "nothing changed": reading it + # that way empties the whole build matrix for a dep bump. Fall open instead - + # even when a sibling token does resolve, because the unmapped one may be the + # family that actually needed the new revision + base = _GD_BASE.replace("'ccc', 'nrf'", "'ccc', 'zz_gone samd5x_e5x'") + self.assertIsNone(self.f(base.replace("'ccc'", "'ccc2'"), base)) + base = _GD_BASE.replace("'ccc', 'nrf'", "'ccc', 'zz_gone'") + self.assertIsNone(self.f(base.replace("'ccc'", "'ccc2'"), base)) + + def test_family_token_change_unions_both_sides(self): + # the family list itself edited: both sides contribute + head = _GD_BASE.replace("'ccc', 'nrf'", "'ccc', 'rp2040 samd5x_e5x'") + self.assertEqual(self.f(head), {'nrf', 'rp2040', 'samd5x_e5x'}) + + def test_known_alias_tokens_select_nothing(self): + # the tokens in _DEPS_ALIAS_TOKENS name no hw/bsp dir: either a pre-rename + # spelling sitting beside the current name in the same entry, or a family with + # no boards in the tree. Changing one selects nothing rather than force-fulling + # every get_deps edit that touches its entry. + base = _GD_BASE.replace("'ccc', 'nrf'", "'ccc', 'stm32l5'") + self.assertEqual(self.f(base.replace("'ccc'", "'ccc2'"), base), set()) + + def test_moving_an_entry_between_the_two_dicts_is_seen(self): + # value untouched, dict changed: mandatory deps are fetched for every family, so + # demoting one stops families fetching it. Merging the dicts before diffing (or + # comparing the ast dump of deps_all) hides this completely. + head = _GD_BASE.replace( + " 'hw/mcu/nordic/nrfx': ['https://github.com/x/nrfx.git', 'ccc', 'nrf'],\n", '') + head = head.replace( + "deps_mandatory = {\n", + "deps_mandatory = {\n 'hw/mcu/nordic/nrfx': ['https://github.com/x/nrfx.git', 'ccc', 'nrf'],\n") + self.assertEqual(self.f(head), {'nrf'}) + + def test_added_entry_selects_its_families(self): + head = _GD_BASE.replace( + "deps_optional = {\n", + "deps_optional = {\n 'hw/mcu/x': ['https://github.com/x/x.git', 'ddd', 'rp2040'],\n") + self.assertEqual(self.f(head), {'rp2040'}) + + def test_removed_entry_selects_its_base_side_families(self): + head = _GD_BASE.replace( + " 'hw/mcu/nordic/nrfx': ['https://github.com/x/nrfx.git', 'ccc', 'nrf'],\n", '') + self.assertEqual(self.f(head), {'nrf'}) + + def test_family_list_change_unions_both_sides(self): + head = _GD_BASE.replace("'stm32f4 stm32f7'", "'stm32f4 stm32h7'") + self.assertEqual(self.f(head), {'stm32f4', 'stm32f7', 'stm32h7'}) + + def test_real_get_deps_parses(self): + with open(os.path.join(REPO, 'tools/get_deps.py')) as f: + real = f.read() + self.assertEqual(ci_select.get_deps_changed_families(real, real, REPO), set()) + # a real optional entry bumped resolves to that entry's real family. The commit + # is read out of get_deps.py rather than pinned here - a routine dep bump must + # not fail this suite, and pinning a hash tests the tree, not the code + sys.path.insert(0, os.path.join(REPO, 'tools')) + import get_deps + commit, tokens = get_deps.deps_optional['hw/mcu/nordic/nrfx'][1:3] + bumped = real.replace(commit, '0' * len(commit)) + self.assertNotEqual(bumped, real) + self.assertEqual(ci_select.get_deps_changed_families(real, bumped, REPO), + set(tokens.split())) + + +class TestGetDepsRule(unittest.TestCase): + """tools/get_deps.py: the changed dep entries' families, or full when unknowable.""" + + def test_build_selects_the_changed_families(self): + s = ci_select.classify_build(['tools/get_deps.py'], REPO, + get_deps_families={'stm32f4'}) + self.assertFalse(s['full']) + self.assertEqual(s['families'], ['stm32f4']) + self.assertNotIn('stm32f4', s['family_examples']) # every example it builds + + def test_build_without_a_base_is_full(self): + # --diff-file mode has no git and so no base content: fail open + self.assertTrue(ci_select.classify_build(['tools/get_deps.py'], REPO)['full']) + + def test_build_no_dep_entry_changed_selects_nothing(self): + s = ci_select.classify_build(['tools/get_deps.py'], REPO, get_deps_families=set()) + self.assertFalse(s['full']) + self.assertEqual(s['families'], []) + + def test_hil_selects_the_changed_families_boards(self): + s = ci_select.classify(['tools/get_deps.py'], REPO, ROSTERS, + get_deps_families={'stm32f4'}) + self.assertFalse(s['full']) + self.assertEqual(list(s['boards']), ['stm32f407disco']) + self.assertEqual(s['families'], ['stm32f4']) + + def test_hil_without_a_base_is_full(self): + self.assertTrue(ci_select.classify(['tools/get_deps.py'], REPO, ROSTERS)['full']) + + def test_hil_no_dep_entry_changed_selects_nothing(self): + s = ci_select.classify(['tools/get_deps.py'], REPO, ROSTERS, get_deps_families=set()) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + def test_cli_diff_file_mode_is_full(self): + import tempfile, json as j + with tempfile.NamedTemporaryFile('w', suffix='.txt', delete=False) as f: + f.write('tools/get_deps.py\n') + path = f.name + r = subprocess.run([sys.executable, os.path.join(REPO, 'tools/ci_select.py'), + '--diff-file', path, os.path.join(REPO, 'test/hil/tinyusb.json')], + capture_output=True, text=True) + os.unlink(path) + self.assertEqual(r.returncode, 0, r.stderr) + out = j.loads(r.stdout) + self.assertTrue(out['full']) + self.assertTrue(out['build']['full']) + + +class TestGetDepsGitPlumbing(unittest.TestCase): + """--base mode: merge-base, the diff, and both blobs come from git, and only + tools/get_deps.py in the diff triggers the blob reads.""" + + HEAD = _GD_BASE.replace("'bbb'", "'bbb2'") + + def run_main(self, diff): + from unittest import mock + calls = [] + + def fake_run(argv, **kw): + calls.append(argv) + if argv[:2] == ['git', 'merge-base']: + out = 'MB123\n' + elif argv[:3] == ci_select.GIT_DIFF_ARGV[:3]: + out = diff + elif argv[:2] == ['git', 'show']: + out = _GD_BASE if argv[2].startswith('MB123:') else self.HEAD + else: + raise AssertionError(f'unexpected git call: {argv}') + return subprocess.CompletedProcess(argv, 0, stdout=out, stderr='') + + buf = io.StringIO() + argv = [sys.executable, '--base', 'origin/master'] + with mock.patch.object(ci_select.subprocess, 'run', fake_run), \ + mock.patch.object(sys, 'argv', argv), \ + contextlib.redirect_stdout(buf), contextlib.redirect_stderr(io.StringIO()): + ci_select.main() + return json.loads(buf.getvalue()), calls + + def test_base_mode_reads_the_merge_base_blob(self): + out, calls = self.run_main('tools/get_deps.py\n') + self.assertIn(['git', 'show', 'MB123:tools/get_deps.py'], calls) + self.assertIn(['git', 'show', 'HEAD:tools/get_deps.py'], calls) + self.assertFalse(out['build']['full']) + self.assertEqual(out['build']['families'], ['stm32f4', 'stm32f7']) + + def test_no_get_deps_in_the_diff_reads_no_blob(self): + out, calls = self.run_main('src/class/cdc/cdc_device.c\n') + self.assertFalse(any(c[:2] == ['git', 'show'] for c in calls)) + self.assertFalse(out['build']['full']) + + def test_git_failure_falls_open(self): + from unittest import mock + + def fake_run(argv, **kw): + if argv[:2] == ['git', 'show']: + raise subprocess.CalledProcessError(128, argv) + out = 'MB123\n' if argv[:2] == ['git', 'merge-base'] else 'tools/get_deps.py\n' + return subprocess.CompletedProcess(argv, 0, stdout=out, stderr='') + + buf = io.StringIO() + with mock.patch.object(ci_select.subprocess, 'run', fake_run), \ + mock.patch.object(sys, 'argv', [sys.executable, '--base', 'origin/master']), \ + contextlib.redirect_stdout(buf), contextlib.redirect_stderr(io.StringIO()): + ci_select.main() + self.assertTrue(json.loads(buf.getvalue())['build']['full']) + + +class TestBuildClassifier(unittest.TestCase): + def b(self, files): + return ci_select.classify_build(files, REPO) + + def test_noncode_and_test_hil_contribute_nothing(self): # rules 1, 2 + s = self.b(['docs/info/index.rst', 'README.rst', 'test/hil/hil_test.py', '.claude/skills/hil/SKILL.md']) + self.assertFalse(s['full']) + self.assertEqual(s['families'], []) + self.assertEqual(s['family_examples'], {}) + + def test_port_device_rule(self): # rule 3 + s = self.b(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + self.assertFalse(s['full']) + self.assertEqual(s['families'], ['rp2040']) + exs = s['family_examples']['rp2040'] + self.assertIn('device/cdc_msc', exs) + self.assertFalse(any(e.startswith(('host/', 'typec/')) for e in exs)) + # dual inclusion asserted on the pure role helper: whether a dual example + # survives Task 4's buildability pruning depends on the environment-gated + # CI board pick, so the classifier-output assertion must not rely on it + self.assertIn('dual/host_info_to_device_cdc', + ci_select.role_examples(REPO, ('device', 'dual'))) + self.assertNotIn('host/bare_api', ci_select.role_examples(REPO, ('device', 'dual'))) + + def test_port_host_rule(self): # rule 4 + s = self.b(['src/portable/analog/max3421/hcd_max3421.c']) + self.assertFalse(s['full']) + # rp2040's family.cmake unconditionally lists hcd_max3421.c as a source of its + # tinyusb_host_max3421 INTERFACE lib (linked only when MAX3421_HOST=1, e.g. the + # real feather_rp2040_max3421 board) and espressif's component CMakeLists also + # references it — so the raw (unpruned) scan legitimately finds both; Task 4's + # buildability post-filter is what may later prune either away + # non-empty FIRST: a subset assertion is satisfied by set(), and since ports are + # now empty-means-empty (fail-closed) an unnoticed regression to zero families + # would select no build leg at all and merge an uncompiled HCD + self.assertTrue(s['families'], 'a host-port change must select some family') + self.assertLessEqual(set(s['families']), {'espressif', 'rp2040'}) + self.assertTrue(s['family_examples'], 'and must name the examples for them') + for exs in s['family_examples'].values(): + self.assertTrue(exs) + self.assertFalse(any(e.startswith(('device/', 'typec/')) for e in exs)) + + def test_port_shared_file_selects_all_examples(self): # rule 5 + s = self.b(['src/portable/synopsys/dwc2/dwc2_common.c']) + self.assertFalse(s['full']) + self.assertIn('stm32f4', s['families']) + self.assertNotIn('rp2040', s['families']) + self.assertNotIn('stm32f4', s['family_examples']) # 'all' => no map key + + def test_bsp_family_rule(self): # rule 6 + s = self.b(['hw/bsp/stm32f4/boards/stm32f407disco/board.h']) + self.assertEqual(s['families'], ['stm32f4']) + self.assertNotIn('stm32f4', s['family_examples']) + + def test_bsp_top_level_file_is_full(self): # rule 16 + self.assertTrue(self.b(['hw/bsp/board.c'])['full']) + self.assertTrue(self.b(['hw/bsp/family_support.cmake'])['full']) + + def test_mcu_rule(self): # rule 7 + s = self.b(['hw/mcu/nordic/nrf5x/nrf_clock.h']) + self.assertEqual(s['families'], ['nrf']) + # empty means empty: no family's build references the path, so no build + # compiles it - nothing to select + s = self.b(['hw/mcu/no_such_vendor/x.c']) + self.assertFalse(s['full']) + self.assertEqual(s['families'], []) + self.assertEqual(s['family_examples'], {}) + + def test_class_device_rule(self): # rule 8 + s = self.b(['src/class/cdc/cdc_device.c']) + self.assertFalse(s['full']) + # near-all families (Task 4's pruning may drop a few); never equality + # against all_bsp_families — that's a tuple, and pruning shrinks the list + self.assertIn('stm32f4', s['families']) + self.assertGreater(len(s['families']), 50) + exs = s['family_examples']['stm32f4'] + self.assertIn('device/cdc_msc', exs) + self.assertNotIn('device/hid_composite', exs) + self.assertNotIn('host/cdc_msc_hid', exs) # TUH_CDC examples are rule 9's + + def test_class_host_rule(self): # rule 9 + s = self.b(['src/class/msc/msc_host.c']) + exs = s['family_examples']['stm32f4'] + self.assertIn('host/msc_file_explorer', exs) + self.assertNotIn('device/cdc_msc', exs) + + def test_class_shared_header_and_include_edge(self): # rule 10 + s = self.b(['src/class/audio/audio.h']) + exs = s['family_examples']['stm32f4'] + self.assertIn('device/audio_test', exs) + self.assertIn('device/midi_test', exs) # midi headers include audio.h + + def test_core_device_rule(self): # rule 11 + s = self.b(['src/device/usbd.c']) + exs = s['family_examples']['stm32f4'] + self.assertIn('device/cdc_msc', exs) + # no dual In-assertion: dual examples are only.txt-gated to max3421/pio-usb + # boards, so pruning legitimately drops them on a plain stm32f4 board + self.assertFalse(any(e.startswith(('host/', 'typec/')) for e in exs)) + + def test_core_host_rule(self): # rule 12 + s = self.b(['src/host/usbh.c']) + exs = s['family_examples']['stm32f4'] + self.assertFalse(any(e.startswith(('device/', 'typec/')) for e in exs)) + + def test_example_rule(self): # rules 13, 14 + s = self.b(['examples/device/cdc_msc/src/main.c']) + self.assertEqual(s['family_examples']['stm32f4'], ['device/cdc_msc']) + s = self.b(['examples/device/board_test/src/main.c']) + self.assertEqual(s['family_examples']['stm32f4'], ['device/board_test']) + s = self.b(['examples/device/no_such_example/src/main.c']) # deleted example: nothing + self.assertFalse(s['full']) + self.assertEqual(s['families'], []) + + def test_full_paths(self): # rules 15-17 + for p in ('src/common/tusb_fifo.c', 'src/osal/osal.h', 'src/tusb.c', + 'src/tusb_option.h', + 'tools/build.py', 'tools/cmake/cpu/cortex-m4.cmake', + 'examples/CMakeLists.txt', 'examples/device/CMakeLists.txt', + 'examples/build_system/cmake/cpu.cmake', '.github/workflows/build.yml', + 'sonar-project.properties', 'some/unknown/path.c'): + self.assertTrue(self.b([p])['full'], p) + + def test_mixed_diff_unions_per_family(self): + s = self.b(['src/portable/raspberrypi/rp2040/dcd_rp2040.c', 'src/class/cdc/cdc_device.c']) + self.assertFalse(s['full']) + self.assertIn('stm32f4', s['families']) + self.assertGreater(len(s['families']), 50) + self.assertIn('device/hid_composite', s['family_examples']['rp2040']) # from the dcd rule + self.assertNotIn('device/hid_composite', s['family_examples']['stm32f4']) # cdc-only there + + def test_example_names_are_real_dirs(self): + for ex in ci_select.all_examples(REPO): + role, name = ex.split('/') + self.assertTrue(os.path.isdir(os.path.join(REPO, 'examples', role, name)), ex) + self.assertRegex(ex, r'^(device|dual|host|typec)/[A-Za-z0-9_]+$') + + +class TestBuildPostFilter(unittest.TestCase): + def test_kept_examples_are_buildable(self): + import build_utils, build as build_py + s = ci_select.classify_build(['src/class/msc/msc_host.c'], REPO) + self.assertFalse(s['full']) + # families that cannot build a single TUH_MSC example drop out entirely + self.assertNotIn('msp430', s['families']) + old = os.getcwd() + os.chdir(REPO) + try: + # buildable on SOME board of the family - CircleCI builds them all + for fam, exs in s['family_examples'].items(): + boards = build_py.get_family_boards(fam, False, False) + for e in exs: + self.assertTrue(any(not build_utils.skip_example(e, b) for b in boards), + f'{fam}: {e}') + finally: + os.chdir(old) + + def test_unfiltered_family_has_no_map_key(self): + s = ci_select.classify_build(['hw/bsp/stm32f4/family.c'], REPO) + self.assertEqual(s['families'], ['stm32f4']) + self.assertEqual(s['family_examples'], {}) + + def test_espressif_prunes_to_what_its_build_path_can_build(self): + # build.py's espressif branch builds get_examples('espressif') only (the + # *_freertos examples plus a short extra list), so keeping espressif for a + # device/mtp diff spins CircleCI's most expensive leg up to skip everything + s = ci_select.classify_build(['examples/device/mtp/src/main.c'], REPO) + self.assertFalse(s['full']) + self.assertNotIn('espressif', s['families']) + + def test_espressif_survives_an_example_it_does_build(self): + s = ci_select.classify_build(['examples/device/cdc_msc_freertos/src/main.c'], REPO) + self.assertFalse(s['full']) + self.assertIn('espressif', s['families']) + + def test_ra_survives_the_dual_example_prune(self): + # ra's only buildable dual example is gated on only.txt's mcu:ra6m5, which + # exists only if the ${MCU_VARIANT} token in FAMILY_MCUS resolves + s = ci_select.classify_build( + ['examples/dual/host_info_to_device_cdc/src/main.c'], REPO) + self.assertFalse(s['full']) + self.assertIn('ra', s['families'], s['families']) + + def test_deleted_family_dir_does_not_crash(self): + # rule 6 extracts a family from the path; a PR that deletes or renames + # hw/bsp/<fam> used to traceback in get_family_boards' scandir + s = ci_select.classify_build(['hw/bsp/no_such_family_xyz/family.cmake'], REPO) + self.assertFalse(s['full']) + self.assertEqual(s['families'], []) + self.assertTrue(any('gone from tree' in r for r in s['reasons']), s['reasons']) + + def test_class_source_selecting_nothing_selects_nothing(self): + # synthetic class-with-no-enabling-config case (vendor_host.c was the live + # instance until its removal): no config enables CFG_TUH_VENDOR, so + # nothing exercises it and nothing builds - empty means empty (maintainer + # decision; the file is still parsed by every full master-push build, which is + # the accepted net for a break outside its #if guard) + s = ci_select.classify_build(['src/class/vendor/vendor_host.c'], REPO) + self.assertFalse(s['full']) + self.assertEqual(s['families'], []) + self.assertTrue(any('no contribution' in r for r in s['reasons']), s['reasons']) + + def test_class_source_with_examples_still_scopes(self): + s = ci_select.classify_build(['src/class/cdc/cdc_device.c'], REPO) + self.assertFalse(s['full']) + + def test_no_stdout_pollution(self): + # get_family_boards prints on odd families; the selector's stdout is JSON + import io, contextlib + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + ci_select.classify_build(['src/class/msc/msc_host.c'], REPO) + self.assertEqual(buf.getvalue(), '') + + +class TestNoContributionPaths(unittest.TestCase): + """Paths that are inside build.yml's code filter but cannot change a compiled byte. + Unclassified means FULL on both axes, so a metrics-only PR would otherwise cost the + whole build matrix plus an exclusive full-rig sweep - where master ran nothing.""" + + def test_metrics_scripts_run_on_no_board_but_still_build(self): + # HIL axis only. tools/metrics.py IS executed by a build - examples/CMakeLists.txt + # makes it the `tinyusb_metrics` target and build_util.yml adds + # `--target tinyusb_metrics` - so the build axis must keep exercising it, or a + # break merges green and reds the next master push. Nothing on the rig runs it. + for p in ('tools/metrics.py', '.github/scripts/metrics_pair_compare.py'): + h = sel([p]) + self.assertFalse(h['full'], p) + self.assertEqual(h['boards'], {}, p) + self.assertTrue(ci_select.classify_build([p], REPO)['full'], p) + + def test_typec_example_builds_but_runs_nothing(self): + # examples/typec is compiled by the build matrix and run by no rig board; the + # HIL walk used to not recognise the role at all -> unclassified -> full rig + p = 'examples/typec/power_delivery/src/main.c' + h = sel([p]) + self.assertFalse(h['full']) + self.assertEqual(h['boards'], {}) + b = ci_select.classify_build([p], REPO) + self.assertFalse(b['full']) + self.assertTrue(b['families'], 'typec still has to be compiled somewhere') + + +class TestHilExamples(unittest.TestCase): + def test_board_test_always_present_and_full_emits(self): + s = ci_select.classify(['src/common/tusb_fifo.c'], REPO, ROSTERS) # full + he = ci_select.hil_examples(s, ROSTERS) + self.assertEqual(set(he), {b['name'] for b in ROSTER}) + for name, exs in he.items(): + self.assertIn('device/board_test', exs) + + def test_narrowed_board_gets_chosen_tests_only(self): + s = ci_select.classify(['examples/device/cdc_msc/src/main.c'], REPO, ROSTERS) + he = ci_select.hil_examples(s, ROSTERS) + self.assertEqual(he['stm32f407disco'], ['device/board_test', 'device/cdc_msc']) + + def test_full_board_gets_its_whole_test_list(self): + s = ci_select.classify(['hw/bsp/stm32f4/boards/stm32f407disco/board.h'], REPO, ROSTERS) + he = ci_select.hil_examples(s, ROSTERS) + want = set(ci_select.board_tests(ROSTER[1])) | {'device/board_test'} + self.assertEqual(set(he['stm32f407disco']), want) + self.assertNotIn('raspberry_pi_pico', he) # deselected board: no firmware needed + + +class TestHilExamplesDuplicateRosters(unittest.TestCase): + """Rosters are disjoint today, but a board moved between rigs (or listed on both + during a migration) must get the UNION of its test lists: superset firmware is + harmless, a missing image fails the run on whichever rig lost the coin toss.""" + + ROSTERS = [ + ('test/hil/a.json', [{'name': 'dup_board', 'uid': 'd1', 'flasher': {'name': 'jlink'}, + 'tests': {'only': ['device/cdc_msc']}}]), + ('test/hil/b.json', [{'name': 'dup_board', 'uid': 'd1', 'flasher': {'name': 'jlink'}, + 'tests': {'only': ['device/hid_boot_interface']}}]), + ] + + def test_duplicate_board_unions_the_test_lists(self): + he = ci_select.hil_examples({'full': True, 'boards': {}}, self.ROSTERS) + self.assertEqual(he['dup_board'], + ['device/board_test', 'device/cdc_msc', + 'device/hid_boot_interface']) + + +class TestCliJson(unittest.TestCase): + def test_build_key_without_rosters(self): + r = subprocess.run([sys.executable, os.path.join(REPO, 'tools/ci_select.py'), + '--diff-file', '/dev/null'], capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + j = json.loads(r.stdout) + self.assertIn('build', j) + self.assertNotIn('hil_examples', j) # rosters not given + + def test_build_and_hil_keys_with_rosters(self): + import tempfile + with tempfile.NamedTemporaryFile('w', suffix='.txt', delete=False) as f: + f.write('src/portable/raspberrypi/rp2040/dcd_rp2040.c\n') + df = f.name + r = subprocess.run([sys.executable, os.path.join(REPO, 'tools/ci_select.py'), + '--diff-file', df, os.path.join(REPO, 'test/hil/tinyusb.json')], + capture_output=True, text=True) + os.unlink(df) + j = json.loads(r.stdout) + self.assertEqual(j['build']['families'], ['rp2040']) + self.assertIn('hil_examples', j) + for exs in j['hil_examples'].values(): + self.assertIn('device/board_test', exs) + + +SET_MATRIX = os.path.join(REPO, '.github/scripts/ci_set_matrix.py') + + +class TestCiSetMatrix(unittest.TestCase): + def run_matrix(self, *args): + return subprocess.run([sys.executable, SET_MATRIX, *args], + capture_output=True, text=True) + + def test_no_flags_is_todays_output(self): + r = self.run_matrix() + self.assertEqual(r.returncode, 0, r.stderr) + self.baseline = json.loads(r.stdout) + self.assertIn('stm32f4', self.baseline['arm-gcc']) + + def test_select_full_is_identical(self): + base = json.loads(self.run_matrix().stdout) + sel = json.dumps({'build': {'full': True, 'families': [], 'family_examples': {}}}) + self.assertEqual(json.loads(self.run_matrix('--select', sel).stdout), base) + + def test_select_narrow_is_a_subset(self): + sel = json.dumps({'build': {'full': False, 'families': ['rp2040', 'stm32f4'], + 'family_examples': {}}}) + m = json.loads(self.run_matrix('--select', sel).stdout) + self.assertEqual(m['arm-gcc'], ['rp2040', 'stm32f4']) + self.assertEqual(m['riscv-gcc'], []) + self.assertEqual(set(m), set(json.loads(self.run_matrix().stdout))) # all keys kept + + def test_malformed_select_falls_open(self): + base = json.loads(self.run_matrix().stdout) + r = self.run_matrix('--select', 'not json {') + self.assertEqual(r.returncode, 0) + self.assertEqual(json.loads(r.stdout), base) + self.assertIn('ci_set_matrix: UNSCOPED', r.stderr) # build.yml greps this + + def test_wrong_shaped_select_falls_open_too(self): + # valid JSON, wrong types: the matrix is built AFTER main()'s try/except, so an + # AttributeError here reds the step - the very outcome that handler exists to + # prevent (GHA and CircleCI only survive it through their own shell `||`) + base = json.loads(self.run_matrix().stdout) + for bad in ('{"build": ["stm32f4"]}', '{"build": {"full": false}}', + '{"build": {"full": false, "families": "stm32f4"}}', '["stm32f4"]'): + r = self.run_matrix('--select', bad) + self.assertEqual(r.returncode, 0, f'{bad}: {r.stderr}') + self.assertEqual(json.loads(r.stdout), base, bad) + + def test_base_flag_with_empty_diff_selects_nothing(self): + # --base HEAD => empty diff => build.families [] => every toolchain scopes to [] + base = json.loads(self.run_matrix().stdout) + r = self.run_matrix('--base', 'HEAD') + self.assertEqual(r.returncode, 0, r.stderr) + m = json.loads(r.stdout) + self.assertEqual(set(m), set(base)) + self.assertTrue(all(v == [] for v in m.values()), m) + + def test_select_file_matches_select(self): + # build.yml hands the selection over as a FILE: a ~128KiB step env var makes + # the step's own exec fail with E2BIG before any fallback can run + import tempfile + sel = json.dumps({'build': {'full': False, 'families': ['rp2040'], + 'family_examples': {}}}) + with tempfile.NamedTemporaryFile('w', suffix='.json', delete=False) as f: + f.write(sel) + path = f.name + try: + self.assertEqual(self.run_matrix('--select-file', path).stdout, + self.run_matrix('--select', sel).stdout) + finally: + os.unlink(path) + + def test_absent_families_key_falls_open(self): + # `{"build": {"full": false}}` with no families key is an unusable selection, + # not "nothing selected": scoping every toolchain to [] would report a + # vacuous green with zero families built + base = json.loads(self.run_matrix().stdout) + r = self.run_matrix('--select', json.dumps({'build': {'full': False}})) + self.assertEqual(r.returncode, 0) + self.assertEqual(json.loads(r.stdout), base) + self.assertIn('ci_set_matrix: UNSCOPED', r.stderr) # build.yml greps this + + def test_families_no_toolchain_builds_falls_open(self): + # hw/bsp/same7x is real but in no toolchain's list, so scoping to it emits an + # all-empty matrix: every leg skips and the PR goes green from a build job that + # ran no compiler. Unusable, not "nothing selected" - and the marker matters, + # because that is what build.yml and CircleCI grep to drop the build extras too. + base = json.loads(self.run_matrix().stdout) + r = self.run_matrix('--select', + json.dumps({'build': {'full': False, 'families': ['same7x']}})) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertEqual(json.loads(r.stdout), base) + self.assertIn('ci_set_matrix: UNSCOPED', r.stderr) + + def test_a_partial_toolchain_miss_still_scopes(self): + # one buildable family is real coverage: scope to it and just note the other + r = self.run_matrix('--select', json.dumps( + {'build': {'full': False, 'families': ['stm32f4', 'same7x']}})) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertEqual(json.loads(r.stdout)['arm-gcc'], ['stm32f4']) + self.assertNotIn('UNSCOPED', r.stderr) + self.assertIn('same7x', r.stderr) + + def test_explicit_empty_families_selects_nothing(self): + # an explicit [] IS a legitimate answer (a diff that builds nothing) + r = self.run_matrix('--select', + json.dumps({'build': {'full': False, 'families': []}})) + self.assertEqual(r.returncode, 0) + self.assertEqual(set().union(*json.loads(r.stdout).values()), set()) + + def test_missing_select_file_falls_open(self): + base = json.loads(self.run_matrix().stdout) + r = self.run_matrix('--select-file', '/no/such/selection.json') + self.assertEqual(r.returncode, 0) + self.assertEqual(json.loads(r.stdout), base) + self.assertIn('ci_set_matrix: UNSCOPED', r.stderr) # build.yml greps this + + def test_base_flag_bad_ref_falls_open(self): + base = json.loads(self.run_matrix().stdout) + r = self.run_matrix('--base', 'no-such-ref-xyz') + self.assertEqual(r.returncode, 0) + self.assertEqual(json.loads(r.stdout), base) + self.assertIn('ci_set_matrix: UNSCOPED', r.stderr) # build.yml greps this + + +HIL_SET_MATRIX = os.path.join(REPO, '.github/scripts/hil_ci_set_matrix.py') + + +class TestHilCiSetMatrixExamples(unittest.TestCase): + def run_matrix(self, *args): + r = subprocess.run([sys.executable, HIL_SET_MATRIX, *args, + os.path.join(REPO, 'test/hil/tinyusb.json')], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + return r.stdout + + def test_no_hil_examples_is_byte_identical(self): + plain = self.run_matrix() + sel = json.dumps({'full': True, 'boards': {}}) + self.assertEqual(self.run_matrix('--select', sel), plain) + + def test_absent_boards_key_falls_open_to_the_full_roster(self): + # the mirror of ci_set_matrix's families guard: reading an ABSENT boards key as + # "nothing selected" filters every board out, so every hil-build leg skips and + # both rig jobs skip through needs: - an all-green PR with zero hardware + # coverage. An explicit boards: {} stays a legitimate nothing-selected. + plain = self.run_matrix() + for bad in ('{"full": false, "hil_examples": {}}', '{"full": false, "boards": []}', + 'not json {', '["a board"]', + # the whole selection is unusable, hil_examples included: keeping the + # -e lists builds a few examples per board while the rig, unfiltered, + # runs that board's whole test list + '{"full": false, "hil_examples": {"frdm_k64f": ["device/cdc_msc"]}}'): + self.assertEqual(self.run_matrix('--select', bad), plain, bad) + self.assertNotEqual(self.run_matrix('--select', '{"full": false, "boards": {}}'), + plain, 'an explicit empty boards map still means nothing') + + def test_select_file_matches_select(self): + # hil-hfp-iar passes the whole selection; as one argv it can exceed + # MAX_ARG_STRLEN on a big diff, so the file form must be equivalent + import tempfile + board = on_roster(self, 'stm32f407disco')[0] + sel = json.dumps({'full': False, 'boards': {board: 'all'}, + 'hil_examples': {board: ['device/board_test']}}) + with tempfile.NamedTemporaryFile('w', suffix='.json', delete=False) as f: + f.write(sel) + path = f.name + try: + self.assertEqual(self.run_matrix('--select-file', path), + self.run_matrix('--select', sel)) + finally: + os.unlink(path) + + def test_examples_appended_per_board(self): + board = on_roster(self, 'stm32f407disco')[0] + sel = json.dumps({'full': False, 'boards': {board: 'all'}, + 'hil_examples': {board: ['device/board_test', 'device/cdc_msc']}}) + m = json.loads(self.run_matrix('--select', sel)) + entries = [e for entries in m.values() for e in entries] + self.assertTrue(entries) + for e in entries: + self.assertIn(f'-b {board}', e) + self.assertIn('-e device/board_test', e) + self.assertIn('-e device/cdc_msc', e) + + +class TestBuildPyExampleFilter(unittest.TestCase): + def setUp(self): + import build as build_py + self.build = build_py + self.old = os.getcwd() + os.chdir(REPO) # skip_example uses repo-relative paths + + def tearDown(self): + os.chdir(self.old) + + def test_all_maps_to_example_names(self): + # ONE group: the examples of a '--target all' build go into a single + # `cmake --build --target a b c`, so they build in parallel + t = self.build.resolve_example_target_groups(['all'], ['device/cdc_msc', 'device/dfu'], + 'stm32f407disco') + self.assertEqual(t, [['cdc_msc', 'dfu']]) + + def test_other_targets_pass_through_in_their_own_group(self): + # a target that is not 'all' keeps its own invocation, so ordering against the + # examples is preserved (tinyusb_metrics runs after them, as it did unfiltered) + t = self.build.resolve_example_target_groups(['all', 'tinyusb_metrics'], + ['device/cdc_msc'], 'stm32f407disco') + self.assertEqual(t, [['cdc_msc'], ['tinyusb_metrics']]) + + def test_unbuildable_examples_drop_and_empty_is_none(self): + # typec/power_delivery only builds on stm32g4-class parts, never on f4 + t = self.build.resolve_example_target_groups(['all'], + ['typec/power_delivery', 'device/cdc_msc'], + 'stm32f407disco') + self.assertEqual(t, [['cdc_msc']]) + self.assertIsNone(self.build.resolve_example_target_groups(['all'], + ['typec/power_delivery'], + 'stm32f407disco')) + + def test_espressif_empty_intersection_skips_without_building(self): + # cmake_board's espressif branch must short-circuit on an empty -e + # intersection the same way the generic cmake/make branches do, and + # must do so before touching idf.py (no real esp-idf build here). + calls = [] + real_run_cmd = self.build.run_cmd # `del` here would drop the real one + self.build.run_cmd = lambda cmd: calls.append(cmd) # would only run for a real build + try: + r = self.build.cmake_board('espressif_s3_devkitc', [], None, [], ['all'], + examples=['nonexistent/example']) + finally: + self.build.run_cmd = real_run_cmd + self.assertEqual(r, [0, 0, 1]) + self.assertEqual(calls, []) + + def test_make_one_example_uses_make_semantics(self): + # F1 end to end: the make path must ask skip_example with build_system='make', + # or lpc54's cmake-only FAMILY_MCUS un-skips a host example whose make build + # compiles no HCD source and fails to link + calls = [] + real_run_cmd = self.build.run_cmd + self.build.run_cmd = lambda cmd: calls.append(cmd) + try: + r = self.build.make_one_example('host/msc_file_explorer_freertos', + 'lpcxpresso54628', '', ['all']) + finally: + self.build.run_cmd = real_run_cmd + self.assertEqual(r, [0, 0, 1]) # skipped, nothing handed to make + self.assertEqual(calls, []) + + def test_example_flag_rejects_a_bare_name(self): + # `-e cdc_msc` (no role) used to IndexError inside the target resolver; + # argparse rejects the shape now, with a message that names it + r = subprocess.run([sys.executable, os.path.join(REPO, 'tools', 'build.py'), + '-b', 'stm32f407disco', '-e', 'cdc_msc'], + capture_output=True, text=True, cwd=REPO) + self.assertEqual(r.returncode, 2, r.stdout + r.stderr) + self.assertIn('role/name', r.stderr) + + def test_no_example_basename_is_reused_across_roles(self): + # -e maps role/name onto the BARE cmake target name, so device/foo and host/foo + # would collapse into one `--target foo`: one of them would never build while + # the post-configure check still reports both as covered. No collision today, + # and the -e lists are machine-generated, so nothing else would notice one. + seen = {} + for ex in ci_select.all_examples(REPO): + role, name = ex.split('/', 1) + self.assertNotIn(name, seen, + f'{ex} and {seen.get(name)}/{name} share a cmake target name; ' + f'build.py -e cannot tell them apart') + seen[name] = role + + def test_example_flag_rejects_a_name_no_example_dir_answers_to(self): + # right shape, no such dir: every board would report Skipped and the run would + # still exit 0 (main returns the FAILED count), so an entirely stale -e list - + # from the example map or from a roster test name - reads as a green build + r = subprocess.run([sys.executable, os.path.join(REPO, 'tools', 'build.py'), + '-b', 'stm32f407disco', '-e', 'device/no_such_example'], + capture_output=True, text=True, cwd=REPO) + self.assertEqual(r.returncode, 2, r.stdout + r.stderr) + self.assertIn('no such example directory', r.stderr) + + def test_pr_filter_answers_before_configuring(self): + # nothing the -e list names is buildable here: the skip.txt mirror needs no + # configure output, so the whole cmake run must be skipped, not just its build + calls = [] + real_run_cmd = self.build.run_cmd + self.build.run_cmd = lambda cmd: calls.append(cmd) + try: + r = self.build.cmake_board('stm32f407disco', [], None, [], ['all'], + examples=['typec/power_delivery']) + finally: + self.build.run_cmd = real_run_cmd + self.assertEqual(r, [0, 0, 1]) + self.assertEqual(calls, []) + + def _cmake_board_with_targets(self, registered, examples): + """cmake_board with the configure/build stubbed and CMake's registered-target + list forced. Returns (result, target names handed to `cmake --build`).""" + class Ok: + returncode = 0 + calls = [] + + def fake_run(cmd): + calls.append(cmd) + return Ok() + real_run_cmd = self.build.run_cmd + real_targets = self.build.cmake_registered_targets + self.build.run_cmd = fake_run + self.build.cmake_registered_targets = lambda d: registered + try: + r = self.build.cmake_board('stm32f407disco', [], None, [], ['all'], + examples=examples) + finally: + self.build.run_cmd = real_run_cmd + self.build.cmake_registered_targets = real_targets + # everything after --target: one invocation carries the whole group + built = [c[c.index('--target') + 1:] for c in calls if '--target' in c] + return r, built + + def test_example_without_a_cmake_target_is_dropped(self): + # an example dir CMake never registered (absent from the role CMakeLists, or + # a stale roster name) must not reach `cmake --build --target <it>`: that is a + # hard red, and skip.txt cannot see it + r, built = self._cmake_board_with_targets({'cdc_msc'}, + ['device/cdc_msc', 'device/dfu']) + self.assertEqual(built, [['cdc_msc']]) + self.assertEqual(r, [1, 0, 0]) + + def test_the_selected_examples_build_in_one_invocation(self): + # one `cmake --build --target a b c`, not one invocation per example: the + # per-example loop serialised every scoped leg, and hil-build gets an -e list + # on EVERY PR (~14 examples per board), so it is on the critical path to the rig + r, built = self._cmake_board_with_targets({'cdc_msc', 'dfu', 'hid_generic_inout'}, + ['device/cdc_msc', 'device/dfu', + 'device/hid_generic_inout']) + self.assertEqual(built, [['cdc_msc', 'dfu', 'hid_generic_inout']]) + + def test_no_registered_target_at_all_skips_the_build(self): + r, built = self._cmake_board_with_targets({'cdc_msc'}, ['device/dfu']) + self.assertEqual(built, []) + self.assertEqual(r, [0, 0, 1]) + + def test_unparseable_target_help_keeps_the_skip_txt_answer(self): + # ground truth unavailable (a non-Ninja generator, an old cmake): fall back + # to the mirror rather than dropping every example + r, built = self._cmake_board_with_targets(None, ['device/cdc_msc']) + self.assertEqual(built, [['cdc_msc']]) + + def test_target_help_parse(self): + text = ('[1/1] All primary targets available:\n' + 'tinyusb_metrics: phony\n' + 'cdc_msc: phony\n' + 'cdc_msc-membrowse-upload: phony\n' + 'device/edit_cache: phony\n' + '/abs/build/device/cdc_msc/CMakeFiles/cdc_msc-jlink: CUSTOM_COMMAND\n') + self.assertEqual(self.build.parse_target_help(text), + {'tinyusb_metrics', 'cdc_msc', 'cdc_msc-membrowse-upload'}) + + def test_build_defines_reach_the_example_filter(self): + # metro_m4_express gets MAX3421_HOST=1 from its roster variant, never + # from its BSP: without threading them through, -e drops the rig's only + # MAX3421 dual firmware that --target all used to build + self.assertIsNone(self.build.resolve_example_target_groups( + ['all'], ['dual/host_info_to_device_cdc'], 'metro_m4_express')) + self.assertEqual(self.build.resolve_example_target_groups( + ['all'], ['dual/host_info_to_device_cdc'], 'metro_m4_express', + extra_defines=('MAX3421_HOST=1',)), [['host_info_to_device_cdc']]) + + + +class TestFamilyMcusFallback(unittest.TestCase): + """A family whose family.cmake sets FAMILY_MCUS only inside if() blocks gets its + whole MCU answer from _board_mcu's CFG_TUSB_MCU scrape (build_utils._family_mcus + does not evaluate cmake conditionals). For mcx that answer is load-bearing - six + examples' skip.txt name mcu:MCXA15 - and it comes out right only because every + mcx board still carries the token in a make-only board.mk the scrape falls + through to. A board.cmake-only board (MCU_VARIANT, no CFG_TUSB_MCU) would scrape + 'NONE' and silently skip EVERY example on it, in CI as well as in -e.""" + + @staticmethod + def conditional_only_families(): + """hw/bsp/<family> dirs whose family.cmake has no unconditional + set(FAMILY_MCUS ...) - computed, not listed, so a family that grows or loses + one moves in and out of this guard on its own.""" + import build_utils + out = [] + for fc in sorted(glob.glob(os.path.join(REPO, 'hw/bsp/*/family.cmake'))): + depth, uncond = 0, False + for line in open(fc).read().splitlines(): + line = line.strip() + if build_utils._FAMILY_MCUS_RE.match(line) and depth == 0: + uncond = True + if re.match(r'if\s*\(', line): + depth += 1 + elif re.match(r'endif\s*\(', line): + depth = max(0, depth - 1) + if not uncond: + out.append(os.path.dirname(fc)) + return out + + def test_every_board_of_such_a_family_scrapes_an_mcu(self): + import build_utils + fams = self.conditional_only_families() + self.assertTrue(fams, 'no family sets FAMILY_MCUS conditionally any more') + for fam_dir in fams: + fam = os.path.basename(fam_dir) + for bd in sorted(glob.glob(os.path.join(fam_dir, 'boards', '*'))): + if not os.path.isdir(bd): + continue + mcu, _ = build_utils._board_mcu(bd, fam_dir, fam) + self.assertNotEqual( + mcu, 'NONE', + f'{fam}/{os.path.basename(bd)}: nothing to scrape a CFG_TUSB_MCU ' + f'token from, and {fam}/family.cmake sets FAMILY_MCUS only inside ' + f'if() - skip_example would skip every example on this board. Fix ' + f'by evaluating the if(MCU_VARIANT STREQUAL ...) branches.') + + +class TestMcuTokensResolve(unittest.TestCase): + """The cmake-side MCU mirror must never answer with an unexpanded ${VAR} or with + nothing at all: both make every `mcu:` token miss, which reads as 'skip' for any + example carrying an only.txt and silently drops compile coverage.""" + + @staticmethod + def _every_board(): + import build as build_py + old = os.getcwd() + os.chdir(REPO) + try: + for fam in sorted(os.path.basename(os.path.dirname(f)) + for f in glob.glob(os.path.join(REPO, 'hw/bsp/*/boards'))): + for b in build_py.get_family_boards(fam, False, False): + yield fam, b + finally: + os.chdir(old) + + def test_no_board_answers_with_an_unexpanded_variable(self): + import build_utils + for fam, board in self._every_board(): + fam_dir, board_dir = f'{REPO}/hw/bsp/{fam}', f'{REPO}/hw/bsp/{fam}/boards/{board}' + mcus = set(build_utils._family_mcus(fam_dir, board_dir)) + mcus.add(build_utils._board_mcu(board_dir, fam_dir, fam)[0]) + self.assertFalse([m for m in mcus if '${' in m], + f'{fam}/{board}: unexpanded cmake variable in {sorted(mcus)} - ' + f'teach build_utils._cmake_expand the construct that produces it') + self.assertTrue(mcus - {'NONE'}, + f'{fam}/{board}: no MCU name resolved at all') + + # skip.txt/only.txt tokens no board in the tree answers to: stale spellings left + # behind by a family rename. Each one silently changes what CI builds, so this list + # must only ever SHRINK - a new entry means either a live token the mirror cannot + # produce, or a rename nobody followed through. `family:samd21` was one of these + # until the nine examples/host/*/only.txt files were corrected to samd2x_l2x. + # + # The `mcu:` entries are NOT all harmless. MIMXRT10XX/MIMXRT11XX and LPC177X_8X sit + # beside a live token in the same file, so they gate nothing either way. MKL25ZXX + # (device/msc_dual_lun) and SAME5X (device/audio_test) do not: those skips are dead, + # and both examples are built today on the boards their skip file meant to exclude - + # successfully, which is why nobody noticed. Correcting them REMOVES working build + # coverage, so it is a maintainer call, not a drive-by fix. + UNREACHABLE_TOKENS = { + 'mcu': {'LPC177X_8X', 'MIMXRT10XX', 'MIMXRT11XX', 'MKL25ZXX', 'SAME5X', 'STM32U3'}, + 'family': set(), + 'board': set(), + } + + def test_every_skip_only_token_is_reachable(self): + import build_utils + wanted = {ns: set() for ns in self.UNREACHABLE_TOKENS} + for f in glob.glob(os.path.join(REPO, 'examples/*/*/*.txt')): + if os.path.basename(f) in ('skip.txt', 'only.txt'): + for tok in open(f).read().split(): + ns, _, name = tok.partition(':') + if ns in wanted and name: + wanted[ns].add(name) + have = {ns: set() for ns in wanted} + have['mcu'].add('MAX3421') # synthetic, from family_support.cmake:940 + for fam, board in self._every_board(): + fam_dir, board_dir = f'{REPO}/hw/bsp/{fam}', f'{REPO}/hw/bsp/{fam}/boards/{board}' + have['family'].add(fam) + have['board'].add(board) + have['mcu'] |= set(build_utils._family_mcus(fam_dir, board_dir)) + have['mcu'].add(build_utils._board_mcu(board_dir, fam_dir, fam)[0]) + have['mcu'].add(build_utils._scrape_mcu(pathlib.Path(fam_dir), + pathlib.Path(board_dir), fam)[0]) # make + for ns in wanted: + self.assertEqual( + wanted[ns] - have[ns], self.UNREACHABLE_TOKENS[ns] & wanted[ns], + f'a skip.txt/only.txt {ns}: token nothing in hw/bsp answers to. Either ' + f'the token is stale (a rename just changed what CI builds), or the ' + f'mirror cannot produce it - both silently skip that example everywhere.') + + def test_the_mcx_skip_tokens_are_still_live(self): + # the reason the mcx scrape is load-bearing rather than academic + named = [os.path.dirname(f) for f in glob.glob(os.path.join(REPO, 'examples/*/*/skip.txt')) + if 'mcu:MCXA15' in open(f).read().split()] + self.assertTrue(named, 'no skip.txt names mcu:MCXA15 any more') + + +class TestSkipExampleMirrorsFamilyFilter(unittest.TestCase): + """build_utils.skip_example is the python mirror of CMake's family_filter + (hw/bsp/family_support.cmake:171-207). family_filter loops over the whole + FAMILY_MCUS list; a per-board CFG_TUSB_MCU scrape alone lets -e ask for a + target CMake never created, and `cmake --build --target <it>` hard-fails.""" + + def setUp(self): + import build_utils + self.build_utils = build_utils + self.old = os.getcwd() + os.chdir(REPO) # skip_example uses repo-relative paths + + def tearDown(self): + os.chdir(self.old) + + def test_any_family_mcu_can_skip(self): + # broadcom_64bit: set(FAMILY_MCUS BCM2711 BCM2835); raspberrypi_cm4 is + # BCM2711, and examples/device/dfu/skip.txt lists mcu:BCM2835 + self.assertTrue(self.build_utils.skip_example('device/dfu', 'raspberrypi_cm4')) + + def test_any_family_mcu_can_satisfy_only(self): + # lpc55: family.mk says LPC55XX, family.cmake sets FAMILY_MCUS LPC55, and + # host/cdc_msc_hid/only.txt lists mcu:LPC55 - CMake builds it + self.assertFalse(self.build_utils.skip_example('host/cdc_msc_hid', 'lpcxpresso55s69')) + + def test_existing_decisions_are_unchanged(self): + self.assertFalse(self.build_utils.skip_example('device/cdc_msc', 'stm32f407disco')) + self.assertTrue(self.build_utils.skip_example('typec/power_delivery', 'stm32f407disco')) + + def test_build_define_enables_max3421_only_list(self): + # family_support.cmake:940 appends MAX3421 to FAMILY_MCUS when + # MAX3421_HOST=1; on metro_m4_express that define comes from the roster + # variant defines, so skip_example has to be told about it + ex = 'dual/host_info_to_device_cdc' + self.assertTrue(self.build_utils.skip_example(ex, 'metro_m4_express')) + self.assertFalse(self.build_utils.skip_example(ex, 'metro_m4_express', + extra_defines=('MAX3421_HOST=1',))) + + def test_family_mcus_variable_token_resolves(self): + """hw/bsp/ra/family.cmake: `set(FAMILY_MCUS RAXXX ${MCU_VARIANT})`, and + ra6m5_ek/board.cmake sets MCU_VARIANT ra6m5 — which is exactly the token + dual/host_info_to_device_cdc/only.txt spells (mcu:ra6m5). Dropping the + ${...} token silently removed ra from every scoped dual-example build.""" + self.assertFalse(self.build_utils.skip_example( + 'dual/host_info_to_device_cdc', 'ra6m5_ek')) + + def test_board_cmake_max3421_counts(self): + """feather_rp2040_max3421/board.cmake sets MAX3421_HOST 1 while the MCU + token comes from rp2040's family.cmake; scanning only the file the token + came from misses it, and only.txt's mcu:MAX3421 never matches.""" + self.assertFalse(self.build_utils.skip_example( + 'host/cdc_msc_hid_freertos', 'feather_rp2040_max3421')) + + +class TestSkipExampleMakeSemantics(unittest.TestCase): + """FAMILY_MCUS is a CMAKE fact. hw/bsp/lpc54/family.cmake sets it to LPC54 and + wires the ohci host sources; family.mk builds OPT_MCU_LPC54XXX and compiles no + HCD source at all — so applying the cmake MCU union to a Make build un-skips + the 9 host examples only.txt gates on mcu:LPC54 and they fail to link + (undefined reference to hcd_init). Make keeps master's exact algorithm.""" + + def setUp(self): + import build_utils + self.build_utils = build_utils + self.old = os.getcwd() + os.chdir(REPO) # skip_example uses repo-relative paths + + def tearDown(self): + os.chdir(self.old) + + def test_make_keeps_cmake_only_family_mcus_out(self): + self.assertTrue(self.build_utils.skip_example( + 'host/msc_file_explorer_freertos', 'lpcxpresso54628', build_system='make')) + + def test_make_does_not_skip_on_a_sibling_family_mcu(self): + # broadcom_64bit sets FAMILY_MCUS "BCM2711 BCM2835"; raspberrypi_cm4 is the + # BCM2711 one and device/dfu/skip.txt names mcu:BCM2835. The aarch64 make leg + # built device/dfu before the union and must keep building it. + for ex in ('device/dfu', 'device/usbtmc'): + self.assertFalse(self.build_utils.skip_example( + ex, 'raspberrypi_cm4', build_system='make'), ex) + + def test_cmake_is_the_default_and_still_unions(self): + self.assertTrue(self.build_utils.skip_example('device/dfu', 'raspberrypi_cm4')) + self.assertEqual( + self.build_utils.skip_example('device/dfu', 'raspberrypi_cm4'), + self.build_utils.skip_example('device/dfu', 'raspberrypi_cm4', + build_system='cmake')) + + def test_build_system_is_part_of_the_cache_key(self): + # one lru_cache shared by both semantics would answer the second caller + # with the first caller's verdict + ex, board = 'host/msc_file_explorer_freertos', 'lpcxpresso54628' + self.assertFalse(self.build_utils.skip_example(ex, board, build_system='cmake')) + self.assertTrue(self.build_utils.skip_example(ex, board, build_system='make')) + self.assertFalse(self.build_utils.skip_example(ex, board, build_system='cmake')) + + +class TestConfigEnables(unittest.TestCase): + """_config_enables decides which examples a class change selects, on BOTH the + build and the HIL axis. A define it cannot evaluate must read as ON: reading + it as OFF is fail-closed, and lets a compile break merge green.""" + + def test_identifier_value_is_enabled(self): + # examples/host/midi_rx: `#define CFG_TUH_MIDI CFG_TUH_DEVICE_MAX` + cfg = os.path.join(REPO, 'examples/host/midi_rx/src/tusb_config.h') + self.assertTrue(ci_select._config_enables(cfg, ['CFG_TUH_MIDI'])) + + def test_literal_zero_is_disabled(self): + import tempfile + with tempfile.TemporaryDirectory() as td: + cfg = os.path.join(td, 'tusb_config.h') + with open(cfg, 'w') as f: + f.write('#define CFG_TUD_CDC 0\n' + '#define CFG_TUD_MSC (0)\n' + '#define CFG_TUD_HID 00\n' + '#define CFG_TUH_HID 0 // typical keyboard + mouse\n' + '#define CFG_TUD_MIDI 01\n' + '#define CFG_TUD_DFU (1)\n') + for m in ('CFG_TUD_CDC', 'CFG_TUD_MSC', 'CFG_TUD_HID', 'CFG_TUH_HID'): + self.assertFalse(ci_select._config_enables(cfg, [m]), m) + for m in ('CFG_TUD_MIDI', 'CFG_TUD_DFU'): + self.assertTrue(ci_select._config_enables(cfg, [m]), m) + self.assertFalse(ci_select._config_enables(cfg, ['CFG_TUD_VIDEO'])) + + def test_two_branch_define_reads_on(self): + # examples/device/uac2_speaker_fb defines CFG_TUD_HID 1 under + # `#if CFG_AUDIO_DEBUG` and 0 in the #else. The default build (CFG_AUDIO_DEBUG + # defaults to 1) compiles the HID class in, so a CFG_TUD_HID change must keep + # this example on both axes - the #else's zero must not decide it. + cfg = os.path.join(REPO, 'examples/device/uac2_speaker_fb/src/tusb_config.h') + self.assertTrue(ci_select._config_enables(cfg, ['CFG_TUD_HID'])) + + def test_any_nonzero_define_wins_over_a_zero_one(self): + import tempfile + with tempfile.TemporaryDirectory() as td: + cfg = os.path.join(td, 'tusb_config.h') + with open(cfg, 'w') as f: + f.write('#if FOO\n#define CFG_TUD_MSC 1\n#else\n' + '#define CFG_TUD_MSC 0\n#endif\n' + '#if BAR\n#define CFG_TUD_CDC 0\n#else\n' + '#define CFG_TUD_CDC (0)\n#endif\n') + self.assertTrue(ci_select._config_enables(cfg, ['CFG_TUD_MSC'])) + self.assertFalse(ci_select._config_enables(cfg, ['CFG_TUD_CDC'])) + + def test_midi_host_change_selects_midi_rx(self): + s = ci_select.classify_build(['src/class/midi/midi_host.c'], REPO) + self.assertFalse(s['full']) + self.assertTrue(s['families'], 'a TUH_MIDI change must select some family') + self.assertTrue(any('host/midi_rx' in exs + for exs in s['family_examples'].values()), + s['family_examples']) + + +class TestPruneUsesEveryFamilyBoard(unittest.TestCase): + """CircleCI's cmake legs build EVERY board of a family, so an example gated to + one board (only.txt board:mimxrt1060_evk) must keep its family even though the + family's one-first board cannot build it.""" + + def test_board_gated_example_keeps_its_family(self): + s = ci_select.classify_build( + ['examples/dual/host_hid_to_device_cdc/src/main.c'], REPO) + self.assertFalse(s['full']) + self.assertIn('imxrt', s['families'], s['families']) + self.assertEqual(s['family_examples'].get('imxrt'), + ['dual/host_hid_to_device_cdc']) + + def test_either_build_system_keeps_the_family(self): + """This one family list gates CircleCI's MAKE legs too, and the two build + systems answer skip.txt differently. device/dfu carries mcu:BCM2835, which the + cmake FAMILY_MCUS union (BCM2711 BCM2835) applies to every broadcom_64bit board + and the make scrape applies to none - asking cmake alone drops the only + aarch64-gcc family in the matrix, so build-make-aarch64-gcc silently stops + compiling dfu at all.""" + import build_utils + old = os.getcwd() + os.chdir(REPO) + try: + self.assertTrue(build_utils.skip_example('device/dfu', 'raspberrypi_cm4')) + self.assertFalse(build_utils.skip_example('device/dfu', 'raspberrypi_cm4', + (), 'make')) + finally: + os.chdir(old) + s = ci_select.classify_build(['examples/device/dfu/src/main.c'], REPO) + self.assertIn('broadcom_64bit', s['families'], s['families']) + + +class TestPrunePoolIsBuildPys(unittest.TestCase): + """_prune_buildable asks build.py what each family's build path can see, the same + way for every family - the espressif carve-out lives in build.py.get_examples and + needs no second copy here. Measured identical on all 82 families.""" + + def setUp(self): + import build as build_py + self.build_py = build_py + self.old = os.getcwd() + os.chdir(REPO) # get_examples scans relative paths + + def tearDown(self): + os.chdir(self.old) + + def test_only_espressif_narrows_the_pool(self): + allex = list(ci_select.all_examples(REPO)) + for fam in ci_select.all_bsp_families(REPO): + pool = [e for e in allex if e in set(self.build_py.get_examples(fam))] + if fam == 'espressif': + self.assertNotEqual(pool, allex) # the carve-out is real + else: + self.assertEqual(pool, allex, f'{fam}: build.py narrows this family') + + def test_selections_are_what_the_espressif_only_rule_gave(self): + # espressif's own list is the one value that ever differed from the unfiltered + # example set. Recomputed from build.py rather than pinned as literals: a new + # board, family or example moves the counts, and a suite that fails for that + # teaches people to edit the numbers instead of reading the diff. What is pinned + # is the RELATION - espressif gets exactly the rule's answer narrowed to its own + # pool, every other family gets the answer unnarrowed. + pool = set(self.build_py.get_examples('espressif')) + # the third diff names an example espressif DOES build, so there is nothing for + # the carve-out to remove - it pins that the narrowing does not over-reach + for files, carve in ((['src/portable/synopsys/dwc2/dcd_dwc2.c'], True), + (['src/class/msc/msc_host.c'], True), + (['examples/device/cdc_msc_freertos/src/main.c'], False)): + s = ci_select.classify_build(files, REPO) + self.assertFalse(s['full'], files) + self.assertIn('espressif', s['families'], files) + esp = set(s['family_examples'].get('espressif') or []) + self.assertTrue(esp, f'{files}: espressif selected nothing') + # the pool narrowing is what _prune_buildable adds here, so it must hold... + self.assertTrue(esp <= pool, f'{files}: {sorted(esp - pool)} is outside the pool') + # ...and it must actually bite: some other family was given an example that + # espressif's build path cannot see, and espressif did not get it + other = set().union(*(set(v) for f, v in s['family_examples'].items() + if f != 'espressif'), set()) + self.assertEqual(bool(other - pool), carve, + f'{files}: carve-out expected={carve}, other-side extras ' + f'{sorted(other - pool)}') + self.assertFalse(esp & (other - pool), files) + + +class TestGetDepsExampleShim(unittest.TestCase): + """hil_ci_set_matrix emits `-b <board> -e role/name` entries that .github/actions/ + get_deps and build.yml's hfp job hand verbatim to get_deps.py. argparse must not + reject -e there (exit 2 = every PR's Get Dependencies step red).""" + + # get_deps.main() with its process pool stubbed out: argparse runs for real, + # nothing is cloned (this suite also runs on GitHub's bare pre-commit runner) + CODE = ('import sys\n' + 'import get_deps\n' + 'class P:\n' + ' def __enter__(self): return self\n' + ' def __exit__(self, *a): return False\n' + ' def map(self, fn, items): return [0] * len(items)\n' + 'get_deps.Pool = P\n' + "sys.argv = ['get_deps.py'] + sys.argv[1:]\n" + 'sys.exit(get_deps.main())\n') + + def run_get_deps(self, *args): + env = dict(os.environ, PYTHONPATH=os.path.join(REPO, 'tools')) + return subprocess.run([sys.executable, '-c', self.CODE, *args], + capture_output=True, text=True, cwd=REPO, env=env) + + def test_example_flag_is_accepted(self): + r = self.run_get_deps('-b', 'stm32f407disco', '-e', 'device/cdc_msc') + self.assertNotIn('unrecognized arguments', r.stderr) + self.assertEqual(r.returncode, 0, r.stderr) + + def test_plain_board_still_works(self): + r = self.run_get_deps('-b', 'stm32f407disco') + self.assertEqual(r.returncode, 0, r.stderr) + + +if __name__ == '__main__': + unittest.main(verbosity=1) diff --git a/test/hil/test/test_hil_select.py b/test/hil/test/test_hil_select.py deleted file mode 100644 index 9a1261878..000000000 --- a/test/hil/test/test_hil_select.py +++ /dev/null @@ -1,689 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MIT -# Unit tests for hil_select.py — pure logic, no hardware, no git. Run directly: -# python3 test/hil/test/test_hil_select.py -# -# Imports stay stdlib + hil_select/hil_util/hil_flash ONLY: the pre-commit hil-test -# hook runs this suite, on GitHub's bare runner in the pre-commit workflow as well as -# locally, and that runner has no pyserial/pymtp. hil_flash is admissible because it -# is stdlib + hil_util only (test_hil_util.BottomLayer enforces the stdlib closure of -# both) and the roster-dispatch tests need its flash_* table; never import hil_test, -# which pulls pyserial. -import glob -import json -import os -import sys -import unittest - -# the modules under test live in the parent dir (test/hil), not here -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -import hil_flash -from helper import hil_select -from helper.hil_util import device_tests, dual_tests - -REPO = os.path.dirname(os.path.dirname(os.path.dirname( - os.path.dirname(os.path.abspath(__file__))))) - - -def real_rosters(): - """The actual rig rosters, for regression tests that need real-world data - (a specific board/family/only-list) rather than the synthetic ROSTER above.""" - rosters = [] - for name in ('tinyusb.json', 'hfp.json'): - path = os.path.join(REPO, 'test/hil', name) - with open(path) as f: - rosters.append((f'test/hil/{name}', json.load(f)['boards'])) - return rosters - - -def roster_flashers(): - """(roster path, board) for every board in the live rosters, `boards-skip` - included: a parked board's flasher name must still dispatch, so that unparking it - is not what discovers the name went stale.""" - for name in ('tinyusb.json', 'hfp.json'): - path = os.path.join(REPO, 'test/hil', name) - with open(path) as f: - cfg = json.load(f) - for key in ('boards', 'boards-skip'): - for b in cfg.get(key, []): - yield f'test/hil/{name}', b - - -def on_roster(tc, *names): - """The subset of `names` currently in the live rig rosters, skipping the test - when none are, because parking/unparking a board is routine rig maintenance. - - That skip now matters MORE than it used to, not less: this suite is a blocking - pre-commit hook AND build.yml's selector steps gate on it (a failing suite falls - open to the full matrix), so an assertion that depends on a specific board being - present goes red on every PR -- including src/-only ones that never touched the - rig -- until someone fixes the roster. Keep roster-dependent assertions behind - on_roster.""" - have = {b['name'] for _, boards in real_rosters() for b in boards} - got = [n for n in names if n in have] - if not got: - tc.skipTest(f'not in the rig roster: {", ".join(names)}') - return got - - -ROSTER = [ - # device-only, rp2040 family - {'name': 'raspberry_pi_pico', 'uid': 'u1', 'flasher': {'name': 'openocd'}, - 'tests': {'device': True, 'host': True, 'dual': True}}, - # device-only, stm32f4 family - {'name': 'stm32f407disco', 'uid': 'u2', 'flasher': {'name': 'jlink'}, - 'tests': {'device': True, 'host': False, 'dual': False}}, - # host-only board - {'name': 'raspberry_pi_pico2', 'uid': 'u3', 'flasher': {'name': 'openocd'}, - 'tests': {'device': False, 'host': True, 'dual': False}}, - # only-list board (espressif-style), flashed by the CI leg that splits on esptool - {'name': 'espressif_s3_devkitm', 'uid': 'u4', 'flasher': {'name': 'esptool'}, - 'tests': {'only': ['device/cdc_msc_freertos', 'host/device_info']}}, -] -ROSTERS = [('test/hil/tinyusb.json', ROSTER)] - - -def sel(files): - return hil_select.classify(files, REPO, ROSTERS) - - -class TestPortRule(unittest.TestCase): - def test_dcd_rp2040_selects_pico_family_only(self): - s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) - self.assertFalse(s['full']) - self.assertIn('raspberry_pi_pico', s['boards']) - self.assertNotIn('stm32f407disco', s['boards']) - self.assertNotIn('espressif_s3_devkitm', s['boards']) - # device role: no host tests in pico's list - self.assertTrue(all(not t.startswith('host/') for t in s['boards']['raspberry_pi_pico'])) - # host-only boards drop out entirely on a device-role change - self.assertNotIn('raspberry_pi_pico2', s['boards']) - - def test_shared_port_file_is_both_roles(self): - s = sel(['src/portable/synopsys/dwc2/dwc2_common.c']) - self.assertFalse(s['full']) - self.assertNotIn('raspberry_pi_pico', s['boards']) # rp2040 is not a dwc2 family - self.assertIn('stm32f407disco', s['boards']) # stm32f4 is - - -class TestCoreRoleRule(unittest.TestCase): - def test_usbd_selects_all_device_tests_everywhere(self): - s = sel(['src/device/usbd.c']) - self.assertFalse(s['full']) - self.assertNotIn('raspberry_pi_pico2', s['boards']) # host-only board dropped - pico = s['boards']['raspberry_pi_pico'] - self.assertTrue(set(device_tests).issubset(set(pico))) - self.assertTrue(set(dual_tests).issubset(set(pico))) # dual survives device role - self.assertTrue(all(not t.startswith('host/') for t in pico)) - # only-list board: selection intersects its only-list - esp = s['boards']['espressif_s3_devkitm'] - self.assertEqual(esp, ['device/cdc_msc_freertos']) - - def test_host_change_drops_device(self): - s = sel(['src/host/usbh.c']) - self.assertFalse(s['full']) - self.assertIn('raspberry_pi_pico2', s['boards']) - self.assertNotIn('stm32f407disco', s['boards']) # device-only board dropped - - -class TestClassRule(unittest.TestCase): - def test_cdc_device_selects_cdc_examples_only(self): - s = sel(['src/class/cdc/cdc_device.c']) - self.assertFalse(s['full']) - pico = s['boards']['raspberry_pi_pico'] - self.assertIn('device/cdc_msc', pico) - self.assertIn('device/cdc_dual_ports', pico) - self.assertNotIn('device/msc_dual_lun', pico) # CFG_TUD_CDC 0 there - self.assertNotIn('device/usbtest', pico) # CFG_TUD_CDC 0 there - self.assertTrue(all(not t.startswith('host/') for t in pico)) - - def test_msc_host_selects_host_side(self): - s = sel(['src/class/msc/msc_host.c']) - self.assertFalse(s['full']) - self.assertNotIn('stm32f407disco', s['boards']) # device-only board - pico2 = s['boards']['raspberry_pi_pico2'] - self.assertIn('host/msc_file_explorer', pico2) - self.assertTrue(all(not t.startswith('device/') for t in pico2)) - - -class TestClassIncludeEdges(unittest.TestCase): - """A class header another class includes reaches that class's examples too. - src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, so - midi_test's firmware contains audio.h - but the class rule derives macros from - the directory name alone, so an audio.h change used to select only - device/audio_test_freertos. On boards that skip that example the per-board - intersection emptied and an audio.h-only PR ran ZERO HIL on them.""" - def test_edges_derived_from_includes(self): - edges = hil_select.class_include_edges(REPO) - self.assertEqual(edges.get('audio/audio.h'), {'midi'}) - self.assertEqual(edges.get('cdc/cdc.h'), {'net'}) - - def test_audio_header_selects_midi_example(self): - s = hil_select.classify(['src/class/audio/audio.h'], REPO, real_rosters()) - self.assertFalse(s['full']) - # every board that runs device/midi_test at all must run it here (boards with - # a tests.only list, e.g. espressif, run the freertos examples instead) - by_name = {b['name']: b for _, bs in real_rosters() for b in bs} - checked = 0 - for name, tests in s['boards'].items(): - if 'device/midi_test' in hil_select.board_tests(by_name[name]): - self.assertIn('device/midi_test', tests, name) - checked += 1 - self.assertTrue(checked) - - def test_audio_header_reaches_boards_that_skip_audio(self): - # both skip device/audio_test_freertos: without the midi edge their - # intersection is empty and they drop out of the selection entirely - boards = on_roster(self, 'metro_m4_express', 'nrf54lm20dk') - s = hil_select.classify(['src/class/audio/audio.h'], REPO, real_rosters()) - for board in boards: - self.assertEqual(s['boards'].get(board), ['device/midi_test'], board) - - def test_edge_is_per_header_not_per_class(self): - # midi includes audio.h, not audio_device.h: an audio_device change must - # not drag midi's examples in - s = hil_select.classify(['src/class/audio/audio_device.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for tests in s['boards'].values(): - if tests != 'all': - self.assertNotIn('device/midi_test', tests) - - -class TestFallbackRules(unittest.TestCase): - def test_unknown_tool_is_full(self): - s = sel(['tools/random_new_script.py']) - self.assertTrue(s['full']) - - def test_docs_only_is_empty_not_full(self): - s = sel(['docs/info/contributing.rst', 'README.rst']) - self.assertFalse(s['full']) - self.assertEqual(s['boards'], {}) - - def test_bsp_family_selects_family_boards(self): - s = sel(['hw/bsp/rp2040/family.cmake']) - self.assertFalse(s['full']) - self.assertIn('raspberry_pi_pico', s['boards']) - self.assertEqual(s['boards']['raspberry_pi_pico'], 'all') - self.assertNotIn('stm32f407disco', s['boards']) - - def test_bsp_board_narrows_to_board(self): - s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) - self.assertFalse(s['full']) - self.assertEqual(list(s['boards'].keys()), ['raspberry_pi_pico']) - - def test_example_change_selects_that_example(self): - s = sel(['examples/device/cdc_msc/src/main.c']) - self.assertFalse(s['full']) - self.assertEqual(s['boards']['raspberry_pi_pico'], ['device/cdc_msc']) - - def test_core_common_is_full(self): - for f in ['src/tusb.c', 'src/common/tusb_fifo.c', 'src/osal/osal_freertos.h']: - self.assertTrue(sel([f])['full'], f) - - def test_board_test_example_is_full(self): - # board_test is the park/teardown firmware hil_test.py flashes on every board, - # not an unlisted example: a regression there must not skip the whole rig - for f in ['examples/device/board_test/src/main.c', - 'examples/device/board_test/CMakeLists.txt']: - self.assertTrue(sel([f])['full'], f) - - def test_harness_is_full(self): - for f in ['test/hil/hil_test.py', '.github/workflows/build.yml', 'hw/mcu/nxp/x.c', 'lib/foo/x.c']: - self.assertTrue(sel([f])['full'], f) - - def test_mixed_roles_no_pruning(self): - s = sel(['src/device/usbd.c', 'src/host/usbh.c']) - self.assertFalse(s['full']) - self.assertIn('raspberry_pi_pico2', s['boards']) - self.assertIn('stm32f407disco', s['boards']) - - def test_cmakelists_and_requirements_are_full(self): - for f in ['src/CMakeLists.txt', 'examples/CMakeLists.txt', - 'examples/device/CMakeLists.txt', 'test/hil/requirements.txt']: - self.assertTrue(sel([f])['full'], f) - - def test_docs_txt_is_noncode(self): - s = sel(['docs/info/changelog.txt']) - self.assertFalse(s['full']) - self.assertEqual(s['boards'], {}) - - -class TestArgsEmission(unittest.TestCase): - def test_args_for_scoped_selection(self): - s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) - args = hil_select.selection_args(s, ROSTERS) - a = args['tinyusb.json'] - self.assertIn('-b raspberry_pi_pico', a) - self.assertNotIn('stm32f407disco', a) - self.assertIn('-bt raspberry_pi_pico:', a) # device-only subset of a device+host board - - def test_args_full_is_empty(self): - s = sel(['tools/random_new_script.py']) - self.assertEqual(hil_select.selection_args(s, ROSTERS), {'tinyusb.json': ''}) - - def test_args_all_board_gets_bare_b(self): - s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) - a = hil_select.selection_args(s, ROSTERS)['tinyusb.json'] - self.assertIn('-b raspberry_pi_pico', a) - self.assertNotIn('-bt', a) - - def test_args_by_flasher_splits_esp_from_the_rest(self): - s = sel(['src/device/usbd.c']) - per = hil_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json'] - self.assertIn('espressif_s3_devkitm', per['esptool']) - self.assertIn('raspberry_pi_pico', per['openocd']) - self.assertNotIn('espressif_s3_devkitm', per.get('openocd', '') + per.get('jlink', '')) - - def test_args_by_flasher_omits_a_flasher_with_no_selected_board(self): - # the esp CI leg must see no args at all here, not a filter matching zero boards - s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) - per = hil_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json'] - self.assertEqual(per, {'openocd': '-b raspberry_pi_pico'}) - - def test_args_by_flasher_full_is_empty(self): - s = sel(['tools/random_new_script.py']) - self.assertEqual(hil_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}}) - - def test_cli_diff_file(self): - import subprocess, tempfile, json as j - with tempfile.NamedTemporaryFile('w', suffix='.txt', delete=False) as f: - f.write('src/class/cdc/cdc_device.c\n') - path = f.name - r = subprocess.run([sys.executable, os.path.join(REPO, 'test/hil/helper/hil_select.py'), - '--diff-file', path, os.path.join(REPO, 'test/hil/tinyusb.json')], - capture_output=True, text=True) - self.assertEqual(r.returncode, 0, r.stderr) - out = j.loads(r.stdout) - self.assertFalse(out['full']) - self.assertIn('tinyusb.json', out['args']) - self.assertTrue(any('cdc_device' in line for line in out['reasons'])) - # A core-class diff must select boards THROUGH THE CLI: the in-process tests - # inject their own repo root, so only this subprocess path catches a broken - # repo_root derivation -- which once made every repo-relative glob match - # nothing and turned this exact diff into a silent full-HIL skip. - self.assertTrue(out['boards'], - 'CLI selected zero boards for a src/class change: repo_root broken?') - os.unlink(path) - - -class TestRealRosterPortFamilies(unittest.TestCase): - """Regression for port_families() missing espressif's dwc2 reference, which - lives in a component CMakeLists.txt rather than family.cmake/family.mk.""" - def test_dwc2_change_selects_espressif_boards(self): - boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') - s = hil_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - self.assertIn(board, s['boards']) - - -class TestOptionGatedPort(unittest.TestCase): - """Regression: family_support.cmake compiles some ports from a build option - (MAX3421_HOST=1 -> hcd_max3421.c), so a board's family file never names them.""" - # host-side option board (max3421 as host controller), off any max3421 family - OPT_ROSTER = [('test/hil/opt.json', [ - {'name': 'fake_dual_board', 'uid': 'o1', 'flasher': {'name': 'jlink'}, - 'build': {'args': ['MAX3421_HOST=1']}, - 'tests': {'device': True, 'host': False, 'dual': True}}, - {'name': 'fake_host_board', 'uid': 'o2', 'flasher': {'name': 'jlink'}, - 'variant': [{'name': 'fake_host_board', 'flags': '-DMAX3421_HOST=1'}], - 'tests': {'device': False, 'host': True, 'dual': False}}, - {'name': 'fake_off_board', 'uid': 'o3', 'flasher': {'name': 'jlink'}, - 'variant': [{'name': 'fake_off_board', 'defines': ['MAX3421_HOST=0']}], - 'tests': {'device': True, 'host': True, 'dual': True}}, - ])] - - def test_real_roster_max3421_selects_option_board(self): - boards = on_roster(self, 'metro_m4_express') - s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - self.assertIn(board, s['boards']) - - def test_option_selects_via_args_defines_and_flags(self): - s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, self.OPT_ROSTER) - self.assertFalse(s['full']) - self.assertIn('fake_dual_board', s['boards']) # build.args - self.assertIn('fake_host_board', s['boards']) # variant flags - self.assertNotIn('fake_off_board', s['boards']) # variant defines, but =0 - - def test_device_role_port_does_not_pull_host_only_option_board(self): - s = hil_select.classify(['src/portable/analog/max3421/dcd_max3421.c'], REPO, self.OPT_ROSTER) - self.assertFalse(s['full']) - self.assertNotIn('fake_host_board', s['boards']) # host-only board, device change - self.assertIn('fake_dual_board', s['boards']) # device-capable option board - - def test_gates_parsed_from_family_support(self): - self.assertEqual(hil_select.port_option_gates(REPO).get('analog/max3421'), - {'MAX3421_HOST'}) - - def test_board_cmake_option_counts(self): - """A board can enable a gated port in its own BSP rather than via the roster - (hw/bsp/espressif/boards/*/board.cmake -> set(MAX3421_HOST 1)); board_options() - must see those too, or such a board joining the roster is silently dropped.""" - self.assertIn('MAX3421_HOST', - hil_select.bsp_board_options('adafruit_feather_esp32s3', REPO)) - self.assertIn('CFG_TUH_RPI_PIO_USB', - hil_select.bsp_board_options('adafruit_fruit_jam', REPO)) - # commented-out `# set(MAX3421_HOST 1)` must not count - self.assertNotIn('MAX3421_HOST', - hil_select.bsp_board_options('feather_nrf52840_express', REPO)) - - def test_board_cmake_option_selects_off_family_board(self): - # adafruit_feather_esp32s3 is not on any rig roster; stand it in as one to - # prove the BSP-sourced option alone pulls a max3421 change onto the board - roster = [('test/hil/opt.json', [ - {'name': 'adafruit_feather_esp32s3', 'uid': 'o1', 'flasher': {'name': 'esptool'}, - 'tests': {'device': False, 'host': True, 'dual': False}}])] - s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster) - self.assertFalse(s['full']) - self.assertIn('adafruit_feather_esp32s3', s['boards']) - - def test_board_mk_option_is_ignored(self): - """Make-only options must not select: HIL CI builds with CMake exclusively, so - hw/bsp/nrf/boards/nrf5340dk/board.mk's MAX3421_HOST compiles nothing here.""" - roster = [('test/hil/opt.json', [ - {'name': 'nrf5340dk', 'uid': 'o1', 'flasher': {'name': 'jlink'}, - 'tests': {'device': False, 'host': True, 'dual': False}}])] - s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster) - self.assertFalse(s['full']) - self.assertEqual(s['boards'], {}) - - -class TestPortFamiliesCmakeOnly(unittest.TestCase): - """port_families() is CMake-only (HIL CI never builds with Make) and matches on - 'port_dir/' so a port dir is not a prefix of a sibling.""" - def test_make_only_family_is_not_a_family(self): - # hw/bsp/pic32mz has family.mk but no family.cmake - self.assertEqual(hil_select.port_families('microchip/pic32mz', REPO), set()) - - def test_prefix_port_does_not_inherit_sibling_families(self): - # bare-substring matching let 'microchip/pic' match '.../microchip/pic32mz/...' - self.assertEqual(hil_select.port_families('microchip/pic', REPO), set()) - - def test_make_only_port_forces_full(self): - s = sel(['src/portable/microchip/pic32mz/dcd_pic32mz.c']) - self.assertTrue(s['full']) - self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons']) - - def test_cmake_families_still_found(self): - self.assertEqual(hil_select.port_families('raspberrypi/rp2040', REPO), {'rp2040'}) - self.assertIn('stm32f4', hil_select.port_families('synopsys/dwc2', REPO)) - - -class TestPortFamiliesCoverage(unittest.TestCase): - """Systematic guard: every real dcd_*/hcd_* port directory should map to at - least one board family, so a future family.cmake/CMakeLists.txt layout that - port_families() doesn't scan fails loudly instead of silently dropping boards - (as espressif's dwc2 reference did - see TestRealRosterPortFamilies).""" - # Ports with no board family: not a bug, just not wired into any rig board. - # Add here (with a reason) only if port_families() legitimately can't find one. - # A port listed here force-fulls (fail-open), so it is never under-selected. - NO_FAMILY = { - 'template', # reference/example port, not built by any board - # hw/bsp/pic32mz has family.mk only (no family.cmake), and port_families() - # is CMake-only because HIL CI builds every board with CMake - so this port - # is compiled for no HIL board. - 'microchip/pic32mz', - 'microchip/pic', # same: only ever referenced from pic32mz's family.mk - } - - @staticmethod - def _dcd_hcd_ports(): - portable_root = os.path.join(REPO, 'src/portable') - ports = [] - for entry in sorted(os.listdir(portable_root)): - d = os.path.join(portable_root, entry) - if not os.path.isdir(d): - continue - if glob.glob(os.path.join(d, 'dcd_*.c')) or glob.glob(os.path.join(d, 'hcd_*.c')): - ports.append(entry) - continue - for sub in sorted(os.listdir(d)): - sd = os.path.join(d, sub) - if os.path.isdir(sd) and (glob.glob(os.path.join(sd, 'dcd_*.c')) or - glob.glob(os.path.join(sd, 'hcd_*.c'))): - ports.append(f'{entry}/{sub}') - return ports - - def test_every_port_maps_to_a_family(self): - ports = self._dcd_hcd_ports() - self.assertTrue(ports) # sanity: the scan itself found something - for port in ports: - if port in self.NO_FAMILY: - continue - fams = hil_select.port_families(port, REPO) - self.assertTrue(fams, f'{port}: no family references this port ' - f'(port_families() scan gap, or add to NO_FAMILY)') - - -class TestRealRosterOnlyListTests(unittest.TestCase): - """Regression for roster-only-list tests (e.g. espressif's hid_composite_freertos) - being invisible to the selector because it only knew the shared hil_util lists.""" - def test_only_list_example_change_selects_it(self): - boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') - s = hil_select.classify(['examples/device/hid_composite_freertos/src/main.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - self.assertEqual(s['boards'][board], ['device/hid_composite_freertos']) - - def test_class_change_includes_only_list_boards(self): - boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') - s = hil_select.classify(['src/class/hid/hid_device.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - self.assertIn(board, s['boards']) - - -class TestPortAndCoreRoleUseExtras(unittest.TestCase): - """Regression: the port rule and core-role rule must thread the roster-only - test universe (extras) the same way the class rule already does, so a DCD - or device-stack change doesn't silently drop espressif's only-list tests - (e.g. hid_composite_freertos) that aren't in the shared device_tests list.""" - def test_dcd_change_includes_only_list_test(self): - boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') - s = hil_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - tests = s['boards'][board] - self.assertIn('device/hid_composite_freertos', tests) - self.assertIn('device/cdc_msc_freertos', tests) - self.assertIn('device/audio_test_freertos', tests) - self.assertIn('device/usbtest', tests) - - def test_core_device_change_includes_only_list_test(self): - boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') - s = hil_select.classify(['src/device/usbd.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - tests = s['boards'][board] - self.assertIn('device/hid_composite_freertos', tests) - self.assertIn('device/cdc_msc_freertos', tests) - self.assertIn('device/audio_test_freertos', tests) - self.assertIn('device/usbtest', tests) - - def test_host_change_does_not_leak_device_only_list_test(self): - s = hil_select.classify(['src/host/usbh.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board, tests in s['boards'].items(): - if tests == 'all': - continue - self.assertNotIn('device/hid_composite_freertos', tests, board) - - -class TestFamilies(unittest.TestCase): - """`families` exists for consumers that build (not just test) the diff: most - families have no rig board, so `boards` alone would compile nothing for them.""" - def test_off_rig_port_still_reports_family(self): - s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c']) - self.assertFalse(s['full']) - self.assertEqual(s['boards'], {}) # no same7x board on the rig - self.assertEqual(s['families'], ['same7x']) - - def test_port_families_are_reported(self): - s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) - self.assertIn('rp2040', s['families']) - - def test_bsp_family_and_board_report_family(self): - self.assertEqual(sel(['hw/bsp/rp2040/family.cmake'])['families'], ['rp2040']) - self.assertEqual(sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h'])['families'], - ['rp2040']) - - def test_docs_only_has_no_families(self): - self.assertEqual(sel(['docs/info/contributing.rst'])['families'], []) - - def test_full_selection_still_reports_families(self): - """A full-matrix file must not hide the families of the other changed files: - consumers that build from `families` (e.g. /pre-pr) ignore `boards` when full.""" - s = sel(['src/common/tusb_fifo.c', 'src/portable/microchip/samx7x/dcd_samx7x.c']) - self.assertTrue(s['full']) - self.assertIn('same7x', s['families']) - # full stays full: every roster board, and no args to narrow the run - self.assertEqual(set(s['boards']), {b['name'] for b in ROSTER}) - self.assertTrue(all(v == 'all' for v in s['boards'].values())) - self.assertEqual(hil_select.selection_args(s, ROSTERS), {'tinyusb.json': ''}) - self.assertEqual(hil_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}}) - - def test_family_order_does_not_matter(self): - # same as above with the full-matrix file last (was the only order that worked) - s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c', 'src/common/tusb_fifo.c']) - self.assertTrue(s['full']) - self.assertIn('same7x', s['families']) - - -class TestGitDiffArgv(unittest.TestCase): - def test_diff_disables_rename_detection(self): - """Without --no-renames git reports only a rename's destination, so moving an - HIL-relevant file to a non-code path would be classified as non-code only.""" - self.assertIn('--no-renames', hil_select.GIT_DIFF_ARGV) - - -class TestPortWithoutFamilyIsFull(unittest.TestCase): - """A port dir no family file references must widen (full matrix), not silently - contribute zero boards — the fail-open contract.""" - def test_unreferenced_port_forces_full(self): - orig = hil_select.port_families - hil_select.port_families = lambda port_dir, repo_root: set() - try: - s = sel(['src/portable/vendor/newip/dcd_newip.c']) - finally: - hil_select.port_families = orig - self.assertTrue(s['full']) - self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons']) - - -class TestOpenocdVidPid(unittest.TestCase): - """The roster's optional flasher `vid_pid` field (openocd-verbatim, e.g. - "0x1a86 0x8010", more pairs appended) pins openocd's probe discovery so it - never opens foreign usbfs nodes. It must be emitted BEFORE the args: the - rescue cfgs run `init` internally (rp2350-rescue.cfg errors on any - config-stage command after its init; rp2040.cfg under RESCUE scans before a - trailing flag is even parsed), and no rig cfg sets a competing list - (the 2026-08-10 convoy mechanism).""" - - def test_vid_pid_flag_precedes_args(self): - cmd = hil_flash._openocd_cmd_base( - {'uid': 'S1', 'args': '-f target/wch-riscv.cfg', 'vid_pid': '0x1a86 0x8010'}) - self.assertIn('-c "adapter usb vid_pid 0x1a86 0x8010" -f target/wch-riscv.cfg', cmd) - self.assertTrue(cmd.endswith('-f target/wch-riscv.cfg'), cmd) - - def test_rescue_cfg_command_keeps_vid_pid_before_init(self): - """rescue_openocd swaps the target cfg for one that runs `init` internally; - a vid_pid flag after the args would error there (rp2350) or be skipped - (rp2040) -- in exactly the wedged-rig scenario the pin exists for.""" - flasher = {'name': 'openocd', 'uid': 'S1', 'vid_pid': '0x2e8a 0x000c', - 'args': '-c "set RESCUE 1" -f target/rp2040.cfg'} - cmd = hil_flash._openocd_cmd_base(flasher) - self.assertLess(cmd.index('adapter usb vid_pid'), cmd.index('-f target/'), cmd) - - def test_vid_pid_multiple_pairs(self): - cmd = hil_flash._openocd_cmd_base( - {'uid': 'S1', 'args': '-f i.cfg', 'vid_pid': '0x2e8a 0x000c 0x2e8a 0x000d'}) - self.assertIn('-c "adapter usb vid_pid 0x2e8a 0x000c 0x2e8a 0x000d"', cmd) - - def test_no_field_no_flag_but_warns(self): - # the roster lint only covers the committed rosters; a dev PC's local.json entry - # without the field must at least say what it is giving up -- on STDERR, since - # hil_test captures stdout per test and would swallow it on a passing run - import io - from contextlib import redirect_stderr - hil_flash._VID_PID_WARNED.discard('S-warn') - cap = io.StringIO() - with redirect_stderr(cap): - cmd = hil_flash._openocd_cmd_base({'uid': 'S-warn', 'args': '-f i.cfg'}) - self.assertNotIn('vid_pid', cmd) - self.assertIn('vid_pid', cap.getvalue()) - - def test_roster_openocd_entries_all_pin_vid_pid(self): - # every openocd probe on the rig has a known VID/PID; a new entry without the - # pin silently reintroduces open-everything discovery - for path, board in roster_flashers(): - f = board['flasher'] - # tinyusb.json only: hfp.json is the hifiphile rig owner's file, and a - # blocking repo-wide lint over someone else's roster would red every PR the - # moment they add an openocd board (hil_flash treats the field as optional) - if f['name'] == 'openocd' and path.endswith('tinyusb.json'): - self.assertIn('vid_pid', f, - f"{path}: {board['name']} openocd flasher lacks vid_pid") - self.assertNotIn('vid_pid', f.get('args', ''), - f"{path}: {board['name']} packs vid_pid into args; use the field") - - -class TestRosterFlashersDispatch(unittest.TestCase): - """hil_test and hil_pool_check resolve a board's flasher with a bare - getattr(hil_flash, f'flash_{name}'), and hil_test does it inside a redirect_stdout — - so a renamed or typo'd roster name raises an AttributeError whose output is swallowed, - with nothing pointing at the roster as the thing to edit. Renaming a flash_*/reset_* - pair without updating every roster must fail here instead.""" - - def test_flash_and_reset_exist_for_every_roster_flasher(self): - for path, board in roster_flashers(): - name = board['flasher']['name'].lower() - for fn in (f'flash_{name}', f'reset_{name}'): - self.assertTrue(callable(getattr(hil_flash, fn, None)), - f'{path}: {board["name"]} uses flasher "{name}" ' - f'but hil_flash.{fn} does not exist') - - def test_firmware_suffix_known_for_every_roster_flasher(self): - """find_firmware falls back to accepting .elf-or-.bin when a flasher is missing - from FLASHER_SUFFIX, silently restoring the mismatch that map exists to catch.""" - for path, board in roster_flashers(): - name = board['flasher']['name'].lower() - self.assertIn(name, hil_flash.FLASHER_SUFFIX, - f'{path}: {board["name"]} uses flasher "{name}" ' - f'with no hil_flash.FLASHER_SUFFIX entry') - - -class FlasherRecoverEntry(unittest.TestCase): - """Optional roster key: a SECOND flasher used only to deliver recovery while a usbfs - node is poisoned. Boards whose primary flasher cannot get past a convoy (jlink, - stlink, lm4flash) name an openocd entry here instead of changing how they are - normally flashed.""" - - def test_recover_flasher_prefers_the_optional_entry(self): - prim = {'name': 'jlink', 'uid': 'X', 'args': '-device MIMXRT1064xxx6A'} - rec = {'name': 'openocd', 'uid': 'X', 'args': '-f interface/jlink.cfg -f target/foo.cfg'} - self.assertEqual(hil_flash.recover_flasher({'flasher': prim, 'flasher_recover': rec}), rec) - self.assertEqual(hil_flash.recover_flasher({'flasher': prim}), prim) - - def test_openocd_over_jlink_is_convoy_safe_without_a_pin(self): - """libjaylink discovery returns early unless idVendor == 0x1366 (SEGGER) and the PID - is in its table, and only THEN calls libusb_open (discovery_usb.c) -- it never opens - a foreign node. `adapter usb vid_pid` is a no-op for this driver: jlink.c reads - adapter_serial / usb address / usb location, never the vid/pid.""" - self.assertTrue(hil_flash.convoy_safe( - {'name': 'openocd', 'args': '-f interface/jlink.cfg -f target/stm32f4x.cfg'})) - - def test_openocd_with_neither_a_pin_nor_jlink_is_not_safe(self): - self.assertFalse(hil_flash.convoy_safe( - {'name': 'openocd', 'args': '-f interface/stlink.cfg -f target/stm32h7x.cfg'})) - - def test_the_existing_rules_are_unchanged(self): - self.assertTrue(hil_flash.convoy_safe( - {'name': 'openocd', 'vid_pid': '0x2e8a 0x000c', 'args': '-f interface/cmsis-dap.cfg'})) - self.assertFalse(hil_flash.convoy_safe({'name': 'jlink', 'uid': 'X'})) - self.assertTrue(hil_flash.convoy_safe({'name': 'esptool'})) - - -if __name__ == '__main__': - unittest.main(verbosity=1) diff --git a/test/hil/test/test_hil_util.py b/test/hil/test/test_hil_util.py index 9c3d5edef..c95e20b6d 100644 --- a/test/hil/test/test_hil_util.py +++ b/test/hil/test/test_hil_util.py @@ -98,7 +98,7 @@ class RunCmdModes(unittest.TestCase): class BottomLayer(unittest.TestCase): def test_bad_timeout_env_falls_back(self): - # hil_select (the PR-diff selector) imports hil_util for the example rosters; + # ci_select (the PR-diff selector) imports hil_util for the example rosters; # a malformed HIL_CMD_TIMEOUT must not crash the selector at import and knock # CI back to the full-matrix fallback import subprocess @@ -108,7 +108,7 @@ class BottomLayer(unittest.TestCase): env={**os.environ, 'HIL_CMD_TIMEOUT': 'bogus'}, capture_output=True, text=True, timeout=30) self.assertEqual(r.returncode, 0, r.stderr) - # the warning must NOT be on stdout: hil_select's stdout is machine-read JSON + # the warning must NOT be on stdout: ci_select's stdout is machine-read JSON self.assertEqual(r.stdout.strip(), '180') self.assertIn('warning', r.stderr) # but a silent fallback hides the misconfiguration @@ -132,22 +132,23 @@ class BottomLayer(unittest.TestCase): # hil_examples.py used to make this structural (a list of strings cannot grow a # dependency); with the rosters folded into hil_util the invariant needs teeth: # everything the bare GitHub runner imports (selector + this suite) must stay - # stdlib + local. Adding pyserial/pymtp here breaks hil_select on CI. + # stdlib + local. Adding pyserial/pymtp here breaks ci_select on CI. import ast hil_dir = Path(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # ONLY the modules the bare runner can import -- not every stem in the tree. # Globbing the directory allowed `import pymtp` (and hil_test, usbtest, # mtp_test) through, so the pymtp case this test names could never fail: that # module runs ctypes.CDLL(find_library('mtp')) at import and raises where there - # is no libmtp, taking hil_select down with it. - local = {'helper', 'hil_util', 'hil_select', 'hil_flash', - 'hil_health', 'hil_lock', 'hil_pool_check'} + # is no libmtp, taking ci_select down with it. + local = {'helper', 'hil_util', 'ci_select', 'hil_flash', + 'hil_health', 'hil_lock', 'hil_pool_check', 'build', 'build_utils'} allowed = set(sys.stdlib_module_names) | local # hil_pool_check included: test_hil_util_is_a_single_module_instance imports it # on the bare runner, and its `import serial` is function-local for exactly # this reason -- hoisting it must fail HERE, not on every PR's pre-commit CI - for mod in ('helper/hil_util', 'hil_flash', 'helper/hil_select', - 'helper/hil_health', 'helper/hil_lock', 'helper/hil_pool_check'): + for mod in ('helper/hil_util', 'hil_flash', '../../tools/ci_select', + 'helper/hil_health', 'helper/hil_lock', 'helper/hil_pool_check', + '../../tools/build', '../../tools/build_utils'): tree = ast.parse((hil_dir / f'{mod}.py').read_text()) # module level only: a deferred import inside a function cannot break # importability (hil_pool_check keeps `import serial` function-local diff --git a/test/hil/tinyusb.json b/test/hil/tinyusb.json index 6f552f126..8fd4683a4 100644 --- a/test/hil/tinyusb.json +++ b/test/hil/tinyusb.json @@ -157,11 +157,14 @@ { "name": "metro_m4_express", "uid": "9995AD485337433231202020FF100A34", - "build": { - "args": [ - "MAX3421_HOST=1" - ] - }, + "variant": [ + { + "name": "metro_m4_express", + "defines": [ + "MAX3421_HOST=1" + ] + } + ], "tests": { "device": true, "host": false, diff --git a/tools/build.py b/tools/build.py index 51d3d0f70..eeefca22d 100755 --- a/tools/build.py +++ b/tools/build.py @@ -2,6 +2,7 @@ import argparse import random import os +import re import sys import time import subprocess @@ -99,6 +100,53 @@ def get_examples(family): return all_examples +def resolve_example_target_groups(build_targets, examples, board, extra_defines=()): + """Map generic targets onto per-example targets for a filtered build (-e), as ONE + GROUP PER REQUESTED TARGET: 'all' -> the example executables, anything else (e.g. + tinyusb_metrics) passes through as its own single-entry group. + + Grouped rather than flattened because each group becomes one `cmake --build + --target a b c` invocation: the examples of a group build in parallel (flattening + them into one target per invocation serialises the whole leg - measured +39% at + -j4 and +220% at -j32 on stm32f407disco), while separate groups stay ordered, so a + target that must run after the examples still does. + + extra_defines are this build's -D tokens: MAX3421_HOST=1 there decides + only.txt for the max3421 examples (see build_utils.skip_example). + Returns None when no requested example is buildable on this board.""" + buildable = [e for e in examples + if not build_utils.skip_example(e, board, extra_defines)] + if not buildable: + return None + names = list(dict.fromkeys(e.split('/', 1)[1] for e in buildable)) + return [list(names) if t == 'all' else [t] for t in build_targets] + + +_TARGET_HELP_RE = re.compile(r'^([A-Za-z0-9_.+-]+):') +# role/name, the only shape resolve_example_target_groups and the CMake target names accept +EXAMPLE_RE = re.compile(r'[A-Za-z0-9_]+/[A-Za-z0-9_]+') + + +def parse_target_help(text): + """Bare target names out of `cmake --build <dir> --target help`; the Ninja + generator prints one '<name>: phony' line per target. Names containing '/' are + per-directory utility targets (device/edit_cache) or absolute CMakeFiles paths, + never an example target.""" + return {m.group(1) for m in map(_TARGET_HELP_RE.match, text.splitlines()) if m} + + +def cmake_registered_targets(build_dir): + """The targets CMake actually created in build_dir, or None when that cannot be + read. Ground truth: skip.txt/only.txt only mirrors family_filter, so an example + the role CMakeLists never lists (or a stale -e name) still looks buildable to it + and `cmake --build --target <it>` hard-fails. None keeps the mirror's answer.""" + r = subprocess.run(['cmake', '--build', build_dir, '--target', 'help'], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + if r.returncode != 0: + return None + return parse_target_help(r.stdout.decode('utf-8', 'replace')) or None + + def print_build_result(board, build_target, status, duration): if isinstance(duration, (int, float)): duration = "{:.2f}s".format(duration) @@ -107,7 +155,7 @@ def print_build_result(board, build_target, status, duration): # ----------------------------- # CMake # ----------------------------- -def cmake_board(board, build_args, build_name, build_cflags, build_targets): +def cmake_board(board, build_args, build_name, build_cflags, build_targets, examples=None, defines=()): ret = [0, 0, 0] start_time = time.monotonic() @@ -120,8 +168,13 @@ def cmake_board(board, build_args, build_name, build_cflags, build_targets): if family == 'espressif': # for espressif, we have to build example individually all_examples = get_examples(family) + if examples is not None: + all_examples = [e for e in all_examples if e in examples] + if not all_examples: + print_build_result(board, 'examples (PR filter)', 2, '-') + return [0, 0, 1] for example in all_examples: - if build_utils.skip_example(example, board): + if build_utils.skip_example(example, board, defines): ret[2] += 1 else: rcmd = run_cmd([ @@ -130,13 +183,40 @@ def cmake_board(board, build_args, build_name, build_cflags, build_targets): ]) ret[0 if rcmd.returncode == 0 else 1] += 1 else: + # the skip.txt/only.txt prefilter reads no configure output: answer it first, + # so a selection this board builds nothing of costs no cmake run at all + if examples is not None: + examples = [e for e in examples + if not build_utils.skip_example(e, board, defines)] + if not examples: + print_build_result(board, 'examples (PR filter)', 2, '-') + return [0, 0, 1] rcmd = run_cmd(['cmake', 'examples', '-B', build_dir, '-GNinja', f'-DBOARD={board}', '-DCMAKE_BUILD_TYPE=MinSizeRel', '-DLINKERMAP_OPTION=-q -f tinyusb/src', *build_args, *build_flags]) if rcmd.returncode == 0: + target_groups = [[t] for t in build_targets] + if examples is not None: + registered = cmake_registered_targets(build_dir) + if registered is not None: + kept = [e for e in examples if e.split('/', 1)[1] in registered] + for e in examples: + if e not in kept: + print_build_result(board, f'{e} (no such target)', 2, '-') + examples = kept + if not examples: + print_build_result(board, 'examples (no such target)', 2, '-') + return [0, 0, 1] + target_groups = resolve_example_target_groups(build_targets, examples, board, defines) + if registered is None: + # ground truth unavailable, so nothing checked these names against + # what CMake created. ninja validates a whole invocation up front: + # one unknown name in the batch builds NOTHING, where a target each + # builds everything up to it. Give up the parallelism, not the work. + target_groups = [[t] for g in target_groups for t in g] cmd = ["cmake", "--build", build_dir, '--parallel', str(parallel_jobs)] - for target in build_targets: - rcmd = run_cmd(cmd + ['--target', target]) + for group in target_groups: + rcmd = run_cmd(cmd + ['--target'] + group) if rcmd.returncode != 0: break ret[0 if rcmd.returncode == 0 else 1] += 1 @@ -148,9 +228,10 @@ def cmake_board(board, build_args, build_name, build_cflags, build_targets): # ----------------------------- # Make # ----------------------------- -def make_one_example(example, board, make_option, build_targets): - # Check if board is skipped - if build_utils.skip_example(example, board): +def make_one_example(example, board, make_option, build_targets, defines=()): + # Check if board is skipped. Make semantics: family.mk decides, not the + # family.cmake MCU list (see build_utils.skip_example). + if build_utils.skip_example(example, board, defines, build_system='make'): print_build_result(board, example, 2, '-') r = 2 else: @@ -171,10 +252,15 @@ def make_one_example(example, board, make_option, build_targets): return ret -def make_board(board, build_args, build_targets): +def make_board(board, build_args, build_targets, examples=None, defines=()): print(build_separator) family = find_family(board); all_examples = get_examples(family) + if examples is not None: + all_examples = [e for e in all_examples if e in examples] + if not all_examples: + print_build_result(board, 'examples (PR filter)', 2, '-') + return [0, 0, 1] start_time = time.monotonic() ret = [0, 0, 0] if family == 'espressif' or family == 'rp2040': @@ -182,7 +268,7 @@ def make_board(board, build_args, build_targets): final_status = 2 else: with Pool(processes=os.cpu_count()) as pool: - pool_args = list((map(lambda e, b=board, o=f"{build_args}", t=build_targets: [e, b, o, t], all_examples))) + pool_args = list((map(lambda e, b=board, o=f"{build_args}", t=build_targets, d=defines: [e, b, o, t, d], all_examples))) r = pool.starmap(make_one_example, pool_args) # sum all element of same index (column sum) ret = list(map(sum, list(zip(*r)))) @@ -194,36 +280,58 @@ def make_board(board, build_args, build_targets): # ----------------------------- # Build Family # ----------------------------- -def build_boards_list(boards, build_defines, build_system, build_name, build_cflags, build_targets): +def build_boards_list(boards, build_defines, build_system, build_name, build_cflags, build_targets, examples=None): ret = [0, 0, 0] + # the -D tokens are part of the skip.txt/only.txt answer (MAX3421_HOST=1), so + # the -e filter has to see them too; sorted+tuple keeps skip_example cacheable + defines = tuple(sorted(build_defines)) for b in boards: r = [0, 0, 0] if build_system == 'cmake': build_args = [f'-D{d}' for d in build_defines] - r = cmake_board(b, build_args, build_name, build_cflags, build_targets) + r = cmake_board(b, build_args, build_name, build_cflags, build_targets, examples, defines) elif build_system == 'make': build_args = ' '.join(f'{d}' for d in build_defines) - r = make_board(b, build_args, build_targets) + r = make_board(b, build_args, build_targets, examples, defines) ret[0] += r[0] ret[1] += r[1] ret[2] += r[2] return ret -def get_family_boards(family, one_random, one_first): +def get_family_boards(family, one_random, one_first, examples=None, build_system='cmake', + extra_defines=(), ci=None): """Get list of boards for a family. Args: family: Family name one_random: If True, return only one random board one_first: If True, return only the first board (alphabetical) + examples: PR example filter (-e). The one-board pick then prefers a board that + can build at least one of them: the family is in the matrix BECAUSE some + board of it builds these examples (ci_select._prune_buildable asks about + every board, since CircleCI builds every board), but GHA builds one. Without + this, lpc54 selected for host/msc_file_explorer picks lpcxpresso54114 - + which every one of those examples skips - and the leg runs to green having + compiled nothing and uploaded no metrics. + build_system: which skip answer to ask for; the two differ (build_utils) + extra_defines: this build's -D tokens, so a board whose only.txt match comes + from -DMAX3421_HOST=1 is not judged unbuildable here and buildable in + cmake_board + ci: force the ci_skip_boards / ci_preferred_boards lists on or off. Default + None reads the environment, which is right for a build but NOT for a caller + asking what CI would do: ci_select must answer the same on a laptop as on a + runner, or /pre-pr and the code-size skill report a family list CI will not + reproduce. Returns: List of board names """ + if ci is None: + ci = bool(os.getenv('GITHUB_ACTIONS') or os.getenv('CIRCLECI')) skip_list = [] preferred_list = [] - if os.getenv('GITHUB_ACTIONS') or os.getenv('CIRCLECI'): + if ci: skip_list = ci_skip_boards.get(family, []) preferred_list = ci_preferred_boards.get(family, []) @@ -238,12 +346,26 @@ def get_family_boards(family, one_random, one_first): # If only-one flags are set, honor select list first, then pick first or random if one_first or one_random: - if preferred_list: + def buildable(board): + # no filter, or nothing in the filter is buildable anywhere: keep today's + # answer rather than inventing a different board + return examples is None or any( + not build_utils.skip_example(e, board, extra_defines, build_system) + for e in examples) + + # the WHOLE preferred list, in order - stopping at entry one would abandon a + # curated list for the raw alphabetical order the moment its first board cannot + # build the filter, which also moves the board the metrics baseline is keyed on + for b in preferred_list: + if buildable(b): + return [b] + if preferred_list and examples is None: return [preferred_list[0]] + candidates = [b for b in all_boards if buildable(b)] or all_boards if one_first: - return [all_boards[0]] + return [candidates[0]] if one_random: - return [random.choice(all_boards)] + return [random.choice(candidates)] return all_boards @@ -272,6 +394,8 @@ def main(): parser.add_argument('-j', '--jobs', type=int, default=os.cpu_count(), help='Number of jobs to run in parallel') parser.add_argument('-T', '--target', action='append', default=[], help='Build target to use, may be specified multiple times (default: all)') + parser.add_argument('-e', '--example', action='append', default=[], + help='Only build these examples (role/name, repeatable). Default: all examples') parser.add_argument('-v', '--verbose', action='store_true', help='Verbose output') args = parser.parse_args() @@ -285,9 +409,20 @@ def main(): one_random = args.one_random one_first = args.one_first build_targets = args.target if args.target else ['all'] + examples = args.example or None verbose = args.verbose parallel_jobs = args.jobs + for e in args.example: + if not EXAMPLE_RE.fullmatch(e): + parser.error(f"-e/--example takes 'role/name' (e.g. device/cdc_msc), got '{e}'") + # a name no example dir answers to would silently build nothing on every board + # and still exit 0 (every row is a Skipped, and main() returns the FAILED count). + # The -e lists are generated - from ci_select's example map and from HIL roster + # test names - so a stale one must be loud, not green + if not os.path.isdir(os.path.join('examples', e)): + parser.error(f"-e/--example '{e}': no such example directory examples/{e}") + build_defines.append(f'TOOLCHAIN={toolchain}') if len(families) == 0 and len(boards) == 0: @@ -317,10 +452,12 @@ def main(): # get boards from families and append to boards list all_boards = list(boards) for f in all_families: - all_boards.extend(get_family_boards(f, one_random, one_first)) + all_boards.extend(get_family_boards(f, one_random, one_first, examples, + build_system, tuple(build_defines))) # build all boards - result = build_boards_list(all_boards, build_defines, build_system, build_name, build_cflags, build_targets) + result = build_boards_list(all_boards, build_defines, build_system, build_name, build_cflags, build_targets, + examples) total_time = time.monotonic() - total_time print(build_separator) diff --git a/tools/build_utils.py b/tools/build_utils.py index d80ceea7c..1eeef0269 100755 --- a/tools/build_utils.py +++ b/tools/build_utils.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import functools import subprocess import pathlib import re @@ -10,32 +11,190 @@ FAILED = "\033[31mfailed\033[0m" SKIPPED = "\033[33mskipped\033[0m" -def skip_example(example, board): - ex_dir = pathlib.Path('examples/') / example - bsp = pathlib.Path("hw/bsp") +# Every read here is a source file, not user text: decode it the same way on every +# machine. Without this the reads take the locale's encoding, and one of the eight +# tracked non-ASCII files this now touches (hw/bsp/nrf/boards/nrf54lm20dk/board.cmake +# among them) raises UnicodeDecodeError under LC_ALL=C - a ValueError, which sails +# straight through the `except OSError` fail-opens. +_TEXT = {'encoding': 'utf-8', 'errors': 'replace'} - # board within family - board_dir = list(bsp.glob("*/boards/" + board)) - if not board_dir: - # Skip unknown boards - return True +_FAMILY_MCUS_RE = re.compile(r'set\s*\(\s*FAMILY_MCUS\s+([^)]*)\)') +_CMAKE_SET_RE = re.compile(r'set\s*\(\s*([A-Za-z_]\w*)\s+([^)\s]+)') +_CMAKE_VAR_RE = re.compile(r'\$\{([A-Za-z_]\w*)\}') +_CMAKE_CASE_RE = re.compile(r'string\s*\(\s*(TOUPPER|TOLOWER)\s+(\S+)\s+([A-Za-z_]\w*)\s*\)') - board_dir = list(board_dir)[0] - family_dir = board_dir.parent.parent - family = family_dir.name - # family.mk [email protected]_cache(maxsize=None) +def _cmake_sets(path): + """One cmake file's variable assignments as NAME -> first definition seen, as + either a literal value or an ('TOUPPER'|'TOLOWER', source) pair. Only used to + expand ${...} tokens; never mutate the cached dict. + + string(TOUPPER ...) is not decoration: hw/bsp/maxim derives its ONLY FAMILY_MCUS + entry that way (`string(TOUPPER ${MAX_DEVICE} MAX_DEVICE_UPPER)`), as do the eight + at32 families, so dropping those lines left nine families with an empty MCU set.""" + try: + text = pathlib.Path(path).read_text(**_TEXT) + except OSError: + return {} + out = {} + for line in text.splitlines(): + line = line.strip() + if line.startswith('#'): + continue + m = _CMAKE_CASE_RE.match(line) + if m: + # strip quotes like the set() branch below: string(TOUPPER "${VAR}" DST) is + # idiomatic cmake, and keeping them yields a '"NAME"' token that can never + # equal a mcu: entry + out.setdefault(m.group(3), (m.group(1), m.group(2).strip('"'))) + continue + m = _CMAKE_SET_RE.match(line) + if m: + out.setdefault(m.group(1), m.group(2).strip('"')) + return out + + +def _cmake_expand(value, files, depth=0): + """`value` with every ${VAR} replaced, resolving each name against `files` in + order, or None when any name resolves nowhere OR the result still carries a `${`. + That last case is the one _CMAKE_VAR_RE cannot see - a hyphen in the name, a nested + ${${X}}, an unterminated brace - where the loop below finds nothing to substitute + and would otherwise hand the raw text back as if it were a resolved MCU name. + Bounded depth: a cmake file may define a var in terms of another one, and a + self-referential set() must not recurse forever.""" + if depth > 4: + return None + out = value + for name in set(_CMAKE_VAR_RE.findall(value)): + val = None + for f in files: + val = _cmake_sets(f).get(name) + if val is not None: + break + if val is None: + return None + if isinstance(val, tuple): # string(TOUPPER src DST) + src = _cmake_expand(val[1], files, depth + 1) + if src is None: + return None + val = src.upper() if val[0] == 'TOUPPER' else src.lower() + else: + val = _cmake_expand(val, files, depth + 1) + if val is None: + return None + out = out.replace('${' + name + '}', val) + return None if '${' in out else out + + [email protected]_cache(maxsize=None) +def _board_dirs(board): + """(board_dir, family_dir) for a board name, or (None, None). Cached: skip_example + is asked (board x example) times - 566k lstat calls per selector run without this, + since the glob rescans every hw/bsp/*/boards for each example.""" + hits = list(pathlib.Path("hw/bsp").glob("*/boards/" + board)) + if not hits: + return None, None + return hits[0], hits[0].parent.parent + + [email protected]_cache(maxsize=None) +def _family_mcus(family_dir, board_dir): + """The MCU names CMake's family_filter iterates. family_support.cmake:176/190 + loop `foreach(MCU IN LISTS FAMILY_MCUS)`, so a family-wide list (broadcom_64bit + sets "BCM2711 BCM2835") makes ANY of its entries decide skip.txt/only.txt -- not + just the one CFG_TUSB_MCU the configured board names. + + ${...} tokens are expanded from `set(VAR value)` and `string(TOUPPER src VAR)` in + the board's board.cmake first, then in family.cmake: hw/bsp/ra sets + `FAMILY_MCUS RAXXX ${MCU_VARIANT}` and ra6m5_ek/board.cmake sets MCU_VARIANT ra6m5, + which is the token dual/host_info_to_device_cdc/only.txt actually spells; hw/bsp/maxim + sets `FAMILY_MCUS ${MAX_DEVICE_UPPER}`, upper-cased from the board's MAX_DEVICE. A + token resolving nowhere is dropped (nothing can be said about it). + + A family that never spells `set(FAMILY_MCUS ...)` at all gets one more chance: the + name is resolved as a variable, which covers the derived form hw/bsp/espressif uses + (`string(TOUPPER ${IDF_TARGET} FAMILY_MCUS)`). + + Only unconditional set() calls count: nrf and mcx pick FAMILY_MCUS per board + inside if() blocks this does not evaluate, so for those two families the whole + cmake-side MCU set is whatever the CFG_TUSB_MCU scrape in _board_mcu finds. + + nrf: the scrape reads the FIRST CFG_TUSB_MCU token of hw/bsp/nrf/family.mk, so + every nrf board answers NRF54, the NRF5X ones included. Harmless only because no + skip.txt/only.txt names an nrf token today. + + mcx: load-bearing, not academic -- mcu:MCXA15 is live in six examples' skip.txt + (device/{cdc_msc,audio_test,hid_composite,audio_4_channel_mic,midi_test}_freertos + and device/net_lwip_webserver). Those answers come out right only because the + scrape falls through to each board's make-only board.mk, which still spells the + token; an mcx board carrying board.cmake alone (MCU_VARIANT and no CFG_TUSB_MCU) + would scrape 'NONE' and skip EVERY example on it, silently. TestFamilyMcusFallback + fails the day such a board lands. The fix then is to evaluate the + if(MCU_VARIANT STREQUAL ...) branches, not to add another scrape. + """ + fam_cmake = pathlib.Path(family_dir) / "family.cmake" + try: + text = fam_cmake.read_text(**_TEXT) + except OSError: + return frozenset() + board_cmake = pathlib.Path(board_dir) / "board.cmake" + out = set() + depth = 0 + any_set = False + for line in text.splitlines(): + line = line.strip() + m = _FAMILY_MCUS_RE.match(line) + if m: + any_set = True + if m and depth == 0: + files = (str(board_cmake), str(fam_cmake)) + for tok in m.group(1).split(): + if tok in ("CACHE", "INTERNAL") or tok.startswith('"'): + continue + val = _cmake_expand(tok, files) + if val: + out.add(val) + if re.match(r'if\s*\(', line): + depth += 1 + elif re.match(r'endif\s*\(', line): + depth = max(0, depth - 1) + if not out and not any_set: + # FAMILY_MCUS can also be produced rather than set: hw/bsp/espressif derives it + # with `string(TOUPPER ${IDF_TARGET} FAMILY_MCUS)`, which _FAMILY_MCUS_RE cannot + # see, leaving espressif's whole cmake answer resting on the IDF_TARGET scrape. + # + # `not any_set` is load-bearing: _cmake_sets is if()-blind and keeps the FIRST + # definition, so on a family that sets FAMILY_MCUS only inside conditionals + # (mcx, nrf) this would leak branch one's value onto every board - mcx/frdm_mcxn947 + # answered MCXA15, which six examples' skip.txt names, dropping 12 firmware + # images CMake actually builds. Those families keep the CFG_TUSB_MCU scrape. + val = _cmake_expand('${FAMILY_MCUS}', (str(board_cmake), str(fam_cmake))) + if val: + out.add(val) + return frozenset(out) + + [email protected]_cache(maxsize=None) +def _scrape_mcu(family_dir, board_dir, family): + """(CFG_TUSB_MCU token of this board, the text it was read from), master's + algorithm verbatim: family.mk (family.cmake when there is none) first, falling + back to the board's board.mk (board.cmake when there is none) only when the + family file names no token at all. espressif spells its MCU as + `set(IDF_TARGET "...")` instead. The text comes back with it because the make + path reads MAX3421_HOST out of that same single file - which file that is IS + part of master's answer, so it cannot be re-derived by the caller.""" family_mk = family_dir / "family.mk" if not family_mk.exists(): family_mk = family_dir / "family.cmake" - mk_contents = family_mk.read_text() + mk_contents = family_mk.read_text(**_TEXT) # Find the mcu, first in family mk then board mk if "CFG_TUSB_MCU=OPT_MCU_" not in mk_contents: board_mk = board_dir / "board.mk" if not board_mk.exists(): board_mk = board_dir / "board.cmake" - mk_contents = board_mk.read_text() + mk_contents = board_mk.read_text(**_TEXT) mcu = "NONE" if family == "espressif": @@ -53,6 +212,95 @@ def skip_example(example, board): mcu = opt_mcu[len("OPT_MCU_"):] if mcu != "NONE": break + return mcu, mk_contents + + [email protected]_cache(maxsize=None) +def _board_mcu(board_dir, family_dir, family): + """(CFG_TUSB_MCU of this board, MAX3421_HOST enabled by its cmake BSP). + + MAX3421_HOST is read from family.cmake AND board.cmake rather than only the file + the MCU token came from: feather_rp2040_max3421 sets it in its board.cmake while + its MCU token comes from rp2040's family file, and family_support.cmake:940 + appends MAX3421 to FAMILY_MCUS for it. board.mk is deliberately not read - a + make-only option compiles nothing in a cmake build (and the make path answers + with master's own single-file scrape, see _skip_example_make).""" + family_dir = pathlib.Path(family_dir) + board_dir = pathlib.Path(board_dir) + mcu, _ = _scrape_mcu(family_dir, board_dir, family) + if "${" in mcu: + # the scrape is textual, so a computed token comes back verbatim + # (tm4c board.cmake spells OPT_MCU_TM4C${MCU_SUB_VARIANT}, maxim + # OPT_MCU_${MAX_DEVICE_UPPER}). Expand it the same way FAMILY_MCUS tokens are; + # what still will not resolve stays as-is and _skip_example treats it as + # "MCU unknown" rather than silently matching no mcu: token at all. + mcu = _cmake_expand(mcu, (str(board_dir / "board.cmake"), + str(family_dir / "family.cmake"))) or mcu + + max3421_enabled = False + for f in (family_dir / "family.cmake", board_dir / "board.cmake"): + try: + text = f.read_text(**_TEXT) + except OSError: + continue + # a commented-out `# set(MAX3421_HOST 1)` (feather_nrf52840_express) enables + # nothing; master never hit one because it only read the MCU token's file + if any(not l.lstrip().startswith('#') and + ("MAX3421_HOST=1" in l or 'MAX3421_HOST 1' in l) + for l in text.splitlines()): + max3421_enabled = True + break + + return mcu, max3421_enabled + + [email protected]_cache(maxsize=None) +def _filter_tokens(path): + """skip.txt / only.txt as a token set, or None when the file does not exist.""" + f = pathlib.Path(path) + return frozenset(f.read_text(**_TEXT).split()) if f.exists() else None + + +def skip_example(example, board, extra_defines=(), build_system='cmake'): + """Is this example unbuildable on this board, for this build system? + + The two build systems ask DIFFERENT questions and must not share an answer: + + 'cmake' mirrors CMake's family_filter (hw/bsp/family_support.cmake:171-207), + including the whole FAMILY_MCUS list the family.cmake sets. + + 'make' is master's original algorithm, unchanged. family.mk and family.cmake are + not the same build: hw/bsp/lpc54/family.cmake sets FAMILY_MCUS LPC54 and wires the + ohci host sources, while family.mk builds OPT_MCU_LPC54XXX and compiles no HCD + source at all -- feeding the cmake MCU union to a make build un-skips the host + examples only.txt gates on mcu:LPC54 and they fail to link (undefined hcd_init). + + extra_defines: NAME=VALUE tokens the build passes on the command line + (build.py -D). MAX3421_HOST=1 there enables the max3421 host controller + exactly like a BSP that sets it, and family_support.cmake:940 appends MAX3421 + to FAMILY_MCUS for it -- so a roster board whose MAX3421 comes from the build + args (metro_m4_express) must resolve its only.txt the same way. cmake only: + master's make algorithm never looked at them. + """ + return _skip_example(example, board, tuple(extra_defines), build_system) + + [email protected]_cache(maxsize=None) +def _skip_example_make(example, board): + """master's skip_example, verbatim (tools/build_utils.py @ 9c202e8c6): the + make build's own answer, derived from family.mk/board.mk with the single + CFG_TUSB_MCU token that file names. Do not "improve" it -- it is the mirror of + what `make BOARD=... all` actually compiles.""" + ex_dir = pathlib.Path('examples/') / example + + # board within family + board_dir, family_dir = _board_dirs(board) + if board_dir is None: + # Skip unknown boards + return True + family = family_dir.name + + mcu, mk_contents = _scrape_mcu(family_dir, board_dir, family) # Skip all OPT_MCU_NONE these are WIP port if mcu == "NONE": @@ -68,14 +316,14 @@ def skip_example(example, board): only_file = ex_dir / "only.txt" if skip_file.exists(): - skips = skip_file.read_text().split() + skips = skip_file.read_text(**_TEXT).split() if ("mcu:" + mcu in skips or "board:" + board in skips or "family:" + family in skips): return True if only_file.exists(): - onlys = only_file.read_text().split() + onlys = only_file.read_text(**_TEXT).split() if not ("mcu:" + mcu in onlys or ("mcu:MAX3421" in onlys and max3421_enabled) or "board:" + board in onlys or @@ -85,6 +333,55 @@ def skip_example(example, board): return False [email protected]_cache(maxsize=None) +def _skip_example(example, board, extra_defines, build_system): + if build_system == 'make': + return _skip_example_make(example, board) + + ex_dir = pathlib.Path('examples/') / example + + # board within family + board_dir, family_dir = _board_dirs(board) + if board_dir is None: + # Skip unknown boards + return True + family = family_dir.name + + mcu, max3421_enabled = _board_mcu(str(board_dir), str(family_dir), family) + + # Skip all OPT_MCU_NONE these are WIP port + if mcu == "NONE": + return True + + if any(t.strip().strip('"') == "MAX3421_HOST=1" for t in extra_defines): + max3421_enabled = True + + mcus = set(_family_mcus(str(family_dir), str(board_dir))) + if "${" not in mcu: + mcus.add(mcu) + if not mcus: + # nothing resolved: neither FAMILY_MCUS nor the scraped CFG_TUSB_MCU token + # yielded a name. Answering "skip" here would silently drop EVERY example on + # the board (an only.txt can then never match), so say "buildable" and let + # the real filter decide - build.py checks the targets CMake actually + # registered, and CMake itself is the authority on the make/cmake legs. + return False + if max3421_enabled: + mcus.add("MAX3421") # family_support.cmake:940 + + keys = {"board:" + board, "family:" + family} | {"mcu:" + m for m in mcus} + + skips = _filter_tokens(str(ex_dir / "skip.txt")) + if skips is not None and (skips & keys): + return True + + onlys = _filter_tokens(str(ex_dir / "only.txt")) + if onlys is not None and not (onlys & keys): + return True + + return False + + def build_size(make_cmd): size_output = subprocess.run(make_cmd + ' size', shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8").splitlines() for i, l in enumerate(size_output): diff --git a/tools/ci_select.py b/tools/ci_select.py new file mode 100755 index 000000000..ced3bbbc0 --- /dev/null +++ b/tools/ci_select.py @@ -0,0 +1,1123 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""PR-diff -> CI selection: which rig boards and which tests a change can affect. + +Lives in tools/ so it can serve both HIL selection and, from Task 3, build-family +selection. Stdlib-only (runs on bare CI runners; imports hil_util for the example +rosters, never hil_test/pyserial — test_hil_util.BottomLayer enforces the stdlib +closure). Fail-open: any file no rule classifies forces the full matrix. See +docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md and +docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md. + +JSON: full, boards (name -> 'all' | [tests]), families (bsp families the diff +touches, including ones with no rig board - build-only consumers such as /pre-pr +sample from these), args (hil_test.py args per config) and args_flasher (the same +args split by each board's flasher, for CI legs that split one rig by flasher). +""" +import argparse +import ast +import contextlib +import functools +import glob +import io +import json +import os +import re +import subprocess +import sys + +# tools/ -> repo root is ONE level up. Guarded by TestModuleMove.test_repo_root_guard: +# a wrong parent count here silently re-points every repo-relative glob (it happened +# at the helper/ move). +_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(_REPO_ROOT, 'test', 'hil')) # for `from helper...` +from helper.hil_util import device_tests, dual_tests, host_test + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # tools/, for build helpers +import build_utils +import build as build_py + +ALL_TESTS = {'device': device_tests, 'dual': dual_tests, 'host': host_test} + +# class dir -> config macro suffix exceptions (rule 3); dfu is per-file, handled inline +NET_MACROS = ('ECM_RNDIS', 'NCM') + + +def _read(path: str) -> str: + """Read a source file with a fixed encoding. The locale's is not it: several tracked + sources carry non-ASCII bytes, and under LC_ALL=C the decode raises UnicodeDecodeError + - a ValueError, which every `except OSError` fail-open below would let through as a + traceback instead of a full matrix.""" + with open(path, encoding='utf-8', errors='replace') as f: + return f.read() + + +_NONCODE_RE = re.compile( + r'^(docs/|\.claude/|.*\.(md|rst)$|LICENSE)') +# Build-size metrics tooling. HIL axis ONLY: nothing on the rig runs any of it, and +# without this rule these paths are unclassified, so a metrics-only PR booked an +# exclusive full 30-board sweep to validate a script no board executes. +# The BUILD axis deliberately keeps its full-matrix answer: `tinyusb_metrics` runs +# tools/metrics.py as a build target (examples/CMakeLists.txt), and build_util.yml adds +# `--target tinyusb_metrics` to every metrics leg - a break in it fails the build, so a +# build has to exercise it. +_METRICS_RE = re.compile( + r'^(tools/metrics[^/]*\.py$|\.github/scripts/metrics_[^/]*\.py$)') +_FULL_RE = re.compile( + r'^(src/common/|src/osal/|src/tusb\.c$|src/tusb\.h$|src/tusb_option\.h$|' + r'test/hil/|\.github/workflows/build.*\.yml$|\.github/actions/|\.github/scripts/|' + r'tools/build\.py$|tools/cmake/|' + r'hw/bsp/(family_support\.cmake|board_api\.h|board\.c|ansi_escape\.h)$|' + r'examples/build_system/|examples/CMakeLists\.txt$|' + # board_test is HIL infrastructure, not a test: hil_test.py flashes it to park + # every board (variant boundary + end-of-board teardown), so every board depends on it + r'examples/device/board_test/)') + +# --no-renames: with rename detection git reports only a rename's destination, so code +# moved out of an HIL-relevant path would be classified by its new path alone +GIT_DIFF_ARGV = ['git', 'diff', '--no-renames', '--name-only'] + + +def test_role(test: str) -> str: + return test.split('/', 1)[0] # 'device' | 'dual' | 'host' + + +def board_roles(board: dict) -> set: + t = board.get('tests', {}) + roles = set() + if t.get('device'): + roles.add('device') + if t.get('host'): + roles.add('host') + if t.get('dual'): + roles.update(('device', 'host')) + for only in t.get('only', []): + r = test_role(only) + roles.update(('device', 'host') if r == 'dual' else (r,)) + return roles + + +def board_tests(board: dict) -> list: + """Every test this board would run today (mirrors hil_test.test_board's default).""" + t = board.get('tests', {}) + if 'only' in t: + run = list(t['only']) + else: + run = [] + if t.get('device'): + run += device_tests + if t.get('dual'): + run += dual_tests + if t.get('host'): + run += host_test + return [x for x in run if x not in t.get('skip', [])] + + +# cached: called per changed file x roster board, and the tree doesn't change mid-run [email protected]_cache(maxsize=None) +def board_family(board_name: str, repo_root: str): + hits = glob.glob(os.path.join(repo_root, 'hw/bsp/*/boards', board_name)) + return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None + + +# `if (OPTION STREQUAL "1")` guards in family_support.cmake, and the option tokens +# a roster entry passes to the build (NAME=VALUE / -DNAME=VALUE) +_CM_IF_RE = re.compile(r'if\s*\(') +_CM_ELSE_RE = re.compile(r'else(if)?\s*\(') +_CM_ENDIF_RE = re.compile(r'endif\s*\(') +_CM_OPT_RE = re.compile(r'if\s*\(\s*\$?\{?([A-Za-z_]\w*)\}?\s+STREQUAL\s+"?1"?\s*\)') +_CM_PORT_RE = re.compile(r'src/portable/((?:[^/\s]+/)?[^/\s]+)/') +_FALSY = ('', '0', 'off', 'false', 'no') + + [email protected]_cache(maxsize=None) +def port_option_gates(repo_root: str) -> dict: + """port dir -> build options that compile it regardless of the board's family + file, e.g. {'analog/max3421': {'MAX3421_HOST'}} from family_support.cmake.""" + gates = {} + try: + text = _read(os.path.join(repo_root, 'hw/bsp/family_support.cmake')) + except OSError: + return gates + stack = [] # one entry per open if(): its option, or None + for line in text.splitlines(): + line = line.strip() + if _CM_IF_RE.match(line): + m = _CM_OPT_RE.match(line) + stack.append(m.group(1) if m else None) + elif _CM_ELSE_RE.match(line): + if stack: + stack[-1] = None # the guard doesn't hold in this branch + elif _CM_ENDIF_RE.match(line): + if stack: + stack.pop() + opts = {o for o in stack if o} + m = _CM_PORT_RE.search(line) + if opts and m: + gates.setdefault(m.group(1), set()).update(opts) + return gates + + +_CM_SET_RE = re.compile(r'set\s*\(\s*([A-Za-z_]\w*)\s+([^)\s]+)\s*\)') + + +# cached: called per changed portable file x roster board [email protected]_cache(maxsize=None) +def bsp_board_options(board_name: str, repo_root: str) -> frozenset: + """Build options a board turns on in its own BSP: `set(<OPT> <value>)` in + hw/bsp/<family>/boards/<board>/board.cmake, e.g. MAX3421_HOST on the espressif + and rp2040 max3421 boards. CMake only - HIL CI builds nothing with Make, so a + board.mk-only option (e.g. nrf5340dk's MAX3421_HOST) compiles no port here.""" + fam = board_family(board_name, repo_root) + if not fam: + return frozenset() + path = os.path.join(repo_root, 'hw/bsp', fam, 'boards', board_name, 'board.cmake') + try: + text = _read(path) + except OSError: + return frozenset() + out = set() + for line in text.splitlines(): + line = line.strip() + if line.startswith('#'): + continue + m = _CM_SET_RE.match(line) + if m and m.group(2).strip('"').lower() not in _FALSY: + out.add(m.group(1)) + return frozenset(out) + + +def board_options(board: dict, repo_root: str) -> set: + """Build options a board has truthy: each variant's defines (NAME=VALUE) and raw + CFLAGS (-DNAME=VALUE), plus whatever its own board.cmake sets (a board can enable a + gated port without the roster saying so). A board whose option is always on carries + a single variant named after itself - metro_m4_express and MAX3421_HOST=1, which is + what makes it the one rig board that compiles hcd_max3421.c.""" + toks = [] + for v in board.get('variant', []): + toks += list(v.get('defines', [])) + toks += v.get('flags', '').split() + out = set(bsp_board_options(board['name'], repo_root)) + for t in toks: + name, _, val = (t[2:] if t.startswith('-D') else t).partition('=') + if name and val.strip().strip('"').lower() not in _FALSY: + out.add(name.strip()) + return out + + [email protected]_cache(maxsize=None) +def path_families(rel_dir: str, repo_root: str) -> set: + """Board families whose family.cmake (or espressif component CMakeLists) + references rel_dir at a directory boundary. CMake only, on every axis: CMake + is the first-class build system and Make follows it, so family.mk is never + read - a port wired up in family.mk alone (microchip/pic32mz) is built by no + CI job and resolves to nothing. Boundary = '/', whitespace, quote, paren, + brace or end: `${TOP}/hw/mcu/nordic/nrfx` has no trailing slash, while bare + 'microchip/pic' must not match '.../microchip/pic32mz/...'.""" + pat = re.compile(re.escape(rel_dir) + r'(?=[/\s"\')}]|$)', re.M) + return {fam for fam, text in _family_file_texts(repo_root) if pat.search(text)} + + [email protected]_cache(maxsize=None) +def _family_file_texts(repo_root: str) -> tuple: + """((family, text), ...) for every family.cmake and espressif component + CMakeLists.txt, read once. path_families is called per distinct directory in the + diff and its own cache only helps repeats: a 6,000-file hw/mcu dep bump re-read + these 84 files 99,892 times (2.2 s) before this.""" + bsp_root = os.path.join(repo_root, 'hw/bsp') + out = [] + for f in sorted(glob.glob(os.path.join(bsp_root, '*/family.cmake')) + + glob.glob(os.path.join(bsp_root, '*/components/*/CMakeLists.txt'))): + try: + out.append((os.path.relpath(f, bsp_root).split(os.sep, 1)[0], _read(f))) + except OSError: + pass + return tuple(out) + + +def port_families(port_dir: str, repo_root: str) -> set: + # 'portable/', not 'src/portable/': family.cmake always spells the full literal + # path ('${TOP}/src/portable/...'), but espressif's component CMakeLists.txt + # assigns 'src' into a ${tusb_src} variable first (`${tusb_src}/portable/...`), + # so a leading 'src/' in the needle would never match there and silently drop + # espressif boards (see TestRealRosterPortFamilies). + return path_families('portable/' + port_dir, repo_root) + + +def mcu_families(path: str, repo_root: str) -> set: + """Families referencing a changed hw/mcu path: longest resolving dir prefix, + hw/mcu/<vendor>/<sub>/... down to hw/mcu/<vendor>.""" + parts = path.split('/') + for n in range(len(parts) - 1, 2, -1): + fams = path_families('/'.join(parts[:n]), repo_root) + if fams: + return fams + return set() + + +GET_DEPS_PATH = 'tools/get_deps.py' +_DEPS_DICTS = ('deps_mandatory', 'deps_optional') + + +def _deps_split(text: str): + """(module dump with the two dep-dict assigns removed, {dict name: entries}). + Parsed with ast, never exec'd: this runs on PR content.""" + mod = ast.parse(text) + dicts, rest = {}, [] + for node in mod.body: + if (isinstance(node, ast.Assign) and len(node.targets) == 1 and + isinstance(node.targets[0], ast.Name) and + node.targets[0].id in _DEPS_DICTS and isinstance(node.value, ast.Dict)): + dicts[node.targets[0].id] = ast.literal_eval(node.value) + else: + rest.append(node) + mod.body = rest + # annotate_fields=False keeps the dump readable-length; line numbers are not + # included unless asked for, so reformatting alone never reads as a logic change + return ast.dump(mod, annotate_fields=False), dicts + + +# Family tokens in tools/get_deps.py that name no hw/bsp directory. get_deps matches a +# token against a requested family name verbatim (`f in deps_optional[d][2].split()`), +# so a token like these matches nothing - a stale spelling in get_deps.py, not a +# selector bug, and out of scope to change here. Pinned so that any OTHER unresolvable +# token (real drift) falls open to the full matrix instead of silently selecting +# nothing, and so TestOrphanInvariant fails the day one is fixed or a new one appears. +# sam3x, samd21, samd51, same5x -> pre-rename spellings, listed alongside the current +# samd2x_l2x / samd5x_e5x / same7x in the same entry +# stm32l1, stm32l5 -> no hw/bsp family in the tree at all +_DEPS_ALIAS_TOKENS = frozenset({'sam3x', 'samd21', 'samd51', 'same5x', + 'stm32l1', 'stm32l5'}) + + +def get_deps_changed_families(base_text: str, head_text: str, repo_root: str): + """Families whose tools/get_deps.py dep entries changed between two versions of + the file, or None meaning 'cannot tell - use the full matrix'. + + None on: anything outside deps_mandatory/deps_optional differing (a logic change + to get_deps affects every family), a mandatory `'all'` entry changing, a token + that resolves to no family and is not a known alias, or text that will not parse. + Callers with no base content at all - `--diff-file` mode has no git and therefore + no merge-base blob - pass None themselves. + + An entry that is added, removed or edited contributes the family tokens of BOTH + sides (a removed entry has only a base side). The two dicts are diffed SEPARATELY: + merging them first would hide a move between deps_mandatory and deps_optional, + which changes which families fetch the dep even though the value is untouched.""" + try: + base_rest, base_d = _deps_split(base_text) + head_rest, head_d = _deps_split(head_text) + except (SyntaxError, ValueError, TypeError): + return None + if base_rest != head_rest: + return None + toks = set() + for name in _DEPS_DICTS: + base_x, head_x = base_d.get(name, {}), head_d.get(name, {}) + for key in set(base_x) | set(head_x): + if base_x.get(key) == head_x.get(key): + continue + for entry in (base_x.get(key), head_x.get(key)): + if entry and len(entry) > 2: + toks.update(str(entry[2]).split()) + if 'all' in toks: + return None + fams = set(all_bsp_families(repo_root)) + if toks - fams - _DEPS_ALIAS_TOKENS: + # a changed entry we cannot map to a family. "changed but unmappable" is NOT + # "nothing changed": reading it as the latter empties the entire build matrix + # for a dep bump, so fall open instead + return None + return toks & fams + + +_CLS_INC_RE = re.compile(r'#\s*include\s*[<"]class/([^/"<>]+)/([^"<>]+)[">]') + + [email protected]_cache(maxsize=None) +def class_include_edges(repo_root: str) -> dict: + """'<class>/<header>' -> the other class dirs that include it. A class header + pulled in by a second class ships in every firmware enabling that second class: + src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, and + net_device.h includes class/cdc/cdc.h. The class rule derives macros from the + directory name alone, so without this edge a change to the included header + selects only its own class's examples - and on a board that skips those (e.g. + metro_m4_express skips audio_test_freertos), nothing at all. + + Derived from the actual #include lines rather than a hand-written table so it + cannot rot when a class picks up or drops a cross-class include.""" + edges = {} + for f in sorted(glob.glob(os.path.join(repo_root, 'src/class/*/*.[ch]'))): + cls = os.path.basename(os.path.dirname(f)) + try: + text = _read(f) + except OSError: + continue + for inc_cls, inc_hdr in _CLS_INC_RE.findall(text): + if inc_cls != cls: + edges.setdefault(f'{inc_cls}/{inc_hdr}', set()).add(cls) + return edges + + +_CLS_STEM_RE = re.compile(r'(.*?)(?:_(?:device|host))?\.[ch]$') + + +def class_macros(cls: str, base: str, prefix: str) -> list: + """Config macros that compile a class dir's code, for role prefix TUD/TUH. + `base` refines dfu (it splits DFU from DFU_RUNTIME per file) and adds the file's + own macro where that differs from the directory's; pass '' for a class reached + through an include edge, where the widest set is correct.""" + if cls == 'net': + return [f'CFG_{prefix}_{m}' for m in NET_MACROS] + if cls == 'dfu': + if base.startswith('dfu_rt'): + return [f'CFG_{prefix}_DFU_RUNTIME'] + if base.startswith('dfu_device') or base.startswith('dfu_host'): + return [f'CFG_{prefix}_DFU'] + return [f'CFG_{prefix}_DFU', f'CFG_{prefix}_DFU_RUNTIME'] + out = [f'CFG_{prefix}_{cls.upper()}'] + # A class directory can hold more than one class. src/class/midi ships MIDI 1.0 + # AND MIDI 2.0: midi2_device.c is `#if CFG_TUD_ENABLED && CFG_TUD_MIDI2`, and + # examples/device/midi2_device is the only example that enables it - so the + # directory macro alone selected the midi_test examples, which do not compile the + # changed file, and none of the ones that do. Union, never replace: the file may + # still be pulled in by the directory's own macro, and over-selecting costs a build + # while under-selecting merges a break. + m = _CLS_STEM_RE.match(base) + if m and m.group(1) and m.group(1) != cls: + out.append(f'CFG_{prefix}_{m.group(1).upper()}') + return out + + +# A define is OFF only when its value is a literal zero (0, 00, (0)), optionally +# followed by a comment. Anything else counts as ON - including a value this cannot +# evaluate, e.g. `#define CFG_TUH_MIDI CFG_TUH_DEVICE_MAX` (examples/host/midi_rx). +# Fail-open: reading such a define as OFF made midi_host.c select zero families and +# let a compile break merge green. +# +# A macro defined more than once is ON if ANY of its defines is non-zero, because +# the preprocessor branches are not evaluated here: uac2_speaker_fb defines +# CFG_TUD_HID 1 under `#if CFG_AUDIO_DEBUG` and 0 in the #else, and the default +# build (CFG_AUDIO_DEBUG defaults to 1) compiles the HID class in. Deciding on the +# LAST/only match found made that example invisible to CFG_TUD_HID changes. +_DEF_VALUE = r'^[ \t]*#[ \t]*define[ \t]+{}[ \t]+(\S[^\n]*?)[ \t]*$' +_DEF_ZERO_VALUE = re.compile(r'\(?\s*0+\s*\)?\s*(?://.*|/\*.*)?') + + +# Shared rule-recognition primitives. The two classifiers walk the same diff with +# different answers, but they must RECOGNISE the same things: one copy each, so a +# new naming convention cannot land in one walk and be missed by the other. +_PORT_PATH_RE = re.compile(r'src/portable/((?:[^/]+/)?[^/]+)/') + + +def _port_roles(base: str) -> set: + """Which USB role a src/portable file serves, from its name: dcd_*/ *_device is + the device-controller side, hcd_*/ *_host the host side, anything else (shared + headers, glue) both.""" + if re.match(r'(dcd_|.*_device)', base): + return {'device'} + if re.match(r'(hcd_|.*_host)', base): + return {'host'} + return {'device', 'host'} + + +def _class_roles(base: str) -> set: + """Same question for a src/class file: <cls>_device.[ch] / <cls>_host.[ch], + else both - the class's shared header ships in either role.""" + if re.search(r'_device\.[ch]$', base): + return {'device'} + if re.search(r'_host\.[ch]$', base): + return {'host'} + return {'device', 'host'} + + +def _config_enables(cfg_path: str, macros) -> bool: + try: + with open(cfg_path, encoding='utf-8', errors='replace') as f: + text = f.read() + except OSError: + return False + for m in macros: + for value in re.findall(_DEF_VALUE.format(m), text, re.M): + if not _DEF_ZERO_VALUE.fullmatch(value): + return True + return False + + +def examples_enabling(pool, macros, repo_root: str) -> set: + """The 'role/name' entries of `pool` whose src/tusb_config.h turns any of + `macros` on. The pool differs per classifier (HIL test lists vs every example), + the question does not.""" + return {ex for ex in pool + if _config_enables(os.path.join(repo_root, 'examples', ex, 'src', + 'tusb_config.h'), macros)} + + +# cached: called per changed lib file, and the tree doesn't change mid-run [email protected]_cache(maxsize=None) +def lib_examples(lib_name: str, repo_root: str) -> set: + """Examples whose OWN examples/<role>/<name>/{CMakeLists.txt,Makefile} references + lib/<lib_name> at a directory boundary (same boundary rule as path_families, so + 'lib/net' cannot inherit lib/networking's example). + + Per-example on purpose: lib/SEGGER_RTT is named by family_support.cmake's + LOGGER=rtt plumbing, which no CI example build turns on (all three references - + family_support.cmake, family_support.mk, rp2040/family.cmake - sit inside a + LOGGER=rtt guard), so a family-file scan would wrongly narrow it to three families + instead of answering 'nobody'. + + The whole example TREE is scanned, not just its top-level files: examples/host/ + msc_file_explorer_freertos/src/CMakeLists.txt names lib/embedded-cli, and that + example survived only because its top-level file happens to name it too.""" + pat = re.compile(re.escape('lib/' + lib_name) + r'(?=[/\s"\')}]|$)', re.M) + out = set() + for ex in all_examples(repo_root): + for f in sorted(glob.glob(os.path.join(repo_root, 'examples', ex, '**', '*'), + recursive=True)): + if os.path.basename(f) not in ('CMakeLists.txt', 'Makefile'): + continue + try: + text = _read(f) + except OSError: + continue + if pat.search(text): + out.add(ex) + break + return out + + +def roster_only_tests(all_boards) -> set: + """Test paths that only appear in a roster board's tests.only list (e.g. + espressif boards), not in the shared device/dual/host_test lists.""" + out = set() + for b in all_boards: + out.update(b.get('tests', {}).get('only', [])) + return out + + +def class_examples(macros, role: str, repo_root: str, extra_tests: set) -> set: + """Tests (from role's + dual lists, plus roster-only-list tests of that role) + whose example config enables any macro.""" + return examples_enabling(role_tests({role}, extra_tests), macros, repo_root) + + +def role_tests(roles: set, extras: set) -> set: + """Every test for the given role(s): each role's own list + dual tests, + plus roster-only-list tests (extras) matching those roles or 'dual'.""" + pool = set(dual_tests) + for r in roles: + pool |= set(ALL_TESTS[r]) + pool |= {t for t in extras if test_role(t) in roles or test_role(t) == 'dual'} + return pool + + +class _Sel: + """Accumulates contributions. board->set(tests) plus 'all-board' markers.""" + def __init__(self): + self.full = False + self.by_board = {} # name -> set of tests, or 'all' + self.roles = set() # roles touched by any contribution + self.families = set() # bsp families touched (incl. off-rig ones: build-only consumers) + self.reasons = [] + + def add(self, boards, tests, reason): + """tests: 'all' or iterable of test paths.""" + self.reasons.append(reason) + for b in boards: + cur = self.by_board.get(b) + if tests == 'all' or cur == 'all': + self.by_board[b] = 'all' + else: + self.by_board[b] = (cur or set()) | set(tests) + + def force_full(self, reason): + self.full = True + self.reasons.append(reason) + + +def _classify_one(path, repo_root, roster_boards, extras: set, s: _Sel, + get_deps_families=None): + base = os.path.basename(path) + if _NONCODE_RE.match(path): + s.reasons.append(f'{path}: non-code, no contribution') + return + if _METRICS_RE.match(path): + s.reasons.append(f'{path}: build-size metrics tooling, no HIL contribution') + return + if _FULL_RE.match(path): + s.force_full(f'{path}: core/infra -> full matrix') + return + + if path == GET_DEPS_PATH: + if get_deps_families is None: + s.force_full(f'{path}: dep changes not resolvable -> full matrix') + return + if not get_deps_families: + s.reasons.append(f'{path}: no dep entry changed, no contribution') + return + fams = sorted(get_deps_families) + s.families.update(fams) + boards = [b['name'] for b in roster_boards + if board_family(b['name'], repo_root) in get_deps_families] + s.roles.update(('device', 'host')) + s.add(boards, 'all', f'{path}: dep entries changed -> families {fams} -> ' + f'boards {boards}') + return + + m = _PORT_PATH_RE.match(path) + if m: + port = m.group(1) + roles = _port_roles(base) + fams = port_families(port, repo_root) + if not fams: + # empty means empty (maintainer ruling), same reading as hw/mcu and as the + # build walk: no family's build references this port, so nothing compiles it + # and there is nothing to run. Forcing the full 30-board rig here bought no + # coverage at all - the build side selected zero families for the same path. + # Live for src/portable/template and the two microchip pic ports; + # TestPortFamiliesCoverage is the drift guard for a port that stops resolving. + s.reasons.append(f'{path}: port {port} maps to no board family, no contribution') + return + s.families.update(fams) + # a board can also pull the port in through a build option (e.g. MAX3421_HOST=1 + # from the roster on metro_m4_express, or from its own board.cmake), which its + # family file never names + gates = port_option_gates(repo_root).get(port, set()) + boards = [b['name'] for b in roster_boards + if (board_family(b['name'], repo_root) in fams or + (gates and board_options(b, repo_root) & gates)) and (board_roles(b) & roles)] + tests = role_tests(roles, extras) + s.roles.update(roles) + why = f'{path}: port {port} -> families {sorted(fams)}' + if gates: + why += f' + option {sorted(gates)}' + s.add(boards, tests, f'{why} -> boards {boards} ({"/".join(sorted(roles))})') + return + + m = re.match(r'src/class/([^/]+)/', path) + if m: + cls = m.group(1) + roles = _class_roles(base) + # this file's own class, plus any class whose headers include it + via = sorted(class_include_edges(repo_root).get(f'{cls}/{base}', ())) + + def macros(prefix): + return (class_macros(cls, base, prefix) + + [m2 for c in via for m2 in class_macros(c, '', prefix)]) + tests = set() + if 'device' in roles: + tests |= class_examples(macros('TUD'), 'device', repo_root, extras) + if 'host' in roles: + tests |= class_examples(macros('TUH'), 'host', repo_root, extras) + boards = [b['name'] for b in roster_boards if board_roles(b) & roles] + s.roles.update(roles) + why = f'{path}: class {cls}' + (f' (+ included by {via})' if via else '') + s.add(boards, tests, f'{why} -> {sorted(tests)} ({"/".join(sorted(roles))})') + return + + m = re.match(r'src/(device|host)/', path) + if m: + role = m.group(1) + boards = [b['name'] for b in roster_boards if role in board_roles(b)] + s.roles.add(role) + s.add(boards, role_tests({role}, extras), f'{path}: core {role} stack -> all {role} tests') + return + + m = re.match(r'hw/bsp/([^/]+)/(?:boards/([^/]+)/)?', path) + if m: + fam, brd = m.group(1), m.group(2) + s.families.add(fam) + if brd: + boards = [b['name'] for b in roster_boards if b['name'] == brd] + why = f'{path}: bsp board {brd}' + else: + boards = [b['name'] for b in roster_boards + if board_family(b['name'], repo_root) == fam] + why = f'{path}: bsp family {fam}' + s.roles.update(('device', 'host')) + s.add(boards, 'all', f'{why} -> boards {boards}') + return + + if re.match(r'hw/mcu/', path): + fams = mcu_families(path, repo_root) + if not fams: + # empty means empty (maintainer ruling): if no family's build references + # the path, no build consumes the change - there is nothing to compile, + # so there is nothing to run either. TestOrphanInvariant's + # test_tracked_mcu_vendors_resolve is the drift guard: a real vendor dir + # that stops resolving fails pre-commit instead of silently vanishing + s.reasons.append(f'{path}: hw/mcu path resolves to no family, no contribution') + return + s.families.update(fams) + boards = [b['name'] for b in roster_boards + if board_family(b['name'], repo_root) in fams] + s.roles.update(('device', 'host')) + s.add(boards, 'all', f'{path}: mcu dir -> families {sorted(fams)} -> boards {boards}') + return + + m = re.match(r'lib/([^/]+)/', path) + if m: + lib = m.group(1) + # only the tests whose example builds the lib, and only those the rig runs + tests = {e for e in lib_examples(lib, repo_root) + if any(e in pool for pool in ALL_TESTS.values()) or e in extras} + if not tests: + s.reasons.append(f'{path}: lib {lib} used by no HIL test, no contribution') + return + roles = set() + for test in tests: + r = test_role(test) + roles.update(('device', 'host') if r == 'dual' else (r,)) + boards = [b['name'] for b in roster_boards] + s.roles.update(roles) + s.add(boards, sorted(tests), f'{path}: lib {lib} -> {sorted(tests)} on all boards') + return + + m = _BUILD_EX_RE.match(path) + if m: + if m.group(1) not in _HIL_EX_ROLES: + # examples/typec: the build matrix compiles it, nothing on the rig runs it + s.reasons.append(f'{path}: {m.group(1)} example, no HIL contribution') + return + test = f'{m.group(1)}/{m.group(2)}' + known = any(test in pool for pool in ALL_TESTS.values()) or test in extras + if known: + boards = [b['name'] for b in roster_boards] + role = test_role(test) + s.roles.update(('device', 'host') if role == 'dual' else (role,)) + s.add(boards, [test], f'{path}: example -> {test} on all boards') + else: + s.reasons.append(f'{path}: example not in HIL lists, no contribution') + return + + s.force_full(f'{path}: unclassified -> full matrix') + + +def classify(changed_files, repo_root, rosters, get_deps_families=None): + all_boards = [] + seen = set() + for _, boards in rosters: + for b in boards: + if b['name'] not in seen: + seen.add(b['name']) + all_boards.append(b) + + extras = roster_only_tests(all_boards) + s = _Sel() + # no early exit once full: keep classifying so `families` still reports every + # family the diff touches (build-only consumers need it). Nothing after the first + # force_full can change full/boards/args - the full branch below ignores by_board. + for path in changed_files: + _classify_one(path, repo_root, all_boards, extras, s, get_deps_families) + + if s.full: + return {'full': True, 'boards': {b['name']: 'all' for b in all_boards}, + 'families': sorted(s.families), 'reasons': s.reasons} + + # role pruning: single-role selections drop the other role's tests and boards + by_name = {b['name']: b for b in all_boards} + out = {} + for name, tests in s.by_board.items(): + allowed = board_tests(by_name[name]) + if tests == 'all': + kept = list(allowed) + else: + kept = [t for t in allowed if t in tests] + if s.roles and s.roles != {'device', 'host'}: + role = next(iter(s.roles)) + kept = [t for t in kept if test_role(t) in (role, 'dual')] + if kept: + out[name] = 'all' if set(kept) == set(allowed) else sorted(kept) + return {'full': False, 'boards': out, 'families': sorted(s.families), + 'reasons': s.reasons} + + +def _board_args(name, chosen) -> list: + parts = [f'-b {name}'] + if chosen != 'all': + parts.append(f'-bt {name}:{",".join(chosen)}') + return parts + + +def hil_examples(sel, rosters): + """{board: examples hil-build must produce}: the board's selected tests plus + device/board_test, which hil_test.py flashes to park at every variant + boundary and at end-of-board teardown. Emitted for full selections too - the + HIL example universe is a fraction of the tree regardless of the diff.""" + by_name = {} + for _, boards in rosters: + for b in boards: + by_name.setdefault(b['name'], []).append(b) + if sel['full']: + chosen = {n: 'all' for n in by_name} + else: + chosen = sel['boards'] + out = {} + for name, tests in chosen.items(): + if tests == 'all': + # a board named by two rosters (rig migration, or shared between rigs) + # may run different tests on each: union them. Superset firmware costs a + # build; a missing image fails the run on whichever rig lost the toss. + run = set().union(*(board_tests(b) for b in by_name[name])) + else: + run = set(tests) + out[name] = sorted(run | {'device/board_test'}) + return out + + +def selection_args(sel, rosters): + """hil_test.py args per config. Empty means either 'full matrix' or 'nothing + selected' - callers must read sel['full'] to tell them apart.""" + args = {} + for cfg_path, boards in rosters: + parts = [] + if not sel['full']: + for b in boards: + chosen = sel['boards'].get(b['name']) + if chosen is not None: + parts += _board_args(b['name'], chosen) + args[os.path.basename(cfg_path)] = ' '.join(parts) + return args + + +def selection_args_by_flasher(sel, rosters): + """{config: {flasher name: args}}. CI runs one rig as several jobs split by + flasher (esptool vs the rest); each must gate on its own subset, otherwise the + other leg runs a filter matching zero boards and reports a vacuous green.""" + out = {} + for cfg_path, boards in rosters: + per = {} + if not sel['full']: + for b in boards: + chosen = sel['boards'].get(b['name']) + if chosen is None: + continue + per.setdefault(b.get('flasher', {}).get('name', ''), []).extend( + _board_args(b['name'], chosen)) + out[os.path.basename(cfg_path)] = {f: ' '.join(p) for f, p in per.items()} + return out + + +def merge_base(base, repo_root): + return subprocess.run(['git', 'merge-base', 'HEAD', base], cwd=repo_root, + capture_output=True, text=True, check=True).stdout.strip() + + +def git_show(spec, repo_root): + return subprocess.run(['git', 'show', spec], cwd=repo_root, + capture_output=True, text=True, check=True).stdout + + +def changed_files_from_git(base, repo_root): + diff = subprocess.run(GIT_DIFF_ARGV + [f'{merge_base(base, repo_root)}..HEAD'], + cwd=repo_root, capture_output=True, text=True, check=True).stdout + return [l for l in diff.splitlines() if l.strip()] + + +def get_deps_families_from_git(base, repo_root): + """The changed dep entries' families for a --base run, or None (-> full matrix) + if git cannot produce both sides of tools/get_deps.py.""" + try: + mb = merge_base(base, repo_root) + return get_deps_changed_families(git_show(f'{mb}:{GET_DEPS_PATH}', repo_root), + git_show(f'HEAD:{GET_DEPS_PATH}', repo_root), + repo_root) + except (subprocess.CalledProcessError, OSError) as e: + print(f'ci_select: {GET_DEPS_PATH}: base content unreadable ({e})', file=sys.stderr) + return None + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + g = ap.add_mutually_exclusive_group(required=True) + g.add_argument('--base', help='git ref to diff against (merge-base..HEAD)') + g.add_argument('--diff-file', help='newline-separated changed-file list') + ap.add_argument('configs', nargs='*', help='rig roster JSON file(s); omit for the build view alone') + a = ap.parse_args() + + repo_root = _REPO_ROOT + rosters = [] + for c in a.configs: + with open(c, encoding='utf-8', errors='replace') as f: + rosters.append((c, json.load(f)['boards'])) + + files = (_read(a.diff_file).splitlines() if a.diff_file + else changed_files_from_git(a.base, repo_root)) + files = [f for f in files if f.strip()] + + # --diff-file has no git and so no base content: the rule falls open to full + gd = (get_deps_families_from_git(a.base, repo_root) + if a.base and GET_DEPS_PATH in files else None) + + s = classify(files, repo_root, rosters, gd) + s['args'] = selection_args(s, rosters) + s['args_flasher'] = selection_args_by_flasher(s, rosters) + if rosters: + s['hil_examples'] = hil_examples(s, rosters) + s['build'] = classify_build(files, repo_root, gd) + for r in s['build']['reasons']: + print(f'ci_select[build]: {r}', file=sys.stderr) + for r in s['reasons']: + print(f'ci_select: {r}', file=sys.stderr) + print(json.dumps(s)) + + +# ------------------------------------------------------------- +# Build-axis classifier (spec rule table, docs/superpowers/specs/ +# 2026-08-19-ci-build-family-filter-design.md). Independent of the HIL +# classifier: same diff, second walk, its own fail-open. +# ------------------------------------------------------------- +# Both walks recognise an example path with the SAME regex, so a role can never be +# known to one walk and unclassified (-> full matrix) to the other. What differs is the +# answer: the rig runs device/host/dual tests, while the build matrix also compiles +# examples/typec, which nothing on the rig runs. +_EX_ROLES = ('device', 'dual', 'host', 'typec') +_HIL_EX_ROLES = ('device', 'host', 'dual') +_BUILD_EX_RE = re.compile(r'examples/(%s)/([^/]+)/' % '|'.join(_EX_ROLES)) + + [email protected]_cache(maxsize=None) +def all_examples(repo_root: str) -> tuple: + """Every examples/<role>/<name> with a CMakeLists.txt, as 'role/name'.""" + out = [] + for role in _EX_ROLES: + for d in sorted(glob.glob(os.path.join(repo_root, 'examples', role, '*/'))): + if os.path.isfile(os.path.join(d, 'CMakeLists.txt')): + out.append(f'{role}/{os.path.basename(d.rstrip(os.sep))}') + return tuple(out) + + +def role_examples(repo_root: str, roles) -> set: + want = set(roles) + return {e for e in all_examples(repo_root) if e.split('/', 1)[0] in want} + + [email protected]_cache(maxsize=None) +def all_bsp_families(repo_root: str) -> tuple: + return tuple(sorted(d for d in os.listdir(os.path.join(repo_root, 'hw/bsp')) + if os.path.isdir(os.path.join(repo_root, 'hw/bsp', d)))) + + +def _build_class_examples(cls: str, base: str, roles: set, repo_root: str) -> set: + """Examples (all 46, not the HIL lists) whose tusb_config.h enables the class's + macros for the given roles, plus classes that #include the changed header.""" + via = sorted(class_include_edges(repo_root).get(f'{cls}/{base}', ())) + out = set() + for prefix, role in (('TUD', 'device'), ('TUH', 'host')): + if role not in roles: + continue + macros = class_macros(cls, base, prefix) + \ + [m for c in via for m in class_macros(c, '', prefix)] + out |= examples_enabling(all_examples(repo_root), macros, repo_root) + return out + + +class _BSel: + """family -> set(examples) | 'all', unioned per family.""" + def __init__(self): + self.full = False + self.fam_ex = {} + self.reasons = [] + + def add(self, fams, examples, reason): + self.reasons.append(reason) + for f in fams: + cur = self.fam_ex.get(f) + if examples == 'all' or cur == 'all': + self.fam_ex[f] = 'all' + else: + self.fam_ex[f] = (cur or set()) | set(examples) + + def force_full(self, reason): + self.full = True + self.reasons.append(reason) + + +def _classify_build_one(path, repo_root, s: _BSel, get_deps_families=None): + base = os.path.basename(path) + if _NONCODE_RE.match(path): # rule 1 + return + if re.match(r'test/hil/', path): # rule 2 + s.reasons.append(f'{path}: HIL harness, no build contribution') + return + if path == GET_DEPS_PATH: # get_deps rule + if get_deps_families is None: + s.force_full(f'{path}: dep changes not resolvable -> full build matrix') + return + if not get_deps_families: + s.reasons.append(f'{path}: no dep entry changed, no contribution') + return + fams = sorted(get_deps_families) + s.add(fams, 'all', f'{path}: dep entries changed -> families {fams}') + return + m = _PORT_PATH_RE.match(path) + if m: # rules 3-5 + port = m.group(1) + fams = port_families(port, repo_root) + roles = _port_roles(base) + exs = 'all' if roles == {'device', 'host'} else \ + role_examples(repo_root, tuple(roles) + ('dual',)) + s.add(fams, exs, f'{path}: port {port} -> families {sorted(fams)}') + return + if re.match(r'hw/bsp/[^/]+/', path): # rule 6 + fam = path.split('/')[2] + s.add({fam}, 'all', f'{path}: bsp family {fam}') + return + if re.match(r'hw/mcu/', path): # rule 7 + fams = mcu_families(path, repo_root) + if not fams: + # empty means empty, same reading as the HIL walk: no family's build + # references the path, so no build compiles it + s.reasons.append(f'{path}: hw/mcu path resolves to no family, no contribution') + return + s.add(fams, 'all', f'{path}: mcu -> families {sorted(fams)}') + return + m = re.match(r'src/class/([^/]+)/', path) + if m: # rules 8-10 + cls = m.group(1) + roles = _class_roles(base) + exs = _build_class_examples(cls, base, roles, repo_root) + if not exs: + # Empty means empty - maintainer decision. No example config enables this + # class, so no build exercises it and + # nothing is selected. The file IS still parsed by every full build + # (src/CMakeLists.txt, src/tinyusb.mk list class sources unconditionally, + # the CFG_ guard sits inside), so a break outside the guard surfaces on the + # next master push - the accepted safety net. + s.reasons.append(f'{path}: class {cls} enabled by no example config, ' + f'no contribution') + return + s.add(all_bsp_families(repo_root), exs, + f'{path}: class {cls} -> {sorted(exs)}') + return + m = re.match(r'src/(device|host)/', path) + if m: # rules 11-12 + role = m.group(1) + s.add(all_bsp_families(repo_root), role_examples(repo_root, (role, 'dual')), + f'{path}: core {role} stack') + return + m = _BUILD_EX_RE.match(path) + if m: # rules 13-14 + ex = f'{m.group(1)}/{m.group(2)}' + if ex in all_examples(repo_root): + s.add(all_bsp_families(repo_root), {ex}, f'{path}: example {ex}') + else: + # a deleted example builds nothing; removing it from the role + # CMakeLists (rule 15) is what forces the full matrix + s.reasons.append(f'{path}: not an example dir, no build contribution') + return + m = re.match(r'lib/([^/]+)/', path) + if m: # lib rule + lib = m.group(1) + exs = lib_examples(lib, repo_root) + if not exs: + # empty means empty: no example's build pulls this lib in, so no build + # compiles it (lib/SEGGER_RTT is only reached through LOGGER=rtt, which + # no CI build sets) + s.reasons.append(f'{path}: lib {lib} built by no example, no contribution') + return + s.add(all_bsp_families(repo_root), exs, f'{path}: lib {lib} -> {sorted(exs)}') + return + s.force_full(f'{path}: unclassified -> full build matrix') # rules 15-17 + + +def _in_repo(repo_root): + """build_utils/build.py use repo-relative paths; scope a chdir around them. + get_family_boards also prints on an empty family - swallow stdout so the + selector's machine-read JSON stays clean (diagnostics belong on stderr).""" + old = os.getcwd() + os.chdir(repo_root) + try: + with contextlib.redirect_stdout(io.StringIO()): + yield + finally: + os.chdir(old) + + +def _prune_buildable(fams, fam_ex, repo_root): + """Intersect each family's selection with what the family can build at all + (build_utils.skip_example - the same skip.txt/only.txt data CMake's + family_filter reads). + + ANY board of the family counts, not just the one GHA's --one-first picks: + CircleCI's cmake legs build every board of a family, so an example gated to a + single board (only.txt board:mimxrt1060_evk) would otherwise lose ALL compile + coverage exactly when a PR touches it. get_family_boards(.., False, False) is + that full list, with the same CI skip lists the build jobs apply. + + EITHER build system counts too. This one list gates CircleCI's make legs as well + as its cmake ones, and the two answer different questions (build_utils.skip_example): + examples/device/dfu carries `mcu:BCM2835` in skip.txt, which the cmake FAMILY_MCUS + union applies to every broadcom_64bit board while the make scrape applies it to + none - asking cmake alone drops the only aarch64-gcc family in the matrix and + `build-make-aarch64-gcc` stops compiling dfu at all.""" + out_fams, out_ex, reasons = [], {}, [] + allex = list(all_examples(repo_root)) + with _in_repo(repo_root): + for fam in fams: + if not os.path.isdir(os.path.join(repo_root, 'hw/bsp', fam, 'boards')): + # a PR that deletes or renames hw/bsp/<fam> still names it in the + # diff (rule 6); the family builds nothing now, and get_family_boards + # would raise FileNotFoundError out of the whole selector + reasons.append(f'{fam}: family dir gone from tree, dropped') + continue + try: + # ci=True unconditionally: this answers "what will CI build", so it must + # not change with GITHUB_ACTIONS/CIRCLECI being set. Locally the lists + # are off by default, and rp2040 would keep feather_rp2040_max3421 - + # the only board satisfying the max3421 only.txt files - giving a + # developer a family list the runner will not reproduce. + boards = build_py.get_family_boards(fam, False, False, ci=True) + except OSError as e: # belt and braces: never traceback here + reasons.append(f'{fam}: boards unreadable ({e}), dropped') + continue + if not boards: + out_fams.append(fam) # unknown layout: keep unfiltered + continue + # what this family's build path can even see, asked the same way for + # every family. build.py's espressif branch builds get_examples('espressif') + # only (the *_freertos examples plus a short extra list); keeping the family + # for anything else spins up CI's most expensive leg to skip every example + # it was given. Identical to the unfiltered list on all 81 other families. + pool = set(build_py.get_examples(fam)) + try: + buildable = [e for e in allex if e in pool and + any(not build_utils.skip_example(e, b) or + not build_utils.skip_example(e, b, (), 'make') + for b in boards)] + except OSError as e: + # a family mid-bring-up (boards/ but no family.cmake/family.mk yet) + # reads as unbuildable to the scrape; keep it rather than tracebacking + # out of the selector and losing the scoping for the whole PR + reasons.append(f'{fam}: mcu scrape unreadable ({e}), kept unfiltered') + out_fams.append(fam) + continue + want = fam_ex.get(fam) + have = set(buildable) + kept = buildable if want is None else [e for e in want if e in have] + if not kept: + continue # this diff builds nothing for this family + out_fams.append(fam) + if set(kept) != set(buildable): + out_ex[fam] = kept + return out_fams, out_ex, reasons + + +def classify_build(changed_files, repo_root, get_deps_families=None): + s = _BSel() + for p in changed_files: + _classify_build_one(p, repo_root, s, get_deps_families) + if s.full: + return {'full': True, 'families': list(all_bsp_families(repo_root)), + 'family_examples': {}, 'reasons': s.reasons} + fams = sorted(s.fam_ex) + fam_ex = {f: sorted(e) for f, e in s.fam_ex.items() if e != 'all'} + fams, fam_ex, pruned = _prune_buildable(fams, fam_ex, repo_root) + s.reasons += pruned + return {'full': False, 'families': fams, 'family_examples': fam_ex, + 'reasons': s.reasons} + + +if __name__ == '__main__': + main() diff --git a/tools/get_deps.py b/tools/get_deps.py index baaf3761f..12bec4861 100755 --- a/tools/get_deps.py +++ b/tools/get_deps.py @@ -33,7 +33,7 @@ deps_mandatory = { deps_optional = { 'hw/mcu/allwinner': ['https://github.com/hathach/allwinner_driver.git', '8e5e89e8e132c0fd90e72d5422e5d3d68232b756', - 'fc100s'], + 'f1c100s'], 'hw/mcu/analog/msdk' : ['https://github.com/analogdevicesinc/msdk.git', 'b20b398d3e5e2007594e54a74ba3d2a2e50ddd75', 'maxim'], @@ -108,7 +108,7 @@ deps_optional = { 'efm32'], 'hw/mcu/sony/cxd56/spresense-exported-sdk': ['https://github.com/sonydevworld/spresense-exported-sdk.git', '2ec2a1538362696118dc3fdf56f33dacaf8f4067', - 'spresense'], + 'cxd56'], 'hw/mcu/st/cmsis_device_c0': ['https://github.com/STMicroelectronics/cmsis_device_c0.git', '517611273f835ffe95318947647bc1408f69120d', 'stm32c0'], @@ -386,6 +386,10 @@ def main(): parser.add_argument('-f1', '--build-flags-on', action='append', default=[], help='Have no effect') parser.add_argument('--build-name', default=None, help='Have no effect') parser.add_argument('--cflag', action='append', default=[], help='Have no effect') + # build-matrix entries carry -e for tools/build.py; they reach get_deps.py + # verbatim (.github/actions/get_deps, build.yml's hil-hfp-iar) and an + # argparse error here reds the Get Dependencies step of every scoped PR + parser.add_argument('-e', '--example', action='append', default=[], help='Have no effect') args = parser.parse_args() families = args.families diff --git a/tools/iar_template.ipcf b/tools/iar_template.ipcf index 035e40b94..922b22426 100644 --- a/tools/iar_template.ipcf +++ b/tools/iar_template.ipcf @@ -81,9 +81,7 @@ </group> <group name="src/class/vendor"> <path>$TUSB_DIR$/src/class/vendor/vendor_device.c</path> - <path>$TUSB_DIR$/src/class/vendor/vendor_host.c</path> <path>$TUSB_DIR$/src/class/vendor/vendor_device.h</path> - <path>$TUSB_DIR$/src/class/vendor/vendor_host.h</path> </group> <group name="src/class/video"> <path>$TUSB_DIR$/src/class/video/video_device.c</path> diff --git a/tools/metrics.py b/tools/metrics.py index 0e29fc1ab..27c995954 100644 --- a/tools/metrics.py +++ b/tools/metrics.py @@ -98,6 +98,24 @@ def combine_files(input_files, filters=None): if fin.endswith(".json"): with open(fin, "r", encoding="utf-8") as f: json_data = json.load(f) + if fin.endswith('_by_example.json') and isinstance(json_data, dict) and \ + all(isinstance(v, dict) and 'files' in v for v in json_data.values()): + # a metrics_by_example.json: one data entry per example. Keyed on + # the filename, which IS the contract (write_by_example, the CMake + # rule and metrics_pair_compare all spell that suffix) - a shape + # sniff would silently reroute any coincidentally-shaped JSON. + for ex in sorted(json_data): + # same TOTAL scrub the shared path below applies: this branch + # `continue`s past it, so do it here or a by-example input keeps + # the fake TOTAL rows an ordinary input has stripped + sub = {'files': [f for f in json_data[ex]['files'] + if str(f.get('file', '')).upper() != 'TOTAL']} + if filters: + sub['files'] = [f for f in sub['files'] + if f.get('path') and any(x in f['path'] for x in filters)] + all_json_data['file_list'].append(f'{fin}:{ex}') + all_json_data['data'].append(sub) + continue if filters: json_data["files"] = [ f @@ -316,6 +334,25 @@ def write_json_output(json_data, path): json.dump(json_data, outf, indent=2) +def write_by_example(all_json_data, path): + """{<role>/<example>: {files: [...]}} from the data combine_files already parsed + - re-reading and re-parsing every input a second time bought nothing. + + Inputs are map.json files laid out as <build>/<role>/<example>/<name>.map.json + (examples/CMakeLists.txt's pattern), so the example name is the last two path + components; a metrics_by_example.json input already carries its own name in the + file_list entry ('<file>.json:<role>/<name>').""" + out = {} + for fin, data in zip(all_json_data["file_list"], all_json_data["data"]): + _, sep, ex = fin.partition('.json:') + if not sep: + d = os.path.dirname(os.path.abspath(fin)) + ex = f'{os.path.basename(os.path.dirname(d))}/{os.path.basename(d)}' + out.setdefault(ex, {'files': []})['files'] += data.get('files', []) + with open(path, 'w', encoding='utf-8') as f: + json.dump(out, f) + + def render_combine_table(json_data, sort_order='name+'): """Render averaged sizes as markdown table lines (no title).""" files = json_data.get("files", []) @@ -594,6 +631,8 @@ def cmd_combine(args): if args.markdown_out: write_combine_markdown(json_average, args.out + '.md', sort_order=args.sort, title="TinyUSB Average Code Size Metrics") + if args.by_example: + write_by_example(all_json_data, args.out + '_by_example.json') def cmd_compare(args): @@ -633,6 +672,8 @@ def main(argv=None): combine_parser.add_argument('-S', '--sort', dest='sort', default='size-', choices=['size', 'size-', 'size+', 'name', 'name-', 'name+'], help='Sort order: size/size- (descending), size+ (ascending), name/name+ (ascending), name- (descending). Default: size-') + combine_parser.add_argument('--by-example', dest='by_example', action='store_true', + help='Also write <out>_by_example.json: per-example file lists keyed by role/example') # Compare subcommand compare_parser = subparsers.add_parser('compare', help='Compare two metrics inputs (bloaty CSV or metrics JSON)') |