From 696c7807f543a6c55656d81a8f6d8969584e9614 Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 21 Aug 2026 11:06:55 +0700 Subject: get_deps: correct two family tokens that matched nothing get_deps matches a family token against a requested family name verbatim (`f in deps_optional[d][2].split()`), so a token naming no hw/bsp directory makes its entry unreachable: hw/mcu/allwinner said 'fc100s'; the family is hw/bsp/f1c100s, and f1c100s/family.cmake sets SDK_DIR to ${TOP}/hw/mcu/allwinner/f1c100s hw/mcu/sony/cxd56/spresense-exported-sdk said 'spresense' (the SDK's name); the family is hw/bsp/cxd56, whose family.cmake points SDK_DIR at it `python3 tools/get_deps.py f1c100s` and `... cxd56` now fetch the SDK each of those families builds against; before, both printed "no additional dependencies found". docs/reference/dependencies.rst is generated from deps_all by tools/gen_doc.py, so it is updated to match - column widths are unchanged (the widest cell is lib/CMSIS_5's, untouched) and every row was cross-checked against deps_all. --- docs/reference/dependencies.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'docs/reference') diff --git a/docs/reference/dependencies.rst b/docs/reference/dependencies.rst index 146192ef8..4118b94c3 100644 --- a/docs/reference/dependencies.rst +++ b/docs/reference/dependencies.rst @@ -7,7 +7,7 @@ MCU low-level peripheral drivers and external libraries for building TinyUSB exa ======================================== ================================================================ ======================================== =================================================================================================================================================================================================================================================================================================================================================== Local Path Repo Commit Required by ======================================== ================================================================ ======================================== =================================================================================================================================================================================================================================================================================================================================================== -hw/mcu/allwinner https://github.com/hathach/allwinner_driver.git 8e5e89e8e132c0fd90e72d5422e5d3d68232b756 fc100s +hw/mcu/allwinner https://github.com/hathach/allwinner_driver.git 8e5e89e8e132c0fd90e72d5422e5d3d68232b756 f1c100s hw/mcu/analog/msdk https://github.com/analogdevicesinc/msdk.git b20b398d3e5e2007594e54a74ba3d2a2e50ddd75 maxim hw/mcu/artery/at32f402_405 https://github.com/ArteryTek/AT32F402_405_Firmware_Library.git 4424515c2663e82438654e0947695295df2abdfe at32f402_405 hw/mcu/artery/at32f403a_407 https://github.com/ArteryTek/AT32F403A_407_Firmware_Library.git f2cb360c3d28fada76b374308b8c4c61d37a090b at32f403a_407 @@ -38,7 +38,7 @@ hw/mcu/raspberry_pi/Pico-PIO-USB https://github.com/sekigon-gonnoc/Pico hw/mcu/renesas/fsp https://github.com/renesas/fsp.git edcc97d684b6f716728a60d7a6fea049d9870bd6 ra hw/mcu/renesas/rx https://github.com/kkitayam/rx_device.git 706b4e0cf485605c32351e2f90f5698267996023 rx hw/mcu/silabs/cmsis-dfp-efm32gg12b https://github.com/cmsis-packs/cmsis-dfp-efm32gg12b.git f1c31b7887669cb230b3ea63f9b56769078960bc efm32 -hw/mcu/sony/cxd56/spresense-exported-sdk https://github.com/sonydevworld/spresense-exported-sdk.git 2ec2a1538362696118dc3fdf56f33dacaf8f4067 spresense +hw/mcu/sony/cxd56/spresense-exported-sdk https://github.com/sonydevworld/spresense-exported-sdk.git 2ec2a1538362696118dc3fdf56f33dacaf8f4067 cxd56 hw/mcu/st/cmsis-device-u0 https://github.com/STMicroelectronics/cmsis-device-u0.git e3a627c6a5bc4eb2388e1885a95cc155e1672253 stm32u0 hw/mcu/st/cmsis-device-wba https://github.com/STMicroelectronics/cmsis-device-wba.git 647d8522e5fd15049e9a1cc30ed19d85e5911eaf stm32wba hw/mcu/st/cmsis_device_c0 https://github.com/STMicroelectronics/cmsis_device_c0.git 517611273f835ffe95318947647bc1408f69120d stm32c0 -- cgit v1.3.1 From 04d0f71984117b8c72349f4584bd9e26a37b129c Mon Sep 17 00:00:00 2001 From: hathach Date: Fri, 21 Aug 2026 11:07:27 +0700 Subject: ci: scope the build matrix and the HIL run to what a PR affects Every PR built all 74 legs (2494 example builds on GHA cmake alone) and flashed all 30 rig boards, whatever it touched. One classifier now walks the PR diff twice and answers three questions: which families to build, which examples per family, and which boards run which tests. Fail-open throughout - anything no rule classifies, any exception, any unusable output falls back to the full matrix, and a master push always builds everything. test/hil/helper/hil_select.py moves to tools/ci_select.py: it is no longer HIL-only, and tools/ is where the build side can import it. test_hil_select.py follows it as test_ci_select.py. Rules (docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md holds the full table): a port selects the families whose family.cmake references it, and its role - a dcd change skips host examples and vice versa; a class selects only the examples whose tusb_config.h enables its CFG_TU[DH]_ macro, following cross-class includes; an example selects itself; hw/bsp selects its family or board; hw/mcu and lib select whoever references them. CMake is the reference for all of it - make follows whatever cmake decides, family.mk is never scanned. Empty means empty (maintainer ruling): a rule that classifies a path to nothing selects nothing. Ports no family references, classes no config enables, libs no example builds and hw/mcu paths that resolve nowhere are all real - nothing compiles them, so nothing can validate them, and the master-push build is the net. Structural tests pin each such case with an explicit allowlist, so the day one stops being empty it fails pre-commit instead of silently narrowing CI. Per-example builds: build.py grows a repeatable -e, resolved against the targets CMake actually registered and batched into one `cmake --build --target a b c`. build_utils mirrors CMake's family_filter (the whole FAMILY_MCUS list, ${...} and string(TOUPPER ...) resolved) for the cmake side, while the make side keeps master's algorithm verbatim - the two build systems answer differently and a shared answer breaks lpc54's make link. hil-build gains this even on a full selection: 1702 example builds become 515. Transport: the selection travels as a file, never an argv or env var - a mass-sweep diff selects 261 KB against a 128 KiB exec limit, and E2BIG would fail the step before its own fallback could run. CircleCI carries the example map inside the generated config (pipeline parameters cap at 512 chars), swapped into the parameter defaults by sentinel match, and drops the scoping wholesale if that rewrite fails. Every PR-derived value written to $GITHUB_ENV/$GITHUB_OUTPUT is character-screened. Code metrics follow the scoping: metrics.py emits per-example totals, and metrics_pair_compare compares the (board, example) pairs present on both sides instead of a scoped run against a full-matrix average. The selector's own suite gates it in both providers: a selector that exits 0 with valid-but-wrong JSON is the one failure fail-open cannot catch, so a red suite means the full matrix. --- .circleci/config.yml | 105 +- .circleci/config2.yml | 52 +- .claude/skills/hil/SKILL.md | 17 +- .claude/skills/pre-pr/SKILL.md | 2 +- .github/scripts/ci_set_matrix.py | 78 +- .github/scripts/hil_ci_set_matrix.py | 47 +- .github/scripts/metrics_pair_compare.py | 130 ++ .github/workflows/build.yml | 211 +- .github/workflows/build_util.yml | 65 +- .pre-commit-config.yaml | 31 +- docs/reference/hardware-in-the-loop.md | 5 +- .../superpowers/followup/pr3803-flasher-recover.md | 24 +- examples/CMakeLists.txt | 2 +- hw/bsp/mcx/family.cmake | 2 +- test/hil/helper/hil_select.py | 524 ----- test/hil/helper/hil_util.py | 2 +- test/hil/hil_ci.sh | 1 - test/hil/hil_flash.py | 2 +- test/hil/test/test_ci_metrics.py | 441 ++++ test/hil/test/test_ci_select.py | 2154 ++++++++++++++++++++ test/hil/test/test_hil_select.py | 689 ------- test/hil/test/test_hil_util.py | 17 +- tools/build.py | 154 +- tools/build_utils.py | 319 ++- tools/ci_select.py | 1084 ++++++++++ tools/get_deps.py | 4 + tools/metrics.py | 46 +- 27 files changed, 4854 insertions(+), 1354 deletions(-) create mode 100755 .github/scripts/metrics_pair_compare.py delete mode 100755 test/hil/helper/hil_select.py create mode 100644 test/hil/test/test_ci_metrics.py create mode 100644 test/hil/test/test_ci_select.py delete mode 100644 test/hil/test/test_hil_select.py create mode 100755 tools/ci_select.py (limited to 'docs/reference') diff --git a/.circleci/config.yml b/.circleci/config.yml index 48fa87899..8c3f09111 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,9 +15,87 @@ jobs: - run: name: Set matrix command: | - MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py) + # PR-scoped selection (best-effort: any failure falls back to the full + # matrix). CircleCI has no base-branch var; tinyusb PRs target master. + # The selection lands in a FILE and never travels as an argv: a mass-sweep + # diff selects hundreds of KB, and E2BIG would fail the step before the + # `||` fallback could fire - leaving a full build labelled scoped, because + # EXAMPLE_MAP/BUILD_FILTERED below have no such limit and stay scoped. + SELECT_FILE=ci_select_out.json + rm -f "$SELECT_FILE" + if [ -n "${CIRCLE_PULL_REQUEST:-}" ]; then + git fetch --no-tags origin master || true + # both suites gate the selector: test_ci_select.py owns the rules, + # test_ci_metrics.py owns the config2 sentinel contract this job rewrites + if python3 test/hil/test/test_ci_select.py >/dev/null 2>&1 && + python3 test/hil/test/test_ci_metrics.py >/dev/null 2>&1; then + python3 tools/ci_select.py --base origin/master > "$SELECT_FILE" || rm -f "$SELECT_FILE" + else + echo "ci_select unit suite failed - using the full matrix" + fi + fi + [ -s "$SELECT_FILE" ] || rm -f "$SELECT_FILE" + + # computed once, up front: it is both the fallback and what the scoping is + # dropped back to further down, and a second invocation there would be an + # unguarded command under `set -e` inside the very branch that exists to + # keep the pipeline green + FULL_MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py 2>/dev/null) || FULL_MATRIX_JSON='' + MATRIX_JSON='' + if [ -f "$SELECT_FILE" ]; then + # ci_set_matrix also falls open with rc 0, saying UNSCOPED on stderr. The + # extras below must follow it, exactly as build.yml does: a full matrix + # paired with a still-scoped -e list builds a fraction of each family and + # tells code-metrics it was an unscoped run. + MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py --select-file "$SELECT_FILE" 2>ci_set_matrix.err) || MATRIX_JSON='' + cat ci_set_matrix.err + if [ -z "$MATRIX_JSON" ] || grep -q 'ci_set_matrix: UNSCOPED' ci_set_matrix.err; then + SELECT_FILE='' + fi + fi + [ -n "$MATRIX_JSON" ] || MATRIX_JSON="$FULL_MATRIX_JSON" echo "MATRIX_JSON=$MATRIX_JSON" + EXAMPLE_MAP='{}' + BUILD_FILTERED='false' + if [ -f "$SELECT_FILE" ]; then + EXAMPLE_MAP=$(jq -c '.build.family_examples // {}' < "$SELECT_FILE") || EXAMPLE_MAP='{}' + BUILD_FILTERED=$(jq -r 'if (.build? | type) == "object" and .build.full == false then "true" else "false" end' < "$SELECT_FILE") || BUILD_FILTERED='false' + fi + + # /pipeline/continue caps parameter values at 512 chars - a scoped map is + # KBs, so both values ride inside the generated config itself (config max + # is 3MB), swapped into the parameter defaults by sentinel-line match. + # Fail-open: a sentinel that drifted (renamed comment, reformatted line) + # must not red EVERY CircleCI pipeline. The rewrite is all-or-nothing + # (config2.yml is only written once both substitutions succeeded). + # + # Done BEFORE the family entries are generated, and a failure drops the + # scoping entirely: the checked-in defaults are {} / false = unfiltered, so + # a scoped FAMILY list with unfiltered defaults would build a subset of + # families while telling code-metrics it had built them all. + if ! EXAMPLE_MAP="$EXAMPLE_MAP" BUILD_FILTERED="$BUILD_FILTERED" python3 - \<<'PYEOF' + import os + p = '.circleci/config2.yml' + t = open(p).read() + def yq(s): # YAML single-quoted scalar + return "'" + s.replace("'", "''") + "'" + for env, tag in (('EXAMPLE_MAP', 'example-map-default'), + ('BUILD_FILTERED', 'build-filtered-default')): + old = [l for l in t.splitlines() if l.strip().endswith(f'# {tag}: rewritten in-place by config.yml set-matrix')] + assert len(old) == 1, f'{tag}: sentinel not found exactly once' + line = old[0] + new = line.split('default:')[0] + 'default: ' + yq(os.environ[env]) + f' # {tag}' + t = t.replace(line, new, 1) + open(p, 'w').write(t) + PYEOF + then + echo "warning: sentinel rewrite failed - dropping the scoping, full build" + MATRIX_JSON="$FULL_MATRIX_JSON" + EXAMPLE_MAP='{}' + BUILD_FILTERED='false' + fi + BUILDSYSTEM_LIST=( "cmake" "make" @@ -75,7 +153,15 @@ jobs: FAMILY=$(echo $MATRIX_JSON | jq -r ".\"$toolchain\"") echo "FAMILY_${toolchain}=$FAMILY" + + if [ "$(echo "$FAMILY" | jq 'length')" = "0" ]; then + # an empty matrix parameter is a hard CircleCI config error, not a skip + echo "skip build-${build_system}-${toolchain}: no families selected" + continue + fi + gen_build_entry "$build_system" "$toolchain" "$FAMILY" + ANY_BUILD=1 # Only add cmake builds: excluding esp-idf or build_args="--one-random" to metrics requirements if [ "$build_system" == "cmake" ] && [ "$toolchain" != "esp-idf" ] && [ "$toolchain" != "arm-iar" ]; then @@ -84,12 +170,17 @@ jobs: done done - # Add code-metrics job that requires all build jobs - echo " - code-metrics:" >> .circleci/config2.yml - echo " requires:" >> .circleci/config2.yml - for alias in "${BUILD_ALIASES[@]}"; do - echo " - $alias" >> .circleci/config2.yml - done + if [ ${#BUILD_ALIASES[@]} -gt 0 ]; then + echo " - code-metrics:" >> .circleci/config2.yml + echo " requires:" >> .circleci/config2.yml + for alias in "${BUILD_ALIASES[@]}"; do + echo " - $alias" >> .circleci/config2.yml + done + fi + if [ "${ANY_BUILD:-0}" != "1" ]; then + # a workflow with zero jobs is invalid config + echo " - no-op" >> .circleci/config2.yml + fi - continuation/continue: configuration_path: .circleci/config2.yml diff --git a/.circleci/config2.yml b/.circleci/config2.yml index e0bd917a4..899cbe24a 100644 --- a/.circleci/config2.yml +++ b/.circleci/config2.yml @@ -1,5 +1,13 @@ version: 2.1 +parameters: + example-map: + type: string + default: "{}" # example-map-default: rewritten in-place by config.yml set-matrix + build-filtered: + type: string + default: "false" # build-filtered-default: rewritten in-place by config.yml set-matrix + commands: setup-toolchain: parameters: @@ -109,9 +117,17 @@ commands: - run: name: Build no_output_timeout: 20m + environment: + EXAMPLE_MAP: << pipeline.parameters.example-map >> command: | + # PR example filter for this family ('{}' or a missing key = build all). + # The map is the PR-derived value, so it must ride via env rather than + # shell-text interpolation (unsafe characters); family is a job + # parameter with charset [a-z0-9_], safe to interpolate directly. + EX_ARGS=$(printf '%s' "$EXAMPLE_MAP" | jq -r --arg fam "<< parameters.family >>" '(.[$fam] // []) | map("-e " + .) | join(" ")' 2>/dev/null) || EX_ARGS='' + if [ << parameters.toolchain >> == esp-idf ]; then - docker run --rm -v $PWD:/project -w /project espressif/idf:v5.5.3 python tools/build.py << parameters.build-args >> --target all << parameters.family >> + docker run --rm -v $PWD:/project -w /project espressif/idf:v5.5.3 python tools/build.py << parameters.build-args >> --target all $EX_ARGS << parameters.family >> else # Toolchain option default is gcc if [ << parameters.toolchain >> == arm-clang ]; then @@ -129,7 +145,7 @@ commands: if [ << parameters.build-system >> == "cmake" ]; then BUILD_PY_ARGS="$BUILD_PY_ARGS --target tinyusb_metrics" fi - python tools/build.py $BUILD_PY_ARGS << parameters.family >> + python tools/build.py $BUILD_PY_ARGS $EX_ARGS << parameters.family >> fi # Only collect and persist metrics for cmake builds (excluding esp-idf and --one-random) @@ -248,8 +264,10 @@ jobs: # Compare with base master metrics on PR branches - when: condition: - not: - equal: [ master, << pipeline.git.branch >> ] + and: + - not: + equal: [ master, << pipeline.git.branch >> ] + - equal: [ "false", << pipeline.parameters.build-filtered >> ] steps: - run: name: Download Base Branch Metrics @@ -276,6 +294,32 @@ jobs: - store_artifacts: path: metrics_compare.md destination: metrics_compare.md + - when: + condition: + and: + - not: + equal: [ master, << pipeline.git.branch >> ] + - equal: [ "true", << pipeline.parameters.build-filtered >> ] + steps: + - run: + name: Scoped build - comparison unavailable + command: | + # CircleCI stores only the averaged metrics.json; the per-example + # baseline lives on GHA. See the GHA code-metrics PR comment. + echo "_Code-size comparison skipped on CircleCI: this PR built a scoped example set._" > metrics_compare.md + + - store_artifacts: + path: metrics_compare.md + destination: metrics_compare.md + + no-op: + docker: + - image: cimg/base:current + resource_class: small + steps: + - run: + name: No families selected + command: echo "PR selection - no families to build on CircleCI" workflows: build: diff --git a/.claude/skills/hil/SKILL.md b/.claude/skills/hil/SKILL.md index f0c449d33..d1e4bdcd5 100644 --- a/.claude/skills/hil/SKILL.md +++ b/.claude/skills/hil/SKILL.md @@ -43,11 +43,11 @@ Use it before a HIL campaign, after rig maintenance/reboot, or when boards fail ## PR-scoped selection -`test/hil/helper/hil_select.py` maps a diff to affected boards/tests (used by CI on PRs; fail-open +`tools/ci_select.py` maps a diff to affected boards/tests (used by CI on PRs; fail-open to the full matrix). Manual use: ```bash -SEL=$(python3 test/hil/helper/hil_select.py --base master test/hil/tinyusb.json) +SEL=$(python3 tools/ci_select.py --base master test/hil/tinyusb.json) FULL=$(printf '%s' "$SEL" | python3 -c "import json,sys; print(json.load(sys.stdin)['full'])") ARGS=$(printf '%s' "$SEL" | python3 -c "import json,sys; print(json.load(sys.stdin)['args']['tinyusb.json'])") if [ "$FULL" = "True" ] || [ -n "$ARGS" ]; then @@ -60,11 +60,14 @@ fi Read `full`, never `args` alone: `args` is empty for BOTH `full: true` (run the whole matrix — a broad or unclassified change) and "nothing selected" (skip). Skip only when `full` is false AND `args` is empty. -Unit suites (no hardware), all four run by the `hil-test`/`hil-select-test` pre-commit -hooks: `test_hil_select.py` covers only board selection. The containment work --- bounded -reads, the kill ladders, the build and pool guards --- lives in `test_hil_bounded.py`, -`test_hil_health.py` and `test_hil_util.py`, so run all four when changing `test/hil`: -`for f in test/hil/test/test_*.py; do python3 "$f"; done` (~55s). +Unit suites (no hardware), all five run by the `hil-test`/`ci-select-test` pre-commit +hooks: `test_ci_select.py` covers only selection, `test_ci_metrics.py` only the code-size +plumbing. The containment work --- bounded reads, the kill ladders, the build and pool +guards --- lives in `test_hil_bounded.py`, `test_hil_health.py` and `test_hil_util.py`, so +run all five when changing `test/hil`: +`for f in test/hil/test/test_*.py; do python3 "$f"; done` (~84s, of which +`test_hil_bounded.py` is ~76s of deliberate hang/timeout simulation; the two `test_ci_*` +suites are ~4s together). ## Pre-flight rig health check diff --git a/.claude/skills/pre-pr/SKILL.md b/.claude/skills/pre-pr/SKILL.md index b96750e4f..8e5c408a6 100644 --- a/.claude/skills/pre-pr/SKILL.md +++ b/.claude/skills/pre-pr/SKILL.md @@ -15,7 +15,7 @@ Run the software + hardware gate for the current branch. The user invoking this ## 2. Map changes to boards -- `python3 test/hil/helper/hil_select.py --base $BASE test/hil/tinyusb.json` → JSON with the affected +- `python3 tools/ci_select.py --base $BASE test/hil/tinyusb.json` → JSON with the affected bsp `families`, the affected rig `boards`, and per-file `reasons`. `full: true` means a broad/infra change. - Affected families = `families` ∪ the family of every name in `boards`. Neither half is diff --git a/.github/scripts/ci_set_matrix.py b/.github/scripts/ci_set_matrix.py index 50ada5964..ee3609bed 100755 --- a/.github/scripts/ci_set_matrix.py +++ b/.github/scripts/ci_set_matrix.py @@ -1,5 +1,9 @@ #!/usr/bin/env python3 +import argparse import json +import os +import subprocess +import sys # toolchain, url toolchain_list = [ @@ -97,15 +101,77 @@ family_list = { } -def set_matrix_json(): +def set_matrix_json(select=None): + sel_fams = None + if select: + # every shape check is explicit: this runs AFTER main()'s fail-open handler, so + # an AttributeError on e.g. {"build": ["stm32f4"]} would red the step instead + # of falling back to the full matrix - the outcome that handler exists to prevent + b = select.get('build') if isinstance(select, dict) else None + if not isinstance(b, dict): + b = {} + if b.get('full') is False: + fams = b.get('families') + if not (isinstance(fams, list) and all(isinstance(f, str) for f in fams)): + # key ABSENT (or not a list of names) is an unusable selection, not + # "nothing selected": scoping every toolchain to [] would build zero + # families and report a vacuous green. An explicit families: [] stays a + # legitimate nothing-selected. + print('ci_set_matrix: UNSCOPED - build.full is false but the families ' + 'list is unusable, emitting the full matrix', file=sys.stderr) + else: + sel_fams = set(fams) matrix = {} for toolchain in toolchain_list: - filtered_families = [family for family, supported_toolchain in family_list.items() if - toolchain in supported_toolchain] - matrix[toolchain] = filtered_families - + fams = [family for family, tc in family_list.items() if toolchain in tc] + if sel_fams is not None: + fams = [f for f in fams if f in sel_fams] + matrix[toolchain] = fams + if sel_fams is not None: + # a family this file does not list builds on no toolchain, so the selection maps + # to an empty matrix and every leg skips - which looks exactly like a working + # scoped run. Say so: hw/bsp holds several families CI has never built + # (efm32, py32f0, ...) and espressif, whose boards are built by hil-build-esp + unbuilt = sorted(f for f in sel_fams if f not in family_list) + if unbuilt: + print(f'ci_set_matrix: selected families built by no toolchain here: ' + f'{", ".join(unbuilt)}', file=sys.stderr) print(json.dumps(matrix)) +def main(): + parser = argparse.ArgumentParser() + group = parser.add_mutually_exclusive_group() + group.add_argument('--select', help='tools/ci_select.py JSON; scopes families when build.full is false') + # a whole selection as one argv/env value can exceed the exec limits on a big + # diff, which fails the calling step BEFORE it can fall open; callers that + # already have the selection on disk pass the path instead + group.add_argument('--select-file', help='file holding the same JSON as --select') + group.add_argument('--base', help='git ref: run tools/ci_select.py --base REF and scope from it') + args = parser.parse_args() + + select = None + try: + if args.select: + select = json.loads(args.select) + elif args.select_file: + with open(args.select_file) as f: + select = json.load(f) + elif args.base: + root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + r = subprocess.run([sys.executable, os.path.join(root, 'tools', 'ci_select.py'), + '--base', args.base], + capture_output=True, text=True, cwd=root, check=True) + select = json.loads(r.stdout) + except Exception as e: # fail-open: an unusable selection must never turn into a red job + # UNSCOPED is the marker build.yml greps for: it must then drop the build extras + # (example map, family regex) too, or a full build gets labelled and filtered as + # a scoped one. Keep the token on every fall-open path. + print(f'ci_set_matrix: UNSCOPED - selection unusable ({e}), emitting the full ' + f'matrix', file=sys.stderr) + select = None + set_matrix_json(select) + + if __name__ == '__main__': - set_matrix_json() + main() diff --git a/.github/scripts/hil_ci_set_matrix.py b/.github/scripts/hil_ci_set_matrix.py index 65f50788e..396c4175a 100644 --- a/.github/scripts/hil_ci_set_matrix.py +++ b/.github/scripts/hil_ci_set_matrix.py @@ -1,6 +1,7 @@ import argparse import json import os +import sys def _resolve_config_path(config_file): @@ -19,13 +20,47 @@ def _resolve_config_path(config_file): def main(): parser = argparse.ArgumentParser() parser.add_argument('config_files', nargs='+', help='Configuration JSON file(s)') - parser.add_argument('--select', help='hil_select.py JSON; scopes boards when full=false') + g = parser.add_mutually_exclusive_group() + g.add_argument('--select', help='ci_select.py JSON; scopes boards when full=false') + # a whole selection as one argv can exceed MAX_ARG_STRLEN on a big diff, which + # would fail the step instead of falling open; callers that already have the + # selection on disk pass the path instead + g.add_argument('--select-file', help='file holding the same JSON as --select') args = parser.parse_args() + raw = args.select + sel = None + try: + if args.select_file: + with open(args.select_file) as f: + raw = f.read() + if raw: + sel = json.loads(raw) + if sel is not None and not isinstance(sel, dict): + raise ValueError(f'selection is {type(sel).__name__}, not an object') + except Exception as e: # fail-open: an unusable selection must never red the job + print(f'hil_ci_set_matrix: selection unusable ({e}) - full roster', + file=sys.stderr) + sel = None + selected = None - sel = json.loads(args.select) if args.select else None if sel and not sel.get('full'): - selected = set(sel.get('boards', {})) + # key ABSENT is an unusable selection, not "nothing selected" - same reading as + # ci_set_matrix.py. Filtering every board out would skip every hil-build leg and, + # through needs:, both rig jobs: an all-green PR with zero hardware coverage. + # An explicit boards: {} stays a legitimate nothing-selected. + if not isinstance(sel.get('boards'), dict): + print('hil_ci_set_matrix: selection has full false but no usable boards ' + 'map - full roster', file=sys.stderr) + sel = None # ALL of it is unusable, hil_examples included: keeping + # the -e lists would build a few examples per board + # while the rig, unfiltered, runs that board's whole + # test list - flash failures on the fail-open path + else: + selected = set(sel['boards']) + ex_map = (sel or {}).get('hil_examples') or {} + if not isinstance(ex_map, dict): + ex_map = {} # Toolchain buckets must match the toolchains instantiated by the hil-build # job in .github/workflows/build.yml. Keep all keys present (even if empty) @@ -71,6 +106,12 @@ def main(): if 'build' in board and 'args' in board['build']: build_board += ' ' + ' '.join(f'-D{a}' for a in board['build']['args']) + # PR selection: build only the examples this board will run (its test + # list plus device/board_test, the parking firmware) - tools/build.py -e. + # Absent key (hand runs, full non-PR builds) keeps --target all. + for ex in ex_map.get(name, []): + build_board += f' -e {ex}' + # Each variant builds into cmake-build- with its own cmake # -D defines and raw CFLAGS. No 'variant' -> a single build named after # the board. diff --git a/.github/scripts/metrics_pair_compare.py b/.github/scripts/metrics_pair_compare.py new file mode 100755 index 000000000..50107cf72 --- /dev/null +++ b/.github/scripts/metrics_pair_compare.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 +"""Board+example-matched code-size compare for PR-scoped builds. + +The averaged metrics baseline (metrics-tinyusb) spans every family and example; +a scoped PR builds a subset, so comparing against it is apples-to-oranges. This +compares the intersection of (board, example) pairs present on BOTH sides, +averaged over exactly those pairs, and names what was dropped. See +docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md #code-metrics. +""" +import argparse +import glob +import json +import os +import sys +import tempfile + +sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', '..', 'tools')) +import metrics + +# dropped (board, example) pairs named in the PR comment before it truncates +DROPPED_SHOWN = 20 + + +def board_family(board, repo_root): + hits = glob.glob(os.path.join(repo_root, 'hw/bsp/*/boards', board)) + return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None + + +def collect(root, repo_root): + """{(board, 'role/example'): [file entries]} from every + **/cmake-build-/metrics_by_example.json under root. + + Keyed on the BOARD, not its family. The two sides are built by + `--one-first`, which returns all_boards[0] for a family with no + ci_preferred_boards entry - so a PR that adds hw/bsp//boards/a_new_board + shifts which board is built, and a family key would file the base run's sizes and + the PR run's sizes under the same name and publish the difference between two + unrelated MCUs as this PR's code-size impact. On the board key that mismatch lands + in `dropped` (reported as not compared), which is the truth.""" + pairs = {} + pat = os.path.join(root, '**', 'metrics_by_example.json') + for f in sorted(glob.glob(pat, recursive=True)): + board = os.path.basename(os.path.dirname(f)) + if not board.startswith('cmake-build-'): + print(f'pair_compare: {f} not under a cmake-build- dir, skipping', file=sys.stderr) + continue + board = board[len('cmake-build-'):] + if not board_family(board, repo_root): + # unknown board: the name is still a usable key, but say so - it means the + # artifact came from a tree whose hw/bsp does not match this checkout + print(f'pair_compare: no family for board {board}', file=sys.stderr) + # parse into a LOCAL dict and merge only once the whole file came out clean: + # a file that blows up half way through must drop WHOLE, or the entries read + # before the malformation stay in the comparison while stderr says the file + # was skipped, and a silently truncated table gets published as the verdict + try: + one = {} + for ex, ent in json.load(open(f)).items(): + one.setdefault((board, ex), []).extend(ent.get('files', [])) + except (OSError, ValueError, AttributeError, TypeError) as e: + print(f'pair_compare: unreadable {f} ({e}), skipping', file=sys.stderr) + continue + for k, v in one.items(): + pairs.setdefault(k, []).extend(v) + return pairs + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument('--base-dir', required=True) + ap.add_argument('--new-dir', required=True) + ap.add_argument('--out', default='metrics_compare') + a = ap.parse_args() + repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + + base = collect(a.base_dir, repo_root) + new = collect(a.new_dir, repo_root) + common = sorted(set(base) & set(new)) + dropped = sorted(set(base) ^ set(new)) + + if not common: + with open(a.out + '.md', 'w') as f: + if new and not base: + # interim state: master has not uploaded a per-example baseline yet. + # Blaming the PR's scoping for that sends people hunting the wrong bug + f.write('_No per-example baseline from the base branch yet (the first ' + 'master push after this feature merges uploads it); comparison ' + 'will appear on the next push._\n') + else: + f.write('_Code-size comparison skipped: no (board, example) pair was ' + 'built on both the base branch and this PR._\n') + return + + def synth(pairs, path): + with open(path, 'w') as f: + json.dump({'files': [e for k in common for e in pairs[k]]}, f) + + with tempfile.TemporaryDirectory() as td: + b, n = os.path.join(td, 'base.json'), os.path.join(td, 'new.json') + synth(base, b) + synth(new, n) + comparison = metrics.compare_files(b, n, ['tinyusb/src']) + if comparison is None: + with open(a.out + '.md', 'w') as f: + f.write('_Code-size comparison failed to produce data._\n') + return + metrics.write_compare_markdown(comparison, a.out + '.md', 'name+') + + with open(a.out + '.md', 'a') as f: + boards = sorted({k[0] for k in common}) + f.write(f'\n_Scoped compare: {len(common)} (board, example) pairs across ' + f'{", ".join(boards)}._\n') + if dropped: + # GitHub caps a comment at 65,536 chars and this footer rides inside the + # sticky code-metrics comment: a broad scoped PR drops hundreds of pairs, + # and the raw list alone reached ~65KB and reddened the whole job. Only a + # summary goes in the comment; the full list goes to the job log. + names = [f'{board}:{ex}' for board, ex in dropped] + print('pair_compare: not compared (missing on one side): ' + + ', '.join(names), file=sys.stderr) + more = len(names) - DROPPED_SHOWN + f.write(f'_Not compared (missing on one side): {len(names)} pairs - ' + + ', '.join(names[:DROPPED_SHOWN]) + + (f', ... and {more} more (see the code-metrics job log)' + if more > 0 else '') + + '._\n') + + +if __name__ == '__main__': + main() diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8f6014f48..2ee124cb3 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,7 +37,10 @@ jobs: - 'hw/**' - 'test/hil/**' - 'tools/build.py' + - 'tools/build_utils.py' + - 'tools/ci_select.py' - 'tools/get_deps.py' + - 'tools/metrics.py' - '.github/actions/**' - '.github/workflows/build.yml' - '.github/workflows/build_util.yml' @@ -48,6 +51,9 @@ jobs: outputs: json: ${{ steps.set-matrix-json.outputs.matrix }} hil_json: ${{ steps.set-matrix-json.outputs.hil_matrix }} + example_map: ${{ steps.set-matrix-json.outputs.example_map }} + build_filtered: ${{ steps.set-matrix-json.outputs.build_filtered }} + build_families_regex: ${{ steps.set-matrix-json.outputs.build_families_regex }} # one pair per rig job: hil-tinyusb (tinyusb.json minus esptool boards), # hil-tinyusb-esp (esptool boards only), hil-tinyusb (hfp.json) hil_args_tinyusb: ${{ steps.hil-select.outputs.args_tinyusb }} @@ -62,7 +68,7 @@ jobs: with: fetch-depth: 0 - - name: HIL selection (PR only) + - name: CI selection (PR only) id: hil-select if: github.event_name == 'pull_request' env: @@ -79,55 +85,124 @@ jobs: # advisory workflow that nothing here can `needs:`. Test-failing selector => # full matrix, same as a crashing one. SELECT_JSON='' - if ! python3 test/hil/test/test_hil_select.py; then - echo "::warning::hil_select unit suite failed - falling back to the full HIL matrix" - elif ! SELECT_JSON=$(python3 test/hil/helper/hil_select.py --base "origin/$BASE_REF" test/hil/tinyusb.json test/hil/hfp.json); then - echo "::warning::hil_select failed - falling back to the full HIL matrix" + if ! python3 test/hil/test/test_ci_select.py; then + echo "::warning::ci_select unit suite failed - falling back to the full HIL matrix" + elif ! SELECT_JSON=$(python3 tools/ci_select.py --base "origin/$BASE_REF" test/hil/tinyusb.json test/hil/hfp.json); then + echo "::warning::ci_select failed - falling back to the full HIL matrix" SELECT_JSON='' fi + # The selection is handed on as a FILE in the workspace, never as a step + # output/env var: it is ~KBs normally but a mass-sweep PR reaches hundreds of + # KB, and an env var that big makes the consuming exec fail with E2BIG BEFORE + # any fallback in it can run. Written here, ahead of its first reader. + # No file (non-PR event, or any fallback) = full matrix. + rm -f ci_select_out.json + if [ -n "$SELECT_JSON" ]; then + printf '%s' "$SELECT_JSON" > ci_select_out.json + fi + # One args/run pair per rig job, split by flasher: a job whose own subset is # empty skips explicitly instead of running a board filter that matches zero # boards ("No tests were run." exits 0 and would read as a green HIL run). OUT='' - if [ -n "$SELECT_JSON" ]; then - OUT=$(SELECT_JSON="$SELECT_JSON" python3 -c ' - import json, os - s = json.loads(os.environ["SELECT_JSON"]) + if [ -s ci_select_out.json ]; then + OUT=$(python3 -c ' + import json, re, sys + s = json.load(open("ci_select_out.json")) + # the same reading hil_ci_set_matrix.py applies: full false with no usable + # boards map is an UNUSABLE selection, not "nothing selected". Both must agree + # - one falling open to the whole roster while the other computes run=false + # buys a full 37-leg build and still zero hardware coverage. + if not s.get("full") and not isinstance(s.get("boards"), dict): + sys.exit("selection has full false but no usable boards map") tin = s.get("args_flasher", {}).get("tinyusb.json", {}) legs = (("tinyusb", " ".join(a for f, a in sorted(tin.items()) if f != "esptool" and a)), ("tinyusb_esp", tin.get("esptool", "")), ("hfp", s.get("args", {}).get("hfp.json", ""))) for key, a in legs: + # roster board names reach $GITHUB_OUTPUT as bare NAME=VALUE lines; a + # newline in one would inject extra run_* lines and flip which rig jobs run. + # ":" and "," are part of the normal shape - a partial filter is + # `-bt :,` (ci_select._board_args) + if not re.fullmatch(r"[-A-Za-z0-9_/ .=+:,]*", a): + sys.exit("unexpected characters in the " + key + " board filter") print("args_" + key + "=" + a) print("run_" + key + "=" + ("true" if (s.get("full") or a) else "false")) ') || OUT='' if [ -z "$OUT" ]; then - echo "::warning::hil_select output unusable - falling back to the full HIL matrix" - SELECT_JSON='' + echo "::warning::ci_select output unusable - falling back to the full HIL matrix" + # the same unusable selection must not stay behind for the build axis + rm -f ci_select_out.json fi fi if [ -z "$OUT" ]; then OUT=$(for k in tinyusb tinyusb_esp hfp; do printf 'args_%s=\nrun_%s=true\n' "$k" "$k"; done) fi echo "$OUT" - { echo "select=$SELECT_JSON"; echo "$OUT"; } >> $GITHUB_OUTPUT + echo "$OUT" >> $GITHUB_OUTPUT - name: Generate matrix json id: set-matrix-json - env: - SELECT: ${{ steps.hil-select.outputs.select }} run: | - # build matrix - MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py) + # Build matrix, scoped by the PR selection when one exists. Best-effort: + # ci_set_matrix falls back to the full matrix itself on unusable JSON, + # and a missing file (non-PR event, selector fallback) means no flags. + SELECT_FILE=ci_select_out.json + [ -s "$SELECT_FILE" ] || SELECT_FILE='' + BUILD_SELECT_FILE="$SELECT_FILE" + MATRIX_JSON='' + if [ -n "$SELECT_FILE" ]; then + # ci_set_matrix falls open on a selection it cannot use with rc 0 - it prints + # the full matrix and says UNSCOPED on stderr. The build extras below must + # not stay scoped when it did, or a nominally full build compiles 1 of 44 + # examples per family and code-metrics compares that partial run against a + # full baseline. Only the BUILD axis is dropped: build.families being + # unusable says nothing about the boards map the HIL matrix reads. + MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py --select-file "$SELECT_FILE" 2>ci_set_matrix.err) || MATRIX_JSON='' + cat ci_set_matrix.err >&2 + if [ -z "$MATRIX_JSON" ] || grep -q 'ci_set_matrix: UNSCOPED' ci_set_matrix.err; then + BUILD_SELECT_FILE='' + fi + fi + [ -z "$MATRIX_JSON" ] && MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py) echo "matrix=$MATRIX_JSON" echo "matrix=$MATRIX_JSON" >> $GITHUB_OUTPUT + # Build-axis extras: the per-family example map rides as a side channel + # (a value inside matrix entries would break CircleCI's family parameter + # and multiply GHA matrix legs). These stay step outputs - they are small + # derived values, unlike the selection they are read from. NOTE jq's // + # treats false like null, so .build.full is compared explicitly. + EXAMPLE_MAP='{}' + BUILD_FILTERED='false' + FAM_REGEX='' + if [ -n "$BUILD_SELECT_FILE" ]; then + EXAMPLE_MAP=$(jq -c '.build.family_examples // {}' "$BUILD_SELECT_FILE") || EXAMPLE_MAP='{}' + BUILD_FILTERED=$(jq -r 'if (.build? | type) == "object" and .build.full == false then "true" else "false" end' "$BUILD_SELECT_FILE") || BUILD_FILTERED='false' + if [ "$BUILD_FILTERED" = "true" ]; then + FAM_REGEX=$(jq -r '.build.families | join("|")' "$BUILD_SELECT_FILE") || FAM_REGEX='' + # family names come from hw/bsp dir names, which rule 6 reads straight out + # of the PR's diff path - and this is interpolated raw into a + # `name_is_regexp` artifact pattern, so a regex metacharacter there would + # silently match another family's baseline + case "$FAM_REGEX" in + *[!-A-Za-z0-9_\|]*) + echo "::warning::unexpected characters in the family list - unscoped metrics" + FAM_REGEX='' ;; + esac + [ -z "$FAM_REGEX" ] && BUILD_FILTERED='false' + fi + fi + echo "example_map=$EXAMPLE_MAP" >> $GITHUB_OUTPUT + echo "build_filtered=$BUILD_FILTERED" >> $GITHUB_OUTPUT + echo "build_families_regex=$FAM_REGEX" >> $GITHUB_OUTPUT + # HIL matrix (merged from tinyusb + hifiphile configs), scoped on PRs. # Scoping is best-effort too: fall back to the unscoped (full) matrix. HIL_MATRIX_JSON='' - if [ -n "$SELECT" ]; then - HIL_MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py --select "$SELECT" test/hil/tinyusb.json test/hil/hfp.json) || HIL_MATRIX_JSON='' + if [ -n "$SELECT_FILE" ]; then + HIL_MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py --select-file "$SELECT_FILE" test/hil/tinyusb.json test/hil/hfp.json) || HIL_MATRIX_JSON='' if [ -z "$HIL_MATRIX_JSON" ]; then echo "::warning::scoped HIL matrix failed - falling back to the full HIL matrix" fi @@ -162,6 +237,7 @@ jobs: toolchain: ${{ matrix.toolchain }} build-args: ${{ toJSON(fromJSON(needs.set-matrix.outputs.json)[matrix.toolchain]) }} build-options: '--one-first' + example-map: ${{ needs.set-matrix.outputs.example_map }} upload-metrics: true upload-artifacts: false upload-membrowse: true @@ -169,8 +245,17 @@ jobs: secrets: inherit code-metrics: - needs: [ check-paths, cmake ] - if: needs.check-paths.outputs.code_changed == 'true' + needs: [ check-paths, cmake, set-matrix ] + # A scoped selection can empty every cmake toolchain (a test/hil-only PR). This + # job must still run then: skipping it leaves the sticky comment showing the + # PREVIOUS push's size table as if it were current. set-matrix must have + # SUCCEEDED though: !cancelled() alone let a failed set-matrix through, and this + # job would then overwrite the sticky comment with a wrong "built no families" + # diagnosis while reporting itself green. + if: | + !cancelled() && needs.check-paths.outputs.code_changed == 'true' && + needs.set-matrix.result == 'success' && + (needs.cmake.result == 'success' || needs.cmake.result == 'skipped') runs-on: ubuntu-latest permissions: pull-requests: write @@ -187,8 +272,21 @@ jobs: pattern: metrics-* path: cmake-build merge-multiple: true + # download-artifact does not fail on a pattern that matches nothing, so a + # scoped PR that built no family simply lands here with an empty dir + + - name: Detect empty metrics set + run: | + # No metrics at all => nothing to aggregate or compare. Write the marker the + # sticky comment will carry, so the size section says "skipped" for THIS push + # instead of silently keeping the previous push's table. + if ! ls cmake-build/*/metrics.json >/dev/null 2>&1; then + echo "_Code-size comparison skipped: PR selection built no families on this push._" > metrics_compare.md + echo "NO_METRICS=true" >> $GITHUB_ENV + fi - name: Aggregate Code Metrics + if: env.NO_METRICS != 'true' run: | python tools/get_deps.py python tools/metrics.py combine -j -m -f tinyusb/src cmake-build/*/metrics.json @@ -201,7 +299,7 @@ jobs: path: metrics.json - name: Download Base Branch Metrics - if: github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' + if: env.NO_METRICS != 'true' && (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch') && needs.set-matrix.outputs.build_filtered != 'true' uses: dawidd6/action-download-artifact@v11 with: workflow: build.yml @@ -211,6 +309,29 @@ jobs: path: base-metrics continue-on-error: true + - name: Download base per-family metrics (scoped PR) + if: env.NO_METRICS != 'true' && github.event_name == 'pull_request' && needs.set-matrix.outputs.build_filtered == 'true' + uses: dawidd6/action-download-artifact@v11 + with: + workflow: build.yml + workflow_conclusion: '' + search_artifacts: true # a docs-only master push uploads no per-family artifacts + branch: ${{ github.base_ref }} + name: ^metrics-(${{ needs.set-matrix.outputs.build_families_regex }})$ + name_is_regexp: true + path: base-family-metrics + continue-on-error: true + + - name: Compare with Base Branch (scoped) + if: env.NO_METRICS != 'true' && github.event_name == 'pull_request' && needs.set-matrix.outputs.build_filtered == 'true' + run: | + # never fall back to the averaged metrics-tinyusb here: a scoped PR vs the + # 64-family/46-example average is exactly the mismatch this path prevents + python .github/scripts/metrics_pair_compare.py \ + --base-dir base-family-metrics --new-dir cmake-build --out metrics_compare || \ + echo "_Code-size comparison failed on the scoped path - see the code-metrics job log._" > metrics_compare.md + cat metrics_compare.md + - name: Download Previous Release Asset if: github.event_name == 'release' env: @@ -224,7 +345,7 @@ jobs: gh release download $PREV_TAG -p metrics.json -D base-metrics || echo "No metrics.json found in $PREV_TAG release" - name: Compare with Base Branch - if: github.event_name != 'push' + if: env.NO_METRICS != 'true' && github.event_name != 'push' && needs.set-matrix.outputs.build_filtered != 'true' run: | if [ -f base-metrics/metrics.json ]; then python tools/metrics.py compare -m -f tinyusb/src base-metrics/metrics.json metrics.json @@ -252,6 +373,9 @@ jobs: path: | metrics_compare.md metrics.json + # metrics.json is absent when the selection built no family; the marker + # in metrics_compare.md is still what the sticky comment needs + if-no-files-found: ignore - name: Post Code Metrics as PR Comment if: (github.event_name == 'workflow_dispatch') || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false) @@ -627,32 +751,32 @@ jobs: run: | # Best-effort: this job is deliberately decoupled from set-matrix so unrelated # failures cannot kill hfp coverage - a selector failure here must likewise - # fall back to the full hfp matrix (no hil_select.json, no SEL_* vars), never + # fall back to the full hfp matrix (no ci_select.json, no SEL_* vars), never # fail the job. - if ! python3 test/hil/test/test_hil_select.py; then - echo "::warning::hil_select unit suite failed - running the full hfp matrix" - rm -f hil_select.json + if ! python3 test/hil/test/test_ci_select.py; then + echo "::warning::ci_select unit suite failed - running the full hfp matrix" + rm -f ci_select.json exit 0 fi - if ! python3 test/hil/helper/hil_select.py --base "origin/$BASE_REF" test/hil/hfp.json > hil_select.json; then - echo "::warning::hil_select failed - running the full hfp matrix" - rm -f hil_select.json + if ! python3 tools/ci_select.py --base "origin/$BASE_REF" test/hil/hfp.json > ci_select.json; then + echo "::warning::ci_select failed - running the full hfp matrix" + rm -f ci_select.json exit 0 fi - # hil_select.json is passed to hil_ci_set_matrix.py --select below to scope the + # ci_select.json is passed to hil_ci_set_matrix.py --select below to scope the # build; it already honours full=true by ignoring the board list. # The hil_test.py args go to a file, never to $GITHUB_ENV: they are derived # from roster board names, which a PR can edit. Only SEL_RUN (a literal # true/false computed here, needed by the step-level `if:`) goes to the env. if ! SEL_RUN=$(python3 -c ' import json - s = json.load(open("hil_select.json")) + s = json.load(open("ci_select.json")) a = s["args"]["hfp.json"] open("hil_sel_args.txt", "w").write(a) print("true" if (s["full"] or a) else "false") '); then - echo "::warning::hil_select output unusable - running the full hfp matrix" - rm -f hil_select.json hil_sel_args.txt + echo "::warning::ci_select output unusable - running the full hfp matrix" + rm -f ci_select.json hil_sel_args.txt exit 0 fi echo "SEL_RUN=$SEL_RUN" @@ -661,9 +785,17 @@ jobs: - name: Get build boards if: env.SEL_RUN != 'false' run: | - if [ -f hil_select.json ]; then - MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py --select "$(cat hil_select.json)" test/hil/hfp.json) - else + # --select-file, never --select "$(cat ...)": a whole selection as one argv + # can exceed MAX_ARG_STRLEN on a big diff, and this job's design is to fall + # back to the full hfp matrix on any selector trouble, not to fail the step. + MATRIX_JSON='' + if [ -f ci_select.json ]; then + MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py --select-file ci_select.json test/hil/hfp.json) || MATRIX_JSON='' + if [ -z "$MATRIX_JSON" ]; then + echo "::warning::scoped hfp matrix failed - building the full hfp matrix" + fi + fi + if [ -z "$MATRIX_JSON" ]; then MATRIX_JSON=$(python .github/scripts/hil_ci_set_matrix.py test/hil/hfp.json) fi # Each variant carries its own --build-name/--cflag, which are global to a @@ -672,6 +804,13 @@ jobs: echo "$MATRIX_JSON" | jq -r '.["arm-gcc"][]' > hil_build_entries.txt cat hil_build_entries.txt BUILD_ARGS=$(echo "$MATRIX_JSON" | jq -r '.["arm-gcc"] | join(" ")') + # board and example names are roster data a PR can edit, and jq -r un-escapes + # them: a newline here writes extra NAME=VALUE lines into GITHUB_ENV for every + # later step of a job that holds the IAR token. Refuse rather than guess. + case "$BUILD_ARGS" in + *[!-A-Za-z0-9_/\ .=+]*) + echo "::error::unexpected characters in the hfp build args"; exit 1 ;; + esac echo "BUILD_ARGS=$BUILD_ARGS" echo "BUILD_ARGS=$BUILD_ARGS" >> $GITHUB_ENV diff --git a/.github/workflows/build_util.yml b/.github/workflows/build_util.yml index 02f16488a..dfbd83ee2 100644 --- a/.github/workflows/build_util.yml +++ b/.github/workflows/build_util.yml @@ -20,6 +20,10 @@ on: required: false default: '' type: string + example-map: + required: false + default: '' + type: string upload-artifacts: required: false default: false @@ -76,19 +80,42 @@ jobs: with: arg: ${{ matrix.arg }} + - name: Resolve PR example filter + if: inputs.example-map != '' && inputs.example-map != '{}' + env: + # values are PR-derived - keep them out of ${{ }} script interpolation + # (env expansion word-splits but never re-parses shell metacharacters) + EXAMPLE_MAP: ${{ inputs.example-map }} + FAMILY: ${{ matrix.arg }} + run: | + # -e flags for this family; a family absent from the map builds everything + EX_ARGS=$(printf '%s' "$EXAMPLE_MAP" | jq -r --arg fam "$FAMILY" '(.[$fam] // []) | map("-e " + .) | join(" ")') || EX_ARGS='' + # the map's values are example dir names from the PR checkout, and `jq -r` + # un-escapes them: a path with a newline (git allows it) would otherwise write + # extra NAME=VALUE lines into GITHUB_ENV for every later step of this job. + # Anything outside the example-name alphabet drops the filter (= build all), + # which is the safe direction. + case "$EX_ARGS" in + *[!-A-Za-z0-9_/\ ]*) + echo "::warning::unexpected characters in the example filter - building all examples" + EX_ARGS='' ;; + esac + echo "EX_ARGS=$EX_ARGS" + echo "EX_ARGS=$EX_ARGS" >> $GITHUB_ENV + - name: Build if: ${{ inputs.code-changed }} env: IAR_LMS_BEARER_TOKEN: ${{ secrets.IAR_LMS_BEARER_TOKEN }} run: | if [ "${{ inputs.toolchain }}" == "esp-idf" ]; then - docker run --rm -e MEMBROWSE_API_KEY="$MEMBROWSE_API_KEY" -e CI="$CI" -v $PWD:/project -w /project espressif/idf:tinyusb python tools/build.py --target all ${{ matrix.arg }} + docker run --rm -e MEMBROWSE_API_KEY="$MEMBROWSE_API_KEY" -e CI="$CI" -v $PWD:/project -w /project espressif/idf:tinyusb python tools/build.py --target all ${{ matrix.arg }} $EX_ARGS else BUILD_PY_ARGS="-s ${{ inputs.build-system }} ${{ steps.setup-toolchain.outputs.build_option }} ${{ inputs.build-options }} --target all" if [ "${{ inputs.upload-metrics }}" = "true" ]; then BUILD_PY_ARGS="$BUILD_PY_ARGS --target tinyusb_metrics" fi - python tools/build.py $BUILD_PY_ARGS ${{ matrix.arg }} + python tools/build.py $BUILD_PY_ARGS ${{ matrix.arg }} $EX_ARGS fi shell: bash @@ -99,6 +126,12 @@ jobs: MEMBROWSE_API_KEY: ${{ secrets.MEMBROWSE_API_KEY }} run: | # if code-changed is false --> there is no elf -> membrowse target upload with --identical flag + # Deliberately NOT scoped by $EX_ARGS: -membrowse-upload has no + # DEPENDS (hw/bsp/family_support.cmake), so the aggregate rebuilds nothing - + # it just records every example, reporting the ones with an elf and + # --identical for the rest. Filtering it here would drop the excluded + # examples from the dataset membrowse-comment.yml reports against, instead + # of recording them as unchanged. BUILD_PY_ARGS="-s ${{ inputs.build-system }} ${{ steps.setup-toolchain.outputs.build_option }} ${{ inputs.build-options }}" python tools/build.py $BUILD_PY_ARGS --target examples-membrowse-upload -j 1 ${{ matrix.arg }} shell: bash @@ -108,13 +141,37 @@ jobs: uses: actions/upload-artifact@v7 with: name: metrics-${{ matrix.arg }} - path: cmake-build/cmake-build-*/metrics.json + path: | + cmake-build/cmake-build-*/metrics.json + cmake-build/cmake-build-*/metrics_by_example.json + + - name: Artifact name + if: inputs.upload-artifacts == true + env: + ARG: ${{ matrix.arg }} + run: | + # -e example filters carry '/', which upload-artifact forbids in artifact + # names; strip them from the NAME only (the build already consumed them). + # Names without -e stay byte-identical to before. Two entries differing + # only in their -e list cannot exist - the -e list is a function of + # (board), and variant suffixes (--build-name/-D/--cflag) survive the + # strip - so the stripped name is still unique per matrix entry. + TAG=$(printf '%s' "$ARG" | sed -E 's/ -e [^ ]+//g') + # board and example names come from the roster, which a PR can edit; a newline + # in one would write extra NAME=VALUE lines into GITHUB_ENV for every later + # step. There is no safe fallback name here - a wrong one mislabels the + # firmware the rig then flashes - so refuse instead. + case "$TAG" in + *[!-A-Za-z0-9_/\ .=+]*) + echo "::error::refusing to build an artifact name from '$ARG'"; exit 1 ;; + esac + echo "ARTIFACT_TAG=$TAG" >> $GITHUB_ENV - name: Upload Artifacts for Hardware Testing if: inputs.upload-artifacts == true && inputs.code-changed == true uses: actions/upload-artifact@v7 with: - name: binaries-${{ inputs.toolchain }}-${{ matrix.arg }} + name: binaries-${{ inputs.toolchain }}-${{ env.ARTIFACT_TAG }} path: | cmake-build/cmake-build-*/*/*/*.elf cmake-build/cmake-build-*/*/*/*.bin diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 17170b7d6..7a29dc89a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -48,18 +48,29 @@ repos: types_or: [c, header] language: system - # Two hooks, split by what each suite actually reads. The full discovery run costs - # ~55s (deliberate hang/timeout simulations); only test_hil_select (~0.1s) reads - # hw/bsp (board.cmake), src (portable dirs + class include graph) and examples - # (tusb_config.h per test) -- renaming a board, port dir or example breaks it without - # touching test/hil, and catching that here beats waiting for pre-commit CI. + # Two hooks, split by what each suite RUNS, not by what it reads: discovery is + # disjoint (test_hil*.py vs the two named suites) so nothing runs twice, but the + # file patterns overlap where both suites care. hil-test runs test_hil*.py only + # (~80s: deliberate hang and timeout simulations) and is scoped to the rig harness + # that owns them. The one part of it the selector depends on - the BottomLayer + # stdlib-closure AST guard over tools/ci_select.py and its imports - is named + # explicitly by ci-select-test instead, so a tools/ or workflow edit costs 4s + # rather than 80s of hang simulations that have nothing to say about it. + # ci-select-test runs the two selector-adjacent suites (~4s together) that read + # hw/bsp (board.cmake, FAMILY_MCUS), src (portable dirs + class include graph), + # examples (tusb_config.h, skip/only.txt), hw/mcu, the rig rosters under test/hil + # (a roster edit changes what the selector emits), .circleci (the sentinel contract + # config.yml rewrites config2.yml through) and .github/workflows (build.yml's own + # file hand-off and GITHUB_ENV guards) -- renaming a board, port dir or example + # breaks them without touching test/hil, and catching that here beats waiting for + # pre-commit CI. # No types_or: the rig rosters (*.json) are inputs too. # examples/device/mtp/src is in scope: test_hil_bounded parses README_TXT_CONTENT # and md5-checks the logo header from there as its MTP fixtures. - id: hil-test name: hil-test files: ^(test/hil/|examples/device/mtp/src/) - entry: python3 -m unittest discover -s test/hil/test + entry: python3 -m unittest discover -s test/hil/test -p 'test_hil*.py' pass_filenames: false language: system # hil-validate.js decides which boards ship. Its result join has been wrong three times -- @@ -71,10 +82,10 @@ repos: entry: node .claude/workflows/test-hil-validate.mjs pass_filenames: false language: system - - id: hil-select-test - name: hil-select-test - files: ^(hw/bsp/|src/|examples/) - entry: python3 test/hil/test/test_hil_select.py + - id: ci-select-test + name: ci-select-test + files: ^(hw/bsp/|hw/mcu/|src/|examples/|test/hil/|tools/(ci_select|build|build_utils|get_deps|metrics)\.py$|\.github/(scripts|workflows)/|\.circleci/) + entry: sh -c "python3 test/hil/test/test_ci_select.py && python3 test/hil/test/test_ci_metrics.py && cd test/hil/test && python3 -m unittest -q test_hil_util.BottomLayer" pass_filenames: false language: system diff --git a/docs/reference/hardware-in-the-loop.md b/docs/reference/hardware-in-the-loop.md index 48c362f4f..cf7e3fe69 100644 --- a/docs/reference/hardware-in-the-loop.md +++ b/docs/reference/hardware-in-the-loop.md @@ -281,8 +281,9 @@ Both files are the source of truth — this table is generated from them. `test/hil/hil_test.py`, which flashes each board and runs its tests. Espressif boards run in `hil-tinyusb-esp`, gated on the slower ESP-IDF build, and `hil-hfp-iar` builds with IAR inside the job. -3. On pull requests, `test/hil/helper/hil_select.py` narrows the run to the boards a diff - can affect, falling open to the full matrix when it cannot tell. +3. On pull requests, `tools/ci_select.py` narrows the run to the boards a diff can + affect — and each board's build to the examples its tests need — falling open to the + full matrix when it cannot tell. The same pass scopes the build matrix. 4. Each board is arbitrated by a kernel flock in `/tmp/tinyusb-hil-locks/`, so interactive work and CI can share the rig without colliding. 5. Each rig job uploads its report as an artifact; `pr_comment.yml` downloads them and diff --git a/docs/superpowers/followup/pr3803-flasher-recover.md b/docs/superpowers/followup/pr3803-flasher-recover.md index e9fff7480..1f71c990f 100644 --- a/docs/superpowers/followup/pr3803-flasher-recover.md +++ b/docs/superpowers/followup/pr3803-flasher-recover.md @@ -19,18 +19,18 @@ libjaylink, J-Link probes. - Roster JSON: `test/hil/tinyusb.json`. `flasher_recover` is OPTIONAL; absent means today's behaviour (`recover_flasher` returns the primary). - Never change the shape of `board['flasher']` — it is read as a dict in `hil_flash`, - `hil_test`, `usbtest`, `hil_pool_check`, `hil_select` and the roster lint, and is shipped + `hil_test`, `usbtest`, `hil_pool_check`, `ci_select` and the roster lint, and is shipped as JSON to a subprocess. - Flasher dispatch is by name: `getattr(hil_flash, f'flash_{name}')` / `reset_{name}`. - `RECOVER_FLASH_TIMEOUT = 90`, `RECOVER_RESET_TIMEOUT = 30` (`usbtest.py`). Any board whose flash cannot finish inside 90 s is not a candidate. -- Tests run offline: `cd test/hil && python3 test/test_hil_select.py`. +- Tests run offline: `cd test/hil && python3 test/test_ci_select.py`. ## What is already established **Landed on PR #3803 and inert without roster entries:** `hil_flash.recover_flasher()`, `convoy_safe()` accepting openocd-over-jlink, `hil_test` substituting the recovery flasher -into `--recover-board`, and `test_hil_select.FlasherRecoverEntry` (4 tests). +into `--recover-board`, and `test_ci_select.FlasherRecoverEntry` (4 tests). **Verified in source:** - openocd's jlink driver ignores `adapter usb vid_pid` — `jlink.c` never reads @@ -77,7 +77,7 @@ is a different scope from containing a wedge; and it needs bench time on seven b - `test/hil/hil_flash.py` — add `flash_openocd_seq` / `reset_openocd_seq`; extend `convoy_safe` to accept the new name. This is the only file that learns the command form. - `test/hil/tinyusb.json` — seven `flasher_recover` entries. -- `test/hil/test/test_hil_select.py` — extend `FlasherRecoverEntry`; add a roster lint. +- `test/hil/test/test_ci_select.py` — extend `FlasherRecoverEntry`; add a roster lint. --- @@ -85,7 +85,7 @@ is a different scope from containing a wedge; and it needs bench time on seven b **Files:** - Modify: `test/hil/hil_flash.py` (beside `flash_openocd`, ~line 100) -- Test: `test/hil/test/test_hil_select.py` +- Test: `test/hil/test/test_ci_select.py` **Interfaces:** - Consumes: `_openocd_cmd_base(flasher)`, `hil_util.run_cmd`. @@ -120,7 +120,7 @@ is a different scope from containing a wedge; and it needs bench time on seven b - [ ] **Step 2: Run test to verify it fails** -Run: `cd test/hil && python3 test/test_hil_select.py FlasherRecoverEntry -v` +Run: `cd test/hil && python3 test/test_ci_select.py FlasherRecoverEntry -v` Expected: FAIL — `module 'hil_flash' has no attribute 'flash_openocd_seq'` - [ ] **Step 3: Write minimal implementation** @@ -155,13 +155,13 @@ In `convoy_safe`, replace `if name != 'openocd':` with: - [ ] **Step 4: Run test to verify it passes** -Run: `cd test/hil && python3 test/test_hil_select.py FlasherRecoverEntry -v` +Run: `cd test/hil && python3 test/test_ci_select.py FlasherRecoverEntry -v` Expected: PASS - [ ] **Step 5: Commit** ```bash -git add test/hil/hil_flash.py test/hil/test/test_hil_select.py +git add test/hil/hil_flash.py test/hil/test/test_ci_select.py git commit -m "hil: add openocd_seq flasher for convoy-safe recovery delivery" ``` @@ -171,7 +171,7 @@ git commit -m "hil: add openocd_seq flasher for convoy-safe recovery delivery" **Files:** - Modify: `test/hil/tinyusb.json` -- Test: `test/hil/test/test_hil_select.py` +- Test: `test/hil/test/test_ci_select.py` **Interfaces:** - Consumes: `flash_openocd_seq` / `reset_openocd_seq` from Task 1. @@ -196,7 +196,7 @@ git commit -m "hil: add openocd_seq flasher for convoy-safe recovery delivery" - [ ] **Step 2: Run test to verify it fails** -Run: `cd test/hil && python3 test/test_hil_select.py FlasherRecoverEntry -v` +Run: `cd test/hil && python3 test/test_ci_select.py FlasherRecoverEntry -v` Expected: FAIL — `0 >= 7` - [ ] **Step 3: Add the entries** @@ -224,13 +224,13 @@ Add to each board below, using the SAME `uid` as its primary jlink entry: - [ ] **Step 4: Run test to verify it passes** -Run: `cd test/hil && python3 test/test_hil_select.py -v` +Run: `cd test/hil && python3 test/test_ci_select.py -v` Expected: PASS, and no other selector test regresses. - [ ] **Step 5: Commit** ```bash -git add test/hil/tinyusb.json test/hil/test/test_hil_select.py +git add test/hil/tinyusb.json test/hil/test/test_ci_select.py git commit -m "hil: give seven J-Link boards a convoy-safe recovery flasher" ``` diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 7669290a8..6122b7d54 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -21,7 +21,7 @@ endforeach () find_package(Python3 REQUIRED COMPONENTS Interpreter) add_custom_target(tinyusb_metrics COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/../tools/metrics.py - combine -f tinyusb/src -j -o ${CMAKE_BINARY_DIR}/metrics + combine -f tinyusb/src -j --by-example -o ${CMAKE_BINARY_DIR}/metrics ${MAPJSON_PATTERNS} COMMENT "Generating average code size metrics" VERBATIM diff --git a/hw/bsp/mcx/family.cmake b/hw/bsp/mcx/family.cmake index b2b4fd45b..00c8c4ead 100644 --- a/hw/bsp/mcx/family.cmake +++ b/hw/bsp/mcx/family.cmake @@ -95,7 +95,7 @@ function(family_configure_example TARGET RTOS) endif() # PORT is set per board (board.cmake), so pick the driver at configure time. Spelled out - # rather than $ so the port path stays greppable: test/hil/helper/hil_select.py + # rather than $ so the port path stays greppable: tools/ci_select.py # maps a portable-driver change to the families whose build file names that directory. if (PORT) set(PORT_SRC ${TOP}/src/portable/chipidea/ci_hs/dcd_ci_hs.c) diff --git a/test/hil/helper/hil_select.py b/test/hil/helper/hil_select.py deleted file mode 100755 index f0d4f0b9f..000000000 --- a/test/hil/helper/hil_select.py +++ /dev/null @@ -1,524 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MIT -"""PR-diff -> HIL selection: which rig boards and which tests a change can affect. - -Stdlib-only (runs on bare CI runners; imports hil_util for the example rosters, -never hil_test/pyserial — test_hil_util.BottomLayer enforces the stdlib closure). -Fail-open: any file no rule classifies forces the full matrix. See -docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md. - -JSON: full, boards (name -> 'all' | [tests]), families (bsp families the diff -touches, including ones with no rig board - build-only consumers such as /pre-pr -sample from these), args (hil_test.py args per config) and args_flasher (the same -args split by each board's flasher, for CI legs that split one rig by flasher). -""" -import argparse -import functools -import glob -import json -import os -import re -import subprocess -import sys - -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # helper/ scripts import via the test/hil root -from helper.hil_util import device_tests, dual_tests, host_test - -ALL_TESTS = {'device': device_tests, 'dual': dual_tests, 'host': host_test} - -# class dir -> config macro suffix exceptions (rule 3); dfu is per-file, handled inline -NET_MACROS = ('ECM_RNDIS', 'NCM') - -_NONCODE_RE = re.compile( - r'^(docs/|\.claude/|.*\.(md|rst)$|LICENSE)') -_FULL_RE = re.compile( - r'^(src/common/|src/osal/|src/tusb\.c$|src/tusb\.h$|src/tusb_option\.h$|' - r'test/hil/|\.github/workflows/build.*\.yml$|\.github/actions/|\.github/scripts/|' - r'tools/build\.py$|tools/get_deps\.py$|tools/cmake/|hw/mcu/|lib/|' - r'hw/bsp/(family_support\.cmake|board_api\.h|board\.c|ansi_escape\.h)$|' - r'examples/build_system/|examples/CMakeLists\.txt$|' - # board_test is HIL infrastructure, not a test: hil_test.py flashes it to park - # every board (variant boundary + end-of-board teardown), so every board depends on it - r'examples/device/board_test/)') - -# --no-renames: with rename detection git reports only a rename's destination, so code -# moved out of an HIL-relevant path would be classified by its new path alone -GIT_DIFF_ARGV = ['git', 'diff', '--no-renames', '--name-only'] - - -def test_role(test: str) -> str: - return test.split('/', 1)[0] # 'device' | 'dual' | 'host' - - -def board_roles(board: dict) -> set: - t = board.get('tests', {}) - roles = set() - if t.get('device'): - roles.add('device') - if t.get('host'): - roles.add('host') - if t.get('dual'): - roles.update(('device', 'host')) - for only in t.get('only', []): - r = test_role(only) - roles.update(('device', 'host') if r == 'dual' else (r,)) - return roles - - -def board_tests(board: dict) -> list: - """Every test this board would run today (mirrors hil_test.test_board's default).""" - t = board.get('tests', {}) - if 'only' in t: - run = list(t['only']) - else: - run = [] - if t.get('device'): - run += device_tests - if t.get('dual'): - run += dual_tests - if t.get('host'): - run += host_test - return [x for x in run if x not in t.get('skip', [])] - - -# cached: called per changed file x roster board, and the tree doesn't change mid-run -@functools.lru_cache(maxsize=None) -def board_family(board_name: str, repo_root: str): - hits = glob.glob(os.path.join(repo_root, 'hw/bsp/*/boards', board_name)) - return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None - - -# `if (OPTION STREQUAL "1")` guards in family_support.cmake, and the option tokens -# a roster entry passes to the build (NAME=VALUE / -DNAME=VALUE) -_CM_IF_RE = re.compile(r'if\s*\(') -_CM_ELSE_RE = re.compile(r'else(if)?\s*\(') -_CM_ENDIF_RE = re.compile(r'endif\s*\(') -_CM_OPT_RE = re.compile(r'if\s*\(\s*\$?\{?([A-Za-z_]\w*)\}?\s+STREQUAL\s+"?1"?\s*\)') -_CM_PORT_RE = re.compile(r'src/portable/((?:[^/\s]+/)?[^/\s]+)/') -_FALSY = ('', '0', 'off', 'false', 'no') - - -@functools.lru_cache(maxsize=None) -def port_option_gates(repo_root: str) -> dict: - """port dir -> build options that compile it regardless of the board's family - file, e.g. {'analog/max3421': {'MAX3421_HOST'}} from family_support.cmake.""" - gates = {} - try: - text = open(os.path.join(repo_root, 'hw/bsp/family_support.cmake')).read() - except OSError: - return gates - stack = [] # one entry per open if(): its option, or None - for line in text.splitlines(): - line = line.strip() - if _CM_IF_RE.match(line): - m = _CM_OPT_RE.match(line) - stack.append(m.group(1) if m else None) - elif _CM_ELSE_RE.match(line): - if stack: - stack[-1] = None # the guard doesn't hold in this branch - elif _CM_ENDIF_RE.match(line): - if stack: - stack.pop() - opts = {o for o in stack if o} - m = _CM_PORT_RE.search(line) - if opts and m: - gates.setdefault(m.group(1), set()).update(opts) - return gates - - -_CM_SET_RE = re.compile(r'set\s*\(\s*([A-Za-z_]\w*)\s+([^)\s]+)\s*\)') - - -# cached: called per changed portable file x roster board -@functools.lru_cache(maxsize=None) -def bsp_board_options(board_name: str, repo_root: str) -> frozenset: - """Build options a board turns on in its own BSP: `set( )` in - hw/bsp//boards//board.cmake, e.g. MAX3421_HOST on the espressif - and rp2040 max3421 boards. CMake only - HIL CI builds nothing with Make, so a - board.mk-only option (e.g. nrf5340dk's MAX3421_HOST) compiles no port here.""" - fam = board_family(board_name, repo_root) - if not fam: - return frozenset() - path = os.path.join(repo_root, 'hw/bsp', fam, 'boards', board_name, 'board.cmake') - try: - text = open(path).read() - except OSError: - return frozenset() - out = set() - for line in text.splitlines(): - line = line.strip() - if line.startswith('#'): - continue - m = _CM_SET_RE.match(line) - if m and m.group(2).strip('"').lower() not in _FALSY: - out.add(m.group(1)) - return frozenset(out) - - -def board_options(board: dict, repo_root: str) -> set: - """Build options a board has truthy: the roster entry's build.args plus each - variant's defines (NAME=VALUE) and raw CFLAGS (-DNAME=VALUE), plus whatever its - own board.cmake sets (a board can enable a gated port without the roster saying so).""" - toks = list(board.get('build', {}).get('args', [])) - for v in board.get('variant', []): - toks += list(v.get('defines', [])) - toks += v.get('flags', '').split() - out = set(bsp_board_options(board['name'], repo_root)) - for t in toks: - name, _, val = (t[2:] if t.startswith('-D') else t).partition('=') - if name and val.strip().strip('"').lower() not in _FALSY: - out.add(name.strip()) - return out - - -@functools.lru_cache(maxsize=None) -def port_families(port_dir: str, repo_root: str) -> set: - """Board families that compile this src/portable dir. CMake only: HIL CI builds - every board with CMake, so a port wired up in family.mk alone is compiled for no - HIL board and must not select one. family.cmake lists portable sources directly - for most families; espressif instead references them from a nested component - CMakeLists.txt (hw/bsp/espressif/components/tinyusb_src/CMakeLists.txt).""" - fams = set() - bsp_root = os.path.join(repo_root, 'hw/bsp') - # trailing '/' so a port dir is not a prefix of a sibling: bare 'microchip/pic' - # would otherwise match '.../microchip/pic32mz/...' and inherit its families - needle = port_dir + '/' - for f in glob.glob(os.path.join(bsp_root, '*/family.cmake')) + \ - glob.glob(os.path.join(bsp_root, '*/components/*/CMakeLists.txt')): - try: - if needle in open(f).read(): - fam = os.path.relpath(f, bsp_root).split(os.sep, 1)[0] - fams.add(fam) - except OSError: - pass - return fams - - -_CLS_INC_RE = re.compile(r'#\s*include\s*[<"]class/([^/"<>]+)/([^"<>]+)[">]') - - -@functools.lru_cache(maxsize=None) -def class_include_edges(repo_root: str) -> dict: - """'/
' -> the other class dirs that include it. A class header - pulled in by a second class ships in every firmware enabling that second class: - src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, and - net_device.h includes class/cdc/cdc.h. The class rule derives macros from the - directory name alone, so without this edge a change to the included header - selects only its own class's examples - and on a board that skips those (e.g. - metro_m4_express skips audio_test_freertos), nothing at all. - - Derived from the actual #include lines rather than a hand-written table so it - cannot rot when a class picks up or drops a cross-class include.""" - edges = {} - for f in sorted(glob.glob(os.path.join(repo_root, 'src/class/*/*.[ch]'))): - cls = os.path.basename(os.path.dirname(f)) - try: - text = open(f).read() - except OSError: - continue - for inc_cls, inc_hdr in _CLS_INC_RE.findall(text): - if inc_cls != cls: - edges.setdefault(f'{inc_cls}/{inc_hdr}', set()).add(cls) - return edges - - -def class_macros(cls: str, base: str, prefix: str) -> list: - """Config macros that compile a class dir's code, for role prefix TUD/TUH. - `base` refines dfu only (it splits DFU from DFU_RUNTIME per file); pass '' for - a class reached through an include edge, where the widest set is correct.""" - if cls == 'net': - return [f'CFG_{prefix}_{m}' for m in NET_MACROS] - if cls == 'dfu': - if base.startswith('dfu_rt'): - return [f'CFG_{prefix}_DFU_RUNTIME'] - if base.startswith('dfu_device') or base.startswith('dfu_host'): - return [f'CFG_{prefix}_DFU'] - return [f'CFG_{prefix}_DFU', f'CFG_{prefix}_DFU_RUNTIME'] - return [f'CFG_{prefix}_{cls.upper()}'] - - -def _config_enables(cfg_path: str, macros) -> bool: - try: - text = open(cfg_path).read() - except OSError: - return False - return any(re.search(rf'#define\s+{m}\s+\(?\s*0*[1-9]', text) for m in macros) - - -def roster_only_tests(all_boards) -> set: - """Test paths that only appear in a roster board's tests.only list (e.g. - espressif boards), not in the shared device/dual/host_test lists.""" - out = set() - for b in all_boards: - out.update(b.get('tests', {}).get('only', [])) - return out - - -def class_examples(macros, role: str, repo_root: str, extra_tests: set) -> set: - """Tests (from role's + dual lists, plus roster-only-list tests of that role) - whose example config enables any macro.""" - pool = role_tests({role}, extra_tests) - out = set() - for test in pool: - cfg = os.path.join(repo_root, 'examples', test, 'src', 'tusb_config.h') - if _config_enables(cfg, macros): - out.add(test) - return out - - -def role_tests(roles: set, extras: set) -> set: - """Every test for the given role(s): each role's own list + dual tests, - plus roster-only-list tests (extras) matching those roles or 'dual'.""" - pool = set(dual_tests) - for r in roles: - pool |= set(ALL_TESTS[r]) - pool |= {t for t in extras if test_role(t) in roles or test_role(t) == 'dual'} - return pool - - -class _Sel: - """Accumulates contributions. board->set(tests) plus 'all-board' markers.""" - def __init__(self): - self.full = False - self.by_board = {} # name -> set of tests, or 'all' - self.roles = set() # roles touched by any contribution - self.families = set() # bsp families touched (incl. off-rig ones: build-only consumers) - self.reasons = [] - - def add(self, boards, tests, reason): - """tests: 'all' or iterable of test paths.""" - self.reasons.append(reason) - for b in boards: - cur = self.by_board.get(b) - if tests == 'all' or cur == 'all': - self.by_board[b] = 'all' - else: - self.by_board[b] = (cur or set()) | set(tests) - - def force_full(self, reason): - self.full = True - self.reasons.append(reason) - - -def _classify_one(path, repo_root, roster_boards, extras: set, s: _Sel): - base = os.path.basename(path) - if _NONCODE_RE.match(path): - s.reasons.append(f'{path}: non-code, no contribution') - return - if _FULL_RE.match(path): - s.force_full(f'{path}: core/infra -> full matrix') - return - - m = re.match(r'src/portable/((?:[^/]+/)?[^/]+)/', path) - if m: - port = m.group(1) - if re.match(r'(dcd_|.*_device)', base): - roles = {'device'} - elif re.match(r'(hcd_|.*_host)', base): - roles = {'host'} - else: - roles = {'device', 'host'} - fams = port_families(port, repo_root) - if not fams: - # no family references this port: either a new/renamed port dir or a - # family.cmake layout the scan misses - widen instead of contributing nothing - s.force_full(f'{path}: port {port} maps to no board family -> full matrix') - return - s.families.update(fams) - # a board can also pull the port in through a build option (e.g. MAX3421_HOST=1 - # from the roster on metro_m4_express, or from its own board.cmake), which its - # family file never names - gates = port_option_gates(repo_root).get(port, set()) - boards = [b['name'] for b in roster_boards - if (board_family(b['name'], repo_root) in fams or - (gates and board_options(b, repo_root) & gates)) and (board_roles(b) & roles)] - tests = role_tests(roles, extras) - s.roles.update(roles) - why = f'{path}: port {port} -> families {sorted(fams)}' - if gates: - why += f' + option {sorted(gates)}' - s.add(boards, tests, f'{why} -> boards {boards} ({"/".join(sorted(roles))})') - return - - m = re.match(r'src/class/([^/]+)/', path) - if m: - cls = m.group(1) - if re.search(r'_device\.[ch]$', base): - roles = {'device'} - elif re.search(r'_host\.[ch]$', base): - roles = {'host'} - else: - roles = {'device', 'host'} - # this file's own class, plus any class whose headers include it - via = sorted(class_include_edges(repo_root).get(f'{cls}/{base}', ())) - - def macros(prefix): - return (class_macros(cls, base, prefix) + - [m2 for c in via for m2 in class_macros(c, '', prefix)]) - tests = set() - if 'device' in roles: - tests |= class_examples(macros('TUD'), 'device', repo_root, extras) - if 'host' in roles: - tests |= class_examples(macros('TUH'), 'host', repo_root, extras) - boards = [b['name'] for b in roster_boards if board_roles(b) & roles] - s.roles.update(roles) - why = f'{path}: class {cls}' + (f' (+ included by {via})' if via else '') - s.add(boards, tests, f'{why} -> {sorted(tests)} ({"/".join(sorted(roles))})') - return - - m = re.match(r'src/(device|host)/', path) - if m: - role = m.group(1) - boards = [b['name'] for b in roster_boards if role in board_roles(b)] - s.roles.add(role) - s.add(boards, role_tests({role}, extras), f'{path}: core {role} stack -> all {role} tests') - return - - m = re.match(r'hw/bsp/([^/]+)/(?:boards/([^/]+)/)?', path) - if m: - fam, brd = m.group(1), m.group(2) - s.families.add(fam) - if brd: - boards = [b['name'] for b in roster_boards if b['name'] == brd] - why = f'{path}: bsp board {brd}' - else: - boards = [b['name'] for b in roster_boards - if board_family(b['name'], repo_root) == fam] - why = f'{path}: bsp family {fam}' - s.roles.update(('device', 'host')) - s.add(boards, 'all', f'{why} -> boards {boards}') - return - - m = re.match(r'examples/(device|host|dual)/([^/]+)/', path) - if m: - test = f'{m.group(1)}/{m.group(2)}' - known = any(test in pool for pool in ALL_TESTS.values()) or test in extras - if known: - boards = [b['name'] for b in roster_boards] - role = test_role(test) - s.roles.update(('device', 'host') if role == 'dual' else (role,)) - s.add(boards, [test], f'{path}: example -> {test} on all boards') - else: - s.reasons.append(f'{path}: example not in HIL lists, no contribution') - return - - s.force_full(f'{path}: unclassified -> full matrix') - - -def classify(changed_files, repo_root, rosters): - all_boards = [] - seen = set() - for _, boards in rosters: - for b in boards: - if b['name'] not in seen: - seen.add(b['name']) - all_boards.append(b) - - extras = roster_only_tests(all_boards) - s = _Sel() - # no early exit once full: keep classifying so `families` still reports every - # family the diff touches (build-only consumers need it). Nothing after the first - # force_full can change full/boards/args - the full branch below ignores by_board. - for path in changed_files: - _classify_one(path, repo_root, all_boards, extras, s) - - if s.full: - return {'full': True, 'boards': {b['name']: 'all' for b in all_boards}, - 'families': sorted(s.families), 'reasons': s.reasons} - - # role pruning: single-role selections drop the other role's tests and boards - by_name = {b['name']: b for b in all_boards} - out = {} - for name, tests in s.by_board.items(): - allowed = board_tests(by_name[name]) - if tests == 'all': - kept = list(allowed) - else: - kept = [t for t in allowed if t in tests] - if s.roles and s.roles != {'device', 'host'}: - role = next(iter(s.roles)) - kept = [t for t in kept if test_role(t) in (role, 'dual')] - if kept: - out[name] = 'all' if set(kept) == set(allowed) else sorted(kept) - return {'full': False, 'boards': out, 'families': sorted(s.families), - 'reasons': s.reasons} - - -def _board_args(name, chosen) -> list: - parts = [f'-b {name}'] - if chosen != 'all': - parts.append(f'-bt {name}:{",".join(chosen)}') - return parts - - -def selection_args(sel, rosters): - """hil_test.py args per config. Empty means either 'full matrix' or 'nothing - selected' - callers must read sel['full'] to tell them apart.""" - args = {} - for cfg_path, boards in rosters: - parts = [] - if not sel['full']: - for b in boards: - chosen = sel['boards'].get(b['name']) - if chosen is not None: - parts += _board_args(b['name'], chosen) - args[os.path.basename(cfg_path)] = ' '.join(parts) - return args - - -def selection_args_by_flasher(sel, rosters): - """{config: {flasher name: args}}. CI runs one rig as several jobs split by - flasher (esptool vs the rest); each must gate on its own subset, otherwise the - other leg runs a filter matching zero boards and reports a vacuous green.""" - out = {} - for cfg_path, boards in rosters: - per = {} - if not sel['full']: - for b in boards: - chosen = sel['boards'].get(b['name']) - if chosen is None: - continue - per.setdefault(b.get('flasher', {}).get('name', ''), []).extend( - _board_args(b['name'], chosen)) - out[os.path.basename(cfg_path)] = {f: ' '.join(p) for f, p in per.items()} - return out - - -def changed_files_from_git(base, repo_root): - mb = subprocess.run(['git', 'merge-base', 'HEAD', base], cwd=repo_root, - capture_output=True, text=True, check=True).stdout.strip() - diff = subprocess.run(GIT_DIFF_ARGV + [f'{mb}..HEAD'], cwd=repo_root, - capture_output=True, text=True, check=True).stdout - return [l for l in diff.splitlines() if l.strip()] - - -def main(): - ap = argparse.ArgumentParser(description=__doc__) - g = ap.add_mutually_exclusive_group(required=True) - g.add_argument('--base', help='git ref to diff against (merge-base..HEAD)') - g.add_argument('--diff-file', help='newline-separated changed-file list') - ap.add_argument('configs', nargs='+', help='rig roster JSON file(s)') - a = ap.parse_args() - - # test/hil/helper/ -> repo root is FOUR levels up; three left this at /test - # after the helper/ move and every repo-relative glob silently matched nothing - repo_root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) - rosters = [] - for c in a.configs: - with open(c) as f: - rosters.append((c, json.load(f)['boards'])) - - files = (open(a.diff_file).read().splitlines() if a.diff_file - else changed_files_from_git(a.base, repo_root)) - files = [f for f in files if f.strip()] - - s = classify(files, repo_root, rosters) - s['args'] = selection_args(s, rosters) - s['args_flasher'] = selection_args_by_flasher(s, rosters) - for r in s['reasons']: - print(f'hil_select: {r}', file=sys.stderr) - print(json.dumps(s)) - - -if __name__ == '__main__': - main() diff --git a/test/hil/helper/hil_util.py b/test/hil/helper/hil_util.py index 54984d20f..0a2a13fca 100644 --- a/test/hil/helper/hil_util.py +++ b/test/hil/helper/hil_util.py @@ -18,7 +18,7 @@ from typing import Any # ------------------------------------------------------------- -# HIL example test lists, shared by hil_test.py (runner) and hil_select.py (PR-diff +# HIL example test lists, shared by hil_test.py (runner) and ci_select.py (PR-diff # selector). Run order is shuffled per board (see test_board); every example carries a # unique hardcoded idProduct (see its usb_descriptors.c). # ------------------------------------------------------------- diff --git a/test/hil/hil_ci.sh b/test/hil/hil_ci.sh index 66f4e48d4..514b0f174 100644 --- a/test/hil/hil_ci.sh +++ b/test/hil/hil_ci.sh @@ -224,7 +224,6 @@ scp -q "$ROOT_DIR/test/hil/helper/__init__.py" \ "$ROOT_DIR/test/hil/helper/hil_health.py" \ "$ROOT_DIR/test/hil/helper/hil_lock.py" \ "$ROOT_DIR/test/hil/helper/hil_summary.py" \ - "$ROOT_DIR/test/hil/helper/hil_select.py" \ "$REMOTE:$REMOTE_DIR/test/hil/helper/" # Copy only firmware binaries (elf/bin/hex) plus esptool metadata diff --git a/test/hil/hil_flash.py b/test/hil/hil_flash.py index f4bed45a6..c4d4e6552 100755 --- a/test/hil/hil_flash.py +++ b/test/hil/hil_flash.py @@ -294,7 +294,7 @@ reset_lm4flash.no_op = True # The one place a flasher's firmware extension is decided. A flasher with no entry falls -# back to .elf-or-.bin and can be handed the wrong file — test_hil_select's +# back to .elf-or-.bin and can be handed the wrong file — test_ci_select's # TestRosterFlashersDispatch fails if a roster names one. FLASHER_SUFFIX = { 'esptool': '.bin', diff --git a/test/hil/test/test_ci_metrics.py b/test/hil/test/test_ci_metrics.py new file mode 100644 index 000000000..89d03aaae --- /dev/null +++ b/test/hil/test/test_ci_metrics.py @@ -0,0 +1,441 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests for the by-example half of tools/metrics.py and the (family, example) +# pair-compare script. Stdlib only; synthetic map.json fixtures, no builds. +# python3 test/hil/test/test_ci_metrics.py +import json +import os +import subprocess +import sys +import tempfile +import unittest + +REPO = os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.abspath(__file__))))) +METRICS = os.path.join(REPO, 'tools', 'metrics.py') + + +def fake_map(path, files): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, 'w') as f: + json.dump({'files': files}, f) + + +def entry(name, size, path_prefix='tinyusb/src'): + return {'file': name, 'path': f'{path_prefix}/{name}', 'size': size, + 'symbols': [{'name': f'{name}_fn', 'size': size}], 'sections': {'.text': size}} + + +class TestByExample(unittest.TestCase): + def build_tree(self, td): + fake_map(os.path.join(td, 'device', 'cdc_msc', 'cdc_msc.map.json'), + [entry('usbd.c', 100), entry('cdc_device.c', 50)]) + fake_map(os.path.join(td, 'host', 'bare_api', 'bare_api.map.json'), + [entry('usbh.c', 200)]) + + def test_by_example_output(self): + with tempfile.TemporaryDirectory() as td: + self.build_tree(td) + out = os.path.join(td, 'metrics') + r = subprocess.run([sys.executable, METRICS, 'combine', '-q', '-j', + '--by-example', '-o', out, + os.path.join(td, '*', '*', '*.map.json')], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + by_ex = json.load(open(out + '_by_example.json')) + self.assertEqual(set(by_ex), {'device/cdc_msc', 'host/bare_api'}) + self.assertEqual({f['file'] for f in by_ex['device/cdc_msc']['files']}, + {'usbd.c', 'cdc_device.c'}) + # the plain averaged output is unchanged by the extra flag + avg = json.load(open(out + '.json')) + self.assertIn('files', avg) + + def test_by_example_json_roundtrips_as_combine_input(self): + with tempfile.TemporaryDirectory() as td: + self.build_tree(td) + out = os.path.join(td, 'metrics') + subprocess.run([sys.executable, METRICS, 'combine', '-q', '-j', '--by-example', + '-o', out, os.path.join(td, '*', '*', '*.map.json')], check=True) + out2 = os.path.join(td, 'sub') + r = subprocess.run([sys.executable, METRICS, 'combine', '-q', '-j', + '--only-examples', 'device/cdc_msc', + '-o', out2, out + '_by_example.json'], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + sub = json.load(open(out2 + '.json')) + names = {f['file'] for f in sub['files']} + self.assertEqual(names, {'usbd.c', 'cdc_device.c'}) # bare_api filtered out + + def test_by_example_expansion_is_keyed_on_the_filename(self): + # the '_by_example.json' suffix IS the contract (write_by_example, the CMake + # rule and metrics_pair_compare all spell it). A shape-sniff would reroute + # any coincidentally-shaped JSON into the per-example branch instead. + with tempfile.TemporaryDirectory() as td: + look_alike = os.path.join(td, 'metrics.json') + with open(look_alike, 'w') as f: + json.dump({'device/cdc_msc': {'files': [entry('usbd.c', 100)]}}, f) + out = os.path.join(td, 'combined') + r = subprocess.run([sys.executable, METRICS, 'combine', '-q', '-j', + '-o', out, look_alike], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + combined = json.load(open(out + '.json')) + self.assertNotIn('usbd.c', {f['file'] for f in combined.get('files', [])}) + + +PAIR_COMPARE = os.path.join(REPO, '.github/scripts/metrics_pair_compare.py') + + +def fake_by_example(root, board, data): + d = os.path.join(root, f'cmake-build-{board}') + os.makedirs(d, exist_ok=True) + with open(os.path.join(d, 'metrics_by_example.json'), 'w') as f: + json.dump(data, f) + + +class TestPairCompare(unittest.TestCase): + def test_intersection_compare(self): + with tempfile.TemporaryDirectory() as td: + base, new = os.path.join(td, 'base'), os.path.join(td, 'new') + # real board names so board->family resolution works against hw/bsp + fake_by_example(base, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('usbd.c', 100)]}, + 'device/dfu': {'files': [entry('dfu_device.c', 10)]}}) + fake_by_example(new, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('usbd.c', 120)]}}) + out = os.path.join(td, 'cmp') + r = subprocess.run([sys.executable, PAIR_COMPARE, '--base-dir', base, + '--new-dir', new, '--out', out], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + md = open(out + '.md').read() + self.assertIn('usbd.c', md) + self.assertNotIn('dfu_device.c', md) # not on both sides + self.assertIn('raspberry_pi_pico', md) # scope footer names the board + self.assertIn('device/dfu', md) # named as dropped + + def test_a_different_board_of_the_same_family_is_not_compared(self): + """--one-first returns all_boards[0], so adding a board can shift which one a + family builds. Keyed on the family, the base run's sizes and the PR run's sizes + would land under one key and the difference between two unrelated MCUs would be + published as this PR's code-size impact.""" + with tempfile.TemporaryDirectory() as td: + base, new = os.path.join(td, 'base'), os.path.join(td, 'new') + # both rp2040, both device/cdc_msc - only the board differs + fake_by_example(base, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('usbd.c', 100)]}}) + fake_by_example(new, 'adafruit_fruit_jam', + {'device/cdc_msc': {'files': [entry('usbd.c', 900)]}}) + out = os.path.join(td, 'cmp') + r = subprocess.run([sys.executable, PAIR_COMPARE, '--base-dir', base, + '--new-dir', new, '--out', out], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + md = open(out + '.md').read() + self.assertIn('skipped', md) + self.assertNotIn('+800', md) + + def test_empty_intersection_writes_note(self): + with tempfile.TemporaryDirectory() as td: + base, new = os.path.join(td, 'base'), os.path.join(td, 'new') + fake_by_example(base, 'raspberry_pi_pico', {'device/dfu': {'files': [entry('a.c', 1)]}}) + fake_by_example(new, 'stm32f407disco', {'device/cdc_msc': {'files': [entry('b.c', 1)]}}) + out = os.path.join(td, 'cmp') + r = subprocess.run([sys.executable, PAIR_COMPARE, '--base-dir', base, + '--new-dir', new, '--out', out], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + self.assertIn('skipped', open(out + '.md').read()) + + def test_malformed_files_are_skipped_with_stderr_note(self): + with tempfile.TemporaryDirectory() as td: + base, new = os.path.join(td, 'base'), os.path.join(td, 'new') + # good pair on both sides -- must survive the malformed siblings below + fake_by_example(base, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('usbd.c', 100)]}}) + fake_by_example(new, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('usbd.c', 120)]}}) + # well-formed JSON, wrong shape (a list, not a {example: {files: [...]}} dict) + wrong_shape = os.path.join(base, 'cmake-build-stm32f407disco', 'metrics_by_example.json') + os.makedirs(os.path.dirname(wrong_shape), exist_ok=True) + with open(wrong_shape, 'w') as f: + json.dump(['not', 'a', 'dict'], f) + # metrics_by_example.json not under a cmake-build- dir + misplaced = os.path.join(base, 'not_a_board_dir', 'metrics_by_example.json') + os.makedirs(os.path.dirname(misplaced), exist_ok=True) + with open(misplaced, 'w') as f: + json.dump({'device/dfu': {'files': [entry('dfu_device.c', 10)]}}, f) + + out = os.path.join(td, 'cmp') + r = subprocess.run([sys.executable, PAIR_COMPARE, '--base-dir', base, + '--new-dir', new, '--out', out], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) # fail-open: never crash the job + md = open(out + '.md').read() + self.assertIn('usbd.c', md) # good pair still compared + self.assertIn(wrong_shape, r.stderr) + self.assertIn(misplaced, r.stderr) + self.assertIn('skipping', r.stderr) + + + def test_missing_base_baseline_gets_its_own_note(self): + # interim state right after this feature merges: master has not uploaded a + # per-example baseline yet, so the BASE side collects nothing. The generic + # "no pair on both sides" note misattributes that to the PR's own scoping. + with tempfile.TemporaryDirectory() as td: + base, new = os.path.join(td, 'base'), os.path.join(td, 'new') + os.makedirs(base) + fake_by_example(new, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('usbd.c', 100)]}}) + out = os.path.join(td, 'cmp') + r = subprocess.run([sys.executable, PAIR_COMPARE, '--base-dir', base, + '--new-dir', new, '--out', out], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + md = open(out + '.md').read() + self.assertIn('No per-example baseline from the base branch yet', md) + self.assertIn('next push', md) + self.assertNotIn('comparison skipped', md) + + def test_a_partially_malformed_file_contributes_nothing(self): + """A file that blows up half way through must drop WHOLE. Entries parsed + before the malformation used to stay in the comparison while stderr claimed + the file had been skipped - a silently truncated table published as the + code-size verdict. A non-list 'files' (TypeError) also has to be caught.""" + with tempfile.TemporaryDirectory() as td: + base, new = os.path.join(td, 'base'), os.path.join(td, 'new') + # good entry FIRST, malformed second: the leak is order-dependent + fake_by_example(base, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('leaked.c', 100)]}, + 'device/dfu': {'files': 42}}) + fake_by_example(new, 'raspberry_pi_pico', + {'device/cdc_msc': {'files': [entry('leaked.c', 120)]}}) + # a sibling file that is fine on both sides must still be compared + fake_by_example(base, 'stm32f407disco', + {'device/cdc_msc': {'files': [entry('good.c', 10)]}}) + fake_by_example(new, 'stm32f407disco', + {'device/cdc_msc': {'files': [entry('good.c', 12)]}}) + out = os.path.join(td, 'cmp') + r = subprocess.run([sys.executable, PAIR_COMPARE, '--base-dir', base, + '--new-dir', new, '--out', out], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + md = open(out + '.md').read() + self.assertIn('good.c', md) + self.assertNotIn('leaked.c', md) + self.assertIn('skipping', r.stderr) + self.assertIn(os.path.join(base, 'cmake-build-raspberry_pi_pico'), r.stderr) + + def test_dropped_footer_is_summarised_not_dumped(self): + """The sticky PR comment is capped at 65,536 chars by GitHub; a broad scoped + PR drops hundreds of (family, example) pairs and the full list alone ran to + tens of KB, pushing the comment past the cap and reddening code-metrics.""" + with tempfile.TemporaryDirectory() as td: + base, new = os.path.join(td, 'base'), os.path.join(td, 'new') + common = {'device/cdc_msc': {'files': [entry('usbd.c', 100)]}} + extra = {f'device/example_{i:03d}': {'files': [entry(f'f{i}.c', i + 1)]} + for i in range(30)} + fake_by_example(base, 'raspberry_pi_pico', dict(common, **extra)) + fake_by_example(new, 'raspberry_pi_pico', common) + out = os.path.join(td, 'cmp') + r = subprocess.run([sys.executable, PAIR_COMPARE, '--base-dir', base, + '--new-dir', new, '--out', out], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + md = open(out + '.md').read() + footer = md[md.index('_Scoped compare:'):] + self.assertLess(len(footer), 2048, footer) + self.assertIn('30', footer) # the count is still reported + self.assertIn('more', footer) # truncation marker + self.assertIn('device/example_029', r.stderr) # full list on stderr + + +CIRCLECI = os.path.join(REPO, '.circleci') +SENTINELS = ('example-map-default', 'build-filtered-default') + + +class TestCircleCiSentinelContract(unittest.TestCase): + """config.yml's set-matrix rewrites config2.yml's parameter defaults by matching + a sentinel comment line — the only way past /pipeline/continue's 512-char + parameter cap. Renaming or reformatting either side is a silent full-build + fallback that no CI job reports, so pin the contract here.""" + + def setUp(self): + self.config = open(os.path.join(CIRCLECI, 'config.yml')).read() + self.config2 = open(os.path.join(CIRCLECI, 'config2.yml')).read() + + def test_each_sentinel_appears_once_on_a_default_line(self): + for tag in SENTINELS: + marker = f'# {tag}: rewritten in-place by config.yml set-matrix' + hits = [l for l in self.config2.splitlines() if l.strip().endswith(marker)] + self.assertEqual(len(hits), 1, f'{tag}: {len(hits)} sentinel lines in config2.yml') + self.assertIn('default:', hits[0], f'{tag}: sentinel is not on a default: line') + + def test_the_selection_travels_as_a_file(self): + # a mass-sweep selection runs to hundreds of KB: handed to ci_set_matrix as one + # argv it E2BIGs the step before the `||` fallback can fire, and EXAMPLE_MAP / + # BUILD_FILTERED (derived with jq, no argv limit) would then label a FULL build + # scoped -- the build and its label disagreeing is worse than either alone + self.assertIn('--select-file', self.config) + self.assertNotIn('--select "', self.config) + + def test_the_rewriter_names_the_same_sentinels(self): + for tag in SENTINELS: + self.assertIn(f"'{tag}'", self.config, + f'{tag}: config.yml rewrite block does not name this sentinel') + self.assertIn("# {tag}: rewritten in-place by config.yml set-matrix", self.config, + 'config.yml no longer builds the sentinel comment it matches on') + + def test_the_rewrite_precedes_the_scoped_entries(self): + # the scoping is all-or-nothing: config2's checked-in defaults are {} / false = + # unfiltered, so a rewrite that fails AFTER the family entries were generated + # leaves a subset of families built and code-metrics told it was a full build. + # Rewrite first, and on failure drop the scoping (back to the full matrix). + rewrite = self.config.index("p = '.circleci/config2.yml'") + entries = self.config.index('gen_build_entry() {') + self.assertLess(rewrite, entries, + 'the sentinel rewrite must run before any build entry is generated') + tail = self.config[rewrite:entries] + self.assertIn('MATRIX_JSON="$FULL_MATRIX_JSON"', tail, + 'a failed rewrite must fall back to the FULL matrix, not keep the ' + 'scoped one') + # and that fallback must be a plain assignment: a second `python ...` here is an + # unguarded command under CircleCI's `set -e`, inside the one branch whose whole + # job is to keep the pipeline green + self.assertNotIn('ci_set_matrix.py)', tail) + + def test_the_selector_gate_runs_both_suites(self): + # test_ci_select.py owns the rules; this file owns the sentinel contract the + # very same job rewrites. Gating on one of the two leaves the other unguarded. + for suite in ('test_ci_select.py', 'test_ci_metrics.py'): + self.assertIn(suite, self.config, f'{suite} does not gate the CircleCI selector') + + +class TestWorkflowSelectionHandOff(unittest.TestCase): + """build.yml's counterpart of the CircleCI contract above: same E2BIG limit, same + consequence (the scoping silently turns itself off on exactly the PRs where it + saves most), plus the GITHUB_ENV lines that carry PR-derived values.""" + + def setUp(self): + wf = os.path.join(os.path.dirname(CIRCLECI), '.github', 'workflows') + self.build = open(os.path.join(wf, 'build.yml')).read() + self.util = open(os.path.join(wf, 'build_util.yml')).read() + + def test_no_step_execs_with_the_selection_in_its_environment(self): + # SELECT_JSON="$SELECT_JSON" python3 -c ... E2BIGs at ~128KiB: measured 261KB + # for a `git ls-files hw/bsp/**` sweep. Every reader takes the file instead. + self.assertNotIn('SELECT_JSON="$SELECT_JSON"', self.build) + self.assertIn('json.load(open("ci_select_out.json"))', self.build) + + def test_the_file_is_written_before_its_first_reader(self): + self.assertLess(self.build.index("printf '%s' \"$SELECT_JSON\" > ci_select_out.json"), + self.build.index('json.load(open("ci_select_out.json"))'), + 'the selection file must exist before the step that reads it') + + def test_pr_derived_env_values_are_character_guarded(self): + # values reach GITHUB_ENV/GITHUB_OUTPUT as bare NAME=VALUE lines; a newline in + # one (git allows it in a path, and both the example map and the roster are + # PR-editable) writes extra variables into every later step of a job that runs + # with secrets - and for run_*, flips which rig jobs execute + for name in ('EX_ARGS', 'ARTIFACT_TAG'): + self.assertIn(f'echo "{name}=', self.util) + self.assertEqual(self.util.count('case "$EX_ARGS" in') + + self.util.count('case "$TAG" in'), 2, + 'both GITHUB_ENV writes must screen their value first') + self.assertIn('case "$BUILD_ARGS" in', self.build) + self.assertIn('unexpected characters in the " + key', self.build, + 'the args_*/run_* emitter must screen each board filter') + + def test_the_guards_accept_what_the_selector_actually_emits(self): + """A guard that rejects a NORMAL value is worse than no guard: build.yml throws + the whole selection away, warns, and both axes fall back to full - silently + turning the feature off. So run the real character classes over real selections + rather than only asserting that the guard text is present. + + The one that got away: `[-A-Za-z0-9_/ .=+]` has no ':' or ',', and every partial + board filter is `-bt :,`.""" + import re, subprocess, sys, tempfile, json + repo = os.path.dirname(CIRCLECI) + # the character classes, lifted from the three places they are written + classes = {} + m = re.search(r're\.fullmatch\(r"\[([^"]+)\]\*"', self.build) + self.assertTrue(m, 'args_*/run_* guard not found in build.yml') + classes['args'] = m.group(1) + for name, text in (('BUILD_ARGS', self.build), ('EX_ARGS', self.util), + ('TAG', self.util)): + m = re.search(r'case "\$%s" in\s*\n\s*\*\[!([^\]]+)\]\*\)' % name, text) + self.assertTrue(m, f'{name} guard not found') + classes[name] = m.group(1).replace('\\', '') + + def ok(cls, value): + return re.fullmatch('[%s]*' % cls.replace('!', ''), value) is not None + + with tempfile.TemporaryDirectory() as d: + for path in ('src/class/cdc/cdc_device.c', 'src/device/usbd.c', + 'src/portable/synopsys/dwc2/dcd_dwc2.c', + 'examples/device/cdc_msc/src/main.c', + 'hw/bsp/stm32f4/family.cmake'): + f = os.path.join(d, 'diff.txt') + with open(f, 'w') as fh: + fh.write(path + '\n') + r = subprocess.run([sys.executable, os.path.join(repo, 'tools/ci_select.py'), + '--diff-file', f, + os.path.join(repo, 'test/hil/tinyusb.json'), + os.path.join(repo, 'test/hil/hfp.json')], + capture_output=True, text=True, cwd=repo) + self.assertEqual(r.returncode, 0, r.stderr) + s = json.loads(r.stdout) + for flasher, a in s.get('args_flasher', {}).get('tinyusb.json', {}).items(): + self.assertTrue(ok(classes['args'], a), + f'{path}/{flasher}: the args guard rejects {a!r}') + hfp = s.get('args', {}).get('hfp.json', '') + self.assertTrue(ok(classes['args'], hfp), f'{path}: hfp {hfp!r}') + # BUILD_ARGS is the hfp job's `-b [-e ...]` list, not the -bt + # test filter above - screen the value that step actually builds + with open(os.path.join(d, 'sel.json'), 'w') as fh: + fh.write(r.stdout) + hm = subprocess.run( + [sys.executable, os.path.join(repo, '.github/scripts/hil_ci_set_matrix.py'), + '--select-file', os.path.join(d, 'sel.json'), + os.path.join(repo, 'test/hil/hfp.json')], + capture_output=True, text=True, cwd=repo) + self.assertEqual(hm.returncode, 0, hm.stderr) + build_args = ' '.join(json.loads(hm.stdout)['arm-gcc']) + self.assertTrue(ok(classes['BUILD_ARGS'], build_args), + f'{path}: the BUILD_ARGS guard rejects {build_args!r}') + for entry in json.loads(hm.stdout)['arm-gcc']: + tag = re.sub(r' -e [^ ]+', '', entry) + self.assertTrue(ok(classes['TAG'], tag), + f'{path}: the artifact-name guard rejects {tag!r}') + for fam, exs in (s.get('build', {}).get('family_examples') or {}).items(): + ex_args = ' '.join('-e ' + e for e in exs) + self.assertTrue(ok(classes['EX_ARGS'], ex_args), + f'{path}/{fam}: the EX_ARGS guard rejects {ex_args!r}') + + def test_an_unusable_selection_is_unusable_for_both_matrices(self): + # hil_ci_set_matrix reads "full false with no boards map" as unusable and falls + # open to the whole roster; if this emitter instead computed run_*=false, the + # rig jobs would skip while all 37 build legs ran - a full build and still zero + # hardware coverage, which is the outcome the guard exists to prevent + self.assertIn('isinstance(s.get("boards"), dict)', self.build) + + def test_the_build_extras_drop_when_the_matrix_falls_open(self): + # ci_set_matrix falls open with rc 0, so the example map and family regex must + # follow it or a nominally full build is filtered and labelled as a scoped one + self.assertIn("grep -q 'ci_set_matrix: UNSCOPED'", self.build) + self.assertIn('BUILD_SELECT_FILE', self.build) + scripts = os.path.join(os.path.dirname(CIRCLECI), '.github', 'scripts') + matrix = open(os.path.join(scripts, 'ci_set_matrix.py')).read() + self.assertEqual(matrix.count('ci_set_matrix: UNSCOPED'), 2, + 'every fall-open path must print the marker build.yml greps for') + + def test_membrowse_upload_is_not_scoped(self): + # -membrowse-upload has no DEPENDS, so the aggregate rebuilds nothing - + # it records every example, --identical for the ones without an elf. Scoping it + # drops the excluded examples from the dataset instead of marking them unchanged. + upload = self.util[self.util.index('--target examples-membrowse-upload'):] + self.assertNotIn('$EX_ARGS', upload.split('\n')[0]) + + +if __name__ == '__main__': + unittest.main() diff --git a/test/hil/test/test_ci_select.py b/test/hil/test/test_ci_select.py new file mode 100644 index 000000000..74e5f48e6 --- /dev/null +++ b/test/hil/test/test_ci_select.py @@ -0,0 +1,2154 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +# Unit tests for ci_select.py — pure logic, no hardware, no git. Run directly: +# python3 test/hil/test/test_ci_select.py +# +# Imports stay stdlib + ci_select/hil_util/hil_flash ONLY: the pre-commit hil-test +# hook runs this suite, on GitHub's bare runner in the pre-commit workflow as well as +# locally, and that runner has no pyserial/pymtp. hil_flash is admissible because it +# is stdlib + hil_util only (test_hil_util.BottomLayer enforces the stdlib closure of +# both) and the roster-dispatch tests need its flash_* table; never import hil_test, +# which pulls pyserial. +import contextlib +import glob +import io +import json +import os +import pathlib +import re +import subprocess +import sys +import unittest + +REPO = os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.abspath(__file__))))) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # test/hil, for hil_flash/helper +sys.path.insert(0, os.path.join(REPO, 'tools')) +import hil_flash +import ci_select +from helper.hil_util import device_tests, dual_tests + + +def real_rosters(): + """The actual rig rosters, for regression tests that need real-world data + (a specific board/family/only-list) rather than the synthetic ROSTER above.""" + rosters = [] + for name in ('tinyusb.json', 'hfp.json'): + path = os.path.join(REPO, 'test/hil', name) + with open(path) as f: + rosters.append((f'test/hil/{name}', json.load(f)['boards'])) + return rosters + + +def roster_flashers(): + """(roster path, board) for every board in the live rosters, `boards-skip` + included: a parked board's flasher name must still dispatch, so that unparking it + is not what discovers the name went stale.""" + for name in ('tinyusb.json', 'hfp.json'): + path = os.path.join(REPO, 'test/hil', name) + with open(path) as f: + cfg = json.load(f) + for key in ('boards', 'boards-skip'): + for b in cfg.get(key, []): + yield f'test/hil/{name}', b + + +def on_roster(tc, *names): + """The subset of `names` currently in the live rig rosters, skipping the test + when none are, because parking/unparking a board is routine rig maintenance. + + That skip now matters MORE than it used to, not less: this suite is a blocking + pre-commit hook AND build.yml's selector steps gate on it (a failing suite falls + open to the full matrix), so an assertion that depends on a specific board being + present goes red on every PR -- including src/-only ones that never touched the + rig -- until someone fixes the roster. Keep roster-dependent assertions behind + on_roster.""" + have = {b['name'] for _, boards in real_rosters() for b in boards} + got = [n for n in names if n in have] + if not got: + tc.skipTest(f'not in the rig roster: {", ".join(names)}') + return got + + +ROSTER = [ + # device-only, rp2040 family + {'name': 'raspberry_pi_pico', 'uid': 'u1', 'flasher': {'name': 'openocd'}, + 'tests': {'device': True, 'host': True, 'dual': True}}, + # device-only, stm32f4 family + {'name': 'stm32f407disco', 'uid': 'u2', 'flasher': {'name': 'jlink'}, + 'tests': {'device': True, 'host': False, 'dual': False}}, + # host-only board + {'name': 'raspberry_pi_pico2', 'uid': 'u3', 'flasher': {'name': 'openocd'}, + 'tests': {'device': False, 'host': True, 'dual': False}}, + # only-list board (espressif-style), flashed by the CI leg that splits on esptool + {'name': 'espressif_s3_devkitm', 'uid': 'u4', 'flasher': {'name': 'esptool'}, + 'tests': {'only': ['device/cdc_msc_freertos', 'host/device_info']}}, +] +ROSTERS = [('test/hil/tinyusb.json', ROSTER)] + + +def sel(files): + return ci_select.classify(files, REPO, ROSTERS) + + +class TestPortRule(unittest.TestCase): + def test_dcd_rp2040_selects_pico_family_only(self): + s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico', s['boards']) + self.assertNotIn('stm32f407disco', s['boards']) + self.assertNotIn('espressif_s3_devkitm', s['boards']) + # device role: no host tests in pico's list + self.assertTrue(all(not t.startswith('host/') for t in s['boards']['raspberry_pi_pico'])) + # host-only boards drop out entirely on a device-role change + self.assertNotIn('raspberry_pi_pico2', s['boards']) + + def test_shared_port_file_is_both_roles(self): + s = sel(['src/portable/synopsys/dwc2/dwc2_common.c']) + self.assertFalse(s['full']) + self.assertNotIn('raspberry_pi_pico', s['boards']) # rp2040 is not a dwc2 family + self.assertIn('stm32f407disco', s['boards']) # stm32f4 is + + +class TestCoreRoleRule(unittest.TestCase): + def test_usbd_selects_all_device_tests_everywhere(self): + s = sel(['src/device/usbd.c']) + self.assertFalse(s['full']) + self.assertNotIn('raspberry_pi_pico2', s['boards']) # host-only board dropped + pico = s['boards']['raspberry_pi_pico'] + self.assertTrue(set(device_tests).issubset(set(pico))) + self.assertTrue(set(dual_tests).issubset(set(pico))) # dual survives device role + self.assertTrue(all(not t.startswith('host/') for t in pico)) + # only-list board: selection intersects its only-list + esp = s['boards']['espressif_s3_devkitm'] + self.assertEqual(esp, ['device/cdc_msc_freertos']) + + def test_host_change_drops_device(self): + s = sel(['src/host/usbh.c']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico2', s['boards']) + self.assertNotIn('stm32f407disco', s['boards']) # device-only board dropped + + +class TestClassRule(unittest.TestCase): + def test_cdc_device_selects_cdc_examples_only(self): + s = sel(['src/class/cdc/cdc_device.c']) + self.assertFalse(s['full']) + pico = s['boards']['raspberry_pi_pico'] + self.assertIn('device/cdc_msc', pico) + self.assertIn('device/cdc_dual_ports', pico) + self.assertNotIn('device/msc_dual_lun', pico) # CFG_TUD_CDC 0 there + self.assertNotIn('device/usbtest', pico) # CFG_TUD_CDC 0 there + self.assertTrue(all(not t.startswith('host/') for t in pico)) + + def test_msc_host_selects_host_side(self): + s = sel(['src/class/msc/msc_host.c']) + self.assertFalse(s['full']) + self.assertNotIn('stm32f407disco', s['boards']) # device-only board + pico2 = s['boards']['raspberry_pi_pico2'] + self.assertIn('host/msc_file_explorer', pico2) + self.assertTrue(all(not t.startswith('device/') for t in pico2)) + + +class TestClassIncludeEdges(unittest.TestCase): + """A class header another class includes reaches that class's examples too. + src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, so + midi_test's firmware contains audio.h - but the class rule derives macros from + the directory name alone, so an audio.h change used to select only + device/audio_test_freertos. On boards that skip that example the per-board + intersection emptied and an audio.h-only PR ran ZERO HIL on them.""" + def test_edges_derived_from_includes(self): + edges = ci_select.class_include_edges(REPO) + self.assertEqual(edges.get('audio/audio.h'), {'midi'}) + self.assertEqual(edges.get('cdc/cdc.h'), {'net'}) + + def test_audio_header_selects_midi_example(self): + s = ci_select.classify(['src/class/audio/audio.h'], REPO, real_rosters()) + self.assertFalse(s['full']) + # every board that runs device/midi_test at all must run it here (boards with + # a tests.only list, e.g. espressif, run the freertos examples instead) + by_name = {b['name']: b for _, bs in real_rosters() for b in bs} + checked = 0 + for name, tests in s['boards'].items(): + if 'device/midi_test' in ci_select.board_tests(by_name[name]): + self.assertIn('device/midi_test', tests, name) + checked += 1 + self.assertTrue(checked) + + def test_audio_header_reaches_boards_that_skip_audio(self): + # both skip device/audio_test_freertos: without the midi edge their + # intersection is empty and they drop out of the selection entirely + boards = on_roster(self, 'metro_m4_express', 'nrf54lm20dk') + s = ci_select.classify(['src/class/audio/audio.h'], REPO, real_rosters()) + for board in boards: + self.assertEqual(s['boards'].get(board), ['device/midi_test'], board) + + def test_edge_is_per_header_not_per_class(self): + # midi includes audio.h, not audio_device.h: an audio_device change must + # not drag midi's examples in + s = ci_select.classify(['src/class/audio/audio_device.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for tests in s['boards'].values(): + if tests != 'all': + self.assertNotIn('device/midi_test', tests) + + +class TestFallbackRules(unittest.TestCase): + def test_unknown_tool_is_full(self): + s = sel(['tools/random_new_script.py']) + self.assertTrue(s['full']) + + def test_docs_only_is_empty_not_full(self): + s = sel(['docs/info/contributing.rst', 'README.rst']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + def test_bsp_family_selects_family_boards(self): + s = sel(['hw/bsp/rp2040/family.cmake']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico', s['boards']) + self.assertEqual(s['boards']['raspberry_pi_pico'], 'all') + self.assertNotIn('stm32f407disco', s['boards']) + + def test_bsp_board_narrows_to_board(self): + s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) + self.assertFalse(s['full']) + self.assertEqual(list(s['boards'].keys()), ['raspberry_pi_pico']) + + def test_example_change_selects_that_example(self): + s = sel(['examples/device/cdc_msc/src/main.c']) + self.assertFalse(s['full']) + self.assertEqual(s['boards']['raspberry_pi_pico'], ['device/cdc_msc']) + + def test_core_common_is_full(self): + for f in ['src/tusb.c', 'src/common/tusb_fifo.c', 'src/osal/osal_freertos.h']: + self.assertTrue(sel([f])['full'], f) + + def test_board_test_example_is_full(self): + # board_test is the park/teardown firmware hil_test.py flashes on every board, + # not an unlisted example: a regression there must not skip the whole rig + for f in ['examples/device/board_test/src/main.c', + 'examples/device/board_test/CMakeLists.txt']: + self.assertTrue(sel([f])['full'], f) + + def test_harness_is_full(self): + # hw/mcu/ is no longer here: it resolves to families/boards via mcu_families() + # instead of forcing full - see TestMcuHilRule + for f in ['test/hil/hil_test.py', '.github/workflows/build.yml']: + self.assertTrue(sel([f])['full'], f) + + def test_mixed_roles_no_pruning(self): + s = sel(['src/device/usbd.c', 'src/host/usbh.c']) + self.assertFalse(s['full']) + self.assertIn('raspberry_pi_pico2', s['boards']) + self.assertIn('stm32f407disco', s['boards']) + + def test_cmakelists_and_requirements_are_full(self): + for f in ['src/CMakeLists.txt', 'examples/CMakeLists.txt', + 'examples/device/CMakeLists.txt', 'test/hil/requirements.txt']: + self.assertTrue(sel([f])['full'], f) + + def test_docs_txt_is_noncode(self): + s = sel(['docs/info/changelog.txt']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + +class TestArgsEmission(unittest.TestCase): + def test_args_for_scoped_selection(self): + s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + args = ci_select.selection_args(s, ROSTERS) + a = args['tinyusb.json'] + self.assertIn('-b raspberry_pi_pico', a) + self.assertNotIn('stm32f407disco', a) + self.assertIn('-bt raspberry_pi_pico:', a) # device-only subset of a device+host board + + def test_args_full_is_empty(self): + s = sel(['tools/random_new_script.py']) + self.assertEqual(ci_select.selection_args(s, ROSTERS), {'tinyusb.json': ''}) + + def test_args_all_board_gets_bare_b(self): + s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) + a = ci_select.selection_args(s, ROSTERS)['tinyusb.json'] + self.assertIn('-b raspberry_pi_pico', a) + self.assertNotIn('-bt', a) + + def test_args_by_flasher_splits_esp_from_the_rest(self): + s = sel(['src/device/usbd.c']) + per = ci_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json'] + self.assertIn('espressif_s3_devkitm', per['esptool']) + self.assertIn('raspberry_pi_pico', per['openocd']) + self.assertNotIn('espressif_s3_devkitm', per.get('openocd', '') + per.get('jlink', '')) + + def test_args_by_flasher_omits_a_flasher_with_no_selected_board(self): + # the esp CI leg must see no args at all here, not a filter matching zero boards + s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) + per = ci_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json'] + self.assertEqual(per, {'openocd': '-b raspberry_pi_pico'}) + + def test_args_by_flasher_full_is_empty(self): + s = sel(['tools/random_new_script.py']) + self.assertEqual(ci_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}}) + + def test_cli_diff_file(self): + import subprocess, tempfile, json as j + with tempfile.NamedTemporaryFile('w', suffix='.txt', delete=False) as f: + f.write('src/class/cdc/cdc_device.c\n') + path = f.name + r = subprocess.run([sys.executable, os.path.join(REPO, 'tools/ci_select.py'), + '--diff-file', path, os.path.join(REPO, 'test/hil/tinyusb.json')], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + out = j.loads(r.stdout) + self.assertFalse(out['full']) + self.assertIn('tinyusb.json', out['args']) + self.assertTrue(any('cdc_device' in line for line in out['reasons'])) + # A core-class diff must select boards THROUGH THE CLI: the in-process tests + # inject their own repo root, so only this subprocess path catches a broken + # repo_root derivation -- which once made every repo-relative glob match + # nothing and turned this exact diff into a silent full-HIL skip. + self.assertTrue(out['boards'], + 'CLI selected zero boards for a src/class change: repo_root broken?') + os.unlink(path) + + +class TestRealRosterPortFamilies(unittest.TestCase): + """Regression for port_families() missing espressif's dwc2 reference, which + lives in a component CMakeLists.txt rather than family.cmake/family.mk.""" + def test_dwc2_change_selects_espressif_boards(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = ci_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertIn(board, s['boards']) + + +class TestOptionGatedPort(unittest.TestCase): + """Regression: family_support.cmake compiles some ports from a build option + (MAX3421_HOST=1 -> hcd_max3421.c), so a board's family file never names them.""" + # host-side option board (max3421 as host controller), off any max3421 family + OPT_ROSTER = [('test/hil/opt.json', [ + {'name': 'fake_dual_board', 'uid': 'o1', 'flasher': {'name': 'jlink'}, + 'build': {'args': ['MAX3421_HOST=1']}, + 'tests': {'device': True, 'host': False, 'dual': True}}, + {'name': 'fake_host_board', 'uid': 'o2', 'flasher': {'name': 'jlink'}, + 'variant': [{'name': 'fake_host_board', 'flags': '-DMAX3421_HOST=1'}], + 'tests': {'device': False, 'host': True, 'dual': False}}, + {'name': 'fake_off_board', 'uid': 'o3', 'flasher': {'name': 'jlink'}, + 'variant': [{'name': 'fake_off_board', 'defines': ['MAX3421_HOST=0']}], + 'tests': {'device': True, 'host': True, 'dual': True}}, + ])] + + def test_real_roster_max3421_selects_option_board(self): + boards = on_roster(self, 'metro_m4_express') + s = ci_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertIn(board, s['boards']) + + def test_option_selects_via_args_defines_and_flags(self): + s = ci_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, self.OPT_ROSTER) + self.assertFalse(s['full']) + self.assertIn('fake_dual_board', s['boards']) # build.args + self.assertIn('fake_host_board', s['boards']) # variant flags + self.assertNotIn('fake_off_board', s['boards']) # variant defines, but =0 + + def test_device_role_port_does_not_pull_host_only_option_board(self): + s = ci_select.classify(['src/portable/analog/max3421/dcd_max3421.c'], REPO, self.OPT_ROSTER) + self.assertFalse(s['full']) + self.assertNotIn('fake_host_board', s['boards']) # host-only board, device change + self.assertIn('fake_dual_board', s['boards']) # device-capable option board + + def test_gates_parsed_from_family_support(self): + self.assertEqual(ci_select.port_option_gates(REPO).get('analog/max3421'), + {'MAX3421_HOST'}) + + def test_board_cmake_option_counts(self): + """A board can enable a gated port in its own BSP rather than via the roster + (hw/bsp/espressif/boards/*/board.cmake -> set(MAX3421_HOST 1)); board_options() + must see those too, or such a board joining the roster is silently dropped.""" + self.assertIn('MAX3421_HOST', + ci_select.bsp_board_options('adafruit_feather_esp32s3', REPO)) + self.assertIn('CFG_TUH_RPI_PIO_USB', + ci_select.bsp_board_options('adafruit_fruit_jam', REPO)) + # commented-out `# set(MAX3421_HOST 1)` must not count + self.assertNotIn('MAX3421_HOST', + ci_select.bsp_board_options('feather_nrf52840_express', REPO)) + + def test_board_cmake_option_selects_off_family_board(self): + # adafruit_feather_esp32s3 is not on any rig roster; stand it in as one to + # prove the BSP-sourced option alone pulls a max3421 change onto the board + roster = [('test/hil/opt.json', [ + {'name': 'adafruit_feather_esp32s3', 'uid': 'o1', 'flasher': {'name': 'esptool'}, + 'tests': {'device': False, 'host': True, 'dual': False}}])] + s = ci_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster) + self.assertFalse(s['full']) + self.assertIn('adafruit_feather_esp32s3', s['boards']) + + def test_board_mk_option_is_ignored(self): + """Make-only options must not select: HIL CI builds with CMake exclusively, so + hw/bsp/nrf/boards/nrf5340dk/board.mk's MAX3421_HOST compiles nothing here.""" + roster = [('test/hil/opt.json', [ + {'name': 'nrf5340dk', 'uid': 'o1', 'flasher': {'name': 'jlink'}, + 'tests': {'device': False, 'host': True, 'dual': False}}])] + s = ci_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + +class TestPortFamiliesCmakeOnly(unittest.TestCase): + """port_families() is CMake-only (HIL CI never builds with Make) and matches on + 'port_dir/' so a port dir is not a prefix of a sibling.""" + def test_make_only_family_is_not_a_family(self): + # hw/bsp/pic32mz has family.mk but no family.cmake + self.assertEqual(ci_select.port_families('microchip/pic32mz', REPO), set()) + + def test_prefix_port_does_not_inherit_sibling_families(self): + # bare-substring matching let 'microchip/pic' match '.../microchip/pic32mz/...' + self.assertEqual(ci_select.port_families('microchip/pic', REPO), set()) + + def test_make_only_port_contributes_nothing(self): + s = sel(['src/portable/microchip/pic32mz/dcd_pic32mz.c']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons']) + + def test_cmake_families_still_found(self): + self.assertEqual(ci_select.port_families('raspberrypi/rp2040', REPO), {'rp2040'}) + self.assertIn('stm32f4', ci_select.port_families('synopsys/dwc2', REPO)) + + +class TestPortFamiliesCoverage(unittest.TestCase): + """Systematic guard: every real dcd_*/hcd_* port directory should map to at + least one board family, so a future family.cmake/CMakeLists.txt layout that + port_families() doesn't scan fails loudly instead of silently dropping boards + (as espressif's dwc2 reference did - see TestRealRosterPortFamilies).""" + # Ports with no board family: not a bug, just not wired into any rig board. + # Add here (with a reason) only if port_families() legitimately can't find one. + # A port listed here contributes NOTHING on either axis (empty means empty), so + # this list is the tripwire: a port that stops resolving must show up as a test + # failure, not as a PR that quietly builds and tests nothing. + NO_FAMILY = { + 'template', # reference/example port, not built by any board + # hw/bsp/pic32mz has family.mk only (no family.cmake), and port_families() + # is CMake-only because HIL CI builds every board with CMake - so this port + # is compiled for no HIL board. + 'microchip/pic32mz', + 'microchip/pic', # same: only ever referenced from pic32mz's family.mk + } + + @staticmethod + def _dcd_hcd_ports(): + portable_root = os.path.join(REPO, 'src/portable') + ports = [] + for entry in sorted(os.listdir(portable_root)): + d = os.path.join(portable_root, entry) + if not os.path.isdir(d): + continue + if glob.glob(os.path.join(d, 'dcd_*.c')) or glob.glob(os.path.join(d, 'hcd_*.c')): + ports.append(entry) + continue + for sub in sorted(os.listdir(d)): + sd = os.path.join(d, sub) + if os.path.isdir(sd) and (glob.glob(os.path.join(sd, 'dcd_*.c')) or + glob.glob(os.path.join(sd, 'hcd_*.c'))): + ports.append(f'{entry}/{sub}') + return ports + + def test_every_port_maps_to_a_family(self): + ports = self._dcd_hcd_ports() + self.assertTrue(ports) # sanity: the scan itself found something + for port in ports: + if port in self.NO_FAMILY: + continue + fams = ci_select.port_families(port, REPO) + self.assertTrue(fams, f'{port}: no family references this port ' + f'(port_families() scan gap, or add to NO_FAMILY)') + + +class TestRealRosterOnlyListTests(unittest.TestCase): + """Regression for roster-only-list tests (e.g. espressif's hid_composite_freertos) + being invisible to the selector because it only knew the shared hil_util lists.""" + def test_only_list_example_change_selects_it(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = ci_select.classify(['examples/device/hid_composite_freertos/src/main.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertEqual(s['boards'][board], ['device/hid_composite_freertos']) + + def test_class_change_includes_only_list_boards(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = ci_select.classify(['src/class/hid/hid_device.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + self.assertIn(board, s['boards']) + + +class TestPortAndCoreRoleUseExtras(unittest.TestCase): + """Regression: the port rule and core-role rule must thread the roster-only + test universe (extras) the same way the class rule already does, so a DCD + or device-stack change doesn't silently drop espressif's only-list tests + (e.g. hid_composite_freertos) that aren't in the shared device_tests list.""" + def test_dcd_change_includes_only_list_test(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = ci_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + tests = s['boards'][board] + self.assertIn('device/hid_composite_freertos', tests) + self.assertIn('device/cdc_msc_freertos', tests) + self.assertIn('device/audio_test_freertos', tests) + self.assertIn('device/usbtest', tests) + + def test_core_device_change_includes_only_list_test(self): + boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') + s = ci_select.classify(['src/device/usbd.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board in boards: + tests = s['boards'][board] + self.assertIn('device/hid_composite_freertos', tests) + self.assertIn('device/cdc_msc_freertos', tests) + self.assertIn('device/audio_test_freertos', tests) + self.assertIn('device/usbtest', tests) + + def test_host_change_does_not_leak_device_only_list_test(self): + s = ci_select.classify(['src/host/usbh.c'], REPO, real_rosters()) + self.assertFalse(s['full']) + for board, tests in s['boards'].items(): + if tests == 'all': + continue + self.assertNotIn('device/hid_composite_freertos', tests, board) + + +class TestFamilies(unittest.TestCase): + """`families` exists for consumers that build (not just test) the diff: most + families have no rig board, so `boards` alone would compile nothing for them.""" + def test_off_rig_port_still_reports_family(self): + s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) # no same7x board on the rig + self.assertEqual(s['families'], ['same7x']) + + def test_port_families_are_reported(self): + s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + self.assertIn('rp2040', s['families']) + + def test_bsp_family_and_board_report_family(self): + self.assertEqual(sel(['hw/bsp/rp2040/family.cmake'])['families'], ['rp2040']) + self.assertEqual(sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h'])['families'], + ['rp2040']) + + def test_docs_only_has_no_families(self): + self.assertEqual(sel(['docs/info/contributing.rst'])['families'], []) + + def test_full_selection_still_reports_families(self): + """A full-matrix file must not hide the families of the other changed files: + consumers that build from `families` (e.g. /pre-pr) ignore `boards` when full.""" + s = sel(['src/common/tusb_fifo.c', 'src/portable/microchip/samx7x/dcd_samx7x.c']) + self.assertTrue(s['full']) + self.assertIn('same7x', s['families']) + # full stays full: every roster board, and no args to narrow the run + self.assertEqual(set(s['boards']), {b['name'] for b in ROSTER}) + self.assertTrue(all(v == 'all' for v in s['boards'].values())) + self.assertEqual(ci_select.selection_args(s, ROSTERS), {'tinyusb.json': ''}) + self.assertEqual(ci_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}}) + + def test_family_order_does_not_matter(self): + # same as above with the full-matrix file last (was the only order that worked) + s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c', 'src/common/tusb_fifo.c']) + self.assertTrue(s['full']) + self.assertIn('same7x', s['families']) + + +class TestGitDiffArgv(unittest.TestCase): + def test_diff_disables_rename_detection(self): + """Without --no-renames git reports only a rename's destination, so moving an + HIL-relevant file to a non-code path would be classified as non-code only.""" + self.assertIn('--no-renames', ci_select.GIT_DIFF_ARGV) + + +class TestPortWithoutFamilyContributesNothing(unittest.TestCase): + """A port dir no family file references contributes nothing on BOTH axes (the + maintainer's empty-means-empty ruling): nothing compiles the file, so there is + nothing to run. Forcing the full 30-board rig here bought no coverage - the build + walk answered the identical condition with zero families for the same path.""" + def test_unreferenced_port_contributes_nothing(self): + orig = ci_select.port_families + ci_select.port_families = lambda port_dir, repo_root: set() + try: + s = sel(['src/portable/vendor/newip/dcd_newip.c']) + b = ci_select.classify_build(['src/portable/vendor/newip/dcd_newip.c'], REPO) + finally: + ci_select.port_families = orig + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons']) + self.assertFalse(b['full']) + self.assertEqual(b['families'], []) + + +class TestOpenocdVidPid(unittest.TestCase): + """The roster's optional flasher `vid_pid` field (openocd-verbatim, e.g. + "0x1a86 0x8010", more pairs appended) pins openocd's probe discovery so it + never opens foreign usbfs nodes. It must be emitted BEFORE the args: the + rescue cfgs run `init` internally (rp2350-rescue.cfg errors on any + config-stage command after its init; rp2040.cfg under RESCUE scans before a + trailing flag is even parsed), and no rig cfg sets a competing list + (the 2026-08-10 convoy mechanism).""" + + def test_vid_pid_flag_precedes_args(self): + cmd = hil_flash._openocd_cmd_base( + {'uid': 'S1', 'args': '-f target/wch-riscv.cfg', 'vid_pid': '0x1a86 0x8010'}) + self.assertIn('-c "adapter usb vid_pid 0x1a86 0x8010" -f target/wch-riscv.cfg', cmd) + self.assertTrue(cmd.endswith('-f target/wch-riscv.cfg'), cmd) + + def test_rescue_cfg_command_keeps_vid_pid_before_init(self): + """rescue_openocd swaps the target cfg for one that runs `init` internally; + a vid_pid flag after the args would error there (rp2350) or be skipped + (rp2040) -- in exactly the wedged-rig scenario the pin exists for.""" + flasher = {'name': 'openocd', 'uid': 'S1', 'vid_pid': '0x2e8a 0x000c', + 'args': '-c "set RESCUE 1" -f target/rp2040.cfg'} + cmd = hil_flash._openocd_cmd_base(flasher) + self.assertLess(cmd.index('adapter usb vid_pid'), cmd.index('-f target/'), cmd) + + def test_vid_pid_multiple_pairs(self): + cmd = hil_flash._openocd_cmd_base( + {'uid': 'S1', 'args': '-f i.cfg', 'vid_pid': '0x2e8a 0x000c 0x2e8a 0x000d'}) + self.assertIn('-c "adapter usb vid_pid 0x2e8a 0x000c 0x2e8a 0x000d"', cmd) + + def test_no_field_no_flag_but_warns(self): + # the roster lint only covers the committed rosters; a dev PC's local.json entry + # without the field must at least say what it is giving up -- on STDERR, since + # hil_test captures stdout per test and would swallow it on a passing run + import io + from contextlib import redirect_stderr + hil_flash._VID_PID_WARNED.discard('S-warn') + cap = io.StringIO() + with redirect_stderr(cap): + cmd = hil_flash._openocd_cmd_base({'uid': 'S-warn', 'args': '-f i.cfg'}) + self.assertNotIn('vid_pid', cmd) + self.assertIn('vid_pid', cap.getvalue()) + + def test_roster_openocd_entries_all_pin_vid_pid(self): + # every openocd probe on the rig has a known VID/PID; a new entry without the + # pin silently reintroduces open-everything discovery + for path, board in roster_flashers(): + f = board['flasher'] + # tinyusb.json only: hfp.json is the hifiphile rig owner's file, and a + # blocking repo-wide lint over someone else's roster would red every PR the + # moment they add an openocd board (hil_flash treats the field as optional) + if f['name'] == 'openocd' and path.endswith('tinyusb.json'): + self.assertIn('vid_pid', f, + f"{path}: {board['name']} openocd flasher lacks vid_pid") + self.assertNotIn('vid_pid', f.get('args', ''), + f"{path}: {board['name']} packs vid_pid into args; use the field") + + +class TestRosterFlashersDispatch(unittest.TestCase): + """hil_test and hil_pool_check resolve a board's flasher with a bare + getattr(hil_flash, f'flash_{name}'), and hil_test does it inside a redirect_stdout — + so a renamed or typo'd roster name raises an AttributeError whose output is swallowed, + with nothing pointing at the roster as the thing to edit. Renaming a flash_*/reset_* + pair without updating every roster must fail here instead.""" + + def test_flash_and_reset_exist_for_every_roster_flasher(self): + for path, board in roster_flashers(): + name = board['flasher']['name'].lower() + for fn in (f'flash_{name}', f'reset_{name}'): + self.assertTrue(callable(getattr(hil_flash, fn, None)), + f'{path}: {board["name"]} uses flasher "{name}" ' + f'but hil_flash.{fn} does not exist') + + def test_firmware_suffix_known_for_every_roster_flasher(self): + """find_firmware falls back to accepting .elf-or-.bin when a flasher is missing + from FLASHER_SUFFIX, silently restoring the mismatch that map exists to catch.""" + for path, board in roster_flashers(): + name = board['flasher']['name'].lower() + self.assertIn(name, hil_flash.FLASHER_SUFFIX, + f'{path}: {board["name"]} uses flasher "{name}" ' + f'with no hil_flash.FLASHER_SUFFIX entry') + + +class FlasherRecoverEntry(unittest.TestCase): + """Optional roster key: a SECOND flasher used only to deliver recovery while a usbfs + node is poisoned. Boards whose primary flasher cannot get past a convoy (jlink, + stlink, lm4flash) name an openocd entry here instead of changing how they are + normally flashed.""" + + def test_recover_flasher_prefers_the_optional_entry(self): + prim = {'name': 'jlink', 'uid': 'X', 'args': '-device MIMXRT1064xxx6A'} + rec = {'name': 'openocd', 'uid': 'X', 'args': '-f interface/jlink.cfg -f target/foo.cfg'} + self.assertEqual(hil_flash.recover_flasher({'flasher': prim, 'flasher_recover': rec}), rec) + self.assertEqual(hil_flash.recover_flasher({'flasher': prim}), prim) + + def test_openocd_over_jlink_is_convoy_safe_without_a_pin(self): + """libjaylink discovery returns early unless idVendor == 0x1366 (SEGGER) and the PID + is in its table, and only THEN calls libusb_open (discovery_usb.c) -- it never opens + a foreign node. `adapter usb vid_pid` is a no-op for this driver: jlink.c reads + adapter_serial / usb address / usb location, never the vid/pid.""" + self.assertTrue(hil_flash.convoy_safe( + {'name': 'openocd', 'args': '-f interface/jlink.cfg -f target/stm32f4x.cfg'})) + + def test_openocd_with_neither_a_pin_nor_jlink_is_not_safe(self): + self.assertFalse(hil_flash.convoy_safe( + {'name': 'openocd', 'args': '-f interface/stlink.cfg -f target/stm32h7x.cfg'})) + + def test_the_existing_rules_are_unchanged(self): + self.assertTrue(hil_flash.convoy_safe( + {'name': 'openocd', 'vid_pid': '0x2e8a 0x000c', 'args': '-f interface/cmsis-dap.cfg'})) + self.assertFalse(hil_flash.convoy_safe({'name': 'jlink', 'uid': 'X'})) + self.assertTrue(hil_flash.convoy_safe({'name': 'esptool'})) + + +class TestModuleMove(unittest.TestCase): + def test_repo_root_guard(self): + # __file__-derived root: moving the module without re-deriving the parent + # count re-points every scan at the wrong tree (it happened once already) + self.assertTrue(os.path.isdir(os.path.join(ci_select._REPO_ROOT, 'src'))) + self.assertTrue(os.path.isdir(os.path.join(ci_select._REPO_ROOT, 'hw', 'bsp'))) + self.assertEqual(os.path.realpath(ci_select._REPO_ROOT), os.path.realpath(REPO)) + + +class TestPathFamilies(unittest.TestCase): + def test_port_wrapper_unchanged(self): + self.assertEqual(ci_select.port_families('raspberrypi/rp2040', REPO), {'rp2040'}) + self.assertIn('stm32f4', ci_select.port_families('synopsys/dwc2', REPO)) + + def test_boundary_without_trailing_slash(self): + # hw/bsp/nrf/family.cmake writes `${TOP}/hw/mcu/nordic/nrfx` — no trailing + # slash; the match must accept a directory-boundary end-of-token + self.assertEqual(ci_select.path_families('hw/mcu/nordic/nrfx', REPO), {'nrf'}) + + def test_boundary_rejects_prefix_sibling(self): + # 'microchip/pic' must not inherit pic32mz's references (and pic32mz itself + # is family.mk-only, which the CMake-only scan never reads) + self.assertEqual(ci_select.port_families('microchip/pic', REPO), set()) + self.assertEqual(ci_select.port_families('microchip/pic32mz', REPO), set()) + + def test_mcu_families_prefix_walk(self): + self.assertEqual(ci_select.mcu_families('hw/mcu/nordic/nrf5x/nrf_clock.h', REPO), {'nrf'}) + self.assertEqual(ci_select.mcu_families('hw/mcu/dialog/da1469x/x.h', REPO), {'da1469x'}) + self.assertEqual(ci_select.mcu_families('hw/mcu/no_such_vendor/x.c', REPO), set()) + + +class TestMcuHilRule(unittest.TestCase): + def test_mcu_no_longer_forces_full(self): + s = ci_select.classify(['hw/mcu/nordic/nrf5x/nrf_clock.h'], REPO, ROSTERS) + self.assertFalse(s['full']) + self.assertIn('nrf', s['families']) # recorded even with no nrf rig board + + def test_mcu_selects_family_boards(self): + got = on_roster(self, 'feather_nrf52840_express', 'pca10056', 'pca10095') + s = ci_select.classify(['hw/mcu/nordic/nrf5x/nrf_clock.h'], REPO, real_rosters()) + self.assertFalse(s['full']) + for b in got: + self.assertIn(b, s['boards']) + + def test_unresolved_mcu_path_selects_nothing(self): + # empty means empty (maintainer ruling): if no family's build references the + # path, no build consumes the change - there is nothing to compile or run. + # test_tracked_mcu_vendors_resolve is the drift guard for a real vendor dir + s = ci_select.classify(['hw/mcu/no_such_vendor/x.c'], REPO, ROSTERS) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + self.assertEqual(s['families'], []) + + +class TestOrphanInvariant(unittest.TestCase): + ALLOW = {'microchip/pic', 'microchip/pic32mz'} # spec: known orphans, CMake builds neither + + def test_every_port_resolves_to_a_family(self): + for d in sorted(glob.glob(os.path.join(REPO, 'src/portable/*/*'))): + if not os.path.isdir(d): + continue + port = os.path.relpath(d, os.path.join(REPO, 'src/portable')).replace(os.sep, '/') + fams = ci_select.port_families(port, REPO) + if port in self.ALLOW: + self.assertEqual(fams, set(), f'{port}: no longer an orphan - drop it from ALLOW') + else: + self.assertTrue(fams, f'{port}: no family.cmake references it - wire it up or allowlist it') + + def test_tracked_mcu_vendors_resolve(self): + import subprocess as sp + r = sp.run(['git', 'ls-files', 'hw/mcu'], cwd=REPO, capture_output=True, text=True) + if r.returncode != 0: + self.skipTest('not a git checkout') + vendors = sorted({'/'.join(p.split('/')[:3]) for p in r.stdout.split()}) + for v in vendors: + self.assertTrue(ci_select.mcu_families(v + '/x.c', REPO), f'{v}: resolves to no family') + + def test_every_get_deps_family_token_resolves_or_is_a_known_alias(self): + """Same drift guard, dep side. A token naming no hw/bsp dir makes the entry + unreachable for its family in get_deps.py itself (`f in entry[2].split()`), and + makes a bump of it select nothing here. The four known ones are pinned; a fifth + appearing is a real bug in get_deps.py, not something to swallow.""" + sys.path.insert(0, os.path.join(REPO, 'tools')) + import get_deps + fams = set(ci_select.all_bsp_families(REPO)) + stale = {} + for name, d in (('deps_mandatory', get_deps.deps_mandatory), + ('deps_optional', get_deps.deps_optional)): + for path, entry in d.items(): + for tok in str(entry[2]).split(): + if tok != 'all' and tok not in fams: + stale.setdefault(tok, []).append(f'{name}[{path}]') + # subset, not equality: correcting a token in get_deps.py (fc100s -> f1c100s) + # should be a one-file change, while a NEW unmappable token - which force-fulls + # every get_deps edit that touches its entry - has to be a deliberate act + self.assertFalse(set(stale) - set(ci_select._DEPS_ALIAS_TOKENS), + f'get_deps family tokens naming no hw/bsp dir: ' + f'{ {k: v for k, v in stale.items() if k not in ci_select._DEPS_ALIAS_TOKENS} }') + + +class TestRostersDoNotOverlap(unittest.TestCase): + """sel['boards'] is one map across every roster, so a board listed in TWO rosters + with different test lists would get the union - and hil_test.py on the rig that + only runs half of them would be handed a -t it has no fixture for. No overlap + exists today; this is the tripwire for the day one is added.""" + + def test_no_board_name_is_in_two_rosters(self): + seen = {} + for name in ('tinyusb.json', 'hfp.json'): + cfg = json.load(open(os.path.join(REPO, 'test/hil', name))) + for b in cfg['boards']: + if b['name'] in seen: + self.assertEqual( + seen[b['name']], b.get('tests'), + f"{b['name']}: on two rosters with different test lists - " + f"selection_args must then filter per roster, not from the union") + seen[b['name']] = b.get('tests') + + +class TestLibRule(unittest.TestCase): + """lib/** is not a full-matrix path: only the examples that build the lib need it.""" + + def b(self, files): + return ci_select.classify_build(files, REPO) + + def test_lib_examples_ground_truth(self): + self.assertEqual(ci_select.lib_examples('embedded-cli', REPO), + {'host/msc_file_explorer', 'host/msc_file_explorer_freertos'}) + self.assertEqual(ci_select.lib_examples('networking', REPO), + {'device/net_lwip_webserver'}) + # only family_support.cmake's LOGGER=rtt plumbing names it, and no CI example + # build turns that on - the scan is per-example on purpose + self.assertEqual(ci_select.lib_examples('SEGGER_RTT', REPO), set()) + self.assertEqual(ci_select.lib_examples('rt-thread', REPO), set()) + + def test_lib_examples_matches_at_a_directory_boundary(self): + # 'lib/net' must not inherit lib/networking's example + self.assertEqual(ci_select.lib_examples('net', REPO), set()) + + def test_build_lib_selects_only_the_using_examples(self): + s = self.b(['lib/embedded-cli/embedded_cli.h']) + self.assertFalse(s['full']) + self.assertTrue(s['families']) + want = {'host/msc_file_explorer', 'host/msc_file_explorer_freertos'} + mapped = set() + for fam, exs in s['family_examples'].items(): + self.assertTrue(set(exs) <= want, f'{fam}: {exs}') + mapped |= set(exs) + self.assertEqual(mapped, want) + + def test_build_lib_nobody_builds_selects_nothing(self): + s = self.b(['lib/SEGGER_RTT/RTT/SEGGER_RTT.c']) + self.assertFalse(s['full']) + self.assertEqual(s['families'], []) + self.assertEqual(s['family_examples'], {}) + + def test_hil_lib_selects_the_using_tests(self): + s = sel(['lib/embedded-cli/embedded_cli.h']) + self.assertFalse(s['full']) + want = {'host/msc_file_explorer', 'host/msc_file_explorer_freertos'} + self.assertEqual(set(s['boards']['raspberry_pi_pico']), want) + self.assertEqual(set(s['boards']['raspberry_pi_pico2']), want) + # device-only board and the only-list board run neither test + self.assertNotIn('stm32f407disco', s['boards']) + self.assertNotIn('espressif_s3_devkitm', s['boards']) + + def test_hil_lib_used_only_by_a_disabled_test_selects_nothing(self): + # device/net_lwip_webserver is commented out of hil_util.device_tests, so the + # intersection with the HIL universe is empty + s = sel(['lib/networking/dhserver.c']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + def test_hil_lib_nobody_builds_selects_nothing(self): + s = sel(['lib/SEGGER_RTT/RTT/SEGGER_RTT.c']) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + +# A miniature get_deps.py: the module shape the parser must cope with (imports, +# both dep dicts, the derived deps_all, a function) without the real 300-entry file. +_GD_BASE = """#!/usr/bin/env python3 +import argparse + +deps_mandatory = { + 'lib/fatfs': ['https://github.com/abbrev/fatfs.git', 'aaa', 'all'], +} + +deps_optional = { + 'hw/mcu/st/cmsis_device_f4': ['https://github.com/x/f4.git', 'bbb', 'stm32f4 stm32f7'], + 'hw/mcu/nordic/nrfx': ['https://github.com/x/nrfx.git', 'ccc', 'nrf'], +} + +deps_all = {**deps_mandatory, **deps_optional} + + +def main(): + return 1 +""" + + +class TestGetDepsChangedFamilies(unittest.TestCase): + """Pure text-in, families-out: no git, no exec of the parsed module.""" + + def f(self, head, base=_GD_BASE): + return ci_select.get_deps_changed_families(base, head, REPO) + + def test_no_change_selects_nothing(self): + self.assertEqual(self.f(_GD_BASE), set()) + + def test_comment_only_change_selects_nothing(self): + self.assertEqual(self.f(_GD_BASE.replace('import argparse', + 'import argparse # noqa')), set()) + + def test_optional_commit_bump_selects_its_families(self): + self.assertEqual(self.f(_GD_BASE.replace("'bbb'", "'bbb2'")), + {'stm32f4', 'stm32f7'}) + + def test_mandatory_all_entry_is_full(self): + self.assertIsNone(self.f(_GD_BASE.replace("'aaa'", "'aaa2'"))) + + def test_logic_change_is_full(self): + self.assertIsNone(self.f(_GD_BASE.replace('return 1', 'return 2'))) + + def test_unparseable_text_is_full(self): + self.assertIsNone(self.f('def broken(:\n')) + + def test_unresolvable_token_is_full(self): + # a changed entry we cannot map to a family is NOT "nothing changed": reading it + # that way empties the whole build matrix for a dep bump. Fall open instead - + # even when a sibling token does resolve, because the unmapped one may be the + # family that actually needed the new revision + base = _GD_BASE.replace("'ccc', 'nrf'", "'ccc', 'zz_gone samd5x_e5x'") + self.assertIsNone(self.f(base.replace("'ccc'", "'ccc2'"), base)) + base = _GD_BASE.replace("'ccc', 'nrf'", "'ccc', 'zz_gone'") + self.assertIsNone(self.f(base.replace("'ccc'", "'ccc2'"), base)) + + def test_family_token_change_unions_both_sides(self): + # the family list itself edited: both sides contribute + head = _GD_BASE.replace("'ccc', 'nrf'", "'ccc', 'rp2040 samd5x_e5x'") + self.assertEqual(self.f(head), {'nrf', 'rp2040', 'samd5x_e5x'}) + + def test_known_alias_tokens_select_nothing(self): + # the tokens in _DEPS_ALIAS_TOKENS name no hw/bsp dir: either a pre-rename + # spelling sitting beside the current name in the same entry, or a family with + # no boards in the tree. Changing one selects nothing rather than force-fulling + # every get_deps edit that touches its entry. + base = _GD_BASE.replace("'ccc', 'nrf'", "'ccc', 'stm32l5'") + self.assertEqual(self.f(base.replace("'ccc'", "'ccc2'"), base), set()) + + def test_moving_an_entry_between_the_two_dicts_is_seen(self): + # value untouched, dict changed: mandatory deps are fetched for every family, so + # demoting one stops families fetching it. Merging the dicts before diffing (or + # comparing the ast dump of deps_all) hides this completely. + head = _GD_BASE.replace( + " 'hw/mcu/nordic/nrfx': ['https://github.com/x/nrfx.git', 'ccc', 'nrf'],\n", '') + head = head.replace( + "deps_mandatory = {\n", + "deps_mandatory = {\n 'hw/mcu/nordic/nrfx': ['https://github.com/x/nrfx.git', 'ccc', 'nrf'],\n") + self.assertEqual(self.f(head), {'nrf'}) + + def test_added_entry_selects_its_families(self): + head = _GD_BASE.replace( + "deps_optional = {\n", + "deps_optional = {\n 'hw/mcu/x': ['https://github.com/x/x.git', 'ddd', 'rp2040'],\n") + self.assertEqual(self.f(head), {'rp2040'}) + + def test_removed_entry_selects_its_base_side_families(self): + head = _GD_BASE.replace( + " 'hw/mcu/nordic/nrfx': ['https://github.com/x/nrfx.git', 'ccc', 'nrf'],\n", '') + self.assertEqual(self.f(head), {'nrf'}) + + def test_family_list_change_unions_both_sides(self): + head = _GD_BASE.replace("'stm32f4 stm32f7'", "'stm32f4 stm32h7'") + self.assertEqual(self.f(head), {'stm32f4', 'stm32f7', 'stm32h7'}) + + def test_real_get_deps_parses(self): + with open(os.path.join(REPO, 'tools/get_deps.py')) as f: + real = f.read() + self.assertEqual(ci_select.get_deps_changed_families(real, real, REPO), set()) + # a real optional entry bumped resolves to that entry's real family. The commit + # is read out of get_deps.py rather than pinned here - a routine dep bump must + # not fail this suite, and pinning a hash tests the tree, not the code + sys.path.insert(0, os.path.join(REPO, 'tools')) + import get_deps + commit, tokens = get_deps.deps_optional['hw/mcu/nordic/nrfx'][1:3] + bumped = real.replace(commit, '0' * len(commit)) + self.assertNotEqual(bumped, real) + self.assertEqual(ci_select.get_deps_changed_families(real, bumped, REPO), + set(tokens.split())) + + +class TestGetDepsRule(unittest.TestCase): + """tools/get_deps.py: the changed dep entries' families, or full when unknowable.""" + + def test_build_selects_the_changed_families(self): + s = ci_select.classify_build(['tools/get_deps.py'], REPO, + get_deps_families={'stm32f4'}) + self.assertFalse(s['full']) + self.assertEqual(s['families'], ['stm32f4']) + self.assertNotIn('stm32f4', s['family_examples']) # every example it builds + + def test_build_without_a_base_is_full(self): + # --diff-file mode has no git and so no base content: fail open + self.assertTrue(ci_select.classify_build(['tools/get_deps.py'], REPO)['full']) + + def test_build_no_dep_entry_changed_selects_nothing(self): + s = ci_select.classify_build(['tools/get_deps.py'], REPO, get_deps_families=set()) + self.assertFalse(s['full']) + self.assertEqual(s['families'], []) + + def test_hil_selects_the_changed_families_boards(self): + s = ci_select.classify(['tools/get_deps.py'], REPO, ROSTERS, + get_deps_families={'stm32f4'}) + self.assertFalse(s['full']) + self.assertEqual(list(s['boards']), ['stm32f407disco']) + self.assertEqual(s['families'], ['stm32f4']) + + def test_hil_without_a_base_is_full(self): + self.assertTrue(ci_select.classify(['tools/get_deps.py'], REPO, ROSTERS)['full']) + + def test_hil_no_dep_entry_changed_selects_nothing(self): + s = ci_select.classify(['tools/get_deps.py'], REPO, ROSTERS, get_deps_families=set()) + self.assertFalse(s['full']) + self.assertEqual(s['boards'], {}) + + def test_cli_diff_file_mode_is_full(self): + import tempfile, json as j + with tempfile.NamedTemporaryFile('w', suffix='.txt', delete=False) as f: + f.write('tools/get_deps.py\n') + path = f.name + r = subprocess.run([sys.executable, os.path.join(REPO, 'tools/ci_select.py'), + '--diff-file', path, os.path.join(REPO, 'test/hil/tinyusb.json')], + capture_output=True, text=True) + os.unlink(path) + self.assertEqual(r.returncode, 0, r.stderr) + out = j.loads(r.stdout) + self.assertTrue(out['full']) + self.assertTrue(out['build']['full']) + + +class TestGetDepsGitPlumbing(unittest.TestCase): + """--base mode: merge-base, the diff, and both blobs come from git, and only + tools/get_deps.py in the diff triggers the blob reads.""" + + HEAD = _GD_BASE.replace("'bbb'", "'bbb2'") + + def run_main(self, diff): + from unittest import mock + calls = [] + + def fake_run(argv, **kw): + calls.append(argv) + if argv[:2] == ['git', 'merge-base']: + out = 'MB123\n' + elif argv[:3] == ci_select.GIT_DIFF_ARGV[:3]: + out = diff + elif argv[:2] == ['git', 'show']: + out = _GD_BASE if argv[2].startswith('MB123:') else self.HEAD + else: + raise AssertionError(f'unexpected git call: {argv}') + return subprocess.CompletedProcess(argv, 0, stdout=out, stderr='') + + buf = io.StringIO() + argv = [sys.executable, '--base', 'origin/master'] + with mock.patch.object(ci_select.subprocess, 'run', fake_run), \ + mock.patch.object(sys, 'argv', argv), \ + contextlib.redirect_stdout(buf), contextlib.redirect_stderr(io.StringIO()): + ci_select.main() + return json.loads(buf.getvalue()), calls + + def test_base_mode_reads_the_merge_base_blob(self): + out, calls = self.run_main('tools/get_deps.py\n') + self.assertIn(['git', 'show', 'MB123:tools/get_deps.py'], calls) + self.assertIn(['git', 'show', 'HEAD:tools/get_deps.py'], calls) + self.assertFalse(out['build']['full']) + self.assertEqual(out['build']['families'], ['stm32f4', 'stm32f7']) + + def test_no_get_deps_in_the_diff_reads_no_blob(self): + out, calls = self.run_main('src/class/cdc/cdc_device.c\n') + self.assertFalse(any(c[:2] == ['git', 'show'] for c in calls)) + self.assertFalse(out['build']['full']) + + def test_git_failure_falls_open(self): + from unittest import mock + + def fake_run(argv, **kw): + if argv[:2] == ['git', 'show']: + raise subprocess.CalledProcessError(128, argv) + out = 'MB123\n' if argv[:2] == ['git', 'merge-base'] else 'tools/get_deps.py\n' + return subprocess.CompletedProcess(argv, 0, stdout=out, stderr='') + + buf = io.StringIO() + with mock.patch.object(ci_select.subprocess, 'run', fake_run), \ + mock.patch.object(sys, 'argv', [sys.executable, '--base', 'origin/master']), \ + contextlib.redirect_stdout(buf), contextlib.redirect_stderr(io.StringIO()): + ci_select.main() + self.assertTrue(json.loads(buf.getvalue())['build']['full']) + + +class TestBuildClassifier(unittest.TestCase): + def b(self, files): + return ci_select.classify_build(files, REPO) + + def test_noncode_and_test_hil_contribute_nothing(self): # rules 1, 2 + s = self.b(['docs/info/index.rst', 'README.rst', 'test/hil/hil_test.py', '.claude/skills/hil/SKILL.md']) + self.assertFalse(s['full']) + self.assertEqual(s['families'], []) + self.assertEqual(s['family_examples'], {}) + + def test_port_device_rule(self): # rule 3 + s = self.b(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) + self.assertFalse(s['full']) + self.assertEqual(s['families'], ['rp2040']) + exs = s['family_examples']['rp2040'] + self.assertIn('device/cdc_msc', exs) + self.assertFalse(any(e.startswith(('host/', 'typec/')) for e in exs)) + # dual inclusion asserted on the pure role helper: whether a dual example + # survives Task 4's buildability pruning depends on the environment-gated + # CI board pick, so the classifier-output assertion must not rely on it + self.assertIn('dual/host_info_to_device_cdc', + ci_select.role_examples(REPO, ('device', 'dual'))) + self.assertNotIn('host/bare_api', ci_select.role_examples(REPO, ('device', 'dual'))) + + def test_port_host_rule(self): # rule 4 + s = self.b(['src/portable/analog/max3421/hcd_max3421.c']) + self.assertFalse(s['full']) + # rp2040's family.cmake unconditionally lists hcd_max3421.c as a source of its + # tinyusb_host_max3421 INTERFACE lib (linked only when MAX3421_HOST=1, e.g. the + # real feather_rp2040_max3421 board) and espressif's component CMakeLists also + # references it — so the raw (unpruned) scan legitimately finds both; Task 4's + # buildability post-filter is what may later prune either away + self.assertLessEqual(set(s['families']), {'espressif', 'rp2040'}) + for exs in s['family_examples'].values(): + self.assertFalse(any(e.startswith(('device/', 'typec/')) for e in exs)) + + def test_port_shared_file_selects_all_examples(self): # rule 5 + s = self.b(['src/portable/synopsys/dwc2/dwc2_common.c']) + self.assertFalse(s['full']) + self.assertIn('stm32f4', s['families']) + self.assertNotIn('rp2040', s['families']) + self.assertNotIn('stm32f4', s['family_examples']) # 'all' => no map key + + def test_bsp_family_rule(self): # rule 6 + s = self.b(['hw/bsp/stm32f4/boards/stm32f407disco/board.h']) + self.assertEqual(s['families'], ['stm32f4']) + self.assertNotIn('stm32f4', s['family_examples']) + + def test_bsp_top_level_file_is_full(self): # rule 16 + self.assertTrue(self.b(['hw/bsp/board.c'])['full']) + self.assertTrue(self.b(['hw/bsp/family_support.cmake'])['full']) + + def test_mcu_rule(self): # rule 7 + s = self.b(['hw/mcu/nordic/nrf5x/nrf_clock.h']) + self.assertEqual(s['families'], ['nrf']) + # empty means empty: no family's build references the path, so no build + # compiles it - nothing to select + s = self.b(['hw/mcu/no_such_vendor/x.c']) + self.assertFalse(s['full']) + self.assertEqual(s['families'], []) + self.assertEqual(s['family_examples'], {}) + + def test_class_device_rule(self): # rule 8 + s = self.b(['src/class/cdc/cdc_device.c']) + self.assertFalse(s['full']) + # near-all families (Task 4's pruning may drop a few); never equality + # against all_bsp_families — that's a tuple, and pruning shrinks the list + self.assertIn('stm32f4', s['families']) + self.assertGreater(len(s['families']), 50) + exs = s['family_examples']['stm32f4'] + self.assertIn('device/cdc_msc', exs) + self.assertNotIn('device/hid_composite', exs) + self.assertNotIn('host/cdc_msc_hid', exs) # TUH_CDC examples are rule 9's + + def test_class_host_rule(self): # rule 9 + s = self.b(['src/class/msc/msc_host.c']) + exs = s['family_examples']['stm32f4'] + self.assertIn('host/msc_file_explorer', exs) + self.assertNotIn('device/cdc_msc', exs) + + def test_class_shared_header_and_include_edge(self): # rule 10 + s = self.b(['src/class/audio/audio.h']) + exs = s['family_examples']['stm32f4'] + self.assertIn('device/audio_test', exs) + self.assertIn('device/midi_test', exs) # midi headers include audio.h + + def test_core_device_rule(self): # rule 11 + s = self.b(['src/device/usbd.c']) + exs = s['family_examples']['stm32f4'] + self.assertIn('device/cdc_msc', exs) + # no dual In-assertion: dual examples are only.txt-gated to max3421/pio-usb + # boards, so pruning legitimately drops them on a plain stm32f4 board + self.assertFalse(any(e.startswith(('host/', 'typec/')) for e in exs)) + + def test_core_host_rule(self): # rule 12 + s = self.b(['src/host/usbh.c']) + exs = s['family_examples']['stm32f4'] + self.assertFalse(any(e.startswith(('device/', 'typec/')) for e in exs)) + + def test_example_rule(self): # rules 13, 14 + s = self.b(['examples/device/cdc_msc/src/main.c']) + self.assertEqual(s['family_examples']['stm32f4'], ['device/cdc_msc']) + s = self.b(['examples/device/board_test/src/main.c']) + self.assertEqual(s['family_examples']['stm32f4'], ['device/board_test']) + s = self.b(['examples/device/no_such_example/src/main.c']) # deleted example: nothing + self.assertFalse(s['full']) + self.assertEqual(s['families'], []) + + def test_full_paths(self): # rules 15-17 + for p in ('src/common/tusb_fifo.c', 'src/osal/osal.h', 'src/tusb.c', + 'src/tusb_option.h', + 'tools/build.py', 'tools/cmake/cpu/cortex-m4.cmake', + 'examples/CMakeLists.txt', 'examples/device/CMakeLists.txt', + 'examples/build_system/cmake/cpu.cmake', '.github/workflows/build.yml', + 'sonar-project.properties', 'some/unknown/path.c'): + self.assertTrue(self.b([p])['full'], p) + + def test_mixed_diff_unions_per_family(self): + s = self.b(['src/portable/raspberrypi/rp2040/dcd_rp2040.c', 'src/class/cdc/cdc_device.c']) + self.assertFalse(s['full']) + self.assertIn('stm32f4', s['families']) + self.assertGreater(len(s['families']), 50) + self.assertIn('device/hid_composite', s['family_examples']['rp2040']) # from the dcd rule + self.assertNotIn('device/hid_composite', s['family_examples']['stm32f4']) # cdc-only there + + def test_example_names_are_real_dirs(self): + for ex in ci_select.all_examples(REPO): + role, name = ex.split('/') + self.assertTrue(os.path.isdir(os.path.join(REPO, 'examples', role, name)), ex) + self.assertRegex(ex, r'^(device|dual|host|typec)/[A-Za-z0-9_]+$') + + +class TestBuildPostFilter(unittest.TestCase): + def test_kept_examples_are_buildable(self): + import build_utils, build as build_py + s = ci_select.classify_build(['src/class/msc/msc_host.c'], REPO) + self.assertFalse(s['full']) + # families that cannot build a single TUH_MSC example drop out entirely + self.assertNotIn('msp430', s['families']) + old = os.getcwd() + os.chdir(REPO) + try: + # buildable on SOME board of the family - CircleCI builds them all + for fam, exs in s['family_examples'].items(): + boards = build_py.get_family_boards(fam, False, False) + for e in exs: + self.assertTrue(any(not build_utils.skip_example(e, b) for b in boards), + f'{fam}: {e}') + finally: + os.chdir(old) + + def test_unfiltered_family_has_no_map_key(self): + s = ci_select.classify_build(['hw/bsp/stm32f4/family.c'], REPO) + self.assertEqual(s['families'], ['stm32f4']) + self.assertEqual(s['family_examples'], {}) + + def test_espressif_prunes_to_what_its_build_path_can_build(self): + # build.py's espressif branch builds get_examples('espressif') only (the + # *_freertos examples plus a short extra list), so keeping espressif for a + # device/mtp diff spins CircleCI's most expensive leg up to skip everything + s = ci_select.classify_build(['examples/device/mtp/src/main.c'], REPO) + self.assertFalse(s['full']) + self.assertNotIn('espressif', s['families']) + + def test_espressif_survives_an_example_it_does_build(self): + s = ci_select.classify_build(['examples/device/cdc_msc_freertos/src/main.c'], REPO) + self.assertFalse(s['full']) + self.assertIn('espressif', s['families']) + + def test_ra_survives_the_dual_example_prune(self): + # ra's only buildable dual example is gated on only.txt's mcu:ra6m5, which + # exists only if the ${MCU_VARIANT} token in FAMILY_MCUS resolves + s = ci_select.classify_build( + ['examples/dual/host_info_to_device_cdc/src/main.c'], REPO) + self.assertFalse(s['full']) + self.assertIn('ra', s['families'], s['families']) + + def test_deleted_family_dir_does_not_crash(self): + # rule 6 extracts a family from the path; a PR that deletes or renames + # hw/bsp/ used to traceback in get_family_boards' scandir + s = ci_select.classify_build(['hw/bsp/no_such_family_xyz/family.cmake'], REPO) + self.assertFalse(s['full']) + self.assertEqual(s['families'], []) + self.assertTrue(any('gone from tree' in r for r in s['reasons']), s['reasons']) + + def test_class_source_selecting_nothing_selects_nothing(self): + # synthetic class-with-no-enabling-config case (vendor_host.c was the live + # instance until its removal): no config enables CFG_TUH_VENDOR, so + # nothing exercises it and nothing builds - empty means empty (maintainer + # decision; the file is still parsed by every full master-push build, which is + # the accepted net for a break outside its #if guard) + s = ci_select.classify_build(['src/class/vendor/vendor_host.c'], REPO) + self.assertFalse(s['full']) + self.assertEqual(s['families'], []) + self.assertTrue(any('no contribution' in r for r in s['reasons']), s['reasons']) + + def test_class_source_with_examples_still_scopes(self): + s = ci_select.classify_build(['src/class/cdc/cdc_device.c'], REPO) + self.assertFalse(s['full']) + + def test_no_stdout_pollution(self): + # get_family_boards prints on odd families; the selector's stdout is JSON + import io, contextlib + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + ci_select.classify_build(['src/class/msc/msc_host.c'], REPO) + self.assertEqual(buf.getvalue(), '') + + +class TestNoContributionPaths(unittest.TestCase): + """Paths that are inside build.yml's code filter but cannot change a compiled byte. + Unclassified means FULL on both axes, so a metrics-only PR would otherwise cost the + whole build matrix plus an exclusive full-rig sweep - where master ran nothing.""" + + def test_metrics_scripts_run_on_no_board_but_still_build(self): + # HIL axis only. tools/metrics.py IS executed by a build - examples/CMakeLists.txt + # makes it the `tinyusb_metrics` target and build_util.yml adds + # `--target tinyusb_metrics` - so the build axis must keep exercising it, or a + # break merges green and reds the next master push. Nothing on the rig runs it. + for p in ('tools/metrics.py', '.github/scripts/metrics_pair_compare.py'): + h = sel([p]) + self.assertFalse(h['full'], p) + self.assertEqual(h['boards'], {}, p) + self.assertTrue(ci_select.classify_build([p], REPO)['full'], p) + + def test_typec_example_builds_but_runs_nothing(self): + # examples/typec is compiled by the build matrix and run by no rig board; the + # HIL walk used to not recognise the role at all -> unclassified -> full rig + p = 'examples/typec/power_delivery/src/main.c' + h = sel([p]) + self.assertFalse(h['full']) + self.assertEqual(h['boards'], {}) + b = ci_select.classify_build([p], REPO) + self.assertFalse(b['full']) + self.assertTrue(b['families'], 'typec still has to be compiled somewhere') + + +class TestHilExamples(unittest.TestCase): + def test_board_test_always_present_and_full_emits(self): + s = ci_select.classify(['src/common/tusb_fifo.c'], REPO, ROSTERS) # full + he = ci_select.hil_examples(s, ROSTERS) + self.assertEqual(set(he), {b['name'] for b in ROSTER}) + for name, exs in he.items(): + self.assertIn('device/board_test', exs) + + def test_narrowed_board_gets_chosen_tests_only(self): + s = ci_select.classify(['examples/device/cdc_msc/src/main.c'], REPO, ROSTERS) + he = ci_select.hil_examples(s, ROSTERS) + self.assertEqual(he['stm32f407disco'], ['device/board_test', 'device/cdc_msc']) + + def test_full_board_gets_its_whole_test_list(self): + s = ci_select.classify(['hw/bsp/stm32f4/boards/stm32f407disco/board.h'], REPO, ROSTERS) + he = ci_select.hil_examples(s, ROSTERS) + want = set(ci_select.board_tests(ROSTER[1])) | {'device/board_test'} + self.assertEqual(set(he['stm32f407disco']), want) + self.assertNotIn('raspberry_pi_pico', he) # deselected board: no firmware needed + + +class TestHilExamplesDuplicateRosters(unittest.TestCase): + """Rosters are disjoint today, but a board moved between rigs (or listed on both + during a migration) must get the UNION of its test lists: superset firmware is + harmless, a missing image fails the run on whichever rig lost the coin toss.""" + + ROSTERS = [ + ('test/hil/a.json', [{'name': 'dup_board', 'uid': 'd1', 'flasher': {'name': 'jlink'}, + 'tests': {'only': ['device/cdc_msc']}}]), + ('test/hil/b.json', [{'name': 'dup_board', 'uid': 'd1', 'flasher': {'name': 'jlink'}, + 'tests': {'only': ['device/hid_boot_interface']}}]), + ] + + def test_duplicate_board_unions_the_test_lists(self): + he = ci_select.hil_examples({'full': True, 'boards': {}}, self.ROSTERS) + self.assertEqual(he['dup_board'], + ['device/board_test', 'device/cdc_msc', + 'device/hid_boot_interface']) + + +class TestCliJson(unittest.TestCase): + def test_build_key_without_rosters(self): + r = subprocess.run([sys.executable, os.path.join(REPO, 'tools/ci_select.py'), + '--diff-file', '/dev/null'], capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + j = json.loads(r.stdout) + self.assertIn('build', j) + self.assertNotIn('hil_examples', j) # rosters not given + + def test_build_and_hil_keys_with_rosters(self): + import tempfile + with tempfile.NamedTemporaryFile('w', suffix='.txt', delete=False) as f: + f.write('src/portable/raspberrypi/rp2040/dcd_rp2040.c\n') + df = f.name + r = subprocess.run([sys.executable, os.path.join(REPO, 'tools/ci_select.py'), + '--diff-file', df, os.path.join(REPO, 'test/hil/tinyusb.json')], + capture_output=True, text=True) + os.unlink(df) + j = json.loads(r.stdout) + self.assertEqual(j['build']['families'], ['rp2040']) + self.assertIn('hil_examples', j) + for exs in j['hil_examples'].values(): + self.assertIn('device/board_test', exs) + + +SET_MATRIX = os.path.join(REPO, '.github/scripts/ci_set_matrix.py') + + +class TestCiSetMatrix(unittest.TestCase): + def run_matrix(self, *args): + return subprocess.run([sys.executable, SET_MATRIX, *args], + capture_output=True, text=True) + + def test_no_flags_is_todays_output(self): + r = self.run_matrix() + self.assertEqual(r.returncode, 0, r.stderr) + self.baseline = json.loads(r.stdout) + self.assertIn('stm32f4', self.baseline['arm-gcc']) + + def test_select_full_is_identical(self): + base = json.loads(self.run_matrix().stdout) + sel = json.dumps({'build': {'full': True, 'families': [], 'family_examples': {}}}) + self.assertEqual(json.loads(self.run_matrix('--select', sel).stdout), base) + + def test_select_narrow_is_a_subset(self): + sel = json.dumps({'build': {'full': False, 'families': ['rp2040', 'stm32f4'], + 'family_examples': {}}}) + m = json.loads(self.run_matrix('--select', sel).stdout) + self.assertEqual(m['arm-gcc'], ['rp2040', 'stm32f4']) + self.assertEqual(m['riscv-gcc'], []) + self.assertEqual(set(m), set(json.loads(self.run_matrix().stdout))) # all keys kept + + def test_malformed_select_falls_open(self): + base = json.loads(self.run_matrix().stdout) + r = self.run_matrix('--select', 'not json {') + self.assertEqual(r.returncode, 0) + self.assertEqual(json.loads(r.stdout), base) + self.assertIn('ci_set_matrix: UNSCOPED', r.stderr) # build.yml greps this + + def test_wrong_shaped_select_falls_open_too(self): + # valid JSON, wrong types: the matrix is built AFTER main()'s try/except, so an + # AttributeError here reds the step - the very outcome that handler exists to + # prevent (GHA and CircleCI only survive it through their own shell `||`) + base = json.loads(self.run_matrix().stdout) + for bad in ('{"build": ["stm32f4"]}', '{"build": {"full": false}}', + '{"build": {"full": false, "families": "stm32f4"}}', '["stm32f4"]'): + r = self.run_matrix('--select', bad) + self.assertEqual(r.returncode, 0, f'{bad}: {r.stderr}') + self.assertEqual(json.loads(r.stdout), base, bad) + + def test_base_flag_with_empty_diff_selects_nothing(self): + # --base HEAD => empty diff => build.families [] => every toolchain scopes to [] + base = json.loads(self.run_matrix().stdout) + r = self.run_matrix('--base', 'HEAD') + self.assertEqual(r.returncode, 0, r.stderr) + m = json.loads(r.stdout) + self.assertEqual(set(m), set(base)) + self.assertTrue(all(v == [] for v in m.values()), m) + + def test_select_file_matches_select(self): + # build.yml hands the selection over as a FILE: a ~128KiB step env var makes + # the step's own exec fail with E2BIG before any fallback can run + import tempfile + sel = json.dumps({'build': {'full': False, 'families': ['rp2040'], + 'family_examples': {}}}) + with tempfile.NamedTemporaryFile('w', suffix='.json', delete=False) as f: + f.write(sel) + path = f.name + try: + self.assertEqual(self.run_matrix('--select-file', path).stdout, + self.run_matrix('--select', sel).stdout) + finally: + os.unlink(path) + + def test_absent_families_key_falls_open(self): + # `{"build": {"full": false}}` with no families key is an unusable selection, + # not "nothing selected": scoping every toolchain to [] would report a + # vacuous green with zero families built + base = json.loads(self.run_matrix().stdout) + r = self.run_matrix('--select', json.dumps({'build': {'full': False}})) + self.assertEqual(r.returncode, 0) + self.assertEqual(json.loads(r.stdout), base) + self.assertIn('ci_set_matrix: UNSCOPED', r.stderr) # build.yml greps this + + def test_explicit_empty_families_selects_nothing(self): + # an explicit [] IS a legitimate answer (a diff that builds nothing) + r = self.run_matrix('--select', + json.dumps({'build': {'full': False, 'families': []}})) + self.assertEqual(r.returncode, 0) + self.assertEqual(set().union(*json.loads(r.stdout).values()), set()) + + def test_missing_select_file_falls_open(self): + base = json.loads(self.run_matrix().stdout) + r = self.run_matrix('--select-file', '/no/such/selection.json') + self.assertEqual(r.returncode, 0) + self.assertEqual(json.loads(r.stdout), base) + self.assertIn('ci_set_matrix: UNSCOPED', r.stderr) # build.yml greps this + + def test_base_flag_bad_ref_falls_open(self): + base = json.loads(self.run_matrix().stdout) + r = self.run_matrix('--base', 'no-such-ref-xyz') + self.assertEqual(r.returncode, 0) + self.assertEqual(json.loads(r.stdout), base) + self.assertIn('ci_set_matrix: UNSCOPED', r.stderr) # build.yml greps this + + +HIL_SET_MATRIX = os.path.join(REPO, '.github/scripts/hil_ci_set_matrix.py') + + +class TestHilCiSetMatrixExamples(unittest.TestCase): + def run_matrix(self, *args): + r = subprocess.run([sys.executable, HIL_SET_MATRIX, *args, + os.path.join(REPO, 'test/hil/tinyusb.json')], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stderr) + return r.stdout + + def test_no_hil_examples_is_byte_identical(self): + plain = self.run_matrix() + sel = json.dumps({'full': True, 'boards': {}}) + self.assertEqual(self.run_matrix('--select', sel), plain) + + def test_absent_boards_key_falls_open_to_the_full_roster(self): + # the mirror of ci_set_matrix's families guard: reading an ABSENT boards key as + # "nothing selected" filters every board out, so every hil-build leg skips and + # both rig jobs skip through needs: - an all-green PR with zero hardware + # coverage. An explicit boards: {} stays a legitimate nothing-selected. + plain = self.run_matrix() + for bad in ('{"full": false, "hil_examples": {}}', '{"full": false, "boards": []}', + 'not json {', '["a board"]', + # the whole selection is unusable, hil_examples included: keeping the + # -e lists builds a few examples per board while the rig, unfiltered, + # runs that board's whole test list + '{"full": false, "hil_examples": {"frdm_k64f": ["device/cdc_msc"]}}'): + self.assertEqual(self.run_matrix('--select', bad), plain, bad) + self.assertNotEqual(self.run_matrix('--select', '{"full": false, "boards": {}}'), + plain, 'an explicit empty boards map still means nothing') + + def test_select_file_matches_select(self): + # hil-hfp-iar passes the whole selection; as one argv it can exceed + # MAX_ARG_STRLEN on a big diff, so the file form must be equivalent + import tempfile + board = on_roster(self, 'stm32f407disco')[0] + sel = json.dumps({'full': False, 'boards': {board: 'all'}, + 'hil_examples': {board: ['device/board_test']}}) + with tempfile.NamedTemporaryFile('w', suffix='.json', delete=False) as f: + f.write(sel) + path = f.name + try: + self.assertEqual(self.run_matrix('--select-file', path), + self.run_matrix('--select', sel)) + finally: + os.unlink(path) + + def test_examples_appended_per_board(self): + board = on_roster(self, 'stm32f407disco')[0] + sel = json.dumps({'full': False, 'boards': {board: 'all'}, + 'hil_examples': {board: ['device/board_test', 'device/cdc_msc']}}) + m = json.loads(self.run_matrix('--select', sel)) + entries = [e for entries in m.values() for e in entries] + self.assertTrue(entries) + for e in entries: + self.assertIn(f'-b {board}', e) + self.assertIn('-e device/board_test', e) + self.assertIn('-e device/cdc_msc', e) + + +class TestBuildPyExampleFilter(unittest.TestCase): + def setUp(self): + import build as build_py + self.build = build_py + self.old = os.getcwd() + os.chdir(REPO) # skip_example uses repo-relative paths + + def tearDown(self): + os.chdir(self.old) + + def test_all_maps_to_example_names(self): + # ONE group: the examples of a '--target all' build go into a single + # `cmake --build --target a b c`, so they build in parallel + t = self.build.resolve_example_target_groups(['all'], ['device/cdc_msc', 'device/dfu'], + 'stm32f407disco') + self.assertEqual(t, [['cdc_msc', 'dfu']]) + + def test_other_targets_pass_through_in_their_own_group(self): + # a target that is not 'all' keeps its own invocation, so ordering against the + # examples is preserved (tinyusb_metrics runs after them, as it did unfiltered) + t = self.build.resolve_example_target_groups(['all', 'tinyusb_metrics'], + ['device/cdc_msc'], 'stm32f407disco') + self.assertEqual(t, [['cdc_msc'], ['tinyusb_metrics']]) + + def test_unbuildable_examples_drop_and_empty_is_none(self): + # typec/power_delivery only builds on stm32g4-class parts, never on f4 + t = self.build.resolve_example_target_groups(['all'], + ['typec/power_delivery', 'device/cdc_msc'], + 'stm32f407disco') + self.assertEqual(t, [['cdc_msc']]) + self.assertIsNone(self.build.resolve_example_target_groups(['all'], + ['typec/power_delivery'], + 'stm32f407disco')) + + def test_espressif_empty_intersection_skips_without_building(self): + # cmake_board's espressif branch must short-circuit on an empty -e + # intersection the same way the generic cmake/make branches do, and + # must do so before touching idf.py (no real esp-idf build here). + calls = [] + real_run_cmd = self.build.run_cmd # `del` here would drop the real one + self.build.run_cmd = lambda cmd: calls.append(cmd) # would only run for a real build + try: + r = self.build.cmake_board('espressif_s3_devkitc', [], None, [], ['all'], + examples=['nonexistent/example']) + finally: + self.build.run_cmd = real_run_cmd + self.assertEqual(r, [0, 0, 1]) + self.assertEqual(calls, []) + + def test_make_one_example_uses_make_semantics(self): + # F1 end to end: the make path must ask skip_example with build_system='make', + # or lpc54's cmake-only FAMILY_MCUS un-skips a host example whose make build + # compiles no HCD source and fails to link + calls = [] + real_run_cmd = self.build.run_cmd + self.build.run_cmd = lambda cmd: calls.append(cmd) + try: + r = self.build.make_one_example('host/msc_file_explorer_freertos', + 'lpcxpresso54628', '', ['all']) + finally: + self.build.run_cmd = real_run_cmd + self.assertEqual(r, [0, 0, 1]) # skipped, nothing handed to make + self.assertEqual(calls, []) + + def test_example_flag_rejects_a_bare_name(self): + # `-e cdc_msc` (no role) used to IndexError inside the target resolver; + # argparse rejects the shape now, with a message that names it + r = subprocess.run([sys.executable, os.path.join(REPO, 'tools', 'build.py'), + '-b', 'stm32f407disco', '-e', 'cdc_msc'], + capture_output=True, text=True, cwd=REPO) + self.assertEqual(r.returncode, 2, r.stdout + r.stderr) + self.assertIn('role/name', r.stderr) + + def test_no_example_basename_is_reused_across_roles(self): + # -e maps role/name onto the BARE cmake target name, so device/foo and host/foo + # would collapse into one `--target foo`: one of them would never build while + # the post-configure check still reports both as covered. No collision today, + # and the -e lists are machine-generated, so nothing else would notice one. + seen = {} + for ex in ci_select.all_examples(REPO): + role, name = ex.split('/', 1) + self.assertNotIn(name, seen, + f'{ex} and {seen.get(name)}/{name} share a cmake target name; ' + f'build.py -e cannot tell them apart') + seen[name] = role + + def test_example_flag_rejects_a_name_no_example_dir_answers_to(self): + # right shape, no such dir: every board would report Skipped and the run would + # still exit 0 (main returns the FAILED count), so an entirely stale -e list - + # from the example map or from a roster test name - reads as a green build + r = subprocess.run([sys.executable, os.path.join(REPO, 'tools', 'build.py'), + '-b', 'stm32f407disco', '-e', 'device/no_such_example'], + capture_output=True, text=True, cwd=REPO) + self.assertEqual(r.returncode, 2, r.stdout + r.stderr) + self.assertIn('no such example directory', r.stderr) + + def test_pr_filter_answers_before_configuring(self): + # nothing the -e list names is buildable here: the skip.txt mirror needs no + # configure output, so the whole cmake run must be skipped, not just its build + calls = [] + real_run_cmd = self.build.run_cmd + self.build.run_cmd = lambda cmd: calls.append(cmd) + try: + r = self.build.cmake_board('stm32f407disco', [], None, [], ['all'], + examples=['typec/power_delivery']) + finally: + self.build.run_cmd = real_run_cmd + self.assertEqual(r, [0, 0, 1]) + self.assertEqual(calls, []) + + def _cmake_board_with_targets(self, registered, examples): + """cmake_board with the configure/build stubbed and CMake's registered-target + list forced. Returns (result, target names handed to `cmake --build`).""" + class Ok: + returncode = 0 + calls = [] + + def fake_run(cmd): + calls.append(cmd) + return Ok() + real_run_cmd = self.build.run_cmd + real_targets = self.build.cmake_registered_targets + self.build.run_cmd = fake_run + self.build.cmake_registered_targets = lambda d: registered + try: + r = self.build.cmake_board('stm32f407disco', [], None, [], ['all'], + examples=examples) + finally: + self.build.run_cmd = real_run_cmd + self.build.cmake_registered_targets = real_targets + # everything after --target: one invocation carries the whole group + built = [c[c.index('--target') + 1:] for c in calls if '--target' in c] + return r, built + + def test_example_without_a_cmake_target_is_dropped(self): + # an example dir CMake never registered (absent from the role CMakeLists, or + # a stale roster name) must not reach `cmake --build --target `: that is a + # hard red, and skip.txt cannot see it + r, built = self._cmake_board_with_targets({'cdc_msc'}, + ['device/cdc_msc', 'device/dfu']) + self.assertEqual(built, [['cdc_msc']]) + self.assertEqual(r, [1, 0, 0]) + + def test_the_selected_examples_build_in_one_invocation(self): + # one `cmake --build --target a b c`, not one invocation per example: the + # per-example loop serialised every scoped leg, and hil-build gets an -e list + # on EVERY PR (~14 examples per board), so it is on the critical path to the rig + r, built = self._cmake_board_with_targets({'cdc_msc', 'dfu', 'hid_generic_inout'}, + ['device/cdc_msc', 'device/dfu', + 'device/hid_generic_inout']) + self.assertEqual(built, [['cdc_msc', 'dfu', 'hid_generic_inout']]) + + def test_no_registered_target_at_all_skips_the_build(self): + r, built = self._cmake_board_with_targets({'cdc_msc'}, ['device/dfu']) + self.assertEqual(built, []) + self.assertEqual(r, [0, 0, 1]) + + def test_unparseable_target_help_keeps_the_skip_txt_answer(self): + # ground truth unavailable (a non-Ninja generator, an old cmake): fall back + # to the mirror rather than dropping every example + r, built = self._cmake_board_with_targets(None, ['device/cdc_msc']) + self.assertEqual(built, [['cdc_msc']]) + + def test_target_help_parse(self): + text = ('[1/1] All primary targets available:\n' + 'tinyusb_metrics: phony\n' + 'cdc_msc: phony\n' + 'cdc_msc-membrowse-upload: phony\n' + 'device/edit_cache: phony\n' + '/abs/build/device/cdc_msc/CMakeFiles/cdc_msc-jlink: CUSTOM_COMMAND\n') + self.assertEqual(self.build.parse_target_help(text), + {'tinyusb_metrics', 'cdc_msc', 'cdc_msc-membrowse-upload'}) + + def test_build_defines_reach_the_example_filter(self): + # metro_m4_express gets MAX3421_HOST=1 from the roster build args, never + # from its BSP: without threading them through, -e drops the rig's only + # MAX3421 dual firmware that --target all used to build + self.assertIsNone(self.build.resolve_example_target_groups( + ['all'], ['dual/host_info_to_device_cdc'], 'metro_m4_express')) + self.assertEqual(self.build.resolve_example_target_groups( + ['all'], ['dual/host_info_to_device_cdc'], 'metro_m4_express', + extra_defines=('MAX3421_HOST=1',)), [['host_info_to_device_cdc']]) + + + +class TestFamilyMcusFallback(unittest.TestCase): + """A family whose family.cmake sets FAMILY_MCUS only inside if() blocks gets its + whole MCU answer from _board_mcu's CFG_TUSB_MCU scrape (build_utils._family_mcus + does not evaluate cmake conditionals). For mcx that answer is load-bearing - six + examples' skip.txt name mcu:MCXA15 - and it comes out right only because every + mcx board still carries the token in a make-only board.mk the scrape falls + through to. A board.cmake-only board (MCU_VARIANT, no CFG_TUSB_MCU) would scrape + 'NONE' and silently skip EVERY example on it, in CI as well as in -e.""" + + @staticmethod + def conditional_only_families(): + """hw/bsp/ dirs whose family.cmake has no unconditional + set(FAMILY_MCUS ...) - computed, not listed, so a family that grows or loses + one moves in and out of this guard on its own.""" + import build_utils + out = [] + for fc in sorted(glob.glob(os.path.join(REPO, 'hw/bsp/*/family.cmake'))): + depth, uncond = 0, False + for line in open(fc).read().splitlines(): + line = line.strip() + if build_utils._FAMILY_MCUS_RE.match(line) and depth == 0: + uncond = True + if re.match(r'if\s*\(', line): + depth += 1 + elif re.match(r'endif\s*\(', line): + depth = max(0, depth - 1) + if not uncond: + out.append(os.path.dirname(fc)) + return out + + def test_every_board_of_such_a_family_scrapes_an_mcu(self): + import build_utils + fams = self.conditional_only_families() + self.assertTrue(fams, 'no family sets FAMILY_MCUS conditionally any more') + for fam_dir in fams: + fam = os.path.basename(fam_dir) + for bd in sorted(glob.glob(os.path.join(fam_dir, 'boards', '*'))): + if not os.path.isdir(bd): + continue + mcu, _ = build_utils._board_mcu(bd, fam_dir, fam) + self.assertNotEqual( + mcu, 'NONE', + f'{fam}/{os.path.basename(bd)}: nothing to scrape a CFG_TUSB_MCU ' + f'token from, and {fam}/family.cmake sets FAMILY_MCUS only inside ' + f'if() - skip_example would skip every example on this board. Fix ' + f'by evaluating the if(MCU_VARIANT STREQUAL ...) branches.') + + +class TestMcuTokensResolve(unittest.TestCase): + """The cmake-side MCU mirror must never answer with an unexpanded ${VAR} or with + nothing at all: both make every `mcu:` token miss, which reads as 'skip' for any + example carrying an only.txt and silently drops compile coverage.""" + + @staticmethod + def _every_board(): + import build as build_py + old = os.getcwd() + os.chdir(REPO) + try: + for fam in sorted(os.path.basename(os.path.dirname(f)) + for f in glob.glob(os.path.join(REPO, 'hw/bsp/*/boards'))): + for b in build_py.get_family_boards(fam, False, False): + yield fam, b + finally: + os.chdir(old) + + def test_no_board_answers_with_an_unexpanded_variable(self): + import build_utils + for fam, board in self._every_board(): + fam_dir, board_dir = f'{REPO}/hw/bsp/{fam}', f'{REPO}/hw/bsp/{fam}/boards/{board}' + mcus = set(build_utils._family_mcus(fam_dir, board_dir)) + mcus.add(build_utils._board_mcu(board_dir, fam_dir, fam)[0]) + self.assertFalse([m for m in mcus if '${' in m], + f'{fam}/{board}: unexpanded cmake variable in {sorted(mcus)} - ' + f'teach build_utils._cmake_expand the construct that produces it') + self.assertTrue(mcus - {'NONE'}, + f'{fam}/{board}: no MCU name resolved at all') + + # skip.txt/only.txt tokens no board in the tree answers to: stale spellings left + # behind by a family rename. Each one silently changes what CI builds, so this list + # must only ever SHRINK - a new entry means either a live token the mirror cannot + # produce, or a rename nobody followed through. `family:samd21` was one of these + # until the nine examples/host/*/only.txt files were corrected to samd2x_l2x. + # + # The `mcu:` entries are NOT all harmless. MIMXRT10XX/MIMXRT11XX and LPC177X_8X sit + # beside a live token in the same file, so they gate nothing either way. MKL25ZXX + # (device/msc_dual_lun) and SAME5X (device/audio_test) do not: those skips are dead, + # and both examples are built today on the boards their skip file meant to exclude - + # successfully, which is why nobody noticed. Correcting them REMOVES working build + # coverage, so it is a maintainer call, not a drive-by fix. + UNREACHABLE_TOKENS = { + 'mcu': {'LPC177X_8X', 'MIMXRT10XX', 'MIMXRT11XX', 'MKL25ZXX', 'SAME5X', 'STM32U3'}, + 'family': set(), + 'board': set(), + } + + def test_every_skip_only_token_is_reachable(self): + import build_utils + wanted = {ns: set() for ns in self.UNREACHABLE_TOKENS} + for f in glob.glob(os.path.join(REPO, 'examples/*/*/*.txt')): + if os.path.basename(f) in ('skip.txt', 'only.txt'): + for tok in open(f).read().split(): + ns, _, name = tok.partition(':') + if ns in wanted and name: + wanted[ns].add(name) + have = {ns: set() for ns in wanted} + have['mcu'].add('MAX3421') # synthetic, from family_support.cmake:940 + for fam, board in self._every_board(): + fam_dir, board_dir = f'{REPO}/hw/bsp/{fam}', f'{REPO}/hw/bsp/{fam}/boards/{board}' + have['family'].add(fam) + have['board'].add(board) + have['mcu'] |= set(build_utils._family_mcus(fam_dir, board_dir)) + have['mcu'].add(build_utils._board_mcu(board_dir, fam_dir, fam)[0]) + have['mcu'].add(build_utils._scrape_mcu(pathlib.Path(fam_dir), + pathlib.Path(board_dir), fam)[0]) # make + for ns in wanted: + self.assertEqual( + wanted[ns] - have[ns], self.UNREACHABLE_TOKENS[ns] & wanted[ns], + f'a skip.txt/only.txt {ns}: token nothing in hw/bsp answers to. Either ' + f'the token is stale (a rename just changed what CI builds), or the ' + f'mirror cannot produce it - both silently skip that example everywhere.') + + def test_the_mcx_skip_tokens_are_still_live(self): + # the reason the mcx scrape is load-bearing rather than academic + named = [os.path.dirname(f) for f in glob.glob(os.path.join(REPO, 'examples/*/*/skip.txt')) + if 'mcu:MCXA15' in open(f).read().split()] + self.assertTrue(named, 'no skip.txt names mcu:MCXA15 any more') + + +class TestSkipExampleMirrorsFamilyFilter(unittest.TestCase): + """build_utils.skip_example is the python mirror of CMake's family_filter + (hw/bsp/family_support.cmake:171-207). family_filter loops over the whole + FAMILY_MCUS list; a per-board CFG_TUSB_MCU scrape alone lets -e ask for a + target CMake never created, and `cmake --build --target ` hard-fails.""" + + def setUp(self): + import build_utils + self.build_utils = build_utils + self.old = os.getcwd() + os.chdir(REPO) # skip_example uses repo-relative paths + + def tearDown(self): + os.chdir(self.old) + + def test_any_family_mcu_can_skip(self): + # broadcom_64bit: set(FAMILY_MCUS BCM2711 BCM2835); raspberrypi_cm4 is + # BCM2711, and examples/device/dfu/skip.txt lists mcu:BCM2835 + self.assertTrue(self.build_utils.skip_example('device/dfu', 'raspberrypi_cm4')) + + def test_any_family_mcu_can_satisfy_only(self): + # lpc55: family.mk says LPC55XX, family.cmake sets FAMILY_MCUS LPC55, and + # host/cdc_msc_hid/only.txt lists mcu:LPC55 - CMake builds it + self.assertFalse(self.build_utils.skip_example('host/cdc_msc_hid', 'lpcxpresso55s69')) + + def test_existing_decisions_are_unchanged(self): + self.assertFalse(self.build_utils.skip_example('device/cdc_msc', 'stm32f407disco')) + self.assertTrue(self.build_utils.skip_example('typec/power_delivery', 'stm32f407disco')) + + def test_build_define_enables_max3421_only_list(self): + # family_support.cmake:940 appends MAX3421 to FAMILY_MCUS when + # MAX3421_HOST=1; on metro_m4_express that define comes from the roster + # build args, so skip_example has to be told about it + ex = 'dual/host_info_to_device_cdc' + self.assertTrue(self.build_utils.skip_example(ex, 'metro_m4_express')) + self.assertFalse(self.build_utils.skip_example(ex, 'metro_m4_express', + extra_defines=('MAX3421_HOST=1',))) + + def test_family_mcus_variable_token_resolves(self): + """hw/bsp/ra/family.cmake: `set(FAMILY_MCUS RAXXX ${MCU_VARIANT})`, and + ra6m5_ek/board.cmake sets MCU_VARIANT ra6m5 — which is exactly the token + dual/host_info_to_device_cdc/only.txt spells (mcu:ra6m5). Dropping the + ${...} token silently removed ra from every scoped dual-example build.""" + self.assertFalse(self.build_utils.skip_example( + 'dual/host_info_to_device_cdc', 'ra6m5_ek')) + + def test_board_cmake_max3421_counts(self): + """feather_rp2040_max3421/board.cmake sets MAX3421_HOST 1 while the MCU + token comes from rp2040's family.cmake; scanning only the file the token + came from misses it, and only.txt's mcu:MAX3421 never matches.""" + self.assertFalse(self.build_utils.skip_example( + 'host/cdc_msc_hid_freertos', 'feather_rp2040_max3421')) + + +class TestSkipExampleMakeSemantics(unittest.TestCase): + """FAMILY_MCUS is a CMAKE fact. hw/bsp/lpc54/family.cmake sets it to LPC54 and + wires the ohci host sources; family.mk builds OPT_MCU_LPC54XXX and compiles no + HCD source at all — so applying the cmake MCU union to a Make build un-skips + the 9 host examples only.txt gates on mcu:LPC54 and they fail to link + (undefined reference to hcd_init). Make keeps master's exact algorithm.""" + + def setUp(self): + import build_utils + self.build_utils = build_utils + self.old = os.getcwd() + os.chdir(REPO) # skip_example uses repo-relative paths + + def tearDown(self): + os.chdir(self.old) + + def test_make_keeps_cmake_only_family_mcus_out(self): + self.assertTrue(self.build_utils.skip_example( + 'host/msc_file_explorer_freertos', 'lpcxpresso54628', build_system='make')) + + def test_make_does_not_skip_on_a_sibling_family_mcu(self): + # broadcom_64bit sets FAMILY_MCUS "BCM2711 BCM2835"; raspberrypi_cm4 is the + # BCM2711 one and device/dfu/skip.txt names mcu:BCM2835. The aarch64 make leg + # built device/dfu before the union and must keep building it. + for ex in ('device/dfu', 'device/usbtmc'): + self.assertFalse(self.build_utils.skip_example( + ex, 'raspberrypi_cm4', build_system='make'), ex) + + def test_cmake_is_the_default_and_still_unions(self): + self.assertTrue(self.build_utils.skip_example('device/dfu', 'raspberrypi_cm4')) + self.assertEqual( + self.build_utils.skip_example('device/dfu', 'raspberrypi_cm4'), + self.build_utils.skip_example('device/dfu', 'raspberrypi_cm4', + build_system='cmake')) + + def test_build_system_is_part_of_the_cache_key(self): + # one lru_cache shared by both semantics would answer the second caller + # with the first caller's verdict + ex, board = 'host/msc_file_explorer_freertos', 'lpcxpresso54628' + self.assertFalse(self.build_utils.skip_example(ex, board, build_system='cmake')) + self.assertTrue(self.build_utils.skip_example(ex, board, build_system='make')) + self.assertFalse(self.build_utils.skip_example(ex, board, build_system='cmake')) + + +class TestConfigEnables(unittest.TestCase): + """_config_enables decides which examples a class change selects, on BOTH the + build and the HIL axis. A define it cannot evaluate must read as ON: reading + it as OFF is fail-closed, and lets a compile break merge green.""" + + def test_identifier_value_is_enabled(self): + # examples/host/midi_rx: `#define CFG_TUH_MIDI CFG_TUH_DEVICE_MAX` + cfg = os.path.join(REPO, 'examples/host/midi_rx/src/tusb_config.h') + self.assertTrue(ci_select._config_enables(cfg, ['CFG_TUH_MIDI'])) + + def test_literal_zero_is_disabled(self): + import tempfile + with tempfile.TemporaryDirectory() as td: + cfg = os.path.join(td, 'tusb_config.h') + with open(cfg, 'w') as f: + f.write('#define CFG_TUD_CDC 0\n' + '#define CFG_TUD_MSC (0)\n' + '#define CFG_TUD_HID 00\n' + '#define CFG_TUH_HID 0 // typical keyboard + mouse\n' + '#define CFG_TUD_MIDI 01\n' + '#define CFG_TUD_DFU (1)\n') + for m in ('CFG_TUD_CDC', 'CFG_TUD_MSC', 'CFG_TUD_HID', 'CFG_TUH_HID'): + self.assertFalse(ci_select._config_enables(cfg, [m]), m) + for m in ('CFG_TUD_MIDI', 'CFG_TUD_DFU'): + self.assertTrue(ci_select._config_enables(cfg, [m]), m) + self.assertFalse(ci_select._config_enables(cfg, ['CFG_TUD_VIDEO'])) + + def test_two_branch_define_reads_on(self): + # examples/device/uac2_speaker_fb defines CFG_TUD_HID 1 under + # `#if CFG_AUDIO_DEBUG` and 0 in the #else. The default build (CFG_AUDIO_DEBUG + # defaults to 1) compiles the HID class in, so a CFG_TUD_HID change must keep + # this example on both axes - the #else's zero must not decide it. + cfg = os.path.join(REPO, 'examples/device/uac2_speaker_fb/src/tusb_config.h') + self.assertTrue(ci_select._config_enables(cfg, ['CFG_TUD_HID'])) + + def test_any_nonzero_define_wins_over_a_zero_one(self): + import tempfile + with tempfile.TemporaryDirectory() as td: + cfg = os.path.join(td, 'tusb_config.h') + with open(cfg, 'w') as f: + f.write('#if FOO\n#define CFG_TUD_MSC 1\n#else\n' + '#define CFG_TUD_MSC 0\n#endif\n' + '#if BAR\n#define CFG_TUD_CDC 0\n#else\n' + '#define CFG_TUD_CDC (0)\n#endif\n') + self.assertTrue(ci_select._config_enables(cfg, ['CFG_TUD_MSC'])) + self.assertFalse(ci_select._config_enables(cfg, ['CFG_TUD_CDC'])) + + def test_midi_host_change_selects_midi_rx(self): + s = ci_select.classify_build(['src/class/midi/midi_host.c'], REPO) + self.assertFalse(s['full']) + self.assertTrue(s['families'], 'a TUH_MIDI change must select some family') + self.assertTrue(any('host/midi_rx' in exs + for exs in s['family_examples'].values()), + s['family_examples']) + + +class TestPruneUsesEveryFamilyBoard(unittest.TestCase): + """CircleCI's cmake legs build EVERY board of a family, so an example gated to + one board (only.txt board:mimxrt1060_evk) must keep its family even though the + family's one-first board cannot build it.""" + + def test_board_gated_example_keeps_its_family(self): + s = ci_select.classify_build( + ['examples/dual/host_hid_to_device_cdc/src/main.c'], REPO) + self.assertFalse(s['full']) + self.assertIn('imxrt', s['families'], s['families']) + self.assertEqual(s['family_examples'].get('imxrt'), + ['dual/host_hid_to_device_cdc']) + + def test_either_build_system_keeps_the_family(self): + """This one family list gates CircleCI's MAKE legs too, and the two build + systems answer skip.txt differently. device/dfu carries mcu:BCM2835, which the + cmake FAMILY_MCUS union (BCM2711 BCM2835) applies to every broadcom_64bit board + and the make scrape applies to none - asking cmake alone drops the only + aarch64-gcc family in the matrix, so build-make-aarch64-gcc silently stops + compiling dfu at all.""" + import build_utils + old = os.getcwd() + os.chdir(REPO) + try: + self.assertTrue(build_utils.skip_example('device/dfu', 'raspberrypi_cm4')) + self.assertFalse(build_utils.skip_example('device/dfu', 'raspberrypi_cm4', + (), 'make')) + finally: + os.chdir(old) + s = ci_select.classify_build(['examples/device/dfu/src/main.c'], REPO) + self.assertIn('broadcom_64bit', s['families'], s['families']) + + +class TestPrunePoolIsBuildPys(unittest.TestCase): + """_prune_buildable asks build.py what each family's build path can see, the same + way for every family - the espressif carve-out lives in build.py.get_examples and + needs no second copy here. Measured identical on all 82 families.""" + + def setUp(self): + import build as build_py + self.build_py = build_py + self.old = os.getcwd() + os.chdir(REPO) # get_examples scans relative paths + + def tearDown(self): + os.chdir(self.old) + + def test_only_espressif_narrows_the_pool(self): + allex = list(ci_select.all_examples(REPO)) + for fam in ci_select.all_bsp_families(REPO): + pool = [e for e in allex if e in set(self.build_py.get_examples(fam))] + if fam == 'espressif': + self.assertNotEqual(pool, allex) # the carve-out is real + else: + self.assertEqual(pool, allex, f'{fam}: build.py narrows this family') + + def test_selections_are_what_the_espressif_only_rule_gave(self): + # espressif's own list is the one value that ever differed from the unfiltered + # example set. Recomputed from build.py rather than pinned as literals: a new + # board, family or example moves the counts, and a suite that fails for that + # teaches people to edit the numbers instead of reading the diff. What is pinned + # is the RELATION - espressif gets exactly the rule's answer narrowed to its own + # pool, every other family gets the answer unnarrowed. + pool = set(self.build_py.get_examples('espressif')) + # the third diff names an example espressif DOES build, so there is nothing for + # the carve-out to remove - it pins that the narrowing does not over-reach + for files, carve in ((['src/portable/synopsys/dwc2/dcd_dwc2.c'], True), + (['src/class/msc/msc_host.c'], True), + (['examples/device/cdc_msc_freertos/src/main.c'], False)): + s = ci_select.classify_build(files, REPO) + self.assertFalse(s['full'], files) + self.assertIn('espressif', s['families'], files) + esp = set(s['family_examples'].get('espressif') or []) + self.assertTrue(esp, f'{files}: espressif selected nothing') + # the pool narrowing is what _prune_buildable adds here, so it must hold... + self.assertTrue(esp <= pool, f'{files}: {sorted(esp - pool)} is outside the pool') + # ...and it must actually bite: some other family was given an example that + # espressif's build path cannot see, and espressif did not get it + other = set().union(*(set(v) for f, v in s['family_examples'].items() + if f != 'espressif'), set()) + self.assertEqual(bool(other - pool), carve, + f'{files}: carve-out expected={carve}, other-side extras ' + f'{sorted(other - pool)}') + self.assertFalse(esp & (other - pool), files) + + +class TestGetDepsExampleShim(unittest.TestCase): + """hil_ci_set_matrix emits `-b -e role/name` entries that .github/actions/ + get_deps and build.yml's hfp job hand verbatim to get_deps.py. argparse must not + reject -e there (exit 2 = every PR's Get Dependencies step red).""" + + # get_deps.main() with its process pool stubbed out: argparse runs for real, + # nothing is cloned (this suite also runs on GitHub's bare pre-commit runner) + CODE = ('import sys\n' + 'import get_deps\n' + 'class P:\n' + ' def __enter__(self): return self\n' + ' def __exit__(self, *a): return False\n' + ' def map(self, fn, items): return [0] * len(items)\n' + 'get_deps.Pool = P\n' + "sys.argv = ['get_deps.py'] + sys.argv[1:]\n" + 'sys.exit(get_deps.main())\n') + + def run_get_deps(self, *args): + env = dict(os.environ, PYTHONPATH=os.path.join(REPO, 'tools')) + return subprocess.run([sys.executable, '-c', self.CODE, *args], + capture_output=True, text=True, cwd=REPO, env=env) + + def test_example_flag_is_accepted(self): + r = self.run_get_deps('-b', 'stm32f407disco', '-e', 'device/cdc_msc') + self.assertNotIn('unrecognized arguments', r.stderr) + self.assertEqual(r.returncode, 0, r.stderr) + + def test_plain_board_still_works(self): + r = self.run_get_deps('-b', 'stm32f407disco') + self.assertEqual(r.returncode, 0, r.stderr) + + +if __name__ == '__main__': + unittest.main(verbosity=1) diff --git a/test/hil/test/test_hil_select.py b/test/hil/test/test_hil_select.py deleted file mode 100644 index 9a1261878..000000000 --- a/test/hil/test/test_hil_select.py +++ /dev/null @@ -1,689 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: MIT -# Unit tests for hil_select.py — pure logic, no hardware, no git. Run directly: -# python3 test/hil/test/test_hil_select.py -# -# Imports stay stdlib + hil_select/hil_util/hil_flash ONLY: the pre-commit hil-test -# hook runs this suite, on GitHub's bare runner in the pre-commit workflow as well as -# locally, and that runner has no pyserial/pymtp. hil_flash is admissible because it -# is stdlib + hil_util only (test_hil_util.BottomLayer enforces the stdlib closure of -# both) and the roster-dispatch tests need its flash_* table; never import hil_test, -# which pulls pyserial. -import glob -import json -import os -import sys -import unittest - -# the modules under test live in the parent dir (test/hil), not here -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -import hil_flash -from helper import hil_select -from helper.hil_util import device_tests, dual_tests - -REPO = os.path.dirname(os.path.dirname(os.path.dirname( - os.path.dirname(os.path.abspath(__file__))))) - - -def real_rosters(): - """The actual rig rosters, for regression tests that need real-world data - (a specific board/family/only-list) rather than the synthetic ROSTER above.""" - rosters = [] - for name in ('tinyusb.json', 'hfp.json'): - path = os.path.join(REPO, 'test/hil', name) - with open(path) as f: - rosters.append((f'test/hil/{name}', json.load(f)['boards'])) - return rosters - - -def roster_flashers(): - """(roster path, board) for every board in the live rosters, `boards-skip` - included: a parked board's flasher name must still dispatch, so that unparking it - is not what discovers the name went stale.""" - for name in ('tinyusb.json', 'hfp.json'): - path = os.path.join(REPO, 'test/hil', name) - with open(path) as f: - cfg = json.load(f) - for key in ('boards', 'boards-skip'): - for b in cfg.get(key, []): - yield f'test/hil/{name}', b - - -def on_roster(tc, *names): - """The subset of `names` currently in the live rig rosters, skipping the test - when none are, because parking/unparking a board is routine rig maintenance. - - That skip now matters MORE than it used to, not less: this suite is a blocking - pre-commit hook AND build.yml's selector steps gate on it (a failing suite falls - open to the full matrix), so an assertion that depends on a specific board being - present goes red on every PR -- including src/-only ones that never touched the - rig -- until someone fixes the roster. Keep roster-dependent assertions behind - on_roster.""" - have = {b['name'] for _, boards in real_rosters() for b in boards} - got = [n for n in names if n in have] - if not got: - tc.skipTest(f'not in the rig roster: {", ".join(names)}') - return got - - -ROSTER = [ - # device-only, rp2040 family - {'name': 'raspberry_pi_pico', 'uid': 'u1', 'flasher': {'name': 'openocd'}, - 'tests': {'device': True, 'host': True, 'dual': True}}, - # device-only, stm32f4 family - {'name': 'stm32f407disco', 'uid': 'u2', 'flasher': {'name': 'jlink'}, - 'tests': {'device': True, 'host': False, 'dual': False}}, - # host-only board - {'name': 'raspberry_pi_pico2', 'uid': 'u3', 'flasher': {'name': 'openocd'}, - 'tests': {'device': False, 'host': True, 'dual': False}}, - # only-list board (espressif-style), flashed by the CI leg that splits on esptool - {'name': 'espressif_s3_devkitm', 'uid': 'u4', 'flasher': {'name': 'esptool'}, - 'tests': {'only': ['device/cdc_msc_freertos', 'host/device_info']}}, -] -ROSTERS = [('test/hil/tinyusb.json', ROSTER)] - - -def sel(files): - return hil_select.classify(files, REPO, ROSTERS) - - -class TestPortRule(unittest.TestCase): - def test_dcd_rp2040_selects_pico_family_only(self): - s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) - self.assertFalse(s['full']) - self.assertIn('raspberry_pi_pico', s['boards']) - self.assertNotIn('stm32f407disco', s['boards']) - self.assertNotIn('espressif_s3_devkitm', s['boards']) - # device role: no host tests in pico's list - self.assertTrue(all(not t.startswith('host/') for t in s['boards']['raspberry_pi_pico'])) - # host-only boards drop out entirely on a device-role change - self.assertNotIn('raspberry_pi_pico2', s['boards']) - - def test_shared_port_file_is_both_roles(self): - s = sel(['src/portable/synopsys/dwc2/dwc2_common.c']) - self.assertFalse(s['full']) - self.assertNotIn('raspberry_pi_pico', s['boards']) # rp2040 is not a dwc2 family - self.assertIn('stm32f407disco', s['boards']) # stm32f4 is - - -class TestCoreRoleRule(unittest.TestCase): - def test_usbd_selects_all_device_tests_everywhere(self): - s = sel(['src/device/usbd.c']) - self.assertFalse(s['full']) - self.assertNotIn('raspberry_pi_pico2', s['boards']) # host-only board dropped - pico = s['boards']['raspberry_pi_pico'] - self.assertTrue(set(device_tests).issubset(set(pico))) - self.assertTrue(set(dual_tests).issubset(set(pico))) # dual survives device role - self.assertTrue(all(not t.startswith('host/') for t in pico)) - # only-list board: selection intersects its only-list - esp = s['boards']['espressif_s3_devkitm'] - self.assertEqual(esp, ['device/cdc_msc_freertos']) - - def test_host_change_drops_device(self): - s = sel(['src/host/usbh.c']) - self.assertFalse(s['full']) - self.assertIn('raspberry_pi_pico2', s['boards']) - self.assertNotIn('stm32f407disco', s['boards']) # device-only board dropped - - -class TestClassRule(unittest.TestCase): - def test_cdc_device_selects_cdc_examples_only(self): - s = sel(['src/class/cdc/cdc_device.c']) - self.assertFalse(s['full']) - pico = s['boards']['raspberry_pi_pico'] - self.assertIn('device/cdc_msc', pico) - self.assertIn('device/cdc_dual_ports', pico) - self.assertNotIn('device/msc_dual_lun', pico) # CFG_TUD_CDC 0 there - self.assertNotIn('device/usbtest', pico) # CFG_TUD_CDC 0 there - self.assertTrue(all(not t.startswith('host/') for t in pico)) - - def test_msc_host_selects_host_side(self): - s = sel(['src/class/msc/msc_host.c']) - self.assertFalse(s['full']) - self.assertNotIn('stm32f407disco', s['boards']) # device-only board - pico2 = s['boards']['raspberry_pi_pico2'] - self.assertIn('host/msc_file_explorer', pico2) - self.assertTrue(all(not t.startswith('device/') for t in pico2)) - - -class TestClassIncludeEdges(unittest.TestCase): - """A class header another class includes reaches that class's examples too. - src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, so - midi_test's firmware contains audio.h - but the class rule derives macros from - the directory name alone, so an audio.h change used to select only - device/audio_test_freertos. On boards that skip that example the per-board - intersection emptied and an audio.h-only PR ran ZERO HIL on them.""" - def test_edges_derived_from_includes(self): - edges = hil_select.class_include_edges(REPO) - self.assertEqual(edges.get('audio/audio.h'), {'midi'}) - self.assertEqual(edges.get('cdc/cdc.h'), {'net'}) - - def test_audio_header_selects_midi_example(self): - s = hil_select.classify(['src/class/audio/audio.h'], REPO, real_rosters()) - self.assertFalse(s['full']) - # every board that runs device/midi_test at all must run it here (boards with - # a tests.only list, e.g. espressif, run the freertos examples instead) - by_name = {b['name']: b for _, bs in real_rosters() for b in bs} - checked = 0 - for name, tests in s['boards'].items(): - if 'device/midi_test' in hil_select.board_tests(by_name[name]): - self.assertIn('device/midi_test', tests, name) - checked += 1 - self.assertTrue(checked) - - def test_audio_header_reaches_boards_that_skip_audio(self): - # both skip device/audio_test_freertos: without the midi edge their - # intersection is empty and they drop out of the selection entirely - boards = on_roster(self, 'metro_m4_express', 'nrf54lm20dk') - s = hil_select.classify(['src/class/audio/audio.h'], REPO, real_rosters()) - for board in boards: - self.assertEqual(s['boards'].get(board), ['device/midi_test'], board) - - def test_edge_is_per_header_not_per_class(self): - # midi includes audio.h, not audio_device.h: an audio_device change must - # not drag midi's examples in - s = hil_select.classify(['src/class/audio/audio_device.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for tests in s['boards'].values(): - if tests != 'all': - self.assertNotIn('device/midi_test', tests) - - -class TestFallbackRules(unittest.TestCase): - def test_unknown_tool_is_full(self): - s = sel(['tools/random_new_script.py']) - self.assertTrue(s['full']) - - def test_docs_only_is_empty_not_full(self): - s = sel(['docs/info/contributing.rst', 'README.rst']) - self.assertFalse(s['full']) - self.assertEqual(s['boards'], {}) - - def test_bsp_family_selects_family_boards(self): - s = sel(['hw/bsp/rp2040/family.cmake']) - self.assertFalse(s['full']) - self.assertIn('raspberry_pi_pico', s['boards']) - self.assertEqual(s['boards']['raspberry_pi_pico'], 'all') - self.assertNotIn('stm32f407disco', s['boards']) - - def test_bsp_board_narrows_to_board(self): - s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) - self.assertFalse(s['full']) - self.assertEqual(list(s['boards'].keys()), ['raspberry_pi_pico']) - - def test_example_change_selects_that_example(self): - s = sel(['examples/device/cdc_msc/src/main.c']) - self.assertFalse(s['full']) - self.assertEqual(s['boards']['raspberry_pi_pico'], ['device/cdc_msc']) - - def test_core_common_is_full(self): - for f in ['src/tusb.c', 'src/common/tusb_fifo.c', 'src/osal/osal_freertos.h']: - self.assertTrue(sel([f])['full'], f) - - def test_board_test_example_is_full(self): - # board_test is the park/teardown firmware hil_test.py flashes on every board, - # not an unlisted example: a regression there must not skip the whole rig - for f in ['examples/device/board_test/src/main.c', - 'examples/device/board_test/CMakeLists.txt']: - self.assertTrue(sel([f])['full'], f) - - def test_harness_is_full(self): - for f in ['test/hil/hil_test.py', '.github/workflows/build.yml', 'hw/mcu/nxp/x.c', 'lib/foo/x.c']: - self.assertTrue(sel([f])['full'], f) - - def test_mixed_roles_no_pruning(self): - s = sel(['src/device/usbd.c', 'src/host/usbh.c']) - self.assertFalse(s['full']) - self.assertIn('raspberry_pi_pico2', s['boards']) - self.assertIn('stm32f407disco', s['boards']) - - def test_cmakelists_and_requirements_are_full(self): - for f in ['src/CMakeLists.txt', 'examples/CMakeLists.txt', - 'examples/device/CMakeLists.txt', 'test/hil/requirements.txt']: - self.assertTrue(sel([f])['full'], f) - - def test_docs_txt_is_noncode(self): - s = sel(['docs/info/changelog.txt']) - self.assertFalse(s['full']) - self.assertEqual(s['boards'], {}) - - -class TestArgsEmission(unittest.TestCase): - def test_args_for_scoped_selection(self): - s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) - args = hil_select.selection_args(s, ROSTERS) - a = args['tinyusb.json'] - self.assertIn('-b raspberry_pi_pico', a) - self.assertNotIn('stm32f407disco', a) - self.assertIn('-bt raspberry_pi_pico:', a) # device-only subset of a device+host board - - def test_args_full_is_empty(self): - s = sel(['tools/random_new_script.py']) - self.assertEqual(hil_select.selection_args(s, ROSTERS), {'tinyusb.json': ''}) - - def test_args_all_board_gets_bare_b(self): - s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) - a = hil_select.selection_args(s, ROSTERS)['tinyusb.json'] - self.assertIn('-b raspberry_pi_pico', a) - self.assertNotIn('-bt', a) - - def test_args_by_flasher_splits_esp_from_the_rest(self): - s = sel(['src/device/usbd.c']) - per = hil_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json'] - self.assertIn('espressif_s3_devkitm', per['esptool']) - self.assertIn('raspberry_pi_pico', per['openocd']) - self.assertNotIn('espressif_s3_devkitm', per.get('openocd', '') + per.get('jlink', '')) - - def test_args_by_flasher_omits_a_flasher_with_no_selected_board(self): - # the esp CI leg must see no args at all here, not a filter matching zero boards - s = sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h']) - per = hil_select.selection_args_by_flasher(s, ROSTERS)['tinyusb.json'] - self.assertEqual(per, {'openocd': '-b raspberry_pi_pico'}) - - def test_args_by_flasher_full_is_empty(self): - s = sel(['tools/random_new_script.py']) - self.assertEqual(hil_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}}) - - def test_cli_diff_file(self): - import subprocess, tempfile, json as j - with tempfile.NamedTemporaryFile('w', suffix='.txt', delete=False) as f: - f.write('src/class/cdc/cdc_device.c\n') - path = f.name - r = subprocess.run([sys.executable, os.path.join(REPO, 'test/hil/helper/hil_select.py'), - '--diff-file', path, os.path.join(REPO, 'test/hil/tinyusb.json')], - capture_output=True, text=True) - self.assertEqual(r.returncode, 0, r.stderr) - out = j.loads(r.stdout) - self.assertFalse(out['full']) - self.assertIn('tinyusb.json', out['args']) - self.assertTrue(any('cdc_device' in line for line in out['reasons'])) - # A core-class diff must select boards THROUGH THE CLI: the in-process tests - # inject their own repo root, so only this subprocess path catches a broken - # repo_root derivation -- which once made every repo-relative glob match - # nothing and turned this exact diff into a silent full-HIL skip. - self.assertTrue(out['boards'], - 'CLI selected zero boards for a src/class change: repo_root broken?') - os.unlink(path) - - -class TestRealRosterPortFamilies(unittest.TestCase): - """Regression for port_families() missing espressif's dwc2 reference, which - lives in a component CMakeLists.txt rather than family.cmake/family.mk.""" - def test_dwc2_change_selects_espressif_boards(self): - boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') - s = hil_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - self.assertIn(board, s['boards']) - - -class TestOptionGatedPort(unittest.TestCase): - """Regression: family_support.cmake compiles some ports from a build option - (MAX3421_HOST=1 -> hcd_max3421.c), so a board's family file never names them.""" - # host-side option board (max3421 as host controller), off any max3421 family - OPT_ROSTER = [('test/hil/opt.json', [ - {'name': 'fake_dual_board', 'uid': 'o1', 'flasher': {'name': 'jlink'}, - 'build': {'args': ['MAX3421_HOST=1']}, - 'tests': {'device': True, 'host': False, 'dual': True}}, - {'name': 'fake_host_board', 'uid': 'o2', 'flasher': {'name': 'jlink'}, - 'variant': [{'name': 'fake_host_board', 'flags': '-DMAX3421_HOST=1'}], - 'tests': {'device': False, 'host': True, 'dual': False}}, - {'name': 'fake_off_board', 'uid': 'o3', 'flasher': {'name': 'jlink'}, - 'variant': [{'name': 'fake_off_board', 'defines': ['MAX3421_HOST=0']}], - 'tests': {'device': True, 'host': True, 'dual': True}}, - ])] - - def test_real_roster_max3421_selects_option_board(self): - boards = on_roster(self, 'metro_m4_express') - s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - self.assertIn(board, s['boards']) - - def test_option_selects_via_args_defines_and_flags(self): - s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, self.OPT_ROSTER) - self.assertFalse(s['full']) - self.assertIn('fake_dual_board', s['boards']) # build.args - self.assertIn('fake_host_board', s['boards']) # variant flags - self.assertNotIn('fake_off_board', s['boards']) # variant defines, but =0 - - def test_device_role_port_does_not_pull_host_only_option_board(self): - s = hil_select.classify(['src/portable/analog/max3421/dcd_max3421.c'], REPO, self.OPT_ROSTER) - self.assertFalse(s['full']) - self.assertNotIn('fake_host_board', s['boards']) # host-only board, device change - self.assertIn('fake_dual_board', s['boards']) # device-capable option board - - def test_gates_parsed_from_family_support(self): - self.assertEqual(hil_select.port_option_gates(REPO).get('analog/max3421'), - {'MAX3421_HOST'}) - - def test_board_cmake_option_counts(self): - """A board can enable a gated port in its own BSP rather than via the roster - (hw/bsp/espressif/boards/*/board.cmake -> set(MAX3421_HOST 1)); board_options() - must see those too, or such a board joining the roster is silently dropped.""" - self.assertIn('MAX3421_HOST', - hil_select.bsp_board_options('adafruit_feather_esp32s3', REPO)) - self.assertIn('CFG_TUH_RPI_PIO_USB', - hil_select.bsp_board_options('adafruit_fruit_jam', REPO)) - # commented-out `# set(MAX3421_HOST 1)` must not count - self.assertNotIn('MAX3421_HOST', - hil_select.bsp_board_options('feather_nrf52840_express', REPO)) - - def test_board_cmake_option_selects_off_family_board(self): - # adafruit_feather_esp32s3 is not on any rig roster; stand it in as one to - # prove the BSP-sourced option alone pulls a max3421 change onto the board - roster = [('test/hil/opt.json', [ - {'name': 'adafruit_feather_esp32s3', 'uid': 'o1', 'flasher': {'name': 'esptool'}, - 'tests': {'device': False, 'host': True, 'dual': False}}])] - s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster) - self.assertFalse(s['full']) - self.assertIn('adafruit_feather_esp32s3', s['boards']) - - def test_board_mk_option_is_ignored(self): - """Make-only options must not select: HIL CI builds with CMake exclusively, so - hw/bsp/nrf/boards/nrf5340dk/board.mk's MAX3421_HOST compiles nothing here.""" - roster = [('test/hil/opt.json', [ - {'name': 'nrf5340dk', 'uid': 'o1', 'flasher': {'name': 'jlink'}, - 'tests': {'device': False, 'host': True, 'dual': False}}])] - s = hil_select.classify(['src/portable/analog/max3421/hcd_max3421.c'], REPO, roster) - self.assertFalse(s['full']) - self.assertEqual(s['boards'], {}) - - -class TestPortFamiliesCmakeOnly(unittest.TestCase): - """port_families() is CMake-only (HIL CI never builds with Make) and matches on - 'port_dir/' so a port dir is not a prefix of a sibling.""" - def test_make_only_family_is_not_a_family(self): - # hw/bsp/pic32mz has family.mk but no family.cmake - self.assertEqual(hil_select.port_families('microchip/pic32mz', REPO), set()) - - def test_prefix_port_does_not_inherit_sibling_families(self): - # bare-substring matching let 'microchip/pic' match '.../microchip/pic32mz/...' - self.assertEqual(hil_select.port_families('microchip/pic', REPO), set()) - - def test_make_only_port_forces_full(self): - s = sel(['src/portable/microchip/pic32mz/dcd_pic32mz.c']) - self.assertTrue(s['full']) - self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons']) - - def test_cmake_families_still_found(self): - self.assertEqual(hil_select.port_families('raspberrypi/rp2040', REPO), {'rp2040'}) - self.assertIn('stm32f4', hil_select.port_families('synopsys/dwc2', REPO)) - - -class TestPortFamiliesCoverage(unittest.TestCase): - """Systematic guard: every real dcd_*/hcd_* port directory should map to at - least one board family, so a future family.cmake/CMakeLists.txt layout that - port_families() doesn't scan fails loudly instead of silently dropping boards - (as espressif's dwc2 reference did - see TestRealRosterPortFamilies).""" - # Ports with no board family: not a bug, just not wired into any rig board. - # Add here (with a reason) only if port_families() legitimately can't find one. - # A port listed here force-fulls (fail-open), so it is never under-selected. - NO_FAMILY = { - 'template', # reference/example port, not built by any board - # hw/bsp/pic32mz has family.mk only (no family.cmake), and port_families() - # is CMake-only because HIL CI builds every board with CMake - so this port - # is compiled for no HIL board. - 'microchip/pic32mz', - 'microchip/pic', # same: only ever referenced from pic32mz's family.mk - } - - @staticmethod - def _dcd_hcd_ports(): - portable_root = os.path.join(REPO, 'src/portable') - ports = [] - for entry in sorted(os.listdir(portable_root)): - d = os.path.join(portable_root, entry) - if not os.path.isdir(d): - continue - if glob.glob(os.path.join(d, 'dcd_*.c')) or glob.glob(os.path.join(d, 'hcd_*.c')): - ports.append(entry) - continue - for sub in sorted(os.listdir(d)): - sd = os.path.join(d, sub) - if os.path.isdir(sd) and (glob.glob(os.path.join(sd, 'dcd_*.c')) or - glob.glob(os.path.join(sd, 'hcd_*.c'))): - ports.append(f'{entry}/{sub}') - return ports - - def test_every_port_maps_to_a_family(self): - ports = self._dcd_hcd_ports() - self.assertTrue(ports) # sanity: the scan itself found something - for port in ports: - if port in self.NO_FAMILY: - continue - fams = hil_select.port_families(port, REPO) - self.assertTrue(fams, f'{port}: no family references this port ' - f'(port_families() scan gap, or add to NO_FAMILY)') - - -class TestRealRosterOnlyListTests(unittest.TestCase): - """Regression for roster-only-list tests (e.g. espressif's hid_composite_freertos) - being invisible to the selector because it only knew the shared hil_util lists.""" - def test_only_list_example_change_selects_it(self): - boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') - s = hil_select.classify(['examples/device/hid_composite_freertos/src/main.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - self.assertEqual(s['boards'][board], ['device/hid_composite_freertos']) - - def test_class_change_includes_only_list_boards(self): - boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') - s = hil_select.classify(['src/class/hid/hid_device.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - self.assertIn(board, s['boards']) - - -class TestPortAndCoreRoleUseExtras(unittest.TestCase): - """Regression: the port rule and core-role rule must thread the roster-only - test universe (extras) the same way the class rule already does, so a DCD - or device-stack change doesn't silently drop espressif's only-list tests - (e.g. hid_composite_freertos) that aren't in the shared device_tests list.""" - def test_dcd_change_includes_only_list_test(self): - boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') - s = hil_select.classify(['src/portable/synopsys/dwc2/dcd_dwc2.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - tests = s['boards'][board] - self.assertIn('device/hid_composite_freertos', tests) - self.assertIn('device/cdc_msc_freertos', tests) - self.assertIn('device/audio_test_freertos', tests) - self.assertIn('device/usbtest', tests) - - def test_core_device_change_includes_only_list_test(self): - boards = on_roster(self, 'espressif_s3_devkitm', 'espressif_p4_function_ev') - s = hil_select.classify(['src/device/usbd.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board in boards: - tests = s['boards'][board] - self.assertIn('device/hid_composite_freertos', tests) - self.assertIn('device/cdc_msc_freertos', tests) - self.assertIn('device/audio_test_freertos', tests) - self.assertIn('device/usbtest', tests) - - def test_host_change_does_not_leak_device_only_list_test(self): - s = hil_select.classify(['src/host/usbh.c'], REPO, real_rosters()) - self.assertFalse(s['full']) - for board, tests in s['boards'].items(): - if tests == 'all': - continue - self.assertNotIn('device/hid_composite_freertos', tests, board) - - -class TestFamilies(unittest.TestCase): - """`families` exists for consumers that build (not just test) the diff: most - families have no rig board, so `boards` alone would compile nothing for them.""" - def test_off_rig_port_still_reports_family(self): - s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c']) - self.assertFalse(s['full']) - self.assertEqual(s['boards'], {}) # no same7x board on the rig - self.assertEqual(s['families'], ['same7x']) - - def test_port_families_are_reported(self): - s = sel(['src/portable/raspberrypi/rp2040/dcd_rp2040.c']) - self.assertIn('rp2040', s['families']) - - def test_bsp_family_and_board_report_family(self): - self.assertEqual(sel(['hw/bsp/rp2040/family.cmake'])['families'], ['rp2040']) - self.assertEqual(sel(['hw/bsp/rp2040/boards/raspberry_pi_pico/board.h'])['families'], - ['rp2040']) - - def test_docs_only_has_no_families(self): - self.assertEqual(sel(['docs/info/contributing.rst'])['families'], []) - - def test_full_selection_still_reports_families(self): - """A full-matrix file must not hide the families of the other changed files: - consumers that build from `families` (e.g. /pre-pr) ignore `boards` when full.""" - s = sel(['src/common/tusb_fifo.c', 'src/portable/microchip/samx7x/dcd_samx7x.c']) - self.assertTrue(s['full']) - self.assertIn('same7x', s['families']) - # full stays full: every roster board, and no args to narrow the run - self.assertEqual(set(s['boards']), {b['name'] for b in ROSTER}) - self.assertTrue(all(v == 'all' for v in s['boards'].values())) - self.assertEqual(hil_select.selection_args(s, ROSTERS), {'tinyusb.json': ''}) - self.assertEqual(hil_select.selection_args_by_flasher(s, ROSTERS), {'tinyusb.json': {}}) - - def test_family_order_does_not_matter(self): - # same as above with the full-matrix file last (was the only order that worked) - s = sel(['src/portable/microchip/samx7x/dcd_samx7x.c', 'src/common/tusb_fifo.c']) - self.assertTrue(s['full']) - self.assertIn('same7x', s['families']) - - -class TestGitDiffArgv(unittest.TestCase): - def test_diff_disables_rename_detection(self): - """Without --no-renames git reports only a rename's destination, so moving an - HIL-relevant file to a non-code path would be classified as non-code only.""" - self.assertIn('--no-renames', hil_select.GIT_DIFF_ARGV) - - -class TestPortWithoutFamilyIsFull(unittest.TestCase): - """A port dir no family file references must widen (full matrix), not silently - contribute zero boards — the fail-open contract.""" - def test_unreferenced_port_forces_full(self): - orig = hil_select.port_families - hil_select.port_families = lambda port_dir, repo_root: set() - try: - s = sel(['src/portable/vendor/newip/dcd_newip.c']) - finally: - hil_select.port_families = orig - self.assertTrue(s['full']) - self.assertTrue(any('no board family' in r for r in s['reasons']), s['reasons']) - - -class TestOpenocdVidPid(unittest.TestCase): - """The roster's optional flasher `vid_pid` field (openocd-verbatim, e.g. - "0x1a86 0x8010", more pairs appended) pins openocd's probe discovery so it - never opens foreign usbfs nodes. It must be emitted BEFORE the args: the - rescue cfgs run `init` internally (rp2350-rescue.cfg errors on any - config-stage command after its init; rp2040.cfg under RESCUE scans before a - trailing flag is even parsed), and no rig cfg sets a competing list - (the 2026-08-10 convoy mechanism).""" - - def test_vid_pid_flag_precedes_args(self): - cmd = hil_flash._openocd_cmd_base( - {'uid': 'S1', 'args': '-f target/wch-riscv.cfg', 'vid_pid': '0x1a86 0x8010'}) - self.assertIn('-c "adapter usb vid_pid 0x1a86 0x8010" -f target/wch-riscv.cfg', cmd) - self.assertTrue(cmd.endswith('-f target/wch-riscv.cfg'), cmd) - - def test_rescue_cfg_command_keeps_vid_pid_before_init(self): - """rescue_openocd swaps the target cfg for one that runs `init` internally; - a vid_pid flag after the args would error there (rp2350) or be skipped - (rp2040) -- in exactly the wedged-rig scenario the pin exists for.""" - flasher = {'name': 'openocd', 'uid': 'S1', 'vid_pid': '0x2e8a 0x000c', - 'args': '-c "set RESCUE 1" -f target/rp2040.cfg'} - cmd = hil_flash._openocd_cmd_base(flasher) - self.assertLess(cmd.index('adapter usb vid_pid'), cmd.index('-f target/'), cmd) - - def test_vid_pid_multiple_pairs(self): - cmd = hil_flash._openocd_cmd_base( - {'uid': 'S1', 'args': '-f i.cfg', 'vid_pid': '0x2e8a 0x000c 0x2e8a 0x000d'}) - self.assertIn('-c "adapter usb vid_pid 0x2e8a 0x000c 0x2e8a 0x000d"', cmd) - - def test_no_field_no_flag_but_warns(self): - # the roster lint only covers the committed rosters; a dev PC's local.json entry - # without the field must at least say what it is giving up -- on STDERR, since - # hil_test captures stdout per test and would swallow it on a passing run - import io - from contextlib import redirect_stderr - hil_flash._VID_PID_WARNED.discard('S-warn') - cap = io.StringIO() - with redirect_stderr(cap): - cmd = hil_flash._openocd_cmd_base({'uid': 'S-warn', 'args': '-f i.cfg'}) - self.assertNotIn('vid_pid', cmd) - self.assertIn('vid_pid', cap.getvalue()) - - def test_roster_openocd_entries_all_pin_vid_pid(self): - # every openocd probe on the rig has a known VID/PID; a new entry without the - # pin silently reintroduces open-everything discovery - for path, board in roster_flashers(): - f = board['flasher'] - # tinyusb.json only: hfp.json is the hifiphile rig owner's file, and a - # blocking repo-wide lint over someone else's roster would red every PR the - # moment they add an openocd board (hil_flash treats the field as optional) - if f['name'] == 'openocd' and path.endswith('tinyusb.json'): - self.assertIn('vid_pid', f, - f"{path}: {board['name']} openocd flasher lacks vid_pid") - self.assertNotIn('vid_pid', f.get('args', ''), - f"{path}: {board['name']} packs vid_pid into args; use the field") - - -class TestRosterFlashersDispatch(unittest.TestCase): - """hil_test and hil_pool_check resolve a board's flasher with a bare - getattr(hil_flash, f'flash_{name}'), and hil_test does it inside a redirect_stdout — - so a renamed or typo'd roster name raises an AttributeError whose output is swallowed, - with nothing pointing at the roster as the thing to edit. Renaming a flash_*/reset_* - pair without updating every roster must fail here instead.""" - - def test_flash_and_reset_exist_for_every_roster_flasher(self): - for path, board in roster_flashers(): - name = board['flasher']['name'].lower() - for fn in (f'flash_{name}', f'reset_{name}'): - self.assertTrue(callable(getattr(hil_flash, fn, None)), - f'{path}: {board["name"]} uses flasher "{name}" ' - f'but hil_flash.{fn} does not exist') - - def test_firmware_suffix_known_for_every_roster_flasher(self): - """find_firmware falls back to accepting .elf-or-.bin when a flasher is missing - from FLASHER_SUFFIX, silently restoring the mismatch that map exists to catch.""" - for path, board in roster_flashers(): - name = board['flasher']['name'].lower() - self.assertIn(name, hil_flash.FLASHER_SUFFIX, - f'{path}: {board["name"]} uses flasher "{name}" ' - f'with no hil_flash.FLASHER_SUFFIX entry') - - -class FlasherRecoverEntry(unittest.TestCase): - """Optional roster key: a SECOND flasher used only to deliver recovery while a usbfs - node is poisoned. Boards whose primary flasher cannot get past a convoy (jlink, - stlink, lm4flash) name an openocd entry here instead of changing how they are - normally flashed.""" - - def test_recover_flasher_prefers_the_optional_entry(self): - prim = {'name': 'jlink', 'uid': 'X', 'args': '-device MIMXRT1064xxx6A'} - rec = {'name': 'openocd', 'uid': 'X', 'args': '-f interface/jlink.cfg -f target/foo.cfg'} - self.assertEqual(hil_flash.recover_flasher({'flasher': prim, 'flasher_recover': rec}), rec) - self.assertEqual(hil_flash.recover_flasher({'flasher': prim}), prim) - - def test_openocd_over_jlink_is_convoy_safe_without_a_pin(self): - """libjaylink discovery returns early unless idVendor == 0x1366 (SEGGER) and the PID - is in its table, and only THEN calls libusb_open (discovery_usb.c) -- it never opens - a foreign node. `adapter usb vid_pid` is a no-op for this driver: jlink.c reads - adapter_serial / usb address / usb location, never the vid/pid.""" - self.assertTrue(hil_flash.convoy_safe( - {'name': 'openocd', 'args': '-f interface/jlink.cfg -f target/stm32f4x.cfg'})) - - def test_openocd_with_neither_a_pin_nor_jlink_is_not_safe(self): - self.assertFalse(hil_flash.convoy_safe( - {'name': 'openocd', 'args': '-f interface/stlink.cfg -f target/stm32h7x.cfg'})) - - def test_the_existing_rules_are_unchanged(self): - self.assertTrue(hil_flash.convoy_safe( - {'name': 'openocd', 'vid_pid': '0x2e8a 0x000c', 'args': '-f interface/cmsis-dap.cfg'})) - self.assertFalse(hil_flash.convoy_safe({'name': 'jlink', 'uid': 'X'})) - self.assertTrue(hil_flash.convoy_safe({'name': 'esptool'})) - - -if __name__ == '__main__': - unittest.main(verbosity=1) diff --git a/test/hil/test/test_hil_util.py b/test/hil/test/test_hil_util.py index 9c3d5edef..c95e20b6d 100644 --- a/test/hil/test/test_hil_util.py +++ b/test/hil/test/test_hil_util.py @@ -98,7 +98,7 @@ class RunCmdModes(unittest.TestCase): class BottomLayer(unittest.TestCase): def test_bad_timeout_env_falls_back(self): - # hil_select (the PR-diff selector) imports hil_util for the example rosters; + # ci_select (the PR-diff selector) imports hil_util for the example rosters; # a malformed HIL_CMD_TIMEOUT must not crash the selector at import and knock # CI back to the full-matrix fallback import subprocess @@ -108,7 +108,7 @@ class BottomLayer(unittest.TestCase): env={**os.environ, 'HIL_CMD_TIMEOUT': 'bogus'}, capture_output=True, text=True, timeout=30) self.assertEqual(r.returncode, 0, r.stderr) - # the warning must NOT be on stdout: hil_select's stdout is machine-read JSON + # the warning must NOT be on stdout: ci_select's stdout is machine-read JSON self.assertEqual(r.stdout.strip(), '180') self.assertIn('warning', r.stderr) # but a silent fallback hides the misconfiguration @@ -132,22 +132,23 @@ class BottomLayer(unittest.TestCase): # hil_examples.py used to make this structural (a list of strings cannot grow a # dependency); with the rosters folded into hil_util the invariant needs teeth: # everything the bare GitHub runner imports (selector + this suite) must stay - # stdlib + local. Adding pyserial/pymtp here breaks hil_select on CI. + # stdlib + local. Adding pyserial/pymtp here breaks ci_select on CI. import ast hil_dir = Path(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # ONLY the modules the bare runner can import -- not every stem in the tree. # Globbing the directory allowed `import pymtp` (and hil_test, usbtest, # mtp_test) through, so the pymtp case this test names could never fail: that # module runs ctypes.CDLL(find_library('mtp')) at import and raises where there - # is no libmtp, taking hil_select down with it. - local = {'helper', 'hil_util', 'hil_select', 'hil_flash', - 'hil_health', 'hil_lock', 'hil_pool_check'} + # is no libmtp, taking ci_select down with it. + local = {'helper', 'hil_util', 'ci_select', 'hil_flash', + 'hil_health', 'hil_lock', 'hil_pool_check', 'build', 'build_utils'} allowed = set(sys.stdlib_module_names) | local # hil_pool_check included: test_hil_util_is_a_single_module_instance imports it # on the bare runner, and its `import serial` is function-local for exactly # this reason -- hoisting it must fail HERE, not on every PR's pre-commit CI - for mod in ('helper/hil_util', 'hil_flash', 'helper/hil_select', - 'helper/hil_health', 'helper/hil_lock', 'helper/hil_pool_check'): + for mod in ('helper/hil_util', 'hil_flash', '../../tools/ci_select', + 'helper/hil_health', 'helper/hil_lock', 'helper/hil_pool_check', + '../../tools/build', '../../tools/build_utils'): tree = ast.parse((hil_dir / f'{mod}.py').read_text()) # module level only: a deferred import inside a function cannot break # importability (hil_pool_check keeps `import serial` function-local diff --git a/tools/build.py b/tools/build.py index 51d3d0f70..e7ca1c839 100755 --- a/tools/build.py +++ b/tools/build.py @@ -2,6 +2,7 @@ import argparse import random import os +import re import sys import time import subprocess @@ -99,6 +100,53 @@ def get_examples(family): return all_examples +def resolve_example_target_groups(build_targets, examples, board, extra_defines=()): + """Map generic targets onto per-example targets for a filtered build (-e), as ONE + GROUP PER REQUESTED TARGET: 'all' -> the example executables, anything else (e.g. + tinyusb_metrics) passes through as its own single-entry group. + + Grouped rather than flattened because each group becomes one `cmake --build + --target a b c` invocation: the examples of a group build in parallel (flattening + them into one target per invocation serialises the whole leg - measured +39% at + -j4 and +220% at -j32 on stm32f407disco), while separate groups stay ordered, so a + target that must run after the examples still does. + + extra_defines are this build's -D tokens: MAX3421_HOST=1 there decides + only.txt for the max3421 examples (see build_utils.skip_example). + Returns None when no requested example is buildable on this board.""" + buildable = [e for e in examples + if not build_utils.skip_example(e, board, extra_defines)] + if not buildable: + return None + names = list(dict.fromkeys(e.split('/', 1)[1] for e in buildable)) + return [list(names) if t == 'all' else [t] for t in build_targets] + + +_TARGET_HELP_RE = re.compile(r'^([A-Za-z0-9_.+-]+):') +# role/name, the only shape resolve_example_target_groups and the CMake target names accept +EXAMPLE_RE = re.compile(r'[A-Za-z0-9_]+/[A-Za-z0-9_]+') + + +def parse_target_help(text): + """Bare target names out of `cmake --build --target help`; the Ninja + generator prints one ': phony' line per target. Names containing '/' are + per-directory utility targets (device/edit_cache) or absolute CMakeFiles paths, + never an example target.""" + return {m.group(1) for m in map(_TARGET_HELP_RE.match, text.splitlines()) if m} + + +def cmake_registered_targets(build_dir): + """The targets CMake actually created in build_dir, or None when that cannot be + read. Ground truth: skip.txt/only.txt only mirrors family_filter, so an example + the role CMakeLists never lists (or a stale -e name) still looks buildable to it + and `cmake --build --target ` hard-fails. None keeps the mirror's answer.""" + r = subprocess.run(['cmake', '--build', build_dir, '--target', 'help'], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + if r.returncode != 0: + return None + return parse_target_help(r.stdout.decode('utf-8', 'replace')) or None + + def print_build_result(board, build_target, status, duration): if isinstance(duration, (int, float)): duration = "{:.2f}s".format(duration) @@ -107,7 +155,7 @@ def print_build_result(board, build_target, status, duration): # ----------------------------- # CMake # ----------------------------- -def cmake_board(board, build_args, build_name, build_cflags, build_targets): +def cmake_board(board, build_args, build_name, build_cflags, build_targets, examples=None, defines=()): ret = [0, 0, 0] start_time = time.monotonic() @@ -120,8 +168,13 @@ def cmake_board(board, build_args, build_name, build_cflags, build_targets): if family == 'espressif': # for espressif, we have to build example individually all_examples = get_examples(family) + if examples is not None: + all_examples = [e for e in all_examples if e in examples] + if not all_examples: + print_build_result(board, 'examples (PR filter)', 2, '-') + return [0, 0, 1] for example in all_examples: - if build_utils.skip_example(example, board): + if build_utils.skip_example(example, board, defines): ret[2] += 1 else: rcmd = run_cmd([ @@ -130,13 +183,40 @@ def cmake_board(board, build_args, build_name, build_cflags, build_targets): ]) ret[0 if rcmd.returncode == 0 else 1] += 1 else: + # the skip.txt/only.txt prefilter reads no configure output: answer it first, + # so a selection this board builds nothing of costs no cmake run at all + if examples is not None: + examples = [e for e in examples + if not build_utils.skip_example(e, board, defines)] + if not examples: + print_build_result(board, 'examples (PR filter)', 2, '-') + return [0, 0, 1] rcmd = run_cmd(['cmake', 'examples', '-B', build_dir, '-GNinja', f'-DBOARD={board}', '-DCMAKE_BUILD_TYPE=MinSizeRel', '-DLINKERMAP_OPTION=-q -f tinyusb/src', *build_args, *build_flags]) if rcmd.returncode == 0: + target_groups = [[t] for t in build_targets] + if examples is not None: + registered = cmake_registered_targets(build_dir) + if registered is not None: + kept = [e for e in examples if e.split('/', 1)[1] in registered] + for e in examples: + if e not in kept: + print_build_result(board, f'{e} (no such target)', 2, '-') + examples = kept + if not examples: + print_build_result(board, 'examples (no such target)', 2, '-') + return [0, 0, 1] + target_groups = resolve_example_target_groups(build_targets, examples, board, defines) + if registered is None: + # ground truth unavailable, so nothing checked these names against + # what CMake created. ninja validates a whole invocation up front: + # one unknown name in the batch builds NOTHING, where a target each + # builds everything up to it. Give up the parallelism, not the work. + target_groups = [[t] for g in target_groups for t in g] cmd = ["cmake", "--build", build_dir, '--parallel', str(parallel_jobs)] - for target in build_targets: - rcmd = run_cmd(cmd + ['--target', target]) + for group in target_groups: + rcmd = run_cmd(cmd + ['--target'] + group) if rcmd.returncode != 0: break ret[0 if rcmd.returncode == 0 else 1] += 1 @@ -148,9 +228,10 @@ def cmake_board(board, build_args, build_name, build_cflags, build_targets): # ----------------------------- # Make # ----------------------------- -def make_one_example(example, board, make_option, build_targets): - # Check if board is skipped - if build_utils.skip_example(example, board): +def make_one_example(example, board, make_option, build_targets, defines=()): + # Check if board is skipped. Make semantics: family.mk decides, not the + # family.cmake MCU list (see build_utils.skip_example). + if build_utils.skip_example(example, board, defines, build_system='make'): print_build_result(board, example, 2, '-') r = 2 else: @@ -171,10 +252,15 @@ def make_one_example(example, board, make_option, build_targets): return ret -def make_board(board, build_args, build_targets): +def make_board(board, build_args, build_targets, examples=None, defines=()): print(build_separator) family = find_family(board); all_examples = get_examples(family) + if examples is not None: + all_examples = [e for e in all_examples if e in examples] + if not all_examples: + print_build_result(board, 'examples (PR filter)', 2, '-') + return [0, 0, 1] start_time = time.monotonic() ret = [0, 0, 0] if family == 'espressif' or family == 'rp2040': @@ -182,7 +268,7 @@ def make_board(board, build_args, build_targets): final_status = 2 else: with Pool(processes=os.cpu_count()) as pool: - pool_args = list((map(lambda e, b=board, o=f"{build_args}", t=build_targets: [e, b, o, t], all_examples))) + pool_args = list((map(lambda e, b=board, o=f"{build_args}", t=build_targets, d=defines: [e, b, o, t, d], all_examples))) r = pool.starmap(make_one_example, pool_args) # sum all element of same index (column sum) ret = list(map(sum, list(zip(*r)))) @@ -194,29 +280,40 @@ def make_board(board, build_args, build_targets): # ----------------------------- # Build Family # ----------------------------- -def build_boards_list(boards, build_defines, build_system, build_name, build_cflags, build_targets): +def build_boards_list(boards, build_defines, build_system, build_name, build_cflags, build_targets, examples=None): ret = [0, 0, 0] + # the -D tokens are part of the skip.txt/only.txt answer (MAX3421_HOST=1), so + # the -e filter has to see them too; sorted+tuple keeps skip_example cacheable + defines = tuple(sorted(build_defines)) for b in boards: r = [0, 0, 0] if build_system == 'cmake': build_args = [f'-D{d}' for d in build_defines] - r = cmake_board(b, build_args, build_name, build_cflags, build_targets) + r = cmake_board(b, build_args, build_name, build_cflags, build_targets, examples, defines) elif build_system == 'make': build_args = ' '.join(f'{d}' for d in build_defines) - r = make_board(b, build_args, build_targets) + r = make_board(b, build_args, build_targets, examples, defines) ret[0] += r[0] ret[1] += r[1] ret[2] += r[2] return ret -def get_family_boards(family, one_random, one_first): +def get_family_boards(family, one_random, one_first, examples=None, build_system='cmake'): """Get list of boards for a family. Args: family: Family name one_random: If True, return only one random board one_first: If True, return only the first board (alphabetical) + examples: PR example filter (-e). The one-board pick then prefers a board that + can build at least one of them: the family is in the matrix BECAUSE some + board of it builds these examples (ci_select._prune_buildable asks about + every board, since CircleCI builds every board), but GHA builds one. Without + this, lpc54 selected for host/msc_file_explorer picks lpcxpresso54114 - + which every one of those examples skips - and the leg runs to green having + compiled nothing and uploaded no metrics. + build_system: which skip answer to ask for; the two differ (build_utils) Returns: List of board names @@ -238,12 +335,19 @@ def get_family_boards(family, one_random, one_first): # If only-one flags are set, honor select list first, then pick first or random if one_first or one_random: - if preferred_list: + def buildable(board): + # no filter, or nothing in the filter is buildable anywhere: keep today's + # answer rather than inventing a different board + return examples is None or any( + not build_utils.skip_example(e, board, (), build_system) for e in examples) + + if preferred_list and buildable(preferred_list[0]): return [preferred_list[0]] + candidates = [b for b in all_boards if buildable(b)] or all_boards if one_first: - return [all_boards[0]] + return [candidates[0]] if one_random: - return [random.choice(all_boards)] + return [random.choice(candidates)] return all_boards @@ -272,6 +376,8 @@ def main(): parser.add_argument('-j', '--jobs', type=int, default=os.cpu_count(), help='Number of jobs to run in parallel') parser.add_argument('-T', '--target', action='append', default=[], help='Build target to use, may be specified multiple times (default: all)') + parser.add_argument('-e', '--example', action='append', default=[], + help='Only build these examples (role/name, repeatable). Default: all examples') parser.add_argument('-v', '--verbose', action='store_true', help='Verbose output') args = parser.parse_args() @@ -285,9 +391,20 @@ def main(): one_random = args.one_random one_first = args.one_first build_targets = args.target if args.target else ['all'] + examples = args.example or None verbose = args.verbose parallel_jobs = args.jobs + for e in args.example: + if not EXAMPLE_RE.fullmatch(e): + parser.error(f"-e/--example takes 'role/name' (e.g. device/cdc_msc), got '{e}'") + # a name no example dir answers to would silently build nothing on every board + # and still exit 0 (every row is a Skipped, and main() returns the FAILED count). + # The -e lists are generated - from ci_select's example map and from HIL roster + # test names - so a stale one must be loud, not green + if not os.path.isdir(os.path.join('examples', e)): + parser.error(f"-e/--example '{e}': no such example directory examples/{e}") + build_defines.append(f'TOOLCHAIN={toolchain}') if len(families) == 0 and len(boards) == 0: @@ -317,10 +434,11 @@ def main(): # get boards from families and append to boards list all_boards = list(boards) for f in all_families: - all_boards.extend(get_family_boards(f, one_random, one_first)) + all_boards.extend(get_family_boards(f, one_random, one_first, examples, build_system)) # build all boards - result = build_boards_list(all_boards, build_defines, build_system, build_name, build_cflags, build_targets) + result = build_boards_list(all_boards, build_defines, build_system, build_name, build_cflags, build_targets, + examples) total_time = time.monotonic() - total_time print(build_separator) diff --git a/tools/build_utils.py b/tools/build_utils.py index d80ceea7c..2af8fd624 100755 --- a/tools/build_utils.py +++ b/tools/build_utils.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import functools import subprocess import pathlib import re @@ -10,32 +11,180 @@ FAILED = "\033[31mfailed\033[0m" SKIPPED = "\033[33mskipped\033[0m" -def skip_example(example, board): - ex_dir = pathlib.Path('examples/') / example - bsp = pathlib.Path("hw/bsp") +# Every read here is a source file, not user text: decode it the same way on every +# machine. Without this the reads take the locale's encoding, and one of the eight +# tracked non-ASCII files this now touches (hw/bsp/nrf/boards/nrf54lm20dk/board.cmake +# among them) raises UnicodeDecodeError under LC_ALL=C - a ValueError, which sails +# straight through the `except OSError` fail-opens. +_TEXT = {'encoding': 'utf-8', 'errors': 'replace'} - # board within family - board_dir = list(bsp.glob("*/boards/" + board)) - if not board_dir: - # Skip unknown boards - return True +_FAMILY_MCUS_RE = re.compile(r'set\s*\(\s*FAMILY_MCUS\s+([^)]*)\)') +_CMAKE_SET_RE = re.compile(r'set\s*\(\s*([A-Za-z_]\w*)\s+([^)\s]+)') +_CMAKE_VAR_RE = re.compile(r'\$\{([A-Za-z_]\w*)\}') +_CMAKE_CASE_RE = re.compile(r'string\s*\(\s*(TOUPPER|TOLOWER)\s+(\S+)\s+([A-Za-z_]\w*)\s*\)') - board_dir = list(board_dir)[0] - family_dir = board_dir.parent.parent - family = family_dir.name - # family.mk +@functools.lru_cache(maxsize=None) +def _cmake_sets(path): + """One cmake file's variable assignments as NAME -> first definition seen, as + either a literal value or an ('TOUPPER'|'TOLOWER', source) pair. Only used to + expand ${...} tokens; never mutate the cached dict. + + string(TOUPPER ...) is not decoration: hw/bsp/maxim derives its ONLY FAMILY_MCUS + entry that way (`string(TOUPPER ${MAX_DEVICE} MAX_DEVICE_UPPER)`), as do the eight + at32 families, so dropping those lines left nine families with an empty MCU set.""" + try: + text = pathlib.Path(path).read_text(**_TEXT) + except OSError: + return {} + out = {} + for line in text.splitlines(): + line = line.strip() + if line.startswith('#'): + continue + m = _CMAKE_CASE_RE.match(line) + if m: + # strip quotes like the set() branch below: string(TOUPPER "${VAR}" DST) is + # idiomatic cmake, and keeping them yields a '"NAME"' token that can never + # equal a mcu: entry + out.setdefault(m.group(3), (m.group(1), m.group(2).strip('"'))) + continue + m = _CMAKE_SET_RE.match(line) + if m: + out.setdefault(m.group(1), m.group(2).strip('"')) + return out + + +def _cmake_expand(value, files, depth=0): + """`value` with every ${VAR} replaced, resolving each name against `files` in + order, or None when any name resolves nowhere OR the result still carries a `${`. + That last case is the one _CMAKE_VAR_RE cannot see - a hyphen in the name, a nested + ${${X}}, an unterminated brace - where the loop below finds nothing to substitute + and would otherwise hand the raw text back as if it were a resolved MCU name. + Bounded depth: a cmake file may define a var in terms of another one, and a + self-referential set() must not recurse forever.""" + if depth > 4: + return None + out = value + for name in set(_CMAKE_VAR_RE.findall(value)): + val = None + for f in files: + val = _cmake_sets(f).get(name) + if val is not None: + break + if val is None: + return None + if isinstance(val, tuple): # string(TOUPPER src DST) + src = _cmake_expand(val[1], files, depth + 1) + if src is None: + return None + val = src.upper() if val[0] == 'TOUPPER' else src.lower() + else: + val = _cmake_expand(val, files, depth + 1) + if val is None: + return None + out = out.replace('${' + name + '}', val) + return None if '${' in out else out + + +@functools.lru_cache(maxsize=None) +def _board_dirs(board): + """(board_dir, family_dir) for a board name, or (None, None). Cached: skip_example + is asked (board x example) times - 566k lstat calls per selector run without this, + since the glob rescans every hw/bsp/*/boards for each example.""" + hits = list(pathlib.Path("hw/bsp").glob("*/boards/" + board)) + if not hits: + return None, None + return hits[0], hits[0].parent.parent + + +@functools.lru_cache(maxsize=None) +def _family_mcus(family_dir, board_dir): + """The MCU names CMake's family_filter iterates. family_support.cmake:176/190 + loop `foreach(MCU IN LISTS FAMILY_MCUS)`, so a family-wide list (broadcom_64bit + sets "BCM2711 BCM2835") makes ANY of its entries decide skip.txt/only.txt -- not + just the one CFG_TUSB_MCU the configured board names. + + ${...} tokens are expanded from `set(VAR value)` and `string(TOUPPER src VAR)` in + the board's board.cmake first, then in family.cmake: hw/bsp/ra sets + `FAMILY_MCUS RAXXX ${MCU_VARIANT}` and ra6m5_ek/board.cmake sets MCU_VARIANT ra6m5, + which is the token dual/host_info_to_device_cdc/only.txt actually spells; hw/bsp/maxim + sets `FAMILY_MCUS ${MAX_DEVICE_UPPER}`, upper-cased from the board's MAX_DEVICE. A + token resolving nowhere is dropped (nothing can be said about it). + + A family that never spells `set(FAMILY_MCUS ...)` at all gets one more chance: the + name is resolved as a variable, which covers the derived form hw/bsp/espressif uses + (`string(TOUPPER ${IDF_TARGET} FAMILY_MCUS)`). + + Only unconditional set() calls count: nrf and mcx pick FAMILY_MCUS per board + inside if() blocks this does not evaluate, so for those two families the whole + cmake-side MCU set is whatever the CFG_TUSB_MCU scrape in _board_mcu finds. + + nrf: the scrape reads the FIRST CFG_TUSB_MCU token of hw/bsp/nrf/family.mk, so + every nrf board answers NRF54, the NRF5X ones included. Harmless only because no + skip.txt/only.txt names an nrf token today. + + mcx: load-bearing, not academic -- mcu:MCXA15 is live in six examples' skip.txt + (device/{cdc_msc,audio_test,hid_composite,audio_4_channel_mic,midi_test}_freertos + and device/net_lwip_webserver). Those answers come out right only because the + scrape falls through to each board's make-only board.mk, which still spells the + token; an mcx board carrying board.cmake alone (MCU_VARIANT and no CFG_TUSB_MCU) + would scrape 'NONE' and skip EVERY example on it, silently. TestFamilyMcusFallback + fails the day such a board lands. The fix then is to evaluate the + if(MCU_VARIANT STREQUAL ...) branches, not to add another scrape. + """ + fam_cmake = pathlib.Path(family_dir) / "family.cmake" + try: + text = fam_cmake.read_text(**_TEXT) + except OSError: + return frozenset() + board_cmake = pathlib.Path(board_dir) / "board.cmake" + out = set() + depth = 0 + for line in text.splitlines(): + line = line.strip() + m = _FAMILY_MCUS_RE.match(line) + if m and depth == 0: + files = (str(board_cmake), str(fam_cmake)) + for tok in m.group(1).split(): + if tok in ("CACHE", "INTERNAL") or tok.startswith('"'): + continue + val = _cmake_expand(tok, files) + if val: + out.add(val) + if re.match(r'if\s*\(', line): + depth += 1 + elif re.match(r'endif\s*\(', line): + depth = max(0, depth - 1) + if not out: + # FAMILY_MCUS can also be produced rather than set: hw/bsp/espressif derives it + # with `string(TOUPPER ${IDF_TARGET} FAMILY_MCUS)`, which _FAMILY_MCUS_RE cannot + # see, leaving espressif's whole cmake answer resting on the IDF_TARGET scrape + val = _cmake_expand('${FAMILY_MCUS}', (str(board_cmake), str(fam_cmake))) + if val: + out.add(val) + return frozenset(out) + + +def _scrape_mcu(family_dir, board_dir, family): + """(CFG_TUSB_MCU token of this board, the text it was read from), master's + algorithm verbatim: family.mk (family.cmake when there is none) first, falling + back to the board's board.mk (board.cmake when there is none) only when the + family file names no token at all. espressif spells its MCU as + `set(IDF_TARGET "...")` instead. The text comes back with it because the make + path reads MAX3421_HOST out of that same single file - which file that is IS + part of master's answer, so it cannot be re-derived by the caller.""" family_mk = family_dir / "family.mk" if not family_mk.exists(): family_mk = family_dir / "family.cmake" - mk_contents = family_mk.read_text() + mk_contents = family_mk.read_text(**_TEXT) # Find the mcu, first in family mk then board mk if "CFG_TUSB_MCU=OPT_MCU_" not in mk_contents: board_mk = board_dir / "board.mk" if not board_mk.exists(): board_mk = board_dir / "board.cmake" - mk_contents = board_mk.read_text() + mk_contents = board_mk.read_text(**_TEXT) mcu = "NONE" if family == "espressif": @@ -53,6 +202,95 @@ def skip_example(example, board): mcu = opt_mcu[len("OPT_MCU_"):] if mcu != "NONE": break + return mcu, mk_contents + + +@functools.lru_cache(maxsize=None) +def _board_mcu(board_dir, family_dir, family): + """(CFG_TUSB_MCU of this board, MAX3421_HOST enabled by its cmake BSP). + + MAX3421_HOST is read from family.cmake AND board.cmake rather than only the file + the MCU token came from: feather_rp2040_max3421 sets it in its board.cmake while + its MCU token comes from rp2040's family file, and family_support.cmake:940 + appends MAX3421 to FAMILY_MCUS for it. board.mk is deliberately not read - a + make-only option compiles nothing in a cmake build (and the make path answers + with master's own single-file scrape, see _skip_example_make).""" + family_dir = pathlib.Path(family_dir) + board_dir = pathlib.Path(board_dir) + mcu, _ = _scrape_mcu(family_dir, board_dir, family) + if "${" in mcu: + # the scrape is textual, so a computed token comes back verbatim + # (tm4c board.cmake spells OPT_MCU_TM4C${MCU_SUB_VARIANT}, maxim + # OPT_MCU_${MAX_DEVICE_UPPER}). Expand it the same way FAMILY_MCUS tokens are; + # what still will not resolve stays as-is and _skip_example treats it as + # "MCU unknown" rather than silently matching no mcu: token at all. + mcu = _cmake_expand(mcu, (str(board_dir / "board.cmake"), + str(family_dir / "family.cmake"))) or mcu + + max3421_enabled = False + for f in (family_dir / "family.cmake", board_dir / "board.cmake"): + try: + text = f.read_text(**_TEXT) + except OSError: + continue + # a commented-out `# set(MAX3421_HOST 1)` (feather_nrf52840_express) enables + # nothing; master never hit one because it only read the MCU token's file + if any(not l.lstrip().startswith('#') and + ("MAX3421_HOST=1" in l or 'MAX3421_HOST 1' in l) + for l in text.splitlines()): + max3421_enabled = True + break + + return mcu, max3421_enabled + + +@functools.lru_cache(maxsize=None) +def _filter_tokens(path): + """skip.txt / only.txt as a token set, or None when the file does not exist.""" + f = pathlib.Path(path) + return frozenset(f.read_text(**_TEXT).split()) if f.exists() else None + + +def skip_example(example, board, extra_defines=(), build_system='cmake'): + """Is this example unbuildable on this board, for this build system? + + The two build systems ask DIFFERENT questions and must not share an answer: + + 'cmake' mirrors CMake's family_filter (hw/bsp/family_support.cmake:171-207), + including the whole FAMILY_MCUS list the family.cmake sets. + + 'make' is master's original algorithm, unchanged. family.mk and family.cmake are + not the same build: hw/bsp/lpc54/family.cmake sets FAMILY_MCUS LPC54 and wires the + ohci host sources, while family.mk builds OPT_MCU_LPC54XXX and compiles no HCD + source at all -- feeding the cmake MCU union to a make build un-skips the host + examples only.txt gates on mcu:LPC54 and they fail to link (undefined hcd_init). + + extra_defines: NAME=VALUE tokens the build passes on the command line + (build.py -D). MAX3421_HOST=1 there enables the max3421 host controller + exactly like a BSP that sets it, and family_support.cmake:940 appends MAX3421 + to FAMILY_MCUS for it -- so a roster board whose MAX3421 comes from the build + args (metro_m4_express) must resolve its only.txt the same way. cmake only: + master's make algorithm never looked at them. + """ + return _skip_example(example, board, tuple(extra_defines), build_system) + + +@functools.lru_cache(maxsize=None) +def _skip_example_make(example, board): + """master's skip_example, verbatim (tools/build_utils.py @ 9c202e8c6): the + make build's own answer, derived from family.mk/board.mk with the single + CFG_TUSB_MCU token that file names. Do not "improve" it -- it is the mirror of + what `make BOARD=... all` actually compiles.""" + ex_dir = pathlib.Path('examples/') / example + + # board within family + board_dir, family_dir = _board_dirs(board) + if board_dir is None: + # Skip unknown boards + return True + family = family_dir.name + + mcu, mk_contents = _scrape_mcu(family_dir, board_dir, family) # Skip all OPT_MCU_NONE these are WIP port if mcu == "NONE": @@ -68,14 +306,14 @@ def skip_example(example, board): only_file = ex_dir / "only.txt" if skip_file.exists(): - skips = skip_file.read_text().split() + skips = skip_file.read_text(**_TEXT).split() if ("mcu:" + mcu in skips or "board:" + board in skips or "family:" + family in skips): return True if only_file.exists(): - onlys = only_file.read_text().split() + onlys = only_file.read_text(**_TEXT).split() if not ("mcu:" + mcu in onlys or ("mcu:MAX3421" in onlys and max3421_enabled) or "board:" + board in onlys or @@ -85,6 +323,55 @@ def skip_example(example, board): return False +@functools.lru_cache(maxsize=None) +def _skip_example(example, board, extra_defines, build_system): + if build_system == 'make': + return _skip_example_make(example, board) + + ex_dir = pathlib.Path('examples/') / example + + # board within family + board_dir, family_dir = _board_dirs(board) + if board_dir is None: + # Skip unknown boards + return True + family = family_dir.name + + mcu, max3421_enabled = _board_mcu(str(board_dir), str(family_dir), family) + + # Skip all OPT_MCU_NONE these are WIP port + if mcu == "NONE": + return True + + if any(t.strip().strip('"') == "MAX3421_HOST=1" for t in extra_defines): + max3421_enabled = True + + mcus = set(_family_mcus(str(family_dir), str(board_dir))) + if "${" not in mcu: + mcus.add(mcu) + if not mcus: + # nothing resolved: neither FAMILY_MCUS nor the scraped CFG_TUSB_MCU token + # yielded a name. Answering "skip" here would silently drop EVERY example on + # the board (an only.txt can then never match), so say "buildable" and let + # the real filter decide - build.py checks the targets CMake actually + # registered, and CMake itself is the authority on the make/cmake legs. + return False + if max3421_enabled: + mcus.add("MAX3421") # family_support.cmake:940 + + keys = {"board:" + board, "family:" + family} | {"mcu:" + m for m in mcus} + + skips = _filter_tokens(str(ex_dir / "skip.txt")) + if skips is not None and (skips & keys): + return True + + onlys = _filter_tokens(str(ex_dir / "only.txt")) + if onlys is not None and not (onlys & keys): + return True + + return False + + def build_size(make_cmd): size_output = subprocess.run(make_cmd + ' size', shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8").splitlines() for i, l in enumerate(size_output): diff --git a/tools/ci_select.py b/tools/ci_select.py new file mode 100755 index 000000000..d253f8c01 --- /dev/null +++ b/tools/ci_select.py @@ -0,0 +1,1084 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""PR-diff -> CI selection: which rig boards and which tests a change can affect. + +Lives in tools/ so it can serve both HIL selection and, from Task 3, build-family +selection. Stdlib-only (runs on bare CI runners; imports hil_util for the example +rosters, never hil_test/pyserial — test_hil_util.BottomLayer enforces the stdlib +closure). Fail-open: any file no rule classifies forces the full matrix. See +docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md and +docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md. + +JSON: full, boards (name -> 'all' | [tests]), families (bsp families the diff +touches, including ones with no rig board - build-only consumers such as /pre-pr +sample from these), args (hil_test.py args per config) and args_flasher (the same +args split by each board's flasher, for CI legs that split one rig by flasher). +""" +import argparse +import ast +import contextlib +import functools +import glob +import io +import json +import os +import re +import subprocess +import sys + +# tools/ -> repo root is ONE level up. Guarded by TestModuleMove.test_repo_root_guard: +# a wrong parent count here silently re-points every repo-relative glob (it happened +# at the helper/ move). +_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(_REPO_ROOT, 'test', 'hil')) # for `from helper...` +from helper.hil_util import device_tests, dual_tests, host_test + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # tools/, for build helpers +import build_utils +import build as build_py + +ALL_TESTS = {'device': device_tests, 'dual': dual_tests, 'host': host_test} + +# class dir -> config macro suffix exceptions (rule 3); dfu is per-file, handled inline +NET_MACROS = ('ECM_RNDIS', 'NCM') + +def _read(path: str) -> str: + """Read a source file with a fixed encoding. The locale's is not it: several tracked + sources carry non-ASCII bytes, and under LC_ALL=C the decode raises UnicodeDecodeError + - a ValueError, which every `except OSError` fail-open below would let through as a + traceback instead of a full matrix.""" + with open(path, encoding='utf-8', errors='replace') as f: + return f.read() + + +_NONCODE_RE = re.compile( + r'^(docs/|\.claude/|.*\.(md|rst)$|LICENSE)') +# Build-size metrics tooling. HIL axis ONLY: nothing on the rig runs any of it, and +# without this rule these paths are unclassified, so a metrics-only PR booked an +# exclusive full 30-board sweep to validate a script no board executes. +# The BUILD axis deliberately keeps its full-matrix answer: `tinyusb_metrics` runs +# tools/metrics.py as a build target (examples/CMakeLists.txt), and build_util.yml adds +# `--target tinyusb_metrics` to every metrics leg - a break in it fails the build, so a +# build has to exercise it. +_METRICS_RE = re.compile( + r'^(tools/metrics[^/]*\.py$|\.github/scripts/metrics_[^/]*\.py$)') +_FULL_RE = re.compile( + r'^(src/common/|src/osal/|src/tusb\.c$|src/tusb\.h$|src/tusb_option\.h$|' + r'test/hil/|\.github/workflows/build.*\.yml$|\.github/actions/|\.github/scripts/|' + r'tools/build\.py$|tools/cmake/|' + r'hw/bsp/(family_support\.cmake|board_api\.h|board\.c|ansi_escape\.h)$|' + r'examples/build_system/|examples/CMakeLists\.txt$|' + # board_test is HIL infrastructure, not a test: hil_test.py flashes it to park + # every board (variant boundary + end-of-board teardown), so every board depends on it + r'examples/device/board_test/)') + +# --no-renames: with rename detection git reports only a rename's destination, so code +# moved out of an HIL-relevant path would be classified by its new path alone +GIT_DIFF_ARGV = ['git', 'diff', '--no-renames', '--name-only'] + + +def test_role(test: str) -> str: + return test.split('/', 1)[0] # 'device' | 'dual' | 'host' + + +def board_roles(board: dict) -> set: + t = board.get('tests', {}) + roles = set() + if t.get('device'): + roles.add('device') + if t.get('host'): + roles.add('host') + if t.get('dual'): + roles.update(('device', 'host')) + for only in t.get('only', []): + r = test_role(only) + roles.update(('device', 'host') if r == 'dual' else (r,)) + return roles + + +def board_tests(board: dict) -> list: + """Every test this board would run today (mirrors hil_test.test_board's default).""" + t = board.get('tests', {}) + if 'only' in t: + run = list(t['only']) + else: + run = [] + if t.get('device'): + run += device_tests + if t.get('dual'): + run += dual_tests + if t.get('host'): + run += host_test + return [x for x in run if x not in t.get('skip', [])] + + +# cached: called per changed file x roster board, and the tree doesn't change mid-run +@functools.lru_cache(maxsize=None) +def board_family(board_name: str, repo_root: str): + hits = glob.glob(os.path.join(repo_root, 'hw/bsp/*/boards', board_name)) + return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None + + +# `if (OPTION STREQUAL "1")` guards in family_support.cmake, and the option tokens +# a roster entry passes to the build (NAME=VALUE / -DNAME=VALUE) +_CM_IF_RE = re.compile(r'if\s*\(') +_CM_ELSE_RE = re.compile(r'else(if)?\s*\(') +_CM_ENDIF_RE = re.compile(r'endif\s*\(') +_CM_OPT_RE = re.compile(r'if\s*\(\s*\$?\{?([A-Za-z_]\w*)\}?\s+STREQUAL\s+"?1"?\s*\)') +_CM_PORT_RE = re.compile(r'src/portable/((?:[^/\s]+/)?[^/\s]+)/') +_FALSY = ('', '0', 'off', 'false', 'no') + + +@functools.lru_cache(maxsize=None) +def port_option_gates(repo_root: str) -> dict: + """port dir -> build options that compile it regardless of the board's family + file, e.g. {'analog/max3421': {'MAX3421_HOST'}} from family_support.cmake.""" + gates = {} + try: + text = _read(os.path.join(repo_root, 'hw/bsp/family_support.cmake')) + except OSError: + return gates + stack = [] # one entry per open if(): its option, or None + for line in text.splitlines(): + line = line.strip() + if _CM_IF_RE.match(line): + m = _CM_OPT_RE.match(line) + stack.append(m.group(1) if m else None) + elif _CM_ELSE_RE.match(line): + if stack: + stack[-1] = None # the guard doesn't hold in this branch + elif _CM_ENDIF_RE.match(line): + if stack: + stack.pop() + opts = {o for o in stack if o} + m = _CM_PORT_RE.search(line) + if opts and m: + gates.setdefault(m.group(1), set()).update(opts) + return gates + + +_CM_SET_RE = re.compile(r'set\s*\(\s*([A-Za-z_]\w*)\s+([^)\s]+)\s*\)') + + +# cached: called per changed portable file x roster board +@functools.lru_cache(maxsize=None) +def bsp_board_options(board_name: str, repo_root: str) -> frozenset: + """Build options a board turns on in its own BSP: `set( )` in + hw/bsp//boards//board.cmake, e.g. MAX3421_HOST on the espressif + and rp2040 max3421 boards. CMake only - HIL CI builds nothing with Make, so a + board.mk-only option (e.g. nrf5340dk's MAX3421_HOST) compiles no port here.""" + fam = board_family(board_name, repo_root) + if not fam: + return frozenset() + path = os.path.join(repo_root, 'hw/bsp', fam, 'boards', board_name, 'board.cmake') + try: + text = _read(path) + except OSError: + return frozenset() + out = set() + for line in text.splitlines(): + line = line.strip() + if line.startswith('#'): + continue + m = _CM_SET_RE.match(line) + if m and m.group(2).strip('"').lower() not in _FALSY: + out.add(m.group(1)) + return frozenset(out) + + +def board_options(board: dict, repo_root: str) -> set: + """Build options a board has truthy: the roster entry's build.args plus each + variant's defines (NAME=VALUE) and raw CFLAGS (-DNAME=VALUE), plus whatever its + own board.cmake sets (a board can enable a gated port without the roster saying so).""" + toks = list(board.get('build', {}).get('args', [])) + for v in board.get('variant', []): + toks += list(v.get('defines', [])) + toks += v.get('flags', '').split() + out = set(bsp_board_options(board['name'], repo_root)) + for t in toks: + name, _, val = (t[2:] if t.startswith('-D') else t).partition('=') + if name and val.strip().strip('"').lower() not in _FALSY: + out.add(name.strip()) + return out + + +@functools.lru_cache(maxsize=None) +def path_families(rel_dir: str, repo_root: str) -> set: + """Board families whose family.cmake (or espressif component CMakeLists) + references rel_dir at a directory boundary. CMake only, on every axis: CMake + is the first-class build system and Make follows it, so family.mk is never + read - a port wired up in family.mk alone (microchip/pic32mz) is built by no + CI job and resolves to nothing. Boundary = '/', whitespace, quote, paren, + brace or end: `${TOP}/hw/mcu/nordic/nrfx` has no trailing slash, while bare + 'microchip/pic' must not match '.../microchip/pic32mz/...'.""" + fams = set() + bsp_root = os.path.join(repo_root, 'hw/bsp') + pat = re.compile(re.escape(rel_dir) + r'(?=[/\s"\')}]|$)', re.M) + for f in glob.glob(os.path.join(bsp_root, '*/family.cmake')) + \ + glob.glob(os.path.join(bsp_root, '*/components/*/CMakeLists.txt')): + try: + if pat.search(_read(f)): + fams.add(os.path.relpath(f, bsp_root).split(os.sep, 1)[0]) + except OSError: + pass + return fams + + +def port_families(port_dir: str, repo_root: str) -> set: + # 'portable/', not 'src/portable/': family.cmake always spells the full literal + # path ('${TOP}/src/portable/...'), but espressif's component CMakeLists.txt + # assigns 'src' into a ${tusb_src} variable first (`${tusb_src}/portable/...`), + # so a leading 'src/' in the needle would never match there and silently drop + # espressif boards (see TestRealRosterPortFamilies). + return path_families('portable/' + port_dir, repo_root) + + +def mcu_families(path: str, repo_root: str) -> set: + """Families referencing a changed hw/mcu path: longest resolving dir prefix, + hw/mcu///... down to hw/mcu/.""" + parts = path.split('/') + for n in range(len(parts) - 1, 2, -1): + fams = path_families('/'.join(parts[:n]), repo_root) + if fams: + return fams + return set() + + +GET_DEPS_PATH = 'tools/get_deps.py' +_DEPS_DICTS = ('deps_mandatory', 'deps_optional') + + +def _deps_split(text: str): + """(module dump with the two dep-dict assigns removed, {dict name: entries}). + Parsed with ast, never exec'd: this runs on PR content.""" + mod = ast.parse(text) + dicts, rest = {}, [] + for node in mod.body: + if (isinstance(node, ast.Assign) and len(node.targets) == 1 and + isinstance(node.targets[0], ast.Name) and + node.targets[0].id in _DEPS_DICTS and isinstance(node.value, ast.Dict)): + dicts[node.targets[0].id] = ast.literal_eval(node.value) + else: + rest.append(node) + mod.body = rest + # annotate_fields=False keeps the dump readable-length; line numbers are not + # included unless asked for, so reformatting alone never reads as a logic change + return ast.dump(mod, annotate_fields=False), dicts + + +# Family tokens in tools/get_deps.py that name no hw/bsp directory. get_deps matches a +# token against a requested family name verbatim (`f in deps_optional[d][2].split()`), +# so a token like these matches nothing - a stale spelling in get_deps.py, not a +# selector bug, and out of scope to change here. Pinned so that any OTHER unresolvable +# token (real drift) falls open to the full matrix instead of silently selecting +# nothing, and so TestOrphanInvariant fails the day one is fixed or a new one appears. +# sam3x, samd21, samd51, same5x -> pre-rename spellings, listed alongside the current +# samd2x_l2x / samd5x_e5x / same7x in the same entry +# stm32l1, stm32l5 -> no hw/bsp family in the tree at all +_DEPS_ALIAS_TOKENS = frozenset({'sam3x', 'samd21', 'samd51', 'same5x', + 'stm32l1', 'stm32l5'}) + + +def get_deps_changed_families(base_text: str, head_text: str, repo_root: str): + """Families whose tools/get_deps.py dep entries changed between two versions of + the file, or None meaning 'cannot tell - use the full matrix'. + + None on: anything outside deps_mandatory/deps_optional differing (a logic change + to get_deps affects every family), a mandatory `'all'` entry changing, a token + that resolves to no family and is not a known alias, or text that will not parse. + Callers with no base content at all - `--diff-file` mode has no git and therefore + no merge-base blob - pass None themselves. + + An entry that is added, removed or edited contributes the family tokens of BOTH + sides (a removed entry has only a base side). The two dicts are diffed SEPARATELY: + merging them first would hide a move between deps_mandatory and deps_optional, + which changes which families fetch the dep even though the value is untouched.""" + try: + base_rest, base_d = _deps_split(base_text) + head_rest, head_d = _deps_split(head_text) + except (SyntaxError, ValueError, TypeError): + return None + if base_rest != head_rest: + return None + toks = set() + for name in _DEPS_DICTS: + base_x, head_x = base_d.get(name, {}), head_d.get(name, {}) + for key in set(base_x) | set(head_x): + if base_x.get(key) == head_x.get(key): + continue + for entry in (base_x.get(key), head_x.get(key)): + if entry and len(entry) > 2: + toks.update(str(entry[2]).split()) + if 'all' in toks: + return None + fams = set(all_bsp_families(repo_root)) + if toks - fams - _DEPS_ALIAS_TOKENS: + # a changed entry we cannot map to a family. "changed but unmappable" is NOT + # "nothing changed": reading it as the latter empties the entire build matrix + # for a dep bump, so fall open instead + return None + return toks & fams + + +_CLS_INC_RE = re.compile(r'#\s*include\s*[<"]class/([^/"<>]+)/([^"<>]+)[">]') + + +@functools.lru_cache(maxsize=None) +def class_include_edges(repo_root: str) -> dict: + """'/
' -> the other class dirs that include it. A class header + pulled in by a second class ships in every firmware enabling that second class: + src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, and + net_device.h includes class/cdc/cdc.h. The class rule derives macros from the + directory name alone, so without this edge a change to the included header + selects only its own class's examples - and on a board that skips those (e.g. + metro_m4_express skips audio_test_freertos), nothing at all. + + Derived from the actual #include lines rather than a hand-written table so it + cannot rot when a class picks up or drops a cross-class include.""" + edges = {} + for f in sorted(glob.glob(os.path.join(repo_root, 'src/class/*/*.[ch]'))): + cls = os.path.basename(os.path.dirname(f)) + try: + text = _read(f) + except OSError: + continue + for inc_cls, inc_hdr in _CLS_INC_RE.findall(text): + if inc_cls != cls: + edges.setdefault(f'{inc_cls}/{inc_hdr}', set()).add(cls) + return edges + + +def class_macros(cls: str, base: str, prefix: str) -> list: + """Config macros that compile a class dir's code, for role prefix TUD/TUH. + `base` refines dfu only (it splits DFU from DFU_RUNTIME per file); pass '' for + a class reached through an include edge, where the widest set is correct.""" + if cls == 'net': + return [f'CFG_{prefix}_{m}' for m in NET_MACROS] + if cls == 'dfu': + if base.startswith('dfu_rt'): + return [f'CFG_{prefix}_DFU_RUNTIME'] + if base.startswith('dfu_device') or base.startswith('dfu_host'): + return [f'CFG_{prefix}_DFU'] + return [f'CFG_{prefix}_DFU', f'CFG_{prefix}_DFU_RUNTIME'] + return [f'CFG_{prefix}_{cls.upper()}'] + + +# A define is OFF only when its value is a literal zero (0, 00, (0)), optionally +# followed by a comment. Anything else counts as ON - including a value this cannot +# evaluate, e.g. `#define CFG_TUH_MIDI CFG_TUH_DEVICE_MAX` (examples/host/midi_rx). +# Fail-open: reading such a define as OFF made midi_host.c select zero families and +# let a compile break merge green. +# +# A macro defined more than once is ON if ANY of its defines is non-zero, because +# the preprocessor branches are not evaluated here: uac2_speaker_fb defines +# CFG_TUD_HID 1 under `#if CFG_AUDIO_DEBUG` and 0 in the #else, and the default +# build (CFG_AUDIO_DEBUG defaults to 1) compiles the HID class in. Deciding on the +# LAST/only match found made that example invisible to CFG_TUD_HID changes. +_DEF_VALUE = r'^[ \t]*#[ \t]*define[ \t]+{}[ \t]+(\S[^\n]*?)[ \t]*$' +_DEF_ZERO_VALUE = re.compile(r'\(?\s*0+\s*\)?\s*(?://.*|/\*.*)?') + + +# Shared rule-recognition primitives. The two classifiers walk the same diff with +# different answers, but they must RECOGNISE the same things: one copy each, so a +# new naming convention cannot land in one walk and be missed by the other. +_PORT_PATH_RE = re.compile(r'src/portable/((?:[^/]+/)?[^/]+)/') + + +def _port_roles(base: str) -> set: + """Which USB role a src/portable file serves, from its name: dcd_*/ *_device is + the device-controller side, hcd_*/ *_host the host side, anything else (shared + headers, glue) both.""" + if re.match(r'(dcd_|.*_device)', base): + return {'device'} + if re.match(r'(hcd_|.*_host)', base): + return {'host'} + return {'device', 'host'} + + +def _class_roles(base: str) -> set: + """Same question for a src/class file: _device.[ch] / _host.[ch], + else both - the class's shared header ships in either role.""" + if re.search(r'_device\.[ch]$', base): + return {'device'} + if re.search(r'_host\.[ch]$', base): + return {'host'} + return {'device', 'host'} + + +def _config_enables(cfg_path: str, macros) -> bool: + try: + with open(cfg_path) as f: + text = f.read() + except OSError: + return False + for m in macros: + for value in re.findall(_DEF_VALUE.format(m), text, re.M): + if not _DEF_ZERO_VALUE.fullmatch(value): + return True + return False + + +def examples_enabling(pool, macros, repo_root: str) -> set: + """The 'role/name' entries of `pool` whose src/tusb_config.h turns any of + `macros` on. The pool differs per classifier (HIL test lists vs every example), + the question does not.""" + return {ex for ex in pool + if _config_enables(os.path.join(repo_root, 'examples', ex, 'src', + 'tusb_config.h'), macros)} + + +# cached: called per changed lib file, and the tree doesn't change mid-run +@functools.lru_cache(maxsize=None) +def lib_examples(lib_name: str, repo_root: str) -> set: + """Examples whose OWN examples///{CMakeLists.txt,Makefile} references + lib/ at a directory boundary (same boundary rule as path_families, so + 'lib/net' cannot inherit lib/networking's example). + + Per-example on purpose: lib/SEGGER_RTT is named by family_support.cmake's + LOGGER=rtt plumbing, which no CI example build turns on, so a family-file scan + would wrongly narrow it to three families instead of answering 'nobody'.""" + pat = re.compile(re.escape('lib/' + lib_name) + r'(?=[/\s"\')}]|$)', re.M) + out = set() + for ex in all_examples(repo_root): + for f in ('CMakeLists.txt', 'Makefile'): + try: + with open(os.path.join(repo_root, 'examples', ex, f)) as fh: + text = fh.read() + except OSError: + continue + if pat.search(text): + out.add(ex) + break + return out + + +def roster_only_tests(all_boards) -> set: + """Test paths that only appear in a roster board's tests.only list (e.g. + espressif boards), not in the shared device/dual/host_test lists.""" + out = set() + for b in all_boards: + out.update(b.get('tests', {}).get('only', [])) + return out + + +def class_examples(macros, role: str, repo_root: str, extra_tests: set) -> set: + """Tests (from role's + dual lists, plus roster-only-list tests of that role) + whose example config enables any macro.""" + return examples_enabling(role_tests({role}, extra_tests), macros, repo_root) + + +def role_tests(roles: set, extras: set) -> set: + """Every test for the given role(s): each role's own list + dual tests, + plus roster-only-list tests (extras) matching those roles or 'dual'.""" + pool = set(dual_tests) + for r in roles: + pool |= set(ALL_TESTS[r]) + pool |= {t for t in extras if test_role(t) in roles or test_role(t) == 'dual'} + return pool + + +class _Sel: + """Accumulates contributions. board->set(tests) plus 'all-board' markers.""" + def __init__(self): + self.full = False + self.by_board = {} # name -> set of tests, or 'all' + self.roles = set() # roles touched by any contribution + self.families = set() # bsp families touched (incl. off-rig ones: build-only consumers) + self.reasons = [] + + def add(self, boards, tests, reason): + """tests: 'all' or iterable of test paths.""" + self.reasons.append(reason) + for b in boards: + cur = self.by_board.get(b) + if tests == 'all' or cur == 'all': + self.by_board[b] = 'all' + else: + self.by_board[b] = (cur or set()) | set(tests) + + def force_full(self, reason): + self.full = True + self.reasons.append(reason) + + +def _classify_one(path, repo_root, roster_boards, extras: set, s: _Sel, + get_deps_families=None): + base = os.path.basename(path) + if _NONCODE_RE.match(path): + s.reasons.append(f'{path}: non-code, no contribution') + return + if _METRICS_RE.match(path): + s.reasons.append(f'{path}: build-size metrics tooling, no HIL contribution') + return + if _FULL_RE.match(path): + s.force_full(f'{path}: core/infra -> full matrix') + return + + if path == GET_DEPS_PATH: + if get_deps_families is None: + s.force_full(f'{path}: dep changes not resolvable -> full matrix') + return + if not get_deps_families: + s.reasons.append(f'{path}: no dep entry changed, no contribution') + return + fams = sorted(get_deps_families) + s.families.update(fams) + boards = [b['name'] for b in roster_boards + if board_family(b['name'], repo_root) in get_deps_families] + s.roles.update(('device', 'host')) + s.add(boards, 'all', f'{path}: dep entries changed -> families {fams} -> ' + f'boards {boards}') + return + + m = _PORT_PATH_RE.match(path) + if m: + port = m.group(1) + roles = _port_roles(base) + fams = port_families(port, repo_root) + if not fams: + # empty means empty (maintainer ruling), same reading as hw/mcu and as the + # build walk: no family's build references this port, so nothing compiles it + # and there is nothing to run. Forcing the full 30-board rig here bought no + # coverage at all - the build side selected zero families for the same path. + # Live for src/portable/template and the two microchip pic ports; + # TestPortFamiliesCoverage is the drift guard for a port that stops resolving. + s.reasons.append(f'{path}: port {port} maps to no board family, no contribution') + return + s.families.update(fams) + # a board can also pull the port in through a build option (e.g. MAX3421_HOST=1 + # from the roster on metro_m4_express, or from its own board.cmake), which its + # family file never names + gates = port_option_gates(repo_root).get(port, set()) + boards = [b['name'] for b in roster_boards + if (board_family(b['name'], repo_root) in fams or + (gates and board_options(b, repo_root) & gates)) and (board_roles(b) & roles)] + tests = role_tests(roles, extras) + s.roles.update(roles) + why = f'{path}: port {port} -> families {sorted(fams)}' + if gates: + why += f' + option {sorted(gates)}' + s.add(boards, tests, f'{why} -> boards {boards} ({"/".join(sorted(roles))})') + return + + m = re.match(r'src/class/([^/]+)/', path) + if m: + cls = m.group(1) + roles = _class_roles(base) + # this file's own class, plus any class whose headers include it + via = sorted(class_include_edges(repo_root).get(f'{cls}/{base}', ())) + + def macros(prefix): + return (class_macros(cls, base, prefix) + + [m2 for c in via for m2 in class_macros(c, '', prefix)]) + tests = set() + if 'device' in roles: + tests |= class_examples(macros('TUD'), 'device', repo_root, extras) + if 'host' in roles: + tests |= class_examples(macros('TUH'), 'host', repo_root, extras) + boards = [b['name'] for b in roster_boards if board_roles(b) & roles] + s.roles.update(roles) + why = f'{path}: class {cls}' + (f' (+ included by {via})' if via else '') + s.add(boards, tests, f'{why} -> {sorted(tests)} ({"/".join(sorted(roles))})') + return + + m = re.match(r'src/(device|host)/', path) + if m: + role = m.group(1) + boards = [b['name'] for b in roster_boards if role in board_roles(b)] + s.roles.add(role) + s.add(boards, role_tests({role}, extras), f'{path}: core {role} stack -> all {role} tests') + return + + m = re.match(r'hw/bsp/([^/]+)/(?:boards/([^/]+)/)?', path) + if m: + fam, brd = m.group(1), m.group(2) + s.families.add(fam) + if brd: + boards = [b['name'] for b in roster_boards if b['name'] == brd] + why = f'{path}: bsp board {brd}' + else: + boards = [b['name'] for b in roster_boards + if board_family(b['name'], repo_root) == fam] + why = f'{path}: bsp family {fam}' + s.roles.update(('device', 'host')) + s.add(boards, 'all', f'{why} -> boards {boards}') + return + + if re.match(r'hw/mcu/', path): + fams = mcu_families(path, repo_root) + if not fams: + # empty means empty (maintainer ruling): if no family's build references + # the path, no build consumes the change - there is nothing to compile, + # so there is nothing to run either. TestOrphanInvariant's + # test_tracked_mcu_vendors_resolve is the drift guard: a real vendor dir + # that stops resolving fails pre-commit instead of silently vanishing + s.reasons.append(f'{path}: hw/mcu path resolves to no family, no contribution') + return + s.families.update(fams) + boards = [b['name'] for b in roster_boards + if board_family(b['name'], repo_root) in fams] + s.roles.update(('device', 'host')) + s.add(boards, 'all', f'{path}: mcu dir -> families {sorted(fams)} -> boards {boards}') + return + + m = re.match(r'lib/([^/]+)/', path) + if m: + lib = m.group(1) + # only the tests whose example builds the lib, and only those the rig runs + tests = {e for e in lib_examples(lib, repo_root) + if any(e in pool for pool in ALL_TESTS.values()) or e in extras} + if not tests: + s.reasons.append(f'{path}: lib {lib} used by no HIL test, no contribution') + return + roles = set() + for test in tests: + r = test_role(test) + roles.update(('device', 'host') if r == 'dual' else (r,)) + boards = [b['name'] for b in roster_boards] + s.roles.update(roles) + s.add(boards, sorted(tests), f'{path}: lib {lib} -> {sorted(tests)} on all boards') + return + + m = _BUILD_EX_RE.match(path) + if m: + if m.group(1) not in _HIL_EX_ROLES: + # examples/typec: the build matrix compiles it, nothing on the rig runs it + s.reasons.append(f'{path}: {m.group(1)} example, no HIL contribution') + return + test = f'{m.group(1)}/{m.group(2)}' + known = any(test in pool for pool in ALL_TESTS.values()) or test in extras + if known: + boards = [b['name'] for b in roster_boards] + role = test_role(test) + s.roles.update(('device', 'host') if role == 'dual' else (role,)) + s.add(boards, [test], f'{path}: example -> {test} on all boards') + else: + s.reasons.append(f'{path}: example not in HIL lists, no contribution') + return + + s.force_full(f'{path}: unclassified -> full matrix') + + +def classify(changed_files, repo_root, rosters, get_deps_families=None): + all_boards = [] + seen = set() + for _, boards in rosters: + for b in boards: + if b['name'] not in seen: + seen.add(b['name']) + all_boards.append(b) + + extras = roster_only_tests(all_boards) + s = _Sel() + # no early exit once full: keep classifying so `families` still reports every + # family the diff touches (build-only consumers need it). Nothing after the first + # force_full can change full/boards/args - the full branch below ignores by_board. + for path in changed_files: + _classify_one(path, repo_root, all_boards, extras, s, get_deps_families) + + if s.full: + return {'full': True, 'boards': {b['name']: 'all' for b in all_boards}, + 'families': sorted(s.families), 'reasons': s.reasons} + + # role pruning: single-role selections drop the other role's tests and boards + by_name = {b['name']: b for b in all_boards} + out = {} + for name, tests in s.by_board.items(): + allowed = board_tests(by_name[name]) + if tests == 'all': + kept = list(allowed) + else: + kept = [t for t in allowed if t in tests] + if s.roles and s.roles != {'device', 'host'}: + role = next(iter(s.roles)) + kept = [t for t in kept if test_role(t) in (role, 'dual')] + if kept: + out[name] = 'all' if set(kept) == set(allowed) else sorted(kept) + return {'full': False, 'boards': out, 'families': sorted(s.families), + 'reasons': s.reasons} + + +def _board_args(name, chosen) -> list: + parts = [f'-b {name}'] + if chosen != 'all': + parts.append(f'-bt {name}:{",".join(chosen)}') + return parts + + +def hil_examples(sel, rosters): + """{board: examples hil-build must produce}: the board's selected tests plus + device/board_test, which hil_test.py flashes to park at every variant + boundary and at end-of-board teardown. Emitted for full selections too - the + HIL example universe is a fraction of the tree regardless of the diff.""" + by_name = {} + for _, boards in rosters: + for b in boards: + by_name.setdefault(b['name'], []).append(b) + if sel['full']: + chosen = {n: 'all' for n in by_name} + else: + chosen = sel['boards'] + out = {} + for name, tests in chosen.items(): + if tests == 'all': + # a board named by two rosters (rig migration, or shared between rigs) + # may run different tests on each: union them. Superset firmware costs a + # build; a missing image fails the run on whichever rig lost the toss. + run = set().union(*(board_tests(b) for b in by_name[name])) + else: + run = set(tests) + out[name] = sorted(run | {'device/board_test'}) + return out + + +def selection_args(sel, rosters): + """hil_test.py args per config. Empty means either 'full matrix' or 'nothing + selected' - callers must read sel['full'] to tell them apart.""" + args = {} + for cfg_path, boards in rosters: + parts = [] + if not sel['full']: + for b in boards: + chosen = sel['boards'].get(b['name']) + if chosen is not None: + parts += _board_args(b['name'], chosen) + args[os.path.basename(cfg_path)] = ' '.join(parts) + return args + + +def selection_args_by_flasher(sel, rosters): + """{config: {flasher name: args}}. CI runs one rig as several jobs split by + flasher (esptool vs the rest); each must gate on its own subset, otherwise the + other leg runs a filter matching zero boards and reports a vacuous green.""" + out = {} + for cfg_path, boards in rosters: + per = {} + if not sel['full']: + for b in boards: + chosen = sel['boards'].get(b['name']) + if chosen is None: + continue + per.setdefault(b.get('flasher', {}).get('name', ''), []).extend( + _board_args(b['name'], chosen)) + out[os.path.basename(cfg_path)] = {f: ' '.join(p) for f, p in per.items()} + return out + + +def merge_base(base, repo_root): + return subprocess.run(['git', 'merge-base', 'HEAD', base], cwd=repo_root, + capture_output=True, text=True, check=True).stdout.strip() + + +def git_show(spec, repo_root): + return subprocess.run(['git', 'show', spec], cwd=repo_root, + capture_output=True, text=True, check=True).stdout + + +def changed_files_from_git(base, repo_root): + diff = subprocess.run(GIT_DIFF_ARGV + [f'{merge_base(base, repo_root)}..HEAD'], + cwd=repo_root, capture_output=True, text=True, check=True).stdout + return [l for l in diff.splitlines() if l.strip()] + + +def get_deps_families_from_git(base, repo_root): + """The changed dep entries' families for a --base run, or None (-> full matrix) + if git cannot produce both sides of tools/get_deps.py.""" + try: + mb = merge_base(base, repo_root) + return get_deps_changed_families(git_show(f'{mb}:{GET_DEPS_PATH}', repo_root), + git_show(f'HEAD:{GET_DEPS_PATH}', repo_root), + repo_root) + except (subprocess.CalledProcessError, OSError) as e: + print(f'ci_select: {GET_DEPS_PATH}: base content unreadable ({e})', file=sys.stderr) + return None + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + g = ap.add_mutually_exclusive_group(required=True) + g.add_argument('--base', help='git ref to diff against (merge-base..HEAD)') + g.add_argument('--diff-file', help='newline-separated changed-file list') + ap.add_argument('configs', nargs='*', help='rig roster JSON file(s); omit for the build view alone') + a = ap.parse_args() + + repo_root = _REPO_ROOT + rosters = [] + for c in a.configs: + with open(c) as f: + rosters.append((c, json.load(f)['boards'])) + + files = (_read(a.diff_file).splitlines() if a.diff_file + else changed_files_from_git(a.base, repo_root)) + files = [f for f in files if f.strip()] + + # --diff-file has no git and so no base content: the rule falls open to full + gd = (get_deps_families_from_git(a.base, repo_root) + if a.base and GET_DEPS_PATH in files else None) + + s = classify(files, repo_root, rosters, gd) + s['args'] = selection_args(s, rosters) + s['args_flasher'] = selection_args_by_flasher(s, rosters) + if rosters: + s['hil_examples'] = hil_examples(s, rosters) + s['build'] = classify_build(files, repo_root, gd) + for r in s['build']['reasons']: + print(f'ci_select[build]: {r}', file=sys.stderr) + for r in s['reasons']: + print(f'ci_select: {r}', file=sys.stderr) + print(json.dumps(s)) + + +# ------------------------------------------------------------- +# Build-axis classifier (spec rule table, docs/superpowers/specs/ +# 2026-08-19-ci-build-family-filter-design.md). Independent of the HIL +# classifier: same diff, second walk, its own fail-open. +# ------------------------------------------------------------- +# Both walks recognise an example path with the SAME regex, so a role can never be +# known to one walk and unclassified (-> full matrix) to the other. What differs is the +# answer: the rig runs device/host/dual tests, while the build matrix also compiles +# examples/typec, which nothing on the rig runs. +_EX_ROLES = ('device', 'dual', 'host', 'typec') +_HIL_EX_ROLES = ('device', 'host', 'dual') +_BUILD_EX_RE = re.compile(r'examples/(%s)/([^/]+)/' % '|'.join(_EX_ROLES)) + + +@functools.lru_cache(maxsize=None) +def all_examples(repo_root: str) -> tuple: + """Every examples// with a CMakeLists.txt, as 'role/name'.""" + out = [] + for role in _EX_ROLES: + for d in sorted(glob.glob(os.path.join(repo_root, 'examples', role, '*/'))): + if os.path.isfile(os.path.join(d, 'CMakeLists.txt')): + out.append(f'{role}/{os.path.basename(d.rstrip(os.sep))}') + return tuple(out) + + +def role_examples(repo_root: str, roles) -> set: + want = set(roles) + return {e for e in all_examples(repo_root) if e.split('/', 1)[0] in want} + + +@functools.lru_cache(maxsize=None) +def all_bsp_families(repo_root: str) -> tuple: + return tuple(sorted(d for d in os.listdir(os.path.join(repo_root, 'hw/bsp')) + if os.path.isdir(os.path.join(repo_root, 'hw/bsp', d)))) + + +def _build_class_examples(cls: str, base: str, roles: set, repo_root: str) -> set: + """Examples (all 46, not the HIL lists) whose tusb_config.h enables the class's + macros for the given roles, plus classes that #include the changed header.""" + via = sorted(class_include_edges(repo_root).get(f'{cls}/{base}', ())) + out = set() + for prefix, role in (('TUD', 'device'), ('TUH', 'host')): + if role not in roles: + continue + macros = class_macros(cls, base, prefix) + \ + [m for c in via for m in class_macros(c, '', prefix)] + out |= examples_enabling(all_examples(repo_root), macros, repo_root) + return out + + +class _BSel: + """family -> set(examples) | 'all', unioned per family.""" + def __init__(self): + self.full = False + self.fam_ex = {} + self.reasons = [] + + def add(self, fams, examples, reason): + self.reasons.append(reason) + for f in fams: + cur = self.fam_ex.get(f) + if examples == 'all' or cur == 'all': + self.fam_ex[f] = 'all' + else: + self.fam_ex[f] = (cur or set()) | set(examples) + + def force_full(self, reason): + self.full = True + self.reasons.append(reason) + + +def _classify_build_one(path, repo_root, s: _BSel, get_deps_families=None): + base = os.path.basename(path) + if _NONCODE_RE.match(path): # rule 1 + return + if re.match(r'test/hil/', path): # rule 2 + s.reasons.append(f'{path}: HIL harness, no build contribution') + return + if path == GET_DEPS_PATH: # get_deps rule + if get_deps_families is None: + s.force_full(f'{path}: dep changes not resolvable -> full build matrix') + return + if not get_deps_families: + s.reasons.append(f'{path}: no dep entry changed, no contribution') + return + fams = sorted(get_deps_families) + s.add(fams, 'all', f'{path}: dep entries changed -> families {fams}') + return + m = _PORT_PATH_RE.match(path) + if m: # rules 3-5 + port = m.group(1) + fams = port_families(port, repo_root) + roles = _port_roles(base) + exs = 'all' if roles == {'device', 'host'} else \ + role_examples(repo_root, tuple(roles) + ('dual',)) + s.add(fams, exs, f'{path}: port {port} -> families {sorted(fams)}') + return + if re.match(r'hw/bsp/[^/]+/', path): # rule 6 + fam = path.split('/')[2] + s.add({fam}, 'all', f'{path}: bsp family {fam}') + return + if re.match(r'hw/mcu/', path): # rule 7 + fams = mcu_families(path, repo_root) + if not fams: + # empty means empty, same reading as the HIL walk: no family's build + # references the path, so no build compiles it + s.reasons.append(f'{path}: hw/mcu path resolves to no family, no contribution') + return + s.add(fams, 'all', f'{path}: mcu -> families {sorted(fams)}') + return + m = re.match(r'src/class/([^/]+)/', path) + if m: # rules 8-10 + cls = m.group(1) + roles = _class_roles(base) + exs = _build_class_examples(cls, base, roles, repo_root) + if not exs: + # Empty means empty - maintainer decision. No example config enables this + # class, so no build exercises it and + # nothing is selected. The file IS still parsed by every full build + # (src/CMakeLists.txt, src/tinyusb.mk list class sources unconditionally, + # the CFG_ guard sits inside), so a break outside the guard surfaces on the + # next master push - the accepted safety net. + s.reasons.append(f'{path}: class {cls} enabled by no example config, ' + f'no contribution') + return + s.add(all_bsp_families(repo_root), exs, + f'{path}: class {cls} -> {sorted(exs)}') + return + m = re.match(r'src/(device|host)/', path) + if m: # rules 11-12 + role = m.group(1) + s.add(all_bsp_families(repo_root), role_examples(repo_root, (role, 'dual')), + f'{path}: core {role} stack') + return + m = _BUILD_EX_RE.match(path) + if m: # rules 13-14 + ex = f'{m.group(1)}/{m.group(2)}' + if ex in all_examples(repo_root): + s.add(all_bsp_families(repo_root), {ex}, f'{path}: example {ex}') + else: + # a deleted example builds nothing; removing it from the role + # CMakeLists (rule 15) is what forces the full matrix + s.reasons.append(f'{path}: not an example dir, no build contribution') + return + m = re.match(r'lib/([^/]+)/', path) + if m: # lib rule + lib = m.group(1) + exs = lib_examples(lib, repo_root) + if not exs: + # empty means empty: no example's build pulls this lib in, so no build + # compiles it (lib/SEGGER_RTT is only reached through LOGGER=rtt, which + # no CI build sets) + s.reasons.append(f'{path}: lib {lib} built by no example, no contribution') + return + s.add(all_bsp_families(repo_root), exs, f'{path}: lib {lib} -> {sorted(exs)}') + return + s.force_full(f'{path}: unclassified -> full build matrix') # rules 15-17 + + +@contextlib.contextmanager +def _in_repo(repo_root): + """build_utils/build.py use repo-relative paths; scope a chdir around them. + get_family_boards also prints on an empty family - swallow stdout so the + selector's machine-read JSON stays clean (diagnostics belong on stderr).""" + old = os.getcwd() + os.chdir(repo_root) + try: + with contextlib.redirect_stdout(io.StringIO()): + yield + finally: + os.chdir(old) + + +def _prune_buildable(fams, fam_ex, repo_root): + """Intersect each family's selection with what the family can build at all + (build_utils.skip_example - the same skip.txt/only.txt data CMake's + family_filter reads). + + ANY board of the family counts, not just the one GHA's --one-first picks: + CircleCI's cmake legs build every board of a family, so an example gated to a + single board (only.txt board:mimxrt1060_evk) would otherwise lose ALL compile + coverage exactly when a PR touches it. get_family_boards(.., False, False) is + that full list, with the same CI skip lists the build jobs apply. + + EITHER build system counts too. This one list gates CircleCI's make legs as well + as its cmake ones, and the two answer different questions (build_utils.skip_example): + examples/device/dfu carries `mcu:BCM2835` in skip.txt, which the cmake FAMILY_MCUS + union applies to every broadcom_64bit board while the make scrape applies it to + none - asking cmake alone drops the only aarch64-gcc family in the matrix and + `build-make-aarch64-gcc` stops compiling dfu at all.""" + out_fams, out_ex, reasons = [], {}, [] + allex = list(all_examples(repo_root)) + with _in_repo(repo_root): + for fam in fams: + if not os.path.isdir(os.path.join(repo_root, 'hw/bsp', fam, 'boards')): + # a PR that deletes or renames hw/bsp/ still names it in the + # diff (rule 6); the family builds nothing now, and get_family_boards + # would raise FileNotFoundError out of the whole selector + reasons.append(f'{fam}: family dir gone from tree, dropped') + continue + try: + boards = build_py.get_family_boards(fam, False, False) + except OSError as e: # belt and braces: never traceback here + reasons.append(f'{fam}: boards unreadable ({e}), dropped') + continue + if not boards: + out_fams.append(fam) # unknown layout: keep unfiltered + continue + # what this family's build path can even see, asked the same way for + # every family. build.py's espressif branch builds get_examples('espressif') + # only (the *_freertos examples plus a short extra list); keeping the family + # for anything else spins up CI's most expensive leg to skip every example + # it was given. Identical to the unfiltered list on all 81 other families. + pool = set(build_py.get_examples(fam)) + try: + buildable = [e for e in allex if e in pool and + any(not build_utils.skip_example(e, b) or + not build_utils.skip_example(e, b, (), 'make') + for b in boards)] + except OSError as e: + # a family mid-bring-up (boards/ but no family.cmake/family.mk yet) + # reads as unbuildable to the scrape; keep it rather than tracebacking + # out of the selector and losing the scoping for the whole PR + reasons.append(f'{fam}: mcu scrape unreadable ({e}), kept unfiltered') + out_fams.append(fam) + continue + want = fam_ex.get(fam) + have = set(buildable) + kept = buildable if want is None else [e for e in want if e in have] + if not kept: + continue # this diff builds nothing for this family + out_fams.append(fam) + if set(kept) != set(buildable): + out_ex[fam] = kept + return out_fams, out_ex, reasons + + +def classify_build(changed_files, repo_root, get_deps_families=None): + s = _BSel() + for p in changed_files: + _classify_build_one(p, repo_root, s, get_deps_families) + if s.full: + return {'full': True, 'families': list(all_bsp_families(repo_root)), + 'family_examples': {}, 'reasons': s.reasons} + fams = sorted(s.fam_ex) + fam_ex = {f: sorted(e) for f, e in s.fam_ex.items() if e != 'all'} + fams, fam_ex, pruned = _prune_buildable(fams, fam_ex, repo_root) + s.reasons += pruned + return {'full': False, 'families': fams, 'family_examples': fam_ex, + 'reasons': s.reasons} + + +if __name__ == '__main__': + main() diff --git a/tools/get_deps.py b/tools/get_deps.py index f8161a933..12bec4861 100755 --- a/tools/get_deps.py +++ b/tools/get_deps.py @@ -386,6 +386,10 @@ def main(): parser.add_argument('-f1', '--build-flags-on', action='append', default=[], help='Have no effect') parser.add_argument('--build-name', default=None, help='Have no effect') parser.add_argument('--cflag', action='append', default=[], help='Have no effect') + # build-matrix entries carry -e for tools/build.py; they reach get_deps.py + # verbatim (.github/actions/get_deps, build.yml's hil-hfp-iar) and an + # argparse error here reds the Get Dependencies step of every scoped PR + parser.add_argument('-e', '--example', action='append', default=[], help='Have no effect') args = parser.parse_args() families = args.families diff --git a/tools/metrics.py b/tools/metrics.py index 0e29fc1ab..b97b2b206 100644 --- a/tools/metrics.py +++ b/tools/metrics.py @@ -83,7 +83,7 @@ def parse_bloaty_csv(csv_text, filters=None): return {"files": files, "TOTAL": total_all} -def combine_files(input_files, filters=None): +def combine_files(input_files, filters=None, only_examples=None): """Combine multiple metrics inputs (bloaty CSV or metrics JSON) into a single data set.""" filters = filters or [] @@ -98,6 +98,22 @@ def combine_files(input_files, filters=None): if fin.endswith(".json"): with open(fin, "r", encoding="utf-8") as f: json_data = json.load(f) + if fin.endswith('_by_example.json') and isinstance(json_data, dict) and \ + all(isinstance(v, dict) and 'files' in v for v in json_data.values()): + # a metrics_by_example.json: one data entry per example. Keyed on + # the filename, which IS the contract (write_by_example, the CMake + # rule and metrics_pair_compare all spell that suffix) - a shape + # sniff would silently reroute any coincidentally-shaped JSON. + for ex in sorted(json_data): + if only_examples and ex not in only_examples: + continue + sub = {'files': list(json_data[ex]['files'])} + if filters: + sub['files'] = [f for f in sub['files'] + if f.get('path') and any(x in f['path'] for x in filters)] + all_json_data['file_list'].append(f'{fin}:{ex}') + all_json_data['data'].append(sub) + continue if filters: json_data["files"] = [ f @@ -316,6 +332,25 @@ def write_json_output(json_data, path): json.dump(json_data, outf, indent=2) +def write_by_example(all_json_data, path): + """{/: {files: [...]}} from the data combine_files already parsed + - re-reading and re-parsing every input a second time bought nothing. + + Inputs are map.json files laid out as ///.map.json + (examples/CMakeLists.txt's pattern), so the example name is the last two path + components; a metrics_by_example.json input already carries its own name in the + file_list entry ('.json:/').""" + out = {} + for fin, data in zip(all_json_data["file_list"], all_json_data["data"]): + _, sep, ex = fin.partition('.json:') + if not sep: + d = os.path.dirname(os.path.abspath(fin)) + ex = f'{os.path.basename(os.path.dirname(d))}/{os.path.basename(d)}' + out.setdefault(ex, {'files': []})['files'] += data.get('files', []) + with open(path, 'w', encoding='utf-8') as f: + json.dump(out, f) + + def render_combine_table(json_data, sort_order='name+'): """Render averaged sizes as markdown table lines (no title).""" files = json_data.get("files", []) @@ -579,7 +614,8 @@ def render_compare_table(rows, include_sum): def cmd_combine(args): """Handle combine subcommand.""" input_files = expand_files(args.files) - all_json_data = combine_files(input_files, args.filters) + only_examples = set(args.only_examples.split(',')) if args.only_examples else None + all_json_data = combine_files(input_files, args.filters, only_examples=only_examples) json_average = compute_avg(all_json_data) if json_average is None: @@ -594,6 +630,8 @@ def cmd_combine(args): if args.markdown_out: write_combine_markdown(json_average, args.out + '.md', sort_order=args.sort, title="TinyUSB Average Code Size Metrics") + if args.by_example: + write_by_example(all_json_data, args.out + '_by_example.json') def cmd_compare(args): @@ -633,6 +671,10 @@ def main(argv=None): combine_parser.add_argument('-S', '--sort', dest='sort', default='size-', choices=['size', 'size-', 'size+', 'name', 'name-', 'name+'], help='Sort order: size/size- (descending), size+ (ascending), name/name+ (ascending), name- (descending). Default: size-') + combine_parser.add_argument('--by-example', dest='by_example', action='store_true', + help='Also write _by_example.json: per-example file lists keyed by role/example') + combine_parser.add_argument('--only-examples', dest='only_examples', default='', + help='Comma-separated role/example ids to keep when reading by-example JSON inputs') # Compare subcommand compare_parser = subparsers.add_parser('compare', help='Compare two metrics inputs (bloaty CSV or metrics JSON)') -- cgit v1.3.1