diff options
Diffstat (limited to 'tools')
| -rwxr-xr-x | tools/build.py | 178 | ||||
| -rwxr-xr-x | tools/build_utils.py | 353 | ||||
| -rwxr-xr-x | tools/ci_select.py | 1292 | ||||
| -rwxr-xr-x | tools/gen_doc.py | 52 | ||||
| -rwxr-xr-x | tools/get_deps.py | 8 | ||||
| -rw-r--r-- | tools/iar_template.ipcf | 2 | ||||
| -rwxr-xr-x | tools/make_release.py | 1 | ||||
| -rw-r--r-- | tools/metrics.py | 41 | ||||
| -rw-r--r-- | tools/metrics_compare_base.py | 4 | ||||
| -rw-r--r-- | tools/rtt.py | 727 |
10 files changed, 2616 insertions, 42 deletions
diff --git a/tools/build.py b/tools/build.py index 51d3d0f70..aa8868cb8 100755 --- a/tools/build.py +++ b/tools/build.py @@ -2,6 +2,7 @@ import argparse import random import os +import re import sys import time import subprocess @@ -33,6 +34,7 @@ ci_skip_boards = { 'adafruit_fruit_jam', 'adafruit_metro_rp2350', 'feather_rp2040_max3421', + 'pico2_etm_trace', 'pico_sdk', 'raspberry_pi_pico_w', ], @@ -99,6 +101,53 @@ def get_examples(family): return all_examples +def resolve_example_target_groups(build_targets, examples, board, extra_defines=()): + """Map generic targets onto per-example targets for a filtered build (-e), as ONE + GROUP PER REQUESTED TARGET: 'all' -> the example executables, anything else (e.g. + tinyusb_metrics) passes through as its own single-entry group. + + Grouped rather than flattened because each group becomes one `cmake --build + --target a b c` invocation: the examples of a group build in parallel (flattening + them into one target per invocation serialises the whole leg - measured +39% at + -j4 and +220% at -j32 on stm32f407disco), while separate groups stay ordered, so a + target that must run after the examples still does. + + extra_defines are this build's -D tokens: MAX3421_HOST=1 there decides + only.txt for the max3421 examples (see build_utils.skip_example). + Returns None when no requested example is buildable on this board.""" + buildable = [e for e in examples + if not build_utils.skip_example(e, board, extra_defines)] + if not buildable: + return None + names = list(dict.fromkeys(e.split('/', 1)[1] for e in buildable)) + return [list(names) if t == 'all' else [t] for t in build_targets] + + +_TARGET_HELP_RE = re.compile(r'^([A-Za-z0-9_.+-]+):') +# role/name, the only shape resolve_example_target_groups and the CMake target names accept +EXAMPLE_RE = re.compile(r'[A-Za-z0-9_]+/[A-Za-z0-9_]+') + + +def parse_target_help(text): + """Bare target names out of `cmake --build <dir> --target help`; the Ninja + generator prints one '<name>: phony' line per target. Names containing '/' are + per-directory utility targets (device/edit_cache) or absolute CMakeFiles paths, + never an example target.""" + return {m.group(1) for m in map(_TARGET_HELP_RE.match, text.splitlines()) if m} + + +def cmake_registered_targets(build_dir): + """The targets CMake actually created in build_dir, or None when that cannot be + read. Ground truth: skip.txt/only.txt only mirrors family_filter, so an example + the role CMakeLists never lists (or a stale -e name) still looks buildable to it + and `cmake --build --target <it>` hard-fails. None keeps the mirror's answer.""" + r = subprocess.run(['cmake', '--build', build_dir, '--target', 'help'], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + if r.returncode != 0: + return None + return parse_target_help(r.stdout.decode('utf-8', 'replace')) or None + + def print_build_result(board, build_target, status, duration): if isinstance(duration, (int, float)): duration = "{:.2f}s".format(duration) @@ -107,7 +156,7 @@ def print_build_result(board, build_target, status, duration): # ----------------------------- # CMake # ----------------------------- -def cmake_board(board, build_args, build_name, build_cflags, build_targets): +def cmake_board(board, build_args, build_name, build_cflags, build_targets, examples=None, defines=()): ret = [0, 0, 0] start_time = time.monotonic() @@ -120,8 +169,13 @@ def cmake_board(board, build_args, build_name, build_cflags, build_targets): if family == 'espressif': # for espressif, we have to build example individually all_examples = get_examples(family) + if examples is not None: + all_examples = [e for e in all_examples if e in examples] + if not all_examples: + print_build_result(board, 'examples (PR filter)', 2, '-') + return [0, 0, 1] for example in all_examples: - if build_utils.skip_example(example, board): + if build_utils.skip_example(example, board, defines): ret[2] += 1 else: rcmd = run_cmd([ @@ -130,13 +184,40 @@ def cmake_board(board, build_args, build_name, build_cflags, build_targets): ]) ret[0 if rcmd.returncode == 0 else 1] += 1 else: + # the skip.txt/only.txt prefilter reads no configure output: answer it first, + # so a selection this board builds nothing of costs no cmake run at all + if examples is not None: + examples = [e for e in examples + if not build_utils.skip_example(e, board, defines)] + if not examples: + print_build_result(board, 'examples (PR filter)', 2, '-') + return [0, 0, 1] rcmd = run_cmd(['cmake', 'examples', '-B', build_dir, '-GNinja', f'-DBOARD={board}', '-DCMAKE_BUILD_TYPE=MinSizeRel', '-DLINKERMAP_OPTION=-q -f tinyusb/src', *build_args, *build_flags]) if rcmd.returncode == 0: + target_groups = [[t] for t in build_targets] + if examples is not None: + registered = cmake_registered_targets(build_dir) + if registered is not None: + kept = [e for e in examples if e.split('/', 1)[1] in registered] + for e in examples: + if e not in kept: + print_build_result(board, f'{e} (no such target)', 2, '-') + examples = kept + if not examples: + print_build_result(board, 'examples (no such target)', 2, '-') + return [0, 0, 1] + target_groups = resolve_example_target_groups(build_targets, examples, board, defines) + if registered is None: + # ground truth unavailable, so nothing checked these names against + # what CMake created. ninja validates a whole invocation up front: + # one unknown name in the batch builds NOTHING, where a target each + # builds everything up to it. Give up the parallelism, not the work. + target_groups = [[t] for g in target_groups for t in g] cmd = ["cmake", "--build", build_dir, '--parallel', str(parallel_jobs)] - for target in build_targets: - rcmd = run_cmd(cmd + ['--target', target]) + for group in target_groups: + rcmd = run_cmd(cmd + ['--target'] + group) if rcmd.returncode != 0: break ret[0 if rcmd.returncode == 0 else 1] += 1 @@ -148,9 +229,10 @@ def cmake_board(board, build_args, build_name, build_cflags, build_targets): # ----------------------------- # Make # ----------------------------- -def make_one_example(example, board, make_option, build_targets): - # Check if board is skipped - if build_utils.skip_example(example, board): +def make_one_example(example, board, make_option, build_targets, defines=()): + # Check if board is skipped. Make semantics: family.mk decides, not the + # family.cmake MCU list (see build_utils.skip_example). + if build_utils.skip_example(example, board, defines, build_system='make'): print_build_result(board, example, 2, '-') r = 2 else: @@ -171,10 +253,15 @@ def make_one_example(example, board, make_option, build_targets): return ret -def make_board(board, build_args, build_targets): +def make_board(board, build_args, build_targets, examples=None, defines=()): print(build_separator) family = find_family(board); all_examples = get_examples(family) + if examples is not None: + all_examples = [e for e in all_examples if e in examples] + if not all_examples: + print_build_result(board, 'examples (PR filter)', 2, '-') + return [0, 0, 1] start_time = time.monotonic() ret = [0, 0, 0] if family == 'espressif' or family == 'rp2040': @@ -182,7 +269,7 @@ def make_board(board, build_args, build_targets): final_status = 2 else: with Pool(processes=os.cpu_count()) as pool: - pool_args = list((map(lambda e, b=board, o=f"{build_args}", t=build_targets: [e, b, o, t], all_examples))) + pool_args = list((map(lambda e, b=board, o=f"{build_args}", t=build_targets, d=defines: [e, b, o, t, d], all_examples))) r = pool.starmap(make_one_example, pool_args) # sum all element of same index (column sum) ret = list(map(sum, list(zip(*r)))) @@ -194,36 +281,58 @@ def make_board(board, build_args, build_targets): # ----------------------------- # Build Family # ----------------------------- -def build_boards_list(boards, build_defines, build_system, build_name, build_cflags, build_targets): +def build_boards_list(boards, build_defines, build_system, build_name, build_cflags, build_targets, examples=None): ret = [0, 0, 0] + # the -D tokens are part of the skip.txt/only.txt answer (MAX3421_HOST=1), so + # the -e filter has to see them too; sorted+tuple keeps skip_example cacheable + defines = tuple(sorted(build_defines)) for b in boards: r = [0, 0, 0] if build_system == 'cmake': build_args = [f'-D{d}' for d in build_defines] - r = cmake_board(b, build_args, build_name, build_cflags, build_targets) + r = cmake_board(b, build_args, build_name, build_cflags, build_targets, examples, defines) elif build_system == 'make': build_args = ' '.join(f'{d}' for d in build_defines) - r = make_board(b, build_args, build_targets) + r = make_board(b, build_args, build_targets, examples, defines) ret[0] += r[0] ret[1] += r[1] ret[2] += r[2] return ret -def get_family_boards(family, one_random, one_first): +def get_family_boards(family, one_random, one_first, examples=None, build_system='cmake', + extra_defines=(), ci=None): """Get list of boards for a family. Args: family: Family name one_random: If True, return only one random board one_first: If True, return only the first board (alphabetical) + examples: PR example filter (-e). The one-board pick then prefers a board that + can build at least one of them: the family is in the matrix BECAUSE some + board of it builds these examples (ci_select._prune_buildable asks about + every board, since CircleCI builds every board), but GHA builds one. Without + this, lpc54 selected for host/msc_file_explorer picks lpcxpresso54114 - + which every one of those examples skips - and the leg runs to green having + compiled nothing and uploaded no metrics. + build_system: which skip answer to ask for; the two differ (build_utils) + extra_defines: this build's -D tokens, so a board whose only.txt match comes + from -DMAX3421_HOST=1 is not judged unbuildable here and buildable in + cmake_board + ci: force the ci_skip_boards / ci_preferred_boards lists on or off. Default + None reads the environment, which is right for a build but NOT for a caller + asking what CI would do: ci_select must answer the same on a laptop as on a + runner, or /pre-pr and the code-size skill report a family list CI will not + reproduce. Returns: List of board names """ + if ci is None: + ci = bool(os.getenv('GITHUB_ACTIONS') or os.getenv('CIRCLECI')) skip_list = [] preferred_list = [] - if os.getenv('GITHUB_ACTIONS') or os.getenv('CIRCLECI'): + if ci: skip_list = ci_skip_boards.get(family, []) preferred_list = ci_preferred_boards.get(family, []) @@ -238,12 +347,26 @@ def get_family_boards(family, one_random, one_first): # If only-one flags are set, honor select list first, then pick first or random if one_first or one_random: - if preferred_list: - return [preferred_list[0]] + def buildable(board): + # no filter, or nothing in the filter is buildable anywhere: keep today's + # answer rather than inventing a different board + return examples is None or any( + not build_utils.skip_example(e, board, extra_defines, build_system) + for e in examples) + + # the WHOLE preferred list, in order - stopping at entry one would abandon a + # curated list for the raw alphabetical order the moment its first board cannot + # build the filter, which also moves the board the metrics baseline is keyed on + # the whole preferred list, in order. Unreachable-when-unfiltered: with + # examples is None, buildable() is True and the loop returns on entry one. + for b in preferred_list: + if buildable(b): + return [b] + candidates = [b for b in all_boards if buildable(b)] or all_boards if one_first: - return [all_boards[0]] + return [candidates[0]] if one_random: - return [random.choice(all_boards)] + return [random.choice(candidates)] return all_boards @@ -272,6 +395,8 @@ def main(): parser.add_argument('-j', '--jobs', type=int, default=os.cpu_count(), help='Number of jobs to run in parallel') parser.add_argument('-T', '--target', action='append', default=[], help='Build target to use, may be specified multiple times (default: all)') + parser.add_argument('-e', '--example', action='append', default=[], + help='Only build these examples (role/name, repeatable). Default: all examples') parser.add_argument('-v', '--verbose', action='store_true', help='Verbose output') args = parser.parse_args() @@ -285,9 +410,20 @@ def main(): one_random = args.one_random one_first = args.one_first build_targets = args.target if args.target else ['all'] + examples = args.example or None verbose = args.verbose parallel_jobs = args.jobs + for e in args.example: + if not EXAMPLE_RE.fullmatch(e): + parser.error(f"-e/--example takes 'role/name' (e.g. device/cdc_msc), got '{e}'") + # a name no example dir answers to would silently build nothing on every board + # and still exit 0 (every row is a Skipped, and main() returns the FAILED count). + # The -e lists are generated - from ci_select's example map and from HIL roster + # test names - so a stale one must be loud, not green + if not os.path.isdir(os.path.join('examples', e)): + parser.error(f"-e/--example '{e}': no such example directory examples/{e}") + build_defines.append(f'TOOLCHAIN={toolchain}') if len(families) == 0 and len(boards) == 0: @@ -317,10 +453,12 @@ def main(): # get boards from families and append to boards list all_boards = list(boards) for f in all_families: - all_boards.extend(get_family_boards(f, one_random, one_first)) + all_boards.extend(get_family_boards(f, one_random, one_first, examples, + build_system, tuple(build_defines))) # build all boards - result = build_boards_list(all_boards, build_defines, build_system, build_name, build_cflags, build_targets) + result = build_boards_list(all_boards, build_defines, build_system, build_name, build_cflags, build_targets, + examples) total_time = time.monotonic() - total_time print(build_separator) diff --git a/tools/build_utils.py b/tools/build_utils.py index d80ceea7c..1b81335e0 100755 --- a/tools/build_utils.py +++ b/tools/build_utils.py @@ -1,4 +1,6 @@ #!/usr/bin/env python3 +import functools +import os import subprocess import pathlib import re @@ -10,32 +12,213 @@ FAILED = "\033[31mfailed\033[0m" SKIPPED = "\033[33mskipped\033[0m" -def skip_example(example, board): - ex_dir = pathlib.Path('examples/') / example - bsp = pathlib.Path("hw/bsp") +# Every read here is a source file, not user text: decode it the same way on every +# machine. Without this the reads take the locale's encoding, and one of the eight +# tracked non-ASCII files this now touches (hw/bsp/nrf/boards/nrf54lm20dk/board.cmake +# among them) raises UnicodeDecodeError under LC_ALL=C - a ValueError, which sails +# straight through the `except OSError` fail-opens. +_TEXT = {'encoding': 'utf-8', 'errors': 'replace'} - # board within family - board_dir = list(bsp.glob("*/boards/" + board)) - if not board_dir: - # Skip unknown boards - return True +_FAMILY_MCUS_RE = re.compile(r'set\s*\(\s*FAMILY_MCUS\s+([^)]*)\)') +_CMAKE_SET_RE = re.compile(r'set\s*\(\s*([A-Za-z_]\w*)\s+([^)\s]+)') +_CMAKE_VAR_RE = re.compile(r'\$\{([A-Za-z_]\w*)\}') +_CMAKE_CASE_RE = re.compile(r'string\s*\(\s*(TOUPPER|TOLOWER)\s+(\S+)\s+([A-Za-z_]\w*)\s*\)') - board_dir = list(board_dir)[0] - family_dir = board_dir.parent.parent - family = family_dir.name - # family.mk + +def _cwd_cache(fn): + """lru_cache, keyed on the working directory as well as the arguments. + + Every cached helper below takes repo-RELATIVE paths ('hw/bsp/<fam>', + 'examples/<ex>/skip.txt', or the literal 'hw/bsp' glob), while ci_select._in_repo() + chdirs around each call so one process can classify more than one tree - the + code-size skill's base-vs-branch worktrees, /pre-pr, a test pointing at a fixture. + Without the cwd in the key the second tree silently gets the first tree's + skip.txt/only.txt and FAMILY_MCUS answers. Master had no caching here, so this + hazard arrived with it.""" + cache = {} + + @functools.wraps(fn) + def wrapper(*args): + key = (os.getcwd(), args) + if key not in cache: + cache[key] = fn(*args) + return cache[key] + + wrapper.cache_clear = cache.clear + return wrapper + +@_cwd_cache +def _cmake_sets(path): + """One cmake file's variable assignments as NAME -> first definition seen, as + either a literal value or an ('TOUPPER'|'TOLOWER', source) pair. Only used to + expand ${...} tokens; never mutate the cached dict. + + string(TOUPPER ...) is not decoration: hw/bsp/maxim derives its ONLY FAMILY_MCUS + entry that way (`string(TOUPPER ${MAX_DEVICE} MAX_DEVICE_UPPER)`), as do the eight + at32 families, so dropping those lines left nine families with an empty MCU set.""" + try: + text = pathlib.Path(path).read_text(**_TEXT) + except OSError: + return {} + out = {} + for line in text.splitlines(): + line = line.strip() + if line.startswith('#'): + continue + m = _CMAKE_CASE_RE.match(line) + if m: + # strip quotes like the set() branch below: string(TOUPPER "${VAR}" DST) is + # idiomatic cmake, and keeping them yields a '"NAME"' token that can never + # equal a mcu: entry + out.setdefault(m.group(3), (m.group(1), m.group(2).strip('"'))) + continue + m = _CMAKE_SET_RE.match(line) + if m: + out.setdefault(m.group(1), m.group(2).strip('"')) + return out + + +def _cmake_expand(value, files, depth=0): + """`value` with every ${VAR} replaced, resolving each name against `files` in + order, or None when any name resolves nowhere OR the result still carries a `${`. + That last case is the one _CMAKE_VAR_RE cannot see - a hyphen in the name, a nested + ${${X}}, an unterminated brace - where the loop below finds nothing to substitute + and would otherwise hand the raw text back as if it were a resolved MCU name. + Bounded depth: a cmake file may define a var in terms of another one, and a + self-referential set() must not recurse forever.""" + if depth > 4: + return None + out = value + for name in set(_CMAKE_VAR_RE.findall(value)): + val = None + for f in files: + val = _cmake_sets(f).get(name) + if val is not None: + break + if val is None: + return None + if isinstance(val, tuple): # string(TOUPPER src DST) + src = _cmake_expand(val[1], files, depth + 1) + if src is None: + return None + val = src.upper() if val[0] == 'TOUPPER' else src.lower() + else: + val = _cmake_expand(val, files, depth + 1) + if val is None: + return None + out = out.replace('${' + name + '}', val) + return None if '${' in out else out + + +@_cwd_cache +def _board_dirs(board): + """(board_dir, family_dir) for a board name, or (None, None). Cached: skip_example + is asked (board x example) times - 566k lstat calls per selector run without this, + since the glob rescans every hw/bsp/*/boards for each example.""" + hits = list(pathlib.Path("hw/bsp").glob("*/boards/" + board)) + if not hits: + return None, None + return hits[0], hits[0].parent.parent + + +@_cwd_cache +def _family_mcus(family_dir, board_dir): + """The MCU names CMake's family_filter iterates. family_support.cmake:176/190 + loop `foreach(MCU IN LISTS FAMILY_MCUS)`, so a family-wide list (broadcom_64bit + sets "BCM2711 BCM2835") makes ANY of its entries decide skip.txt/only.txt -- not + just the one CFG_TUSB_MCU the configured board names. + + ${...} tokens are expanded from `set(VAR value)` and `string(TOUPPER src VAR)` in + the board's board.cmake first, then in family.cmake: hw/bsp/ra sets + `FAMILY_MCUS RAXXX ${MCU_VARIANT}` and ra6m5_ek/board.cmake sets MCU_VARIANT ra6m5, + which is the token dual/host_info_to_device_cdc/only.txt actually spells; hw/bsp/maxim + sets `FAMILY_MCUS ${MAX_DEVICE_UPPER}`, upper-cased from the board's MAX_DEVICE. A + token resolving nowhere is dropped (nothing can be said about it). + + A family that never spells `set(FAMILY_MCUS ...)` at all gets one more chance: the + name is resolved as a variable, which covers the derived form hw/bsp/espressif uses + (`string(TOUPPER ${IDF_TARGET} FAMILY_MCUS)`). + + Only unconditional set() calls count: nrf and mcx pick FAMILY_MCUS per board + inside if() blocks this does not evaluate, so for those two families the whole + cmake-side MCU set is whatever the CFG_TUSB_MCU scrape in _board_mcu finds. + + nrf: the scrape reads the FIRST CFG_TUSB_MCU token of hw/bsp/nrf/family.mk, so + every nrf board answers NRF54, the NRF5X ones included. Harmless only because no + skip.txt/only.txt names an nrf token today. + + mcx: load-bearing, not academic -- mcu:MCXA15 is live in six examples' skip.txt + (device/{cdc_msc,audio_test,hid_composite,audio_4_channel_mic,midi_test}_freertos + and device/net_lwip_webserver). Those answers come out right only because the + scrape falls through to each board's make-only board.mk, which still spells the + token; an mcx board carrying board.cmake alone (MCU_VARIANT and no CFG_TUSB_MCU) + would scrape 'NONE' and skip EVERY example on it, silently. TestFamilyMcusFallback + fails the day such a board lands. The fix then is to evaluate the + if(MCU_VARIANT STREQUAL ...) branches, not to add another scrape. + """ + fam_cmake = pathlib.Path(family_dir) / "family.cmake" + try: + text = fam_cmake.read_text(**_TEXT) + except OSError: + return frozenset() + board_cmake = pathlib.Path(board_dir) / "board.cmake" + out = set() + depth = 0 + any_set = False + for line in text.splitlines(): + line = line.strip() + m = _FAMILY_MCUS_RE.match(line) + if m: + any_set = True + if m and depth == 0: + files = (str(board_cmake), str(fam_cmake)) + for tok in m.group(1).split(): + if tok in ("CACHE", "INTERNAL") or tok.startswith('"'): + continue + val = _cmake_expand(tok, files) + if val: + out.add(val) + if re.match(r'if\s*\(', line): + depth += 1 + elif re.match(r'endif\s*\(', line): + depth = max(0, depth - 1) + if not out and not any_set: + # FAMILY_MCUS can also be produced rather than set: hw/bsp/espressif derives it + # with `string(TOUPPER ${IDF_TARGET} FAMILY_MCUS)`, which _FAMILY_MCUS_RE cannot + # see, leaving espressif's whole cmake answer resting on the IDF_TARGET scrape. + # + # `not any_set` is load-bearing: _cmake_sets is if()-blind and keeps the FIRST + # definition, so on a family that sets FAMILY_MCUS only inside conditionals + # (mcx, nrf) this would leak branch one's value onto every board - mcx/frdm_mcxn947 + # answered MCXA15, which six examples' skip.txt names, dropping 12 firmware + # images CMake actually builds. Those families keep the CFG_TUSB_MCU scrape. + val = _cmake_expand('${FAMILY_MCUS}', (str(board_cmake), str(fam_cmake))) + if val: + out.add(val) + return frozenset(out) + + +@_cwd_cache +def _scrape_mcu(family_dir, board_dir, family): + """(CFG_TUSB_MCU token of this board, the text it was read from), master's + algorithm verbatim: family.mk (family.cmake when there is none) first, falling + back to the board's board.mk (board.cmake when there is none) only when the + family file names no token at all. espressif spells its MCU as + `set(IDF_TARGET "...")` instead. The text comes back with it because the make + path reads MAX3421_HOST out of that same single file - which file that is IS + part of master's answer, so it cannot be re-derived by the caller.""" family_mk = family_dir / "family.mk" if not family_mk.exists(): family_mk = family_dir / "family.cmake" - mk_contents = family_mk.read_text() + mk_contents = family_mk.read_text(**_TEXT) # Find the mcu, first in family mk then board mk if "CFG_TUSB_MCU=OPT_MCU_" not in mk_contents: board_mk = board_dir / "board.mk" if not board_mk.exists(): board_mk = board_dir / "board.cmake" - mk_contents = board_mk.read_text() + mk_contents = board_mk.read_text(**_TEXT) mcu = "NONE" if family == "espressif": @@ -53,6 +236,95 @@ def skip_example(example, board): mcu = opt_mcu[len("OPT_MCU_"):] if mcu != "NONE": break + return mcu, mk_contents + + +@_cwd_cache +def _board_mcu(board_dir, family_dir, family): + """(CFG_TUSB_MCU of this board, MAX3421_HOST enabled by its cmake BSP). + + MAX3421_HOST is read from family.cmake AND board.cmake rather than only the file + the MCU token came from: feather_rp2040_max3421 sets it in its board.cmake while + its MCU token comes from rp2040's family file, and family_support.cmake:940 + appends MAX3421 to FAMILY_MCUS for it. board.mk is deliberately not read - a + make-only option compiles nothing in a cmake build (and the make path answers + with master's own single-file scrape, see _skip_example_make).""" + family_dir = pathlib.Path(family_dir) + board_dir = pathlib.Path(board_dir) + mcu, _ = _scrape_mcu(family_dir, board_dir, family) + if "${" in mcu: + # the scrape is textual, so a computed token comes back verbatim + # (tm4c board.cmake spells OPT_MCU_TM4C${MCU_SUB_VARIANT}, maxim + # OPT_MCU_${MAX_DEVICE_UPPER}). Expand it the same way FAMILY_MCUS tokens are; + # what still will not resolve stays as-is and _skip_example treats it as + # "MCU unknown" rather than silently matching no mcu: token at all. + mcu = _cmake_expand(mcu, (str(board_dir / "board.cmake"), + str(family_dir / "family.cmake"))) or mcu + + max3421_enabled = False + for f in (family_dir / "family.cmake", board_dir / "board.cmake"): + try: + text = f.read_text(**_TEXT) + except OSError: + continue + # a commented-out `# set(MAX3421_HOST 1)` (feather_nrf52840_express) enables + # nothing; master never hit one because it only read the MCU token's file + if any(not l.lstrip().startswith('#') and + ("MAX3421_HOST=1" in l or 'MAX3421_HOST 1' in l) + for l in text.splitlines()): + max3421_enabled = True + break + + return mcu, max3421_enabled + + +@_cwd_cache +def _filter_tokens(path): + """skip.txt / only.txt as a token set, or None when the file does not exist.""" + f = pathlib.Path(path) + return frozenset(f.read_text(**_TEXT).split()) if f.exists() else None + + +def skip_example(example, board, extra_defines=(), build_system='cmake'): + """Is this example unbuildable on this board, for this build system? + + The two build systems ask DIFFERENT questions and must not share an answer: + + 'cmake' mirrors CMake's family_filter (hw/bsp/family_support.cmake:171-207), + including the whole FAMILY_MCUS list the family.cmake sets. + + 'make' is master's original algorithm, unchanged. family.mk and family.cmake are + not the same build: hw/bsp/lpc54/family.cmake sets FAMILY_MCUS LPC54 and wires the + ohci host sources, while family.mk builds OPT_MCU_LPC54XXX and compiles no HCD + source at all -- feeding the cmake MCU union to a make build un-skips the host + examples only.txt gates on mcu:LPC54 and they fail to link (undefined hcd_init). + + extra_defines: NAME=VALUE tokens the build passes on the command line + (build.py -D). MAX3421_HOST=1 there enables the max3421 host controller + exactly like a BSP that sets it, and family_support.cmake:940 appends MAX3421 + to FAMILY_MCUS for it -- so a roster board whose MAX3421 comes from the build + args (metro_m4_express) must resolve its only.txt the same way. cmake only: + master's make algorithm never looked at them. + """ + return _skip_example(example, board, tuple(extra_defines), build_system) + + +@_cwd_cache +def _skip_example_make(example, board): + """master's skip_example, verbatim (tools/build_utils.py @ 9c202e8c6): the + make build's own answer, derived from family.mk/board.mk with the single + CFG_TUSB_MCU token that file names. Do not "improve" it -- it is the mirror of + what `make BOARD=... all` actually compiles.""" + ex_dir = pathlib.Path('examples/') / example + + # board within family + board_dir, family_dir = _board_dirs(board) + if board_dir is None: + # Skip unknown boards + return True + family = family_dir.name + + mcu, mk_contents = _scrape_mcu(family_dir, board_dir, family) # Skip all OPT_MCU_NONE these are WIP port if mcu == "NONE": @@ -68,14 +340,14 @@ def skip_example(example, board): only_file = ex_dir / "only.txt" if skip_file.exists(): - skips = skip_file.read_text().split() + skips = skip_file.read_text(**_TEXT).split() if ("mcu:" + mcu in skips or "board:" + board in skips or "family:" + family in skips): return True if only_file.exists(): - onlys = only_file.read_text().split() + onlys = only_file.read_text(**_TEXT).split() if not ("mcu:" + mcu in onlys or ("mcu:MAX3421" in onlys and max3421_enabled) or "board:" + board in onlys or @@ -85,6 +357,55 @@ def skip_example(example, board): return False +@_cwd_cache +def _skip_example(example, board, extra_defines, build_system): + if build_system == 'make': + return _skip_example_make(example, board) + + ex_dir = pathlib.Path('examples/') / example + + # board within family + board_dir, family_dir = _board_dirs(board) + if board_dir is None: + # Skip unknown boards + return True + family = family_dir.name + + mcu, max3421_enabled = _board_mcu(str(board_dir), str(family_dir), family) + + # Skip all OPT_MCU_NONE these are WIP port + if mcu == "NONE": + return True + + if any(t.strip().strip('"') == "MAX3421_HOST=1" for t in extra_defines): + max3421_enabled = True + + mcus = set(_family_mcus(str(family_dir), str(board_dir))) + if "${" not in mcu: + mcus.add(mcu) + if not mcus: + # nothing resolved: neither FAMILY_MCUS nor the scraped CFG_TUSB_MCU token + # yielded a name. Answering "skip" here would silently drop EVERY example on + # the board (an only.txt can then never match), so say "buildable" and let + # the real filter decide - build.py checks the targets CMake actually + # registered, and CMake itself is the authority on the make/cmake legs. + return False + if max3421_enabled: + mcus.add("MAX3421") # family_support.cmake:940 + + keys = {"board:" + board, "family:" + family} | {"mcu:" + m for m in mcus} + + skips = _filter_tokens(str(ex_dir / "skip.txt")) + if skips is not None and (skips & keys): + return True + + onlys = _filter_tokens(str(ex_dir / "only.txt")) + if onlys is not None and not (onlys & keys): + return True + + return False + + def build_size(make_cmd): size_output = subprocess.run(make_cmd + ' size', shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8").splitlines() for i, l in enumerate(size_output): diff --git a/tools/ci_select.py b/tools/ci_select.py new file mode 100755 index 000000000..1526f2064 --- /dev/null +++ b/tools/ci_select.py @@ -0,0 +1,1292 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""PR-diff -> CI selection: which rig boards and which tests a change can affect. + +Lives in tools/ so it can serve both HIL selection and, from Task 3, build-family +selection. Stdlib-only (runs on bare CI runners; imports hil_util for the example +rosters, never hil_test/pyserial — test_hil_util.BottomLayer enforces the stdlib +closure). Fail-open: any file no rule classifies forces the full matrix. See +docs/superpowers/specs/2026-07-29-hil-pr-scoped-selection-design.md and +docs/superpowers/specs/2026-08-19-ci-build-family-filter-design.md. + +JSON: full, boards (name -> 'all' | [tests]), families (bsp families the diff +touches, including ones with no rig board - build-only consumers such as /pre-pr +sample from these), args (hil_test.py args per config) and args_flasher (the same +args split by each board's flasher, for CI legs that split one rig by flasher). + +THE RULE TABLE. First match wins; answers union per family (build) and per board +(HIL). A CARBON COPY of the table in the design spec above - edit both, or +TestRuleTableIsCarbonOfTheSpec fails. `FAM` = the families whose family.cmake +references the changed path (CMake only; make follows it). `DEV`/`HOST`/`DUAL`/ +`TYPEC`/`ALL` are the example role sets. The Build families column is PRE-PRUNE: +_prune_buildable then intersects each family with what it can actually build. + +| # | Changed path | Build families | Build examples | HIL boards → tests | +| 1 | `docs/`, `.claude/`, `*.md`, `*.rst`, `LICENSE` | — | — | — | +| 1b | `.gitignore`, `.clang-format`, `.idea/**`, `test/{fuzz,unit-test}/**`, `test/hil/test/**`, non-build `.github/**`, packaging manifests | — | — | — | +| 2 | `test/hil/**` (not `test/hil/test/**`) | — | — | all boards → all tests | +| 2b | `tools/metrics.py`, `.github/scripts/metrics_*.py` | `ALL` (unchanged — `tinyusb_metrics` runs `metrics.py` as a build target) | `ALL` | — (nothing on the rig runs it) | +| 3 | `src/portable/<port>/dcd_*`, `*_device.[ch]` | `FAM` | `DEV`+`DUAL` | `FAM`'s device-role boards → device+dual tests | +| 4 | `src/portable/<port>/hcd_*`, `*_host.[ch]` | `FAM` | `HOST`+`DUAL` | `FAM`'s host-role boards → host+dual tests | +| 5 | `src/portable/<port>/**` (anything else) | `FAM` | `ALL` | `FAM`'s boards → all their tests | +| 5b | `src/portable/<port>/**` where `FAM` is empty | — | — | — (empty resolves to nothing on BOTH axes) | +| 6 | `hw/bsp/<family>/**` | that family | `ALL` | that family's boards → all tests (a `boards/<board>/` path narrows to that board) | +| 7 | `hw/mcu/<vendor>/**` | `FAM` — empty resolves to nothing (maintainer ruling) | `ALL` | `FAM`'s boards → all tests; empty resolves to nothing (maintainer ruling) ⚠ *see below* | +| 8 | `src/class/<cls>/*_device.[ch]` | `ALL` | examples enabling `CFG_TUD_<CLS>` | device-role boards → HIL tests enabling `CFG_TUD_<CLS>` | +| 9 | `src/class/<cls>/*_host.[ch]` | `ALL` | examples enabling `CFG_TUH_<CLS>` | host-role boards → HIL tests enabling `CFG_TUH_<CLS>` | +| 10 | `src/class/<cls>/**` (shared header) | `ALL` | either, **plus include-edge classes** | both roles → same, plus include-edge classes | +| 11 | `src/device/**` | `ALL` | `DEV`+`DUAL` | device-role boards → device+dual tests | +| 12 | `src/host/**` | `ALL` | `HOST`+`DUAL` | host-role boards → host+dual tests | +| 12b | `src/typec/**` | `ALL` | examples enabling `CFG_TUC_ENABLED` | — (no rig board runs a typec test) | +| 13 | `examples/<role>/<name>/**` | `ALL` | just `<name>` | if `<name>` is a HIL test: all boards → that test; else nothing | +| 14 | `examples/device/board_test/**` | `ALL` | just `board_test` | all boards → all tests (HIL parking firmware) | +| 15 | `examples/build_system/**`, `examples/CMakeLists.txt`, `examples/<role>/CMakeLists.txt` | `ALL` | `ALL` | all boards → all tests | +| 16 | `src/common/`, `src/osal/`, `src/tusb.[ch]`, `src/tusb_option.h`, `tools/{build,build_utils,ci_select}.py`, `tools/cmake/**`, `src/CMakeLists.txt`, `src/tinyusb.mk`, `hw/bsp/{family_support.{cmake,mk},family_rules.mk,zephyr_board_aliases.cmake,board.c,board_api.h,ansi_escape.h}`, `.github/**`, `.circleci/**` | `ALL` | `ALL` | all boards → all tests | +| 16a | `lib/<name>/**` | `ALL` | examples whose own `CMakeLists.txt`/`Makefile` names `lib/<name>` | those examples that are HIL tests, on all boards; empty resolves to nothing | +| 16b | `tools/get_deps.py` | families whose `deps_mandatory`/`deps_optional` entries changed | `ALL` | those families' boards → all tests; a logic change, an `'all'` entry, no base content or a changed token naming no family → full | +| 17 | anything unclassified (no tracked file reaches this — TestNoTrackedFileIsUnclassified) | `ALL` | `ALL` | all boards → all tests (fail-open) | +""" +import argparse +import ast +import contextlib +import functools +import glob +import io +import json +import os +import re +import subprocess +import sys + +# tools/ -> repo root is ONE level up. Guarded by TestModuleMove.test_repo_root_guard: +# a wrong parent count here silently re-points every repo-relative glob (it happened +# at the helper/ move). +_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, os.path.join(_REPO_ROOT, 'test', 'hil')) # for `from helper...` +from helper.hil_util import device_tests, dual_tests, host_test + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) # tools/, for build helpers +import build_utils +import build as build_py + +ALL_TESTS = {'device': device_tests, 'dual': dual_tests, 'host': host_test} + +# class dir -> config macro suffix exceptions (rule 3); dfu is per-file, handled inline +NET_MACROS = ('ECM_RNDIS', 'NCM') + + +def _read(path: str) -> str: + """Read a source file with a fixed encoding. The locale's is not it: several tracked + sources carry non-ASCII bytes, and under LC_ALL=C the decode raises UnicodeDecodeError + - a ValueError, which every `except OSError` fail-open below would let through as a + traceback instead of a full matrix.""" + with open(path, encoding='utf-8', errors='replace') as f: + return f.read() + + +_NONCODE_RE = re.compile( + # LICENSE is anchored and LICENSES/ named separately: a bare `LICENSE` alternative + # also swallowed anything merely STARTING with it (a future LICENSE_extra.c), + # which is the silent-under-selection direction + r'^(docs/|\.claude/|.*\.(md|rst)$|LICENSE$|LICENSES/)') +# Repo metadata and tooling that no CI build reads. Enumerated rather than left to +# rule 17, which widens BOTH axes: a PR touching only .gitignore and a README was +# creating 74 cmake legs (each a runner doing checkout + toolchain + get_deps before +# skipping the build) and booking the whole 30-board rig. +# +# Deliberately NOT here, and still full: .circleci/**, .github/workflows/build*.yml, +# .github/actions/**, .github/scripts/** - those decide what gets built. The line is +# "does any Build step read this file", not "is it source". +# +# test/{fuzz,unit-test} have their own jobs (cifuzz.yml, the unit-test pre-commit hook +# and workflow); the Build matrix never compiles them, and test/hil is rule 2. +_META_RE = re.compile( + r'^(' + r'\.(gitignore|gitattributes|clang-format|codespellrc|readthedocs\.yaml)$|' + r'\.pre-commit-config\.yaml$|\.PVS-Studio/|\.idea/|\.vscode/|' + r'sonar-project\.properties$|library\.json$|pkg\.yml$|repository\.yml$|' + r'version\.yml$|SConscript$|' + r'.*CMakePresets\.json$|hw/bsp/BoardPresets\.json$|examples/west\.yml$|' + r'.*/[0-9]+-tinyusb[^/]*\.rules$|tools/usb_drivers/|tools/codespell/|' + # test/hil/test/ holds the harness's own unit tests, not the harness: nothing on + # the rig runs them (pre-commit does, and build.yml runs test_ci_select.py as the + # gate before trusting a selection), so they cannot change what the rig does. + # The harness itself stays under _FULL_RE's test/hil/ prefix. + r'test/(fuzz|unit-test)/|test/hil/test/|' + # .github, minus the build machinery named in _FULL_RE + r'\.github/(FUNDING\.yml$|labeler\.yml$|membrowse_pr_message\.j2$|ISSUE_TEMPLATE/|' + r'workflows/(cifuzz|claude|claude-code-review|labeler|membrowse-comment|' + r'membrowse-onboard|pr_comment|pre-commit|static_analysis|trigger)\.yml$)|' + # tools/ scripts no build invokes (tools/build*.py and metrics are handled above) + r'tools/(build_doc|check_example_pids|file2carray|gen_doc|gen_presets|iar_gen|' + r'make_release|mksunxi|pcapng_to_corpus)\.py$|tools/iar_template\.ipcf$' + r')') +# Build-size metrics tooling. HIL axis ONLY: nothing on the rig runs any of it, and +# without this rule these paths are unclassified, so a metrics-only PR booked an +# exclusive full 30-board sweep to validate a script no board executes. +# The BUILD axis deliberately keeps its full-matrix answer: `tinyusb_metrics` runs +# tools/metrics.py as a build target (examples/CMakeLists.txt), and build_util.yml adds +# `--target tinyusb_metrics` to every metrics leg - a break in it fails the build, so a +# build has to exercise it. +_METRICS_RE = re.compile( + r'^(tools/metrics[^/]*\.py$|\.github/scripts/metrics_[^/]*\.py$)') +_FULL_RE = re.compile( + r'^(src/common/|src/osal/|src/tusb\.c$|src/tusb\.h$|src/tusb_option\.h$|' + # tools/rtt.py is part of the harness, not a standalone tool: hil_util imports it + # at module load, so a break in it breaks every rig run the same way a test/hil/ + # edit can (the pre-commit hil-test hook runs its unit tests for the same reason) + r'test/hil/|tools/rtt\.py$|' + r'\.github/workflows/build.*\.yml$|\.github/actions/|\.github/scripts/|' + # generates the whole CircleCI matrix, same authority as .github/** + r'\.circleci/|' + # rule 16 says `tools/build*.py`; name the two siblings the glob implies. Both + # decide what gets built, so neither can be trusted to narrow its own change. + r'tools/(build|build_utils|ci_select)\.py$|tools/cmake/|' + # the make twins of family_support.cmake are the same authority for the make legs + r'hw/bsp/(family_support\.(cmake|mk)|family_rules\.mk|zephyr_board_aliases\.cmake|' + r'board_api\.h|board\.c|ansi_escape\.h)$|' + # rule 15 lists examples/<role>/CMakeLists.txt - it registers every target in that + # role, so it was only ever reaching `full` through rule 17's fall-through + r'examples/build_system/|examples/CMakeLists\.txt$|' + r'examples/[^/]+/CMakeLists\.txt$|' + # every firmware compiles these unconditionally (src/CMakeLists.txt, src/tinyusb.mk) + r'src/CMakeLists\.txt$|src/tinyusb\.mk$|' + # board_test is HIL infrastructure, not a test: hil_test.py flashes it to park + # every board (variant boundary + end-of-board teardown), so every board depends on it + r'examples/device/board_test/)') + +# --no-renames: with rename detection git reports only a rename's destination, so code +# moved out of an HIL-relevant path would be classified by its new path alone +GIT_DIFF_ARGV = ['git', 'diff', '--no-renames', '--name-only'] + + +def test_role(test: str) -> str: + return test.split('/', 1)[0] # 'device' | 'dual' | 'host' + + +def board_roles(board: dict) -> set: + t = board.get('tests', {}) + roles = set() + if t.get('device'): + roles.add('device') + if t.get('host'): + roles.add('host') + if t.get('dual'): + roles.update(('device', 'host')) + for only in t.get('only', []): + r = test_role(only) + roles.update(('device', 'host') if r == 'dual' else (r,)) + return roles + + +def board_tests(board: dict) -> list: + """Every test this board would run today (mirrors hil_test.test_board's default).""" + t = board.get('tests', {}) + if 'only' in t: + run = list(t['only']) + else: + run = [] + if t.get('device'): + run += device_tests + if t.get('dual'): + run += dual_tests + if t.get('host'): + run += host_test + return [x for x in run if x not in t.get('skip', [])] + + + +def _rg(repo_root: str, *parts: str) -> str: + """A glob pattern rooted at repo_root, with the ROOT escaped and the parts left as + patterns. The root is a filesystem path, not a pattern: a checkout at + /w/pr[1]/tinyusb (a worktree named after a PR, a CI workspace with brackets) makes + an unescaped '[1]' a character class that matches nothing, and every lookup below + then resolves to zero - families=0 instead of 30, i.e. the selector fails CLOSED + and the whole matrix compiles nothing while reporting green.""" + return os.path.join(glob.escape(repo_root), *parts) + +# cached: called per changed file x roster board, and the tree doesn't change mid-run [email protected]_cache(maxsize=None) +def board_family(board_name: str, repo_root: str): + hits = glob.glob(_rg(repo_root, 'hw/bsp/*/boards', board_name)) + return os.path.basename(os.path.dirname(os.path.dirname(hits[0]))) if hits else None + + +# `if (OPTION STREQUAL "1")` guards in family_support.cmake, and the option tokens +# a roster entry passes to the build (NAME=VALUE / -DNAME=VALUE) +_CM_IF_RE = re.compile(r'if\s*\(') +_CM_ELSE_RE = re.compile(r'else(if)?\s*\(') +_CM_ENDIF_RE = re.compile(r'endif\s*\(') +_CM_OPT_RE = re.compile(r'if\s*\(\s*\$?\{?([A-Za-z_]\w*)\}?\s+STREQUAL\s+"?1"?\s*\)') +_CM_PORT_RE = re.compile(r'src/portable/((?:[^/\s]+/)?[^/\s]+)/') +_FALSY = ('', '0', 'off', 'false', 'no') + + [email protected]_cache(maxsize=None) +def port_option_gates(repo_root: str) -> dict: + """port dir -> build options that compile it regardless of the board's family + file, e.g. {'analog/max3421': {'MAX3421_HOST'}} from family_support.cmake.""" + gates = {} + try: + text = _read(os.path.join(repo_root, 'hw/bsp/family_support.cmake')) + except OSError: + return gates + stack = [] # one entry per open if(): its option, or None + for line in text.splitlines(): + line = line.strip() + if _CM_IF_RE.match(line): + m = _CM_OPT_RE.match(line) + stack.append(m.group(1) if m else None) + elif _CM_ELSE_RE.match(line): + if stack: + stack[-1] = None # the guard doesn't hold in this branch + elif _CM_ENDIF_RE.match(line): + if stack: + stack.pop() + opts = {o for o in stack if o} + m = _CM_PORT_RE.search(line) + if opts and m: + gates.setdefault(m.group(1), set()).update(opts) + return gates + + +_CM_SET_RE = re.compile(r'set\s*\(\s*([A-Za-z_]\w*)\s+([^)\s]+)\s*\)') + + +# cached: called per changed portable file x roster board [email protected]_cache(maxsize=None) +def bsp_board_options(board_name: str, repo_root: str) -> frozenset: + """Build options a board turns on in its own BSP: `set(<OPT> <value>)` in + hw/bsp/<family>/boards/<board>/board.cmake, e.g. MAX3421_HOST on the espressif + and rp2040 max3421 boards. CMake only - HIL CI builds nothing with Make, so a + board.mk-only option (e.g. nrf5340dk's MAX3421_HOST) compiles no port here.""" + fam = board_family(board_name, repo_root) + if not fam: + return frozenset() + path = os.path.join(repo_root, 'hw/bsp', fam, 'boards', board_name, 'board.cmake') + try: + text = _read(path) + except OSError: + return frozenset() + out = set() + for line in text.splitlines(): + line = line.strip() + if line.startswith('#'): + continue + m = _CM_SET_RE.match(line) + if m and m.group(2).strip('"').lower() not in _FALSY: + out.add(m.group(1)) + return frozenset(out) + + +def board_options(board: dict, repo_root: str) -> set: + """Build options a board has truthy: each variant's defines (NAME=VALUE) and raw + CFLAGS (-DNAME=VALUE), plus whatever its own board.cmake sets (a board can enable a + gated port without the roster saying so). A board whose option is always on carries + a single variant named after itself - metro_m4_express and MAX3421_HOST=1, which is + what makes it the one rig board that compiles hcd_max3421.c.""" + toks = [] + for v in board.get('variant', []): + toks += list(v.get('defines', [])) + toks += v.get('flags', '').split() + out = set(bsp_board_options(board['name'], repo_root)) + for t in toks: + name, _, val = (t[2:] if t.startswith('-D') else t).partition('=') + if name and val.strip().strip('"').lower() not in _FALSY: + out.add(name.strip()) + return out + + [email protected]_cache(maxsize=None) +def path_families(rel_dir: str, repo_root: str) -> set: + """Board families whose family.cmake (or espressif component CMakeLists) + references rel_dir at a directory boundary. CMake only, on every axis: CMake + is the first-class build system and Make follows it, so family.mk is never + read - a port wired up in family.mk alone (microchip/pic32mz) is built by no + CI job and resolves to nothing. Boundary = '/', whitespace, quote, paren, + brace or end: `${TOP}/hw/mcu/nordic/nrfx` has no trailing slash, while bare + 'microchip/pic' must not match '.../microchip/pic32mz/...'.""" + pat = re.compile(re.escape(rel_dir) + r'(?=[/\s"\')}]|$)', re.M) + return {fam for fam, text in _family_file_texts(repo_root) if pat.search(text)} + + [email protected]_cache(maxsize=None) +def _family_file_texts(repo_root: str) -> tuple: + """((family, text), ...) for every family.cmake and espressif component + CMakeLists.txt, read once. path_families is called per distinct directory in the + diff and its own cache only helps repeats: a 6,000-file hw/mcu dep bump re-read + these 84 files 99,892 times (2.2 s) before this.""" + bsp_root = os.path.join(repo_root, 'hw/bsp') # escaped by _rg below + out = [] + for f in sorted(glob.glob(_rg(bsp_root, '*/family.cmake')) + + glob.glob(_rg(bsp_root, '*/components/*/CMakeLists.txt'))): + try: + out.append((os.path.relpath(f, bsp_root).split(os.sep, 1)[0], _read(f))) + except OSError: + pass + return tuple(out) + + +def port_families(port_dir: str, repo_root: str) -> set: + # 'portable/', not 'src/portable/': family.cmake always spells the full literal + # path ('${TOP}/src/portable/...'), but espressif's component CMakeLists.txt + # assigns 'src' into a ${tusb_src} variable first (`${tusb_src}/portable/...`), + # so a leading 'src/' in the needle would never match there and silently drop + # espressif boards (see TestRealRosterPortFamilies). + return path_families('portable/' + port_dir, repo_root) + + +def mcu_families(path: str, repo_root: str) -> set: + """Families referencing a changed hw/mcu path: longest resolving dir prefix, + hw/mcu/<vendor>/<sub>/... down to hw/mcu/<vendor>.""" + parts = path.split('/') + for n in range(len(parts) - 1, 2, -1): + fams = path_families('/'.join(parts[:n]), repo_root) + if fams: + return fams + return set() + + +GET_DEPS_PATH = 'tools/get_deps.py' +_DEPS_DICTS = ('deps_mandatory', 'deps_optional') + + +def _deps_split(text: str): + """(module dump with the two dep-dict assigns removed, {dict name: entries}). + Parsed with ast, never exec'd: this runs on PR content.""" + mod = ast.parse(text) + dicts, rest = {}, [] + for node in mod.body: + if (isinstance(node, ast.Assign) and len(node.targets) == 1 and + isinstance(node.targets[0], ast.Name) and + node.targets[0].id in _DEPS_DICTS and isinstance(node.value, ast.Dict)): + dicts[node.targets[0].id] = ast.literal_eval(node.value) + else: + rest.append(node) + mod.body = rest + # annotate_fields=False keeps the dump readable-length; line numbers are not + # included unless asked for, so reformatting alone never reads as a logic change + return ast.dump(mod, annotate_fields=False), dicts + + +# Family tokens in tools/get_deps.py that name no hw/bsp directory. get_deps matches a +# token against a requested family name verbatim (`f in deps_optional[d][2].split()`), +# so a token like these matches nothing - a stale spelling in get_deps.py, not a +# selector bug, and out of scope to change here. Pinned so that any OTHER unresolvable +# token (real drift) falls open to the full matrix instead of silently selecting +# nothing, and so TestOrphanInvariant fails the day one is fixed or a new one appears. +# sam3x, samd21, samd51, same5x -> pre-rename spellings, listed alongside the current +# samd2x_l2x / samd5x_e5x / same7x in the same entry +# stm32l1, stm32l5 -> no hw/bsp family in the tree at all +_DEPS_ALIAS_TOKENS = frozenset({'sam3x', 'samd21', 'samd51', 'same5x', + 'stm32l1', 'stm32l5'}) + + +def get_deps_changed_families(base_text: str, head_text: str, repo_root: str): + """Families whose tools/get_deps.py dep entries changed between two versions of + the file, or None meaning 'cannot tell - use the full matrix'. + + None on: anything outside deps_mandatory/deps_optional differing (a logic change + to get_deps affects every family), a mandatory `'all'` entry changing, a token + that resolves to no family and is not a known alias, or text that will not parse. + Callers with no base content at all - `--diff-file` mode has no git and therefore + no merge-base blob - pass None themselves. + + An entry that is added, removed or edited contributes the family tokens of BOTH + sides (a removed entry has only a base side). The two dicts are diffed SEPARATELY: + merging them first would hide a move between deps_mandatory and deps_optional, + which changes which families fetch the dep even though the value is untouched.""" + try: + base_rest, base_d = _deps_split(base_text) + head_rest, head_d = _deps_split(head_text) + except (SyntaxError, ValueError, TypeError): + return None + if base_rest != head_rest: + return None + toks = set() + for name in _DEPS_DICTS: + base_x, head_x = base_d.get(name, {}), head_d.get(name, {}) + for key in set(base_x) | set(head_x): + if base_x.get(key) == head_x.get(key): + continue + for entry in (base_x.get(key), head_x.get(key)): + if entry and len(entry) > 2: + toks.update(str(entry[2]).split()) + if 'all' in toks: + return None + fams = set(all_bsp_families(repo_root)) + if toks - fams - _DEPS_ALIAS_TOKENS: + # a changed entry we cannot map to a family. "changed but unmappable" is NOT + # "nothing changed": reading it as the latter empties the entire build matrix + # for a dep bump, so fall open instead + return None + return toks & fams + + +_CLS_INC_RE = re.compile(r'#\s*include\s*[<"]class/([^/"<>]+)/([^"<>]+)[">]') + + [email protected]_cache(maxsize=None) +def class_include_edges(repo_root: str) -> dict: + """'<class>/<header>' -> the other class dirs that include it. A class header + pulled in by a second class ships in every firmware enabling that second class: + src/class/midi/midi{,2}_{device,host}.h include class/audio/audio.h, and + net_device.h includes class/cdc/cdc.h. The class rule derives macros from the + directory name alone, so without this edge a change to the included header + selects only its own class's examples - and on a board that skips those (e.g. + metro_m4_express skips audio_test_freertos), nothing at all. + + Derived from the actual #include lines rather than a hand-written table so it + cannot rot when a class picks up or drops a cross-class include.""" + edges = {} + for f in sorted(glob.glob(_rg(repo_root, 'src/class/*/*.[ch]'))): + cls = os.path.basename(os.path.dirname(f)) + try: + text = _read(f) + except OSError: + continue + for inc_cls, inc_hdr in _CLS_INC_RE.findall(text): + if inc_cls != cls: + edges.setdefault(f'{inc_cls}/{inc_hdr}', set()).add(cls) + return edges + + +_CLS_STEM_RE = re.compile(r'(.*?)(?:_(?:device|host))?\.[ch]$') + + +def class_macros(cls: str, base: str, prefix: str) -> list: + """Config macros that compile a class dir's code, for role prefix TUD/TUH. + `base` refines dfu (it splits DFU from DFU_RUNTIME per file) and adds the file's + own macro where that differs from the directory's; pass '' for a class reached + through an include edge, where the widest set is correct.""" + if cls == 'net': + return [f'CFG_{prefix}_{m}' for m in NET_MACROS] + if cls == 'dfu': + if base.startswith('dfu_rt'): + return [f'CFG_{prefix}_DFU_RUNTIME'] + if base.startswith('dfu_device') or base.startswith('dfu_host'): + return [f'CFG_{prefix}_DFU'] + return [f'CFG_{prefix}_DFU', f'CFG_{prefix}_DFU_RUNTIME'] + out = [f'CFG_{prefix}_{cls.upper()}'] + # A class directory can hold more than one class. src/class/midi ships MIDI 1.0 + # AND MIDI 2.0: midi2_device.c is `#if CFG_TUD_ENABLED && CFG_TUD_MIDI2`, and + # examples/device/midi2_device is the only example that enables it - so the + # directory macro alone selected the midi_test examples, which do not compile the + # changed file, and none of the ones that do. Union, never replace: the file may + # still be pulled in by the directory's own macro, and over-selecting costs a build + # while under-selecting merges a break. + m = _CLS_STEM_RE.match(base) + if m and m.group(1) and m.group(1) != cls: + out.append(f'CFG_{prefix}_{m.group(1).upper()}') + return out + + +# A define is OFF only when its value is a literal zero (0, 00, (0)), optionally +# followed by a comment. Anything else counts as ON - including a value this cannot +# evaluate, e.g. `#define CFG_TUH_MIDI CFG_TUH_DEVICE_MAX` (examples/host/midi_rx). +# Fail-open: reading such a define as OFF made midi_host.c select zero families and +# let a compile break merge green. +# +# A macro defined more than once is ON if ANY of its defines is non-zero, because +# the preprocessor branches are not evaluated here: uac2_speaker_fb defines +# CFG_TUD_HID 1 under `#if CFG_AUDIO_DEBUG` and 0 in the #else, and the default +# build (CFG_AUDIO_DEBUG defaults to 1) compiles the HID class in. Deciding on the +# LAST/only match found made that example invisible to CFG_TUD_HID changes. +_DEF_VALUE = r'^[ \t]*#[ \t]*define[ \t]+{}[ \t]+(\S[^\n]*?)[ \t]*$' +_DEF_ZERO_VALUE = re.compile(r'\(?\s*0+\s*\)?\s*(?://.*|/\*.*)?') + + +# Shared rule-recognition primitives. The two classifiers walk the same diff with +# different answers, but they must RECOGNISE the same things: one copy each, so a +# new naming convention cannot land in one walk and be missed by the other. +_PORT_PATH_RE = re.compile(r'src/portable/((?:[^/]+/)?[^/]+)/') + + +def _port_roles(base: str) -> set: + """Which USB role a src/portable file serves, from its name: dcd_*/ *_device is + the device-controller side, hcd_*/ *_host the host side, anything else (shared + headers, glue) both.""" + if re.match(r'(dcd_|.*_device)', base): + return {'device'} + if re.match(r'(hcd_|.*_host)', base): + return {'host'} + return {'device', 'host'} + + +def _class_roles(base: str) -> set: + """Same question for a src/class file: <cls>_device.[ch] / <cls>_host.[ch], + else both - the class's shared header ships in either role.""" + if re.search(r'_device\.[ch]$', base): + return {'device'} + if re.search(r'_host\.[ch]$', base): + return {'host'} + return {'device', 'host'} + + [email protected]_cache(maxsize=None) +def _config_text(cfg_path: str) -> str: + """An example's tusb_config.h, read once. Every class path re-asks the same 46 + configs on both axes, so the reads go up with the diff: 4,240 of the same 46 files + for a diff touching all of src/class (0.48s -> 0.13s), and they cannot change + mid-run. Cached here rather than on _config_enables so the macros argument stays an + ordinary list at every call site.""" + try: + with open(cfg_path, encoding='utf-8', errors='replace') as f: + return f.read() + except OSError: + return '' + + +def _config_enables(cfg_path: str, macros) -> bool: + text = _config_text(cfg_path) + if not text: + return False + for m in macros: + for value in re.findall(_DEF_VALUE.format(m), text, re.M): + if not _DEF_ZERO_VALUE.fullmatch(value): + return True + return False + + +def examples_enabling(pool, macros, repo_root: str) -> set: + """The 'role/name' entries of `pool` whose src/tusb_config.h turns any of + `macros` on. The pool differs per classifier (HIL test lists vs every example), + the question does not.""" + return {ex for ex in pool + if _config_enables(os.path.join(repo_root, 'examples', ex, 'src', + 'tusb_config.h'), macros)} + + +# cached: called per changed lib file, and the tree doesn't change mid-run [email protected]_cache(maxsize=None) +def lib_examples(lib_name: str, repo_root: str) -> set: + """Examples whose OWN examples/<role>/<name>/{CMakeLists.txt,Makefile} references + lib/<lib_name> at a directory boundary (same boundary rule as path_families, so + 'lib/net' cannot inherit lib/networking's example). + + Per-example on purpose: lib/SEGGER_RTT is named by family_support.cmake's + LOGGER=rtt plumbing, which no CI example build turns on (all three references - + family_support.cmake, family_support.mk, rp2040/family.cmake - sit inside a + LOGGER=rtt guard), so a family-file scan would wrongly narrow it to three families + instead of answering 'nobody'. + + The whole example TREE is scanned, not just its top-level files: examples/host/ + msc_file_explorer_freertos/src/CMakeLists.txt names lib/embedded-cli, and that + example survived only because its top-level file happens to name it too.""" + pat = re.compile(re.escape('lib/' + lib_name) + r'(?=[/\s"\')}]|$)', re.M) + out = set() + for ex in all_examples(repo_root): + # the two filenames directly: '**/*' enumerated 489 entries per lib against a + # clean tree to use 107, and grows without bound once `make BOARD=... all` has + # written examples/<role>/<name>/_build/ - which is where /pre-pr runs + for f in sorted(glob.glob(_rg(repo_root, 'examples', ex, '**', 'CMakeLists.txt'), + recursive=True) + + glob.glob(_rg(repo_root, 'examples', ex, '**', 'Makefile'), + recursive=True)): + try: + text = _read(f) + except OSError: + continue + if pat.search(text): + out.add(ex) + break + return out + + +def roster_only_tests(all_boards) -> set: + """Test paths that only appear in a roster board's tests.only list (e.g. + espressif boards), not in the shared device/dual/host_test lists.""" + out = set() + for b in all_boards: + out.update(b.get('tests', {}).get('only', [])) + return out + + +def class_examples(macros, role: str, repo_root: str, extra_tests: set) -> set: + """Tests (from role's + dual lists, plus roster-only-list tests of that role) + whose example config enables any macro.""" + return examples_enabling(role_tests({role}, extra_tests), macros, repo_root) + + +def role_tests(roles: set, extras: set) -> set: + """Every test for the given role(s): each role's own list + dual tests, + plus roster-only-list tests (extras) matching those roles or 'dual'.""" + pool = set(dual_tests) + for r in roles: + pool |= set(ALL_TESTS[r]) + pool |= {t for t in extras if test_role(t) in roles or test_role(t) == 'dual'} + return pool + + +class _Sel: + """Accumulates contributions. board->set(tests) plus 'all-board' markers.""" + def __init__(self): + self.full = False + self.by_board = {} # name -> set of tests, or 'all' + self.roles = set() # roles touched by any contribution + self.families = set() # bsp families touched (incl. off-rig ones: build-only consumers) + self.reasons = [] + + def add(self, boards, tests, reason): + """tests: 'all' or iterable of test paths.""" + self.reasons.append(reason) + for b in boards: + cur = self.by_board.get(b) + if tests == 'all' or cur == 'all': + self.by_board[b] = 'all' + else: + self.by_board[b] = (cur or set()) | set(tests) + + def force_full(self, reason): + self.full = True + self.reasons.append(reason) + + +def _classify_one(path, repo_root, roster_boards, extras: set, s: _Sel, + get_deps_families=None): + base = os.path.basename(path) + if _NONCODE_RE.match(path) or _META_RE.match(path): + s.reasons.append(f'{path}: non-code, no contribution') + return + if _METRICS_RE.match(path): # rule 2b + s.reasons.append(f'{path}: build-size metrics tooling, no HIL contribution') + return + if _FULL_RE.match(path): + s.force_full(f'{path}: core/infra -> full matrix') + return + + if path == GET_DEPS_PATH: + if get_deps_families is None: + s.force_full(f'{path}: dep changes not resolvable -> full matrix') + return + if not get_deps_families: + s.reasons.append(f'{path}: no dep entry changed, no contribution') + return + fams = sorted(get_deps_families) + s.families.update(fams) + boards = [b['name'] for b in roster_boards + if board_family(b['name'], repo_root) in get_deps_families] + s.roles.update(('device', 'host')) + s.add(boards, 'all', f'{path}: dep entries changed -> families {fams} -> ' + f'boards {boards}') + return + + m = _PORT_PATH_RE.match(path) + if m: + port = m.group(1) + roles = _port_roles(base) + fams = port_families(port, repo_root) + if not fams: + # empty means empty (maintainer ruling), same reading as hw/mcu and as the + # build walk: no family's build references this port, so nothing compiles it + # and there is nothing to run. Forcing the full 30-board rig here bought no + # coverage at all - the build side selected zero families for the same path. + # Live for src/portable/template and the two microchip pic ports; + # TestPortFamiliesCoverage is the drift guard for a port that stops resolving. + s.reasons.append(f'{path}: port {port} maps to no board family, no contribution') + return + s.families.update(fams) + # a board can also pull the port in through a build option (e.g. MAX3421_HOST=1 + # from the roster on metro_m4_express, or from its own board.cmake), which its + # family file never names + gates = port_option_gates(repo_root).get(port, set()) + boards = [b['name'] for b in roster_boards + if (board_family(b['name'], repo_root) in fams or + (gates and board_options(b, repo_root) & gates)) and (board_roles(b) & roles)] + tests = role_tests(roles, extras) + s.roles.update(roles) + why = f'{path}: port {port} -> families {sorted(fams)}' + if gates: + why += f' + option {sorted(gates)}' + s.add(boards, tests, f'{why} -> boards {boards} ({"/".join(sorted(roles))})') + return + + m = re.match(r'src/class/([^/]+)/', path) + if m: + cls = m.group(1) + roles = _class_roles(base) + # this file's own class, plus any class whose headers include it + via = sorted(class_include_edges(repo_root).get(f'{cls}/{base}', ())) + + def macros(prefix): + return (class_macros(cls, base, prefix) + + [m2 for c in via for m2 in class_macros(c, '', prefix)]) + tests = set() + if 'device' in roles: + tests |= class_examples(macros('TUD'), 'device', repo_root, extras) + if 'host' in roles: + tests |= class_examples(macros('TUH'), 'host', repo_root, extras) + boards = [b['name'] for b in roster_boards if board_roles(b) & roles] + s.roles.update(roles) + why = f'{path}: class {cls}' + (f' (+ included by {via})' if via else '') + s.add(boards, tests, f'{why} -> {sorted(tests)} ({"/".join(sorted(roles))})') + return + + m = re.match(r'src/(device|host)/', path) + if m: + role = m.group(1) + boards = [b['name'] for b in roster_boards if role in board_roles(b)] + s.roles.add(role) + s.add(boards, role_tests({role}, extras), f'{path}: core {role} stack -> all {role} tests') + return + + m = re.match(r'hw/bsp/([^/]+)/(?:boards/([^/]+)/)?', path) + if m: + fam, brd = m.group(1), m.group(2) + s.families.add(fam) + if brd: + boards = [b['name'] for b in roster_boards if b['name'] == brd] + why = f'{path}: bsp board {brd}' + else: + boards = [b['name'] for b in roster_boards + if board_family(b['name'], repo_root) == fam] + why = f'{path}: bsp family {fam}' + s.roles.update(('device', 'host')) + s.add(boards, 'all', f'{why} -> boards {boards}') + return + + if re.match(r'hw/mcu/', path): + fams = mcu_families(path, repo_root) + if not fams: + # empty means empty (maintainer ruling): if no family's build references + # the path, no build consumes the change - there is nothing to compile, + # so there is nothing to run either. TestOrphanInvariant's + # test_tracked_mcu_vendors_resolve is the drift guard: a real vendor dir + # that stops resolving fails pre-commit instead of silently vanishing + s.reasons.append(f'{path}: hw/mcu path resolves to no family, no contribution') + return + s.families.update(fams) + boards = [b['name'] for b in roster_boards + if board_family(b['name'], repo_root) in fams] + s.roles.update(('device', 'host')) + s.add(boards, 'all', f'{path}: mcu dir -> families {sorted(fams)} -> boards {boards}') + return + + m = re.match(r'lib/([^/]+)/', path) + if m: + lib = m.group(1) + # only the tests whose example builds the lib, and only those the rig runs + tests = {e for e in lib_examples(lib, repo_root) + if any(e in pool for pool in ALL_TESTS.values()) or e in extras} + if lib == 'SEGGER_RTT': + # no example names this lib, but a board whose roster entry says + # "logger": "rtt" (variant defines LOGGER=rtt) reads EVERY test's console + # through it -- a break here silently breaks all of that board's rows + rtt_boards = [b['name'] for b in roster_boards if b.get('logger') == 'rtt'] + if rtt_boards: + s.roles.update(('device', 'host')) + s.add(rtt_boards, 'all', + f'{path}: SEGGER_RTT is the rtt console on {rtt_boards} -> all tests') + return + if not tests: + s.reasons.append(f'{path}: lib {lib} used by no HIL test, no contribution') + return + roles = set() + for test in tests: + r = test_role(test) + roles.update(('device', 'host') if r == 'dual' else (r,)) + boards = [b['name'] for b in roster_boards] + s.roles.update(roles) + s.add(boards, sorted(tests), f'{path}: lib {lib} -> {sorted(tests)} on all boards') + return + + if re.match(r'src/typec/', path): + # only examples/typec enables CFG_TUC_ENABLED, and no rig board runs a typec + # test (see _HIL_EX_ROLES) - so the build axis covers it and the rig cannot + s.reasons.append(f'{path}: typec, no HIL contribution') + return + m = _BUILD_EX_RE.match(path) + if m: + if m.group(1) not in _HIL_EX_ROLES: + # examples/typec: the build matrix compiles it, nothing on the rig runs it + s.reasons.append(f'{path}: {m.group(1)} example, no HIL contribution') + return + test = f'{m.group(1)}/{m.group(2)}' + known = any(test in pool for pool in ALL_TESTS.values()) or test in extras + if known: + boards = [b['name'] for b in roster_boards] + role = test_role(test) + s.roles.update(('device', 'host') if role == 'dual' else (role,)) + s.add(boards, [test], f'{path}: example -> {test} on all boards') + else: + s.reasons.append(f'{path}: example not in HIL lists, no contribution') + return + + s.force_full(f'{path}: unclassified -> full matrix') + + +def classify(changed_files, repo_root, rosters, get_deps_families=None): + all_boards = [] + seen = set() + for _, boards in rosters: + for b in boards: + if b['name'] not in seen: + seen.add(b['name']) + all_boards.append(b) + + extras = roster_only_tests(all_boards) + s = _Sel() + # no early exit once full: keep classifying so `families` still reports every + # family the diff touches (build-only consumers need it). Nothing after the first + # force_full can change full/boards/args - the full branch below ignores by_board. + for path in changed_files: + _classify_one(path, repo_root, all_boards, extras, s, get_deps_families) + + if s.full: + return {'full': True, 'boards': {b['name']: 'all' for b in all_boards}, + 'families': sorted(s.families), 'reasons': s.reasons} + + # role pruning: single-role selections drop the other role's tests and boards + by_name = {b['name']: b for b in all_boards} + out = {} + for name, tests in s.by_board.items(): + allowed = board_tests(by_name[name]) + if tests == 'all': + kept = list(allowed) + else: + kept = [t for t in allowed if t in tests] + if s.roles and s.roles != {'device', 'host'}: + role = next(iter(s.roles)) + kept = [t for t in kept if test_role(t) in (role, 'dual')] + if kept: + out[name] = 'all' if set(kept) == set(allowed) else sorted(kept) + return {'full': False, 'boards': out, 'families': sorted(s.families), + 'reasons': s.reasons} + + +def _board_args(name, chosen) -> list: + parts = [f'-b {name}'] + if chosen != 'all': + parts.append(f'-bt {name}:{",".join(chosen)}') + return parts + + +def hil_examples(sel, rosters): + """{board: examples hil-build must produce}: the board's selected tests plus + device/board_test, which hil_test.py flashes to park at every variant + boundary and at end-of-board teardown. Emitted for full selections too - the + HIL example universe is a fraction of the tree regardless of the diff.""" + by_name = {} + for _, boards in rosters: + for b in boards: + by_name.setdefault(b['name'], []).append(b) + if sel['full']: + chosen = {n: 'all' for n in by_name} + else: + chosen = sel['boards'] + out = {} + for name, tests in chosen.items(): + if tests == 'all': + # a board named by two rosters (rig migration, or shared between rigs) + # may run different tests on each: union them. Superset firmware costs a + # build; a missing image fails the run on whichever rig lost the toss. + run = set().union(*(board_tests(b) for b in by_name[name])) + else: + run = set(tests) + out[name] = sorted(run | {'device/board_test'}) + return out + + +def selection_args(sel, rosters): + """hil_test.py args per config. Empty means either 'full matrix' or 'nothing + selected' - callers must read sel['full'] to tell them apart.""" + args = {} + for cfg_path, boards in rosters: + parts = [] + if not sel['full']: + for b in boards: + chosen = sel['boards'].get(b['name']) + if chosen is not None: + parts += _board_args(b['name'], chosen) + args[os.path.basename(cfg_path)] = ' '.join(parts) + return args + + +def selection_args_by_flasher(sel, rosters): + """{config: {flasher name: args}}. CI runs one rig as several jobs split by + flasher (esptool vs the rest); each must gate on its own subset, otherwise the + other leg runs a filter matching zero boards and reports a vacuous green.""" + out = {} + for cfg_path, boards in rosters: + per = {} + if not sel['full']: + for b in boards: + chosen = sel['boards'].get(b['name']) + if chosen is None: + continue + per.setdefault(b.get('flasher', {}).get('name', ''), []).extend( + _board_args(b['name'], chosen)) + out[os.path.basename(cfg_path)] = {f: ' '.join(p) for f, p in per.items()} + return out + + +def merge_base(base, repo_root): + return subprocess.run(['git', 'merge-base', 'HEAD', base], cwd=repo_root, + capture_output=True, text=True, check=True).stdout.strip() + + +def git_show(spec, repo_root): + return subprocess.run(['git', 'show', spec], cwd=repo_root, + capture_output=True, text=True, check=True).stdout + + +def changed_files_from_git(base, repo_root): + diff = subprocess.run(GIT_DIFF_ARGV + [f'{merge_base(base, repo_root)}..HEAD'], + cwd=repo_root, capture_output=True, text=True, check=True).stdout + return [l for l in diff.splitlines() if l.strip()] + + +def get_deps_families_from_git(base, repo_root): + """The changed dep entries' families for a --base run, or None (-> full matrix) + if git cannot produce both sides of tools/get_deps.py.""" + try: + mb = merge_base(base, repo_root) + return get_deps_changed_families(git_show(f'{mb}:{GET_DEPS_PATH}', repo_root), + git_show(f'HEAD:{GET_DEPS_PATH}', repo_root), + repo_root) + except (subprocess.CalledProcessError, OSError) as e: + print(f'ci_select: {GET_DEPS_PATH}: base content unreadable ({e})', file=sys.stderr) + return None + + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + g = ap.add_mutually_exclusive_group(required=True) + g.add_argument('--base', help='git ref to diff against (merge-base..HEAD)') + g.add_argument('--diff-file', help='newline-separated changed-file list') + ap.add_argument('configs', nargs='*', help='rig roster JSON file(s); omit for the build view alone') + a = ap.parse_args() + + repo_root = _REPO_ROOT + rosters = [] + for c in a.configs: + with open(c, encoding='utf-8', errors='replace') as f: + rosters.append((c, json.load(f)['boards'])) + + files = (_read(a.diff_file).splitlines() if a.diff_file + else changed_files_from_git(a.base, repo_root)) + files = [f for f in files if f.strip()] + + # --diff-file has no git and so no base content: the rule falls open to full + gd = (get_deps_families_from_git(a.base, repo_root) + if a.base and GET_DEPS_PATH in files else None) + + s = classify(files, repo_root, rosters, gd) + s['args'] = selection_args(s, rosters) + s['args_flasher'] = selection_args_by_flasher(s, rosters) + if rosters: + s['hil_examples'] = hil_examples(s, rosters) + s['build'] = classify_build(files, repo_root, gd) + for r in s['build']['reasons']: + print(f'ci_select[build]: {r}', file=sys.stderr) + for r in s['reasons']: + print(f'ci_select: {r}', file=sys.stderr) + # reasons go to stderr ONLY - they are a human diagnostic and no consumer reads them + # back. They are also ~97% of the payload (a whole-tree diff: 453 KB -> 12 KB), which + # build.yml re-parses with ci_set_matrix, hil_ci_set_matrix, an inline python and + # three jq calls. The in-process dicts still carry them, for the log and the tests. + out = {k: v for k, v in s.items() if k != 'reasons'} + out['build'] = {k: v for k, v in s['build'].items() if k != 'reasons'} + print(json.dumps(out)) + + +# ------------------------------------------------------------- +# Build-axis classifier (spec rule table, docs/superpowers/specs/ +# 2026-08-19-ci-build-family-filter-design.md). Independent of the HIL +# classifier: same diff, second walk, its own fail-open. +# ------------------------------------------------------------- +# Both walks recognise an example path with the SAME regex, so a role can never be +# known to one walk and unclassified (-> full matrix) to the other. What differs is the +# answer: the rig runs device/host/dual tests, while the build matrix also compiles +# examples/typec, which nothing on the rig runs. +_EX_ROLES = ('device', 'dual', 'host', 'typec') +_HIL_EX_ROLES = ('device', 'host', 'dual') +_BUILD_EX_RE = re.compile(r'examples/(%s)/([^/]+)/' % '|'.join(_EX_ROLES)) + + [email protected]_cache(maxsize=None) +def all_examples(repo_root: str) -> tuple: + """Every examples/<role>/<name> with a CMakeLists.txt, as 'role/name'.""" + out = [] + for role in _EX_ROLES: + for d in sorted(glob.glob(_rg(repo_root, 'examples', role, '*/'))): + if os.path.isfile(os.path.join(d, 'CMakeLists.txt')): + out.append(f'{role}/{os.path.basename(d.rstrip(os.sep))}') + return tuple(out) + + +def role_examples(repo_root: str, roles) -> set: + want = set(roles) + return {e for e in all_examples(repo_root) if e.split('/', 1)[0] in want} + + [email protected]_cache(maxsize=None) +def all_bsp_families(repo_root: str) -> tuple: + return tuple(sorted(d for d in os.listdir(os.path.join(repo_root, 'hw/bsp')) + if os.path.isdir(os.path.join(repo_root, 'hw/bsp', d)))) + + +def _build_class_examples(cls: str, base: str, roles: set, repo_root: str) -> set: + """Examples (all 46, not the HIL lists) whose tusb_config.h enables the class's + macros for the given roles, plus classes that #include the changed header.""" + via = sorted(class_include_edges(repo_root).get(f'{cls}/{base}', ())) + out = set() + for prefix, role in (('TUD', 'device'), ('TUH', 'host')): + if role not in roles: + continue + macros = class_macros(cls, base, prefix) + \ + [m for c in via for m in class_macros(c, '', prefix)] + out |= examples_enabling(all_examples(repo_root), macros, repo_root) + return out + + +class _BSel: + """family -> set(examples) | 'all', unioned per family.""" + def __init__(self): + self.full = False + self.fam_ex = {} + self.reasons = [] + + def add(self, fams, examples, reason): + self.reasons.append(reason) + for f in fams: + cur = self.fam_ex.get(f) + if examples == 'all' or cur == 'all': + self.fam_ex[f] = 'all' + else: + self.fam_ex[f] = (cur or set()) | set(examples) + + def force_full(self, reason): + self.full = True + self.reasons.append(reason) + + +def _classify_build_one(path, repo_root, s: _BSel, get_deps_families=None): + base = os.path.basename(path) + if _NONCODE_RE.match(path) or _META_RE.match(path): # rules 1, 1b + s.reasons.append(f'{path}: non-code, no build contribution') + return + if re.match(r'test/hil/', path): # rule 2 + s.reasons.append(f'{path}: HIL harness, no build contribution') + return + if path == GET_DEPS_PATH: # rule 16b + if get_deps_families is None: + s.force_full(f'{path}: dep changes not resolvable -> full build matrix') + return + if not get_deps_families: + s.reasons.append(f'{path}: no dep entry changed, no contribution') + return + fams = sorted(get_deps_families) + s.add(fams, 'all', f'{path}: dep entries changed -> families {fams}') + return + m = _PORT_PATH_RE.match(path) + if m: # rules 3-5 + port = m.group(1) + fams = port_families(port, repo_root) + roles = _port_roles(base) + exs = 'all' if roles == {'device', 'host'} else \ + role_examples(repo_root, tuple(roles) + ('dual',)) + # rule 5b: fams empty -> s.add iterates nothing -> no contribution + s.add(fams, exs, f'{path}: port {port} -> families {sorted(fams)}') + return + if re.match(r'hw/bsp/[^/]+/', path): # rule 6 + fam = path.split('/')[2] + s.add({fam}, 'all', f'{path}: bsp family {fam}') + return + if re.match(r'hw/mcu/', path): # rule 7 + fams = mcu_families(path, repo_root) + if not fams: + # empty means empty, same reading as the HIL walk: no family's build + # references the path, so no build compiles it + s.reasons.append(f'{path}: hw/mcu path resolves to no family, no contribution') + return + s.add(fams, 'all', f'{path}: mcu -> families {sorted(fams)}') + return + m = re.match(r'src/class/([^/]+)/', path) + if m: # rules 8-10 + cls = m.group(1) + roles = _class_roles(base) + exs = _build_class_examples(cls, base, roles, repo_root) + if not exs: + # Empty means empty - maintainer decision. No example config enables this + # class, so no build exercises it and + # nothing is selected. The file IS still parsed by every full build + # (src/CMakeLists.txt, src/tinyusb.mk list class sources unconditionally, + # the CFG_ guard sits inside), so a break outside the guard surfaces on the + # next master push - the accepted safety net. + s.reasons.append(f'{path}: class {cls} enabled by no example config, ' + f'no contribution') + return + s.add(all_bsp_families(repo_root), exs, + f'{path}: class {cls} -> {sorted(exs)}') + return + m = re.match(r'src/(device|host)/', path) + if m: # rules 11-12 + role = m.group(1) + s.add(all_bsp_families(repo_root), role_examples(repo_root, (role, 'dual')), + f'{path}: core {role} stack') + return + m = _BUILD_EX_RE.match(path) + if m: # rules 13-14 + ex = f'{m.group(1)}/{m.group(2)}' + if ex in all_examples(repo_root): + s.add(all_bsp_families(repo_root), {ex}, f'{path}: example {ex}') + else: + # a deleted example builds nothing; removing it from the role + # CMakeLists (rule 15) is what forces the full matrix + s.reasons.append(f'{path}: not an example dir, no build contribution') + return + if re.match(r'src/typec/', path): # rule 12b + # listed unconditionally by src/CMakeLists.txt and src/tinyusb.mk, but the whole + # body is `#if CFG_TUC_ENABLED` - so it is PARSED by every build and COMPILED + # only for examples that enable it. Same shape as the class rule, same answer: + # the examples whose tusb_config.h turns it on, and empty means empty. + exs = examples_enabling(role_examples(repo_root, ('typec',)), + ('CFG_TUC_ENABLED',), repo_root) + if not exs: + s.reasons.append(f'{path}: typec enabled by no example config, no contribution') + return + s.add(all_bsp_families(repo_root), exs, f'{path}: typec -> {sorted(exs)}') + return + m = re.match(r'lib/([^/]+)/', path) + if m: # rule 16a + lib = m.group(1) + exs = lib_examples(lib, repo_root) + if not exs: + # empty means empty: no example's build pulls this lib in, so no MAIN- + # matrix build compiles it. (lib/SEGGER_RTT is reached through LOGGER=rtt, + # which the main matrix never sets; the hil-build legs set it only for + # roster boards whose variant defines carry it, via the HIL SEGGER_RTT rule. + # No committed CI roster has such a board yet, so a SEGGER_RTT edit is + # currently neither built nor HIL-tested by CI -- verify vendor bumps + # manually until a rig board adopts "logger": "rtt".) + s.reasons.append(f'{path}: lib {lib} built by no example, no contribution') + return + s.add(all_bsp_families(repo_root), exs, f'{path}: lib {lib} -> {sorted(exs)}') + return + if _METRICS_RE.match(path): + # HIL-suppressed above; on this axis they stay full - tools/metrics.py runs as + # the `tinyusb_metrics` build target, so a break in it fails the build + s.force_full(f'{path}: metrics tooling runs in the build -> full build matrix') + return + if _FULL_RE.match(path): # rules 15-16 + # attribution, not behaviour: these already reached `full` through the + # fall-through below. Naming them means a future narrowing of rule 17 cannot + # silently change what they do. Deliberately last, so every earlier rule keeps + # priority - examples/device/board_test is rule 14 (just board_test), not ALL. + s.force_full(f'{path}: core/infra -> full build matrix') + return + s.force_full(f'{path}: unclassified -> full build matrix') # rule 17 + + +def _in_repo(repo_root): + """build_utils/build.py use repo-relative paths; scope a chdir around them. + get_family_boards also prints on an empty family - swallow stdout so the + selector's machine-read JSON stays clean (diagnostics belong on stderr).""" + old = os.getcwd() + os.chdir(repo_root) + try: + with contextlib.redirect_stdout(io.StringIO()): + yield + finally: + os.chdir(old) + + +def _prune_buildable(fams, fam_ex, repo_root): + """Intersect each family's selection with what the family can build at all + (build_utils.skip_example - the same skip.txt/only.txt data CMake's + family_filter reads). + + ANY board of the family counts, not just the one GHA's --one-first picks: + CircleCI's cmake legs build every board of a family, so an example gated to a + single board (only.txt board:mimxrt1060_evk) would otherwise lose ALL compile + coverage exactly when a PR touches it. get_family_boards(.., False, False) is + that full list, with the same CI skip lists the build jobs apply. + + EITHER build system counts too. This one list gates CircleCI's make legs as well + as its cmake ones, and the two answer different questions (build_utils.skip_example): + examples/device/dfu carries `mcu:BCM2835` in skip.txt, which the cmake FAMILY_MCUS + union applies to every broadcom_64bit board while the make scrape applies it to + none - asking cmake alone drops the only aarch64-gcc family in the matrix and + `build-make-aarch64-gcc` stops compiling dfu at all.""" + out_fams, out_ex, reasons = [], {}, [] + allex = list(all_examples(repo_root)) + with _in_repo(repo_root): + for fam in fams: + if not os.path.isdir(os.path.join(repo_root, 'hw/bsp', fam, 'boards')): + # a PR that deletes or renames hw/bsp/<fam> still names it in the + # diff (rule 6); the family builds nothing now, and get_family_boards + # would raise FileNotFoundError out of the whole selector + reasons.append(f'{fam}: family dir gone from tree, dropped') + continue + try: + # ci=True unconditionally: this answers "what will CI build", so it must + # not change with GITHUB_ACTIONS/CIRCLECI being set. Locally the lists + # are off by default, and rp2040 would keep feather_rp2040_max3421 - + # the only board satisfying the max3421 only.txt files - giving a + # developer a family list the runner will not reproduce. + boards = build_py.get_family_boards(fam, False, False, ci=True) + except OSError as e: # belt and braces: never traceback here + reasons.append(f'{fam}: boards unreadable ({e}), dropped') + continue + if not boards: + out_fams.append(fam) # unknown layout: keep unfiltered + continue + # what this family's build path can even see, asked the same way for + # every family. build.py's espressif branch builds get_examples('espressif') + # only (the *_freertos examples plus a short extra list); keeping the family + # for anything else spins up CI's most expensive leg to skip every example + # it was given. Identical to the unfiltered list on all 81 other families. + pool = set(build_py.get_examples(fam)) + + # asked per example instead of materialising the family's whole buildable + # list: skip_example is by far the hottest call in the selector, and every + # question below short-circuits (one cdc_device.c diff: 6,883 calls -> 1,889) + def can_build(ex): + # EITHER build system: this one list gates CircleCI's make legs too, and + # the two answer differently (build_utils.skip_example) + return ex in pool and any( + not build_utils.skip_example(ex, b) or + not build_utils.skip_example(ex, b, (), 'make') for b in boards) + + want = fam_ex.get(fam) + try: + if want is None: + kept = None if any(can_build(e) for e in allex) else [] + else: + kept = [e for e in want if can_build(e)] + if kept and not any(can_build(e) for e in allex if e not in want): + kept = None # already everything the family can build + except OSError as e: + # a family mid-bring-up (boards/ but no family.cmake/family.mk yet) + # reads as unbuildable to the scrape; keep it rather than tracebacking + # out of the selector and losing the scoping for the whole PR + reasons.append(f'{fam}: mcu scrape unreadable ({e}), kept unfiltered') + out_fams.append(fam) + continue + if kept == []: + continue # this diff builds nothing for this family + out_fams.append(fam) + if kept is not None: + out_ex[fam] = kept + return out_fams, out_ex, reasons + + +def classify_build(changed_files, repo_root, get_deps_families=None): + s = _BSel() + for p in changed_files: + _classify_build_one(p, repo_root, s, get_deps_families) + if s.full: + return {'full': True, 'families': list(all_bsp_families(repo_root)), + 'family_examples': {}, 'reasons': s.reasons} + fams = sorted(s.fam_ex) + fam_ex = {f: sorted(e) for f, e in s.fam_ex.items() if e != 'all'} + fams, fam_ex, pruned = _prune_buildable(fams, fam_ex, repo_root) + s.reasons += pruned + return {'full': False, 'families': fams, 'family_examples': fam_ex, + 'reasons': s.reasons} + + +if __name__ == '__main__': + main() diff --git a/tools/gen_doc.py b/tools/gen_doc.py index 3920531d5..41a60c0b6 100755 --- a/tools/gen_doc.py +++ b/tools/gen_doc.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import json import re import pandas as pd from tabulate import tabulate @@ -110,8 +111,59 @@ Following boards are supported""" # ----------------------------------------- +# HIL rig rosters +# ----------------------------------------- +def hil_cell(text): + """A '|' in free-form roster text would silently split the markdown row.""" + return ' '.join((text or '').split()).replace('|', '\\|') + + +def hil_rows(boards): + rows = [] + for b in boards: + tests = b.get('tests', {}) + if 'only' in tests: + roles = sorted({t.split('/')[0] for t in tests['only']}) + else: + roles = [r for r in ('device', 'host', 'dual') if tests.get(r)] + rows.append([ + b['name'], + ', '.join(roles), + b.get('flasher', {}).get('name', ''), + hil_cell(', '.join(v['name'] for v in b.get('variant') or [])), + hil_cell(b.get('comment') or tests.get('comment')), + ]) + return rows + + +def gen_hil_boards_doc(): + tinyusb = json.loads((Path(TOP) / "test/hil/tinyusb.json").read_text()) + hfp = json.loads((Path(TOP) / "test/hil/hfp.json").read_text()) + sections = [ + ("ci rig", "test/hil/tinyusb.json", tinyusb.get('boards', [])), + ("hfp rig", "test/hil/hfp.json", hfp.get('boards', [])), + ] + headers = ['Board', 'Roles', 'Flasher', 'Variants', 'Note'] + + out = ["<!-- Generated by tools/gen_doc.py - do not edit. -->", ""] + for title, src, boards in sections: + if not boards: + continue + out.append(f"### {title}") + out.append("") + out.append(f"{len(boards)} boards, from `{src}`.") + out.append("") + out.append(tabulate(hil_rows(boards), headers=headers, tablefmt='github')) + out.append("") + + hil_md = Path(TOP) / "docs/reference/hil_boards.md" + hil_md.write_text('\n'.join(out)) + + +# ----------------------------------------- # Main # ----------------------------------------- if __name__ == "__main__": gen_deps_doc() gen_boards_doc() + gen_hil_boards_doc() diff --git a/tools/get_deps.py b/tools/get_deps.py index baaf3761f..12bec4861 100755 --- a/tools/get_deps.py +++ b/tools/get_deps.py @@ -33,7 +33,7 @@ deps_mandatory = { deps_optional = { 'hw/mcu/allwinner': ['https://github.com/hathach/allwinner_driver.git', '8e5e89e8e132c0fd90e72d5422e5d3d68232b756', - 'fc100s'], + 'f1c100s'], 'hw/mcu/analog/msdk' : ['https://github.com/analogdevicesinc/msdk.git', 'b20b398d3e5e2007594e54a74ba3d2a2e50ddd75', 'maxim'], @@ -108,7 +108,7 @@ deps_optional = { 'efm32'], 'hw/mcu/sony/cxd56/spresense-exported-sdk': ['https://github.com/sonydevworld/spresense-exported-sdk.git', '2ec2a1538362696118dc3fdf56f33dacaf8f4067', - 'spresense'], + 'cxd56'], 'hw/mcu/st/cmsis_device_c0': ['https://github.com/STMicroelectronics/cmsis_device_c0.git', '517611273f835ffe95318947647bc1408f69120d', 'stm32c0'], @@ -386,6 +386,10 @@ def main(): parser.add_argument('-f1', '--build-flags-on', action='append', default=[], help='Have no effect') parser.add_argument('--build-name', default=None, help='Have no effect') parser.add_argument('--cflag', action='append', default=[], help='Have no effect') + # build-matrix entries carry -e for tools/build.py; they reach get_deps.py + # verbatim (.github/actions/get_deps, build.yml's hil-hfp-iar) and an + # argparse error here reds the Get Dependencies step of every scoped PR + parser.add_argument('-e', '--example', action='append', default=[], help='Have no effect') args = parser.parse_args() families = args.families diff --git a/tools/iar_template.ipcf b/tools/iar_template.ipcf index 035e40b94..922b22426 100644 --- a/tools/iar_template.ipcf +++ b/tools/iar_template.ipcf @@ -81,9 +81,7 @@ </group> <group name="src/class/vendor"> <path>$TUSB_DIR$/src/class/vendor/vendor_device.c</path> - <path>$TUSB_DIR$/src/class/vendor/vendor_host.c</path> <path>$TUSB_DIR$/src/class/vendor/vendor_device.h</path> - <path>$TUSB_DIR$/src/class/vendor/vendor_host.h</path> </group> <group name="src/class/video"> <path>$TUSB_DIR$/src/class/video/video_device.c</path> diff --git a/tools/make_release.py b/tools/make_release.py index 65226834f..ec4755f34 100755 --- a/tools/make_release.py +++ b/tools/make_release.py @@ -59,6 +59,7 @@ with open(f_sonar_properties, 'w') as f: # gen docs gen_doc.gen_deps_doc() gen_doc.gen_boards_doc() +gen_doc.gen_hil_boards_doc() # gen presets gen_presets.main() diff --git a/tools/metrics.py b/tools/metrics.py index 0e29fc1ab..27c995954 100644 --- a/tools/metrics.py +++ b/tools/metrics.py @@ -98,6 +98,24 @@ def combine_files(input_files, filters=None): if fin.endswith(".json"): with open(fin, "r", encoding="utf-8") as f: json_data = json.load(f) + if fin.endswith('_by_example.json') and isinstance(json_data, dict) and \ + all(isinstance(v, dict) and 'files' in v for v in json_data.values()): + # a metrics_by_example.json: one data entry per example. Keyed on + # the filename, which IS the contract (write_by_example, the CMake + # rule and metrics_pair_compare all spell that suffix) - a shape + # sniff would silently reroute any coincidentally-shaped JSON. + for ex in sorted(json_data): + # same TOTAL scrub the shared path below applies: this branch + # `continue`s past it, so do it here or a by-example input keeps + # the fake TOTAL rows an ordinary input has stripped + sub = {'files': [f for f in json_data[ex]['files'] + if str(f.get('file', '')).upper() != 'TOTAL']} + if filters: + sub['files'] = [f for f in sub['files'] + if f.get('path') and any(x in f['path'] for x in filters)] + all_json_data['file_list'].append(f'{fin}:{ex}') + all_json_data['data'].append(sub) + continue if filters: json_data["files"] = [ f @@ -316,6 +334,25 @@ def write_json_output(json_data, path): json.dump(json_data, outf, indent=2) +def write_by_example(all_json_data, path): + """{<role>/<example>: {files: [...]}} from the data combine_files already parsed + - re-reading and re-parsing every input a second time bought nothing. + + Inputs are map.json files laid out as <build>/<role>/<example>/<name>.map.json + (examples/CMakeLists.txt's pattern), so the example name is the last two path + components; a metrics_by_example.json input already carries its own name in the + file_list entry ('<file>.json:<role>/<name>').""" + out = {} + for fin, data in zip(all_json_data["file_list"], all_json_data["data"]): + _, sep, ex = fin.partition('.json:') + if not sep: + d = os.path.dirname(os.path.abspath(fin)) + ex = f'{os.path.basename(os.path.dirname(d))}/{os.path.basename(d)}' + out.setdefault(ex, {'files': []})['files'] += data.get('files', []) + with open(path, 'w', encoding='utf-8') as f: + json.dump(out, f) + + def render_combine_table(json_data, sort_order='name+'): """Render averaged sizes as markdown table lines (no title).""" files = json_data.get("files", []) @@ -594,6 +631,8 @@ def cmd_combine(args): if args.markdown_out: write_combine_markdown(json_average, args.out + '.md', sort_order=args.sort, title="TinyUSB Average Code Size Metrics") + if args.by_example: + write_by_example(all_json_data, args.out + '_by_example.json') def cmd_compare(args): @@ -633,6 +672,8 @@ def main(argv=None): combine_parser.add_argument('-S', '--sort', dest='sort', default='size-', choices=['size', 'size-', 'size+', 'name', 'name-', 'name+'], help='Sort order: size/size- (descending), size+ (ascending), name/name+ (ascending), name- (descending). Default: size-') + combine_parser.add_argument('--by-example', dest='by_example', action='store_true', + help='Also write <out>_by_example.json: per-example file lists keyed by role/example') # Compare subcommand compare_parser = subparsers.add_parser('compare', help='Compare two metrics inputs (bloaty CSV or metrics JSON)') diff --git a/tools/metrics_compare_base.py b/tools/metrics_compare_base.py index 799a96800..844130097 100644 --- a/tools/metrics_compare_base.py +++ b/tools/metrics_compare_base.py @@ -81,7 +81,7 @@ def symlink_deps(main_root, worktree_dir): def ci_first_boards(): """Return the first board (alphabetical) of each arm-gcc CI family.""" - matrix_py = os.path.join(TINYUSB_ROOT, '.github', 'workflows', 'ci_set_matrix.py') + matrix_py = os.path.join(TINYUSB_ROOT, '.github', 'scripts', 'ci_set_matrix.py') if not os.path.isfile(matrix_py): return [] ret = run([sys.executable, matrix_py]) @@ -188,7 +188,7 @@ def main(): args.combined = True ci_boards = ci_first_boards() if not ci_boards: - parser.error('--ci: failed to derive boards from .github/workflows/ci_set_matrix.py') + parser.error('--ci: failed to derive boards from .github/scripts/ci_set_matrix.py') # Append, dedup, preserve order seen = set(args.board) for b in ci_boards: diff --git a/tools/rtt.py b/tools/rtt.py new file mode 100644 index 000000000..e3aef36f2 --- /dev/null +++ b/tools/rtt.py @@ -0,0 +1,727 @@ +#!/usr/bin/env python3 +"""RTT console/capture over a debug probe — importable classes + CLI (the rtt +skill's SKILL.md is the manual). + +Three routes (see the skill's transport matrix for which route a probe gets). +--backend is always explicit: + + J-Link route (console/capture, channel 0 only) + rtt.py --backend jlink --probe <sn> --device <JLINK_DEVICE> [--seconds N] [-i] + OpenOCD route (native probes: ST-Link/CMSIS-DAP; console/capture, any channel) + rtt.py --backend openocd [--probe <sn>] [--vid-pid "0xVVVV 0xPPPP"] \\ + --cfg "-f interface/stlink.cfg -f target/stm32h7x.cfg" \\ + (--elf <flashed.elf> | --addr 0x2000xxxx) [--channel N] [--seconds N] [-i] + [--reset-before-attach] # capture from the target's boot (SystemView) + Post-mortem ring dump (J-Link, no halt — debug-AP reads) + rtt.py --backend jlink --dump <out.bin> --probe <sn> --device <JLINK_DEVICE> \\ + (--elf <flashed.elf> | --addr 0x...) + +The probe is owned for the whole run: flash and reset BEFORE starting this, never +reset the target while it is attached. Pin the probe: rigs and benches run +several (jlink: --probe serial; openocd: --probe and/or --vid-pid). + +The classes (JlinkRtt for J-Link, OpenocdRtt for openocd-driven probes) expose +the slice of pyserial the HIL harness uses — read/in_waiting/write/close/timeout, +reset_input_buffer, context-manager use, plus an `eof` latch — and are imported +by test/hil/helper/hil_util.py, so this file is HARNESS-CRITICAL: a change here +is classified like a test/hil/ harness change (tools/ci_select.py) and runs the +console unit tests (pre-commit hil-test hook, test/hil/test/test_hil_rtt.py). +Stdlib only — hil_util imports this file, never the other way around. +""" +import argparse +import contextlib +import os +import re +import select +import shlex +import signal +import socket +import subprocess +import sys +import tempfile +import threading +import time + + +class RttError(RuntimeError): + """Every way a console can break: stall, closed, dead or reset server. + + A RuntimeError subclass so existing `except RuntimeError` callers keep working, + but named so the harness can tell a console failure from an unrelated + NotImplementedError / 'dictionary changed size during iteration' and stop + reporting harness bugs as board failures.""" + + +def _pos_float_env(name: str, default: float) -> float: + # mirrors hil_util.pos_float_env, including its rejection of inf/nan: an infinite + # write timeout is an unbounded write, the very thing this knob exists to bound + raw = os.environ.get(name) + if raw is None: + return default + try: + v = float(raw) + except ValueError: + print(f'warning: {name} is not a number; using {default}', file=sys.stderr, flush=True) + return default + if not (v > 0 and v < float('inf')): + print(f'warning: {name}={v} is not usable; using {default}', file=sys.stderr, flush=True) + return default + return v + + +# whole-call deadline for write() — same env knob as the harness's serial twin +RTT_WRITE_TIMEOUT = _pos_float_env('HIL_SERIAL_WRITE_TIMEOUT', 10) + +# J-Link Commander's telnet greeting, sent at connect BEFORE (or without) the control +# block being found: never target output. Three lines; the middle one is the PROBE +# MODEL string, which in libjlinkarm carries no 'SEGGER ' prefix (J-Link OH3, +# J-Trace H9, ...) though some builds do prefix it — match both shapes. Consumers +# judging "did the target speak" must strip these lines first. +RTT_BANNER_RE = re.compile(r'^(SEGGER J-|J-Link[ 0-9]|J-Trace[ 0-9]|Process:\s)') + + +def strip_banner(data: bytes, complete_only: bool = False) -> bytes: + """Target bytes only: drop the J-Link server banner lines and blanks. + + Both harness consumers (hil_test's device_info verdict, hil_pool_check's + aliveness score) must judge "did the target speak" through this one filter, + or the same byte stream scores differently per consumer. complete_only=True + additionally drops a trailing unterminated line — for poll loops judging a + growing buffer, where a banner FRAGMENT at a read boundary (b'SEGG', b'Proce') + would defeat the prefix regex and count as target output; the final verdict + after the window should pass complete_only=False to keep a genuine + unterminated tail.""" + lines = data.splitlines(keepends=False) + if complete_only and data and not data.endswith((b'\n', b'\r')) and lines: + lines = lines[:-1] + return b'\n'.join(l for l in lines + if l.strip() and not RTT_BANNER_RE.match(l.decode('utf-8', errors='ignore'))) + + +def free_ports(count: int) -> list: + """Bind ephemeral ports and hand back the numbers. Boards run in parallel, so the + RTT/GDB ports cannot be the SEGGER defaults or two boards collide. + + Known TOCTOU: the port is free when released here, but another process can claim + it before the server binds it. Accepted — the server binds the port itself, so + there is no fd to hand over. The post-connect re-poll catches the common outcome + (our server lost the bind and died); a foreign listener that stays alive is not + detectable here and would need the connected peer to be validated.""" + socks = [] + try: + for _ in range(count): + s = socket.socket() + s.bind(('127.0.0.1', 0)) + socks.append(s) + return [s.getsockname()[1] for s in socks] + finally: + for s in socks: + s.close() + + +def nm_rtt_addr(elf: str, nm: str = None) -> int: + """Control-block address from the FLASHED elf's symbol table. --addr is the way + out when nm cannot read the file (another architecture, no toolchain).""" + nm = nm or os.environ.get('RTT_NM', 'arm-none-eabi-nm') + try: + r = subprocess.run([nm, elf], capture_output=True, text=True, timeout=30) + except FileNotFoundError: + raise SystemExit(f'{nm} not on PATH — set RTT_NM=<your-nm>, or pass --addr') + except subprocess.TimeoutExpired: + raise SystemExit(f'{nm} did not finish reading {elf} in 30 s — pass --addr instead') + if r.returncode != 0: + raise SystemExit(f'{nm} could not read {elf}: {r.stderr.strip()[:200]}\n' + f'(wrong architecture? set RTT_NM=<your-nm>, or pass --addr)') + for line in r.stdout.splitlines(): + # "<addr> <type> _SEGGER_RTT": a defined data symbol only — an undefined one + # (" U _SEGGER_RTT") has no address and would int('U', 16) + m = re.match(r'^([0-9a-fA-F]+)\s+[bBdD]\s+_SEGGER_RTT$', line.strip()) + if m: + return int(m.group(1), 16) + raise SystemExit(f'no defined _SEGGER_RTT symbol in {elf} — was it built with LOGGER=rtt?') + + +class _SocketRtt: + """Shared console core: a TCP socket onto an RTT server owned by self._proc. + + Subclasses build their server argv and call _spawn() + _connect() in __init__. + One failure contract: RttError for every way the console can break (stall, + closed, dead server) — callers are written for exactly it. A dead or resetting + server LATCHES `eof` rather than raising from the read side, so read loops and + the harness's `assert not ser.eof` triage see it without an exception racing + them to a generic handler.""" + + server = 'RTT server' # for error messages + + def __init__(self, timeout: float = 0.1): + self.timeout = timeout + self._buf = b'' + self._eof = False + self._sock = None + self._proc = None + self._log = None + self._lock = threading.Lock() # _buf is touched by the CLI pump thread too + + def _spawn(self, cmd: list, stdin=None) -> None: + # server output spools to a temp file: a PIPE nobody drains blocks a + # single-threaded server once 64 KiB of log accumulates (openocd at + # polling_interval 1 against a resetting target fills that in minutes) and + # the console goes silent with no error; the file also feeds _server_tail + self._log = tempfile.NamedTemporaryFile(prefix='rtt-server-', suffix='.log') + try: + self._proc = subprocess.Popen(cmd, stdin=stdin, stdout=self._log, + stderr=subprocess.STDOUT, start_new_session=True) + except FileNotFoundError as e: + self.close() + raise RttError(f'RTT console: {e.filename or cmd[0]} not on PATH') from e + except BaseException: + # any other spawn failure (PermissionError...) must not leak the log fd + self.close() + raise + + def _connect(self, port: int) -> None: + try: + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + try: + self._sock = socket.create_connection(('127.0.0.1', port), timeout=2) + break + except OSError: + if self._proc.poll() is not None: + break + time.sleep(0.2) + if self._sock is None: + tail = self._server_tail() + self.close() + raise RttError(f'RTT console: {self.server} did not serve port {port}{tail}') + if self._proc.poll() is not None: + # the connect succeeded but our server is dead: a foreign process claimed + # the port in the free_ports window — refuse a console wired to a stranger + self.close() + raise RttError(f'RTT console: {self.server} died after connect (port {port} hijacked?)') + self._sock.setblocking(False) + except (KeyboardInterrupt, SystemExit): + # a signal mid-construction must not orphan the server we just spawned + self.close() + raise + + def _server_tail(self) -> str: + log = getattr(self, '_log', None) + if log: + with contextlib.suppress(OSError, ValueError): + with open(log.name, 'rb') as fh: + tail = fh.read()[-400:].decode(errors='replace') + if tail: + return ' — ' + tail + return '' + + def _drain(self) -> None: + # LATCH, never raise: a peer reset or a socket closed under us ends the + # stream exactly like an orderly EOF. Raising here raced the harness's + # `assert not ser.eof` triage into a generic handler that re-flashes the + # board, and leaked ConnectionResetError/ValueError to in_waiting callers. + # the WHOLE body under the lock, not just the append: the CLI's -i pump thread + # and the read loop drain the same socket concurrently, and recv->append being + # non-atomic let chunks land out of order (measured: transposed 64-byte + # segments in 3/6 stress trials) + try: + with self._lock: + while self._sock and select.select([self._sock], [], [], 0)[0]: + try: + chunk = self._sock.recv(65536) + except (BlockingIOError, InterruptedError): + return + if not chunk: + self._eof = True + return + self._buf += chunk + except (OSError, ValueError, TypeError, AttributeError): + self._eof = True + + @property + def eof(self) -> bool: + """True once the server hung up AND everything it sent has been read out.""" + if self._sock is None: + return True + self._drain() + return self._eof and not self._buf + + @property + def in_waiting(self) -> int: + if self._sock is None: + # pyserial raises on a closed port; answering "N bytes waiting" from a + # closed dead console would let a caller bug look like a healthy board + raise RttError('RTT console is closed') + self._drain() + return len(self._buf) + + def read(self, size: int = 1) -> bytes: + if size is None or size <= 0: + # pyserial's read(0) returns b'' and consumes nothing; a negative size + # must not silently hand over (or destroy) buffered bytes + return b'' + if self._sock is None: + raise RttError('RTT console is closed') + self._drain() + deadline = None if self.timeout is None else time.monotonic() + self.timeout + while (len(self._buf) < size and not self._eof + and (deadline is None or time.monotonic() < deadline)): + time.sleep(0.005) + self._drain() + if self._eof and len(self._buf) < size: + # dead server: pace the empty returns like a serial timeout would, so a + # caller's read loop cannot busy-spin at 100% CPU (416k empty reads/s + # measured unpaced). timeout=None deliberately diverges from pyserial's + # block-forever: the eof latch makes "server is gone" knowable, and an + # eternal block on it helps nobody -- paced empties + .eof is the contract. + pace = self.timeout if self.timeout is not None else 0.1 + remaining = (deadline - time.monotonic()) if deadline is not None else pace + time.sleep(max(0.0, min(remaining, pace))) + with self._lock: + out, self._buf = self._buf[:size], self._buf[size:] + return out + + def reset_input_buffer(self) -> None: + # pyserial surface: the host tests flush pre-reset backlog through this + if self._sock is None: + raise RttError('RTT console is closed') + self._drain() + with self._lock: + self._buf = b'' + + def write(self, data: bytes) -> int: + # select+send, not sendall(): the socket is non-blocking for reads, and sendall() + # on a non-blocking socket raises BlockingIOError as soon as the send buffer is + # full, with no count of what already went out -- a caller cannot resume without + # duplicating bytes. Same reason serial_write_all treats a short write as fatal. + sock = self._sock # snapshot: close() from another thread nulls the attribute + if sock is None: + raise RttError('RTT console is closed') + self._drain() + if self._eof: + # TCP accepts exactly one send after peer death — without this the bytes + # would "succeed" into the void and the read timeout gets blamed on the target + raise RttError(f'RTT console write to a dead server ({self.server} gone)') + sent = 0 + deadline = time.monotonic() + RTT_WRITE_TIMEOUT + while sent < len(data): + if time.monotonic() > deadline: + raise RttError(f'RTT console write stalled after {sent}/{len(data)} bytes') + try: + if not select.select([], [sock], [], 0.1)[1]: + continue + sent += sock.send(data[sent:]) + except (BlockingIOError, InterruptedError): + continue + except (OSError, ValueError, TypeError, AttributeError) as e: + # peer death (BrokenPipe/ConnectionReset) or the socket closed under us + # mid-call: keep the class's one failure contract + raise RttError(f'RTT console write failed after {sent}/{len(data)} bytes: {e}') from e + return sent + + def _gentle_stop(self, proc) -> None: + """Subclass hook: ask the server to exit before the group takedown.""" + + def close(self) -> None: + self._eof = True # latch: post-close eof reads True, like a hung-up server + if getattr(self, '_sock', None): + self._sock.close() + self._sock = None + with self._lock: + self._buf = b'' # pyserial contract: nothing is readable after close + proc = getattr(self, '_proc', None) + if proc: + if proc.poll() is None: + self._gentle_stop(proc) + with contextlib.suppress(subprocess.TimeoutExpired): + proc.wait(timeout=5) + if proc and proc.poll() is None: + # own session (start_new_session), so the group takedown gets the server and + # anything it spawned; leaving one alive would hold the probe for the next test + try: + os.killpg(proc.pid, signal.SIGTERM) + proc.wait(timeout=5) + except (ProcessLookupError, PermissionError): + pass + except subprocess.TimeoutExpired: + with contextlib.suppress(ProcessLookupError, PermissionError): + os.killpg(proc.pid, signal.SIGKILL) + # reap, or the server stays a zombie for the caller's lifetime + with contextlib.suppress(subprocess.TimeoutExpired): + proc.wait(timeout=2) + if proc: + for pipe in (proc.stdin, proc.stdout): + if pipe: + with contextlib.suppress(OSError, ValueError): + pipe.close() + # the server spool file: one fd plus a /tmp file per console, and the server + # grows it while alive -- GC is not a release policy on a rig + log = getattr(self, '_log', None) + if log: + with contextlib.suppress(OSError, ValueError): + log.close() + self._log = None + + # a console dropped without close() must not hold the probe for the process's life + def __enter__(self): + return self + + def __exit__(self, *exc): + self.close() + + def __del__(self): + with contextlib.suppress(Exception): + self.close() + + +class JlinkRtt(_SocketRtt): + """Bidirectional console over SEGGER RTT channel 0, for J-Link probes (the only + console on boards whose probe has no VCOM or whose BSP has no UART). + + J-Link Commander (JLinkExe) owns the probe and serves RTT channel 0 on + -RTTTelnetPort -- what JLinkRTTClient talks to, minus its banner. It keeps + hunting for the control block and streams whatever the buffer already holds, + where JLinkRTTLogger searches once when it attaches and gives up. It also + carries input, which the host tests that drive a menu need. + + The probe is held for as long as this is open, so flashing and resetting the + board must happen before it is created or after close(). Select the probe by + serial: rigs run more than one.""" + + server = 'JLinkExe' + + def __init__(self, board: dict, timeout: float = 0.1): + super().__init__(timeout) + flasher = board['flasher'] + args = shlex.split(flasher.get('args', '')) + if '-device' not in args: + # fail with the real cause now: JLinkExe without a device blocks prompting + # and would surface 15 s later as a misleading port error + raise RttError(f'RTT console: no -device in flasher args: {flasher.get("args")!r}') + port = free_ports(1)[0] + # defaults first, the roster's args after so they can override (-if jtag, + # -JLinkScriptFile, an explicit -speed). NOTE: hil_flash orders it the other + # way (roster args first, its own -if/-speed last, so ITS defaults win) -- + # a roster override honored here is ignored by flash/reset; align them if a + # roster ever carries such args. -ExitOnError makes a failed target connect + # EXIT Commander + # (a clean error with the log tail) instead of leaving a banner-only console + cmd = ['JLinkExe', '-USB', str(flasher['uid']), '-if', 'swd', + '-JTAGConf', '-1,-1', '-speed', 'auto', '-NoGui', '1', + '-ExitOnError', '1', '-AutoConnect', '1', + *args, '-RTTTelnetPort', str(port)] + # stdin stays open: Commander exits when it runs out of input; close() writes + # 'exit' there. + self._spawn(cmd, stdin=subprocess.PIPE) + self._connect(port) + + def _gentle_stop(self, proc) -> None: + with contextlib.suppress(OSError, ValueError): + proc.stdin.write(b'exit\n') + proc.stdin.flush() + # close our pipe end in its own suppress: a BrokenPipe on the write above must + # not skip it (the base close also closes it for the server-already-dead path) + with contextlib.suppress(OSError, ValueError): + proc.stdin.close() + + +class OpenocdRtt(_SocketRtt): + """The console surface over an openocd `rtt server` (native probes: + ST-Link/CMSIS-DAP — never point openocd at ea4088's LPC-Link2, measured to + knock that probe off USB; other J-Link-OB probes untested). + + Exact control-block address (never a full-RAM scan), polling_interval 1 + (default 100 ms polling loses most of a busy stream), attach WITHOUT reset — + flash and reset before starting; `rtt start` needs the block to exist. + reset_before_attach opts into an in-session reset for streams that only + decode from byte 0 (SystemView).""" + + server = 'openocd' + + def __init__(self, cfg: str, addr: int, channel: int, serial_no: str = None, + vid_pid: str = None, timeout: float = 0.1, reset_before_attach: bool = False): + super().__init__(timeout) + port = free_ports(1)[0] + # argv, never a shell string: cfg/serial/vid_pid come from roster JSON and the + # command line, and a '$', backtick or quote in any of them would otherwise be + # substituted by the shell or break out of it + cmd = ['openocd', '-c', 'tcl_port disabled', '-c', 'gdb_port disabled', + '-c', 'telnet_port disabled'] + # probe pin: vid_pid keeps discovery from opening foreign usbfs nodes (a + # wedged one hangs the open), serial disambiguates same-model probes — + # both before the -f scripts, like hil_flash does + if vid_pid: + if not re.fullmatch(r'0x[0-9a-fA-F]{1,4} 0x[0-9a-fA-F]{1,4}', vid_pid.strip()): + # openocd only WARNS and exits 0 on a malformed value, so the pin + # silently does not apply and discovery reopens every usbfs node -- + # the convoy hil_flash.valid_vid_pid exists to stop + raise RttError(f'--vid-pid must be "0xVVVV 0xPPPP", got {vid_pid!r}') + cmd += ['-c', f'adapter usb vid_pid {vid_pid.strip()}'] + if serial_no: + cmd += ['-c', f'adapter serial {serial_no}'] + cmd += shlex.split(cfg) + cmd += ['-c', 'init'] + # opt-in: reset the target INSIDE this session, give it 2 s to boot, THEN + # attach and drain. The order is forced: `rtt start` needs the control block + # to already exist in RAM (the firmware creates it at init), and attaching + # ahead of the reset would latch the PREVIOUS run's stale block. Byte 0 still + # reaches the consumer because NO_BLOCK_SKIP retains the ring's HEAD: a boot + # burst bigger than the ring loses its tail until the drain catches up, never + # its first bytes -- which is the part a boot-anchored decoder needs + # (SystemView's Init record, carrying the timestamp frequency, is emitted once + # at boot; a mid-flight attach yields a stream no decoder can lock onto; size + # BUFFER_SIZE_UP to the boot burst if the tail matters too). Costs the tool's + # usual no-reset invariant, and is unsafe on parts where an in-session reset + # leaves the core held (SAMD5x DSU) or perturbs the target (WCH SDI). + if reset_before_attach: + cmd += ['-c', 'reset run', '-c', 'sleep 2000'] + cmd += ['-c', f'rtt setup 0x{addr:x} 0x800 "SEGGER RTT"', + '-c', 'rtt polling_interval 1', '-c', 'rtt start', + '-c', f'rtt server start {port} {channel}'] + self._spawn(cmd) + self._connect(port) + + def _gentle_stop(self, proc) -> None: + # no stdin channel to ask openocd to exit, and it keeps its listener up after + # the client disconnects: go straight to the group takedown instead of blocking + # the base class's 5 s wait on a process that has no reason to leave + with contextlib.suppress(ProcessLookupError, PermissionError): + os.killpg(proc.pid, signal.SIGTERM) + + +def dump_ring(probe: str, device: str, addr: int, out_path: str, channel: int = 0) -> int: + """Post-mortem: read aUp[channel]'s ring over the debug AP (no halt) via JLinkExe. + NO_BLOCK_SKIP means an undrained ring holds the FIRST KB after boot, not the + tail — interpretation rules in the target-debug skill.""" + if re.search(r'[\s"\']', out_path): + raise SystemExit(f'--dump path must not contain whitespace or quotes: {out_path!r} ' + f'(it is spliced into a JLinkExe script line)') + # a stale file from an earlier run must not satisfy the success check below + with contextlib.suppress(OSError): + os.remove(out_path) + # SEGGER_RTT_CB: acID[16], MaxNumUpBuffers, MaxNumDownBuffers, then aUp[] at 0x18, + # each ring 6 words {sName, pBuffer, SizeOfBuffer, WrOff, RdOff, Flags}. Read the + # counts with the descriptor so an out-of-range channel is rejected instead of + # reading whatever RAM follows the array. + jlink = ['JLinkExe', '-USB', probe, '-device', device, '-if', 'swd', + '-speed', '4000', '-NoGui', '1', '-AutoConnect', '1'] + + def _jlink_run(script: str): + # same clean-exit contract as nm_rtt_addr/_spawn: a missing binary or a wedged + # probe must not reach the CLI as a traceback + try: + return subprocess.run(jlink, input=script, capture_output=True, text=True, timeout=60) + except FileNotFoundError: + raise SystemExit('JLinkExe not on PATH — the --dump route needs J-Link Commander') + except subprocess.TimeoutExpired: + raise SystemExit('JLinkExe did not finish in 60 s — probe wedged or target unreachable?') + + script = f'mem32 {addr + 0x10:#x}, 2\nmem32 {addr + 0x18 + channel * 24:#x}, 6\nexit\n' + r = _jlink_run(script) + words = [] + for line in r.stdout.splitlines(): + # UNANCHORED: when the script arrives on stdin, some JLinkExe versions glue + # the 'J-Link>' prompt onto the result line with no newline between + m = re.search(r'([0-9A-Fa-f]{8}) = ((?:[0-9A-Fa-f]{8} ?)+)$', line.strip()) + if m: + words += [int(w, 16) for w in m.group(2).split()] + if len(words) < 8: + print(r.stdout[-500:], file=sys.stderr) + raise SystemExit(f'could not read the aUp[{channel}] descriptor — wrong control block address?') + max_up = words[0] + if not 0 < max_up <= 32: + raise SystemExit(f'control block at {addr:#x} looks uninitialized ' + f'(MaxNumUpBuffers={max_up}) — the target has not written to RTT yet, ' + f'or the address is wrong') + if channel >= max_up: + raise SystemExit(f'--channel {channel}: this firmware has {max_up} up-buffer(s) (0..{max_up - 1})') + _, pbuf, size, wroff, rdoff, _ = words[2:8] + if not pbuf or not size: + raise SystemExit(f'up-buffer {channel} is not initialized (pBuffer={pbuf:#x} size={size}) — ' + f'the target has not written to it yet') + script = f'savebin {out_path}, {pbuf:#x}, {size:#x}\nexit\n' + _jlink_run(script) + # JLinkExe exits 0 even when a command inside its script fails, so the only proof + # savebin worked is the file itself: it must hold the WHOLE ring, since a read that + # dies partway (probe disconnect, unreadable address) still leaves a short file that + # would otherwise be reported as a complete dump. Removing it also keeps the + # invariant above -- no stale file can satisfy a later run's check. + got = os.path.getsize(out_path) if os.path.exists(out_path) else 0 + if got < size: + with contextlib.suppress(OSError): + os.remove(out_path) + if got == 0: + raise SystemExit(f'savebin produced no data at {out_path} — probe or address problem') + raise SystemExit(f'savebin wrote {got}/{size} B to {out_path} (truncated dump removed) ' + f'— probe or address problem') + print(f'ring: {size} B at {pbuf:#x}, WrOff={wroff:#x} RdOff={rdoff:#x} -> {out_path}\n' + f'valid bytes wrap at WrOff; default NO_BLOCK_SKIP holds the FIRST data after ' + f'boot, not the tail', file=sys.stderr) + return 0 + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument('--backend', choices=['jlink', 'openocd'], required=True, + help='transport route — explicit, no default (skill transport matrix)') + ap.add_argument('--probe', help='probe serial (JLinkExe -USB / openocd "adapter serial")') + ap.add_argument('--vid-pid', help='openocd probe pin by USB IDs, e.g. "0x2e8a 0x000c" ' + '(with or instead of --probe)') + ap.add_argument('--device', help='JLINK_DEVICE from board.cmake/family.cmake (jlink backend)') + ap.add_argument('--cfg', help='openocd -f/-c args, e.g. "-f interface/stlink.cfg -f target/stm32h7x.cfg"') + ap.add_argument('--elf', help='the FLASHED elf: exact _SEGGER_RTT address via nm (openocd/--dump)') + ap.add_argument('--addr', help='SEGGER RTT control block address (hex), instead of --elf') + ap.add_argument('--channel', type=int, default=0, help='up-buffer index (0 console, 1 SysView)') + ap.add_argument('--seconds', type=float, default=0, help='capture duration; 0 = until Ctrl-C/EOF') + ap.add_argument('-i', '--interactive', action='store_true', help='forward stdin to the target') + ap.add_argument('--reset-before-attach', action='store_true', + help='openocd: reset the target inside the capture session so the ' + 'server is draining when it boots (needed for streams that must ' + 'include the boot preamble, e.g. SystemView); unsafe on SAMD5x/WCH') + ap.add_argument('--dump', metavar='OUT.bin', + help='post-mortem ring dump (jlink backend; needs --elf or --addr)') + args = ap.parse_args() + + if args.seconds < 0 or args.seconds != args.seconds: # negative or nan + ap.error(f'--seconds must be >= 0 (0 = until Ctrl-C/EOF), got {args.seconds}') + if args.channel < 0: + # a negative index would walk backwards off aUp[] into the control-block + # header and read garbage as a descriptor + ap.error(f'--channel must be >= 0, got {args.channel}') + + def rtt_addr(): + if args.addr: + try: + return int(args.addr, 16) + except ValueError: + ap.error(f'--addr must be hex, got {args.addr!r}') + if args.elf: + return nm_rtt_addr(args.elf) + ap.error('need --elf (flashed elf, address via nm) or --addr') + + if args.backend == 'jlink': + if args.reset_before_attach: + ap.error('--reset-before-attach is openocd-only (the J-Link route attaches ' + 'to a running target; flash and reset before starting it)') + if args.channel and not args.dump: + # -RTTTelnetPort serves the Terminal buffer only; --dump can read any ring + ap.error('the jlink backend streams channel 0 only (use --backend openocd ' + 'for another channel, or --dump to read one)') + if args.vid_pid: + ap.error('--vid-pid is openocd-only; J-Link probes are selected by serial (--probe)') + if not (args.probe and args.device): + ap.error('the jlink backend needs --probe and --device') + elif not (args.probe or args.vid_pid): + ap.error('the openocd backend needs --probe and/or --vid-pid') + + if args.dump: + if args.backend != 'jlink': + ap.error('--dump uses the jlink backend (debug-AP reads via JLinkExe)') + return dump_ring(args.probe, args.device, rtt_addr(), args.dump, args.channel) + + # install BEFORE the console exists: an external `timeout`/kill during the + # up-to-15 s connect window must still reach the cleanup below, or the openocd + # route leaves a server holding the probe and the port (JLinkExe would exit on + # stdin EOF; openocd has no such channel and its own session shields it) + def _terminate(signum, _frame): + raise KeyboardInterrupt + for _sig in (signal.SIGTERM, signal.SIGHUP): + with contextlib.suppress(ValueError, OSError): + signal.signal(_sig, _terminate) + + try: + if args.backend == 'openocd': + if not args.cfg: + ap.error('--backend openocd needs --cfg') + con = OpenocdRtt(args.cfg, rtt_addr(), args.channel, + serial_no=args.probe, vid_pid=args.vid_pid, + reset_before_attach=args.reset_before_attach) + else: + con = JlinkRtt({'flasher': {'uid': args.probe, 'args': f'-device {args.device}'}}, + timeout=0.1) + except RttError as e: + print(e, file=sys.stderr) + return 1 + except KeyboardInterrupt: + return 130 # constructors clean up after themselves on the way out + + saw_output = threading.Event() + forwarded = threading.Event() + if args.interactive: + def pump_stdin(): + # Hold input until the capture side has seen TARGET output (or 5 s for a + # quiet firmware): the J-Link telnet route silently DROPS client bytes + # until Commander locates the control block, so input forwarded at attach + # vanishes (measured on the rig: instant 'ping' lost, delayed 'ping' + # echoed). The gate must ignore the server's own banner — it arrives at + # connect, BEFORE the block is found. Raw os.read, not sys.stdin.buffer: + # bytes with no newline wait, and no BufferedReader lock — a daemon + # thread blocked holding that lock at interpreter shutdown aborts + # CPython (_enter_buffered_busy). + saw_output.wait(5) + try: + while True: + data = os.read(0, 4096) + if not data: + return + con.write(data) + forwarded.set() + except (RttError, OSError, ValueError): + return # console closed/stalled/dead; capture side reports the state + threading.Thread(target=pump_stdin, daemon=True).start() + + deadline = time.monotonic() + args.seconds if args.seconds else None + rc = 0 + seen = b'' # pre-release accumulator for the banner check only + try: + while deadline is None or time.monotonic() < deadline: + try: + chunk = con.read(con.in_waiting or 1) + except RttError as e: + print(f'rtt: {e}', file=sys.stderr) + rc = 1 + break + if chunk: + if args.interactive and not saw_output.is_set(): + # target data = anything past the J-Link banner's final line + # ('Process: <name>'); the openocd server has no banner + seen = (seen + chunk)[-65536:] + if args.backend != 'jlink': + saw_output.set() + else: + i = seen.find(b'Process: ') + j = seen.find(b'\n', i) if i >= 0 else -1 + if j >= 0 and len(seen) > j + 1: + saw_output.set() + sys.stdout.buffer.write(chunk) + sys.stdout.buffer.flush() + elif con.eof: + print('rtt: server closed the connection', file=sys.stderr) + rc = 1 + break + except KeyboardInterrupt: + pass + except BrokenPipeError: + # downstream consumer (head/grep -m) closed the pipe: a normal way to end a + # capture, not an error. Point stdout at devnull so interpreter shutdown does + # not raise on the final implicit flush. + os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno()) + finally: + if args.interactive and not forwarded.is_set(): + # only claim what is true: the gate releases after 5 s and forwards anyway, + # so "never forwarded" must come from the forwarded flag, not the gate + print('rtt: -i stdin was never forwarded to the target (no input arrived, ' + 'or the console closed first)', file=sys.stderr) + if args.interactive and not saw_output.is_set(): + print('rtt: no target output within the window', file=sys.stderr) + # a late TERM landing during the up-to-12 s teardown must not skip the kill + # escalation and orphan the server -- cleanup is committed at this point + for _sig in (signal.SIGTERM, signal.SIGHUP): + with contextlib.suppress(ValueError, OSError): + signal.signal(_sig, signal.SIG_IGN) + con.close() + return rc + + +if __name__ == '__main__': + sys.exit(main()) |