diff options
Diffstat (limited to 'tools/build.py')
| -rwxr-xr-x | tools/build.py | 175 |
1 files changed, 156 insertions, 19 deletions
diff --git a/tools/build.py b/tools/build.py index 51d3d0f70..eeefca22d 100755 --- a/tools/build.py +++ b/tools/build.py @@ -2,6 +2,7 @@ import argparse import random import os +import re import sys import time import subprocess @@ -99,6 +100,53 @@ def get_examples(family): return all_examples +def resolve_example_target_groups(build_targets, examples, board, extra_defines=()): + """Map generic targets onto per-example targets for a filtered build (-e), as ONE + GROUP PER REQUESTED TARGET: 'all' -> the example executables, anything else (e.g. + tinyusb_metrics) passes through as its own single-entry group. + + Grouped rather than flattened because each group becomes one `cmake --build + --target a b c` invocation: the examples of a group build in parallel (flattening + them into one target per invocation serialises the whole leg - measured +39% at + -j4 and +220% at -j32 on stm32f407disco), while separate groups stay ordered, so a + target that must run after the examples still does. + + extra_defines are this build's -D tokens: MAX3421_HOST=1 there decides + only.txt for the max3421 examples (see build_utils.skip_example). + Returns None when no requested example is buildable on this board.""" + buildable = [e for e in examples + if not build_utils.skip_example(e, board, extra_defines)] + if not buildable: + return None + names = list(dict.fromkeys(e.split('/', 1)[1] for e in buildable)) + return [list(names) if t == 'all' else [t] for t in build_targets] + + +_TARGET_HELP_RE = re.compile(r'^([A-Za-z0-9_.+-]+):') +# role/name, the only shape resolve_example_target_groups and the CMake target names accept +EXAMPLE_RE = re.compile(r'[A-Za-z0-9_]+/[A-Za-z0-9_]+') + + +def parse_target_help(text): + """Bare target names out of `cmake --build <dir> --target help`; the Ninja + generator prints one '<name>: phony' line per target. Names containing '/' are + per-directory utility targets (device/edit_cache) or absolute CMakeFiles paths, + never an example target.""" + return {m.group(1) for m in map(_TARGET_HELP_RE.match, text.splitlines()) if m} + + +def cmake_registered_targets(build_dir): + """The targets CMake actually created in build_dir, or None when that cannot be + read. Ground truth: skip.txt/only.txt only mirrors family_filter, so an example + the role CMakeLists never lists (or a stale -e name) still looks buildable to it + and `cmake --build --target <it>` hard-fails. None keeps the mirror's answer.""" + r = subprocess.run(['cmake', '--build', build_dir, '--target', 'help'], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + if r.returncode != 0: + return None + return parse_target_help(r.stdout.decode('utf-8', 'replace')) or None + + def print_build_result(board, build_target, status, duration): if isinstance(duration, (int, float)): duration = "{:.2f}s".format(duration) @@ -107,7 +155,7 @@ def print_build_result(board, build_target, status, duration): # ----------------------------- # CMake # ----------------------------- -def cmake_board(board, build_args, build_name, build_cflags, build_targets): +def cmake_board(board, build_args, build_name, build_cflags, build_targets, examples=None, defines=()): ret = [0, 0, 0] start_time = time.monotonic() @@ -120,8 +168,13 @@ def cmake_board(board, build_args, build_name, build_cflags, build_targets): if family == 'espressif': # for espressif, we have to build example individually all_examples = get_examples(family) + if examples is not None: + all_examples = [e for e in all_examples if e in examples] + if not all_examples: + print_build_result(board, 'examples (PR filter)', 2, '-') + return [0, 0, 1] for example in all_examples: - if build_utils.skip_example(example, board): + if build_utils.skip_example(example, board, defines): ret[2] += 1 else: rcmd = run_cmd([ @@ -130,13 +183,40 @@ def cmake_board(board, build_args, build_name, build_cflags, build_targets): ]) ret[0 if rcmd.returncode == 0 else 1] += 1 else: + # the skip.txt/only.txt prefilter reads no configure output: answer it first, + # so a selection this board builds nothing of costs no cmake run at all + if examples is not None: + examples = [e for e in examples + if not build_utils.skip_example(e, board, defines)] + if not examples: + print_build_result(board, 'examples (PR filter)', 2, '-') + return [0, 0, 1] rcmd = run_cmd(['cmake', 'examples', '-B', build_dir, '-GNinja', f'-DBOARD={board}', '-DCMAKE_BUILD_TYPE=MinSizeRel', '-DLINKERMAP_OPTION=-q -f tinyusb/src', *build_args, *build_flags]) if rcmd.returncode == 0: + target_groups = [[t] for t in build_targets] + if examples is not None: + registered = cmake_registered_targets(build_dir) + if registered is not None: + kept = [e for e in examples if e.split('/', 1)[1] in registered] + for e in examples: + if e not in kept: + print_build_result(board, f'{e} (no such target)', 2, '-') + examples = kept + if not examples: + print_build_result(board, 'examples (no such target)', 2, '-') + return [0, 0, 1] + target_groups = resolve_example_target_groups(build_targets, examples, board, defines) + if registered is None: + # ground truth unavailable, so nothing checked these names against + # what CMake created. ninja validates a whole invocation up front: + # one unknown name in the batch builds NOTHING, where a target each + # builds everything up to it. Give up the parallelism, not the work. + target_groups = [[t] for g in target_groups for t in g] cmd = ["cmake", "--build", build_dir, '--parallel', str(parallel_jobs)] - for target in build_targets: - rcmd = run_cmd(cmd + ['--target', target]) + for group in target_groups: + rcmd = run_cmd(cmd + ['--target'] + group) if rcmd.returncode != 0: break ret[0 if rcmd.returncode == 0 else 1] += 1 @@ -148,9 +228,10 @@ def cmake_board(board, build_args, build_name, build_cflags, build_targets): # ----------------------------- # Make # ----------------------------- -def make_one_example(example, board, make_option, build_targets): - # Check if board is skipped - if build_utils.skip_example(example, board): +def make_one_example(example, board, make_option, build_targets, defines=()): + # Check if board is skipped. Make semantics: family.mk decides, not the + # family.cmake MCU list (see build_utils.skip_example). + if build_utils.skip_example(example, board, defines, build_system='make'): print_build_result(board, example, 2, '-') r = 2 else: @@ -171,10 +252,15 @@ def make_one_example(example, board, make_option, build_targets): return ret -def make_board(board, build_args, build_targets): +def make_board(board, build_args, build_targets, examples=None, defines=()): print(build_separator) family = find_family(board); all_examples = get_examples(family) + if examples is not None: + all_examples = [e for e in all_examples if e in examples] + if not all_examples: + print_build_result(board, 'examples (PR filter)', 2, '-') + return [0, 0, 1] start_time = time.monotonic() ret = [0, 0, 0] if family == 'espressif' or family == 'rp2040': @@ -182,7 +268,7 @@ def make_board(board, build_args, build_targets): final_status = 2 else: with Pool(processes=os.cpu_count()) as pool: - pool_args = list((map(lambda e, b=board, o=f"{build_args}", t=build_targets: [e, b, o, t], all_examples))) + pool_args = list((map(lambda e, b=board, o=f"{build_args}", t=build_targets, d=defines: [e, b, o, t, d], all_examples))) r = pool.starmap(make_one_example, pool_args) # sum all element of same index (column sum) ret = list(map(sum, list(zip(*r)))) @@ -194,36 +280,58 @@ def make_board(board, build_args, build_targets): # ----------------------------- # Build Family # ----------------------------- -def build_boards_list(boards, build_defines, build_system, build_name, build_cflags, build_targets): +def build_boards_list(boards, build_defines, build_system, build_name, build_cflags, build_targets, examples=None): ret = [0, 0, 0] + # the -D tokens are part of the skip.txt/only.txt answer (MAX3421_HOST=1), so + # the -e filter has to see them too; sorted+tuple keeps skip_example cacheable + defines = tuple(sorted(build_defines)) for b in boards: r = [0, 0, 0] if build_system == 'cmake': build_args = [f'-D{d}' for d in build_defines] - r = cmake_board(b, build_args, build_name, build_cflags, build_targets) + r = cmake_board(b, build_args, build_name, build_cflags, build_targets, examples, defines) elif build_system == 'make': build_args = ' '.join(f'{d}' for d in build_defines) - r = make_board(b, build_args, build_targets) + r = make_board(b, build_args, build_targets, examples, defines) ret[0] += r[0] ret[1] += r[1] ret[2] += r[2] return ret -def get_family_boards(family, one_random, one_first): +def get_family_boards(family, one_random, one_first, examples=None, build_system='cmake', + extra_defines=(), ci=None): """Get list of boards for a family. Args: family: Family name one_random: If True, return only one random board one_first: If True, return only the first board (alphabetical) + examples: PR example filter (-e). The one-board pick then prefers a board that + can build at least one of them: the family is in the matrix BECAUSE some + board of it builds these examples (ci_select._prune_buildable asks about + every board, since CircleCI builds every board), but GHA builds one. Without + this, lpc54 selected for host/msc_file_explorer picks lpcxpresso54114 - + which every one of those examples skips - and the leg runs to green having + compiled nothing and uploaded no metrics. + build_system: which skip answer to ask for; the two differ (build_utils) + extra_defines: this build's -D tokens, so a board whose only.txt match comes + from -DMAX3421_HOST=1 is not judged unbuildable here and buildable in + cmake_board + ci: force the ci_skip_boards / ci_preferred_boards lists on or off. Default + None reads the environment, which is right for a build but NOT for a caller + asking what CI would do: ci_select must answer the same on a laptop as on a + runner, or /pre-pr and the code-size skill report a family list CI will not + reproduce. Returns: List of board names """ + if ci is None: + ci = bool(os.getenv('GITHUB_ACTIONS') or os.getenv('CIRCLECI')) skip_list = [] preferred_list = [] - if os.getenv('GITHUB_ACTIONS') or os.getenv('CIRCLECI'): + if ci: skip_list = ci_skip_boards.get(family, []) preferred_list = ci_preferred_boards.get(family, []) @@ -238,12 +346,26 @@ def get_family_boards(family, one_random, one_first): # If only-one flags are set, honor select list first, then pick first or random if one_first or one_random: - if preferred_list: + def buildable(board): + # no filter, or nothing in the filter is buildable anywhere: keep today's + # answer rather than inventing a different board + return examples is None or any( + not build_utils.skip_example(e, board, extra_defines, build_system) + for e in examples) + + # the WHOLE preferred list, in order - stopping at entry one would abandon a + # curated list for the raw alphabetical order the moment its first board cannot + # build the filter, which also moves the board the metrics baseline is keyed on + for b in preferred_list: + if buildable(b): + return [b] + if preferred_list and examples is None: return [preferred_list[0]] + candidates = [b for b in all_boards if buildable(b)] or all_boards if one_first: - return [all_boards[0]] + return [candidates[0]] if one_random: - return [random.choice(all_boards)] + return [random.choice(candidates)] return all_boards @@ -272,6 +394,8 @@ def main(): parser.add_argument('-j', '--jobs', type=int, default=os.cpu_count(), help='Number of jobs to run in parallel') parser.add_argument('-T', '--target', action='append', default=[], help='Build target to use, may be specified multiple times (default: all)') + parser.add_argument('-e', '--example', action='append', default=[], + help='Only build these examples (role/name, repeatable). Default: all examples') parser.add_argument('-v', '--verbose', action='store_true', help='Verbose output') args = parser.parse_args() @@ -285,9 +409,20 @@ def main(): one_random = args.one_random one_first = args.one_first build_targets = args.target if args.target else ['all'] + examples = args.example or None verbose = args.verbose parallel_jobs = args.jobs + for e in args.example: + if not EXAMPLE_RE.fullmatch(e): + parser.error(f"-e/--example takes 'role/name' (e.g. device/cdc_msc), got '{e}'") + # a name no example dir answers to would silently build nothing on every board + # and still exit 0 (every row is a Skipped, and main() returns the FAILED count). + # The -e lists are generated - from ci_select's example map and from HIL roster + # test names - so a stale one must be loud, not green + if not os.path.isdir(os.path.join('examples', e)): + parser.error(f"-e/--example '{e}': no such example directory examples/{e}") + build_defines.append(f'TOOLCHAIN={toolchain}') if len(families) == 0 and len(boards) == 0: @@ -317,10 +452,12 @@ def main(): # get boards from families and append to boards list all_boards = list(boards) for f in all_families: - all_boards.extend(get_family_boards(f, one_random, one_first)) + all_boards.extend(get_family_boards(f, one_random, one_first, examples, + build_system, tuple(build_defines))) # build all boards - result = build_boards_list(all_boards, build_defines, build_system, build_name, build_cflags, build_targets) + result = build_boards_list(all_boards, build_defines, build_system, build_name, build_cflags, build_targets, + examples) total_time = time.monotonic() - total_time print(build_separator) |