1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
|
version: 2.1
setup: true
orbs:
continuation: circleci/continuation@1
jobs:
set-matrix:
executor: continuation/default
docker:
- image: cimg/base:current
resource_class: large
steps:
- checkout
- run:
name: Set matrix
command: |
# PR-scoped selection (best-effort: any failure falls back to the full
# matrix). CircleCI has no base-branch var; tinyusb PRs target master.
# The selection lands in a FILE and never travels as an argv: a mass-sweep
# diff selects hundreds of KB, and E2BIG would fail the step before the
# `||` fallback could fire - leaving a full build labelled scoped, because
# EXAMPLE_MAP/BUILD_FILTERED below have no such limit and stay scoped.
SELECT_FILE=ci_select_out.json
rm -f "$SELECT_FILE"
if [ -n "${CIRCLE_PULL_REQUEST:-}" ]; then
git fetch --no-tags origin master || true
# both suites gate the selector: test_ci_select.py owns the rules,
# test_ci_metrics.py owns the config2 sentinel contract this job rewrites
if python3 test/hil/test/test_ci_select.py >/dev/null 2>&1 &&
python3 test/hil/test/test_ci_metrics.py >/dev/null 2>&1; then
python3 tools/ci_select.py --base origin/master > "$SELECT_FILE" || rm -f "$SELECT_FILE"
else
echo "ci_select unit suite failed - using the full matrix"
fi
fi
[ -s "$SELECT_FILE" ] || rm -f "$SELECT_FILE"
# computed once, up front: it is both the fallback and what the scoping is
# dropped back to further down, and a second invocation there would be an
# unguarded command under `set -e` inside the very branch that exists to
# keep the pipeline green
FULL_MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py 2>/dev/null) || FULL_MATRIX_JSON=''
MATRIX_JSON=''
if [ -f "$SELECT_FILE" ]; then
# ci_set_matrix also falls open with rc 0, saying UNSCOPED on stderr. The
# extras below must follow it, exactly as build.yml does: a full matrix
# paired with a still-scoped -e list builds a fraction of each family and
# tells code-metrics it was an unscoped run.
MATRIX_JSON=$(python .github/scripts/ci_set_matrix.py --select-file "$SELECT_FILE" 2>ci_set_matrix.err) || MATRIX_JSON=''
cat ci_set_matrix.err
if [ -z "$MATRIX_JSON" ] || grep -q 'ci_set_matrix: UNSCOPED' ci_set_matrix.err; then
SELECT_FILE=''
fi
fi
[ -n "$MATRIX_JSON" ] || MATRIX_JSON="$FULL_MATRIX_JSON"
echo "MATRIX_JSON=$MATRIX_JSON"
EXAMPLE_MAP='{}'
BUILD_FILTERED='false'
if [ -f "$SELECT_FILE" ]; then
EXAMPLE_MAP=$(jq -c '.build.family_examples // {}' < "$SELECT_FILE") || EXAMPLE_MAP='{}'
BUILD_FILTERED=$(jq -r 'if (.build? | type) == "object" and .build.full == false then "true" else "false" end' < "$SELECT_FILE") || BUILD_FILTERED='false'
fi
# /pipeline/continue caps parameter values at 512 chars - a scoped map is
# KBs, so both values ride inside the generated config itself (config max
# is 3MB), swapped into the parameter defaults by sentinel-line match.
# Fail-open: a sentinel that drifted (renamed comment, reformatted line)
# must not red EVERY CircleCI pipeline. The rewrite is all-or-nothing
# (config2.yml is only written once both substitutions succeeded).
#
# Done BEFORE the family entries are generated, and a failure drops the
# scoping entirely: the checked-in defaults are {} / false = unfiltered, so
# a scoped FAMILY list with unfiltered defaults would build a subset of
# families while telling code-metrics it had built them all.
if ! EXAMPLE_MAP="$EXAMPLE_MAP" BUILD_FILTERED="$BUILD_FILTERED" python3 - \<<'PYEOF'
import os
p = '.circleci/config2.yml'
t = open(p).read()
def yq(s): # YAML single-quoted scalar
return "'" + s.replace("'", "''") + "'"
for env, tag in (('EXAMPLE_MAP', 'example-map-default'),
('BUILD_FILTERED', 'build-filtered-default')):
old = [l for l in t.splitlines() if l.strip().endswith(f'# {tag}: rewritten in-place by config.yml set-matrix')]
assert len(old) == 1, f'{tag}: sentinel not found exactly once'
line = old[0]
new = line.split('default:')[0] + 'default: ' + yq(os.environ[env]) + f' # {tag}'
t = t.replace(line, new, 1)
open(p, 'w').write(t)
PYEOF
then
echo "warning: sentinel rewrite failed - dropping the scoping, full build"
MATRIX_JSON="$FULL_MATRIX_JSON"
EXAMPLE_MAP='{}'
BUILD_FILTERED='false'
fi
BUILDSYSTEM_LIST=(
"cmake"
"make"
)
TOOLCHAIN_LIST=(
"aarch64-gcc"
"arm-clang"
"arm-gcc"
"esp-idf"
"ft9xx-gcc"
"msp430-gcc"
"riscv-gcc"
)
# only build IAR if not forked PR, since IAR token is not shared
if [ -z $CIRCLE_PR_USERNAME ]; then
TOOLCHAIN_LIST+=("arm-iar")
fi
gen_build_entry() {
local build_system="$1"
local toolchain="$2"
local family="$3"
local build_args=""
if [[ "$toolchain" == "arm-iar" || "$build_system" == "make" ]]; then
build_args="--one-random"
fi
if [[ "$toolchain" == "esp-idf" ]]; then
echo " - build-vm:" >> .circleci/config2.yml
else
echo " - build:" >> .circleci/config2.yml
fi
echo " matrix:" >> .circleci/config2.yml
echo " alias: build-${build_system}-${toolchain}" >> .circleci/config2.yml
echo " parameters:" >> .circleci/config2.yml
echo " build-system: ['$build_system']" >> .circleci/config2.yml
echo " toolchain: ['$toolchain']" >> .circleci/config2.yml
echo " family: $family" >> .circleci/config2.yml
echo " resource_class: ['large']" >> .circleci/config2.yml
echo " build-args: ['$build_args']" >> .circleci/config2.yml
}
# Collect all build aliases for code-metrics requires (cmake only, exclude esp-idf)
BUILD_ALIASES=()
for build_system in "${BUILDSYSTEM_LIST[@]}"; do
for toolchain in "${TOOLCHAIN_LIST[@]}"; do
# make does not support these toolchains
if [ "$build_system" == "make" ] && { [ "$toolchain" == "arm-clang" ] || [ "$toolchain" == "arm-iar" ] || [ "$toolchain" == "esp-idf" ]; }; then
continue
fi
FAMILY=$(echo $MATRIX_JSON | jq -r ".\"$toolchain\"")
echo "FAMILY_${toolchain}=$FAMILY"
if [ "$(echo "$FAMILY" | jq 'length')" = "0" ]; then
# an empty matrix parameter is a hard CircleCI config error, not a skip
echo "skip build-${build_system}-${toolchain}: no families selected"
continue
fi
gen_build_entry "$build_system" "$toolchain" "$FAMILY"
ANY_BUILD=1
# Only add cmake builds: excluding esp-idf or build_args="--one-random" to metrics requirements
if [ "$build_system" == "cmake" ] && [ "$toolchain" != "esp-idf" ] && [ "$toolchain" != "arm-iar" ]; then
BUILD_ALIASES+=("build-${build_system}-${toolchain}")
fi
done
done
if [ ${#BUILD_ALIASES[@]} -gt 0 ]; then
echo " - code-metrics:" >> .circleci/config2.yml
echo " requires:" >> .circleci/config2.yml
for alias in "${BUILD_ALIASES[@]}"; do
echo " - $alias" >> .circleci/config2.yml
done
fi
if [ "${ANY_BUILD:-0}" != "1" ]; then
# a workflow with zero jobs is invalid config
echo " - no-op" >> .circleci/config2.yml
fi
- continuation/continue:
configuration_path: .circleci/config2.yml
workflows:
set-matrix:
jobs:
- set-matrix
|
