diff options
| author | hathach <[email protected]> | 2026-06-11 17:20:15 +0700 |
|---|---|---|
| committer | hathach <[email protected]> | 2026-06-11 17:20:15 +0700 |
| commit | 5145b67f7946763ca04954f5494bfa88eec2acad (patch) | |
| tree | 9130540204c2404d532613adab01e88f4e918daa /test | |
| parent | f52d9c7b27dc0234fb4738762a8a6da01817e2bf (diff) | |
| parent | 0244a4f12e019406a4b73c75cd96f8efa096bbdc (diff) | |
Merge remote-tracking branch 'origin/master' into stm32c5
# Conflicts:
# README.rst
Diffstat (limited to 'test')
| -rw-r--r-- | test/hil/hfp.json | 4 | ||||
| -rw-r--r-- | test/hil/hil_ci.sh | 68 | ||||
| -rw-r--r-- | test/hil/hil_ci_set_matrix.py | 26 | ||||
| -rwxr-xr-x | test/hil/hil_test.py | 297 | ||||
| -rw-r--r-- | test/hil/requirements.txt | 11 | ||||
| -rw-r--r-- | test/hil/tinyusb.json | 299 |
6 files changed, 551 insertions, 154 deletions
diff --git a/test/hil/hfp.json b/test/hil/hfp.json index 8ba7a8f44..bb146d2fc 100644 --- a/test/hil/hfp.json +++ b/test/hil/hfp.json @@ -15,6 +15,10 @@ { "name": "stm32f746disco", "uid": "210041000C51343237303334", + "variant": [ + { "name": "stm32f746disco", "flags": "" }, + { "name": "stm32f746disco-DMA", "flags": "-DCFG_TUD_DWC2_DMA_ENABLE=1 -DCFG_TUH_DWC2_DMA_ENABLE=1" } + ], "tests": { "device": true, "host": false, "dual": false }, diff --git a/test/hil/hil_ci.sh b/test/hil/hil_ci.sh index 96872e2e1..3ec907979 100644 --- a/test/hil/hil_ci.sh +++ b/test/hil/hil_ci.sh @@ -54,27 +54,60 @@ scp -q "$ROOT_DIR/test/hil/hil_test.py" \ "$CONFIG" \ "$REMOTE:$REMOTE_DIR/test/hil/" -# Copy only firmware binaries (elf/bin/hex), preserving directory structure +# Copy only firmware binaries (elf/bin/hex) plus esptool metadata +# (config.env + flash_args needed by the esptool flasher), preserving structure copy_board_binaries() { local src="$1" rsync -a --prune-empty-dirs \ - --include='*/' --include='*.elf' --include='*.bin' --include='*.hex' --exclude='*' \ + --include='*/' --include='*.elf' --include='*.bin' --include='*.hex' \ + --include='config.env' --include='flash_args' \ + --exclude='*' \ "$src" "$REMOTE:$REMOTE_DIR/examples/" } if [ -n "$BOARD" ]; then - BUILD_DIR="$ROOT_DIR/examples/cmake-build-$BOARD" - if [ ! -d "$BUILD_DIR" ]; then - echo "Error: build directory not found: $BUILD_DIR" - echo "Build first with: cd examples && cmake -DBOARD=$BOARD -G Ninja -B cmake-build-$BOARD . && cmake --build cmake-build-$BOARD" + # Copy the board's build dir plus its variant dirs. Variant names come from + # $CONFIG (they are not required to be prefixed with the board name); the + # cmake-build-<BOARD>-* glob is kept as a fallback for ad-hoc local builds. + # Collect only dirs that actually exist, deduplicated. + declare -A SEEN_DIRS=() + BUILD_DIRS=() + add_build_dir() { + [[ -d "$1" && -z "${SEEN_DIRS[$1]:-}" ]] || return 0 + SEEN_DIRS[$1]=1 + BUILD_DIRS+=("$1") + } + shopt -s nullglob + for d in "$ROOT_DIR"/examples/cmake-build-"$BOARD" "$ROOT_DIR"/examples/cmake-build-"$BOARD"-*; do + add_build_dir "$d" + done + shopt -u nullglob + while IFS= read -r v; do + add_build_dir "$ROOT_DIR/examples/cmake-build-$v" + done < <(python3 -c ' +import json, sys +cfg = json.load(open(sys.argv[1])) +for b in cfg.get("boards", []): + if b["name"] == sys.argv[2]: + for v in b.get("variant") or []: + print(v["name"]) +' "$CONFIG" "$BOARD") + if [ ${#BUILD_DIRS[@]} -eq 0 ]; then + echo "Error: no build directory found for $BOARD under $ROOT_DIR/examples/" + echo "Build first with: cd examples && cmake --preset $BOARD && cmake --build --preset $BOARD" exit 1 fi - echo "==> Copying binaries for $BOARD" - copy_board_binaries "$BUILD_DIR" + echo "==> Copying binaries for $BOARD (${#BUILD_DIRS[@]} build dir(s))" + for d in "${BUILD_DIRS[@]}"; do + copy_board_binaries "$d" + done else echo "==> Copying all built binaries" + # Use `%/` parameter expansion to strip the trailing slash from the glob — + # rsync needs the bare dir name so the per-board cmake-build-<BOARD>/ subdir + # is preserved on the remote (hil_test.py looks up binaries by that path). for dir in "$ROOT_DIR"/examples/cmake-build-*/; do - [ -d "$dir" ] && copy_board_binaries "$dir" + [ -d "$dir" ] && copy_board_binaries "${dir%/}" done fi @@ -82,8 +115,21 @@ fi # parameters; quoting and metacharacters in args are preserved. CONFIG_BASENAME="$(basename "$CONFIG")" echo "==> Running HIL test on $REMOTE" -ssh "$REMOTE" bash -s -- "$REMOTE_DIR" "${ARGS[@]}" "test/hil/$CONFIG_BASENAME" <<'REMOTE' +rc=0 +ssh "$REMOTE" bash -s -- "$REMOTE_DIR" "${ARGS[@]}" "test/hil/$CONFIG_BASENAME" <<'REMOTE' || rc=$? cd -- "$1" shift -exec python3 -u test/hil/hil_test.py -B examples "$@" +# Flasher CLIs live in the user bin dirs on ci.lan (esptool/idf in ~/.local/bin, +# STM32CubeProgrammer's STM32_Programmer_CLI in ~/bin); the non-interactive shell +# subprocess used for flashing doesn't source profile/rc, so add them explicitly. +export PATH="$HOME/.local/bin:$HOME/bin:$PATH" +python3 -u test/hil/hil_test.py -B examples "$@" REMOTE + +# Copy the generated report back to the local checkout (best-effort; the run's +# exit code is preserved regardless of whether a report was produced). +scp -q "$REMOTE:$REMOTE_DIR/hil_report.md" "$ROOT_DIR/hil_report.md" \ + && echo "==> Report copied to $ROOT_DIR/hil_report.md" \ + || echo "==> warning: no hil_report.md copied back" >&2 + +exit $rc diff --git a/test/hil/hil_ci_set_matrix.py b/test/hil/hil_ci_set_matrix.py index 2cce35ae2..baa24afb1 100644 --- a/test/hil/hil_ci_set_matrix.py +++ b/test/hil/hil_ci_set_matrix.py @@ -44,19 +44,19 @@ def main(): toolchain = 'arm-gcc' build_board = f'-b {name}' - if 'build' in board: - if 'args' in board['build']: - build_board += ' ' + ' '.join(f'-D{a}' for a in board['build']['args']) - if 'flags_on' in board['build']: - for f in board['build']['flags_on']: - if f == '': - append_build_arg(toolchain, build_board) - else: - append_build_arg(toolchain, f'{build_board} -f1 {f.replace(" ", " -f1 ")}') - else: - append_build_arg(toolchain, build_board) - else: - append_build_arg(toolchain, build_board) + if 'build' in board and 'args' in board['build']: + build_board += ' ' + ' '.join(f'-D{a}' for a in board['build']['args']) + + # Each variant builds into cmake-build-<variant.name> with its raw CFLAGS. + # No 'variant' -> a single build named after the board. + variants = board.get('variant') or [{'name': name, 'flags': ''}] + for v in variants: + arg = build_board + if v['name'] != name: + arg += f' --build-name {v["name"]}' + for tok in v.get('flags', '').split(): + arg += f' --cflag={tok}' + append_build_arg(toolchain, arg) print(json.dumps(matrix)) diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py index ed9ebbf1a..da13fcbaf 100755 --- a/test/hil/hil_test.py +++ b/test/hil/hil_test.py @@ -22,6 +22,14 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. +# Host setup: +# - System packages: sudo apt install mtools libmtp9 alsa-utils iperf +# mtools - read_disk_file (device/cdc_msc, device/msc_dual_lun) +# libmtp9 - pymtp ctypes load (device/mtp); Debian 13 uses libmtp9t64 +# alsa-utils - arecord (device/audio_test_freertos) +# iperf - throughput tests (device/net_lwip_*) +# - Python packages: pip install -r requirements.txt +# # udev rules : # ACTION=="add", SUBSYSTEM=="tty", SUBSYSTEMS=="usb", MODE="0666", PROGRAM="/bin/sh -c 'echo $$ID_SERIAL_SHORT | rev | cut -c -8 | rev'", SYMLINK+="ttyUSB_%c.%s{bInterfaceNumber}" # ACTION=="add", SUBSYSTEM=="block", SUBSYSTEMS=="usb", ENV{ID_FS_USAGE}=="filesystem", MODE="0666", PROGRAM="/bin/sh -c 'echo $$ID_SERIAL_SHORT | rev | cut -c -8 | rev'", RUN{program}+="/usr/bin/systemd-mount --no-block --automount=yes --collect $devnode /media/blkUSB_%c.%s{bInterfaceNumber}" @@ -34,17 +42,11 @@ import re import select import sys import time -import warnings import signal from contextlib import redirect_stdout from pathlib import Path from typing import Any, TypedDict, NotRequired, cast -# Suppress pkg_resources deprecation warning from fs module -warnings.filterwarnings("ignore", message="pkg_resources is deprecated") -# Suppress pyfatfs unclean unmount warning -warnings.filterwarnings("ignore", message="Filesystem was not cleanly unmounted") - import serial import subprocess import json @@ -52,7 +54,6 @@ import glob import shutil from multiprocessing import Pool, Lock from multiprocessing import TimeoutError as MpTimeoutError -import fs import hashlib import ctypes from pymtp import MTP @@ -64,6 +65,10 @@ STATUS_OK = "\033[32mOK\033[0m" STATUS_FAILED = "\033[31mFailed\033[0m" STATUS_SKIPPED = "\033[33mSkipped\033[0m" +# Plain (non-ANSI) cell symbols for the markdown matrix report (hil_report.md). +# A missing binary is reported as skipped too. +REPORT_CELL = {'pass': '✅', 'fail': '❌', 'skip': '⚪'} + verbose = False test_only = [] board_test = {} @@ -117,16 +122,21 @@ class TestsCfg(TypedDict, total=False): class BuildCfg(TypedDict, total=False): - flags_on: list[str] args: list[str] +class VariantCfg(TypedDict, total=False): + name: str # build dir (cmake-build-<name>) and HIL report row + flags: str # raw CFLAGS, e.g. "-DCFG_TUD_DWC2_DMA_ENABLE=1" + + class Board(TypedDict): name: str uid: str tests: TestsCfg flasher: FlasherCfg build: NotRequired[BuildCfg] + variant: NotRequired[list[VariantCfg]] class HilConfig(TypedDict): @@ -218,7 +228,9 @@ def open_serial_dev(port: str): while timeout > 0: if os.path.exists(port): try: - ser = serial.Serial(port, baudrate=115200, timeout=5) + # write_timeout: a wedged device otherwise blocks ser.write() forever, + # hanging the worker until the pool/job timeout kills the whole run + ser = serial.Serial(port, baudrate=115200, timeout=5, write_timeout=5) break except serial.SerialException: print(f'serial {port} not reaady {timeout} sec') @@ -232,23 +244,24 @@ def open_serial_dev(port: str): def read_disk_file(uid: str, lun: int, fname: str) -> bytes: - # open_fs("fat://{dev}) require 'pip install pyfatfs' + # Reads a file from a FAT volume on a block device without mounting it. + # Requires mtools: `apt install mtools` (no pip dependency). dev = get_disk_dev(uid, 'TinyUSB', lun) timeout = ENUM_TIMEOUT + last_err = None while timeout > 0: if os.path.exists(dev): - fat = fs.open_fs(f'fat://{dev}?read_only=true') try: - with fat.open(fname, 'rb') as f: - data = f.read() - finally: - fat.close() - assert data, f'Cannot read file {fname} from {dev}' - return data + data = subprocess.check_output( + ['mtype', '-i', dev, f'::/{fname}'], stderr=subprocess.PIPE) + assert data, f'Cannot read file {fname} from {dev}' + return data + except subprocess.CalledProcessError as e: + last_err = e.stderr.decode(errors='replace').strip() time.sleep(1) timeout -= 1 - raise AssertionError(f'Storage {dev} not existed') + raise AssertionError(f'mtype failed on {dev}: {last_err}' if last_err else f'Storage {dev} not existed') def open_mtp_dev(uid): @@ -494,6 +507,19 @@ def reset_uniflash(board): return subprocess.CompletedProcess(args=['dummy'], returncode=0) +def flash_lm4flash(board, firmware): + # TI Tiva-C / Stellaris ICDI: lightweight lm4flash, resets and runs after write + flasher = board['flasher'] + ret = run_cmd(f'lm4flash -s {flasher["uid"]} {flasher["args"]} {firmware}.bin') + return ret + + +def reset_lm4flash(board): + # lm4flash has no reset-only mode; it resets+runs on flash, so reset is a no-op + flasher = board['flasher'] + return subprocess.CompletedProcess(args=['dummy'], returncode=0) + + # ------------------------------------------------------------- # Tests: dual # ------------------------------------------------------------- @@ -788,12 +814,21 @@ def test_host_msc_file_explorer(board): t -= 0.05 resp_text = resp.decode('utf-8', errors='ignore') + speed = None for line in resp_text.splitlines(): if 'KB/s' in line: print(f'{line.strip()} ', end='') + m = re.search(r'([\d.]+\s*[KMG]B/s)', line) # MSC read speed for the report cell + if m: + speed = 'rd ' + m.group(1).replace(' ', '') break ser.close() + return speed + + +def test_host_msc_file_explorer_freertos(board): + return test_host_msc_file_explorer(board) # ------------------------------------------------------------- @@ -948,6 +983,9 @@ def test_device_cdc_msc_throughput(board): pass print(f' CDC read {cdc_r} write {cdc_w}, MSC read {msc_r} write {msc_w} ', end='') + # compact read/write speed for the report cell, e.g. "✅ CDC 652k/422k MSC 1.1M/783k" + short = lambda s: (s.split()[0].rstrip('0').rstrip('.') + s.split()[-1][0]) if ' ' in s else s + return f'{REPORT_CELL["pass"]} CDC {short(cdc_r)}/{short(cdc_w)} MSC {short(msc_r)}/{short(msc_w)}' def test_device_dfu(board): @@ -1402,7 +1440,7 @@ def test_device_audio_test_freertos(board): def test_device_hid_generic_inout(board): uid = board['uid'] - import hid + import hid # cython-hidapi (pip: hidapi, apt: python3-hid) # Find HID device by UID (VID=0xCafe) timeout = ENUM_TIMEOUT @@ -1418,22 +1456,23 @@ def test_device_hid_generic_inout(board): timeout -= 1 assert dev is not None, f'HID device not found for {uid}' - h = hid.Device(vid=dev['vendor_id'], pid=dev['product_id'], serial=uid) - - # Echo test: send random data and verify echo - for size in [8, 32, 63]: - # Report ID (0) + payload, padded to 64 bytes - payload = bytes([random.randint(1, 255) for _ in range(size)]) - report = bytes([0]) + payload + bytes(64 - size) - h.write(report) - echo = h.read(64, timeout=2000) - assert echo is not None and len(echo) >= size, ( - f'HID echo timeout or short read ({size} bytes)') - assert bytes(echo[:size]) == payload, ( - f'HID echo wrong data ({size} bytes):\n' - f' expected: {payload.hex()}\n received: {bytes(echo[:size]).hex()}') - - h.close() + h = hid.device() + h.open(dev['vendor_id'], dev['product_id'], uid) + try: + # Echo test: send random data and verify echo + for size in [8, 32, 63]: + # Report ID (0) + payload, padded to 64 bytes + payload = bytes([random.randint(1, 255) for _ in range(size)]) + report = bytes([0]) + payload + bytes(64 - size) + h.write(report) + echo = h.read(64, 2000) + assert echo and len(echo) >= size, ( + f'HID echo timeout or short read ({size} bytes)') + assert bytes(echo[:size]) == payload, ( + f'HID echo wrong data ({size} bytes):\n' + f' expected: {payload.hex()}\n received: {bytes(echo[:size]).hex()}') + finally: + h.close() # ------------------------------------------------------------- @@ -1465,32 +1504,47 @@ dual_tests = [ host_test = [ 'host/cdc_msc_hid', 'host/msc_file_explorer', + 'host/msc_file_explorer_freertos', 'host/device_info', ] -def test_example(board: Board, f1: str, example: str) -> int: +def find_firmware(variant: str, example: str): + """Locate a built example's firmware base path (no extension) under + cmake-build-<variant>/<example>/. Accepts the single-config layout (firmware + directly in the example dir) or Ninja Multi-Config (a per-config subdir like + RelWithDebInfo/). Returns the base Path, or None if not built.""" + fw_dir = TINYUSB_ROOT / build_dir / f'cmake-build-{variant}' / example + base = Path(example).name + if fw_dir.is_dir(): + for cand in [fw_dir / base, fw_dir / 'RelWithDebInfo' / base, + *(p.with_suffix('') for p in sorted(fw_dir.glob(f'*/{base}.elf')))]: + if cand.with_suffix('.elf').exists() or cand.with_suffix('.bin').exists(): + return cand + return None + + +def test_example(board: Board, variant: str, example: str) -> tuple[int, str]: """ Test example firmware :param board: board dict - :param f1: flags on + :param variant: build variant name = build dir (cmake-build-<variant>) and report row :param example: example name - :return: 0 if success/skip, 1 if failed + :return: (err_count, status, metric) where err_count is 0 on success/skip or + 1 on failure, status is one of 'pass'/'fail'/'skip' (a missing binary + counts as 'skip'), and metric is an optional string a test returns to + show in its report cell instead of the pass symbol (e.g. speed) """ - name = board['name'] err_count = 0 + result_status = 'fail' + metric = None - f1_str = "" - if f1 != "": - f1_str = '-f1_' + f1.replace(' ', '_') - - fw_dir = TINYUSB_ROOT / build_dir / f'cmake-build-{name}{f1_str}' / example - fw_name = fw_dir / Path(example).name - test_name = f'{name+f1_str:40} {example:30} ...' + test_name = f'{variant:40} {example:30} ...' - if not fw_dir.exists() or not ((fw_name.with_suffix('.elf')).exists() or (fw_name.with_suffix('.bin')).exists()): + fw_name = find_firmware(variant, example) + if fw_name is None: log_line(f'{test_name} Skip (no binary)') - return 0 + return 0, 'skip', None if verbose: log_line(f'Flashing {fw_name}.elf') @@ -1513,8 +1567,12 @@ def test_example(board: Board, f1: str, example: str) -> int: last_detail = compact_output(attempt_out.getvalue()) if tret == 'skipped': status = STATUS_SKIPPED + result_status = 'skip' else: status = STATUS_OK + result_status = 'pass' + # a test may return a string to show in its report cell (e.g. speed) + metric = tret if isinstance(tret, str) else None msg = f'{test_name} {status}' if last_detail: msg += f' {last_detail}' @@ -1557,26 +1615,27 @@ def test_example(board: Board, f1: str, example: str) -> int: msg += f' in {time.time() - start_s:.1f}s' log_line(msg) - return err_count + return err_count, result_status, metric def build_board(board: Board) -> tuple[str, int]: """Build firmware for this board via tools/build.py. - Honors board config's build.flags_on variants and build.args defines. - Output goes to cmake-build/cmake-build-BOARD[-f1_...]/ (tools/build.py layout).""" + Honors board config's variant list and build.args defines. + Output goes to cmake-build/cmake-build-<variant>/ (tools/build.py layout).""" name = board['name'] bcfg = cast(BuildCfg, board.get('build', {})) - flags_on_list = bcfg.get('flags_on', ['']) extra_defs = bcfg.get('args', []) + variants = board.get('variant') or [{'name': name, 'flags': ''}] failed = 0 - for f1 in flags_on_list: + for v in variants: cmd = [sys.executable, str(TINYUSB_ROOT / 'tools' / 'build.py'), '-b', name] for d in extra_defs: cmd += ['-D', d] - if f1: - for flag in f1.split(): - cmd += ['-f1', flag] + if v['name'] != name: + cmd += ['--build-name', v['name']] + for tok in v.get('flags', '').split(): + cmd += [f'--cflag={tok}'] if verbose: cmd.append('-v') print(f' + {" ".join(cmd)}') @@ -1586,7 +1645,7 @@ def build_board(board: Board) -> tuple[str, int]: return name, failed -def test_board(board: Board) -> tuple[str, int, list[str]]: +def test_board(board: Board) -> tuple[str, int, list[str], list]: name = board['name'] flasher = board['flasher'] @@ -1596,7 +1655,18 @@ def test_board(board: Board) -> tuple[str, int, list[str]]: if name in board_test: test_list = board_test[name] elif len(test_only) > 0: - test_list = test_only + # Explicit -t: filter against the board's capabilities so a device-only + # board doesn't try to run host/dual tests (the test functions need a + # `dev_attached` entry in the board config that won't exist). + board_tests = board.get('tests', {}) + if 'only' in board_tests: + allowed = set(board_tests['only']) + test_list = [t for t in test_only if t in allowed] + else: + for t in test_only: + category = t.split('/', 1)[0] + if board_tests.get(category) is True: + test_list.append(t) else: if 'tests' in board: board_tests = board['tests'] @@ -1616,22 +1686,96 @@ def test_board(board: Board) -> tuple[str, int, list[str]]: err_count = 0 failed_tests = [] - flags_on_list = [""] - if 'build' in board and 'flags_on' in board['build']: - flags_on_list = board['build']['flags_on'] + rows = [] # list of (row_label, {example: status}) — one row per build variant + variants = board.get('variant') or [{'name': name, 'flags': ''}] - for f1 in flags_on_list: + for v in variants: + vname = v['name'] + cells = {} for test in test_list: - ec = test_example(board, f1, test) + ec, status, metric = test_example(board, vname, test) err_count += ec + cells[test] = metric if metric else status if ec > 0: failed_tests.append(test) + rows.append((vname, cells)) - # flash board_test last to disable board's usb (skipped when --skip-flash is set) + # flash board_test last to disable board's usb (skipped when --skip-flash is set); + # this is teardown/park, not a test — not recorded in the report if not skip_flash: - test_example(board, flags_on_list[0], 'device/board_test') + test_example(board, variants[0]['name'], 'device/board_test') + + return name, err_count, sorted(set(failed_tests)), rows + + +REPORT_MD = 'hil_report.md' +REPORT_JSON = 'hil_report.json' + + +def render_matrix(rows_all: list) -> str: + """Render rows (list of (row_label, {example: status})) as an aligned markdown + matrix: columns = tests (bare names) centered, boards left-aligned.""" + canonical = device_tests + dual_tests + host_test + seen = set() + for _, cells in rows_all: + seen.update(cells) + if not seen: + return 'No tests were run.' + + # columns: canonical order first, then any extras (e.g. from -t) alphabetically + columns = [t for t in canonical if t in seen] + columns += [t for t in sorted(seen) if t not in canonical] + headers = [c.rsplit('/', 1)[-1] for c in columns] # bare example name + + def cell(cells, col): + v = cells.get(col) + if v is None: + return '' + return REPORT_CELL.get(v, v) # status symbol, or a metric string (e.g. speed) verbatim + + board_hdr = 'Board' + board_w = max([len(board_hdr)] + [len(lbl) for lbl, _ in rows_all]) + col_w = [max([len(h)] + [len(cell(cells, c)) for _, cells in rows_all]) + for h, c in zip(headers, columns)] - return name, err_count, sorted(set(failed_tests)) + def line(label, values): + padded = [label.ljust(board_w)] + [v.center(w) for v, w in zip(values, col_w)] + return '| ' + ' | '.join(padded) + ' |' + + header = line(board_hdr, headers) + sep = '| ' + '-' * board_w + ' | ' + ' | '.join(':' + '-' * (w - 2) + ':' for w in col_w) + ' |' + body = [line(lbl, [cell(cells, c) for c in columns]) for lbl, cells in rows_all] + + legend = 'Legend: ✅ pass · ❌ fail · ⚪ skipped · blank not run' + return '\n'.join([header, sep] + body) + '\n\n' + legend + + +def accumulate_report(mret: list, report_dir: Path, fresh: bool) -> str: + """Merge this run's results into hil_report.json in report_dir, then (re)write + the markdown matrix to hil_report.md. `fresh` (a full run, no --skip-board/-bt) + starts a new report; otherwise a re-run accumulates so boards/tests that + already passed are preserved while re-run cells are updated. Returns the md.""" + acc = {} # ordered {row_label: {example: status}} + jpath = report_dir / REPORT_JSON + if not fresh and jpath.is_file(): + try: + for entry in json.loads(jpath.read_text()).get('rows', []): + acc[entry['board']] = dict(entry['cells']) + except (ValueError, KeyError, TypeError): + pass # corrupt/old sidecar: start fresh + + # merge this run: current cells override prior for boards/tests that ran + for _, _, _, rows in mret: + for row_label, cells in rows: + acc.setdefault(row_label, {}).update(cells) + + report_dir.mkdir(parents=True, exist_ok=True) + jpath.write_text(json.dumps({'rows': [{'board': k, 'cells': v} for k, v in acc.items()]}, + indent=2) + '\n') + + md = render_matrix(list(acc.items())) + (report_dir / REPORT_MD).write_text(md + '\n', encoding='utf-8') + return md def main() -> None: @@ -1701,6 +1845,17 @@ def main() -> None: print(f'Build phase done: {build_err} failed') print('-' * 30) + # HIL report sidecar (hil_report.json/.md). A full run starts fresh; a re-run + # (--skip-board / -bt, i.e. the .skip file) accumulates so already-passed + # boards/tests are preserved. Clear any prior report up front on a fresh run so + # a crash mid-run can't leave stale results to be merged by a retry or posted. + report_dir = Path(os.environ.get('HIL_REPORT_DIR', '.')) + fresh = not (args.skip_board or args.board_test) + if fresh: + report_dir.mkdir(parents=True, exist_ok=True) + for f in (REPORT_JSON, REPORT_MD): + (report_dir / f).unlink(missing_ok=True) + with Pool(processes=os.cpu_count() or 1, initializer=init_worker, initargs=(Lock(),)) as pool: async_ret = pool.map_async(test_board, config_boards) try: @@ -1715,14 +1870,20 @@ def main() -> None: # and emit -bt BOARD:t1,t2 so each failed board only re-runs its own failed tests. skip_fname = config_file.with_suffix(config_file.suffix + '.skip') if err_count > 0: - skip_boards += [name for name, err, _ in mret if err == 0] + skip_boards += [name for name, err, _, _ in mret if err == 0] parts = [f'--skip-board {i}' for i in skip_boards] - parts += [f'-bt {name}:{",".join(fts)}' for name, err, fts in mret if err > 0 and fts] + parts += [f'-bt {name}:{",".join(fts)}' for name, err, fts, _ in mret if err > 0 and fts] with skip_fname.open('w') as f: f.write(' '.join(parts)) elif skip_fname.exists(): skip_fname.unlink() + # board x test result matrix -> hil_report.md (accumulates across re-runs) + stdout + report = accumulate_report(mret, report_dir, fresh) + print() + print(report) + print(f'\nReport written to {(report_dir / REPORT_MD).resolve()}') + duration = time.time() - duration print() print("-" * 30) diff --git a/test/hil/requirements.txt b/test/hil/requirements.txt index ef2fecebe..ef1cf575b 100644 --- a/test/hil/requirements.txt +++ b/test/hil/requirements.txt @@ -1,4 +1,9 @@ -fs -hid -pyfatfs +# System packages (install separately): +# sudo apt install mtools libmtp9 alsa-utils iperf +# mtools - read_disk_file (device/cdc_msc, device/msc_dual_lun) +# libmtp9 - pymtp ctypes load (device/mtp); Debian 13 uses libmtp9t64 +# alsa-utils - arecord (device/audio_test_freertos) +# iperf - throughput tests (device/net_lwip_*) +hidapi pyserial +esptool diff --git a/test/hil/tinyusb.json b/test/hil/tinyusb.json index cba7677cf..afe3c4d03 100644 --- a/test/hil/tinyusb.json +++ b/test/hil/tinyusb.json @@ -1,14 +1,49 @@ { "boards": [ { + "name": "ek_tm4c123gxl", + "uid": "010105186C60A110", + "tests": { + "device": true, + "host": false, + "dual": false + }, + "flasher": { + "name": "lm4flash", + "uid": "0E205D19", + "args": "-v" + } + }, + { "name": "espressif_p4_function_ev", "uid": "6055F9F98715", - "build" : { - "flags_on": ["", "CFG_TUD_DWC2_DMA_ENABLE CFG_TUH_DWC2_DMA_ENABLE"] - }, + "variant": [ + { "name": "espressif_p4_function_ev", "flags": "" }, + { "name": "espressif_p4_function_ev-DMA", "flags": "-DCFG_TUD_DWC2_DMA_ENABLE=1 -DCFG_TUH_DWC2_DMA_ENABLE=1" } + ], "tests": { - "only": ["device/cdc_msc_freertos", "device/hid_composite_freertos", "device/audio_test_freertos", "host/device_info"], - "dev_attached": [{"vid_pid": "1a86_55d4", "serial": "52D2002427", "is_cdc": true}] + "only": [ + "device/cdc_msc_freertos", + "device/hid_composite_freertos", + "device/audio_test_freertos", + "host/device_info", + "host/msc_file_explorer_freertos" + ], + "dev_attached": [ + { + "vid_pid": "1a86_55d4", + "serial": "52D2002427", + "is_cdc": true + }, + { + "vid_pid": "21c4_0cc7", + "serial": "900058944CB80A53", + "is_msc": true, + "block_size": 512, + "block_count": 60620800, + "msc_inquiry": "Lexar USB Flash Drive PMAP" + } + ] }, "flasher": { "name": "esptool", @@ -21,12 +56,34 @@ { "name": "espressif_s3_devkitm", "uid": "84F703C084E4", - "build" : { - "flags_on": ["", "CFG_TUD_DWC2_DMA_ENABLE CFG_TUH_DWC2_DMA_ENABLE"] - }, + "variant": [ + { "name": "espressif_s3_devkitm", "flags": "" }, + { "name": "espressif_s3_devkitm-DMA", "flags": "-DCFG_TUD_DWC2_DMA_ENABLE=1 -DCFG_TUH_DWC2_DMA_ENABLE=1" } + ], "tests": { - "only": ["device/cdc_msc_freertos", "device/hid_composite_freertos", "device/audio_test_freertos", "host/device_info"], - "dev_attached": [{"vid_pid": "1a86_55d4", "serial": "52D2005402", "is_cdc": true}] + "only": [ + "device/cdc_msc_freertos", + "device/hid_composite_freertos", + "device/audio_test_freertos", + "host/device_info", + "host/msc_file_explorer_freertos" + ], + "dev_attached": [ + { + "vid_pid": "1a86_55d4", + "serial": "52D2005402", + "is_cdc": true + }, + { + "vid_pid": "048d_04d2", + "serial": "\u0409", + "is_msc": true, + "block_size": 512, + "block_count": 30720000, + "msc_inquiry": "General UDisk 5.00", + "comment": "General UDisk reports iSerialNumber=U+0409" + } + ] }, "flasher": { "name": "esptool", @@ -39,7 +96,9 @@ "name": "feather_nrf52840_express", "uid": "1F0479CD0F764471", "tests": { - "device": true, "host": false, "dual": false + "device": true, + "host": false, + "dual": false }, "flasher": { "name": "jlink", @@ -51,7 +110,9 @@ "name": "max32666fthr", "uid": "0C81464124010B20FF0A08CC2C", "tests": { - "device": true, "host": false, "dual": false + "device": true, + "host": false, + "dual": false }, "flasher": { "name": "openocd_adi", @@ -71,7 +132,13 @@ "device": true, "host": false, "dual": true, - "dev_attached": [{"vid_pid": "067b_2303", "serial": "0", "is_cdc": true}], + "dev_attached": [ + { + "vid_pid": "067b_2303", + "serial": "0", + "is_cdc": true + } + ], "comment": "pl23x" }, "flasher": { @@ -84,7 +151,9 @@ "name": "mimxrt1015_evk", "uid": "DC28F865D2111D228D00B0543A70463C", "tests": { - "device": true, "host": false, "dual": false + "device": true, + "host": false, + "dual": false }, "flasher": { "name": "jlink", @@ -96,9 +165,25 @@ "name": "mimxrt1064_evk", "uid": "BAE96FB95AFA6DBB8F00005002001200", "tests": { - "device": true, "host": true, "dual": true, - "dev_attached": [{"vid_pid": "10c4_ea60", "serial": "0001", "is_cdc": true}], - "comment": "cp2102" + "device": true, + "host": true, + "dual": true, + "dev_attached": [ + { + "vid_pid": "10c4_ea60", + "serial": "0001", + "is_cdc": true, + "comment": "cp2102" + }, + { + "vid_pid": "21c4_0cc7", + "serial": "900058874D871F66", + "is_msc": true, + "block_size": 512, + "block_count": 60620800, + "msc_inquiry": "Lexar USB Flash Drive PMAP" + } + ] }, "flasher": { "name": "jlink", @@ -110,7 +195,9 @@ "name": "lpcxpresso11u37", "uid": "17121919", "tests": { - "device": true, "host": false, "dual": false + "device": true, + "host": false, + "dual": false }, "flasher": { "name": "jlink", @@ -135,13 +222,30 @@ { "name": "raspberry_pi_pico", "uid": "E6614C311B764A37", - "build" : { - "flags_on": ["CFG_TUH_RPI_PIO_USB"] - }, + "variant": [ + { "name": "raspberry_pi_pico", "flags": "-DCFG_TUH_RPI_PIO_USB=1" } + ], "tests": { - "device": true, "host": true, "dual": true, - "dev_attached": [{"vid_pid": "1a86_7523", "serial": "0", "is_cdc": true}], - "comment": "ch34x" + "device": true, + "host": true, + "dual": true, + "dev_attached": [ + { + "vid_pid": "1a86_7523", + "serial": "0", + "is_cdc": true, + "comment": "ch34x" + }, + { + "vid_pid": "048d_04d2", + "serial": "\u0409", + "is_msc": true, + "block_size": 512, + "block_count": 30720000, + "msc_inquiry": "General UDisk 5.00", + "comment": "General UDisk reports iSerialNumber=U+0409" + } + ] }, "flasher": { "name": "openocd", @@ -153,9 +257,16 @@ "name": "raspberry_pi_pico_w", "uid": "E6614864D35DAE36", "tests": { - "device": false, "host": true, "dual": false, + "device": false, + "host": true, + "dual": false, "dev_attached": [ { + "vid_pid": "1a86_55d4", + "serial": "52D2002694", + "is_cdc": true + }, + { "vid_pid": "2008_2018", "serial": "O20070925A002746", "is_msc": true, @@ -176,7 +287,9 @@ "name": "raspberry_pi_pico2", "uid": "560AE75E1C7152C9", "tests": { - "device": false, "host": true, "dual": false, + "device": false, + "host": true, + "dual": false, "dev_attached": [ { "vid_pid": "0951_1603", @@ -184,7 +297,7 @@ "is_msc": true, "block_size": 512, "block_count": 3987456, - "msc_inquiry": "Kingston DataTraveler 2.0 1.0" + "msc_inquiry": "Kingston DataTraveler 2.0 1.00" } ] }, @@ -202,14 +315,24 @@ "host": true, "dual": true, "dev_attached": [ - {"vid_pid": "0403_6001", "serial": "0", "is_cdc": true}, - {"vid_pid": "058f_6387", "serial": "A8BEE062633D", "is_msc": true, - "block_size": 512, "block_count": 7639040, "msc_inquiry": "Generic Flash Disk 8.07"} + { + "vid_pid": "0403_6001", + "serial": "0", + "is_cdc": true + }, + { + "vid_pid": "058f_6387", + "serial": "A8BEE062633D", + "is_msc": true, + "block_size": 512, + "block_count": 7639040, + "msc_inquiry": "Generic Flash Disk 8.07" + } ] }, "flasher": { "name": "openocd", - "uid": "E6614103E78E8324", + "uid": "E663AC91D3359B38", "args": "-f interface/cmsis-dap.cfg -f target/rp2350.cfg -c \"adapter speed 5000\"" } }, @@ -217,7 +340,9 @@ "name": "stm32f072disco", "uid": "3A001A001357364230353532", "tests": { - "device": true, "host": false, "dual": false + "device": true, + "host": false, + "dual": false }, "flasher": { "name": "jlink", @@ -227,14 +352,45 @@ "comment": "2x16 access scheme with 1KB USB SRAM" }, { + "name": "stm32f407disco", + "uid": "30001A000647313332353735", + "tests": { + "device": true, + "host": false, + "dual": false + }, + "flasher": { + "name": "jlink", + "uid": "000773661813", + "args": "-device stm32f407vg" + } + }, + { "name": "stm32f723disco", "uid": "460029001951373031313335", - "build" : { - "flags_on": ["", "CFG_TUH_DWC2_DMA_ENABLE"] - }, + "variant": [ + { "name": "stm32f723disco", "flags": "" }, + { "name": "stm32f723disco-DMA", "flags": "-DCFG_TUH_DWC2_DMA_ENABLE=1" } + ], "tests": { - "device": true, "host": true, "dual": false, - "dev_attached": [{"vid_pid": "1a86_55d4", "serial": "52D2003414", "is_cdc": true}] + "device": true, + "host": true, + "dual": false, + "dev_attached": [ + { + "vid_pid": "1a86_55d4", + "serial": "52D2003414", + "is_cdc": true + }, + { + "vid_pid": "21c4_0cc7", + "serial": "90005893730A1A63", + "is_msc": true, + "block_size": 512, + "block_count": 60620800, + "msc_inquiry": "Lexar USB Flash Drive PMAP" + } + ] }, "flasher": { "name": "jlink", @@ -246,11 +402,14 @@ { "name": "stm32h743nucleo", "uid": "110018000951383432343236", - "build" : { - "flags_on": ["", "CFG_TUD_DWC2_DMA_ENABLE"] - }, + "variant": [ + { "name": "stm32h743nucleo", "flags": "" }, + { "name": "stm32h743nucleo-DMA", "flags": "-DCFG_TUD_DWC2_DMA_ENABLE=1 -DCFG_TUH_DWC2_DMA_ENABLE=1" } + ], "tests": { - "device": true, "host": false, "dual": false + "device": true, + "host": false, + "dual": false }, "flasher": { "name": "openocd", @@ -262,7 +421,9 @@ "name": "stm32g0b1nucleo", "uid": "4D0038000450434E37343120", "tests": { - "device": true, "host": false, "dual": false + "device": true, + "host": false, + "dual": false }, "flasher": { "name": "openocd", @@ -270,17 +431,47 @@ "args": "-f interface/stlink.cfg -f target/stm32g0x.cfg" }, "comment": "32-bit scheme, 2KB USB SRAM" + }, + { + "name": "stm32l476disco", + "uid": "3C0050001150334258343920", + "tests": { + "device": true, + "host": false, + "dual": false + }, + "flasher": { + "name": "jlink", + "uid": "777632258", + "args": "-device STM32L476VG" + } + }, + { + "name": "stm32u083nucleo", + "uid": "300044000D5036394E373620", + "tests": { + "device": true, + "host": false, + "dual": false + }, + "flasher": { + "name": "stlink", + "uid": "0668FF575457657187061314" + } } ], "boards-skip": [ { "name": "stm32f769disco", "uid": "21002F000F51363531383437", - "build" : { - "flags_on": ["", "CFG_TUD_DWC2_DMA_ENABLE"] - }, + "variant": [ + { "name": "stm32f769disco", "flags": "" }, + { "name": "stm32f769disco-DMA", "flags": "-DCFG_TUD_DWC2_DMA_ENABLE=1 -DCFG_TUH_DWC2_DMA_ENABLE=1" } + ], "tests": { - "device": true, "host": false, "dual": false + "device": true, + "host": false, + "dual": false }, "flasher": { "name": "jlink", @@ -292,25 +483,15 @@ "name": "nanoch32v203", "uid": "CDAB277B0FBC03E339E339E3", "tests": { - "device": true, "host": false, "dual": false + "device": true, + "host": false, + "dual": false }, "flasher": { "name": "openocd_wch", "uid": "EBCA8F0670AF", "args": "" } - }, - { - "name": "stm32f407disco", - "uid": "30001A000647313332353735", - "tests": { - "device": true, "host": false, "dual": false - }, - "flasher": { - "name": "jlink", - "uid": "000773661813", - "args": "-device stm32f407vg" - } } ] } |
