diff options
| -rw-r--r-- | .github/workflows/build.yml | 97 | ||||
| -rw-r--r-- | .github/workflows/build_util.yml | 2 | ||||
| -rwxr-xr-x | test/hil/hil_test.py | 375 | ||||
| -rw-r--r-- | test/hil/tinyusb.json | 85 |
4 files changed, 384 insertions, 175 deletions
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index f24f3ae1f..818e7ba81 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -274,13 +274,25 @@ jobs: toolchain: - 'arm-gcc' - 'riscv-gcc' - - 'esp-idf' with: build-system: 'cmake' toolchain: ${{ matrix.toolchain }} build-args: ${{ toJSON(fromJSON(needs.set-matrix.outputs.hil_json)[matrix.toolchain]) }} upload-artifacts: true + # esp-idf builds are by far the slowest; keep them out of hil-build so the main + # hil-tinyusb run starts as soon as the fast toolchains finish (esp boards get + # their own hil-tinyusb-esp run gated only on this job) + hil-build-esp: + needs: [ check-paths, set-matrix ] + if: needs.check-paths.outputs.code_changed == 'true' && github.repository_owner == 'hathach' + uses: ./.github/workflows/build_util.yml + with: + build-system: 'cmake' + toolchain: 'esp-idf' + build-args: ${{ toJSON(fromJSON(needs.set-matrix.outputs.hil_json)['esp-idf']) }} + upload-artifacts: true + # --------------------------------------- # Hardware in the loop (HIL) # self-hosted on local VM, for attached hardware checkout HIL_JSON @@ -295,9 +307,13 @@ jobs: - display: tinyusb.json runner: [ self-hosted, X64, hathach, hardware-in-the-loop ] hil_json: test/hil/tinyusb.json + # esptool-flashed (espressif) boards run in hil-tinyusb-esp, + # gated on the slow esp-idf build + test_args: '--exclude-flasher esptool' - display: hfp.json runner: [ self-hosted, Linux, X64, hifiphile ] hil_json: test/hil/hfp.json + test_args: '' runs-on: ${{ matrix.runner }} env: HIL_JSON: ${{ matrix.hil_json }} @@ -305,16 +321,19 @@ jobs: - name: Set HIL report dir (sibling of workspace; persists across run attempts) run: echo "HIL_REPORT_DIR=$(dirname "$GITHUB_WORKSPACE")/hil-report" >> "$GITHUB_ENV" - - name: Get Skip Boards from previous run + - name: Get re-run spec from previous attempt if: github.run_attempt != '1' run: | - if [ -f "${{ env.HIL_JSON }}.skip" ]; then - SKIP_BOARDS=$(cat "${{ env.HIL_JSON }}.skip") + # only honor a spec stamped by THIS run: a spec left by another run (attempt 1 + # died or was skipped before hil_test.py could clear it) must not be consumed + SPEC="$HIL_REPORT_DIR/$(basename "${{ env.HIL_JSON }}").failed" + if [ -f "$SPEC" ] && [ "$(cat "$SPEC.run" 2>/dev/null)" = "$GITHUB_RUN_ID" ]; then + RERUN_ARGS=$(cat "$SPEC") else - SKIP_BOARDS="" + RERUN_ARGS="" fi - echo "SKIP_BOARDS=$SKIP_BOARDS" - echo "SKIP_BOARDS=$SKIP_BOARDS" >> $GITHUB_ENV + echo "RERUN_ARGS=$RERUN_ARGS" + echo "RERUN_ARGS=$RERUN_ARGS" >> $GITHUB_ENV - name: Clean workspace run: | @@ -335,9 +354,7 @@ jobs: - name: Test on actual hardware # Single attempt per test (--retry 1), no in-run second pass: a broken fixture # fails fast instead of holding the runner (and other PRs' HIL jobs) for hours. - # hil_test.py still writes ${HIL_JSON}.skip, so a manual re-run attempt only - # retests what failed (see "Get Skip Boards from previous run"). - run: python3 test/hil/hil_test.py --retry 1 ${{ env.HIL_JSON }} $SKIP_BOARDS + run: python3 test/hil/hil_test.py --retry 1 ${{ matrix.test_args }} ${{ env.HIL_JSON }} $RERUN_ARGS - name: Upload HIL report if: always() && github.event_name == 'pull_request' @@ -349,6 +366,66 @@ jobs: overwrite: true # --------------------------------------- + # Hardware in the loop (HIL) - espressif boards only + # Same rig as hil-tinyusb (tinyusb.json) but gated only on the slow esp-idf build, + # so the main run does not wait for it. Per-board flocks arbitrate the shared rig; + # the runner has a single job slot, so the two HIL jobs never overlap - adding a + # second slot would double the per-controller flash/usbtest budgets. + # --------------------------------------- + hil-tinyusb-esp: + needs: hil-build-esp + name: hil-tinyusb (tinyusb-esp.json) + runs-on: [ self-hosted, X64, hathach, hardware-in-the-loop ] + env: + HIL_JSON: test/hil/tinyusb.json + TEST_ARGS: '--flasher esptool' + steps: + - name: Set HIL report dir (sibling of workspace; persists across run attempts) + run: echo "HIL_REPORT_DIR=$(dirname "$GITHUB_WORKSPACE")/hil-report-esp" >> "$GITHUB_ENV" + + - name: Get re-run spec from previous attempt + if: github.run_attempt != '1' + run: | + # only honor a spec stamped by THIS run: a spec left by another run (attempt 1 + # died or was skipped before hil_test.py could clear it) must not be consumed + SPEC="$HIL_REPORT_DIR/$(basename "${{ env.HIL_JSON }}").failed" + if [ -f "$SPEC" ] && [ "$(cat "$SPEC.run" 2>/dev/null)" = "$GITHUB_RUN_ID" ]; then + RERUN_ARGS=$(cat "$SPEC") + else + RERUN_ARGS="" + fi + echo "RERUN_ARGS=$RERUN_ARGS" + echo "RERUN_ARGS=$RERUN_ARGS" >> $GITHUB_ENV + + - name: Clean workspace + run: | + echo "Cleaning up for the first run" + rm -rf "${{ github.workspace }}" + mkdir -p "${{ github.workspace }}" + + - name: Checkout TinyUSB + uses: actions/checkout@v6 + + - name: Download Artifacts + uses: actions/download-artifact@v5 + with: + pattern: binaries-esp-idf-* + path: cmake-build + merge-multiple: true + + - name: Test on actual hardware + run: python3 test/hil/hil_test.py --retry 1 $TEST_ARGS ${{ env.HIL_JSON }} $RERUN_ARGS + + - name: Upload HIL report + if: always() && github.event_name == 'pull_request' + uses: actions/upload-artifact@v7 + with: + name: hil-report-tinyusb-esp.json + path: ${{ env.HIL_REPORT_DIR }}/hil_report.md + if-no-files-found: ignore + overwrite: true + + # --------------------------------------- # Hardware in the loop (HIL) # self-hosted by HFP, build with IAR toolchain, for attached hardware checkout test/hil/hfp.json # Since IAR Token secret is not passed to forked PR, only build non-forked PR diff --git a/.github/workflows/build_util.yml b/.github/workflows/build_util.yml index 2532caebe..90115862b 100644 --- a/.github/workflows/build_util.yml +++ b/.github/workflows/build_util.yml @@ -99,7 +99,7 @@ jobs: if: inputs.upload-artifacts == true && inputs.code-changed == true uses: actions/upload-artifact@v7 with: - name: binaries-${{ matrix.arg }} + name: binaries-${{ inputs.toolchain }}-${{ matrix.arg }} path: | cmake-build/cmake-build-*/*/*/*.elf cmake-build/cmake-build-*/*/*/*.bin diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py index 40ae80572..448b236fc 100755 --- a/test/hil/hil_test.py +++ b/test/hil/hil_test.py @@ -38,6 +38,7 @@ import argparse import io +import itertools import os import random import re @@ -121,7 +122,7 @@ ENUM_TIMEOUT_RETRY = 4 _enum_timeout = ENUM_TIMEOUT -def enum_timeout_s() -> int: +def enum_timeout() -> int: """Enumeration wait budget for the current test attempt.""" return _enum_timeout @@ -130,7 +131,7 @@ def wait_until(predicate, step: float = 1.0): """Poll predicate under the per-attempt enum budget. Deadline-based so a slow predicate body (subprocess, libmtp scan) counts against the budget. Returns the first truthy predicate value, or None on timeout.""" - deadline = time.monotonic() + enum_timeout_s() + deadline = time.monotonic() + enum_timeout() while True: r = predicate() if r: @@ -157,6 +158,7 @@ class TestFail(AssertionError): verbose = False +PROFILE = os.environ.get('HIL_PROFILE') == '1' # timestamped logs + permit/flash timing + ctrl-map dump test_only = [] board_test = {} build_dir = 'cmake-build' @@ -164,35 +166,41 @@ skip_flash = False print_lock = None shuffle_seed = None # per-run seed for the per-board test-order shuffle (HIL_SHUFFLE_SEED to replay) -# Per-host-controller concurrency (see controller_of/ctrl_slot below): a usbtest battery +# Per-host-controller concurrency (see controller_of/controller_slot below): a usbtest battery # saturates its DUT's host controller, so batteries and flashes are budgeted per controller. -# NOTE: a Renesas uPD720201 host card must run its latest firmware (>= 2.0.2.6; RAM-uploaded, -# so it must be re-loaded every power cycle) - its ROM firmware dies under battery + -# flash/re-enumeration churn, and usbtest.py refuses the unlink-stress cases on old firmware. -# Widths profiled 2026-07-13/14 on fw 2.0.2.6 (8/1 through 12/8): wall time falls -# 22.2/14.3/12.5/10.8 min at usbtest width 1/2/3/4 and plateaus there; flash width beyond 8 -# buys nothing and only amplifies flasher-hub contention; the first battery case failures -# (bandwidth stretch on shared leaf-hub uplinks) appear at 12/8. Hence the 8/4 defaults. -FLASH_PARALLEL = max(1, int(os.getenv('HIL_FLASH_PARALLEL', '8'))) -USBTEST_PARALLEL = max(1, int(os.getenv('HIL_USBTEST_PARALLEL', '4'))) -CTRL_SLOTS = 12 # lock slots; controllers are assigned to slots on first sight -usbtest_sems = None # CTRL_SLOTS semaphores: up to USBTEST_PARALLEL batteries per controller -flash_sems = None # CTRL_SLOTS semaphores(FLASH_PARALLEL): flash permits per controller -ctrl_map = None # shared dict: 'pci:<addr>' -> slot, 'uid:<uid>' -> pci addr cache -ctrl_meta = None # guards slot assignment in ctrl_map +# - uPD720201 cards need their latest firmware (>= 2.0.2.6; RAM-uploaded, reloads every +# power cycle): ROM firmware dies under battery + re-enumeration churn, and usbtest.py +# refuses the unlink-stress cases on it. +# - widths (profiled 2026-07-13/14): wall time 22.2/14.3/12.5/10.8 min at usbtest width +# 1/2/3/4, plateau after; flash width beyond 8 only adds flasher-hub contention; +# battery case failures start at 12/8 (bandwidth stretch on shared leaf-hub uplinks). +# - a marginal DUT port bouncing during concurrent batteries can wedge/kill a uPD720201 +# ("xHCI host not responding to stop endpoint command"): fix the port/cable or pull +# the board, don't lower the widths (2026-07-16: every death traced to one board's port). +FLASH_PARALLEL = int(os.getenv('HIL_FLASH_PARALLEL', '8')) +USBTEST_PARALLEL = int(os.getenv('HIL_USBTEST_PARALLEL', '4')) +CONTROLLER_SLOTS = 12 # lock slots; controllers are assigned to slots on first sight +usbtest_sems = None # CONTROLLER_SLOTS semaphores: per-slot usbtest-battery permits +flash_sems = None # CONTROLLER_SLOTS semaphores: per-slot flash permits +controller_map = None # shared dict: 'pci:<addr>' -> slot, 'uid:<uid>' -> pci addr cache +controller_meta = None # guards slot assignment in controller_map +controller_hints = {} # static uid -> pci from the last run's cache (read-only per worker) -def init_worker(lock, seed, b_mutexes, f_sems, cmap, cmeta): - global print_lock, shuffle_seed, usbtest_sems, flash_sems, ctrl_map, ctrl_meta +def init_worker(lock, seed, b_mutexes, f_sems, cmap, cmeta, hints_by_uid): + global print_lock, shuffle_seed, usbtest_sems, flash_sems, controller_map, controller_meta, controller_hints print_lock = lock shuffle_seed = seed usbtest_sems = b_mutexes flash_sems = f_sems - ctrl_map = cmap - ctrl_meta = cmeta + controller_map = cmap + controller_meta = cmeta + controller_hints = hints_by_uid def log_line(msg: str) -> None: + if PROFILE: + msg = f'{time.time():.3f} {msg}' out = sys.__stdout__ if sys.__stdout__ is not None else sys.stdout if print_lock is not None: with print_lock: @@ -210,9 +218,9 @@ def controller_of(uid: str): resolutions are cached — cabling does not change mid-run. Dual-port parts (e.g. CH32V307 usbhs/usbfs variants) share one uid and one cache entry: budgeting is only exact when both ports sit on the same controller (true on this rig).""" - if ctrl_map is None: + if controller_map is None: return None - cached = ctrl_map.get(f'uid:{uid}') + cached = controller_map.get(f'uid:{uid}') if cached: return cached for f in glob.glob('/sys/bus/usb/devices/*/serial'): @@ -224,29 +232,29 @@ def controller_of(uid: str): root = os.path.realpath(f'/sys/bus/usb/devices/usb{bus}') m = re.findall(r'[0-9a-f]{4}:[0-9a-f]{2}:[0-9a-f]{2}\.[0-9a-f]', root) if m: - ctrl_map[f'uid:{uid}'] = m[-1] + controller_map[f'uid:{uid}'] = m[-1] return m[-1] except (OSError, ValueError): continue return None -def ctrl_slot(pci: str) -> int: +def controller_slot(pci: str) -> int: """Map a controller PCI address to a lock slot (assigned on first sight).""" key = f'pci:{pci}' - with ctrl_meta: - slot = ctrl_map.get(key) + with controller_meta: + slot = controller_map.get(key) if slot is None: - slot = ctrl_map.get('nslots', 0) - if slot >= CTRL_SLOTS: + slot = controller_map.get('nslots', 0) + if slot >= CONTROLLER_SLOTS: slot = 0 # more controllers than slots: overflow shares slot 0 (safe, over-serialized) else: - ctrl_map['nslots'] = slot + 1 - ctrl_map[key] = slot + controller_map['nslots'] = slot + 1 + controller_map[key] = slot return slot -class ctrl_permit: +class controller_permit: """Context manager: one permit from `sems` on the board's controller slot. If the controller is unknown, fail closed: take one permit from EVERY slot, in order, so the operation respects the budget wherever it might land. `warn_unknown` logs that fallback @@ -254,21 +262,34 @@ class ctrl_permit: def __init__(self, sems, uid: str, warn_unknown: bool = False): self.sems = sems self.slots = None + self.uid = uid if sems is None: return pci = controller_of(uid) + if pci is None and not warn_unknown: + # last-run cabling hint, flash budgeting only: a mis-budgeted flash is harmless, + # but a battery must never trust a stale hint (it could stack two batteries on + # one controller). In practice only a board's first flash lands here - batteries + # assert enumeration before taking their permit. + pci = controller_hints.get(uid) if pci is None and warn_unknown: log_line(f'warning: cannot resolve {uid} to a host controller; ' 'taking a permit on every slot (over-serialized)') - self.slots = [ctrl_slot(pci)] if pci else list(range(CTRL_SLOTS)) + self.slots = [controller_slot(pci)] if pci else list(range(CONTROLLER_SLOTS)) def __enter__(self): if self.slots: + t0 = time.monotonic() taken = [] try: for s in self.slots: self.sems[s].acquire() taken.append(s) + # stays inside the try: if this raises (e.g. broken stdout), the permits + # must be released - a failed __enter__ never gets its __exit__ + if PROFILE and time.monotonic() - t0 > 1.0: + log_line(f'[prof] permit wait {time.monotonic() - t0:.1f}s ' + f'(uid {self.uid}, slots {self.slots})') except BaseException: for s in reversed(taken): self.sems[s].release() @@ -282,12 +303,12 @@ class ctrl_permit: return False -def flash_permit(uid: str) -> ctrl_permit: - return ctrl_permit(flash_sems, uid) +def flash_permit(uid: str) -> controller_permit: + return controller_permit(flash_sems, uid) -def usbtest_permit(uid: str) -> ctrl_permit: - return ctrl_permit(usbtest_sems, uid, warn_unknown=True) +def usbtest_permit(uid: str) -> controller_permit: + return controller_permit(usbtest_sems, uid, warn_unknown=True) def compact_output(raw: str) -> str: @@ -426,7 +447,7 @@ def get_alsa_capture_dev(id): def open_serial_dev(port: str): - timeout = enum_timeout_s() + timeout = enum_timeout() ser = None while timeout > 0: if os.path.exists(port): @@ -749,7 +770,7 @@ def test_dual_host_info_to_device_cdc(board): # read until all expected devices are enumerated data = b'' - timeout = enum_timeout_s() + timeout = enum_timeout() while timeout > 0: new_data = ser.read(ser.in_waiting or 1) if new_data: @@ -801,7 +822,7 @@ def test_host_device_info(board): # read until all expected devices are enumerated data = b'' - timeout = enum_timeout_s() + timeout = enum_timeout() while timeout > 0: new_data = ser.read(ser.in_waiting or 1) if new_data: @@ -880,7 +901,7 @@ def test_host_cdc_msc_hid(board): # Wait for all expected mount messages data = b'' - timeout = enum_timeout_s() + timeout = enum_timeout() wait_cdc = len(cdc_devs) > 0 wait_msc = len(msc_devs) > 0 while timeout > 0: @@ -973,7 +994,7 @@ def test_host_msc_file_explorer(board): # Wait for MSC mount (Disk Size message) data = b'' - timeout = enum_timeout_s() + timeout = enum_timeout() while timeout > 0: new_data = ser.read(ser.in_waiting or 1) if new_data: @@ -1032,9 +1053,9 @@ def test_host_msc_file_explorer(board): for line in resp_text.splitlines(): if 'KB/s' in line: print(f'{line.strip()} ', end='') - m = re.search(r'([\d.]+\s*[KMG]B/s)', line) # MSC read speed for the report cell + m = re.search(r'([\d.]+)\s*([KMG]B/s)', line) # MSC read speed for the report cell if m: - speed = 'rd ' + m.group(1).replace(' ', '') + speed = f'{m.group(1)} {m.group(2)}' break ser.close() @@ -1138,7 +1159,7 @@ def test_device_cdc_msc_throughput(board): # Wait for MSC disk enumeration dev = get_disk_dev(uid, 'TinyUSB', 0) - timeout = enum_timeout_s() + timeout = enum_timeout() while timeout > 0: if os.path.exists(dev): break @@ -1147,7 +1168,7 @@ def test_device_cdc_msc_throughput(board): # Wait for CDC tty enumeration tty = get_serial_dev(uid, 'TinyUSB', 'Throughput', 0) - timeout = enum_timeout_s() + timeout = enum_timeout() while timeout > 0: if os.path.exists(tty): break @@ -1196,9 +1217,19 @@ def test_device_cdc_msc_throughput(board): pass print(f' CDC read {cdc_r} write {cdc_w}, MSC read {msc_r} write {msc_w} ', end='') - # compact read/write speed for the report cell, e.g. "✅ CDC 652k/422k MSC 1.1M/783k" - short = lambda s: (s.split()[0].rstrip('0').rstrip('.') + s.split()[-1][0]) if ' ' in s else s - return f'{REPORT_CELL["pass"]} CDC {short(cdc_r)}/{short(cdc_w)} MSC {short(msc_r)}/{short(msc_w)}' + + # compact read/write speeds for the report cell, e.g. "✅ C 652/422k M 1.1M/783k" + # (C=CDC, M=MSC; the unit is shown once when both sides share it) + def short(s): + return (s.split()[0].rstrip('0').rstrip('.') + s.split()[-1][0]) if ' ' in s else s + + def pair(r, w): + r, w = short(r), short(w) + if r[-1:] == w[-1:] and r[-1:].isalpha(): + r = r[:-1] + return f'{r}/{w}' + + return f'{REPORT_CELL["pass"]} C {pair(cdc_r, cdc_w)} M {pair(msc_r, msc_w)}' def test_device_dfu(board): @@ -1206,7 +1237,7 @@ def test_device_dfu(board): # Wait device enum. Deadline-based: dfu-util -l itself takes ~1 s per call, which a # per-iteration countdown would not charge against the budget. - deadline = time.monotonic() + enum_timeout_s() + deadline = time.monotonic() + enum_timeout() found = False while time.monotonic() < deadline: ret = run_cmd(f'dfu-util -l') @@ -1247,7 +1278,7 @@ def test_device_dfu(board): def test_device_dfu_runtime(board): uid = board['uid'] # Wait device enum (deadline-based, see test_device_dfu) - deadline = time.monotonic() + enum_timeout_s() + deadline = time.monotonic() + enum_timeout() found = False while time.monotonic() < deadline: ret = run_cmd(f'dfu-util -l') @@ -1266,7 +1297,7 @@ def test_device_hid_boot_interface(board): mouse1 = get_hid_dev(uid, 'TinyUSB', 'TinyUSB_Device', 'if01-event-mouse') mouse2 = get_hid_dev(uid, 'TinyUSB', 'TinyUSB_Device', 'if01-mouse') # Wait device enum - timeout = enum_timeout_s() + timeout = enum_timeout() while timeout > 0: if os.path.exists(kbd) and os.path.exists(mouse1) and os.path.exists(mouse2): break @@ -1478,7 +1509,7 @@ def test_device_net_lwip_webserver(board): # Poll the iperf TCP port until the device is accepting. The net stack comes up a bit # after DHCP completes; iperf server binding isn't instantaneous after reflash. - deadline = time.monotonic() + enum_timeout_s() + deadline = time.monotonic() + enum_timeout() last_err = None while time.monotonic() < deadline: try: @@ -1488,7 +1519,7 @@ def test_device_net_lwip_webserver(board): except OSError as e: last_err = e time.sleep(0.3) - assert last_err is None, f'iperf TCP {device_ip}:{iperf_port} not accepting within {enum_timeout_s()}s: {last_err}' + assert last_err is None, f'iperf TCP {device_ip}:{iperf_port} not accepting within {enum_timeout()}s: {last_err}' # Throughput: 5-second iperf2 TCP test, CSV output for stable parsing. # iperf2 CSV final summary line: timestamp,src_ip,src_port,dst_ip,dst_port,id,interval,bytes,bps @@ -1528,7 +1559,7 @@ def test_device_midi_test(board): uid = board['uid'] # Find MIDI device via /dev/snd/by-id using board UID - timeout = enum_timeout_s() + timeout = enum_timeout() midi_port = None while timeout > 0: pattern = f'/dev/snd/by-id/usb-*_{uid}-*' @@ -1587,7 +1618,7 @@ def test_device_audio_test_freertos(board): return 'skipped' pcm = None - timeout = enum_timeout_s() + timeout = enum_timeout() while timeout > 0: pcm = get_alsa_capture_dev(uid) if pcm: @@ -1654,7 +1685,7 @@ def test_device_hid_generic_inout(board): import hid # cython-hidapi (pip: hidapi, apt: python3-hid) # Find HID device by UID (VID=0xCafe) - timeout = enum_timeout_s() + timeout = enum_timeout() dev = None while timeout > 0: for d in hid.enumerate(0xCafe): @@ -1705,12 +1736,15 @@ def test_device_usbtest(board): pass return False - end = time.monotonic() + enum_timeout_s() + end = time.monotonic() + enum_timeout() while time.monotonic() < end and not usbtest_enumerated(): time.sleep(0.2) # fail before usbtest_permit: an absent device would otherwise queue on the battery # mutex for minutes behind real batteries just to have usbtest.py report "no device" - assert usbtest_enumerated(), f'no cafe:4010 device with serial {uid}' + if not usbtest_enumerated(): + # 0/30 rather than a bare cell: the battery never ran (30 = standard case count) + raise TestFail(f'no cafe:4010 device with serial {uid}', + metric=f'{REPORT_CELL["fail"]} 0/30') # settle: right after flashing the enumeration can bounce once (and on dual-port parts like # CH32V307 the other port's stale usbtest node — same serial and PID — lingers a moment); # running testusb into that gap sees the device drop mid-case @@ -1732,7 +1766,8 @@ def test_device_usbtest(board): data = json.loads(out[brace:]) passed, failed = int(data['passed']), int(data['failed']) except (ValueError, KeyError, json.JSONDecodeError): - raise AssertionError(f'usbtest did not run: {compact_output(out) or cmd_stdout_text(r.stderr)}') + raise TestFail(f'usbtest did not run: {compact_output(out) or cmd_stdout_text(r.stderr)}', + metric=f'{REPORT_CELL["fail"]} 0/30') total = passed + failed if failed == 0 and total > 0: @@ -1745,12 +1780,12 @@ def test_device_usbtest(board): # ------------------------------------------------------------- # Main # ------------------------------------------------------------- + +# The per-board run order is shuffled (see test_board). +# Every example carries a unique hardcoded idProduct (see its usb_descriptors.c) + # device tests device_tests = [ - # The per-board run order is shuffled (see test_board). Every example carries a unique - # hardcoded idProduct (see its usb_descriptors.c), so any two different examples always - # re-enumerate back-to-back — even on boards whose CPU-reset does not drop D+ (e.g. WCH - # CH58x via openocd), which only re-enumerate when the PID changes. 'device/cdc_dual_ports', 'device/cdc_msc', 'device/dfu', @@ -1833,7 +1868,11 @@ def test_example(board: Board, variant: str, example: str) -> tuple[int, str, st with redirect_stdout(attempt_out): if not skip_flash: with flash_permit(board['uid']): + t_flash = time.monotonic() ret = globals()[f'flash_{board["flasher"]["name"].lower()}'](board, str(fw_name)) + if PROFILE: + log_line(f'[prof] {variant} {example} flash attempt {i + 1}: ' + f'{time.monotonic() - t_flash:.1f}s rc={ret.returncode}') flash_ok = (ret.returncode == 0) if flash_ok: try: @@ -1923,7 +1962,7 @@ def build_board(board: Board) -> tuple[str, int]: return name, failed -def test_board(board: Board) -> tuple[str, int, list[str], list]: +def test_board(board: Board) -> tuple[str, int, list[str], list, float]: name = board['name'] flasher = board['flasher'] @@ -1933,7 +1972,9 @@ def test_board(board: Board) -> tuple[str, int, list[str], list]: log_line(f'{name:25} {STATUS_FAILED}: {e}') # visible report row so the ❌ matches the exit code; failed-tests stays # empty so a re-run repeats the whole board (no bogus -bt test filter) - return name, 1, [], [(name, {'board-locked': 'fail'})] + return name, 1, [], [(name, {'board-locked': 'fail'}, None)], 0.0 + # after the lock: flock wait behind a concurrent run is not board cost + t_board = time.monotonic() try: # default to all tests test_list = [] @@ -1972,7 +2013,10 @@ def test_board(board: Board) -> tuple[str, int, list[str], list]: err_count = 0 failed_tests = [] - rows = [] # list of (row_label, {example: status}) — one row per build variant + rows = [] # list of (row_label, {example: status}, duration) — one row per build variant + # a -t/-bt filtered run times only a subset; report no duration so an accumulate + # re-run keeps the previous full-run value + partial = bool(test_only) or name in board_test variants = board.get('variant') or [{'name': name, 'flags': ''}] prev_last = None # last test of the previous variant: the variant boundary is an adjacency too @@ -1988,9 +2032,9 @@ def test_board(board: Board) -> tuple[str, int, list[str], list]: random.Random(f'{shuffle_seed}:{name}:{vname}').shuffle(run_list) if run_list[0] == prev_last: run_list[0], run_list[-1] = run_list[-1], run_list[0] - log_line(f'{vname:40} test order: {", ".join(t.rsplit("/", 1)[-1] for t in run_list)}') if run_list: prev_last = run_list[-1] + t_variant = time.monotonic() cells = {} for test in run_list: ec, status, metric = test_example(board, vname, test) @@ -1998,14 +2042,19 @@ def test_board(board: Board) -> tuple[str, int, list[str], list]: cells[test] = metric if metric else status if ec > 0: failed_tests.append(test) - rows.append((vname, cells)) + dur = f'{time.monotonic() - t_variant:.0f}s' if run_list and not partial else None + rows.append((vname, cells, dur)) + + # board duration excludes the teardown park-flash below; a partial (filtered) + # run reports 0.0 so it never overwrites a cached full-run duration + t_total = 0.0 if partial else time.monotonic() - t_board # flash board_test last to disable board's usb (skipped when --skip-flash is set); # this is teardown/park, not a test — not recorded in the report if not skip_flash: test_example(board, variants[0]['name'], 'device/board_test') - return name, err_count, sorted(set(failed_tests)), rows + return name, err_count, sorted(set(failed_tests)), rows, t_total finally: if _lock_fh: try: @@ -2021,13 +2070,30 @@ def test_board(board: Board) -> tuple[str, int, list[str], list]: REPORT_MD = 'hil_report.md' REPORT_JSON = 'hil_report.json' +# controller hints learned from previous runs: uid -> {'name', 'pci', 'duration'}. Only +# 'pci' is consumed (dispatch order and first-flash budgeting, never battery +# serialization); name/duration are informational. PCI addresses are boot-stable (bus +# numbers are not), so the cache survives reboots and only goes stale on re-cabling. +CONTROLLER_CACHE = Path.home() / '.cache' / 'tinyusb-hil' / 'controller_cache.json' + + +def schedule_boards(boards: list, pci_of_uid: dict) -> list: + """Dispatch order: round-robin across host controllers so every controller's + serialized usbtest battery chain is fed from t=0 instead of one card's boards + convoying at the head of the queue. Boards without a controller hint form their + own bucket; config order is kept within a bucket.""" + buckets = {} + for b in boards: + buckets.setdefault(pci_of_uid.get(b['uid'], '?'), []).append(b) + return [b for grp in itertools.zip_longest(*buckets.values()) for b in grp if b is not None] def render_matrix(rows_all: list) -> str: - """Render rows (list of (row_label, {example: status})) as an aligned markdown - matrix: columns = tests (bare names) centered, boards left-aligned.""" + """Render rows (list of (row_label, {example: status}, duration)) as an aligned + markdown matrix: columns = tests (bare names) centered, boards left-aligned, + per-row duration as the trailing column.""" seen = set() - for _, cells in rows_all: + for _, cells, _ in rows_all: seen.update(cells) if not seen: return 'No tests were run.' @@ -2041,7 +2107,7 @@ def render_matrix(rows_all: list) -> str: return (pinned.index(name) if name in pinned else len(pinned), name, t) columns = sorted(seen, key=col_key) - headers = [c.rsplit('/', 1)[-1] for c in columns] # bare example name + headers = [c.rsplit('/', 1)[-1] for c in columns] + ['duration'] # bare example names def cell(cells, col): v = cells.get(col) @@ -2049,10 +2115,12 @@ def render_matrix(rows_all: list) -> str: return '' return REPORT_CELL.get(v, v) # status symbol, or a metric string (e.g. speed) verbatim + rows_vals = [(lbl, [cell(cells, c) for c in columns] + [dur or '']) + for lbl, cells, dur in rows_all] board_hdr = 'Board' - board_w = max([len(board_hdr)] + [len(lbl) for lbl, _ in rows_all]) - col_w = [max([len(h)] + [len(cell(cells, c)) for _, cells in rows_all]) - for h, c in zip(headers, columns)] + board_w = max([len(board_hdr)] + [len(lbl) for lbl, _ in rows_vals]) + col_w = [max([len(h)] + [len(vals[i]) for _, vals in rows_vals]) + for i, h in enumerate(headers)] def line(label, values): padded = [label.ljust(board_w)] + [v.center(w) for v, w in zip(values, col_w)] @@ -2060,7 +2128,7 @@ def render_matrix(rows_all: list) -> str: header = line(board_hdr, headers) sep = '| ' + '-' * board_w + ' | ' + ' | '.join(':' + '-' * (w - 2) + ':' for w in col_w) + ' |' - body = [line(lbl, [cell(cells, c) for c in columns]) for lbl, cells in rows_all] + body = [line(lbl, vals) for lbl, vals in rows_vals] # tally run cells (blank/not-run cells are absent from the dicts). A cell is a bare status # ('pass'/'fail'/'skip') or a metric string that carries its own icon (e.g. "❌ 29/30" is a @@ -2071,7 +2139,7 @@ def render_matrix(rows_all: list) -> str: if v == 'skip' or (isinstance(v, str) and v.startswith(REPORT_CELL['skip'])): return 'skip' return 'pass' - kinds = [cell_kind(v) for _, cells in rows_all for v in cells.values()] + kinds = [cell_kind(v) for _, cells, _ in rows_all for v in cells.values()] failed = kinds.count('fail') skipped = kinds.count('skip') passed = kinds.count('pass') @@ -2083,38 +2151,42 @@ def render_matrix(rows_all: list) -> str: def accumulate_report(mret: list, report_dir: Path, fresh: bool) -> str: """Merge this run's results into hil_report.json in report_dir, then (re)write - the markdown matrix to hil_report.md. `fresh` (a full run, no --skip-board/-bt) + the markdown matrix to hil_report.md. `fresh` (a full run, no --accumulate/-bt) starts a new report; otherwise a re-run accumulates so boards/tests that already passed are preserved while re-run cells are updated. Returns the md.""" - acc = {} # ordered {row_label: {example: status}} + acc = {} # ordered {row_label: [cells dict, duration str|None]} jpath = report_dir / REPORT_JSON if not fresh and jpath.is_file(): try: for entry in json.loads(jpath.read_text()).get('rows', []): - acc[entry['board']] = dict(entry['cells']) + acc[entry['board']] = [dict(entry['cells']), entry.get('duration')] except (ValueError, KeyError, TypeError): pass # corrupt/old sidecar: start fresh - # merge this run: current cells override prior for boards/tests that ran - for name, _, _, rows in mret: - if rows and not any('board-locked' in cells for _, cells in rows): + # merge this run: current cells override prior for boards/tests that ran; a filtered + # run reports duration None, keeping the previous full-run value + for name, _, _, rows, _ in mret: + if rows and not any('board-locked' in cells for _, cells, _ in rows): # board ran for real this time: clear a stale lock-failure cell # (its row is keyed by board name; test rows may be variant names) stale = acc.get(name) if stale is not None: - stale.pop('board-locked', None) - if not stale: + stale[0].pop('board-locked', None) + if not stale[0]: # variant-keyed boards never repopulate the board-name row — # drop it or it renders as a blank ghost row del acc[name] - for row_label, cells in rows: - acc.setdefault(row_label, {}).update(cells) + for row_label, cells, dur in rows: + row = acc.setdefault(row_label, [{}, None]) + row[0].update(cells) + if dur is not None: + row[1] = dur report_dir.mkdir(parents=True, exist_ok=True) - jpath.write_text(json.dumps({'rows': [{'board': k, 'cells': v} for k, v in acc.items()]}, - indent=2) + '\n') + jpath.write_text(json.dumps({'rows': [{'board': k, 'cells': c, 'duration': d} + for k, (c, d) in acc.items()]}, indent=2) + '\n') - md = render_matrix(list(acc.items())) + md = render_matrix([(k, c, d) for k, (c, d) in acc.items()]) (report_dir / REPORT_MD).write_text(md + '\n', encoding='utf-8') return md @@ -2135,7 +2207,14 @@ def main() -> None: parser = argparse.ArgumentParser() parser.add_argument('config_file', help='Configuration JSON file') parser.add_argument('-b', '--board', action='append', default=[], help='Boards to test, all if not specified') - parser.add_argument('-s', '--skip-board', action='append', default=[], help='Skip boards from test') + parser.add_argument('--flasher', action='append', default=[], + help='Only boards using these flashers, e.g. esptool ' + '(for splitting one config across CI jobs)') + parser.add_argument('--exclude-flasher', action='append', default=[], + help='Exclude boards using these flashers') + parser.add_argument('-a', '--accumulate', action='store_true', + help='Merge results into the existing report instead of starting fresh ' + '(re-runs; the .failed file starts with this)') parser.add_argument('-sf', '--skip-flash', action='store_true', help='Run tests without flashing firmware (use whatever is already on the board)') parser.add_argument('-t', '--test-only', action='append', default=[], help='Tests to run, all if not specified') parser.add_argument('-bt', '--board-test', action='append', default=[], @@ -2148,7 +2227,6 @@ def main() -> None: config_file = Path(args.config_file) boards = args.board - skip_boards = args.skip_board verbose = args.verbose test_only = args.test_only for entry in args.board_test: @@ -2167,7 +2245,7 @@ def main() -> None: config = cast(HilConfig, json.load(f)) if len(boards) == 0: - config_boards = [e for e in config['boards'] if e['name'] not in skip_boards] + config_boards = list(config['boards']) else: unknown = [b for b in boards if b not in {e['name'] for e in config['boards']}] if unknown: @@ -2175,6 +2253,8 @@ def main() -> None: print(f'ERROR: board(s) not in {config_file.name}: {", ".join(unknown)}') sys.exit(1) config_boards = [e for e in config['boards'] if e['name'] in boards] + config_boards = [e for e in config_boards if e['flasher']['name'] not in args.exclude_flasher + and (not args.flasher or e['flasher']['name'] in args.flasher)] build_err = 0 if args.build: @@ -2191,26 +2271,46 @@ def main() -> None: print(f'Build phase done: {build_err} failed') print('-' * 30) - # HIL report sidecar (hil_report.json/.md). A full run starts fresh; a re-run - # (--skip-board / -bt, i.e. the .skip file) accumulates so already-passed - # boards/tests are preserved. Clear any prior report up front on a fresh run so - # a crash mid-run can't leave stale results to be merged by a retry or posted. + # HIL report sidecar (hil_report.json/.md) and the .failed re-run spec live in + # report_dir (persists across CI run attempts). A full run starts fresh; a re-run + # (--accumulate / -bt, i.e. the .failed file) merges so already-passed boards/tests + # are preserved. Clear prior state up front on a fresh run so a crash mid-run can't + # leave a stale report - or worse, a stale re-run spec from another commit - to be + # consumed by a retry. report_dir = Path(os.environ.get('HIL_REPORT_DIR', '.')) - fresh = not (args.skip_board or args.board_test) + failed_fname = report_dir / (config_file.name + '.failed') + fresh = not (args.accumulate or args.board_test) if fresh: report_dir.mkdir(parents=True, exist_ok=True) for f in (REPORT_JSON, REPORT_MD): (report_dir / f).unlink(missing_ok=True) + failed_fname.unlink(missing_ok=True) + failed_fname.with_suffix(failed_fname.suffix + '.run').unlink(missing_ok=True) seed = os.getenv('HIL_SHUFFLE_SEED') or str(int(time.time())) log_line(f'test-order shuffle seed: {seed} (HIL_SHUFFLE_SEED={seed} to replay); ' f'flash/usbtest parallel per controller: {FLASH_PARALLEL}/{USBTEST_PARALLEL}; ' f'enum timeout first/retry: {ENUM_TIMEOUT}/{ENUM_TIMEOUT_RETRY}s') + + hints = {} + try: + with CONTROLLER_CACHE.open() as f: + loaded = json.load(f) + # tolerate a hand-edited/torn cache: keep only the expected uid -> dict shape + if isinstance(loaded, dict): + hints = {k: v for k, v in loaded.items() if isinstance(v, dict)} + except (OSError, ValueError): + pass + hints_by_uid = {uid: h['pci'] for uid, h in hints.items() if h.get('pci')} + config_boards = schedule_boards(config_boards, hints_by_uid) + log_line('dispatch order: ' + ', '.join(b['name'] for b in config_boards)) + mgr = Manager() + cmap = mgr.dict() initargs = (Lock(), seed, - [Semaphore(USBTEST_PARALLEL) for _ in range(CTRL_SLOTS)], - [Semaphore(FLASH_PARALLEL) for _ in range(CTRL_SLOTS)], - mgr.dict(), Lock()) + [Semaphore(USBTEST_PARALLEL) for _ in range(CONTROLLER_SLOTS)], + [Semaphore(FLASH_PARALLEL) for _ in range(CONTROLLER_SLOTS)], + cmap, Lock(), hints_by_uid) with Pool(processes=os.cpu_count() or 1, initializer=init_worker, initargs=initargs) as pool: async_ret = pool.map_async(test_board, config_boards) try: @@ -2221,17 +2321,66 @@ def main() -> None: raise RuntimeError(f'HIL worker pool timed out after {POOL_TIMEOUT}s') err_count = build_err + sum(e[1] for e in mret) - # generate skip list for next re-run if failed: skip boards that fully passed, - # and emit -bt BOARD:t1,t2 so each failed board only re-runs its own failed tests. - skip_fname = config_file.with_suffix(config_file.suffix + '.skip') - if err_count > 0: - skip_boards += [name for name, err, _, _ in mret if err == 0] - parts = [f'--skip-board {i}' for i in skip_boards] - parts += [f'-bt {name}:{",".join(fts)}' for name, err, fts, _ in mret if err > 0 and fts] - with skip_fname.open('w') as f: + # generate the re-run spec if anything failed: run ONLY the failed boards (-b), + # each restricted to its own failed tests (-bt); a board with failures but no + # test list (e.g. board-locked) re-runs entirely. --accumulate preserves the + # already-passed cells in the report. + parts = ['--accumulate'] + for name, err, fts, _, _ in mret: + if err > 0: + parts.append(f'-b {name}') + if fts: + parts.append(f'-bt {name}:{",".join(fts)}') + stamp_fname = failed_fname.with_suffix(failed_fname.suffix + '.run') + if len(parts) > 1: # build-only failures have no boards to re-run + report_dir.mkdir(parents=True, exist_ok=True) + with failed_fname.open('w') as f: f.write(' '.join(parts)) - elif skip_fname.exists(): - skip_fname.unlink() + # CI stamps the spec with its run id: a later run's retry must not consume a + # spec left by an attempt of a DIFFERENT run (e.g. attempt 1 skipped entirely) + stamp_fname.write_text(os.environ.get('GITHUB_RUN_ID', '')) + else: + failed_fname.unlink(missing_ok=True) + stamp_fname.unlink(missing_ok=True) + + # refresh controller hints: pci resolved this run, plus board durations when the + # full test list ran (a -t/-bt filtered run would understate the board's real cost) + try: + if PROFILE: + # debug snapshot of the run's live uid->PCI / PCI->slot resolutions + report_dir.mkdir(parents=True, exist_ok=True) + with (report_dir / 'hil_profile_ctrl.json').open('w') as f: + json.dump(dict(cmap), f, indent=1, sort_keys=True) + uid_of = {b['name']: b['uid'] for b in config['boards']} + for name, _, _, _, dur in mret: + uid = uid_of.get(name) + if uid is None: + continue + h = dict(hints.get(uid) or {}) + h['name'] = name # informational: cache is keyed by uid + h['pci'] = cmap.get(f'uid:{uid}') or h.get('pci') + if dur > 0: # test_board reports 0.0 for filtered (partial) runs + h['duration'] = round(dur, 1) + hints[uid] = h + # merge-on-write: another HIL job (e.g. the esp split) may have finished since + # our startup read - re-read and overlay only this run's boards so its entries + # survive, then replace atomically so a concurrent reader never sees a torn file + merged = {} + try: + with CONTROLLER_CACHE.open() as f: + cur = json.load(f) + if isinstance(cur, dict): + merged = {k: v for k, v in cur.items() if isinstance(v, dict)} + except (OSError, ValueError): + pass + merged.update({uid_of[n]: hints[uid_of[n]] for n, *_ in mret if n in uid_of}) + CONTROLLER_CACHE.parent.mkdir(parents=True, exist_ok=True) + tmp = CONTROLLER_CACHE.with_suffix('.json.tmp') + with tmp.open('w') as f: + json.dump(merged, f, indent=1, sort_keys=True) + tmp.replace(CONTROLLER_CACHE) + except OSError as e: + print(f'warning: cannot persist controller hints to {CONTROLLER_CACHE}: {e}') # board x test result matrix -> hil_report.md (accumulates across re-runs) + stdout report = accumulate_report(mret, report_dir, fresh) diff --git a/test/hil/tinyusb.json b/test/hil/tinyusb.json index 8ed33c8a2..6871d812f 100644 --- a/test/hil/tinyusb.json +++ b/test/hil/tinyusb.json @@ -22,7 +22,6 @@ { "name": "espressif_p4_function_ev-DMA", "flags": "-DCFG_TUD_DWC2_DMA_ENABLE=1 -DCFG_TUH_DWC2_DMA_ENABLE=1" } ], "tests": { - "comment": "espressif fleet build = IDF/FreeRTOS examples plus the IDF-buildable bare-metal-style ones tools/build.py allowlists (board_test, usbtest, video_capture)", "only": [ "device/cdc_msc_freertos", "device/hid_composite_freertos", @@ -152,51 +151,6 @@ } }, { - "name": "mimxrt1015_evk", - "uid": "DC28F865D2111D228D00B0543A70463C", - "tests": { - "device": true, - "host": false, - "dual": false - }, - "flasher": { - "name": "jlink", - "uid": "000726284213", - "args": "-device MIMXRT1015DAF5A" - } - }, - { - "name": "mimxrt1064_evk", - "uid": "BAE96FB95AFA6DBB8F00005002001200", - "tests": { - "skip": ["host/cdc_msc_hid"], - "comment-cdc-echo": "CH9102+Lexar bundle (moved here from stm32f723disco) mounts fine but echo returns nothing - TX-RX loopback jumper likely lost in the move; re-check wiring then re-enable", - "device": true, - "host": true, - "dual": true, - "dev_attached": [ - { - "vid_pid": "1a86_55d4", - "serial": "52D2003414", - "is_cdc": true - }, - { - "vid_pid": "21c4_0cc7", - "serial": "90005893730A1A63", - "is_msc": true, - "block_size": 512, - "block_count": 60620800, - "msc_inquiry": "Lexar USB Flash Drive PMAP" - } - ] - }, - "flasher": { - "name": "jlink", - "uid": "000725299165", - "args": "-device MIMXRT1064xxx6A" - } - }, - { "name": "lpcxpresso11u37", "uid": "17121919", "tests": { @@ -234,8 +188,6 @@ "device": true, "host": true, "dual": true, - "skip": ["host/cdc_msc_hid", "host/device_info", "host/msc_file_explorer", "host/msc_file_explorer_freertos", "dual/host_info_to_device_cdc"], - "comment-skip": "PIO-USB host port enumerates nothing since the board moves (CH340+UDisk bundle unplugged or unpowered) - re-attach the bundle then drop these skips", "dev_attached": [ { "vid_pid": "1a86_7523", @@ -550,10 +502,43 @@ "uid": "001050076405", "args": "-device NRF5340_XXAA_APP" } + } + ], + "boards-skip": [ + { + "name": "mimxrt1064_evk", + "uid": "BAE96FB95AFA6DBB8F00005002001200", + "comment-skip": "device-port cable degraded from enum drops to killing the uPD720201 mid-battery (2026-07-17); replace the cable, verify enum, then move back", + "tests": { + "device": true, + "host": true, + "dual": true, + "dev_attached": [ + { + "vid_pid": "1a86_55d4", + "serial": "52D2003414", + "is_cdc": true + }, + { + "vid_pid": "21c4_0cc7", + "serial": "90005893730A1A63", + "is_msc": true, + "block_size": 512, + "block_count": 60620800, + "msc_inquiry": "Lexar USB Flash Drive PMAP" + } + ] + }, + "flasher": { + "name": "jlink", + "uid": "000725299165", + "args": "-device MIMXRT1064xxx6A" + } }, { "name": "nrf54lm20dk", "uid": "899C3DE5B0F4D5CA", + "comment-skip": "J-Link probe fails most flashes (2026-07-16); replug/repair the probe, then move back", "tests": { "device": true, "host": false, @@ -566,9 +551,7 @@ "uid": "1051856258", "args": "-device NRF54LM20A_M33" } - } - ], - "boards-skip": [ + }, { "name": "ra6m5_ek", "uid": "8419032D32363657364EF4622D294B4E", |