test/hil: report board x test results as a markdown matrix

hil_test.py now writes hil_report.md and prints it to stdout: rows are
boards, columns are tests (bare example names), cells are pass/fail/skip.
test_example returns a per-test status, test_board collects a board x test
grid (one row per flags-on variant), and main() renders an aligned table.
A missing binary counts as skipped. hil_ci.sh copies the report back from
the remote after a run; hil_report.md is gitignored.

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>

2 files changed, 90 insertions, 15 deletions

diff --git a/test/hil/hil_ci.sh b/test/hil/hil_ci.sh
index 4bb459af3..4f68ed067 100644
--- a/test/hil/hil_ci.sh
+++ b/test/hil/hil_ci.sh
@@ -88,12 +88,21 @@ fi
 # parameters; quoting and metacharacters in args are preserved.
 CONFIG_BASENAME="$(basename "$CONFIG")"
 echo "==> Running HIL test on $REMOTE"
-ssh "$REMOTE" bash -s -- "$REMOTE_DIR" "${ARGS[@]}" "test/hil/$CONFIG_BASENAME" <<'REMOTE'
+rc=0
+ssh "$REMOTE" bash -s -- "$REMOTE_DIR" "${ARGS[@]}" "test/hil/$CONFIG_BASENAME" <<'REMOTE' || rc=$?
 cd -- "$1"
 shift
 # Flasher CLIs live in the user bin dirs on ci.lan (esptool/idf in ~/.local/bin,
 # STM32CubeProgrammer's STM32_Programmer_CLI in ~/bin); the non-interactive shell
 # subprocess used for flashing doesn't source profile/rc, so add them explicitly.
 export PATH="$HOME/.local/bin:$HOME/bin:$PATH"
-exec python3 -u test/hil/hil_test.py -B examples "$@"
+python3 -u test/hil/hil_test.py -B examples "$@"
 REMOTE
+
+# Copy the generated report back to the local checkout (best-effort; the run's
+# exit code is preserved regardless of whether a report was produced).
+scp -q "$REMOTE:$REMOTE_DIR/hil_report.md" "$ROOT_DIR/hil_report.md" \
+  && echo "==> Report copied to $ROOT_DIR/hil_report.md" \
+  || echo "==> warning: no hil_report.md copied back" >&2
+
+exit $rc
diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py
index 609199d1b..2fb5f6b3f 100755
--- a/test/hil/hil_test.py
+++ b/test/hil/hil_test.py
@@ -65,6 +65,10 @@ STATUS_OK = "\033[32mOK\033[0m"
 STATUS_FAILED = "\033[31mFailed\033[0m"
 STATUS_SKIPPED = "\033[33mSkipped\033[0m"
 
+# Plain (non-ANSI) cell symbols for the markdown matrix report (hil_report.md).
+# A missing binary is reported as skipped too.
+REPORT_CELL = {'pass': '✔', 'fail': '✖', 'skip': '➖'}
+
 verbose = False
 test_only = []
 board_test = {}
@@ -1490,20 +1494,26 @@ host_test = [
 ]
 
 
-def test_example(board: Board, f1: str, example: str) -> int:
+def f1_suffix(f1: str) -> str:
+    """Build dir / row-label suffix for a flags-on variant ('' for the default)."""
+    return '-f1_' + f1.replace(' ', '_') if f1 else ''
+
+
+def test_example(board: Board, f1: str, example: str) -> tuple[int, str]:
     """
     Test example firmware
     :param board: board dict
     :param f1: flags on
     :param example: example name
-    :return: 0 if success/skip, 1 if failed
+    :return: (err_count, status) where err_count is 0 on success/skip or 1 on
+             failure, and status is one of 'pass'/'fail'/'skip' (a missing
+             binary counts as 'skip')
     """
     name = board['name']
     err_count = 0
+    result_status = 'fail'
 
-    f1_str = ""
-    if f1 != "":
-        f1_str = '-f1_' + f1.replace(' ', '_')
+    f1_str = f1_suffix(f1)
 
     fw_dir = TINYUSB_ROOT / build_dir / f'cmake-build-{name}{f1_str}' / example
     fw_name = fw_dir / Path(example).name
@@ -1511,7 +1521,7 @@ def test_example(board: Board, f1: str, example: str) -> int:
 
     if not fw_dir.exists() or not ((fw_name.with_suffix('.elf')).exists() or (fw_name.with_suffix('.bin')).exists()):
         log_line(f'{test_name} Skip (no binary)')
-        return 0
+        return 0, 'skip'
 
     if verbose:
         log_line(f'Flashing {fw_name}.elf')
@@ -1534,8 +1544,10 @@ def test_example(board: Board, f1: str, example: str) -> int:
                     last_detail = compact_output(attempt_out.getvalue())
                     if tret == 'skipped':
                         status = STATUS_SKIPPED
+                        result_status = 'skip'
                     else:
                         status = STATUS_OK
+                        result_status = 'pass'
                     msg = f'{test_name}  {status}'
                     if last_detail:
                         msg += f'  {last_detail}'
@@ -1578,7 +1590,7 @@ def test_example(board: Board, f1: str, example: str) -> int:
         msg += f'  in {time.time() - start_s:.1f}s'
         log_line(msg)
 
-    return err_count
+    return err_count, result_status
 
 
 def build_board(board: Board) -> tuple[str, int]:
@@ -1607,7 +1619,7 @@ def build_board(board: Board) -> tuple[str, int]:
     return name, failed
 
 
-def test_board(board: Board) -> tuple[str, int, list[str]]:
+def test_board(board: Board) -> tuple[str, int, list[str], list]:
     name = board['name']
     flasher = board['flasher']
 
@@ -1648,22 +1660,68 @@ def test_board(board: Board) -> tuple[str, int, list[str]]:
 
     err_count = 0
     failed_tests = []
+    rows = []  # list of (row_label, {example: status}) — one row per board[-f1] variant
     flags_on_list = [""]
     if 'build' in board and 'flags_on' in board['build']:
         flags_on_list = board['build']['flags_on']
 
     for f1 in flags_on_list:
+        cells = {}
         for test in test_list:
-            ec = test_example(board, f1, test)
+            ec, status = test_example(board, f1, test)
             err_count += ec
+            cells[test] = status
             if ec > 0:
                 failed_tests.append(test)
+        rows.append((name + f1_suffix(f1), cells))
 
     # flash board_test last to disable board's usb (skipped when --skip-flash is set)
     if not skip_flash:
-        test_example(board, flags_on_list[0], 'device/board_test')
+        _ec, status = test_example(board, flags_on_list[0], 'device/board_test')
+        if rows:
+            rows[0][1]['device/board_test'] = status
+
+    return name, err_count, sorted(set(failed_tests)), rows
+
+
+def generate_report(mret: list) -> str:
+    """Build a markdown matrix (rows = boards, columns = tests) from test_board
+    results. Each mret entry is (name, err, failed_tests, rows) where rows is a
+    list of (row_label, {example: status}). Columns are padded so the raw table
+    is aligned in plain text (boards left-aligned, test cells centered)."""
+    canonical = device_tests + dual_tests + host_test + ['device/board_test']
+    rows_all = []  # flattened (row_label, cells), preserving board/f1 order
+    seen = set()
+    for _, _, _, rows in mret:
+        for row_label, cells in rows:
+            rows_all.append((row_label, cells))
+            seen.update(cells)
+    if not seen:
+        return 'No tests were run.'
+
+    # columns: canonical order first, then any extras (e.g. from -t) alphabetically
+    columns = [t for t in canonical if t in seen]
+    columns += [t for t in sorted(seen) if t not in canonical]
+    headers = [c.rsplit('/', 1)[-1] for c in columns]  # bare example name
 
-    return name, err_count, sorted(set(failed_tests))
+    def cell(cells, col):
+        return REPORT_CELL.get(cells.get(col), '')
+
+    board_hdr = 'Board'
+    board_w = max([len(board_hdr)] + [len(lbl) for lbl, _ in rows_all])
+    col_w = [max([len(h)] + [len(cell(cells, c)) for _, cells in rows_all])
+             for h, c in zip(headers, columns)]
+
+    def line(label, values):
+        padded = [label.ljust(board_w)] + [v.center(w) for v, w in zip(values, col_w)]
+        return '| ' + ' | '.join(padded) + ' |'
+
+    header = line(board_hdr, headers)
+    sep = '| ' + '-' * board_w + ' | ' + ' | '.join(':' + '-' * (w - 2) + ':' for w in col_w) + ' |'
+    body = [line(lbl, [cell(cells, c) for c in columns]) for lbl, cells in rows_all]
+
+    legend = 'Legend: ✔ pass · ✖ fail · ➖ skipped · blank not run'
+    return '\n'.join([header, sep] + body) + '\n\n' + legend
 
 
 def main() -> None:
@@ -1747,14 +1805,22 @@ def main() -> None:
         # and emit -bt BOARD:t1,t2 so each failed board only re-runs its own failed tests.
         skip_fname = config_file.with_suffix(config_file.suffix + '.skip')
         if err_count > 0:
-            skip_boards += [name for name, err, _ in mret if err == 0]
+            skip_boards += [name for name, err, _, _ in mret if err == 0]
             parts = [f'--skip-board {i}' for i in skip_boards]
-            parts += [f'-bt {name}:{",".join(fts)}' for name, err, fts in mret if err > 0 and fts]
+            parts += [f'-bt {name}:{",".join(fts)}' for name, err, fts, _ in mret if err > 0 and fts]
             with skip_fname.open('w') as f:
                 f.write(' '.join(parts))
         elif skip_fname.exists():
             skip_fname.unlink()
 
+    # board x test result matrix -> hil_report.md and stdout
+    report = generate_report(mret)
+    report_path = Path('hil_report.md')
+    report_path.write_text(report + '\n', encoding='utf-8')
+    print()
+    print(report)
+    print(f'\nReport written to {report_path.resolve()}')
+
     duration = time.time() - duration
     print()
     print("-" * 30)