test/hil: accumulate HIL report across re-runs; post as sticky PR comment

hil_test.py persists results in a hil_report.json sidecar and regenerates
hil_report.md from it. A full run starts fresh; a re-run (--skip-board / -bt,
i.e. the .skip file) merges into the existing report so already-passed
boards/tests are preserved while only re-run cells update. The report dir is
configurable via HIL_REPORT_DIR.

build.yml: each HIL rig writes the report to a workspace-sibling dir that
survives the per-attempt workspace clean, and uploads it as an artifact. A new
hil-report job merges the rigs' reports into one sticky PR comment (marocchino)
with one table per rig.

Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>

3 files changed, 99 insertions, 15 deletions

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index e5075f46f..0ff29cdda 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -306,6 +306,9 @@ jobs:
     env:
       HIL_JSON: ${{ matrix.hil_json }}
     steps:
+      - name: Set HIL report dir (sibling of workspace; persists across run attempts)
+        run: echo "HIL_REPORT_DIR=$(dirname "$GITHUB_WORKSPACE")/hil-report" >> "$GITHUB_ENV"
+
       - name: Get Skip Boards from previous run
         if: github.run_attempt != '1'
         run: |
@@ -344,6 +347,15 @@ jobs:
             exit 1
           fi)
 
+      - name: Upload HIL report
+        if: always() && github.event_name == 'pull_request'
+        uses: actions/upload-artifact@v7
+        with:
+          name: hil-report-${{ matrix.display }}
+          path: ${{ env.HIL_REPORT_DIR }}/hil_report.md
+          if-no-files-found: ignore
+          overwrite: true
+
   # ---------------------------------------
   # Hardware in the loop (HIL)
   # self-hosted by HFP, build with IAR toolchain, for attached hardware checkout test/hil/hfp.json
@@ -390,3 +402,45 @@ jobs:
       - name: Test on actual hardware (hardware in the loop)
         run: |
           python3 test/hil/hil_test.py hfp.json
+
+  # ---------------------------------------
+  # Combine HIL results from the rigs into a single sticky PR comment (one table per rig)
+  # ---------------------------------------
+  hil-report:
+    needs: hil-tinyusb
+    if: |
+      always() &&
+      needs.hil-tinyusb.result != 'skipped' &&
+      github.event_name == 'pull_request' &&
+      github.repository_owner == 'hathach' &&
+      github.event.pull_request.head.repo.fork == false
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+    steps:
+      - name: Download HIL reports
+        uses: actions/download-artifact@v5
+        with:
+          pattern: hil-report-*
+          path: hil-reports
+
+      - name: Combine rig reports (one table per rig)
+        run: |
+          {
+            echo "## HIL test results"
+            echo
+            for d in hil-reports/hil-report-*; do
+              [ -d "$d" ] || continue
+              echo "### ${d#hil-reports/hil-report-}"
+              echo
+              cat "$d/hil_report.md" 2>/dev/null || echo "_no report produced_"
+              echo
+            done
+          } > hil_combined.md
+          cat hil_combined.md
+
+      - name: Post HIL report as sticky PR comment
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          header: hil-report
+          path: hil_combined.md
diff --git a/.gitignore b/.gitignore
index ba1574558..0f3c9ce49 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,7 @@
 html
 latex
 hil_report.md
+hil_report.json
 *.a
 *.d
 *.o
diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py
index 2758d093c..515c20e75 100755
--- a/test/hil/hil_test.py
+++ b/test/hil/hil_test.py
@@ -1694,18 +1694,17 @@ def test_board(board: Board) -> tuple[str, int, list[str], list]:
     return name, err_count, sorted(set(failed_tests)), rows
 
 
-def generate_report(mret: list) -> str:
-    """Build a markdown matrix (rows = boards, columns = tests) from test_board
-    results. Each mret entry is (name, err, failed_tests, rows) where rows is a
-    list of (row_label, {example: status}). Columns are padded so the raw table
-    is aligned in plain text (boards left-aligned, test cells centered)."""
+REPORT_MD = 'hil_report.md'
+REPORT_JSON = 'hil_report.json'
+
+
+def render_matrix(rows_all: list) -> str:
+    """Render rows (list of (row_label, {example: status})) as an aligned markdown
+    matrix: columns = tests (bare names) centered, boards left-aligned."""
     canonical = device_tests + dual_tests + host_test + ['device/board_test']
-    rows_all = []  # flattened (row_label, cells), preserving board/f1 order
     seen = set()
-    for _, _, _, rows in mret:
-        for row_label, cells in rows:
-            rows_all.append((row_label, cells))
-            seen.update(cells)
+    for _, cells in rows_all:
+        seen.update(cells)
     if not seen:
         return 'No tests were run.'
 
@@ -1734,6 +1733,34 @@ def generate_report(mret: list) -> str:
     return '\n'.join([header, sep] + body) + '\n\n' + legend
 
 
+def accumulate_report(mret: list, report_dir: Path, fresh: bool) -> str:
+    """Merge this run's results into hil_report.json in report_dir, then (re)write
+    the markdown matrix to hil_report.md. `fresh` (a full run, no --skip-board/-bt)
+    starts a new report; otherwise a re-run accumulates so boards/tests that
+    already passed are preserved while re-run cells are updated. Returns the md."""
+    acc = {}  # ordered {row_label: {example: status}}
+    jpath = report_dir / REPORT_JSON
+    if not fresh and jpath.is_file():
+        try:
+            for entry in json.loads(jpath.read_text()).get('rows', []):
+                acc[entry['board']] = dict(entry['cells'])
+        except (ValueError, KeyError, TypeError):
+            pass  # corrupt/old sidecar: start fresh
+
+    # merge this run: current cells override prior for boards/tests that ran
+    for _, _, _, rows in mret:
+        for row_label, cells in rows:
+            acc.setdefault(row_label, {}).update(cells)
+
+    report_dir.mkdir(parents=True, exist_ok=True)
+    jpath.write_text(json.dumps({'rows': [{'board': k, 'cells': v} for k, v in acc.items()]},
+                                indent=2) + '\n')
+
+    md = render_matrix(list(acc.items()))
+    (report_dir / REPORT_MD).write_text(md + '\n', encoding='utf-8')
+    return md
+
+
 def main() -> None:
     """
     Hardware test on specified boards
@@ -1823,13 +1850,15 @@ def main() -> None:
         elif skip_fname.exists():
             skip_fname.unlink()
 
-    # board x test result matrix -> hil_report.md and stdout
-    report = generate_report(mret)
-    report_path = Path('hil_report.md')
-    report_path.write_text(report + '\n', encoding='utf-8')
+    # board x test result matrix -> hil_report.md (accumulates across re-runs) + stdout.
+    # A full run starts fresh; a re-run (--skip-board / -bt, i.e. the .skip file) merges
+    # into the existing report so already-passed boards/tests are preserved.
+    report_dir = Path(os.environ.get('HIL_REPORT_DIR', '.'))
+    fresh = not (args.skip_board or args.board_test)
+    report = accumulate_report(mret, report_dir, fresh)
     print()
     print(report)
-    print(f'\nReport written to {report_path.resolve()}')
+    print(f'\nReport written to {(report_dir / REPORT_MD).resolve()}')
 
     duration = time.time() - duration
     print()