Key HIL report dir by run id so re-runs and other PRs cannot clobber it

A re-run attempt merged into an empty base: another PR's HIL job ran
between attempt 1 and the retry and rewrote the shared hil_report.json,
so the run-stamp guard (correctly) refused the foreign base but the
full-fleet results were lost - the retry report contained only the
re-run cells.

Give each (run id, job) its own report dir instead:
- attempts of the same run share a dir, so the retry always finds its
  own sidecar and .failed spec intact
- interleaved runs of other PRs/jobs write elsewhere and cannot clobber
- the run-stamp mechanism (.failed.run file) becomes redundant and is
  removed
- stale per-run dirs are pruned after 2 weeks

2 files changed, 33 insertions, 30 deletions

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 818e7ba81..76e19ee02 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -318,20 +318,23 @@ jobs:
     env:
       HIL_JSON: ${{ matrix.hil_json }}
     steps:
-      - name: Set HIL report dir (sibling of workspace; persists across run attempts)
-        run: echo "HIL_REPORT_DIR=$(dirname "$GITHUB_WORKSPACE")/hil-report" >> "$GITHUB_ENV"
+      - name: Set HIL report dir (per run+job; persists across run attempts)
+        run: |
+          # one report dir per (run id, job): re-run attempts find their own report/spec,
+          # and interleaved runs of other PRs/jobs on the same runner cannot clobber them
+          BASE="$(dirname "$GITHUB_WORKSPACE")/hil-report"
+          # prune per-run dirs older than 2 weeks
+          find "$BASE" -mindepth 1 -maxdepth 1 -type d -mtime +14 -exec rm -rf {} + 2>/dev/null || true
+          echo "HIL_REPORT_DIR=$BASE/${GITHUB_RUN_ID}-$(basename "${{ matrix.display }}" .json)" >> "$GITHUB_ENV"
 
       - name: Get re-run spec from previous attempt
         if: github.run_attempt != '1'
         run: |
-          # only honor a spec stamped by THIS run: a spec left by another run (attempt 1
-          # died or was skipped before hil_test.py could clear it) must not be consumed
+          # the report dir is keyed by run id, so a spec here can only have been
+          # written by an earlier attempt of THIS run
           SPEC="$HIL_REPORT_DIR/$(basename "${{ env.HIL_JSON }}").failed"
-          if [ -f "$SPEC" ] && [ "$(cat "$SPEC.run" 2>/dev/null)" = "$GITHUB_RUN_ID" ]; then
-            RERUN_ARGS=$(cat "$SPEC")
-          else
-            RERUN_ARGS=""
-          fi
+          RERUN_ARGS=""
+          [ -f "$SPEC" ] && RERUN_ARGS=$(cat "$SPEC")
           echo "RERUN_ARGS=$RERUN_ARGS"
           echo "RERUN_ARGS=$RERUN_ARGS" >> $GITHUB_ENV
 
@@ -380,20 +383,23 @@ jobs:
       HIL_JSON: test/hil/tinyusb.json
       TEST_ARGS: '--flasher esptool'
     steps:
-      - name: Set HIL report dir (sibling of workspace; persists across run attempts)
-        run: echo "HIL_REPORT_DIR=$(dirname "$GITHUB_WORKSPACE")/hil-report-esp" >> "$GITHUB_ENV"
+      - name: Set HIL report dir (per run+job; persists across run attempts)
+        run: |
+          # one report dir per (run id, job): re-run attempts find their own report/spec,
+          # and interleaved runs of other PRs/jobs on the same runner cannot clobber them
+          BASE="$(dirname "$GITHUB_WORKSPACE")/hil-report"
+          # prune per-run dirs older than 2 weeks
+          find "$BASE" -mindepth 1 -maxdepth 1 -type d -mtime +14 -exec rm -rf {} + 2>/dev/null || true
+          echo "HIL_REPORT_DIR=$BASE/${GITHUB_RUN_ID}-tinyusb-esp" >> "$GITHUB_ENV"
 
       - name: Get re-run spec from previous attempt
         if: github.run_attempt != '1'
         run: |
-          # only honor a spec stamped by THIS run: a spec left by another run (attempt 1
-          # died or was skipped before hil_test.py could clear it) must not be consumed
+          # the report dir is keyed by run id, so a spec here can only have been
+          # written by an earlier attempt of THIS run
           SPEC="$HIL_REPORT_DIR/$(basename "${{ env.HIL_JSON }}").failed"
-          if [ -f "$SPEC" ] && [ "$(cat "$SPEC.run" 2>/dev/null)" = "$GITHUB_RUN_ID" ]; then
-            RERUN_ARGS=$(cat "$SPEC")
-          else
-            RERUN_ARGS=""
-          fi
+          RERUN_ARGS=""
+          [ -f "$SPEC" ] && RERUN_ARGS=$(cat "$SPEC")
           echo "RERUN_ARGS=$RERUN_ARGS"
           echo "RERUN_ARGS=$RERUN_ARGS" >> $GITHUB_ENV
 
diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py
index 448b236fc..0efc6826f 100755
--- a/test/hil/hil_test.py
+++ b/test/hil/hil_test.py
@@ -2158,7 +2158,10 @@ def accumulate_report(mret: list, report_dir: Path, fresh: bool) -> str:
     jpath = report_dir / REPORT_JSON
     if not fresh and jpath.is_file():
         try:
-            for entry in json.loads(jpath.read_text()).get('rows', []):
+            saved = json.loads(jpath.read_text())
+            # CI keys the report dir by run id, so the sidecar can only have been
+            # written by an earlier attempt of the same run
+            for entry in saved.get('rows', []):
                 acc[entry['board']] = [dict(entry['cells']), entry.get('duration')]
         except (ValueError, KeyError, TypeError):
             pass  # corrupt/old sidecar: start fresh
@@ -2272,11 +2275,11 @@ def main() -> None:
         print('-' * 30)
 
     # HIL report sidecar (hil_report.json/.md) and the .failed re-run spec live in
-    # report_dir (persists across CI run attempts). A full run starts fresh; a re-run
-    # (--accumulate / -bt, i.e. the .failed file) merges so already-passed boards/tests
-    # are preserved. Clear prior state up front on a fresh run so a crash mid-run can't
-    # leave a stale report - or worse, a stale re-run spec from another commit - to be
-    # consumed by a retry.
+    # report_dir (CI keys it by run id, so it persists across run attempts but is
+    # private to one run). A full run starts fresh; a re-run (--accumulate / -bt,
+    # i.e. the .failed file) merges so already-passed boards/tests are preserved.
+    # Clear prior state up front on a fresh run so a crash mid-run can't leave a
+    # stale report or re-run spec to be consumed by a retry.
     report_dir = Path(os.environ.get('HIL_REPORT_DIR', '.'))
     failed_fname = report_dir / (config_file.name + '.failed')
     fresh = not (args.accumulate or args.board_test)
@@ -2285,7 +2288,6 @@ def main() -> None:
         for f in (REPORT_JSON, REPORT_MD):
             (report_dir / f).unlink(missing_ok=True)
         failed_fname.unlink(missing_ok=True)
-        failed_fname.with_suffix(failed_fname.suffix + '.run').unlink(missing_ok=True)
 
     seed = os.getenv('HIL_SHUFFLE_SEED') or str(int(time.time()))
     log_line(f'test-order shuffle seed: {seed} (HIL_SHUFFLE_SEED={seed} to replay); '
@@ -2331,17 +2333,12 @@ def main() -> None:
                 parts.append(f'-b {name}')
                 if fts:
                     parts.append(f'-bt {name}:{",".join(fts)}')
-        stamp_fname = failed_fname.with_suffix(failed_fname.suffix + '.run')
         if len(parts) > 1:  # build-only failures have no boards to re-run
             report_dir.mkdir(parents=True, exist_ok=True)
             with failed_fname.open('w') as f:
                 f.write(' '.join(parts))
-            # CI stamps the spec with its run id: a later run's retry must not consume a
-            # spec left by an attempt of a DIFFERENT run (e.g. attempt 1 skipped entirely)
-            stamp_fname.write_text(os.environ.get('GITHUB_RUN_ID', ''))
         else:
             failed_fname.unlink(missing_ok=True)
-            stamp_fname.unlink(missing_ok=True)
 
     # refresh controller hints: pci resolved this run, plus board durations when the
     # full test list ran (a -t/-bt filtered run would understate the board's real cost)