Merge pull request #3166 from hathach/hil-skip-passed-board-in-rerun

try to skip passed board in hil test when re-run

2 files changed, 39 insertions, 15 deletions

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 5e11f8a29..ff59421ce 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -199,17 +199,20 @@ jobs:
     runs-on: [self-hosted, X64, hathach, hardware-in-the-loop]
     steps:
       - name: Clean workspace
+        if: github.run_attempt == '1'
         run: |
-          echo "Cleaning up previous run"
+          echo "Cleaning up for the first run"
           rm -rf "${{ github.workspace }}"
           mkdir -p "${{ github.workspace }}"
 
       - name: Checkout TinyUSB
+        if: github.run_attempt == '1'
         uses: actions/checkout@v4
         with:
           sparse-checkout: test/hil
 
       - name: Download Artifacts
+        if: github.run_attempt == '1'
         uses: actions/download-artifact@v4
         with:
           path: cmake-build
@@ -218,7 +221,15 @@ jobs:
       - name: Test on actual hardware
         run: |
           ls cmake-build/
-          python3 test/hil/hil_test.py ${{ env.HIL_JSON }}
+
+          # Skip boards that passed with previous run, file is generated by hil_test.py
+          SKIP_BOARDS=""
+          if [ -f ${{ env.HIL_JSON }}.skip ]; then
+            SKIP_BOARDS=$(cat "${HIL_JSON}.skip")
+          fi
+          echo "SKIP_BOARDS=$SKIP_BOARDS"
+
+          python3 test/hil/hil_test.py ${{ env.HIL_JSON }} $SKIP_BOARDS
 
   # ---------------------------------------
   # Hardware in the loop (HIL)
diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py
index f292bca15..93cc0a750 100755
--- a/test/hil/hil_test.py
+++ b/test/hil/hil_test.py
@@ -533,7 +533,7 @@ def test_example(board, f1, example):
     if not os.path.exists(fw_dir):
         fw_dir = f'{TINYUSB_ROOT}/examples/cmake-build-{name}{f1_str}/{example}'
     fw_name = f'{fw_dir}/{os.path.basename(example)}'
-    print(f'{name+f1_str:40} {example:30} ... ', end='')
+    print(f'{name+f1_str:40} {example:30} ...', end='')
 
     if not os.path.exists(fw_dir) or not (os.path.exists(f'{fw_name}.elf') or os.path.exists(f'{fw_name}.bin')):
         print('Skip (no binary)')
@@ -544,29 +544,30 @@ def test_example(board, f1, example):
 
     # flash firmware. It may fail randomly, retry a few times
     max_rety = 3
+    start_s = time.time()
     for i in range(max_rety):
         ret = globals()[f'flash_{board["flasher"]["name"].lower()}'](board, fw_name)
         if ret.returncode == 0:
             try:
                 globals()[f'test_{example.replace("/", "_")}'](board)
-                print('OK')
+                print('  OK', end='')
                 break
             except Exception as e:
                 if i == max_rety - 1:
                     err_count += 1
-                    print(STATUS_FAILED)
-                    print(f'  {e}')
+                    print(f'{STATUS_FAILED}: {e}')
                 else:
-                    print()
-                    print(f'  Test failed: {e}, retry {i+2}/{max_rety}')
-                    time.sleep(1)
+                    print(f'\n  Test failed: {e}, retry {i+2}/{max_rety}', end='')
+                    time.sleep(0.5)
         else:
-            print(f'Flashing failed, retry {i+2}/{max_rety}')
-            time.sleep(1)
+            print(f'\n  Flash failed, retry {i+2}/{max_rety}', end='')
+            time.sleep(0.5)
 
     if ret.returncode != 0:
         err_count += 1
-        print(f'Flash {STATUS_FAILED}')
+        print(f'  Flash {STATUS_FAILED}', end='')
+
+    print(f'  in {time.time() - start_s:.1f}s')
 
     return err_count
 
@@ -606,7 +607,7 @@ def test_board(board):
     # flash board_test last to disable board's usb
     test_example(board, flags_on_list[0], 'device/board_test')
 
-    return err_count
+    return name, err_count
 
 
 def main():
@@ -620,11 +621,13 @@ def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('config_file', help='Configuration JSON file')
     parser.add_argument('-b', '--board', action='append', default=[], help='Boards to test, all if not specified')
+    parser.add_argument('-s', '--skip', action='append', default=[], help='Skip boards from test')
     parser.add_argument('-v', '--verbose', action='store_true', help='Verbose output')
     args = parser.parse_args()
 
     config_file = args.config_file
     boards = args.board
+    skip_boards = args.skip
     verbose = args.verbose
 
     # if config file is not found, try to find it in the same directory as this script
@@ -634,12 +637,22 @@ def main():
         config = json.load(f)
 
     if len(boards) == 0:
-        config_boards = config['boards']
+        config_boards = [e for e in config['boards'] if e['name'] not in skip_boards]
     else:
         config_boards = [e for e in config['boards'] if e['name'] in boards]
 
+    err_count = 0
     with Pool(processes=os.cpu_count()) as pool:
-        err_count = sum(pool.map(test_board, config_boards))
+        mret = pool.map(test_board, config_boards)
+        err_count = sum(e[1] for e in mret)
+        # generate skip list for next re-run if failed
+        skip_fname = f'{config_file}.skip'
+        if err_count > 0:
+            skip_boards += [name for name, err in mret if err == 0]
+            with open(skip_fname, 'w') as f:
+                f.write(' '.join(f'-s {i}' for i in skip_boards))
+        elif os.path.exists(skip_fname):
+            os.remove(skip_fname)
 
     duration = time.time() - duration
     print()