Add RTT console/capture tooling (tools/rtt.py), rtt skill, and HIL harness support (#3853)

Promote SEGGER RTT from an inline debugging technique to a standalone skill
backed by one stdlib-only implementation in tools/rtt.py: a CLI and importable
module for console/capture over J-Link (RTTTelnetPort) and OpenOCD (rtt server)
probes, with probe selection by serial or VID:PID, control-block address via
--elf or --addr, bidirectional console, post-mortem ring dump, and
--reset-before-attach for boot-time capture. The HIL harness reads a board's
console over RTT when its probe has no VCOM ("logger": "rtt" plus a LOGGER=rtt
variant define), covering device_info, pool-check aliveness, and CI wiring.
Validated on 22 boards across both backends; 26 unit tests run in pre-commit.

1 files changed, 126 insertions, 26 deletions

diff --git a/test/hil/hil_test.py b/test/hil/hil_test.py
index e32998420..233627ec7 100755
--- a/test/hil/hil_test.py
+++ b/test/hil/hil_test.py
@@ -206,6 +206,7 @@ class Board(TypedDict):
     # needs one carries a single variant named after itself (metro_m4_express /
     # MAX3421_HOST=1), which is exactly what the `or [...]` default below synthesises
     variant: NotRequired[list[VariantCfg]]
+    logger: NotRequired[str]  # "rtt": console = the debug probe's RTT channel 0, not a VCOM (rtt skill)
     toolchain: NotRequired[str]  # CI build bucket override, e.g. "riscv-gcc" (consumed by hil_ci_set_matrix.py)
 
 
@@ -292,6 +293,25 @@ def open_serial_dev(port: str):
     return ser
 
 
+def open_board_console(board: Board):
+    """The board's log console: its probe's VCOM, or RTT when the probe has none.
+
+    Both ends expose the same read/in_waiting/write/close surface, so the tests read one
+    the same way they read the other."""
+    if board.get('logger') == 'rtt':
+        # JlinkRtt speaks JLinkExe only; an openocd/stlink flasher would yield
+        # `-device ''` and fail 15 s later with a misleading port error. The OpenOCD
+        # RTT route is validated manually on native probes but has no harness backend
+        # yet (rtt skill; followup doc) — and never point it at ea4088's LPC-Link2
+        # (measured: knocks that probe off USB; other J-Link-OB probes untested)
+        assert board['flasher']['name'].lower() == 'jlink', \
+            f'{board["name"]}: "logger": "rtt" needs a jlink flasher, not {board["flasher"]["name"]}'
+        return hil_util.JlinkRtt(board)
+    ser = open_serial_dev(hil_util.get_serial_dev(board['flasher']["uid"], None, None, 0))
+    ser.timeout = 0.1
+    return ser
+
+
 def serial_write_all(ser: serial.Serial, data: bytes):
     # write_timeout is a deadline for the whole call. A timeout means the device stopped
     # draining, and it is fatal: pyserial loses the partial-write count on raise, so
@@ -300,8 +320,19 @@ def serial_write_all(ser: serial.Serial, data: bytes):
         ser.write(data)
     except serial.SerialTimeoutException:
         raise AssertionError(f'Serial write timeout after {SERIAL_WRITE_TIMEOUT:.1f}s')
+    except hil_util.RttError as e:
+        # the RTT console's failure contract (stall/closed/peer death): same
+        # drain-stopped meaning as the serial timeout -- a test failure, not a harness
+        # crash. Deliberately NOT bare RuntimeError: NotImplementedError and CPython's
+        # own 'dictionary changed size during iteration' are RuntimeErrors too, and a
+        # harness bug must not be reported as this board misbehaving.
+        raise AssertionError(f'Console write failed: {e}')
 
 
+# J-Link Commander's telnet greeting: never target output (defined with the console
+# in tools/rtt.py; hil_pool_check strips it through the same object)
+RTT_BANNER_RE = hil_util.RTT_BANNER_RE
+
 LP_OPEN_TIMEOUT = 5   # bound on opening the printer lp node; see test_device_printer_to_cdc
 # Runs under hil_util.run_alongside as `python3 -c`. Inline rather than a file so hil_ci.sh's
 # staging list does not need another entry to keep the rig working.
@@ -508,34 +539,53 @@ def test_host_device_info(board):
     flasher = board['flasher']
     declared_devs = [f'{d["vid_pid"]}_{d["serial"]}' for d in board['tests']['dev_attached']]
 
-    port = hil_util.get_serial_dev(flasher["uid"], None, None, 0)
-    ser = open_serial_dev(port)
-    ser.timeout = 0.1
-
-    # reset device since we can miss the first line
-    ret = getattr(hil_flash, f'reset_{flasher["name"].lower()}')(board)
-    assert ret.returncode == 0, 'Failed to reset device'
+    if board.get('logger') == 'rtt':
+        # The RTT console owns the probe, so reset BEFORE opening it (Commander then
+        # delivers the buffered boot burst). Unconditional, not only under --skip-flash:
+        # a previous run's console drained the ring, and the enumeration lines print
+        # only once — without this a re-run on unchanged firmware reads an empty ring.
+        ret = getattr(hil_flash, f'reset_{flasher["name"].lower()}')(board)
+        assert ret.returncode == 0, 'Failed to reset device'
+    ser = open_board_console(board)
+    try:
+        if board.get('logger') != 'rtt':
+            # reset device since we can miss the first line; on the VCOM the console
+            # survives the reset, so resetting after open catches the boot banner.
+            ret = getattr(hil_flash, f'reset_{flasher["name"].lower()}')(board)
+            assert ret.returncode == 0, 'Failed to reset device'
 
-    data = b''
-    timeout = enum_timeout()
-    while timeout > 0:
-        new_data = ser.read(ser.in_waiting or 1)
-        if new_data:
-            data += new_data
-        enum_dev_sn = []
-        for l in data.decode('utf-8', errors='ignore').splitlines():
-            vid_pid_sn = re.search(r'ID ([0-9a-fA-F]+):([0-9a-fA-F]+) SN (\w+)', l)
-            if vid_pid_sn:
-                enum_dev_sn.append(f'{vid_pid_sn.group(1)}_{vid_pid_sn.group(2)}_{vid_pid_sn.group(3)}')
-        if set(declared_devs).issubset(set(enum_dev_sn)):
-            break
-        time.sleep(0.1)
-        timeout -= 0.1
-    ser.close()
+        data = b''
+        timeout = enum_timeout()
+        while timeout > 0:
+            # infra death is not a board failure: without this a dead JLinkExe/probe
+            # would burn the whole timeout and report as 'No data from device'
+            assert not getattr(ser, 'eof', False), \
+                'RTT console died (its server exited or the probe dropped off USB)'
+            new_data = ser.read(ser.in_waiting or 1)
+            if new_data:
+                data += new_data
+            enum_dev_sn = []
+            for l in data.decode('utf-8', errors='ignore').splitlines():
+                vid_pid_sn = re.search(r'ID ([0-9a-fA-F]+):([0-9a-fA-F]+) SN (\w+)', l)
+                if vid_pid_sn:
+                    enum_dev_sn.append(f'{vid_pid_sn.group(1)}_{vid_pid_sn.group(2)}_{vid_pid_sn.group(3)}')
+            if set(declared_devs).issubset(set(enum_dev_sn)):
+                break
+            time.sleep(0.1)
+            timeout -= 0.1
+    finally:
+        ser.close()
 
-    if len(data) == 0:
-        assert False, 'No data from device'
     lines = data.decode('utf-8', errors='ignore').splitlines()
+    if board.get('logger') == 'rtt':
+        # JLinkExe's telnet banner is delivered at connect, whether or not it ever
+        # finds the control block, so len(data) alone cannot tell "board said nothing"
+        # from "console never attached to the ring" -- drop the banner first
+        target_lines = hil_util.strip_banner(data).splitlines()
+        assert target_lines, ('No data from device: the RTT console attached but the target '
+                              'produced nothing -- firmware built without LOGGER=rtt, or SWD lost')
+    elif len(data) == 0:
+        assert False, 'No data from device'
 
     enum_dev_sn = []
     for l in lines:
@@ -1729,7 +1779,7 @@ def test_example(board: Board, variant: str, example: str) -> tuple[int, str, st
 
 def build_board(board: Board) -> tuple[str, int]:
     """Build firmware for this board via tools/build.py.
-    Honors board config's variant list (name, defines, flags).
+    Honors board config's variant list.
     Output goes to cmake-build/cmake-build-<variant>/ (tools/build.py layout).
 
     Unbounded on purpose: --build is a local convenience (no CI workflow passes it), so
@@ -2337,6 +2387,56 @@ def main() -> None:
         config_boards = [e for e in config['boards'] if e['name'] in boards]
     config_boards = [e for e in config_boards if e['flasher']['name'] not in args.exclude_flasher
                      and (not args.flasher or e['flasher']['name'] in args.flasher)]
+
+    # fail rtt misconfigurations before the first flash cycle -- but only for boards
+    # this run actually touches: one bad roster entry must not abort other runs' subsets
+    def _rtt_config_abort(msg: str):
+        # loud AND leaving evidence, like the no-boards branch below: exiting with no
+        # report at all lets the PR comment keep the previous push's stale table
+        print(f'ERROR: {msg}', flush=True)
+        rd = Path(os.environ.get('HIL_REPORT_DIR', '.'))
+        hil_report.mark_report_no_boards(rd, f'config error: {msg}', fresh=not args.accumulate)
+        sys.exit(1)
+
+    bad_logger = [e['name'] for e in config_boards if e.get('logger') not in (None, 'rtt')]
+    if bad_logger:
+        # only the exact string activates RTT handling; anything else would silently
+        # mean VCOM and reproduce the misleading 'No serial device found' failure
+        _rtt_config_abort(f'unknown "logger" value (only "rtt" is supported): {", ".join(bad_logger)}')
+    bad_rtt = [e['name'] for e in config_boards
+               if e.get('logger') == 'rtt' and e['flasher']['name'].lower() != 'jlink']
+    if bad_rtt:
+        # JlinkRtt speaks JLinkExe only (the OpenOCD RTT route is manual — rtt skill)
+        _rtt_config_abort(f'"logger": "rtt" needs a jlink flasher: {", ".join(bad_rtt)}')
+    rtt_no_logger_def = [e['name'] for e in config_boards
+                         if e.get('logger') == 'rtt'
+                         and any('LOGGER=rtt' not in (v.get('defines') or [])
+                                 for v in (e.get('variant') or [{}]))]
+    if rtt_no_logger_def:
+        # a prebuilt cmake-build-<board> configured with -DLOGGER=rtt is a legitimate
+        # build path the roster need not describe, so warn there -- but when this run is
+        # responsible for the firmware (--build, or CI where the hil-build job compiled
+        # the artifact from these same defines) the flashed image is UART-logger and every
+        # test times out as 'the target produced nothing'. An always-on define is
+        # expressed as a single self-named variant (see the Board comment).
+        msg = (f'"logger": "rtt" board has a variant without LOGGER=rtt in its defines '
+               f'({", ".join(rtt_no_logger_def)})')
+        if args.build or os.environ.get('GITHUB_ACTIONS'):
+            _rtt_config_abort(f'{msg} -- the firmware built for this run cannot serve the '
+                              f'configured RTT console')
+        print(f'warning: {msg} -- fine for prebuilt example sets, wrong for --build/CI '
+              f'builds', flush=True)
+    rtt_fixture = [e['name'] for e in config_boards
+                   if e.get('logger') == 'rtt'
+                   and any(d.get('is_cdc') or d.get('is_msc')
+                           for d in e.get('tests', {}).get('dev_attached', []))]
+    if rtt_fixture:
+        # interim guard, removed when the followup lands: cdc_msc_hid/msc_file_explorer
+        # still open the flasher VCOM directly and would die mid-run on an rtt board
+        _rtt_config_abort(f'"logger": "rtt" boards cannot carry is_cdc/is_msc fixtures yet '
+                          f'(host cdc/msc tests bypass the RTT console — see '
+                          f'the rtt harness-adoption doc in docs/superpowers/followup/): {", ".join(rtt_fixture)}')
+
     if not config_boards:
         # same reason the unknown -b board exits 1: 'No tests were run.' with rc 0 reads as
         # a green HIL leg, so a roster edit emptying a leg's filter stops testing silently