From 39c717dad8caa31567aae8473386f41097e725bc Mon Sep 17 00:00:00 2001 From: Tom Rini Date: Tue, 18 Nov 2025 15:00:12 -0600 Subject: Gitlab: Add a "sage-lab" stage to access a board farm This is the Gitlab side of adding support for the board lab connected to the "konsulko-sage" runner. On the software side, this lab uses only upstream labgrid. On the hardware side, each device under test is connected to its own exporter (typically a Raspberry Pi 4) that must be turned on (and cleanly turned off) as part of a given test cycle. Add support for testing on a SolidRun Hummingboard 2 (imx6), Raspberry Pi 3 and Raspberry Pi 4. In all cases, we enable additional options to run more tests on the board. As we have some networking tests, we test both the legacy network stack and lwIP. In the case of Pi platforms, we test all of 32bit configuration, plain configuration and rpi_arm64, and again with and without lwIP. Signed-off-by: Tom Rini --- .gitlab-ci-sage-lab.yml | 186 ++++++++++++++++++++++++++++++++++++++++++++++++ .gitlab-ci.yml | 5 ++ 2 files changed, 191 insertions(+) create mode 100644 .gitlab-ci-sage-lab.yml diff --git a/.gitlab-ci-sage-lab.yml b/.gitlab-ci-sage-lab.yml new file mode 100644 index 00000000000..d29af92565a --- /dev/null +++ b/.gitlab-ci-sage-lab.yml @@ -0,0 +1,186 @@ +# Definition for the lab connected to the "sage" host and managed with labgrid + +.sage_lab_template: &sage_lab_dfn + stage: sage-lab + rules: + - if: $SAGE_LAB == "1" + when: always + - if: $SAGE_LAB != "1" + when: manual + allow_failure: true + # USB can be unreliable. + retry: 2 + dependencies: [] + needs: [ "sandbox test.py" ] + tags: [ 'konsulko-sage-labgrid' ] + before_script: + # Clone uboot-test-hooks + - git config --global --add safe.directory "${CI_PROJECT_DIR}" + - git clone --depth=1 https://source.denx.de/u-boot/u-boot-test-hooks /tmp/uboot-test-hooks + - ln -s sage /tmp/uboot-test-hooks/bin/`hostname` + - ln -s sage /tmp/uboot-test-hooks/py/`hostname` + - export LG_ENV="/tmp/uboot-test-hooks/bin/sage/labgrid-sage-env.yaml"; + export LG_COORDINATOR=172.17.0.1:20408 + # Prepare python environment + - python3 -m venv /tmp/venv; + . /tmp/venv/bin/activate; + pip install -r test/py/requirements.txt -r tools/binman/requirements.txt + -r tools/buildman/requirements.txt -r tools/u_boot_pylib/requirements.txt + labgrid setuptools + # Add more tools + - sudo apt-get update && sudo apt-get -y install snmp rsync + # Acquire and turn on the exporter. + - labgrid-client reserve --wait board=${LABGRID_EXPORTER} && + labgrid-client -p ${LABGRID_EXPORTER} acquire && + labgrid-client -p ${LABGRID_EXPORTER} power on + # Prepare ssh + - mkdir --mode=0700 ~/.ssh; + printf 'Host sage-exporter-*\n\tUser labgrid\n' > ~/.ssh/config + script: + - export UBOOT_TRAVIS_BUILD_DIR=/tmp/${TEST_PY_BD} + # Don't stop on non-zero exit codes now, to provide as much chance as + # possible to ensure we don't leave lab resources acquired. + - set +e; + # Sage is 16 threads and has 3 devices attached, so 6 builder threads. + - tools/buildman/buildman -o ${UBOOT_TRAVIS_BUILD_DIR} -w -E -W -e + --board ${TEST_PY_BD} ${OVERRIDE} -T 6; + RC=`echo $?`; + # Wait for the exporter to have the device ready + - while test `labgrid-client resources | + grep -q ${LABGRID_EXPORTER}/${LG_PLACE}/; echo $?` -ne 0; do sleep 1; + done + # Fingerprint the exporter + - ssh-keyscan ${LABGRID_EXPORTER} > ~/.ssh/known_hosts 2>/dev/null + # If we built, run the tests and save the results. + - export PATH=/tmp/uboot-test-hooks/bin:${PATH}; + export PYTHONPATH=/tmp/uboot-test-hooks/py/sage; + test $RC -eq 0 && labgrid-client reserve --wait board=${LG_PLACE} && + labgrid-client -p ${LG_PLACE} acquire && + ./test/py/test.py -ra --bd ${TEST_PY_BD} ${TEST_PY_ID} + ${TEST_PY_EXTRA:-"--capture=tee-sys"} + ${TEST_PY_TEST_SPEC:+"-k ${TEST_PY_TEST_SPEC}"} + --build-dir "$UBOOT_TRAVIS_BUILD_DIR" + --junitxml=/tmp/${TEST_PY_BD}/results.xml; + RC=`echo $?`; + cp -v /tmp/${TEST_PY_BD}/*.{html,css,xml} .; + # Release and power off everything. + - labgrid-client -p ${LG_PLACE} power off + - labgrid-client -p ${LG_PLACE} release + - labgrid-client -p ${LABGRID_EXPORTER} ssh sudo poweroff && sleep 15 && + labgrid-client -p ${LABGRID_EXPORTER} power off + - labgrid-client -p ${LABGRID_EXPORTER} release + - exit $RC + artifacts: + when: always + paths: + - "*.html" + - "*.css" + - results.xml + reports: + junit: results.xml + expire_in: 1 week + +SolidRun Hummingboard 2: + <<: *sage_lab_dfn + variables: + LABGRID_EXPORTER: "sage-exporter-hb-1" + LG_PLACE: "hb-1" + TEST_PY_BD: "mx6cuboxi" + OVERRIDE: "-a UNIT_TEST -a ~CMD_EFIDEBUG -a CMD_BOOTMENU -a CMD_LOG" + +SolidRun Hummingboard 2 (lwIP): + <<: *sage_lab_dfn + needs: [ "SolidRun Hummingboard 2" ] + variables: + LABGRID_EXPORTER: "sage-exporter-hb-1" + LG_PLACE: "hb-1" + TEST_PY_BD: "mx6cuboxi" + OVERRIDE: "-a UNIT_TEST -a ~CMD_EFIDEBUG -a CMD_BOOTMENU -a CMD_LOG -a NET_LWIP" + +Raspberry Pi 4: + <<: *sage_lab_dfn + variables: + LABGRID_EXPORTER: "sage-exporter-rpi4-1" + LG_PLACE: "rpi4-1" + TEST_PY_BD: "rpi_4" + OVERRIDE: "-a UNIT_TEST -a ~CMD_EFIDEBUG -a CMD_BOOTMENU -a CMD_LOG -a ~CMD_BOOTEFI_SELFTEST -a CMD_TFTPPUT -a FIT -a FIT_SIGNATURE -a FIT_BEST_MATCH -a SYS_BOOTM_LEN=0x4000000 -a BOOTSTAGE -a BOOTSTAGE_STASH -a CMD_BOOTSTAGE -a BOOTSTAGE_STASH_ADDR=0x02400000" + +Raspberry Pi 4 (rpi_arm64): + <<: *sage_lab_dfn + needs: [ "Raspberry Pi 4" ] + variables: + LABGRID_EXPORTER: "sage-exporter-rpi4-1" + LG_PLACE: "rpi4-1" + TEST_PY_BD: "rpi_arm64" + OVERRIDE: "-a UNIT_TEST -a ~CMD_EFIDEBUG -a CMD_BOOTMENU -a CMD_LOG -a ~CMD_BOOTEFI_SELFTEST -a CMD_TFTPPUT -a FIT -a FIT_SIGNATURE -a FIT_BEST_MATCH -a SYS_BOOTM_LEN=0x4000000 -a BOOTSTAGE -a BOOTSTAGE_STASH -a CMD_BOOTSTAGE -a BOOTSTAGE_STASH_ADDR=0x02400000" + +Raspberry Pi 4 (rpi_arm64, lwIP): + <<: *sage_lab_dfn + needs: [ "Raspberry Pi 4 (rpi_arm64)" ] + variables: + LABGRID_EXPORTER: "sage-exporter-rpi4-1" + LG_PLACE: "rpi4-1" + TEST_PY_BD: "rpi_arm64" + OVERRIDE: "-a UNIT_TEST -a ~CMD_EFIDEBUG -a CMD_BOOTMENU -a CMD_LOG -a ~CMD_BOOTEFI_SELFTEST -a FIT -a FIT_SIGNATURE -a FIT_BEST_MATCH -a SYS_BOOTM_LEN=0x4000000 -a BOOTSTAGE -a BOOTSTAGE_STASH -a CMD_BOOTSTAGE -a BOOTSTAGE_STASH_ADDR=0x02400000 -a NET_LWIP" + +Raspberry Pi 4 (rpi_4_32b): + <<: *sage_lab_dfn + needs: [ "Raspberry Pi 4 (rpi_arm64, lwIP)" ] + variables: + LABGRID_EXPORTER: "sage-exporter-rpi4-1" + LG_PLACE: "rpi4-1" + TEST_PY_BD: "rpi_4_32b" + OVERRIDE: "-a UNIT_TEST -a ~CMD_EFIDEBUG -a CMD_BOOTMENU -a CMD_LOG -a CMD_BOOTEFI_SELFTEST -a CMD_TFTPPUT -a FIT -a FIT_SIGNATURE -a FIT_BEST_MATCH -a SYS_BOOTM_LEN=0x4000000 -a BOOTSTAGE -a BOOTSTAGE_STASH -a CMD_BOOTSTAGE -a BOOTSTAGE_STASH_ADDR=0x02400000" + +Raspberry Pi 4 (rpi_4_32b, lwIP): + <<: *sage_lab_dfn + needs: [ "Raspberry Pi 4 (rpi_4_32b)" ] + variables: + LABGRID_EXPORTER: "sage-exporter-rpi4-1" + LG_PLACE: "rpi4-1" + TEST_PY_BD: "rpi_4_32b" + OVERRIDE: "-a UNIT_TEST -a ~CMD_EFIDEBUG -a CMD_BOOTMENU -a CMD_LOG -a CMD_BOOTEFI_SELFTEST -a FIT -a FIT_SIGNATURE -a FIT_BEST_MATCH -a SYS_BOOTM_LEN=0x4000000 -a BOOTSTAGE -a BOOTSTAGE_STASH -a CMD_BOOTSTAGE -a BOOTSTAGE_STASH_ADDR=0x02400000 -a NET_LWIP" + +Raspberry Pi 3: + <<: *sage_lab_dfn + variables: + LABGRID_EXPORTER: "sage-exporter-rpi3-1" + LG_PLACE: "rpi3-1" + TEST_PY_BD: "rpi_3" + OVERRIDE: "-a UNIT_TEST -a ~CMD_EFIDEBUG -a CMD_BOOTMENU -a CMD_LOG -a CMD_BOOTEFI_SELFTEST -a CMD_TFTPPUT -a FIT -a FIT_SIGNATURE -a FIT_BEST_MATCH -a SYS_BOOTM_LEN=0x4000000 -a BOOTSTAGE -a BOOTSTAGE_STASH -a CMD_BOOTSTAGE -a BOOTSTAGE_STASH_ADDR=0x02400000" + +Raspberry Pi 3 (rpi_arm64): + <<: *sage_lab_dfn + needs: [ "Raspberry Pi 3" ] + variables: + LABGRID_EXPORTER: "sage-exporter-rpi3-1" + LG_PLACE: "rpi3-1" + TEST_PY_BD: "rpi_arm64" + OVERRIDE: "-a UNIT_TEST -a ~CMD_EFIDEBUG -a CMD_BOOTMENU -a CMD_LOG -a ~CMD_BOOTEFI_SELFTEST -a CMD_TFTPPUT -a FIT -a FIT_SIGNATURE -a FIT_BEST_MATCH -a SYS_BOOTM_LEN=0x4000000 -a BOOTSTAGE -a BOOTSTAGE_STASH -a CMD_BOOTSTAGE -a BOOTSTAGE_STASH_ADDR=0x02400000" + +Raspberry Pi 3 (rpi_arm64, lwIP): + <<: *sage_lab_dfn + needs: [ "Raspberry Pi 3 (rpi_arm64)" ] + variables: + LABGRID_EXPORTER: "sage-exporter-rpi3-1" + LG_PLACE: "rpi3-1" + TEST_PY_BD: "rpi_arm64" + OVERRIDE: "-a UNIT_TEST -a ~CMD_EFIDEBUG -a CMD_BOOTMENU -a CMD_LOG -a ~CMD_BOOTEFI_SELFTEST -a FIT -a FIT_SIGNATURE -a FIT_BEST_MATCH -a SYS_BOOTM_LEN=0x4000000 -a BOOTSTAGE -a BOOTSTAGE_STASH -a CMD_BOOTSTAGE -a BOOTSTAGE_STASH_ADDR=0x02400000 -a NET_LWIP" + +Raspberry Pi 3 (rpi_3_32b): + <<: *sage_lab_dfn + needs: [ "Raspberry Pi 3 (rpi_arm64, lwIP)" ] + variables: + LABGRID_EXPORTER: "sage-exporter-rpi3-1" + LG_PLACE: "rpi3-1" + TEST_PY_BD: "rpi_3_32b" + OVERRIDE: "-a UNIT_TEST -a ~CMD_EFIDEBUG -a CMD_BOOTMENU -a CMD_LOG -a CMD_BOOTEFI_SELFTEST -a CMD_TFTPPUT -a FIT -a FIT_SIGNATURE -a FIT_BEST_MATCH -a SYS_BOOTM_LEN=0x4000000 -a BOOTSTAGE -a BOOTSTAGE_STASH -a CMD_BOOTSTAGE -a BOOTSTAGE_STASH_ADDR=0x02400000" + +Raspberry Pi 3 (rpi_3_32b, lwIP): + <<: *sage_lab_dfn + needs: [ "Raspberry Pi 3 (rpi_3_32b)" ] + variables: + LABGRID_EXPORTER: "sage-exporter-rpi3-1" + LG_PLACE: "rpi3-1" + TEST_PY_BD: "rpi_3_32b" + OVERRIDE: "-a UNIT_TEST -a ~CMD_EFIDEBUG -a CMD_BOOTMENU -a CMD_LOG -a CMD_BOOTEFI_SELFTEST -a FIT -a FIT_SIGNATURE -a FIT_BEST_MATCH -a SYS_BOOTM_LEN=0x4000000 -a BOOTSTAGE -a BOOTSTAGE_STASH -a CMD_BOOTSTAGE -a BOOTSTAGE_STASH_ADDR=0x02400000 -a NET_LWIP" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5e9995aed6a..8aab61dcb62 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,6 +5,7 @@ variables: DEFAULT_AMD64_TAG: "amd64" DEFAULT_FAST_TAG: "fast" MIRROR_DOCKER: docker.io + SAGE_LAB: "" SJG_LAB: "" PLATFORM: linux/amd64,linux/arm64 @@ -24,6 +25,7 @@ image: ${MIRROR_DOCKER}/trini/u-boot-gitlab-ci-runner:noble-20251001-11Nov2025 stages: - testsuites - test.py + - sage-lab - sjg-lab - world build @@ -593,6 +595,9 @@ coreboot test.py: TEST_PY_ID: "--id qemu" <<: *buildman_and_testpy_dfn +# Add sage-lab stage +include: .gitlab-ci-sage-lab.yml + .sjg_lab_template: &sjg_lab_dfn stage: sjg-lab rules: -- cgit v1.2.3 From 0c4990c5b84431db53f88292905b5aedd7d1ff28 Mon Sep 17 00:00:00 2001 From: Tom Rini Date: Tue, 18 Nov 2025 15:00:13 -0600 Subject: Docker: Add building TF-A for pine64_plus In order to add pine64_plus to the sage lab we will need to have a build of TF-A available for it as bl31.bin. Add this to the existing build loop of TF-A targets. Signed-off-by: Tom Rini --- tools/docker/Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile index 2dc4bec33bb..fd8765af458 100644 --- a/tools/docker/Dockerfile +++ b/tools/docker/Dockerfile @@ -276,6 +276,10 @@ RUN git clone --depth=1 https://git.trustedfirmware.org/TF-A/trusted-firmware-a. mkdir -p /opt/tf-a/qemu_arm64_fw_handoff_tfa_optee && \ cp build/qemu/release/fip.bin build/qemu/release/bl1.bin \ /opt/tf-a/qemu_arm64_fw_handoff_tfa_optee/ && \ + make CROSS_COMPILE=/opt/gcc-${TCVER}-nolibc/aarch64-linux/bin/aarch64-linux- \ + PLAT=sun50i_a64 -j$(nproc) all && \ + mkdir -p /opt/tf-a/pine64_plus && \ + cp build/sun50i_a64/release/bl31.bin /opt/tf-a/pine64_plus/ && \ rm -rf /tmp/optee_os && \ rm -rf /tmp/mbedtls && \ rm -rf /tmp/tf-a -- cgit v1.2.3 From 04c913a3672848c95c15c0fb560e0f6de2fdd220 Mon Sep 17 00:00:00 2001 From: Tom Rini Date: Tue, 18 Nov 2025 15:00:14 -0600 Subject: Gitlab: sage: Add Pine64+ platform This adds the Pine64+ platform to the sage lab, for both legacy and lwIP networking stacks. In order to build this platform we need to copy certain files that were built in the container to /tmp and then set BINMAN_INDIRS to /tmp in order to find them when building. For now, we disable the test_net_pxe_boot_config test on lwIP as it leads to a crash that needs to be investigated. Signed-off-by: Tom Rini --- .gitlab-ci-sage-lab.yml | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci-sage-lab.yml b/.gitlab-ci-sage-lab.yml index d29af92565a..911132b6b00 100644 --- a/.gitlab-ci-sage-lab.yml +++ b/.gitlab-ci-sage-lab.yml @@ -36,14 +36,21 @@ # Prepare ssh - mkdir --mode=0700 ~/.ssh; printf 'Host sage-exporter-*\n\tUser labgrid\n' > ~/.ssh/config + # If we have TF-A binaries, we need to use them. + - if [[ -d /opt/tf-a/${TEST_PY_BD}${TEST_PY_ID//--id /_} ]]; then + cp /opt/tf-a/${TEST_PY_BD}${TEST_PY_ID//--id /_}/* /tmp/; + elif [[ -d /opt/tf-a/${TEST_PY_BD} ]]; then + cp /opt/tf-a/${TEST_PY_BD}/* /tmp/; + fi; + export BINMAN_INDIRS=/tmp; script: - export UBOOT_TRAVIS_BUILD_DIR=/tmp/${TEST_PY_BD} # Don't stop on non-zero exit codes now, to provide as much chance as # possible to ensure we don't leave lab resources acquired. - set +e; - # Sage is 16 threads and has 3 devices attached, so 6 builder threads. + # Sage is 16 threads and has 4 devices attached, so 4 builder threads. - tools/buildman/buildman -o ${UBOOT_TRAVIS_BUILD_DIR} -w -E -W -e - --board ${TEST_PY_BD} ${OVERRIDE} -T 6; + --board ${TEST_PY_BD} ${OVERRIDE} -T 4; RC=`echo $?`; # Wait for the exporter to have the device ready - while test `labgrid-client resources | @@ -80,6 +87,25 @@ junit: results.xml expire_in: 1 week +Pine64+: + <<: *sage_lab_dfn + variables: + LABGRID_EXPORTER: "sage-exporter-pine64-1" + LG_PLACE: "pine64-1" + TEST_PY_BD: "pine64_plus" + OVERRIDE: "-a CMD_BOOTMENU -a CMD_LOG" + +Pine64+ (lwIP): + <<: *sage_lab_dfn + needs: [ "Pine64+" ] + variables: + LABGRID_EXPORTER: "sage-exporter-pine64-1" + LG_PLACE: "pine64-1" + TEST_PY_BD: "pine64_plus" + # Leads to crash on lwIP, needs investigation. + TEST_PY_TEST_SPEC: "not test_net_pxe_boot_config" + OVERRIDE: "-a CMD_BOOTMENU -a CMD_LOG -a NET_LWIP" + SolidRun Hummingboard 2: <<: *sage_lab_dfn variables: -- cgit v1.2.3 From f84a6d94c7bd28f5d450f997b395e94b1cff36ae Mon Sep 17 00:00:00 2001 From: Tom Rini Date: Tue, 18 Nov 2025 15:00:15 -0600 Subject: Gitlab sage, Docker: Add snmp and rsync Add snmp and rsync to the normal docker image. While these tools are only required for the lab on sage, I think it's a small enough addition to be worth always including at this point. A higher level TODO I think may be to see if we can / should split the resulting container up, or if there's host tooling we can remove at a later step, after building all the software we require. Signed-off-by: Tom Rini --- .gitlab-ci-sage-lab.yml | 2 -- tools/docker/Dockerfile | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci-sage-lab.yml b/.gitlab-ci-sage-lab.yml index 911132b6b00..f2fa0e3e771 100644 --- a/.gitlab-ci-sage-lab.yml +++ b/.gitlab-ci-sage-lab.yml @@ -27,8 +27,6 @@ pip install -r test/py/requirements.txt -r tools/binman/requirements.txt -r tools/buildman/requirements.txt -r tools/u_boot_pylib/requirements.txt labgrid setuptools - # Add more tools - - sudo apt-get update && sudo apt-get -y install snmp rsync # Acquire and turn on the exporter. - labgrid-client reserve --wait board=${LABGRID_EXPORTER} && labgrid-client -p ${LABGRID_EXPORTER} acquire && diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile index fd8765af458..bcb95a8cae3 100644 --- a/tools/docker/Dockerfile +++ b/tools/docker/Dockerfile @@ -131,7 +131,9 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ python3-tomli \ python3-venv \ rpm2cpio \ + rsync \ sbsigntool \ + snmp \ socat \ softhsm2 \ sparse \ -- cgit v1.2.3