path: root/.github/workflows/regression_template.yml

# This is a basic workflow that is manually triggered

name: regression_template

on:
  workflow_call:
    inputs:
      install_script:
        default: './scripts/install.sh'
        required: false
        type: string
      build_script:
        default: './scripts/build.sh'
        required: false
        type: string
      test_script:
        default: './scripts/test.sh'
        required: false
        type: string
      cmake_path:
        default: './test/cmake'
        required: false
        type: string
      skip_test:
        default: false
        required: false
        type: boolean
      skip_coverage:
        default: false
        required: false
        type: boolean
      # The merged report, not one configuration's. Every configuration is
      # instrumented now (TX_COVERAGE below) and coverage.sh --merge unions
      # them; default_build_coverage was one build of five, and the code the
      # other four select was absent from its denominator rather than uncovered
      # in it.
      coverage_name:
        default: 'merged'
        required: false
        type: string
      skip_deploy:
        default: false
        required: false
        type: boolean
      deploy_list:
        default: ''
        required: false
        type: string
      result_affix:
        default: ''
        required: false
        type: string
      # Extra options for the container the tests run in, empty for every job
      # that does not ask for them.
      #
      # It exists for one measured case. NetX Duo's two interoperability suites
      # build a veth pair per test and address it, which needs CAP_NET_ADMIN in
      # the container's network namespace. A container started with the default
      # capability set holds CAP_NET_RAW, so tcpdump and libpcap work, and does
      # not hold CAP_NET_ADMIN, so "ip link add ... type veth" fails with
      # "RTNETLINK answers: Operation not permitted" on the first test of every
      # configuration -- measured, not predicted.
      #
      # A caller that needs it passes "--cap-add=NET_ADMIN" and nothing else
      # changes. Granting it here for every component instead would widen the
      # privileges of five suites that have no use for it.
      container_options:
        default: ''
        required: false
        type: string

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  # This workflow contains a single job called "linux_job"
  run_tests:
    if: ${{ !inputs.skip_test}}
    permissions:
      contents: read
      issues: read
      checks: write
      pull-requests: write
      
    # The type of runner that the job will run on. Pinned rather than "latest",
    # for the same reason the container below is.
    runs-on: ubuntu-24.04

    # The build environment is pinned, because the coverage numbers this job
    # produces are certification evidence and have to be reproducible from a
    # recorded version rather than from whatever the hosted image carries today.
    #
    # The kernel sources declare cmake_minimum_required(VERSION 3.0.0), and CMake
    # removed compatibility below 3.5 in 4.0. On ubuntu-latest with no pin, this
    # branch would stop configuring the day GitHub moves that image to CMake 4.x,
    # with no change here and no warning. Debian trixie carries CMake 3.31.6, so
    # the distribution package settles it with no policy override and no edit to
    # the sources.
    #
    # trixie is also GCC 14, and gcc-multilib with it: the coverage build is 32
    # bit (-m32 in cmake/linux.cmake and in both test trees), so a compiler
    # without multilib cannot build this suite at all.
    #
    # The tag is pinned by digest as well as by name. A tag is a pointer Debian
    # moves on every point release, so a certificate that names one names an
    # image that can change underneath it; a digest cannot move. This is the
    # image the published coverage figures were measured on.
    #
    # It fixes the base and not the whole environment: the toolchain is
    # installed from the Debian archive at run time, so gcc, CMake and Ninja
    # still arrive at whatever version the archive carries. scripts/install.sh
    # prints and asserts all three, which is what makes a drifting toolchain
    # fail the job by name and number rather than quietly produce evidence built
    # with something else.
    container:
      image: debian:trixie@sha256:f324c7ff54321e8d9c588493a20244965938ce0aa50bbd1022d38010e9ffc4b1
      options: ${{ inputs.container_options }}

    # Steps represent a sequence of tasks that will be executed as part of the job
    steps:
    # Before the checkout, because actions/checkout needs git and the pinned
    # image is a bare Debian. Without it the action silently falls back to
    # downloading a tarball over the REST API, which leaves no .git behind -- and
    # the commit this evidence was built from is part of the evidence.
    - name: Install git
      run: |
        apt-get update
        apt-get install -y --no-install-recommends ca-certificates git

    - name: Check out the repository
      uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      with:
        submodules: true
    
    - name: Install softwares
      run: ${{ inputs.install_script }}

    # TX_COVERAGE instruments every build configuration rather than only the one
    # whose name ends in _coverage. It has to be set for the build as well as the
    # test: the build is where -fprofile-arcs is decided, and the test run is
    # where the reports are collected and merged.
    - name: Build
      env:
        TX_COVERAGE: ${{ inputs.skip_coverage && 'OFF' || 'ON' }}
      run: ${{ inputs.build_script }}

    # CTEST_REPEAT_FAIL gives ctest --repeat until-pass:N, and the runner script
    # defaults it to 2. One attempt is what this branch wants, because a second
    # attempt reports an intermittent failure as a pass and these runs are
    # certification evidence. Measured on this commit: over 112 configuration
    # runs, four were not green on the first attempt -- two SMP tests that hung
    # for the full 1000-second timeout and two that failed outright -- and all
    # fourteen runs reported success. A retry is a reasonable default for
    # day-to-day CI and cannot be one here, where a green run has to mean the
    # suite passed rather than that it passed eventually. Reading it back out of
    # the logs afterwards is not an equivalent: a timed-out attempt prints its
    # own 1000-second line, so the run's total matches the sum of its test times
    # and the discrepancy that would betray it does not exist.
    - name: Test
      env:
        TX_COVERAGE: ${{ inputs.skip_coverage && 'OFF' || 'ON' }}
        CTEST_REPEAT_FAIL: 1
      run: ${{ inputs.test_script }}
      
    - name: Publish Test Results
      uses: EnricoMi/publish-unit-test-result-action@d0a4676d0e0b938bc201470d88276b7c74c712b3 # v2.24.0
      if: always()
      with:
        check_name: Test Results ${{ inputs.result_affix }}
        files: |
          ${{ inputs.cmake_path }}/build/*/*.xml
          
    - name: Upload Test Results
      if: success() || failure()
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
      with:
        name: test_reports ${{ inputs.result_affix }}
        path: |
          ${{ inputs.cmake_path }}/build/*.txt
          ${{ inputs.cmake_path }}/build/*/Testing/**/*.xml
          ${{ inputs.cmake_path }}/build/**/regression/output_files/*.bin
    
    - name: Configure GitHub Pages
      uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0

    # The coverage steps below run even when a test failed. cmake_bootstrap.sh
    # now produces the report in that case, and the run whose behaviour changed
    # is the one whose coverage is worth reading; previously a single flaky test
    # suppressed the report for the whole run. !cancelled() rather than always(),
    # so a cancelled run still stops promptly -- the same idiom the
    # deploy_code_coverage job below already uses. The ${{ }} is required: a bare
    # ! opens a YAML tag, and the expression will not parse without it.
    - name: Generate Code Coverage Results Summary
      if: ${{ !cancelled() && (!inputs.skip_coverage) }}
      uses: irongut/CodeCoverageSummary@51cc3a756ddcd398d447c044c02cb6aa83fdae95 # v1.3.0
      with:
        filename: ${{ inputs.cmake_path }}/coverage_report/${{ inputs.coverage_name }}.xml
        format: markdown
        badge: true
        hide_complexity: true
        output: file

    - name: Write Code Coverage Summary
      if: ${{ !cancelled() && (!inputs.skip_coverage) }}
      run: |
        echo "## Coverage Report ${{ inputs.result_affix }}" >> $GITHUB_STEP_SUMMARY
        cat code-coverage-results.md >> $GITHUB_STEP_SUMMARY

    - name: Create CheckRun for Code Coverage
      if: ${{ !cancelled() && ((github.event_name == 'push') || (github.event_name == 'workflow_dispatch') || (github.event.pull_request.head.repo.full_name == github.repository)) && (!inputs.skip_coverage) }}
      uses: LouisBrunner/checks-action@937cbbcde3259005b50746dc91cde29098aac2ff # v3.1.0
      with:
        token: ${{ secrets.GITHUB_TOKEN }}
        name: Code Coverage ${{ inputs.result_affix }}
        conclusion: ${{ job.status }}
        output: |
          {"summary":"Coverage Report"}
        output_text_description_file: code-coverage-results.md

    - name: Add Code Coverage PR Comment
      if: ${{ !cancelled() && ((github.event_name == 'push') || (github.event.pull_request.head.repo.full_name == github.repository)) && (!inputs.skip_coverage) }}
      uses: marocchino/sticky-pull-request-comment@5770ad5eb8f42dd2c4f34da00c94c5381e49af88 # v3.0.5
      with:
        header: Code Coverage ${{ inputs.result_affix }}
        path: code-coverage-results.md

    # No sudo: the steps above run as root inside the pinned container, and the
    # image does not carry sudo at all.
    - name: Prepare Coverage GitHub Pages
      if: ${{ !cancelled() && (!inputs.skip_coverage) }}
      run: >-
        if [ "${{ inputs.result_affix }}" != "" ] && ${{ inputs.skip_deploy }}; then
          mv ${{ inputs.cmake_path }}/coverage_report/${{ inputs.coverage_name }} \
            ${{ inputs.cmake_path }}/coverage_report/${{ inputs.result_affix }}
        fi
          
    # Every suite used to upload an artifact called coverage_report and rely on
    # them merging. upload-artifact refuses a name another job has already used
    # from v4 onwards, so the name is made unique here and the deploy job
    # collects them by pattern.
    - name: Coverage Report name
      id: artifact
      if: ${{ !cancelled() && (!inputs.skip_coverage) }}
      run: echo "coverage_report=coverage_report-$(date +%s)" >> $GITHUB_OUTPUT

    # per_configuration is excluded deliberately. The deploy job merges every
    # coverage_report-* artifact into one tree, and each suite's per-configuration
    # directories carry the same names, so ThreadX's would overwrite SMP's. The
    # top level therefore holds only the suite directory and the merged XML,
    # which is the shape the deploy already expects.
    - name: Upload Code Coverage Artifacts
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
      if: ${{ !cancelled() && (inputs.skip_deploy && !inputs.skip_coverage) }}
      with:
        name: ${{ steps.artifact.outputs.coverage_report }}
        path: |
          ${{ inputs.cmake_path }}/coverage_report
          !${{ inputs.cmake_path }}/coverage_report/per_configuration/**
        # A day is the right life for a report that exists to answer "what moved
        # in this run". On the certification branch the same report is evidence,
        # and evidence an assessor cannot fetch from the run that produced it is
        # evidence that has to be taken on trust. That branch therefore keeps
        # them for the repository's own default of 90 days, which is already
        # what the test results above get, and every other branch is unchanged.
        retention-days: ${{ github.ref == 'refs/heads/v6.4.1_cert' && 90 || 1 }}

    # The per-configuration reports, kept separately so they are downloadable
    # when the merged number moves and the question is which configuration moved
    # it. It is a distinct artifact name, so the deploy job never picks it up and
    # it does not reach the published site.
    - name: Upload Per-Configuration Coverage
      uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
      if: ${{ !cancelled() && (!inputs.skip_coverage) }}
      with:
        name: coverage_detail ${{ inputs.result_affix }}
        path: ${{ inputs.cmake_path }}/coverage_report/per_configuration
        # Kept as long as the merged report above, and for the same reason: these
        # are the tracefiles the union figure and the gap inventory are computed
        # from, so they are the ones a figure is checked against.
        retention-days: ${{ github.ref == 'refs/heads/v6.4.1_cert' && 90 || 1 }}
          
    - name: Upload Code Coverage Pages
      uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0
      if: ${{ !cancelled() && (!inputs.skip_deploy && !inputs.skip_coverage) }}
      with:
        path: ${{ inputs.cmake_path }}/coverage_report/${{ inputs.coverage_name }}
  
  deploy_code_coverage:
    runs-on: ubuntu-24.04
    if: ((github.event_name == 'push') || (github.event_name == 'workflow_dispatch')) && !inputs.skip_coverage && !inputs.skip_deploy && !failure() && !cancelled()
    needs: run_tests
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    permissions:
      pages: write
      id-token: write

    steps:
    # By pattern, because the artifacts are now coverage_report-<epoch> and the
    # step that names them runs in a different job, whose steps context does not
    # reach here. merge-multiple puts both suites' contents directly into
    # coverage_report rather than under a directory named for each artifact, so
    # the published path holds the two suite directories and not a run timestamp.
    - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
      if: ${{ inputs.skip_test }}
      with:
        pattern: coverage_report-*
        merge-multiple: true
        path: ${{ inputs.cmake_path }}/coverage_report

    - name: Upload Code Coverage Pages
      uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0
      if: ${{ inputs.skip_test }}
      with:
        path: .

    - name: Delete Duplicate Code Coverage Artifact
      uses: geekyeggo/delete-artifact@176a747ab7e287e3ff4787bf8a148716375ca118 # v6.0.0
      with:
          name: coverage_report-*

    - name: Deploy GitHub Pages site
      id: deployment
      uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0

    - name: Write Code Coverage Report URL
      run: >-
        if [ "${{ inputs.deploy_list }}" != "" ]; then
          for i in ${{ inputs.deploy_list }}; do            
            echo 'Coverage report for ' $i ':${{ steps.deployment.outputs.page_url }}'$i >>  $GITHUB_STEP_SUMMARY
          done
        else
          echo 'Coverage report: ${{ steps.deployment.outputs.page_url }}' >>  $GITHUB_STEP_SUMMARY
        fi