add build target argument to improve flexibility of build scripts and workflows

membrowse-upload upload with --identical if elf file does not exist

4 files changed, 103 insertions, 55 deletions

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 2b0c38c4f..412d52bb8 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -47,8 +47,6 @@ jobs:
               - '.github/workflows/ci_set_matrix.py'
 
   set-matrix:
-    needs: [ check-paths ]
-    if: needs.check-paths.outputs.code_changed == 'true'
     runs-on: ubuntu-latest
     outputs:
       json: ${{ steps.set-matrix-json.outputs.matrix }}
@@ -75,7 +73,7 @@ jobs:
   # For Make and IAR build: will be done on CircleCI only (one random per family as well)
   # ------------------------------------------------------------------------------
   cmake:
-    needs: set-matrix
+    needs: [ check-paths, set-matrix ]
     uses: ./.github/workflows/build_util.yml
     strategy:
       fail-fast: false
@@ -95,10 +93,12 @@ jobs:
       upload-metrics: true
       upload-artifacts: false
       upload-membrowse: true
+      code-changed: ${{ needs.check-paths.outputs.code_changed == 'true' }}
     secrets: inherit
 
   code-metrics:
-    needs: cmake
+    needs: [ check-paths, cmake ]
+    if: needs.check-paths.outputs.code_changed == true
     runs-on: ubuntu-latest
     permissions:
       pull-requests: write
@@ -196,17 +196,18 @@ jobs:
   # ---------------------------------------
   build-os:
     needs: [ check-paths ]
-    if: needs.check-paths.outputs.code_changed == 'true'
+    if: needs.check-paths.outputs.code_changed == true
     uses: ./.github/workflows/build_util.yml
     strategy:
       fail-fast: false
       matrix:
         os: [ windows-latest, macos-latest ]
+        build-system: [ 'make', 'cmake' ]
     with:
       os: ${{ matrix.os }}
-      build-system: 'cmake-make'
+      build-system: ${{ matrix.build-system }}
       toolchain: 'arm-gcc-${{ matrix.os }}'
-      build-args: '["stm32h7"]'
+      build-args: '["stm32h7rs"]'
       build-options: '--one-random'
 
   # ---------------------------------------
@@ -215,7 +216,8 @@ jobs:
   zephyr:
     needs: [ check-paths ]
     # skip zephyr build due to failed build, fix later
-    if: false && needs.check-paths.outputs.code_changed == 'true'
+    if: false
+    #if: needs.check-paths.outputs.code_changed == 'true'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout TinyUSB
@@ -237,8 +239,8 @@ jobs:
   # Run on PR only (hil-tinyusb), hil-hfp only run on non-forked PR
   # ---------------------------------------
   hil-build:
-    needs: set-matrix
-    if: github.repository_owner == 'hathach'
+    needs: [ check-paths, set-matrix ]
+    if: needs.check-paths.outputs.code_changed == true && github.repository_owner == 'hathach'
     uses: ./.github/workflows/build_util.yml
     strategy:
       fail-fast: false
@@ -299,6 +301,7 @@ jobs:
   hil-hfp:
     needs: [ check-paths ]
     if: |
+      needs.check-paths.outputs.code_changed == true &&
       github.repository_owner == 'hathach' &&
       !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == true)
     runs-on: [ self-hosted, Linux, X64, hifiphile ]
diff --git a/.github/workflows/build_util.yml b/.github/workflows/build_util.yml
index eb3c4df89..4200c11bd 100644
--- a/.github/workflows/build_util.yml
+++ b/.github/workflows/build_util.yml
@@ -3,6 +3,10 @@ name: Reusable build util
 on:
   workflow_call:
     inputs:
+      os:
+        required: false
+        type: string
+        default: 'ubuntu-latest'
       build-system:
         required: true
         type: string
@@ -28,10 +32,10 @@ on:
         required: false
         default: false
         type: boolean
-      os:
+      code-changed:
         required: false
-        type: string
-        default: 'ubuntu-latest'
+        default: false
+        type: boolean
 
 jobs:
   family:
@@ -58,19 +62,29 @@ jobs:
           arg: ${{ matrix.arg }}
 
       - name: Build
+        if: ${{ inputs.code-changed }}
         env:
           IAR_LMS_BEARER_TOKEN: ${{ secrets.IAR_LMS_BEARER_TOKEN }}
-          MEMBROWSE_API_KEY: ${{ secrets.MEMBROWSE_API_KEY }}
-          MEMBROWSE_UPLOAD_OPTION: ${{ inputs.upload-membrowse && '--membrowse-upload' || '' }}
-          TOOLCHAIN: ${{ inputs.toolchain }}
         run: |
-          if [ "$TOOLCHAIN" == "esp-idf" ]; then
-            docker run --rm -e MEMBROWSE_API_KEY="$MEMBROWSE_API_KEY" -v $PWD:/project -w /project espressif/idf:tinyusb python tools/build.py $MEMBROWSE_UPLOAD_OPTION ${{ matrix.arg }}
-          elif [ "${{ inputs.build-system }}" == "cmake-make" ] || [ "${{ inputs.build-system }}" == "make-cmake" ]; then
-            python tools/build.py -s make $MEMBROWSE_UPLOAD_OPTION ${{ steps.setup-toolchain.outputs.build_option }} ${{ inputs.build-options }} ${{ matrix.arg }}
-            python tools/build.py -s cmake $MEMBROWSE_UPLOAD_OPTION ${{ steps.setup-toolchain.outputs.build_option }} ${{ inputs.build-options }} ${{ matrix.arg }}
+          if [ "${{ inputs.toolchain }}" == "esp-idf" ]; then
+            docker run --rm -e MEMBROWSE_API_KEY="$MEMBROWSE_API_KEY" -v $PWD:/project -w /project espressif/idf:tinyusb python tools/build.py -T all ${{ matrix.arg }}
           else
-            python tools/build.py -s ${{ inputs.build-system }} $MEMBROWSE_UPLOAD_OPTION ${{ steps.setup-toolchain.outputs.build_option }} ${{ inputs.build-options }} ${{ matrix.arg }}
+            BUILD_PY_ARGS="-s ${{ inputs.build-system }} ${{ steps.setup-toolchain.outputs.build_option }} ${{ inputs.build-options }}"
+            python tools/build.py $BUILD_PY_ARGS --target all ${{ matrix.arg }}
+
+            if [ "${{ inputs.upload-metrics }}" = "true" ]; then
+              python tools/build.py $BUILD_PY_ARGS --target tinyusb_metrics ${{ matrix.arg }}
+            fi
+          fi
+        shell: bash
+
+      - name: Membrowse Upload
+        if: inputs.toolchain != 'esp-idf'
+        env:
+          MEMBROWSE_API_KEY: ${{ secrets.MEMBROWSE_API_KEY }}
+        run: |
+          if  [ "${{ inputs.upload-membrowse }}" = "true" ]; then
+            python tools/build.py $BUILD_PY_ARGS --target examples-membrowse-upload ${{ matrix.arg }}
           fi
         shell: bash
 
diff --git a/hw/bsp/family_support.cmake b/hw/bsp/family_support.cmake
index 57a323c4a..fdf7b78ed 100644
--- a/hw/bsp/family_support.cmake
+++ b/hw/bsp/family_support.cmake
@@ -280,21 +280,62 @@ function(family_add_membrowse TARGET)
     string(APPEND OPTION " ${MEMBROWSE_OPTION}")
   endif ()
 
-  # For Ninja generator, extract all linker scripts from Ninja commands and pass them to membrowse.
+  # For Ninja generator, extract all linker scripts from Ninja commands (with INCLUDE) and pass them to membrowse.
   if (CMAKE_GENERATOR MATCHES "Ninja")
+    set(TARGET_ELF_PATH "$<TARGET_FILE_DIR:${TARGET}>/$<TARGET_FILE_NAME:${TARGET}>")
+    set(MEMBROWSE_LD_SCRIPTS_CMD
+      "ld_scripts=\"$(${CMAKE_MAKE_PROGRAM} -C ${CMAKE_BINARY_DIR} -t commands ${TARGET} | grep -oP '(?:-Wl,--script=|-T\\s*)\\K[A-Za-z0-9_./-]+\\.ld' | xargs)\"; \
+all_ld_scripts=\"\"; \
+pending_ld_scripts=\"$ld_scripts\"; \
+while [ -n \"$pending_ld_scripts\" ]; do \
+  next_pending=\"\"; \
+  for script in $pending_ld_scripts; do \
+    case \" $all_ld_scripts \" in *\" $script \"*) continue ;; esac; \
+    all_ld_scripts=\"$all_ld_scripts $script\"; \
+    script_dir=$(dirname \"$script\"); \
+    include_scripts=$(grep -hoP '^\\s*INCLUDE\\s+[<\"]?\\K[^\">[:space:]]+\\.ld' \"$script\" 2>/dev/null | xargs); \
+    for include_script in $include_scripts; do \
+      resolved_script=\"\"; \
+      if [ -f \"$include_script\" ]; then \
+        resolved_script=\"$include_script\"; \
+      elif [ -f \"$script_dir/$include_script\" ]; then \
+        resolved_script=\"$script_dir/$include_script\"; \
+      fi; \
+      if [ -n \"$resolved_script\" ]; then \
+        case \" $all_ld_scripts $next_pending \" in *\" $resolved_script \"*) ;; *) next_pending=\"$next_pending $resolved_script\" ;; esac; \
+      fi; \
+    done; \
+  done; \
+  pending_ld_scripts=\"$(echo \"$next_pending\" | xargs)\"; \
+done; \
+ld_scripts=\"$(echo \"$all_ld_scripts\" | xargs)\"")
+
     set(MEMBROWSE_CMD
-      "ld_scripts=\"$(${CMAKE_MAKE_PROGRAM} -C ${CMAKE_BINARY_DIR} -t commands ${TARGET} | grep -oP '(?<=-Wl,--script=)[A-Za-z0-9_./-]+\\.ld' | xargs)\"; \
-${MEMBROWSE_EXE} report ${OPTION} $<TARGET_FILE:${TARGET}> \"$ld_scripts\"")
+      "if [ -f \"${TARGET_ELF_PATH}\" ]; then \
+  ${MEMBROWSE_LD_SCRIPTS_CMD}; \
+  echo ld_scripts=\"$ld_scripts\"; \
+  if [ \"$MEMBROWSE_UPLOAD\" = \"1\" ]; then \
+    ${MEMBROWSE_EXE} report ${OPTION} \"${TARGET_ELF_PATH}\" \"$ld_scripts\" --upload --github --target-name ${BOARD}-${TARGET} --api-key $ENV{MEMBROWSE_API_KEY}; \
+  else \
+    ${MEMBROWSE_EXE} report ${OPTION} \"${TARGET_ELF_PATH}\" \"$ld_scripts\"; \
+  fi; \
+else \
+  if [ \"$MEMBROWSE_UPLOAD\" = \"1\" ]; then \
+    ${MEMBROWSE_EXE} report ${OPTION} --identical --upload --github --target-name ${BOARD}-${TARGET} --api-key $ENV{MEMBROWSE_API_KEY}; \
+  else \
+    ${MEMBROWSE_EXE} report ${OPTION} --identical; \
+  fi; \
+fi")
 
     add_custom_target(${TARGET}-membrowse
       DEPENDS ${TARGET}
-      COMMAND bash -lc "${MEMBROWSE_CMD}"
+      COMMAND ${CMAKE_COMMAND} -E env MEMBROWSE_UPLOAD=0 bash -lc "${MEMBROWSE_CMD}"
       VERBATIM
       )
+    set_property(TARGET ${TARGET}-membrowse PROPERTY FOLDER ${TARGET})
 
     add_custom_target(${TARGET}-membrowse-upload
-      DEPENDS ${TARGET}
-      COMMAND bash -lc "${MEMBROWSE_CMD} --upload --github --target-name ${BOARD}-${TARGET} --api-key $ENV{MEMBROWSE_API_KEY}"
+      COMMAND ${CMAKE_COMMAND} -E env MEMBROWSE_UPLOAD=1 bash -lc "${MEMBROWSE_CMD}"
       VERBATIM
       )
 
@@ -303,7 +344,6 @@ ${MEMBROWSE_EXE} report ${OPTION} $<TARGET_FILE:${TARGET}> \"$ld_scripts\"")
     endif ()
     add_dependencies(examples-membrowse-upload ${TARGET}-membrowse-upload)
 
-    set_property(TARGET ${TARGET}-membrowse PROPERTY FOLDER ${TARGET})
     set_property(TARGET ${TARGET}-membrowse-upload PROPERTY FOLDER ${TARGET})
   endif ()
 endfunction()
diff --git a/tools/build.py b/tools/build.py
index bc032fb91..d26028c51 100755
--- a/tools/build.py
+++ b/tools/build.py
@@ -98,15 +98,15 @@ def get_examples(family):
     return all_examples
 
 
-def print_build_result(board, example, status, duration):
+def print_build_result(board, build_target, status, duration):
     if isinstance(duration, (int, float)):
         duration = "{:.2f}s".format(duration)
-    print(build_format.format(board, example, build_status[status], duration))
+    print(build_format.format(board, build_target, build_status[status], duration))
 
 # -----------------------------
 # CMake
 # -----------------------------
-def cmake_board(board, build_args, build_flags_on, membrowse_upload):
+def cmake_board(board, build_args, build_flags_on, build_target):
     ret = [0, 0, 0]
     start_time = time.monotonic()
 
@@ -137,26 +137,18 @@ def cmake_board(board, build_args, build_flags_on, membrowse_upload):
         if rcmd.returncode == 0:
             if clean_build:
                 run_cmd(["cmake", "--build", build_dir, '--target', 'clean'])
-            cmd = ["cmake", "--build", build_dir, '--parallel', str(parallel_jobs)]
+            cmd = ["cmake", "--build", build_dir, '--target', build_target, '--parallel', str(parallel_jobs)]
             rcmd = run_cmd(cmd)
-        if rcmd.returncode == 0:
-            ret[0] += 1
-            run_cmd(["cmake", "--build", build_dir, '--target', 'tinyusb_metrics'])
-            if membrowse_upload:
-                run_cmd(["cmake", "--build", build_dir, '--target', 'examples-membrowse-upload'])
-            # print(rcmd.stdout.decode("utf-8"))
-        else:
-            ret[1] += 1
+        ret[0 if rcmd.returncode == 0 else 1] += 1
 
-    example = 'all'
-    print_build_result(board, example, 0 if ret[1] == 0 else 1, time.monotonic() - start_time)
+    print_build_result(board, build_target, 0 if ret[1] == 0 else 1, time.monotonic() - start_time)
     return ret
 
 
 # -----------------------------
 # Make
 # -----------------------------
-def make_one_example(example, board, make_option):
+def make_one_example(example, board, make_option, build_target):
     # Check if board is skipped
     if build_utils.skip_example(example, board):
         print_build_result(board, example, 2, '-')
@@ -168,7 +160,7 @@ def make_one_example(example, board, make_option):
             make_args += shlex.split(make_option)
         if clean_build:
             run_cmd(make_args + ["clean"])
-        build_result = run_cmd(make_args + ['all'])
+        build_result = run_cmd(make_args + [build_target])
         r = 0 if build_result.returncode == 0 else 1
         print_build_result(board, example, r, time.monotonic() - start_time)
 
@@ -177,7 +169,7 @@ def make_one_example(example, board, make_option):
     return ret
 
 
-def make_board(board, build_args):
+def make_board(board, build_args, build_target):
     print(build_separator)
     family = find_family(board);
     all_examples = get_examples(family)
@@ -188,7 +180,7 @@ def make_board(board, build_args):
         final_status = 2
     else:
         with Pool(processes=os.cpu_count()) as pool:
-            pool_args = list((map(lambda e, b=board, o=f"{build_args}": [e, b, o], all_examples)))
+            pool_args = list((map(lambda e, b=board, o=f"{build_args}", t=build_target: [e, b, o, t], all_examples)))
             r = pool.starmap(make_one_example, pool_args)
             # sum all element of same index (column sum)
             ret = list(map(sum, list(zip(*r))))
@@ -200,16 +192,16 @@ def make_board(board, build_args):
 # -----------------------------
 # Build Family
 # -----------------------------
-def build_boards_list(boards, build_defines, build_system, build_flags_on, membrowse_upload):
+def build_boards_list(boards, build_defines, build_system, build_flags_on, build_target):
     ret = [0, 0, 0]
     for b in boards:
         r = [0, 0, 0]
         if build_system == 'cmake':
             build_args = [f'-D{d}' for d in build_defines]
-            r = cmake_board(b, build_args, build_flags_on, membrowse_upload)
+            r = cmake_board(b, build_args, build_flags_on, build_target)
         elif build_system == 'make':
             build_args = ' '.join(f'{d}' for d in build_defines)
-            r = make_board(b, build_args)
+            r = make_board(b, build_args, build_target)
         ret[0] += r[0]
         ret[1] += r[1]
         ret[2] += r[2]
@@ -275,8 +267,7 @@ def main():
     parser.add_argument('--one-first', action='store_true', default=False,
                         help='Build only the first board (alphabetical) of each specified family')
     parser.add_argument('-j', '--jobs', type=int, default=os.cpu_count(), help='Number of jobs to run in parallel')
-    parser.add_argument('--membrowse-upload', action='store_true', default=False,
-                        help='Run examples-membrowse-upload target after successful CMake build')
+    parser.add_argument('-T', '--target', default='all', help='Build target to use, default is all')
     parser.add_argument('-v', '--verbose', action='store_true', help='Verbose output')
     args = parser.parse_args()
 
@@ -288,7 +279,7 @@ def main():
     build_flags_on = args.build_flags_on
     one_random = args.one_random
     one_first = args.one_first
-    membrowse_upload = args.membrowse_upload
+    build_target = args.target
     verbose = args.verbose
     clean_build = args.clean
     parallel_jobs = args.jobs
@@ -300,7 +291,7 @@ def main():
         return 1
 
     print(build_separator)
-    print(build_format.format('Board', 'Example', '\033[39mResult\033[0m', 'Time'))
+    print(build_format.format('Board', 'Target', '\033[39mResult\033[0m', 'Time'))
     total_time = time.monotonic()
 
     # get all families
@@ -319,7 +310,7 @@ def main():
         all_boards.extend(get_family_boards(f, one_random, one_first))
 
     # build all boards
-    result = build_boards_list(all_boards, build_defines, build_system, build_flags_on, membrowse_upload)
+    result = build_boards_list(all_boards, build_defines, build_system, build_flags_on, build_target)
 
     total_time = time.monotonic() - total_time
     print(build_separator)