seperate ci build for family and orphaned boards

6 files changed, 240 insertions, 63 deletions

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 4f18790a0..de1b9bafd 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -20,8 +20,8 @@ jobs:
         cd test
         ceedling test:all
 
-  # Build most of the ports
-  build:
+  # Build all no-family (opharned) boards
+  build-board:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -40,6 +40,7 @@ jobs:
         - 'device/midi_test'
         - 'device/msc_dual_lun'
         - 'device/net_lwip_webserver'
+        - 'device/uac2_headset'
         - 'device/usbtmc'
         - 'device/webusb_serial'
         - 'host/cdc_msc_hid'
@@ -86,11 +87,71 @@ jobs:
       run: |
         # some submodule has it own submodule that need to be fetched as well
         git submodule update --init --recursive hw/mcu/microchip
-        #git submodule update --init --recursive hw/mcu/microchip
+        git submodule update --init --recursive lib/FreeRTOS
 
     - name: Build
+      run: python3 tools/build_board.py ${{ matrix.example }}
+
+  # build all example for each family
+  build-family:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        family:
+        - 'imxrt'
+        - 'nrf'
+        #- 'rp2040'
+        - 'samd21'
+        - 'samd51'
+        - 'stm32f4'
+        - 'stm32f7'
+    steps:
+    - name: Setup Python
+      uses: actions/setup-python@v1
+
+    - name: Setup Node.js
+      uses: actions/setup-node@v1
+
+    - name: Cache MSP430 Toolchain
+      id: cache-msp430
+      uses: actions/cache@v1
+      with:
+        path: /tmp/dl/
+        # Increment gcc version number at end when updating downloads
+        key: msp430-${{ runner.os }}-9.2.0.50
+
+    - name: Install Toolchains
+      env:
+        MSP430GCC_URL: http://software-dl.ti.com/msp430/msp430_public_sw/mcu/msp430/MSPGCC/9_2_0_0/export/msp430-gcc-9.2.0.50_linux64.tar.bz2
+      run: |
+        # ARM & RISC-V GCC from xpack
+        npm install --global xpm
+        xpm install --global @xpack-dev-tools/arm-none-eabi-gcc@latest
+        xpm install --global @xpack-dev-tools/riscv-none-embed-gcc@latest
+        echo `echo $HOME/opt/xPacks/@xpack-dev-tools/arm-none-eabi-gcc/*/.content/bin` >> $GITHUB_PATH
+        echo `echo $HOME/opt/xPacks/@xpack-dev-tools/riscv-none-embed-gcc/*/.content/bin` >> $GITHUB_PATH
+
+        # TI MSP430 GCC
+        mkdir -p /tmp/dl/
+        [ -f "/tmp/dl/msp430-gcc.tar.bz2" ] || wget --progress=dot:mega $MSP430GCC_URL -O /tmp/dl/msp430-gcc.tar.bz2
+        tar -C $HOME -xaf /tmp/dl/msp430-gcc.tar.bz2
+        echo `echo $HOME/msp430-gcc-*_linux64/bin` >> $GITHUB_PATH
+
+    - name: Checkout TinyUSB
+      uses: actions/checkout@v2
+      with:
+        # Cannot do submodule checkout here since LWIP's git server cannot checkout unadventised commits (it must use tags)
+        submodules: 'true'
+
+    - name: Checkout Sub-Submodules
       run: |
-        python3 tools/build_all.py ${{ matrix.example }}
+        # some submodule has it own submodule that need to be fetched as well
+        git submodule update --init --recursive hw/mcu/microchip
+        git submodule update --init --recursive lib/FreeRTOS
+
+    - name: Build
+      run: python3 tools/build_family.py ${{ matrix.family }}
 
   # Build ESP32S
   build-esp32s:
diff --git a/examples/device/uac2_headset/.skip.MCU_SAMD11 b/examples/device/uac2_headset/.skip.MCU_SAMD11
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/examples/device/uac2_headset/.skip.MCU_SAMD11
diff --git a/examples/device/uac2_headset/.skip.MCU_SAME5X b/examples/device/uac2_headset/.skip.MCU_SAME5X
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/examples/device/uac2_headset/.skip.MCU_SAME5X
diff --git a/examples/device/uac2_headset/.skip.MCU_SAMG b/examples/device/uac2_headset/.skip.MCU_SAMG
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/examples/device/uac2_headset/.skip.MCU_SAMG
diff --git a/tools/build_all.py b/tools/build_board.py
index 787011e26..cf3a8fb94 100644
--- a/tools/build_all.py
+++ b/tools/build_board.py
@@ -18,48 +18,60 @@ total_time = time.monotonic()
 build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |'
 build_separator = '-' * 106
 
+def filter_with_input(mylist):
+    if len(sys.argv) > 1:
+        input_args = list(set(mylist).intersection(sys.argv))
+        if len(input_args) > 0:
+            mylist[:] = input_args
+
 # If examples are not specified in arguments, build all
 all_examples = []
-
 for entry in os.scandir("examples/device"):
     if entry.is_dir():
         all_examples.append("device/" + entry.name)
-
 for entry in os.scandir("examples/host"):
     if entry.is_dir():
         all_examples.append("host/" + entry.name)
-
-if len(sys.argv) > 1:
-    input_examples = list(set(all_examples).intersection(sys.argv))
-    if len(input_examples) > 0:
-        all_examples = input_examples
-
+filter_with_input(all_examples)
 all_examples.sort()
 
 # If boards are not specified in arguments, build all
 all_boards = []
-
 for entry in os.scandir("hw/bsp"):
-    if entry.is_dir() and entry.name != "esp32s2":
-        if os.path.isdir(entry.path + "/boards"):
-            # family directory
-            for subentry in os.scandir(entry.path + "/boards"):
-                if subentry.is_dir(): all_boards.append(subentry.name)
-        else:
-            all_boards.append(entry.name)
+    if entry.is_dir() and os.path.exists(entry.path + "/board.mk"):
+        all_boards.append(entry.name)
+filter_with_input(all_boards)
+all_boards.sort()
 
-if len(sys.argv) > 1:
-    input_boards = list(set(all_boards).intersection(sys.argv))
-    if len(input_boards) > 0:
-        all_boards = input_boards
+def build_board(example, board):
+    global success_count, fail_count, skip_count
+    start_time = time.monotonic()
+    flash_size = "-"
+    sram_size = "-"
 
-all_boards.sort()
+    # Check if board is skipped
+    if skip_example(example, board):
+        success = SKIPPED
+        skip_count += 1
+        print(build_format.format(example, board, success, '-', flash_size, sram_size))
+    else:
+        subprocess.run("make -C examples/{} BOARD={} clean".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+        if build_result.returncode == 0:
+            success = SUCCEEDED
+            success_count += 1
+            (flash_size, sram_size) = build_size(example, board)
+        else:
+            exit_status = build_result.returncode
+            success = FAILED
+            fail_count += 1
 
-def build_example(example, board):
-    subprocess.run("make -C examples/{} BOARD={} clean".format(example, board), shell=True,
-                   stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    return subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True,
-                          stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        build_duration = time.monotonic() - start_time
+        print(build_format.format(example, board, success, "{:.2f}s".format(build_duration), flash_size, sram_size))
+
+        if build_result.returncode != 0:
+            print(build_result.stdout.decode("utf-8"))
 
 def build_size(example, board):
     #elf_file = 'examples/device/{}/_build/build-{}/{}-firmware.elf'.format(example, board, board)
@@ -72,11 +84,7 @@ def build_size(example, board):
 
 def skip_example(example, board):
     ex_dir = 'examples/' + example
-
     board_mk = 'hw/bsp/{}/board.mk'.format(board)
-    if not os.path.exists(board_mk):
-        board_mk = list(glob.iglob('hw/bsp/*/boards/{}/../../family.mk'.format(board)))[0]
-
     with open(board_mk) as mk:
         mk_contents = mk.read()
 
@@ -107,35 +115,7 @@ print(build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', '
 for example in all_examples:
     print(build_separator)
     for board in all_boards:
-        start_time = time.monotonic()
-
-        flash_size = "-"
-        sram_size = "-"
-
-        # Check if board is skipped
-        if skip_example(example, board):
-            success = SKIPPED
-            skip_count += 1
-            print(build_format.format(example, board, success, '-', flash_size, sram_size))
-        else:
-            build_result = build_example(example, board)
-
-            if build_result.returncode == 0:
-                success = SUCCEEDED
-                success_count += 1
-                (flash_size, sram_size) = build_size(example, board)
-            else:
-                exit_status = build_result.returncode
-                success = FAILED
-                fail_count += 1
-
-            build_duration = time.monotonic() - start_time
-            print(build_format.format(example, board, success, "{:.2f}s".format(build_duration), flash_size, sram_size))
-
-            if build_result.returncode != 0:
-                print(build_result.stdout.decode("utf-8"))
-
-
+        build_board(example, board)
 
 total_time = time.monotonic() - total_time
 print(build_separator)
diff --git a/tools/build_family.py b/tools/build_family.py
new file mode 100644
index 000000000..7f2c1a706
--- /dev/null
+++ b/tools/build_family.py
@@ -0,0 +1,136 @@
+import os
+import glob
+import sys
+import subprocess
+import time
+
+SUCCEEDED = "\033[32msucceeded\033[0m"
+FAILED = "\033[31mfailed\033[0m"
+SKIPPED = "\033[33mskipped\033[0m"
+
+success_count = 0
+fail_count = 0
+skip_count = 0
+exit_status = 0
+
+total_time = time.monotonic()
+
+build_format = '| {:29} | {:30} | {:18} | {:7} | {:6} | {:6} |'
+build_separator = '-' * 106
+
+def filter_with_input(mylist):
+    if len(sys.argv) > 1:
+        input_args = list(set(mylist).intersection(sys.argv))
+        if len(input_args) > 0:
+            mylist[:] = input_args
+
+# If examples are not specified in arguments, build all
+all_examples = []
+for entry in os.scandir("examples/device"):
+    if entry.is_dir():
+        all_examples.append("device/" + entry.name)
+for entry in os.scandir("examples/host"):
+    if entry.is_dir():
+        all_examples.append("host/" + entry.name)
+filter_with_input(all_examples)
+all_examples.sort()
+
+# If family are not specified in arguments, build all
+all_families = []
+for entry in os.scandir("hw/bsp"):
+    if entry.is_dir() and os.path.isdir(entry.path + "/boards") and entry.name != "esp32s2":
+        all_families.append(entry.name)
+            
+filter_with_input(all_families)
+all_families.sort()
+
+def build_family(example, family):
+    all_boards = []
+    for entry in os.scandir("hw/bsp/{}/boards".format(family)):
+        if entry.is_dir():
+            all_boards.append(entry.name)
+    all_boards.sort()
+    
+    for board in all_boards:
+        build_board(example, board)
+    
+def build_board(example, board):
+    global success_count, fail_count, skip_count
+    start_time = time.monotonic()
+    flash_size = "-"
+    sram_size = "-"
+
+    # Check if board is skipped
+    if skip_example(example, board):
+        success = SKIPPED
+        skip_count += 1
+        print(build_format.format(example, board, success, '-', flash_size, sram_size))
+    else:   
+        subprocess.run("make -C examples/{} BOARD={} clean".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        build_result = subprocess.run("make -j -C examples/{} BOARD={} all".format(example, board), shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+
+        if build_result.returncode == 0:
+            success = SUCCEEDED
+            success_count += 1
+            (flash_size, sram_size) = build_size(example, board)
+        else:
+            exit_status = build_result.returncode
+            success = FAILED
+            fail_count += 1
+
+        build_duration = time.monotonic() - start_time
+        print(build_format.format(example, board, success, "{:.2f}s".format(build_duration), flash_size, sram_size))
+
+        if build_result.returncode != 0:
+            print(build_result.stdout.decode("utf-8"))            
+
+def build_size(example, board):
+    #elf_file = 'examples/device/{}/_build/build-{}/{}-firmware.elf'.format(example, board, board)
+    elf_file = 'examples/{}/_build/build-{}/*.elf'.format(example, board)
+    size_output = subprocess.run('size {}'.format(elf_file), shell=True, stdout=subprocess.PIPE).stdout.decode("utf-8")
+    size_list = size_output.split('\n')[1].split('\t')
+    flash_size = int(size_list[0])
+    sram_size = int(size_list[1]) + int(size_list[2])
+    return (flash_size, sram_size)
+
+def skip_example(example, board):
+    ex_dir = 'examples/' + example
+    board_mk = 'hw/bsp/{}/family.mk'.format(family)
+
+    with open(board_mk) as mk:
+        mk_contents = mk.read()
+
+        # Skip all OPT_MCU_NONE these are WIP port
+        if '-DCFG_TUSB_MCU=OPT_MCU_NONE' in mk_contents:
+            return 1
+
+        # Skip if CFG_TUSB_MCU in board.mk to match skip file
+        for skip_file in glob.iglob(ex_dir + '/.skip.MCU_*'):
+            mcu_cflag = '-DCFG_TUSB_MCU=OPT_' + os.path.basename(skip_file).split('.')[2]
+            if mcu_cflag in mk_contents:
+                return 1
+
+        # Build only list, if exists only these MCU are built
+        only_list = list(glob.iglob(ex_dir + '/.only.MCU_*'))
+        if len(only_list) > 0:
+            for only_file in only_list:
+                mcu_cflag = '-DCFG_TUSB_MCU=OPT_' + os.path.basename(only_file).split('.')[2]
+                if mcu_cflag in mk_contents:
+                    return 0
+            return 1
+    return 0
+
+print(build_separator)
+print(build_format.format('Example', 'Board', '\033[39mResult\033[0m', 'Time', 'Flash', 'SRAM'))
+
+for example in all_examples:
+    print(build_separator)
+    for family in all_families:
+        build_family(example, family)
+
+total_time = time.monotonic() - total_time
+print(build_separator)
+print("Build Summary: {} {}, {} {}, {} {} and took {:.2f}s".format(success_count, SUCCEEDED, fail_count, FAILED, skip_count, SKIPPED, total_time))
+print(build_separator)
+
+sys.exit(exit_status)