Run CI build with fixed set of boards (#3389)

* run cmake ci build on github with a fixed set of board to keep the size stable
* Size Difference Report contain major >1% and minor <1& table

2 files changed, 98 insertions, 44 deletions

diff --git a/tools/build.py b/tools/build.py
index c4f1558c0..87064b7a0 100755
--- a/tools/build.py
+++ b/tools/build.py
@@ -27,6 +27,23 @@ verbose = False
 clean_build = False
 parallel_jobs = os.cpu_count()
 
+# CI board control lists (used when running under CI)
+ci_skip_boards = {
+    'rp2040': [
+        'adafruit_feather_rp2040_usb_host',
+        'adafruit_fruit_jam',
+        'adafruit_metro_rp2350',
+        'feather_rp2040_max3421',
+        'pico_sdk',
+        'raspberry_pi_pico_w',
+    ],
+}
+
+ci_preferred_boards = {
+    'stm32h7': ['stm32h743eval'],
+}
+
+
 # -----------------------------
 # Helper
 # -----------------------------
@@ -195,35 +212,40 @@ def build_boards_list(boards, build_defines, build_system, build_flags_on):
     return ret
 
 
-def get_family_boards(family, one_per_family, boards):
+def get_family_boards(family, one_random, one_first):
     """Get list of boards for a family.
 
     Args:
         family: Family name
-        one_per_family: If True, return only one random board
-        boards: List of boards already specified via -b flag
+        one_random: If True, return only one random board
+        one_first: If True, return only the first board (alphabetical)
 
     Returns:
         List of board names
     """
-    skip_ci = []
+    skip_list = []
+    preferred_list = []
     if os.getenv('GITHUB_ACTIONS') or os.getenv('CIRCLECI'):
-        skip_ci_file = Path(f"hw/bsp/{family}/skip_ci.txt")
-        if skip_ci_file.exists():
-            skip_ci = skip_ci_file.read_text().split()
+        skip_list = ci_skip_boards.get(family, [])
+        preferred_list = ci_preferred_boards.get(family, [])
+
     all_boards = []
     for entry in os.scandir(f"hw/bsp/{family}/boards"):
-        if entry.is_dir() and not entry.name in skip_ci:
+        if entry.is_dir() and entry.name not in skip_list:
             all_boards.append(entry.name)
+    if not all_boards:
+        print(f"No boards found for family '{family}'")
+        return []
     all_boards.sort()
 
-    # If only-one flag is set, select one random board
-    if one_per_family:
-        for b in boards:
-            # skip if -b already specify one in this family
-            if find_family(b) == family:
-                return []
-        all_boards = [random.choice(all_boards)]
+    # If only-one flags are set, honor select list first, then pick first or random
+    if one_first or one_random:
+        if preferred_list:
+            return [preferred_list[0]]
+        if one_first:
+            return [all_boards[0]]
+        if one_random:
+            return [random.choice(all_boards)]
 
     return all_boards
 
@@ -244,7 +266,10 @@ def main():
     parser.add_argument('-s', '--build-system', default='cmake', help='Build system to use, default is cmake')
     parser.add_argument('-D', '--define-symbol', action='append', default=[], help='Define to pass to build system')
     parser.add_argument('-f1', '--build-flags-on', action='append', default=[], help='Build flag to pass to build system')
-    parser.add_argument('-1', '--one-per-family', action='store_true', default=False, help='Build only one random board inside a family')
+    parser.add_argument('--one-random', action='store_true', default=False,
+                        help='Build only one random board of each specified family')
+    parser.add_argument('--one-first', action='store_true', default=False,
+                        help='Build only the first board (alphabetical) of each specified family')
     parser.add_argument('-j', '--jobs', type=int, default=os.cpu_count(), help='Number of jobs to run in parallel')
     parser.add_argument('-v', '--verbose', action='store_true', help='Verbose output')
     args = parser.parse_args()
@@ -255,7 +280,8 @@ def main():
     build_system = args.build_system
     build_defines = args.define_symbol
     build_flags_on = args.build_flags_on
-    one_per_family = args.one_per_family
+    one_random = args.one_random
+    one_first = args.one_first
     verbose = args.verbose
     clean_build = args.clean
     parallel_jobs = args.jobs
@@ -283,7 +309,7 @@ def main():
     # get boards from families and append to boards list
     all_boards = list(boards)
     for f in all_families:
-        all_boards.extend(get_family_boards(f, one_per_family, boards))
+        all_boards.extend(get_family_boards(f, one_random, one_first))
 
     # build all boards
     result = build_boards_list(all_boards, build_defines, build_system, build_flags_on)
diff --git a/tools/metrics.py b/tools/metrics.py
index c3b366e42..bdc64fccc 100644
--- a/tools/metrics.py
+++ b/tools/metrics.py
@@ -245,8 +245,18 @@ def write_compare_markdown(comparison, path, sort_order='size'):
     header += " Total |"
     separator += "------:|"
 
-    md_lines.append(header)
-    md_lines.append(separator)
+    def is_significant(file_row):
+        for s in sections:
+            sd = file_row["sections"][s]
+            diff = abs(sd["diff"])
+            base = sd["base"]
+            if base == 0:
+                if diff != 0:
+                    return True
+            else:
+                if (diff / base) * 100 > 1.0:
+                    return True
+        return False
 
     # Sort files based on sort_order
     if sort_order == 'size-':
@@ -263,38 +273,56 @@ def write_compare_markdown(comparison, path, sort_order='size'):
         reverse = False
     sorted_files = sorted(comparison["files"], key=key_func, reverse=reverse)
 
-    sum_base = {s: 0 for s in sections}
-    sum_base["total"] = 0
-    sum_new = {s: 0 for s in sections}
-    sum_new["total"] = 0
-
+    significant = []
+    minor = []
     for f in sorted_files:
         # Skip files with no changes
         if f["total"]["diff"] == 0 and all(f["sections"][s]["diff"] == 0 for s in sections):
             continue
+        (significant if is_significant(f) else minor).append(f)
 
-        row = f"| {f['file']} |"
-        for s in sections:
-            sd = f["sections"][s]
-            sum_base[s] += sd["base"]
-            sum_new[s] += sd["new"]
-            row += f" {format_diff(sd['base'], sd['new'], sd['diff'])} |"
+    def render_table(title, rows):
+        md_lines.append(f"## {title}")
+        if not rows:
+            md_lines.append("No entries.")
+            md_lines.append("")
+            return
 
-        td = f["total"]
-        sum_base["total"] += td["base"]
-        sum_new["total"] += td["new"]
-        row += f" {format_diff(td['base'], td['new'], td['diff'])} |"
+        md_lines.append(header)
+        md_lines.append(separator)
 
-        md_lines.append(row)
+        sum_base = {s: 0 for s in sections}
+        sum_base["total"] = 0
+        sum_new = {s: 0 for s in sections}
+        sum_new["total"] = 0
 
-    # Add sum row
-    sum_row = "| **SUM** |"
-    for s in sections:
-        diff = sum_new[s] - sum_base[s]
-        sum_row += f" {format_diff(sum_base[s], sum_new[s], diff)} |"
-    total_diff = sum_new["total"] - sum_base["total"]
-    sum_row += f" {format_diff(sum_base['total'], sum_new['total'], total_diff)} |"
-    md_lines.append(sum_row)
+        for f in rows:
+            row = f"| {f['file']} |"
+            for s in sections:
+                sd = f["sections"][s]
+                sum_base[s] += sd["base"]
+                sum_new[s] += sd["new"]
+                row += f" {format_diff(sd['base'], sd['new'], sd['diff'])} |"
+
+            td = f["total"]
+            sum_base["total"] += td["base"]
+            sum_new["total"] += td["new"]
+            row += f" {format_diff(td['base'], td['new'], td['diff'])} |"
+
+            md_lines.append(row)
+
+        # Add sum row
+        sum_row = "| **SUM** |"
+        for s in sections:
+            diff = sum_new[s] - sum_base[s]
+            sum_row += f" {format_diff(sum_base[s], sum_new[s], diff)} |"
+        total_diff = sum_new["total"] - sum_base["total"]
+        sum_row += f" {format_diff(sum_base['total'], sum_new['total'], total_diff)} |"
+        md_lines.append(sum_row)
+        md_lines.append("")
+
+    render_table("Changes >1% in any section", significant)
+    render_table("Changes <1% in all sections", minor)
 
     with open(path, "w", encoding="utf-8") as f:
         f.write("\n".join(md_lines))