Merge pull request #3384 from hathach/pr-size-diff

Size metrics

3 files changed, 442 insertions, 46 deletions

diff --git a/tools/build.py b/tools/build.py
index ce4d0ef1a..e4909f45f 100755
--- a/tools/build.py
+++ b/tools/build.py
@@ -5,6 +5,7 @@ import os
 import sys
 import time
 import subprocess
+import shlex
 from pathlib import Path
 from multiprocessing import Pool
 
@@ -23,15 +24,19 @@ build_separator = '-' * 95
 build_status = [STATUS_OK, STATUS_FAILED, STATUS_SKIPPED]
 
 verbose = False
+clean_build = False
 parallel_jobs = os.cpu_count()
 
 # -----------------------------
 # Helper
 # -----------------------------
 def run_cmd(cmd):
-    #print(cmd)
-    r = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-    title = f'Command Error: {cmd}'
+    if isinstance(cmd, str):
+        raise TypeError("run_cmd expects a list/tuple of args, not a string")
+    args = cmd
+    cmd_display = " ".join(args)
+    r = subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    title = f'Command Error: {cmd_display}'
     if r.returncode != 0:
         # print build output if failed
         if os.getenv('GITHUB_ACTIONS'):
@@ -42,7 +47,7 @@ def run_cmd(cmd):
             print(title)
             print(r.stdout.decode("utf-8"))
     elif verbose:
-        print(cmd)
+        print(cmd_display)
         print(r.stdout.decode("utf-8"))
     return r
 
@@ -87,10 +92,10 @@ def cmake_board(board, build_args, build_flags_on):
     start_time = time.monotonic()
 
     build_dir = f'cmake-build/cmake-build-{board}'
-    build_flags = ''
+    build_flags = []
     if len(build_flags_on) > 0:
-        build_flags =  ' '.join(f'-D{flag}=1' for flag in build_flags_on)
-        build_flags = f'-DCFLAGS_CLI="{build_flags}"'
+        cli_flags = ' '.join(f'-D{flag}=1' for flag in build_flags_on)
+        build_flags.append(f'-DCFLAGS_CLI={cli_flags}')
         build_dir += '-f1_' + '_'.join(build_flags_on)
 
     family = find_family(board)
@@ -101,27 +106,26 @@ def cmake_board(board, build_args, build_flags_on):
             if build_utils.skip_example(example, board):
                 ret[2] += 1
             else:
-                rcmd = run_cmd(f'idf.py -C examples/{example} -B {build_dir}/{example} -G Ninja '
-                               f'-DBOARD={board} {build_flags} build')
+                rcmd = run_cmd([
+                    'idf.py', '-C', f'examples/{example}', '-B', f'{build_dir}/{example}', '-GNinja',
+                    f'-DBOARD={board}', *build_flags, 'build'
+                ])
                 ret[0 if rcmd.returncode == 0 else 1] += 1
     else:
-        rcmd = run_cmd(f'cmake examples -B {build_dir} -G Ninja -DBOARD={board} -DCMAKE_BUILD_TYPE=MinSizeRel '
-                       f'{build_args} {build_flags}')
+        rcmd = run_cmd(['cmake', 'examples', '-B', build_dir, '-GNinja',
+                        f'-DBOARD={board}', '-DCMAKE_BUILD_TYPE=MinSizeRel', '-DLINKERMAP_OPTION=-q -f tinyusb/src',
+                        *build_args, *build_flags])
         if rcmd.returncode == 0:
-            cmd = f"cmake --build {build_dir}"
-            njobs = parallel_jobs
-
-            # circleci docker return $nproc as 36 core, limit parallel according to resource class.
-            # Required for IAR, also prevent crashed/killed by docker
-            if os.getenv('CIRCLECI'):
-                resource_class = { 'small': 1, 'medium': 2, 'medium+': 3, 'large': 4 }
-                for rc in resource_class:
-                    if rc in os.getenv('CIRCLE_JOB'):
-                        njobs = resource_class[rc]
-                        break
-            cmd += f' --parallel {njobs}'
+            if clean_build:
+                run_cmd(["cmake", "--build", build_dir, '--target', 'clean'])
+            cmd = ["cmake", "--build", build_dir, '--parallel', str(parallel_jobs)]
             rcmd = run_cmd(cmd)
-        ret[0 if rcmd.returncode == 0 else 1] += 1
+        if rcmd.returncode == 0:
+            ret[0] += 1
+            run_cmd(["cmake", "--build", build_dir, '--target', 'tinyusb_metrics'])
+            # print(rcmd.stdout.decode("utf-8"))
+        else:
+            ret[1] += 1
 
     example = 'all'
     print_build_result(board, example, 0 if ret[1] == 0 else 1, time.monotonic() - start_time)
@@ -141,9 +145,13 @@ def make_one_example(example, board, make_option):
         # skip -j for circleci
         if not os.getenv('CIRCLECI'):
             make_option += ' -j'
-        make_cmd = f"make -C examples/{example} BOARD={board} {make_option}"
-        # run_cmd(f"{make_cmd} clean")
-        build_result = run_cmd(f"{make_cmd} all")
+        make_args = ["make", "-C", f"examples/{example}", f"BOARD={board}"]
+        if make_option:
+            make_args += shlex.split(make_option)
+        make_args.append("all")
+        if clean_build:
+            run_cmd(make_args + ["clean"])
+        build_result = run_cmd(make_args)
         r = 0 if build_result.returncode == 0 else 1
         print_build_result(board, example, r, time.monotonic() - start_time)
 
@@ -180,7 +188,7 @@ def build_boards_list(boards, build_defines, build_system, build_flags_on):
     for b in boards:
         r = [0, 0, 0]
         if build_system == 'cmake':
-            build_args = ' '.join(f'-D{d}' for d in build_defines)
+            build_args = [f'-D{d}' for d in build_defines]
             r = cmake_board(b, build_args, build_flags_on)
         elif build_system == 'make':
             build_args = ' '.join(f'{d}' for d in build_defines)
@@ -191,8 +199,18 @@ def build_boards_list(boards, build_defines, build_system, build_flags_on):
     return ret
 
 
-def build_family(family, build_defines, build_system, build_flags_on, one_per_family, boards):
-    skip_ci = ['pico_sdk']
+def get_family_boards(family, one_per_family, boards):
+    """Get list of boards for a family.
+
+    Args:
+        family: Family name
+        one_per_family: If True, return only one random board
+        boards: List of boards already specified via -b flag
+
+    Returns:
+        List of board names
+    """
+    skip_ci = []
     if os.getenv('GITHUB_ACTIONS') or os.getenv('CIRCLECI'):
         skip_ci_file = Path(f"hw/bsp/{family}/skip_ci.txt")
         if skip_ci_file.exists():
@@ -203,17 +221,15 @@ def build_family(family, build_defines, build_system, build_flags_on, one_per_fa
             all_boards.append(entry.name)
     all_boards.sort()
 
-    ret = [0, 0, 0]
     # If only-one flag is set, select one random board
     if one_per_family:
         for b in boards:
             # skip if -b already specify one in this family
             if find_family(b) == family:
-                return ret
+                return []
         all_boards = [random.choice(all_boards)]
 
-    ret = build_boards_list(all_boards, build_defines, build_system, build_flags_on)
-    return ret
+    return all_boards
 
 
 # -----------------------------
@@ -221,11 +237,13 @@ def build_family(family, build_defines, build_system, build_flags_on, one_per_fa
 # -----------------------------
 def main():
     global verbose
+    global clean_build
     global parallel_jobs
 
     parser = argparse.ArgumentParser()
     parser.add_argument('families', nargs='*', default=[], help='Families to build')
     parser.add_argument('-b', '--board', action='append', default=[], help='Boards to build')
+    parser.add_argument('-c', '--clean', action='store_true', default=False, help='Clean before build')
     parser.add_argument('-t', '--toolchain', default='gcc', help='Toolchain to use, default is gcc')
     parser.add_argument('-s', '--build-system', default='cmake', help='Build system to use, default is cmake')
     parser.add_argument('-D', '--define-symbol', action='append', default=[], help='Define to pass to build system')
@@ -243,6 +261,7 @@ def main():
     build_flags_on = args.build_flags_on
     one_per_family = args.one_per_family
     verbose = args.verbose
+    clean_build = args.clean
     parallel_jobs = args.jobs
 
     build_defines.append(f'TOOLCHAIN={toolchain}')
@@ -254,9 +273,8 @@ def main():
     print(build_separator)
     print(build_format.format('Board', 'Example', '\033[39mResult\033[0m', 'Time'))
     total_time = time.monotonic()
-    result = [0, 0, 0]
 
-    # build families
+    # get all families
     all_families = []
     if 'all' in families:
         for entry in os.scandir("hw/bsp"):
@@ -266,23 +284,19 @@ def main():
         all_families = list(families)
     all_families.sort()
 
-    # succeeded, failed, skipped
+    # get boards from families and append to boards list
+    all_boards = list(boards)
     for f in all_families:
-        r = build_family(f, build_defines, build_system, build_flags_on, one_per_family, boards)
-        result[0] += r[0]
-        result[1] += r[1]
-        result[2] += r[2]
+        all_boards.extend(get_family_boards(f, one_per_family, boards))
 
-    # build boards
-    r = build_boards_list(boards, build_defines, build_system, build_flags_on)
-    result[0] += r[0]
-    result[1] += r[1]
-    result[2] += r[2]
+    # build all boards
+    result = build_boards_list(all_boards, build_defines, build_system, build_flags_on)
 
     total_time = time.monotonic() - total_time
     print(build_separator)
     print(f"Build Summary: {result[0]} {STATUS_OK}, {result[1]} {STATUS_FAILED} and took {total_time:.2f}s")
     print(build_separator)
+
     return result[1]
 
 
diff --git a/tools/get_deps.py b/tools/get_deps.py
index d749e4c84..99e406ce7 100755
--- a/tools/get_deps.py
+++ b/tools/get_deps.py
@@ -14,6 +14,9 @@ deps_mandatory = {
     'lib/lwip': ['https://github.com/lwip-tcpip/lwip.git',
                  '159e31b689577dbf69cf0683bbaffbd71fa5ee10',
                  'all'],
+    'tools/linkermap': ['https://github.com/hathach/linkermap.git',
+                        '8a8206c39d0dfd7abfa615a676b3291165fcd65c',
+                        'all'],
     'tools/uf2': ['https://github.com/microsoft/uf2.git',
                   'c594542b2faa01cc33a2b97c9fbebc38549df80a',
                   'all'],
diff --git a/tools/metrics.py b/tools/metrics.py
new file mode 100644
index 000000000..bb84f803e
--- /dev/null
+++ b/tools/metrics.py
@@ -0,0 +1,379 @@
+#!/usr/bin/env python3
+"""Calculate average size from multiple linker map files."""
+
+import argparse
+import glob
+import json
+import sys
+import os
+
+# Add linkermap module to path
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), 'linkermap'))
+import linkermap
+
+
+def expand_files(file_patterns):
+    """Expand file patterns (globs) to list of files.
+
+    Args:
+        file_patterns: List of file paths or glob patterns
+
+    Returns:
+        List of expanded file paths
+    """
+    expanded = []
+    for pattern in file_patterns:
+        if '*' in pattern or '?' in pattern:
+            expanded.extend(glob.glob(pattern))
+        else:
+            expanded.append(pattern)
+    return expanded
+
+
+def combine_maps(map_files, filters=None):
+    """Combine multiple map files into a list of json_data.
+
+    Args:
+        map_files: List of paths to linker map files or JSON files
+        filters: List of path substrings to filter object files (default: [])
+
+    Returns:
+        all_json_data: Dictionary with mapfiles list and data from each map file
+    """
+    filters = filters or []
+    all_json_data = {"mapfiles": [], "data": []}
+
+    for map_file in map_files:
+        if not os.path.exists(map_file):
+            print(f"Warning: {map_file} not found, skipping", file=sys.stderr)
+            continue
+
+        try:
+            if map_file.endswith('.json'):
+                with open(map_file, 'r', encoding='utf-8') as f:
+                    json_data = json.load(f)
+                # Apply path filters to JSON data
+                if filters:
+                    filtered_files = [
+                        f for f in json_data.get("files", [])
+                        if f.get("path") and any(filt in f["path"] for filt in filters)
+                    ]
+                    json_data["files"] = filtered_files
+            else:
+                json_data = linkermap.analyze_map(map_file, filters=filters)
+            all_json_data["mapfiles"].append(map_file)
+            all_json_data["data"].append(json_data)
+        except Exception as e:
+            print(f"Warning: Failed to analyze {map_file}: {e}", file=sys.stderr)
+            continue
+
+    return all_json_data
+
+
+def compute_avg(all_json_data):
+    """Compute average sizes from combined json_data.
+
+    Args:
+        all_json_data: Dictionary with mapfiles and data from combine_maps()
+
+    Returns:
+        json_average: Dictionary with averaged size data
+    """
+    if not all_json_data["data"]:
+        return None
+
+    # Collect all sections preserving order
+    all_sections = []
+    for json_data in all_json_data["data"]:
+        for s in json_data["sections"]:
+            if s not in all_sections:
+                all_sections.append(s)
+
+    # Merge files with the same 'file' value and compute averages
+    file_accumulator = {}  # key: file name, value: {"sections": {section: [sizes]}, "totals": [totals]}
+
+    for json_data in all_json_data["data"]:
+        for f in json_data["files"]:
+            fname = f["file"]
+            if fname not in file_accumulator:
+                file_accumulator[fname] = {"sections": {}, "totals": [], "path": f.get("path")}
+            file_accumulator[fname]["totals"].append(f["total"])
+            for section, size in f["sections"].items():
+                if section in file_accumulator[fname]["sections"]:
+                    file_accumulator[fname]["sections"][section].append(size)
+                else:
+                    file_accumulator[fname]["sections"][section] = [size]
+
+    # Build json_average with averaged values
+    files_average = []
+    for fname, data in file_accumulator.items():
+        avg_total = round(sum(data["totals"]) / len(data["totals"]))
+        avg_sections = {}
+        for section, sizes in data["sections"].items():
+            avg_sections[section] = round(sum(sizes) / len(sizes))
+        files_average.append({
+            "file": fname,
+            "path": data["path"],
+            "sections": avg_sections,
+            "total": avg_total
+        })
+
+    json_average = {
+        "mapfiles": all_json_data["mapfiles"],
+        "sections": all_sections,
+        "files": files_average
+    }
+
+    return json_average
+
+
+def compare_maps(base_file, new_file, filters=None):
+    """Compare two map/json files and generate difference report.
+
+    Args:
+        base_file: Path to base map/json file
+        new_file: Path to new map/json file
+        filters: List of path substrings to filter object files
+
+    Returns:
+        Dictionary with comparison data
+    """
+    filters = filters or []
+
+    # Load both files
+    base_data = combine_maps([base_file], filters)
+    new_data = combine_maps([new_file], filters)
+
+    if not base_data["data"] or not new_data["data"]:
+        return None
+
+    base_avg = compute_avg(base_data)
+    new_avg = compute_avg(new_data)
+
+    if not base_avg or not new_avg:
+        return None
+
+    # Collect all sections from both
+    all_sections = list(base_avg["sections"])
+    for s in new_avg["sections"]:
+        if s not in all_sections:
+            all_sections.append(s)
+
+    # Build file lookup
+    base_files = {f["file"]: f for f in base_avg["files"]}
+    new_files = {f["file"]: f for f in new_avg["files"]}
+
+    # Get all file names
+    all_file_names = set(base_files.keys()) | set(new_files.keys())
+
+    # Build comparison data
+    comparison = []
+    for fname in sorted(all_file_names):
+        base_f = base_files.get(fname)
+        new_f = new_files.get(fname)
+
+        row = {"file": fname, "sections": {}, "total": {}}
+
+        for section in all_sections:
+            base_val = base_f["sections"].get(section, 0) if base_f else 0
+            new_val = new_f["sections"].get(section, 0) if new_f else 0
+            row["sections"][section] = {"base": base_val, "new": new_val, "diff": new_val - base_val}
+
+        base_total = base_f["total"] if base_f else 0
+        new_total = new_f["total"] if new_f else 0
+        row["total"] = {"base": base_total, "new": new_total, "diff": new_total - base_total}
+
+        comparison.append(row)
+
+    return {
+        "base_file": base_file,
+        "new_file": new_file,
+        "sections": all_sections,
+        "files": comparison
+    }
+
+
+def format_diff(base, new, diff):
+    """Format a diff value with percentage."""
+    if base == 0 and new == 0:
+        return "0"
+    if base == 0:
+        return f"{new} (new)"
+    if new == 0:
+        return f"{base} ➡ 0"
+    if diff == 0:
+        return f"{base} ➡ {new}"
+    pct = (diff / base) * 100
+    sign = "+" if diff > 0 else ""
+    return f"{base} ➡ {new} ({sign}{diff}, {sign}{pct:.1f}%)"
+
+
+def get_sort_key(sort_order):
+    """Get sort key function based on sort order.
+
+    Args:
+        sort_order: One of 'size-', 'size+', 'name-', 'name+'
+
+    Returns:
+        Tuple of (key_func, reverse)
+    """
+    if sort_order == 'size-':
+        return lambda x: x.get('total', 0) if isinstance(x.get('total'), int) else x['total']['new'], True
+    elif sort_order == 'size+':
+        return lambda x: x.get('total', 0) if isinstance(x.get('total'), int) else x['total']['new'], False
+    elif sort_order == 'name-':
+        return lambda x: x.get('file', ''), True
+    else:  # name+
+        return lambda x: x.get('file', ''), False
+
+
+def write_compare_markdown(comparison, path, sort_order='size'):
+    """Write comparison data to markdown file."""
+    sections = comparison["sections"]
+
+    md_lines = [
+        "# TinyUSB Code Size Different Report",
+        "",
+        f"**Base:** `{comparison['base_file']}`",
+        f"**New:** `{comparison['new_file']}`",
+        "",
+    ]
+
+    # Build header
+    header = "| File |"
+    separator = "|:-----|"
+    for s in sections:
+        header += f" {s} |"
+        separator += "-----:|"
+    header += " Total |"
+    separator += "------:|"
+
+    md_lines.append(header)
+    md_lines.append(separator)
+
+    # Sort files based on sort_order
+    if sort_order == 'size-':
+        key_func = lambda x: abs(x["total"]["diff"])
+        reverse = True
+    elif sort_order in ('size', 'size+'):
+        key_func = lambda x: abs(x["total"]["diff"])
+        reverse = False
+    elif sort_order == 'name-':
+        key_func = lambda x: x['file']
+        reverse = True
+    else:  # name or name+
+        key_func = lambda x: x['file']
+        reverse = False
+    sorted_files = sorted(comparison["files"], key=key_func, reverse=reverse)
+
+    sum_base = {s: 0 for s in sections}
+    sum_base["total"] = 0
+    sum_new = {s: 0 for s in sections}
+    sum_new["total"] = 0
+
+    for f in sorted_files:
+        # Skip files with no changes
+        if f["total"]["diff"] == 0 and all(f["sections"][s]["diff"] == 0 for s in sections):
+            continue
+
+        row = f"| {f['file']} |"
+        for s in sections:
+            sd = f["sections"][s]
+            sum_base[s] += sd["base"]
+            sum_new[s] += sd["new"]
+            row += f" {format_diff(sd['base'], sd['new'], sd['diff'])} |"
+
+        td = f["total"]
+        sum_base["total"] += td["base"]
+        sum_new["total"] += td["new"]
+        row += f" {format_diff(td['base'], td['new'], td['diff'])} |"
+
+        md_lines.append(row)
+
+    # Add sum row
+    sum_row = "| **SUM** |"
+    for s in sections:
+        diff = sum_new[s] - sum_base[s]
+        sum_row += f" {format_diff(sum_base[s], sum_new[s], diff)} |"
+    total_diff = sum_new["total"] - sum_base["total"]
+    sum_row += f" {format_diff(sum_base['total'], sum_new['total'], total_diff)} |"
+    md_lines.append(sum_row)
+
+    with open(path, "w", encoding="utf-8") as f:
+        f.write("\n".join(md_lines))
+
+
+def cmd_combine(args):
+    """Handle combine subcommand."""
+    map_files = expand_files(args.files)
+    all_json_data = combine_maps(map_files, args.filters)
+    json_average = compute_avg(all_json_data)
+
+    if json_average is None:
+        print("No valid map files found", file=sys.stderr)
+        sys.exit(1)
+
+    if not args.quiet:
+        linkermap.print_summary(json_average, False, args.sort)
+    if args.json_out:
+        linkermap.write_json(json_average, args.out + '.json')
+    if args.markdown_out:
+        linkermap.write_markdown(json_average, args.out + '.md', sort_opt=args.sort,
+                                 title="TinyUSB Average Code Size Metrics")
+
+
+def cmd_compare(args):
+    """Handle compare subcommand."""
+    comparison = compare_maps(args.base, args.new, args.filters)
+
+    if comparison is None:
+        print("Failed to compare files", file=sys.stderr)
+        sys.exit(1)
+
+    write_compare_markdown(comparison, args.out + '.md', args.sort)
+    print(f"Comparison written to {args.out}.md")
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description='Code size metrics tool')
+    subparsers = parser.add_subparsers(dest='command', required=True, help='Available commands')
+
+    # Combine subcommand
+    combine_parser = subparsers.add_parser('combine', help='Combine and average multiple map files')
+    combine_parser.add_argument('files', nargs='+', help='Path to map file(s) or glob pattern(s)')
+    combine_parser.add_argument('-f', '--filter', dest='filters', action='append', default=[],
+                                help='Only include object files whose path contains this substring (can be repeated)')
+    combine_parser.add_argument('-o', '--out', dest='out', default='metrics',
+                                help='Output path basename for JSON and Markdown files (default: metrics)')
+    combine_parser.add_argument('-j', '--json', dest='json_out', action='store_true',
+                                help='Write JSON output file')
+    combine_parser.add_argument('-m', '--markdown', dest='markdown_out', action='store_true',
+                                help='Write Markdown output file')
+    combine_parser.add_argument('-q', '--quiet', dest='quiet', action='store_true',
+                                help='Suppress summary output')
+    combine_parser.add_argument('-S', '--sort', dest='sort', default='name+',
+                                choices=['size', 'size-', 'size+', 'name', 'name-', 'name+'],
+                                help='Sort order: size/size- (descending), size+ (ascending), name/name+ (ascending), name- (descending). Default: name+')
+
+    # Compare subcommand
+    compare_parser = subparsers.add_parser('compare', help='Compare two map files')
+    compare_parser.add_argument('base', help='Base map/json file')
+    compare_parser.add_argument('new', help='New map/json file')
+    compare_parser.add_argument('-f', '--filter', dest='filters', action='append', default=[],
+                                help='Only include object files whose path contains this substring (can be repeated)')
+    compare_parser.add_argument('-o', '--out', dest='out', default='metrics_compare',
+                                help='Output path basename for Markdown file (default: metrics_compare)')
+    compare_parser.add_argument('-S', '--sort', dest='sort', default='name+',
+                                choices=['size', 'size-', 'size+', 'name', 'name-', 'name+'],
+                                help='Sort order: size/size- (descending), size+ (ascending), name/name+ (ascending), name- (descending). Default: name+')
+
+    args = parser.parse_args(argv)
+
+    if args.command == 'combine':
+        cmd_combine(args)
+    elif args.command == 'compare':
+        cmd_compare(args)
+
+
+if __name__ == '__main__':
+    main()