From 25c03648e9faf334d5f97ab8a37b3b199a60fcfb Mon Sep 17 00:00:00 2001 From: Anton Moryakov Date: Thu, 30 Jan 2025 16:42:43 +0300 Subject: tools: fix NULL_AFTER_DEREF in image-host.c Report of the static analyzer: 1. NULL_AFTER_DEREF Pointer 'str', which is dereferenced at image-host.c:688 by calling function 'strdup', is compared to a NULL value at image-host.c:691. 2. NULL_AFTER_DEREF Pointer 'list', which is dereferenced at image-host.c:689, is compared to a NULL value at image-host.c:691. Corrections explained: 1. Checking for NULL before using pointers: The if (!list || !str) check is now performed before calling strdup and realloc, which prevents null pointer dereferences. 2. Checking the result of strdup: strdup can return NULL if memory allocation fails. This also needs to be checked. 3. Checking the result of realloc: If realloc returns NULL, then memory has not been allocated and dup must be freed to avoid memory leaks. Triggers found by static analyzer Svace. Signed-off-by: Anton Moryakov --- tools/image-host.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/image-host.c b/tools/image-host.c index 84095d760c1..05d8c898209 100644 --- a/tools/image-host.c +++ b/tools/image-host.c @@ -716,11 +716,20 @@ static int strlist_add(struct strlist *list, const char *str) { char *dup; + if (!list || !str) + return -1; + dup = strdup(str); + if(!dup) + return -1; + list->strings = realloc(list->strings, (list->count + 1) * sizeof(char *)); - if (!list || !str) + if (!list->strings) { + free(dup); return -1; + } + list->strings[list->count++] = dup; return 0; -- cgit v1.3.1 From 4545880c3c86ff48fc88073f390ccab9cff7c11b Mon Sep 17 00:00:00 2001 From: Maks Mishin Date: Sun, 2 Feb 2025 20:05:42 +0300 Subject: tools: ublimage: Fix memory leak in parse_cfg_file() Dynamic memory, referenced by 'line', is allocated at ublimage.c:159 by calling function 'getline' and lost at ublimage.c:184. Signed-off-by: Maks Mishin --- tools/ublimage.c | 1 + 1 file changed, 1 insertion(+) (limited to 'tools') diff --git a/tools/ublimage.c b/tools/ublimage.c index 8f9b58c7983..a1bd807bfa0 100644 --- a/tools/ublimage.c +++ b/tools/ublimage.c @@ -178,6 +178,7 @@ static uint32_t parse_cfg_file(struct ubl_header *ublhdr, char *name) lineno, fld, &dcd_len); } } + free(line); fclose(fd); return dcd_len; -- cgit v1.3.1 From 4e18b47c73cdffb1dbd9ea196aeaff20e5030ade Mon Sep 17 00:00:00 2001 From: Maks Mishin Date: Wed, 5 Feb 2025 19:26:07 +0300 Subject: tools: proftool: Fix potential memory leaks Dynamic memory, referenced by 'line', is allocated by calling function 'calloc' and lost when the function terminates with code -1. Signed-off-by: Maks Mishin --- tools/proftool.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'tools') diff --git a/tools/proftool.c b/tools/proftool.c index af2cdb6d584..c7b427f3078 100644 --- a/tools/proftool.c +++ b/tools/proftool.c @@ -676,6 +676,7 @@ static int read_trace_config(FILE *fin) if (!tok) { error("Invalid trace config data on line %d\n", linenum); + free(line); return -1; } if (0 == strcmp(tok, "include-func")) { @@ -685,6 +686,7 @@ static int read_trace_config(FILE *fin) } else { error("Unknown command in trace config data line %d\n", linenum); + free(line); return -1; } @@ -692,6 +694,7 @@ static int read_trace_config(FILE *fin) if (!tok) { error("Missing pattern in trace config data line %d\n", linenum); + free(line); return -1; } -- cgit v1.3.1 From 5818fcf32e1c4a994d520a86d0cc287ebbd2cfbd Mon Sep 17 00:00:00 2001 From: Tom Rini Date: Tue, 4 Feb 2025 17:12:04 -0600 Subject: python: Create requirements.txt files for each "project" Rather than have a requirements.txt file that's shared between multiple python projects within U-Boot, create one for each using "pipreqs". Signed-off-by: Tom Rini --- tools/binman/requirements.txt | 5 +++++ tools/patman/requirements.txt | 5 +++++ tools/u_boot_pylib/requirements.txt | 1 + 3 files changed, 11 insertions(+) create mode 100644 tools/binman/requirements.txt create mode 100644 tools/patman/requirements.txt create mode 100644 tools/u_boot_pylib/requirements.txt (limited to 'tools') diff --git a/tools/binman/requirements.txt b/tools/binman/requirements.txt new file mode 100644 index 00000000000..f068ef75a30 --- /dev/null +++ b/tools/binman/requirements.txt @@ -0,0 +1,5 @@ +importlib_resources==6.5.2 +jsonschema==4.23.0 +pycryptodomex==3.21.0 +pyelftools==0.31 +yamllint==1.35.1 diff --git a/tools/patman/requirements.txt b/tools/patman/requirements.txt new file mode 100644 index 00000000000..e8cbc6cf0c3 --- /dev/null +++ b/tools/patman/requirements.txt @@ -0,0 +1,5 @@ +ConfigParser==7.1.0 +importlib_resources==6.5.2 +pygit2==1.13.3 +Requests==2.32.3 +setuptools==75.8.0 diff --git a/tools/u_boot_pylib/requirements.txt b/tools/u_boot_pylib/requirements.txt new file mode 100644 index 00000000000..1087e6f2857 --- /dev/null +++ b/tools/u_boot_pylib/requirements.txt @@ -0,0 +1 @@ +concurrencytest==0.1.2 -- cgit v1.3.1 From 859621b47f9e660340665fc18af6d4a01f4ed749 Mon Sep 17 00:00:00 2001 From: Tom Rini Date: Tue, 4 Feb 2025 17:12:08 -0600 Subject: python: Recreate test/py and tools/buildman requirements.txt files Use the "pipreqs" tool to re-create these files, with a few manual corrections. We still need to include pytest-xdist which the tool does not detect. We also for now don't upgrade most of the required tools as that creates problems with various tests, which should be resolved independently. Signed-off-by: Tom Rini --- test/py/requirements.txt | 28 +--------------------------- tools/buildman/requirements.txt | 7 ++----- 2 files changed, 3 insertions(+), 32 deletions(-) (limited to 'tools') diff --git a/test/py/requirements.txt b/test/py/requirements.txt index 75760f96e56..acfe17dce9f 100644 --- a/test/py/requirements.txt +++ b/test/py/requirements.txt @@ -1,30 +1,4 @@ -atomicwrites==1.4.1 -attrs==19.3.0 -concurrencytest==0.1.2 -coverage==6.2 -extras==1.0.0 filelock==3.0.12 -fixtures==3.0.0 -importlib-metadata==0.23 -linecache2==1.0.0 -more-itertools==7.2.0 -packaging==24.1 -pbr==5.4.3 -pluggy==0.13.0 -py==1.11.0 -pycryptodomex==3.19.1 -pyelftools==0.27 -pygit2==1.13.3 -pyparsing==3.0.7 +pycryptodomex==3.21.0 pytest==6.2.5 pytest-xdist==2.5.0 -python-mimeparse==1.6.0 -python-subunit==1.3.0 -requests==2.32.3 -setuptools==70.3.0 -six==1.16.0 -testtools==2.3.0 -traceback2==1.4.0 -unittest2==1.1.0 -wcwidth==0.1.7 -zipp==3.19.2 diff --git a/tools/buildman/requirements.txt b/tools/buildman/requirements.txt index 052d0ed5c6f..d48650cd1e5 100644 --- a/tools/buildman/requirements.txt +++ b/tools/buildman/requirements.txt @@ -1,5 +1,2 @@ -coverage==6.2 -jsonschema==4.17.3 -pycryptodome==3.20 -pyyaml==6.0 -yamllint==1.26.3 +filelock==3.0.12 +importlib_resources==6.5.2 -- cgit v1.3.1 From fc9f3dd2e91ad2da73d011fd624cb46591a51933 Mon Sep 17 00:00:00 2001 From: Tom Rini Date: Tue, 4 Feb 2025 17:12:09 -0600 Subject: Dockerfile: Update for having more requirements.txt files Now that we have more requirements.txt files we need to grab all of them for creating our cache. Also, we do longer should install python3-pyelftools on the host as it's not used. Signed-off-by: Tom Rini --- tools/docker/Dockerfile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'tools') diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile index d2848ab85f3..85d67848327 100644 --- a/tools/docker/Dockerfile +++ b/tools/docker/Dockerfile @@ -122,7 +122,6 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ python3 \ python3-dev \ python3-pip \ - python3-pyelftools \ python3-sphinx \ python3-virtualenv \ rpm2cpio \ @@ -308,12 +307,18 @@ USER uboot:uboot # COPY / ADD directives don't work as we need them to. RUN wget -O /tmp/pytest-requirements.txt https://source.denx.de/u-boot/u-boot/-/raw/master/test/py/requirements.txt RUN wget -O /tmp/sphinx-requirements.txt https://source.denx.de/u-boot/u-boot/-/raw/master/doc/sphinx/requirements.txt +RUN wget -O /tmp/binman-requirements.txt https://source.denx.de/u-boot/u-boot/-/raw/master/tools/binman/requirements.txt RUN wget -O /tmp/buildman-requirements.txt https://source.denx.de/u-boot/u-boot/-/raw/master/tools/buildman/requirements.txt +RUN wget -O /tmp/patman-requirements.txt https://source.denx.de/u-boot/u-boot/-/raw/master/tools/patman/requirements.txt +RUN wget -O /tmp/u_boot_pylib-requirements.txt https://source.denx.de/u-boot/u-boot/-/raw/master/tools/u_boot_pylib/requirements.txt RUN virtualenv -p /usr/bin/python3 /tmp/venv && \ . /tmp/venv/bin/activate && \ pip install -r /tmp/pytest-requirements.txt \ -r /tmp/sphinx-requirements.txt \ - -r /tmp/buildman-requirements.txt && \ + -r /tmp/binman-requirements.txt \ + -r /tmp/buildman-requirements.txt \ + -r /tmp/patman-requirements.txt \ + -r /tmp/u_boot_pylib-requirements.txt && \ deactivate && \ rm -rf /tmp/venv /tmp/*-requirements.txt -- cgit v1.3.1 From 5c33fb028865bdc490aa0db2980987149786b00f Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sun, 9 Feb 2025 14:26:00 -0700 Subject: u_boot_pylib: Move gitutil into the library Move this file into U-Boot's Python library, so that it is no-longer part of patman. This makes a start on: https://source.denx.de/u-boot/custodians/u-boot-dm/-/issues/35 Signed-off-by: Simon Glass --- tools/buildman/builder.py | 2 +- tools/buildman/builderthread.py | 2 +- tools/buildman/control.py | 2 +- tools/buildman/func_test.py | 2 +- tools/buildman/main.py | 3 +- tools/patman/__init__.py | 4 +- tools/patman/__main__.py | 2 +- tools/patman/checkpatch.py | 2 +- tools/patman/cmdline.py | 2 +- tools/patman/control.py | 2 +- tools/patman/func_test.py | 2 +- tools/patman/get_maintainer.py | 2 +- tools/patman/gitutil.py | 706 ---------------------------------------- tools/patman/patchstream.py | 2 +- tools/patman/project.py | 2 +- tools/patman/series.py | 2 +- tools/patman/settings.py | 2 +- tools/patman/test_checkpatch.py | 2 +- tools/u_boot_pylib/__init__.py | 4 +- tools/u_boot_pylib/__main__.py | 1 - tools/u_boot_pylib/gitutil.py | 706 ++++++++++++++++++++++++++++++++++++++++ 21 files changed, 726 insertions(+), 728 deletions(-) delete mode 100644 tools/patman/gitutil.py create mode 100644 tools/u_boot_pylib/gitutil.py (limited to 'tools') diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index cbf1345281b..2568e4e8423 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -19,8 +19,8 @@ import time from buildman import builderthread from buildman import toolchain -from patman import gitutil from u_boot_pylib import command +from u_boot_pylib import gitutil from u_boot_pylib import terminal from u_boot_pylib import tools from u_boot_pylib.terminal import tprint diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index 29e6cf32af1..78c95a67095 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -17,8 +17,8 @@ import sys import threading from buildman import cfgutil -from patman import gitutil from u_boot_pylib import command +from u_boot_pylib import gitutil from u_boot_pylib import tools RETURN_CODE_RETRY = -1 diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 55d4d770c5c..5109b1cd5ce 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -20,9 +20,9 @@ from buildman import bsettings from buildman import cfgutil from buildman import toolchain from buildman.builder import Builder -from patman import gitutil from patman import patchstream from u_boot_pylib import command +from u_boot_pylib import gitutil from u_boot_pylib import terminal from u_boot_pylib import tools from u_boot_pylib.terminal import print_clear, tprint diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index 4e12c671a3d..c7c4f506a65 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -18,8 +18,8 @@ from buildman import bsettings from buildman import cmdline from buildman import control from buildman import toolchain -from patman import gitutil from u_boot_pylib import command +from u_boot_pylib import gitutil from u_boot_pylib import terminal from u_boot_pylib import test_util from u_boot_pylib import tools diff --git a/tools/buildman/main.py b/tools/buildman/main.py index a948f36d9c0..72571b226d9 100755 --- a/tools/buildman/main.py +++ b/tools/buildman/main.py @@ -50,8 +50,7 @@ def run_tests(skip_net_tests, debug, verbose, args): # 'entry' module. result = test_util.run_test_suites( 'buildman', debug, verbose, False, args.threads, test_name, [], - [test.TestBuild, func_test.TestFunctional, - 'buildman.toolchain', 'patman.gitutil']) + [test.TestBuild, func_test.TestFunctional, 'buildman.toolchain']) return (0 if result.wasSuccessful() else 1) diff --git a/tools/patman/__init__.py b/tools/patman/__init__.py index 08eeffdf6d2..6de0e9fba10 100644 --- a/tools/patman/__init__.py +++ b/tools/patman/__init__.py @@ -1,5 +1,5 @@ # SPDX-License-Identifier: GPL-2.0+ __all__ = ['checkpatch', 'commit', 'control', 'func_test', 'get_maintainer', - 'gitutil', '__main__', 'patchstream', 'project', 'series', - 'settings','setup', 'status', 'test_checkpatch', 'test_settings'] + '__main__', 'patchstream', 'project', 'series', + 'settings', 'setup', 'status', 'test_checkpatch', 'test_settings'] diff --git a/tools/patman/__main__.py b/tools/patman/__main__.py index f645b38b647..36f1c08507c 100755 --- a/tools/patman/__main__.py +++ b/tools/patman/__main__.py @@ -49,7 +49,7 @@ def run_patman(): result = test_util.run_test_suites( 'patman', False, False, False, None, None, None, [test_checkpatch.TestPatch, func_test.TestFunctional, - 'gitutil', 'settings']) + 'settings']) sys.exit(0 if result.wasSuccessful() else 1) diff --git a/tools/patman/checkpatch.py b/tools/patman/checkpatch.py index e03cac115e4..2975881705c 100644 --- a/tools/patman/checkpatch.py +++ b/tools/patman/checkpatch.py @@ -8,8 +8,8 @@ import os import re import sys -from patman import gitutil from u_boot_pylib import command +from u_boot_pylib import gitutil from u_boot_pylib import terminal EMACS_PREFIX = r'(?:[0-9]{4}.*\.patch:[0-9]+: )?' diff --git a/tools/patman/cmdline.py b/tools/patman/cmdline.py index d6496c0cb78..562bc823f60 100644 --- a/tools/patman/cmdline.py +++ b/tools/patman/cmdline.py @@ -13,8 +13,8 @@ import os import pathlib import sys -from patman import gitutil from patman import project +from u_boot_pylib import gitutil from patman import settings PATMAN_DIR = pathlib.Path(__file__).parent diff --git a/tools/patman/control.py b/tools/patman/control.py index b292da9dc27..fb5a4246ced 100644 --- a/tools/patman/control.py +++ b/tools/patman/control.py @@ -12,8 +12,8 @@ import os import sys from patman import checkpatch -from patman import gitutil from patman import patchstream +from u_boot_pylib import gitutil from u_boot_pylib import terminal diff --git a/tools/patman/func_test.py b/tools/patman/func_test.py index af6c025a441..bf333dc557b 100644 --- a/tools/patman/func_test.py +++ b/tools/patman/func_test.py @@ -18,11 +18,11 @@ import unittest from patman.commit import Commit from patman import control -from patman import gitutil from patman import patchstream from patman.patchstream import PatchStream from patman.series import Series from patman import settings +from u_boot_pylib import gitutil from u_boot_pylib import terminal from u_boot_pylib import tools from u_boot_pylib.test_util import capture_sys_output diff --git a/tools/patman/get_maintainer.py b/tools/patman/get_maintainer.py index 8df3d124bac..200ee96551d 100644 --- a/tools/patman/get_maintainer.py +++ b/tools/patman/get_maintainer.py @@ -7,8 +7,8 @@ import os import shlex import shutil -from patman import gitutil from u_boot_pylib import command +from u_boot_pylib import gitutil def find_get_maintainer(script_file_name): diff --git a/tools/patman/gitutil.py b/tools/patman/gitutil.py deleted file mode 100644 index 10ea5ff39f5..00000000000 --- a/tools/patman/gitutil.py +++ /dev/null @@ -1,706 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0+ -# Copyright (c) 2011 The Chromium OS Authors. -# - -import os -import sys - -from patman import settings -from u_boot_pylib import command -from u_boot_pylib import terminal - -# True to use --no-decorate - we check this in setup() -use_no_decorate = True - - -def log_cmd(commit_range, git_dir=None, oneline=False, reverse=False, - count=None): - """Create a command to perform a 'git log' - - Args: - commit_range: Range expression to use for log, None for none - git_dir: Path to git repository (None to use default) - oneline: True to use --oneline, else False - reverse: True to reverse the log (--reverse) - count: Number of commits to list, or None for no limit - Return: - List containing command and arguments to run - """ - cmd = ['git'] - if git_dir: - cmd += ['--git-dir', git_dir] - cmd += ['--no-pager', 'log', '--no-color'] - if oneline: - cmd.append('--oneline') - if use_no_decorate: - cmd.append('--no-decorate') - if reverse: - cmd.append('--reverse') - if count is not None: - cmd.append('-n%d' % count) - if commit_range: - cmd.append(commit_range) - - # Add this in case we have a branch with the same name as a directory. - # This avoids messages like this, for example: - # fatal: ambiguous argument 'test': both revision and filename - cmd.append('--') - return cmd - - -def count_commits_to_branch(branch): - """Returns number of commits between HEAD and the tracking branch. - - This looks back to the tracking branch and works out the number of commits - since then. - - Args: - branch: Branch to count from (None for current branch) - - Return: - Number of patches that exist on top of the branch - """ - if branch: - us, msg = get_upstream('.git', branch) - rev_range = '%s..%s' % (us, branch) - else: - rev_range = '@{upstream}..' - pipe = [log_cmd(rev_range, oneline=True)] - result = command.run_pipe(pipe, capture=True, capture_stderr=True, - oneline=True, raise_on_error=False) - if result.return_code: - raise ValueError('Failed to determine upstream: %s' % - result.stderr.strip()) - patch_count = len(result.stdout.splitlines()) - return patch_count - - -def name_revision(commit_hash): - """Gets the revision name for a commit - - Args: - commit_hash: Commit hash to look up - - Return: - Name of revision, if any, else None - """ - pipe = ['git', 'name-rev', commit_hash] - stdout = command.run_pipe([pipe], capture=True, oneline=True).stdout - - # We expect a commit, a space, then a revision name - name = stdout.split(' ')[1].strip() - return name - - -def guess_upstream(git_dir, branch): - """Tries to guess the upstream for a branch - - This lists out top commits on a branch and tries to find a suitable - upstream. It does this by looking for the first commit where - 'git name-rev' returns a plain branch name, with no ! or ^ modifiers. - - Args: - git_dir: Git directory containing repo - branch: Name of branch - - Returns: - Tuple: - Name of upstream branch (e.g. 'upstream/master') or None if none - Warning/error message, or None if none - """ - pipe = [log_cmd(branch, git_dir=git_dir, oneline=True, count=100)] - result = command.run_pipe(pipe, capture=True, capture_stderr=True, - raise_on_error=False) - if result.return_code: - return None, "Branch '%s' not found" % branch - for line in result.stdout.splitlines()[1:]: - commit_hash = line.split(' ')[0] - name = name_revision(commit_hash) - if '~' not in name and '^' not in name: - if name.startswith('remotes/'): - name = name[8:] - return name, "Guessing upstream as '%s'" % name - return None, "Cannot find a suitable upstream for branch '%s'" % branch - - -def get_upstream(git_dir, branch): - """Returns the name of the upstream for a branch - - Args: - git_dir: Git directory containing repo - branch: Name of branch - - Returns: - Tuple: - Name of upstream branch (e.g. 'upstream/master') or None if none - Warning/error message, or None if none - """ - try: - remote = command.output_one_line('git', '--git-dir', git_dir, 'config', - 'branch.%s.remote' % branch) - merge = command.output_one_line('git', '--git-dir', git_dir, 'config', - 'branch.%s.merge' % branch) - except Exception: - upstream, msg = guess_upstream(git_dir, branch) - return upstream, msg - - if remote == '.': - return merge, None - elif remote and merge: - # Drop the initial refs/heads from merge - leaf = merge.split('/', maxsplit=2)[2:] - return '%s/%s' % (remote, '/'.join(leaf)), None - else: - raise ValueError("Cannot determine upstream branch for branch " - "'%s' remote='%s', merge='%s'" - % (branch, remote, merge)) - - -def get_range_in_branch(git_dir, branch, include_upstream=False): - """Returns an expression for the commits in the given branch. - - Args: - git_dir: Directory containing git repo - branch: Name of branch - Return: - Expression in the form 'upstream..branch' which can be used to - access the commits. If the branch does not exist, returns None. - """ - upstream, msg = get_upstream(git_dir, branch) - if not upstream: - return None, msg - rstr = '%s%s..%s' % (upstream, '~' if include_upstream else '', branch) - return rstr, msg - - -def count_commits_in_range(git_dir, range_expr): - """Returns the number of commits in the given range. - - Args: - git_dir: Directory containing git repo - range_expr: Range to check - Return: - Number of patches that exist in the supplied range or None if none - were found - """ - pipe = [log_cmd(range_expr, git_dir=git_dir, oneline=True)] - result = command.run_pipe(pipe, capture=True, capture_stderr=True, - raise_on_error=False) - if result.return_code: - return None, "Range '%s' not found or is invalid" % range_expr - patch_count = len(result.stdout.splitlines()) - return patch_count, None - - -def count_commits_in_branch(git_dir, branch, include_upstream=False): - """Returns the number of commits in the given branch. - - Args: - git_dir: Directory containing git repo - branch: Name of branch - Return: - Number of patches that exist on top of the branch, or None if the - branch does not exist. - """ - range_expr, msg = get_range_in_branch(git_dir, branch, include_upstream) - if not range_expr: - return None, msg - return count_commits_in_range(git_dir, range_expr) - - -def count_commits(commit_range): - """Returns the number of commits in the given range. - - Args: - commit_range: Range of commits to count (e.g. 'HEAD..base') - Return: - Number of patches that exist on top of the branch - """ - pipe = [log_cmd(commit_range, oneline=True), - ['wc', '-l']] - stdout = command.run_pipe(pipe, capture=True, oneline=True).stdout - patch_count = int(stdout) - return patch_count - - -def checkout(commit_hash, git_dir=None, work_tree=None, force=False): - """Checkout the selected commit for this build - - Args: - commit_hash: Commit hash to check out - """ - pipe = ['git'] - if git_dir: - pipe.extend(['--git-dir', git_dir]) - if work_tree: - pipe.extend(['--work-tree', work_tree]) - pipe.append('checkout') - if force: - pipe.append('-f') - pipe.append(commit_hash) - result = command.run_pipe([pipe], capture=True, raise_on_error=False, - capture_stderr=True) - if result.return_code != 0: - raise OSError('git checkout (%s): %s' % (pipe, result.stderr)) - - -def clone(git_dir, output_dir): - """Checkout the selected commit for this build - - Args: - commit_hash: Commit hash to check out - """ - pipe = ['git', 'clone', git_dir, '.'] - result = command.run_pipe([pipe], capture=True, cwd=output_dir, - capture_stderr=True) - if result.return_code != 0: - raise OSError('git clone: %s' % result.stderr) - - -def fetch(git_dir=None, work_tree=None): - """Fetch from the origin repo - - Args: - commit_hash: Commit hash to check out - """ - pipe = ['git'] - if git_dir: - pipe.extend(['--git-dir', git_dir]) - if work_tree: - pipe.extend(['--work-tree', work_tree]) - pipe.append('fetch') - result = command.run_pipe([pipe], capture=True, capture_stderr=True) - if result.return_code != 0: - raise OSError('git fetch: %s' % result.stderr) - - -def check_worktree_is_available(git_dir): - """Check if git-worktree functionality is available - - Args: - git_dir: The repository to test in - - Returns: - True if git-worktree commands will work, False otherwise. - """ - pipe = ['git', '--git-dir', git_dir, 'worktree', 'list'] - result = command.run_pipe([pipe], capture=True, capture_stderr=True, - raise_on_error=False) - return result.return_code == 0 - - -def add_worktree(git_dir, output_dir, commit_hash=None): - """Create and checkout a new git worktree for this build - - Args: - git_dir: The repository to checkout the worktree from - output_dir: Path for the new worktree - commit_hash: Commit hash to checkout - """ - # We need to pass --detach to avoid creating a new branch - pipe = ['git', '--git-dir', git_dir, 'worktree', 'add', '.', '--detach'] - if commit_hash: - pipe.append(commit_hash) - result = command.run_pipe([pipe], capture=True, cwd=output_dir, - capture_stderr=True) - if result.return_code != 0: - raise OSError('git worktree add: %s' % result.stderr) - - -def prune_worktrees(git_dir): - """Remove administrative files for deleted worktrees - - Args: - git_dir: The repository whose deleted worktrees should be pruned - """ - pipe = ['git', '--git-dir', git_dir, 'worktree', 'prune'] - result = command.run_pipe([pipe], capture=True, capture_stderr=True) - if result.return_code != 0: - raise OSError('git worktree prune: %s' % result.stderr) - - -def create_patches(branch, start, count, ignore_binary, series, signoff=True): - """Create a series of patches from the top of the current branch. - - The patch files are written to the current directory using - git format-patch. - - Args: - branch: Branch to create patches from (None for current branch) - start: Commit to start from: 0=HEAD, 1=next one, etc. - count: number of commits to include - ignore_binary: Don't generate patches for binary files - series: Series object for this series (set of patches) - Return: - Filename of cover letter (None if none) - List of filenames of patch files - """ - cmd = ['git', 'format-patch', '-M'] - if signoff: - cmd.append('--signoff') - if ignore_binary: - cmd.append('--no-binary') - if series.get('cover'): - cmd.append('--cover-letter') - prefix = series.GetPatchPrefix() - if prefix: - cmd += ['--subject-prefix=%s' % prefix] - brname = branch or 'HEAD' - cmd += ['%s~%d..%s~%d' % (brname, start + count, brname, start)] - - stdout = command.run_list(cmd) - files = stdout.splitlines() - - # We have an extra file if there is a cover letter - if series.get('cover'): - return files[0], files[1:] - else: - return None, files - - -def build_email_list(in_list, tag=None, alias=None, warn_on_error=True): - """Build a list of email addresses based on an input list. - - Takes a list of email addresses and aliases, and turns this into a list - of only email address, by resolving any aliases that are present. - - If the tag is given, then each email address is prepended with this - tag and a space. If the tag starts with a minus sign (indicating a - command line parameter) then the email address is quoted. - - Args: - in_list: List of aliases/email addresses - tag: Text to put before each address - alias: Alias dictionary - warn_on_error: True to raise an error when an alias fails to match, - False to just print a message. - - Returns: - List of email addresses - - >>> alias = {} - >>> alias['fred'] = ['f.bloggs@napier.co.nz'] - >>> alias['john'] = ['j.bloggs@napier.co.nz'] - >>> alias['mary'] = ['Mary Poppins '] - >>> alias['boys'] = ['fred', ' john'] - >>> alias['all'] = ['fred ', 'john', ' mary '] - >>> build_email_list(['john', 'mary'], None, alias) - ['j.bloggs@napier.co.nz', 'Mary Poppins '] - >>> build_email_list(['john', 'mary'], '--to', alias) - ['--to "j.bloggs@napier.co.nz"', \ -'--to "Mary Poppins "'] - >>> build_email_list(['john', 'mary'], 'Cc', alias) - ['Cc j.bloggs@napier.co.nz', 'Cc Mary Poppins '] - """ - quote = '"' if tag and tag[0] == '-' else '' - raw = [] - for item in in_list: - raw += lookup_email(item, alias, warn_on_error=warn_on_error) - result = [] - for item in raw: - if item not in result: - result.append(item) - if tag: - return ['%s %s%s%s' % (tag, quote, email, quote) for email in result] - return result - - -def check_suppress_cc_config(): - """Check if sendemail.suppresscc is configured correctly. - - Returns: - True if the option is configured correctly, False otherwise. - """ - suppresscc = command.output_one_line( - 'git', 'config', 'sendemail.suppresscc', raise_on_error=False) - - # Other settings should be fine. - if suppresscc == 'all' or suppresscc == 'cccmd': - col = terminal.Color() - - print((col.build(col.RED, "error") + - ": git config sendemail.suppresscc set to %s\n" - % (suppresscc)) + - " patman needs --cc-cmd to be run to set the cc list.\n" + - " Please run:\n" + - " git config --unset sendemail.suppresscc\n" + - " Or read the man page:\n" + - " git send-email --help\n" + - " and set an option that runs --cc-cmd\n") - return False - - return True - - -def email_patches(series, cover_fname, args, dry_run, warn_on_error, cc_fname, - self_only=False, alias=None, in_reply_to=None, thread=False, - smtp_server=None, get_maintainer_script=None): - """Email a patch series. - - Args: - series: Series object containing destination info - cover_fname: filename of cover letter - args: list of filenames of patch files - dry_run: Just return the command that would be run - warn_on_error: True to print a warning when an alias fails to match, - False to ignore it. - cc_fname: Filename of Cc file for per-commit Cc - self_only: True to just email to yourself as a test - in_reply_to: If set we'll pass this to git as --in-reply-to. - Should be a message ID that this is in reply to. - thread: True to add --thread to git send-email (make - all patches reply to cover-letter or first patch in series) - smtp_server: SMTP server to use to send patches - get_maintainer_script: File name of script to get maintainers emails - - Returns: - Git command that was/would be run - - # For the duration of this doctest pretend that we ran patman with ./patman - >>> _old_argv0 = sys.argv[0] - >>> sys.argv[0] = './patman' - - >>> alias = {} - >>> alias['fred'] = ['f.bloggs@napier.co.nz'] - >>> alias['john'] = ['j.bloggs@napier.co.nz'] - >>> alias['mary'] = ['m.poppins@cloud.net'] - >>> alias['boys'] = ['fred', ' john'] - >>> alias['all'] = ['fred ', 'john', ' mary '] - >>> alias[os.getenv('USER')] = ['this-is-me@me.com'] - >>> series = {} - >>> series['to'] = ['fred'] - >>> series['cc'] = ['mary'] - >>> email_patches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \ - False, alias) - 'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \ -"m.poppins@cloud.net" --cc-cmd "./patman send --cc-cmd cc-fname" cover p1 p2' - >>> email_patches(series, None, ['p1'], True, True, 'cc-fname', False, \ - alias) - 'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \ -"m.poppins@cloud.net" --cc-cmd "./patman send --cc-cmd cc-fname" p1' - >>> series['cc'] = ['all'] - >>> email_patches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \ - True, alias) - 'git send-email --annotate --to "this-is-me@me.com" --cc-cmd "./patman \ -send --cc-cmd cc-fname" cover p1 p2' - >>> email_patches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \ - False, alias) - 'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \ -"f.bloggs@napier.co.nz" --cc "j.bloggs@napier.co.nz" --cc \ -"m.poppins@cloud.net" --cc-cmd "./patman send --cc-cmd cc-fname" cover p1 p2' - - # Restore argv[0] since we clobbered it. - >>> sys.argv[0] = _old_argv0 - """ - to = build_email_list(series.get('to'), '--to', alias, warn_on_error) - if not to: - git_config_to = command.output('git', 'config', 'sendemail.to', - raise_on_error=False) - if not git_config_to: - print("No recipient.\n" - "Please add something like this to a commit\n" - "Series-to: Fred Bloggs \n" - "Or do something like this\n" - "git config sendemail.to u-boot@lists.denx.de") - return - cc = build_email_list(list(set(series.get('cc')) - set(series.get('to'))), - '--cc', alias, warn_on_error) - if self_only: - to = build_email_list([os.getenv('USER')], '--to', - alias, warn_on_error) - cc = [] - cmd = ['git', 'send-email', '--annotate'] - if smtp_server: - cmd.append('--smtp-server=%s' % smtp_server) - if in_reply_to: - cmd.append('--in-reply-to="%s"' % in_reply_to) - if thread: - cmd.append('--thread') - - cmd += to - cmd += cc - cmd += ['--cc-cmd', '"%s send --cc-cmd %s"' % (sys.argv[0], cc_fname)] - if cover_fname: - cmd.append(cover_fname) - cmd += args - cmdstr = ' '.join(cmd) - if not dry_run: - os.system(cmdstr) - return cmdstr - - -def lookup_email(lookup_name, alias=None, warn_on_error=True, level=0): - """If an email address is an alias, look it up and return the full name - - TODO: Why not just use git's own alias feature? - - Args: - lookup_name: Alias or email address to look up - alias: Dictionary containing aliases (None to use settings default) - warn_on_error: True to print a warning when an alias fails to match, - False to ignore it. - - Returns: - tuple: - list containing a list of email addresses - - Raises: - OSError if a recursive alias reference was found - ValueError if an alias was not found - - >>> alias = {} - >>> alias['fred'] = ['f.bloggs@napier.co.nz'] - >>> alias['john'] = ['j.bloggs@napier.co.nz'] - >>> alias['mary'] = ['m.poppins@cloud.net'] - >>> alias['boys'] = ['fred', ' john', 'f.bloggs@napier.co.nz'] - >>> alias['all'] = ['fred ', 'john', ' mary '] - >>> alias['loop'] = ['other', 'john', ' mary '] - >>> alias['other'] = ['loop', 'john', ' mary '] - >>> lookup_email('mary', alias) - ['m.poppins@cloud.net'] - >>> lookup_email('arthur.wellesley@howe.ro.uk', alias) - ['arthur.wellesley@howe.ro.uk'] - >>> lookup_email('boys', alias) - ['f.bloggs@napier.co.nz', 'j.bloggs@napier.co.nz'] - >>> lookup_email('all', alias) - ['f.bloggs@napier.co.nz', 'j.bloggs@napier.co.nz', 'm.poppins@cloud.net'] - >>> lookup_email('odd', alias) - Alias 'odd' not found - [] - >>> lookup_email('loop', alias) - Traceback (most recent call last): - ... - OSError: Recursive email alias at 'other' - >>> lookup_email('odd', alias, warn_on_error=False) - [] - >>> # In this case the loop part will effectively be ignored. - >>> lookup_email('loop', alias, warn_on_error=False) - Recursive email alias at 'other' - Recursive email alias at 'john' - Recursive email alias at 'mary' - ['j.bloggs@napier.co.nz', 'm.poppins@cloud.net'] - """ - if not alias: - alias = settings.alias - lookup_name = lookup_name.strip() - if '@' in lookup_name: # Perhaps a real email address - return [lookup_name] - - lookup_name = lookup_name.lower() - col = terminal.Color() - - out_list = [] - if level > 10: - msg = "Recursive email alias at '%s'" % lookup_name - if warn_on_error: - raise OSError(msg) - else: - print(col.build(col.RED, msg)) - return out_list - - if lookup_name: - if lookup_name not in alias: - msg = "Alias '%s' not found" % lookup_name - if warn_on_error: - print(col.build(col.RED, msg)) - return out_list - for item in alias[lookup_name]: - todo = lookup_email(item, alias, warn_on_error, level + 1) - for new_item in todo: - if new_item not in out_list: - out_list.append(new_item) - - return out_list - - -def get_top_level(): - """Return name of top-level directory for this git repo. - - Returns: - Full path to git top-level directory - - This test makes sure that we are running tests in the right subdir - - >>> os.path.realpath(os.path.dirname(__file__)) == \ - os.path.join(get_top_level(), 'tools', 'patman') - True - """ - return command.output_one_line('git', 'rev-parse', '--show-toplevel') - - -def get_alias_file(): - """Gets the name of the git alias file. - - Returns: - Filename of git alias file, or None if none - """ - fname = command.output_one_line('git', 'config', 'sendemail.aliasesfile', - raise_on_error=False) - if not fname: - return None - - fname = os.path.expanduser(fname.strip()) - if os.path.isabs(fname): - return fname - - return os.path.join(get_top_level(), fname) - - -def get_default_user_name(): - """Gets the user.name from .gitconfig file. - - Returns: - User name found in .gitconfig file, or None if none - """ - uname = command.output_one_line('git', 'config', '--global', '--includes', 'user.name') - return uname - - -def get_default_user_email(): - """Gets the user.email from the global .gitconfig file. - - Returns: - User's email found in .gitconfig file, or None if none - """ - uemail = command.output_one_line('git', 'config', '--global', '--includes', 'user.email') - return uemail - - -def get_default_subject_prefix(): - """Gets the format.subjectprefix from local .git/config file. - - Returns: - Subject prefix found in local .git/config file, or None if none - """ - sub_prefix = command.output_one_line( - 'git', 'config', 'format.subjectprefix', raise_on_error=False) - - return sub_prefix - - -def setup(): - """Set up git utils, by reading the alias files.""" - # Check for a git alias file also - global use_no_decorate - - alias_fname = get_alias_file() - if alias_fname: - settings.ReadGitAliases(alias_fname) - cmd = log_cmd(None, count=0) - use_no_decorate = (command.run_pipe([cmd], raise_on_error=False) - .return_code == 0) - - -def get_head(): - """Get the hash of the current HEAD - - Returns: - Hash of HEAD - """ - return command.output_one_line('git', 'show', '-s', '--pretty=format:%H') - - -if __name__ == "__main__": - import doctest - - doctest.testmod() diff --git a/tools/patman/patchstream.py b/tools/patman/patchstream.py index 4955f6aaab9..08795c4a0a8 100644 --- a/tools/patman/patchstream.py +++ b/tools/patman/patchstream.py @@ -15,9 +15,9 @@ import shutil import tempfile from patman import commit -from patman import gitutil from patman.series import Series from u_boot_pylib import command +from u_boot_pylib import gitutil # Tags that we detect and remove RE_REMOVE = re.compile(r'^BUG=|^TEST=|^BRANCH=|^Review URL:' diff --git a/tools/patman/project.py b/tools/patman/project.py index 4459042b5d4..d6143a67066 100644 --- a/tools/patman/project.py +++ b/tools/patman/project.py @@ -4,7 +4,7 @@ import os.path -from patman import gitutil +from u_boot_pylib import gitutil def detect_project(): """Autodetect the name of the current project. diff --git a/tools/patman/series.py b/tools/patman/series.py index 6866e1dbd08..d7f2f010f5d 100644 --- a/tools/patman/series.py +++ b/tools/patman/series.py @@ -12,8 +12,8 @@ import sys import time from patman import get_maintainer -from patman import gitutil from patman import settings +from u_boot_pylib import gitutil from u_boot_pylib import terminal from u_boot_pylib import tools diff --git a/tools/patman/settings.py b/tools/patman/settings.py index 68c93e313b3..d66b22be1df 100644 --- a/tools/patman/settings.py +++ b/tools/patman/settings.py @@ -12,7 +12,7 @@ import argparse import os import re -from patman import gitutil +from u_boot_pylib import gitutil """Default settings per-project. diff --git a/tools/patman/test_checkpatch.py b/tools/patman/test_checkpatch.py index db7860f551d..11d003bc4e7 100644 --- a/tools/patman/test_checkpatch.py +++ b/tools/patman/test_checkpatch.py @@ -11,10 +11,10 @@ import tempfile import unittest from patman import checkpatch -from patman import gitutil from patman import patchstream from patman import series from patman import commit +from u_boot_pylib import gitutil class Line: diff --git a/tools/u_boot_pylib/__init__.py b/tools/u_boot_pylib/__init__.py index 63c88e85ec0..807a62e0743 100644 --- a/tools/u_boot_pylib/__init__.py +++ b/tools/u_boot_pylib/__init__.py @@ -1,4 +1,4 @@ # SPDX-License-Identifier: GPL-2.0+ -__all__ = ['command', 'cros_subprocess','terminal', 'test_util', 'tools', - 'tout'] +__all__ = ['command', 'cros_subprocess', 'gitutil', 'terminal', 'test_util', + 'tools', 'tout'] diff --git a/tools/u_boot_pylib/__main__.py b/tools/u_boot_pylib/__main__.py index 8f98d7bd9f8..c0762bca733 100755 --- a/tools/u_boot_pylib/__main__.py +++ b/tools/u_boot_pylib/__main__.py @@ -13,7 +13,6 @@ if __name__ == "__main__": sys.path.append(os.path.join(our_path, '..')) # Run tests - from u_boot_pylib import terminal from u_boot_pylib import test_util result = test_util.run_test_suites( diff --git a/tools/u_boot_pylib/gitutil.py b/tools/u_boot_pylib/gitutil.py new file mode 100644 index 00000000000..10ea5ff39f5 --- /dev/null +++ b/tools/u_boot_pylib/gitutil.py @@ -0,0 +1,706 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2011 The Chromium OS Authors. +# + +import os +import sys + +from patman import settings +from u_boot_pylib import command +from u_boot_pylib import terminal + +# True to use --no-decorate - we check this in setup() +use_no_decorate = True + + +def log_cmd(commit_range, git_dir=None, oneline=False, reverse=False, + count=None): + """Create a command to perform a 'git log' + + Args: + commit_range: Range expression to use for log, None for none + git_dir: Path to git repository (None to use default) + oneline: True to use --oneline, else False + reverse: True to reverse the log (--reverse) + count: Number of commits to list, or None for no limit + Return: + List containing command and arguments to run + """ + cmd = ['git'] + if git_dir: + cmd += ['--git-dir', git_dir] + cmd += ['--no-pager', 'log', '--no-color'] + if oneline: + cmd.append('--oneline') + if use_no_decorate: + cmd.append('--no-decorate') + if reverse: + cmd.append('--reverse') + if count is not None: + cmd.append('-n%d' % count) + if commit_range: + cmd.append(commit_range) + + # Add this in case we have a branch with the same name as a directory. + # This avoids messages like this, for example: + # fatal: ambiguous argument 'test': both revision and filename + cmd.append('--') + return cmd + + +def count_commits_to_branch(branch): + """Returns number of commits between HEAD and the tracking branch. + + This looks back to the tracking branch and works out the number of commits + since then. + + Args: + branch: Branch to count from (None for current branch) + + Return: + Number of patches that exist on top of the branch + """ + if branch: + us, msg = get_upstream('.git', branch) + rev_range = '%s..%s' % (us, branch) + else: + rev_range = '@{upstream}..' + pipe = [log_cmd(rev_range, oneline=True)] + result = command.run_pipe(pipe, capture=True, capture_stderr=True, + oneline=True, raise_on_error=False) + if result.return_code: + raise ValueError('Failed to determine upstream: %s' % + result.stderr.strip()) + patch_count = len(result.stdout.splitlines()) + return patch_count + + +def name_revision(commit_hash): + """Gets the revision name for a commit + + Args: + commit_hash: Commit hash to look up + + Return: + Name of revision, if any, else None + """ + pipe = ['git', 'name-rev', commit_hash] + stdout = command.run_pipe([pipe], capture=True, oneline=True).stdout + + # We expect a commit, a space, then a revision name + name = stdout.split(' ')[1].strip() + return name + + +def guess_upstream(git_dir, branch): + """Tries to guess the upstream for a branch + + This lists out top commits on a branch and tries to find a suitable + upstream. It does this by looking for the first commit where + 'git name-rev' returns a plain branch name, with no ! or ^ modifiers. + + Args: + git_dir: Git directory containing repo + branch: Name of branch + + Returns: + Tuple: + Name of upstream branch (e.g. 'upstream/master') or None if none + Warning/error message, or None if none + """ + pipe = [log_cmd(branch, git_dir=git_dir, oneline=True, count=100)] + result = command.run_pipe(pipe, capture=True, capture_stderr=True, + raise_on_error=False) + if result.return_code: + return None, "Branch '%s' not found" % branch + for line in result.stdout.splitlines()[1:]: + commit_hash = line.split(' ')[0] + name = name_revision(commit_hash) + if '~' not in name and '^' not in name: + if name.startswith('remotes/'): + name = name[8:] + return name, "Guessing upstream as '%s'" % name + return None, "Cannot find a suitable upstream for branch '%s'" % branch + + +def get_upstream(git_dir, branch): + """Returns the name of the upstream for a branch + + Args: + git_dir: Git directory containing repo + branch: Name of branch + + Returns: + Tuple: + Name of upstream branch (e.g. 'upstream/master') or None if none + Warning/error message, or None if none + """ + try: + remote = command.output_one_line('git', '--git-dir', git_dir, 'config', + 'branch.%s.remote' % branch) + merge = command.output_one_line('git', '--git-dir', git_dir, 'config', + 'branch.%s.merge' % branch) + except Exception: + upstream, msg = guess_upstream(git_dir, branch) + return upstream, msg + + if remote == '.': + return merge, None + elif remote and merge: + # Drop the initial refs/heads from merge + leaf = merge.split('/', maxsplit=2)[2:] + return '%s/%s' % (remote, '/'.join(leaf)), None + else: + raise ValueError("Cannot determine upstream branch for branch " + "'%s' remote='%s', merge='%s'" + % (branch, remote, merge)) + + +def get_range_in_branch(git_dir, branch, include_upstream=False): + """Returns an expression for the commits in the given branch. + + Args: + git_dir: Directory containing git repo + branch: Name of branch + Return: + Expression in the form 'upstream..branch' which can be used to + access the commits. If the branch does not exist, returns None. + """ + upstream, msg = get_upstream(git_dir, branch) + if not upstream: + return None, msg + rstr = '%s%s..%s' % (upstream, '~' if include_upstream else '', branch) + return rstr, msg + + +def count_commits_in_range(git_dir, range_expr): + """Returns the number of commits in the given range. + + Args: + git_dir: Directory containing git repo + range_expr: Range to check + Return: + Number of patches that exist in the supplied range or None if none + were found + """ + pipe = [log_cmd(range_expr, git_dir=git_dir, oneline=True)] + result = command.run_pipe(pipe, capture=True, capture_stderr=True, + raise_on_error=False) + if result.return_code: + return None, "Range '%s' not found or is invalid" % range_expr + patch_count = len(result.stdout.splitlines()) + return patch_count, None + + +def count_commits_in_branch(git_dir, branch, include_upstream=False): + """Returns the number of commits in the given branch. + + Args: + git_dir: Directory containing git repo + branch: Name of branch + Return: + Number of patches that exist on top of the branch, or None if the + branch does not exist. + """ + range_expr, msg = get_range_in_branch(git_dir, branch, include_upstream) + if not range_expr: + return None, msg + return count_commits_in_range(git_dir, range_expr) + + +def count_commits(commit_range): + """Returns the number of commits in the given range. + + Args: + commit_range: Range of commits to count (e.g. 'HEAD..base') + Return: + Number of patches that exist on top of the branch + """ + pipe = [log_cmd(commit_range, oneline=True), + ['wc', '-l']] + stdout = command.run_pipe(pipe, capture=True, oneline=True).stdout + patch_count = int(stdout) + return patch_count + + +def checkout(commit_hash, git_dir=None, work_tree=None, force=False): + """Checkout the selected commit for this build + + Args: + commit_hash: Commit hash to check out + """ + pipe = ['git'] + if git_dir: + pipe.extend(['--git-dir', git_dir]) + if work_tree: + pipe.extend(['--work-tree', work_tree]) + pipe.append('checkout') + if force: + pipe.append('-f') + pipe.append(commit_hash) + result = command.run_pipe([pipe], capture=True, raise_on_error=False, + capture_stderr=True) + if result.return_code != 0: + raise OSError('git checkout (%s): %s' % (pipe, result.stderr)) + + +def clone(git_dir, output_dir): + """Checkout the selected commit for this build + + Args: + commit_hash: Commit hash to check out + """ + pipe = ['git', 'clone', git_dir, '.'] + result = command.run_pipe([pipe], capture=True, cwd=output_dir, + capture_stderr=True) + if result.return_code != 0: + raise OSError('git clone: %s' % result.stderr) + + +def fetch(git_dir=None, work_tree=None): + """Fetch from the origin repo + + Args: + commit_hash: Commit hash to check out + """ + pipe = ['git'] + if git_dir: + pipe.extend(['--git-dir', git_dir]) + if work_tree: + pipe.extend(['--work-tree', work_tree]) + pipe.append('fetch') + result = command.run_pipe([pipe], capture=True, capture_stderr=True) + if result.return_code != 0: + raise OSError('git fetch: %s' % result.stderr) + + +def check_worktree_is_available(git_dir): + """Check if git-worktree functionality is available + + Args: + git_dir: The repository to test in + + Returns: + True if git-worktree commands will work, False otherwise. + """ + pipe = ['git', '--git-dir', git_dir, 'worktree', 'list'] + result = command.run_pipe([pipe], capture=True, capture_stderr=True, + raise_on_error=False) + return result.return_code == 0 + + +def add_worktree(git_dir, output_dir, commit_hash=None): + """Create and checkout a new git worktree for this build + + Args: + git_dir: The repository to checkout the worktree from + output_dir: Path for the new worktree + commit_hash: Commit hash to checkout + """ + # We need to pass --detach to avoid creating a new branch + pipe = ['git', '--git-dir', git_dir, 'worktree', 'add', '.', '--detach'] + if commit_hash: + pipe.append(commit_hash) + result = command.run_pipe([pipe], capture=True, cwd=output_dir, + capture_stderr=True) + if result.return_code != 0: + raise OSError('git worktree add: %s' % result.stderr) + + +def prune_worktrees(git_dir): + """Remove administrative files for deleted worktrees + + Args: + git_dir: The repository whose deleted worktrees should be pruned + """ + pipe = ['git', '--git-dir', git_dir, 'worktree', 'prune'] + result = command.run_pipe([pipe], capture=True, capture_stderr=True) + if result.return_code != 0: + raise OSError('git worktree prune: %s' % result.stderr) + + +def create_patches(branch, start, count, ignore_binary, series, signoff=True): + """Create a series of patches from the top of the current branch. + + The patch files are written to the current directory using + git format-patch. + + Args: + branch: Branch to create patches from (None for current branch) + start: Commit to start from: 0=HEAD, 1=next one, etc. + count: number of commits to include + ignore_binary: Don't generate patches for binary files + series: Series object for this series (set of patches) + Return: + Filename of cover letter (None if none) + List of filenames of patch files + """ + cmd = ['git', 'format-patch', '-M'] + if signoff: + cmd.append('--signoff') + if ignore_binary: + cmd.append('--no-binary') + if series.get('cover'): + cmd.append('--cover-letter') + prefix = series.GetPatchPrefix() + if prefix: + cmd += ['--subject-prefix=%s' % prefix] + brname = branch or 'HEAD' + cmd += ['%s~%d..%s~%d' % (brname, start + count, brname, start)] + + stdout = command.run_list(cmd) + files = stdout.splitlines() + + # We have an extra file if there is a cover letter + if series.get('cover'): + return files[0], files[1:] + else: + return None, files + + +def build_email_list(in_list, tag=None, alias=None, warn_on_error=True): + """Build a list of email addresses based on an input list. + + Takes a list of email addresses and aliases, and turns this into a list + of only email address, by resolving any aliases that are present. + + If the tag is given, then each email address is prepended with this + tag and a space. If the tag starts with a minus sign (indicating a + command line parameter) then the email address is quoted. + + Args: + in_list: List of aliases/email addresses + tag: Text to put before each address + alias: Alias dictionary + warn_on_error: True to raise an error when an alias fails to match, + False to just print a message. + + Returns: + List of email addresses + + >>> alias = {} + >>> alias['fred'] = ['f.bloggs@napier.co.nz'] + >>> alias['john'] = ['j.bloggs@napier.co.nz'] + >>> alias['mary'] = ['Mary Poppins '] + >>> alias['boys'] = ['fred', ' john'] + >>> alias['all'] = ['fred ', 'john', ' mary '] + >>> build_email_list(['john', 'mary'], None, alias) + ['j.bloggs@napier.co.nz', 'Mary Poppins '] + >>> build_email_list(['john', 'mary'], '--to', alias) + ['--to "j.bloggs@napier.co.nz"', \ +'--to "Mary Poppins "'] + >>> build_email_list(['john', 'mary'], 'Cc', alias) + ['Cc j.bloggs@napier.co.nz', 'Cc Mary Poppins '] + """ + quote = '"' if tag and tag[0] == '-' else '' + raw = [] + for item in in_list: + raw += lookup_email(item, alias, warn_on_error=warn_on_error) + result = [] + for item in raw: + if item not in result: + result.append(item) + if tag: + return ['%s %s%s%s' % (tag, quote, email, quote) for email in result] + return result + + +def check_suppress_cc_config(): + """Check if sendemail.suppresscc is configured correctly. + + Returns: + True if the option is configured correctly, False otherwise. + """ + suppresscc = command.output_one_line( + 'git', 'config', 'sendemail.suppresscc', raise_on_error=False) + + # Other settings should be fine. + if suppresscc == 'all' or suppresscc == 'cccmd': + col = terminal.Color() + + print((col.build(col.RED, "error") + + ": git config sendemail.suppresscc set to %s\n" + % (suppresscc)) + + " patman needs --cc-cmd to be run to set the cc list.\n" + + " Please run:\n" + + " git config --unset sendemail.suppresscc\n" + + " Or read the man page:\n" + + " git send-email --help\n" + + " and set an option that runs --cc-cmd\n") + return False + + return True + + +def email_patches(series, cover_fname, args, dry_run, warn_on_error, cc_fname, + self_only=False, alias=None, in_reply_to=None, thread=False, + smtp_server=None, get_maintainer_script=None): + """Email a patch series. + + Args: + series: Series object containing destination info + cover_fname: filename of cover letter + args: list of filenames of patch files + dry_run: Just return the command that would be run + warn_on_error: True to print a warning when an alias fails to match, + False to ignore it. + cc_fname: Filename of Cc file for per-commit Cc + self_only: True to just email to yourself as a test + in_reply_to: If set we'll pass this to git as --in-reply-to. + Should be a message ID that this is in reply to. + thread: True to add --thread to git send-email (make + all patches reply to cover-letter or first patch in series) + smtp_server: SMTP server to use to send patches + get_maintainer_script: File name of script to get maintainers emails + + Returns: + Git command that was/would be run + + # For the duration of this doctest pretend that we ran patman with ./patman + >>> _old_argv0 = sys.argv[0] + >>> sys.argv[0] = './patman' + + >>> alias = {} + >>> alias['fred'] = ['f.bloggs@napier.co.nz'] + >>> alias['john'] = ['j.bloggs@napier.co.nz'] + >>> alias['mary'] = ['m.poppins@cloud.net'] + >>> alias['boys'] = ['fred', ' john'] + >>> alias['all'] = ['fred ', 'john', ' mary '] + >>> alias[os.getenv('USER')] = ['this-is-me@me.com'] + >>> series = {} + >>> series['to'] = ['fred'] + >>> series['cc'] = ['mary'] + >>> email_patches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \ + False, alias) + 'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \ +"m.poppins@cloud.net" --cc-cmd "./patman send --cc-cmd cc-fname" cover p1 p2' + >>> email_patches(series, None, ['p1'], True, True, 'cc-fname', False, \ + alias) + 'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \ +"m.poppins@cloud.net" --cc-cmd "./patman send --cc-cmd cc-fname" p1' + >>> series['cc'] = ['all'] + >>> email_patches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \ + True, alias) + 'git send-email --annotate --to "this-is-me@me.com" --cc-cmd "./patman \ +send --cc-cmd cc-fname" cover p1 p2' + >>> email_patches(series, 'cover', ['p1', 'p2'], True, True, 'cc-fname', \ + False, alias) + 'git send-email --annotate --to "f.bloggs@napier.co.nz" --cc \ +"f.bloggs@napier.co.nz" --cc "j.bloggs@napier.co.nz" --cc \ +"m.poppins@cloud.net" --cc-cmd "./patman send --cc-cmd cc-fname" cover p1 p2' + + # Restore argv[0] since we clobbered it. + >>> sys.argv[0] = _old_argv0 + """ + to = build_email_list(series.get('to'), '--to', alias, warn_on_error) + if not to: + git_config_to = command.output('git', 'config', 'sendemail.to', + raise_on_error=False) + if not git_config_to: + print("No recipient.\n" + "Please add something like this to a commit\n" + "Series-to: Fred Bloggs \n" + "Or do something like this\n" + "git config sendemail.to u-boot@lists.denx.de") + return + cc = build_email_list(list(set(series.get('cc')) - set(series.get('to'))), + '--cc', alias, warn_on_error) + if self_only: + to = build_email_list([os.getenv('USER')], '--to', + alias, warn_on_error) + cc = [] + cmd = ['git', 'send-email', '--annotate'] + if smtp_server: + cmd.append('--smtp-server=%s' % smtp_server) + if in_reply_to: + cmd.append('--in-reply-to="%s"' % in_reply_to) + if thread: + cmd.append('--thread') + + cmd += to + cmd += cc + cmd += ['--cc-cmd', '"%s send --cc-cmd %s"' % (sys.argv[0], cc_fname)] + if cover_fname: + cmd.append(cover_fname) + cmd += args + cmdstr = ' '.join(cmd) + if not dry_run: + os.system(cmdstr) + return cmdstr + + +def lookup_email(lookup_name, alias=None, warn_on_error=True, level=0): + """If an email address is an alias, look it up and return the full name + + TODO: Why not just use git's own alias feature? + + Args: + lookup_name: Alias or email address to look up + alias: Dictionary containing aliases (None to use settings default) + warn_on_error: True to print a warning when an alias fails to match, + False to ignore it. + + Returns: + tuple: + list containing a list of email addresses + + Raises: + OSError if a recursive alias reference was found + ValueError if an alias was not found + + >>> alias = {} + >>> alias['fred'] = ['f.bloggs@napier.co.nz'] + >>> alias['john'] = ['j.bloggs@napier.co.nz'] + >>> alias['mary'] = ['m.poppins@cloud.net'] + >>> alias['boys'] = ['fred', ' john', 'f.bloggs@napier.co.nz'] + >>> alias['all'] = ['fred ', 'john', ' mary '] + >>> alias['loop'] = ['other', 'john', ' mary '] + >>> alias['other'] = ['loop', 'john', ' mary '] + >>> lookup_email('mary', alias) + ['m.poppins@cloud.net'] + >>> lookup_email('arthur.wellesley@howe.ro.uk', alias) + ['arthur.wellesley@howe.ro.uk'] + >>> lookup_email('boys', alias) + ['f.bloggs@napier.co.nz', 'j.bloggs@napier.co.nz'] + >>> lookup_email('all', alias) + ['f.bloggs@napier.co.nz', 'j.bloggs@napier.co.nz', 'm.poppins@cloud.net'] + >>> lookup_email('odd', alias) + Alias 'odd' not found + [] + >>> lookup_email('loop', alias) + Traceback (most recent call last): + ... + OSError: Recursive email alias at 'other' + >>> lookup_email('odd', alias, warn_on_error=False) + [] + >>> # In this case the loop part will effectively be ignored. + >>> lookup_email('loop', alias, warn_on_error=False) + Recursive email alias at 'other' + Recursive email alias at 'john' + Recursive email alias at 'mary' + ['j.bloggs@napier.co.nz', 'm.poppins@cloud.net'] + """ + if not alias: + alias = settings.alias + lookup_name = lookup_name.strip() + if '@' in lookup_name: # Perhaps a real email address + return [lookup_name] + + lookup_name = lookup_name.lower() + col = terminal.Color() + + out_list = [] + if level > 10: + msg = "Recursive email alias at '%s'" % lookup_name + if warn_on_error: + raise OSError(msg) + else: + print(col.build(col.RED, msg)) + return out_list + + if lookup_name: + if lookup_name not in alias: + msg = "Alias '%s' not found" % lookup_name + if warn_on_error: + print(col.build(col.RED, msg)) + return out_list + for item in alias[lookup_name]: + todo = lookup_email(item, alias, warn_on_error, level + 1) + for new_item in todo: + if new_item not in out_list: + out_list.append(new_item) + + return out_list + + +def get_top_level(): + """Return name of top-level directory for this git repo. + + Returns: + Full path to git top-level directory + + This test makes sure that we are running tests in the right subdir + + >>> os.path.realpath(os.path.dirname(__file__)) == \ + os.path.join(get_top_level(), 'tools', 'patman') + True + """ + return command.output_one_line('git', 'rev-parse', '--show-toplevel') + + +def get_alias_file(): + """Gets the name of the git alias file. + + Returns: + Filename of git alias file, or None if none + """ + fname = command.output_one_line('git', 'config', 'sendemail.aliasesfile', + raise_on_error=False) + if not fname: + return None + + fname = os.path.expanduser(fname.strip()) + if os.path.isabs(fname): + return fname + + return os.path.join(get_top_level(), fname) + + +def get_default_user_name(): + """Gets the user.name from .gitconfig file. + + Returns: + User name found in .gitconfig file, or None if none + """ + uname = command.output_one_line('git', 'config', '--global', '--includes', 'user.name') + return uname + + +def get_default_user_email(): + """Gets the user.email from the global .gitconfig file. + + Returns: + User's email found in .gitconfig file, or None if none + """ + uemail = command.output_one_line('git', 'config', '--global', '--includes', 'user.email') + return uemail + + +def get_default_subject_prefix(): + """Gets the format.subjectprefix from local .git/config file. + + Returns: + Subject prefix found in local .git/config file, or None if none + """ + sub_prefix = command.output_one_line( + 'git', 'config', 'format.subjectprefix', raise_on_error=False) + + return sub_prefix + + +def setup(): + """Set up git utils, by reading the alias files.""" + # Check for a git alias file also + global use_no_decorate + + alias_fname = get_alias_file() + if alias_fname: + settings.ReadGitAliases(alias_fname) + cmd = log_cmd(None, count=0) + use_no_decorate = (command.run_pipe([cmd], raise_on_error=False) + .return_code == 0) + + +def get_head(): + """Get the hash of the current HEAD + + Returns: + Hash of HEAD + """ + return command.output_one_line('git', 'show', '-s', '--pretty=format:%H') + + +if __name__ == "__main__": + import doctest + + doctest.testmod() -- cgit v1.3.1 From 320ba79911511d7f29d3092fb4cc4f5b7a03d7da Mon Sep 17 00:00:00 2001 From: Maks Mishin Date: Sun, 9 Feb 2025 18:46:21 +0300 Subject: tools: Fix potential null-deref with result of strtok_r Return value of a function 'strtok_r' is dereferenced at kwbimage.c:1655 without checking for NULL, but it is usually checked for this function. Signed-off-by: Maks Mishin --- tools/kwbimage.c | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'tools') diff --git a/tools/kwbimage.c b/tools/kwbimage.c index d1cbced28fc..3dcf5ba66b9 100644 --- a/tools/kwbimage.c +++ b/tools/kwbimage.c @@ -1653,6 +1653,12 @@ static int image_create_config_parse_oneline(char *line, char *unknown_msg = "Ignoring unknown line '%s'\n"; keyword = strtok_r(line, delimiters, &saveptr); + + if (!keyword) { + fprintf(stderr, "Parameter missing in line '%s'\n", line); + return -1; + } + keyword_id = recognize_keyword(keyword); if (!keyword_id) { -- cgit v1.3.1 From 2fc363c7005829b28449ee2fe4c0673ae667e135 Mon Sep 17 00:00:00 2001 From: Tom Rini Date: Wed, 12 Feb 2025 16:23:46 -0600 Subject: dtoc: Switch to setuptools With the distutils module having been removed with Python 3.12, switch to using setuptools instead. Signed-off-by: Tom Rini --- tools/dtoc/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/dtoc/setup.py b/tools/dtoc/setup.py index 5e092fe0872..ae9ad043b01 100644 --- a/tools/dtoc/setup.py +++ b/tools/dtoc/setup.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0+ -from distutils.core import setup +from setuptools import setup setup(name='dtoc', version='1.0', license='GPL-2.0+', -- cgit v1.3.1 From c128ec4647267c8d7d667cbb1dd9037a72f70934 Mon Sep 17 00:00:00 2001 From: Tom Rini Date: Wed, 12 Feb 2025 16:23:39 -0600 Subject: binman: Switch to setuptools With the distutils module having been removed with Python 3.12, switch to using setuptools instead. Signed-off-by: Tom Rini --- tools/binman/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/binman/setup.py b/tools/binman/setup.py index 9a9206eb044..bec078a3d9b 100644 --- a/tools/binman/setup.py +++ b/tools/binman/setup.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: GPL-2.0+ -from distutils.core import setup +from setuptools import setup setup(name='binman', version='1.0', license='GPL-2.0+', -- cgit v1.3.1 From a21f6efaf58cf3df6537f1549d509339de5aeabc Mon Sep 17 00:00:00 2001 From: Tom Rini Date: Wed, 12 Feb 2025 16:23:33 -0600 Subject: tools: binman: ti_board_cfg: Fix pylint error over 'br' With a newer pylint, we get a warning that 'br' could be used before assignment. Fix this by declaring br first as an empty bytearray. Reviewed-by: Neha Malcom Francis Signed-off-by: Tom Rini --- tools/binman/etype/ti_board_config.py | 1 + 1 file changed, 1 insertion(+) (limited to 'tools') diff --git a/tools/binman/etype/ti_board_config.py b/tools/binman/etype/ti_board_config.py index c10d66edcb1..cc7075eeebe 100644 --- a/tools/binman/etype/ti_board_config.py +++ b/tools/binman/etype/ti_board_config.py @@ -119,6 +119,7 @@ class Entry_ti_board_config(Entry_section): array of bytes representing value """ size = 0 + br = bytearray() if (data_type == '#/definitions/u8'): size = 1 elif (data_type == '#/definitions/u16'): -- cgit v1.3.1 From 8e233cca9d9b8876f60e8fa2a17eceda165e649e Mon Sep 17 00:00:00 2001 From: Tom Rini Date: Wed, 12 Feb 2025 16:23:17 -0600 Subject: tools/patman: Don't call a non-existent suite With a newer pylint we get a warning that gitutil.RunTests does not exist, so remove the line. Signed-off-by: Tom Rini --- tools/patman/test_checkpatch.py | 1 - 1 file changed, 1 deletion(-) (limited to 'tools') diff --git a/tools/patman/test_checkpatch.py b/tools/patman/test_checkpatch.py index 11d003bc4e7..3bf16febbf6 100644 --- a/tools/patman/test_checkpatch.py +++ b/tools/patman/test_checkpatch.py @@ -530,4 +530,3 @@ index 0000000..2234c87 if __name__ == "__main__": unittest.main() - gitutil.RunTests() -- cgit v1.3.1 From 6e628c221ebf19a869542d31187e3ac29dba20fb Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Wed, 19 Feb 2025 08:11:16 -0700 Subject: tools: Fix pylint 3.3.4 errors This newer pylint produces errors about variables possibly being used before being set. Adjust the code to pass these checks. Signed-off-by: Simon Glass Reported-by: Tom Rini --- tools/binman/etype/fdtmap.py | 5 +++-- tools/binman/etype/image_header.py | 1 + tools/binman/etype/pre_load.py | 2 ++ tools/binman/etype/ti_board_config.py | 1 + tools/binman/etype/x509_cert.py | 1 + tools/binman/ftest.py | 1 + tools/binman/state.py | 3 +++ tools/buildman/builder.py | 6 +++--- tools/microcode-tool.py | 3 +++ 9 files changed, 18 insertions(+), 5 deletions(-) (limited to 'tools') diff --git a/tools/binman/etype/fdtmap.py b/tools/binman/etype/fdtmap.py index f1f6217940f..2259404180c 100644 --- a/tools/binman/etype/fdtmap.py +++ b/tools/binman/etype/fdtmap.py @@ -106,6 +106,9 @@ class Entry_fdtmap(Entry): Returns: FDT map binary data """ + fsw = libfdt.FdtSw() + fsw.finish_reservemap() + def _AddNode(node): """Add a node to the FDT map""" for pname, prop in node.props.items(): @@ -134,8 +137,6 @@ class Entry_fdtmap(Entry): # Build a new tree with all nodes and properties starting from that # node - fsw = libfdt.FdtSw() - fsw.finish_reservemap() with fsw.add_node(''): fsw.property_string('image-node', node.name) _AddNode(node) diff --git a/tools/binman/etype/image_header.py b/tools/binman/etype/image_header.py index 24011884958..2114df8159f 100644 --- a/tools/binman/etype/image_header.py +++ b/tools/binman/etype/image_header.py @@ -62,6 +62,7 @@ class Entry_image_header(Entry): def _GetHeader(self): image_pos = self.GetSiblingImagePos('fdtmap') + offset = None if image_pos == False: self.Raise("'image_header' section must have an 'fdtmap' sibling") elif image_pos is None: diff --git a/tools/binman/etype/pre_load.py b/tools/binman/etype/pre_load.py index 2e4c72359ff..00f1a896767 100644 --- a/tools/binman/etype/pre_load.py +++ b/tools/binman/etype/pre_load.py @@ -112,6 +112,8 @@ class Entry_pre_load(Entry_collection): # Compute the signature if padding_name is None: padding_name = "pkcs-1.5" + padding = None + padding_args = None if padding_name == "pss": salt_len = key.size_in_bytes() - hash_image.digest_size - 2 padding = pss diff --git a/tools/binman/etype/ti_board_config.py b/tools/binman/etype/ti_board_config.py index cc7075eeebe..7c6773ac7bc 100644 --- a/tools/binman/etype/ti_board_config.py +++ b/tools/binman/etype/ti_board_config.py @@ -126,6 +126,7 @@ class Entry_ti_board_config(Entry_section): size = 2 else: size = 4 + br = None if type(val) == int: br = val.to_bytes(size, byteorder='little') return br diff --git a/tools/binman/etype/x509_cert.py b/tools/binman/etype/x509_cert.py index 29630d1b86c..25e6808b7f9 100644 --- a/tools/binman/etype/x509_cert.py +++ b/tools/binman/etype/x509_cert.py @@ -84,6 +84,7 @@ class Entry_x509_cert(Entry_collection): input_fname = tools.get_output_filename('input.%s' % uniq) config_fname = tools.get_output_filename('config.%s' % uniq) tools.write_file(input_fname, input_data) + stdout = None if type == 'generic': stdout = self.openssl.x509_cert( cert_fname=output_fname, diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index a553ca9e564..d2802f67e2d 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -6381,6 +6381,7 @@ fdt fdtmap Extract the devicetree blob from the fdtmap ename, prop = entry_m.group(1), entry_m.group(3) entry, entry_name, prop_name = image.LookupEntry(entries, name, msg) + expect_val = None if prop_name == 'offset': expect_val = entry.offset elif prop_name == 'image_pos': diff --git a/tools/binman/state.py b/tools/binman/state.py index 45bae40c525..6772d3678fe 100644 --- a/tools/binman/state.py +++ b/tools/binman/state.py @@ -406,10 +406,13 @@ def CheckSetHashValue(node, get_data_func): hash_node = node.FindNode('hash') if hash_node: algo = hash_node.props.get('algo').value + data = None if algo == 'sha256': m = hashlib.sha256() m.update(get_data_func()) data = m.digest() + if data is None: + raise ValueError(f"Node '{node.path}': Unknown hash algorithm '{algo}'") for n in GetUpdateNodes(hash_node): n.SetData('value', data) diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index 2568e4e8423..23b1016d0f9 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -1095,14 +1095,13 @@ class Builder: diff = result[name] if name.startswith('_'): continue - if diff != 0: - color = self.col.RED if diff > 0 else self.col.GREEN + colour = self.col.RED if diff > 0 else self.col.GREEN msg = ' %s %+d' % (name, diff) if not printed_target: tprint('%10s %-15s:' % ('', result['_target']), newline=False) printed_target = True - tprint(msg, colour=color, newline=False) + tprint(msg, colour=colour, newline=False) if printed_target: tprint() if show_bloat: @@ -1353,6 +1352,7 @@ class Builder: for line in lines: if not line: continue + col = None if line[0] == '+': col = self.col.GREEN elif line[0] == '-': diff --git a/tools/microcode-tool.py b/tools/microcode-tool.py index 24c02c4fca1..b726794751a 100755 --- a/tools/microcode-tool.py +++ b/tools/microcode-tool.py @@ -279,6 +279,9 @@ def MicrocodeTool(): if (not not options.mcfile) != (not not options.mcfile): parser.error("You must specify either header files or a microcode file, not both") + date = None + microcodes = None + license_text = None if options.headerfile: date, license_text, microcodes = ParseHeaderFiles(options.headerfile) elif options.mcfile: -- cgit v1.3.1 From 77718437862ee849bd8818c482208aaa92363c8d Mon Sep 17 00:00:00 2001 From: Paul HENRYS Date: Wed, 12 Feb 2025 10:31:21 +0100 Subject: rsa: Add rsa_verify_openssl() to use openssl for host builds rsa_verify_openssl() is used in lib/rsa/rsa-verify.c to authenticate data when building host tools. Signed-off-by: Paul HENRYS --- include/image.h | 18 +++++++ lib/rsa/rsa-verify.c | 5 ++ tools/image-host.c | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 164 insertions(+) (limited to 'tools') diff --git a/include/image.h b/include/image.h index 8a9f779d3ff..54b1557d6c6 100644 --- a/include/image.h +++ b/include/image.h @@ -1687,6 +1687,24 @@ struct sig_header_s { */ int image_pre_load(ulong addr); +#if defined(USE_HOSTCC) +/** + * rsa_verify_openssl() - Verify a signature against some data with openssl API + * + * Verify a RSA PKCS1.5/PSS signature against an expected hash. + * + * @info: Specifies the key and algorithms + * @region: Pointer to the input data + * @region_count: Number of region + * @sig: Signature + * @sig_len: Number of bytes in the signature + * Return: 0 if verified, -ve on error + */ +int rsa_verify_openssl(struct image_sign_info *info, + const struct image_region region[], int region_count, + uint8_t *sig, uint sig_len); +#endif + /** * fit_image_verify_required_sigs() - Verify signatures marked as 'required' * diff --git a/lib/rsa/rsa-verify.c b/lib/rsa/rsa-verify.c index d3b4f71d6be..b74aaf86e6d 100644 --- a/lib/rsa/rsa-verify.c +++ b/lib/rsa/rsa-verify.c @@ -565,6 +565,11 @@ int rsa_verify(struct image_sign_info *info, uint8_t hash[info->crypto->key_len]; int ret; +#ifdef USE_HOSTCC + if (!info->fdt_blob) + return rsa_verify_openssl(info, region, region_count, sig, sig_len); +#endif + /* * Verify that the checksum-length does not exceed the * rsa-signature-length diff --git a/tools/image-host.c b/tools/image-host.c index 84095d760c1..3a643367f2b 100644 --- a/tools/image-host.c +++ b/tools/image-host.c @@ -19,6 +19,11 @@ #include #endif +#if CONFIG_IS_ENABLED(IMAGE_PRE_LOAD) +#include +#include +#endif + /** * fit_set_hash_value - set hash value in requested has node * @fit: pointer to the FIT format image header @@ -1388,3 +1393,139 @@ int fit_check_sign(const void *fit, const void *key, return ret; } #endif + +#if CONFIG_IS_ENABLED(IMAGE_PRE_LOAD) +/** + * rsa_verify_openssl() - Verify a signature against some data with openssl API + * + * Verify a RSA PKCS1.5/PSS signature against an expected hash. + * + * @info: Specifies the key and algorithms + * @region: Pointer to the input data + * @region_count: Number of region + * @sig: Signature + * @sig_len: Number of bytes in the signature + * Return: 0 if verified, -ve on error + */ +int rsa_verify_openssl(struct image_sign_info *info, + const struct image_region region[], int region_count, + uint8_t *sig, uint sig_len) +{ + EVP_PKEY *pkey = NULL; + EVP_PKEY_CTX *ckey = NULL; + EVP_MD_CTX *ctx = NULL; + int pad; + int size; + int i; + int ret = 0; + + if (!info) { + fprintf(stderr, "No info provided\n"); + ret = -EINVAL; + goto out; + } + + if (!info->key) { + fprintf(stderr, "No key provided\n"); + ret = -EINVAL; + goto out; + } + + if (!info->checksum) { + fprintf(stderr, "No checksum information\n"); + ret = -EINVAL; + goto out; + } + + if (!info->padding) { + fprintf(stderr, "No padding information\n"); + ret = -EINVAL; + goto out; + } + + if (region_count < 1) { + fprintf(stderr, "Invalid value for region_count: %d\n", region_count); + ret = -EINVAL; + goto out; + } + + pkey = (EVP_PKEY *)info->key; + + ckey = EVP_PKEY_CTX_new(pkey, NULL); + if (!ckey) { + ret = -ENOMEM; + fprintf(stderr, "EVK key context setup failed: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + goto out; + } + + size = EVP_PKEY_size(pkey); + if (size > sig_len) { + fprintf(stderr, "Invalid signature size (%d bytes)\n", + size); + ret = -EINVAL; + goto out; + } + + ctx = EVP_MD_CTX_new(); + if (!ctx) { + ret = -ENOMEM; + fprintf(stderr, "EVP context creation failed: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + goto out; + } + EVP_MD_CTX_init(ctx); + + if (EVP_DigestVerifyInit(ctx, &ckey, + EVP_get_digestbyname(info->checksum->name), + NULL, pkey) <= 0) { + ret = -EINVAL; + fprintf(stderr, "Verifier setup failed: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + goto out; + } + + if (!strcmp(info->padding->name, "pkcs-1.5")) { + pad = RSA_PKCS1_PADDING; + } else if (!strcmp(info->padding->name, "pss")) { + pad = RSA_PKCS1_PSS_PADDING; + } else { + ret = -ENOMSG; + fprintf(stderr, "Unsupported padding: %s\n", + info->padding->name); + goto out; + } + + if (EVP_PKEY_CTX_set_rsa_padding(ckey, pad) <= 0) { + ret = -EINVAL; + fprintf(stderr, "padding setup has failed: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + goto out; + } + + for (i=0 ; i < region_count ; ++i) { + if (EVP_DigestVerifyUpdate(ctx, region[i].data, + region[i].size) <= 0) { + ret = -EINVAL; + fprintf(stderr, "Hashing data failed: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + goto out; + } + } + + if (EVP_DigestVerifyFinal(ctx, sig, sig_len) <= 0) { + ret = -EINVAL; + fprintf(stderr, "Verifying digest failed: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + goto out; + } +out: + if (ctx) + EVP_MD_CTX_free(ctx); + + if (ret) + fprintf(stderr, "Failed to verify signature\n"); + + return ret; +} +#endif -- cgit v1.3.1 From a85a67a16075f3515d8c4d6b909d7895ce136178 Mon Sep 17 00:00:00 2001 From: Paul HENRYS Date: Wed, 12 Feb 2025 10:31:22 +0100 Subject: image: Add an inline declaration of unmap_sysmem() Add an empty inline declaration when compiling tools for a host where unmap_sysmem() is not defined. Signed-off-by: Paul HENRYS --- tools/mkimage.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'tools') diff --git a/tools/mkimage.h b/tools/mkimage.h index 15741f250fd..5d6bcc9301a 100644 --- a/tools/mkimage.h +++ b/tools/mkimage.h @@ -37,6 +37,10 @@ static inline void *map_sysmem(ulong paddr, unsigned long len) return (void *)(uintptr_t)paddr; } +static inline void unmap_sysmem(const void *vaddr) +{ +} + static inline ulong map_to_sysmem(const void *ptr) { return (ulong)(uintptr_t)ptr; -- cgit v1.3.1 From 7dd0bf52790eda703af591de76c3a2837f545f7c Mon Sep 17 00:00:00 2001 From: Paul HENRYS Date: Wed, 12 Feb 2025 10:31:24 +0100 Subject: tools: Add preload_check_sign to authenticate images with a pre-load preload_check_sign is added so that it can be used to authenticate images signed with the pre-load signature supported by binman and U-Boot. It could also be used to test the signature in binman tests signing images with the pre-load. Signed-off-by: Paul HENRYS --- tools/.gitignore | 1 + tools/Kconfig | 5 ++ tools/Makefile | 5 ++ tools/preload_check_sign.c | 161 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 172 insertions(+) create mode 100644 tools/preload_check_sign.c (limited to 'tools') diff --git a/tools/.gitignore b/tools/.gitignore index 0108c567309..6a5c613f772 100644 --- a/tools/.gitignore +++ b/tools/.gitignore @@ -29,6 +29,7 @@ /mxsboot /ncb /prelink-riscv +/preload_check_sign /printinitialenv /proftool /relocate-rela diff --git a/tools/Kconfig b/tools/Kconfig index 01ff0fcf748..8e272ee99a8 100644 --- a/tools/Kconfig +++ b/tools/Kconfig @@ -9,6 +9,11 @@ config MKIMAGE_DTC_PATH some cases the system dtc may not support all required features and the path to a different version should be given here. +config TOOLS_IMAGE_PRE_LOAD + def_bool y + help + Enable pre-load signature support in the tools builds. + config TOOLS_CRC16 def_bool y help diff --git a/tools/Makefile b/tools/Makefile index 237fa900a24..e5f5eea47c7 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -66,6 +66,7 @@ mkenvimage-objs := mkenvimage.o os_support.o generated/lib/crc32.o hostprogs-y += dumpimage mkimage hostprogs-$(CONFIG_TOOLS_LIBCRYPTO) += fit_info fit_check_sign hostprogs-$(CONFIG_TOOLS_LIBCRYPTO) += fdt_add_pubkey +hostprogs-$(CONFIG_TOOLS_LIBCRYPTO) += preload_check_sign ifneq ($(CONFIG_CMD_BOOTEFI_SELFTEST)$(CONFIG_FWU_MDATA_GPT_BLK),) hostprogs-y += file2include @@ -89,6 +90,8 @@ ECDSA_OBJS-$(CONFIG_TOOLS_LIBCRYPTO) := $(addprefix generated/lib/ecdsa/, ecdsa- AES_OBJS-$(CONFIG_TOOLS_LIBCRYPTO) := $(addprefix generated/lib/aes/, \ aes-encrypt.o aes-decrypt.o) +PRELOAD_OBJS-$(CONFIG_TOOLS_LIBCRYPTO) := generated/boot/image-pre-load.o + # Cryptographic helpers and image types that depend on openssl/libcrypto LIBCRYPTO_OBJS-$(CONFIG_TOOLS_LIBCRYPTO) := \ generated/lib/fdt-libcrypto.o \ @@ -158,6 +161,7 @@ fit_info-objs := $(dumpimage-mkimage-objs) fit_info.o fit_check_sign-objs := $(dumpimage-mkimage-objs) fit_check_sign.o fdt_add_pubkey-objs := $(dumpimage-mkimage-objs) fdt_add_pubkey.o file2include-objs := file2include.o +preload_check_sign-objs := $(dumpimage-mkimage-objs) $(PRELOAD_OBJS-y) preload_check_sign.o ifneq ($(CONFIG_MX23)$(CONFIG_MX28)$(CONFIG_TOOLS_LIBCRYPTO),) # Add CFG_MXS into host CFLAGS, so we can check whether or not register @@ -195,6 +199,7 @@ HOSTLDLIBS_dumpimage := $(HOSTLDLIBS_mkimage) HOSTLDLIBS_fit_info := $(HOSTLDLIBS_mkimage) HOSTLDLIBS_fit_check_sign := $(HOSTLDLIBS_mkimage) HOSTLDLIBS_fdt_add_pubkey := $(HOSTLDLIBS_mkimage) +HOSTLDLIBS_preload_check_sign := $(HOSTLDLIBS_mkimage) hostprogs-$(CONFIG_EXYNOS5250) += mkexynosspl hostprogs-$(CONFIG_EXYNOS5420) += mkexynosspl diff --git a/tools/preload_check_sign.c b/tools/preload_check_sign.c new file mode 100644 index 00000000000..63a778203f0 --- /dev/null +++ b/tools/preload_check_sign.c @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Check a file including a preload header including a signature + * + * Copyright (c) 2025 Paul HENRYS + * + * Binman makes it possible to generate a preload header signing part or the + * complete file. The tool preload_check_sign allows to verify and authenticate + * a file starting with a preload header. + */ +#include +#include +#include +#include +#include +#include +#include + +extern void image_pre_load_sig_set_info(struct image_sig_info *info); +extern int image_pre_load_sig(ulong addr); + +static void usage(char *cmdname) +{ + fprintf(stderr, "Usage: %s -f file -k PEM key file\n" + " -f ==> set file which should be checked\n" + " -k ==> PEM key file\n" + " -a ==> algo (default: sha256,rsa2048)\n" + " -p ==> padding (default: pkcs-1.5)\n" + " -h ==> help\n", + cmdname); + exit(EXIT_FAILURE); +} + +int main(int argc, char **argv) +{ + int ret = 0; + char cmdname[256]; + char *file = NULL; + char *keyfile = NULL; + int c; + FILE *fp = NULL; + FILE *fp_key = NULL; + size_t bytes; + long filesize; + void *buffer = NULL; + EVP_PKEY *pkey = NULL; + char *algo = "sha256,rsa2048"; + char *padding = "pkcs-1.5"; + struct image_sig_info info = {0}; + + strncpy(cmdname, *argv, sizeof(cmdname) - 1); + cmdname[sizeof(cmdname) - 1] = '\0'; + while ((c = getopt(argc, argv, "f:k:a:p:h")) != -1) + switch (c) { + case 'f': + file = optarg; + break; + case 'k': + keyfile = optarg; + break; + case 'a': + algo = optarg; + break; + case 'p': + padding = optarg; + break; + default: + usage(cmdname); + break; + } + + if (!file) { + fprintf(stderr, "%s: Missing file\n", *argv); + usage(*argv); + } + + if (!keyfile) { + fprintf(stderr, "%s: Missing key file\n", *argv); + usage(*argv); + } + + fp = fopen(file, "r"); + if (!fp) { + fprintf(stderr, "Error opening file: %s\n", file); + ret = EXIT_FAILURE; + goto out; + } + + fseek(fp, 0, SEEK_END); + filesize = ftell(fp); + rewind(fp); + + buffer = malloc(filesize); + if (!buffer) { + fprintf(stderr, "Memory allocation failed"); + ret = EXIT_FAILURE; + goto out; + } + + bytes = fread(buffer, 1, filesize, fp); + if (bytes != filesize) { + fprintf(stderr, "Error reading file\n"); + ret = EXIT_FAILURE; + goto out; + } + + fp_key = fopen(keyfile, "r"); + if (!fp_key) { + fprintf(stderr, "Error opening file: %s\n", keyfile); + ret = EXIT_FAILURE; + goto out; + } + + /* Attempt to read the private key */ + pkey = PEM_read_PrivateKey(fp_key, NULL, NULL, NULL); + if (!pkey) { + /* If private key reading fails, try reading as a public key */ + fseek(fp_key, 0, SEEK_SET); + pkey = PEM_read_PUBKEY(fp_key, NULL, NULL, NULL); + } + if (!pkey) { + fprintf(stderr, "Unable to retrieve the public key: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + ret = EXIT_FAILURE; + goto out; + } + + info.algo_name = algo; + info.padding_name = padding; + info.key = (uint8_t *)pkey; + info.mandatory = 1; + info.sig_size = (EVP_PKEY_get_bits(pkey) + 7) / 8; + if (info.sig_size < 0) { + fprintf(stderr, "Fail to retrieve the signature size: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + ret = EXIT_FAILURE; + goto out; + } + + /* Compute signature information */ + info.sig_info.name = info.algo_name; + info.sig_info.padding = image_get_padding_algo(info.padding_name); + info.sig_info.checksum = image_get_checksum_algo(info.sig_info.name); + info.sig_info.crypto = image_get_crypto_algo(info.sig_info.name); + info.sig_info.key = info.key; + info.sig_info.keylen = info.key_len; + + /* Check the signature */ + image_pre_load_sig_set_info(&info); + ret = image_pre_load_sig((ulong)buffer); +out: + if (fp) + fclose(fp); + if (fp_key) + fclose(fp_key); + if (info.key) + EVP_PKEY_free(pkey); + free(buffer); + + exit(ret); +} -- cgit v1.3.1 From a9842ac6347e2e0e7f6f8b66b5fe254739cdd298 Mon Sep 17 00:00:00 2001 From: Paul HENRYS Date: Wed, 12 Feb 2025 10:31:26 +0100 Subject: binman: Authenticate the image when testing the preload signature Use preload_check_sign to authenticate the generated image when testing the preload signature in testPreLoad(). Signed-off-by: Paul HENRYS --- tools/binman/ftest.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'tools') diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index a553ca9e564..8cf867fd3fe 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -762,6 +762,16 @@ class TestFunctional(unittest.TestCase): return False return True + def _CheckPreload(self, image, key, algo="sha256,rsa2048", + padding="pkcs-1.5"): + try: + tools.run('preload_check_sign', '-k', key, '-a', algo, '-p', + padding, '-f', image) + except: + self.fail('Expected image signed with a pre-load') + return False + return True + def testRun(self): """Test a basic run with valid args""" result = self._RunBinman('-h') @@ -5781,9 +5791,14 @@ fdt fdtmap Extract the devicetree blob from the fdtmap data = self._DoReadFileDtb( '230_pre_load.dts', entry_args=entry_args, extra_indirs=[os.path.join(self._binman_dir, 'test')])[0] + + image_fname = tools.get_output_filename('image.bin') + is_signed = self._CheckPreload(image_fname, self.TestFile("dev.key")) + self.assertEqual(PRE_LOAD_MAGIC, data[:len(PRE_LOAD_MAGIC)]) self.assertEqual(PRE_LOAD_VERSION, data[4:4 + len(PRE_LOAD_VERSION)]) self.assertEqual(PRE_LOAD_HDR_SIZE, data[8:8 + len(PRE_LOAD_HDR_SIZE)]) + self.assertEqual(is_signed, True) def testPreLoadNoKey(self): """Test an image with a pre-load heade0r with missing key""" -- cgit v1.3.1 From 7520827be3dbde7fb7324d63d263d650d427c035 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 4 Feb 2025 16:33:53 -0700 Subject: buildman: Update tests for newer filelock module Recent versions of this module call time.perf_counter() so add a patch for this also. Signed-off-by: Simon Glass Reported-by: Tom Rini --- tools/buildman/test.py | 1 + 1 file changed, 1 insertion(+) (limited to 'tools') diff --git a/tools/buildman/test.py b/tools/buildman/test.py index 385a34e5254..c5feb74a105 100644 --- a/tools/buildman/test.py +++ b/tools/buildman/test.py @@ -836,6 +836,7 @@ class TestBuild(unittest.TestCase): tmpdir = self.base_dir with (patch('time.time', side_effect=self.get_time), + patch('time.perf_counter', side_effect=self.get_time), patch('time.monotonic', side_effect=self.get_time), patch('time.sleep', side_effect=self.inc_time), patch('os.kill', side_effect=self.kill)): -- cgit v1.3.1 From 523a56cc54637a0c04a1e87c262599faf26d7d69 Mon Sep 17 00:00:00 2001 From: Tom Rini Date: Mon, 24 Feb 2025 10:32:04 -0600 Subject: Revert "Merge patch series "Add preload_check_sign tool"" This reverts commit c8750efe02c20725388dd4279896aaf306acfad4, reversing changes made to 8c6cf8aeea7e57ca686de8b765e4baf3a7ef1fa7. Unfortunately these changes do not build on macOS hosts. Signed-off-by: Tom Rini --- boot/image-pre-load.c | 57 +-------------- configs/tools-only_defconfig | 3 +- include/image.h | 18 ----- lib/rsa/rsa-verify.c | 5 -- tools/.gitignore | 1 - tools/Kconfig | 5 -- tools/Makefile | 5 -- tools/binman/ftest.py | 15 ---- tools/image-host.c | 141 ------------------------------------- tools/mkimage.h | 4 -- tools/preload_check_sign.c | 161 ------------------------------------------- 11 files changed, 3 insertions(+), 412 deletions(-) delete mode 100644 tools/preload_check_sign.c (limited to 'tools') diff --git a/boot/image-pre-load.c b/boot/image-pre-load.c index adf3b341a20..cc19017404c 100644 --- a/boot/image-pre-load.c +++ b/boot/image-pre-load.c @@ -3,23 +3,12 @@ * Copyright (C) 2021 Philippe Reynes */ -#ifdef USE_HOSTCC -#include "mkimage.h" -#else #include -#include DECLARE_GLOBAL_DATA_PTR; -#endif /* !USE_HOSTCC*/ - #include -#include +#include -#ifdef USE_HOSTCC -/* Define compat stuff for use in tools. */ -typedef uint8_t u8; -typedef uint16_t u16; -typedef uint32_t u32; -#endif +#include /* * Offset of the image @@ -28,47 +17,6 @@ typedef uint32_t u32; */ ulong image_load_offset; -#ifdef USE_HOSTCC -/* Host tools use these implementations to setup information related to the - * pre-load signatures - */ -static struct image_sig_info *host_info; - -#define log_info(fmt, args...) printf(fmt, ##args) -#define log_err(fmt, args...) printf(fmt, ##args) - -void image_pre_load_sig_set_info(struct image_sig_info *info) -{ - host_info = info; -} - -/* - * This function sets a pointer to information for the signature check. - * It expects that host_info has been initially provision by the host - * application. - * - * return: - * < 0 => an error has occurred - * 0 => OK - */ -static int image_pre_load_sig_setup(struct image_sig_info *info) -{ - if (!info) { - log_err("ERROR: info is NULL\n"); - return -EINVAL; - } - - if (!host_info) { - log_err("ERROR: host_info is NULL\n"); - log_err("ERROR: Set it with image_pre_load_sig_set_info()\n"); - return -EINVAL; - } - - memcpy(info, host_info, sizeof(struct image_sig_info)); - - return 0; -} -#else /* * This function gathers information about the signature check * that could be done before launching the image. @@ -158,7 +106,6 @@ static int image_pre_load_sig_setup(struct image_sig_info *info) out: return ret; } -#endif /* !USE_HOSTCC */ static int image_pre_load_sig_get_magic(ulong addr, u32 *magic) { diff --git a/configs/tools-only_defconfig b/configs/tools-only_defconfig index e64bb768440..cecd26175d1 100644 --- a/configs/tools-only_defconfig +++ b/configs/tools-only_defconfig @@ -9,11 +9,10 @@ CONFIG_EFI_LOADER=n CONFIG_ANDROID_BOOT_IMAGE=y CONFIG_TIMESTAMP=y CONFIG_FIT=y +CONFIG_FIT_SIGNATURE=y CONFIG_BOOTSTD_FULL=n CONFIG_BOOTMETH_CROS=n CONFIG_BOOTMETH_VBE=n -CONFIG_IMAGE_PRE_LOAD=y -CONFIG_IMAGE_PRE_LOAD_SIG=y CONFIG_USE_BOOTCOMMAND=y CONFIG_BOOTCOMMAND="run distro_bootcmd" CONFIG_CMD_BOOTD=n diff --git a/include/image.h b/include/image.h index 54b1557d6c6..8a9f779d3ff 100644 --- a/include/image.h +++ b/include/image.h @@ -1687,24 +1687,6 @@ struct sig_header_s { */ int image_pre_load(ulong addr); -#if defined(USE_HOSTCC) -/** - * rsa_verify_openssl() - Verify a signature against some data with openssl API - * - * Verify a RSA PKCS1.5/PSS signature against an expected hash. - * - * @info: Specifies the key and algorithms - * @region: Pointer to the input data - * @region_count: Number of region - * @sig: Signature - * @sig_len: Number of bytes in the signature - * Return: 0 if verified, -ve on error - */ -int rsa_verify_openssl(struct image_sign_info *info, - const struct image_region region[], int region_count, - uint8_t *sig, uint sig_len); -#endif - /** * fit_image_verify_required_sigs() - Verify signatures marked as 'required' * diff --git a/lib/rsa/rsa-verify.c b/lib/rsa/rsa-verify.c index b74aaf86e6d..d3b4f71d6be 100644 --- a/lib/rsa/rsa-verify.c +++ b/lib/rsa/rsa-verify.c @@ -565,11 +565,6 @@ int rsa_verify(struct image_sign_info *info, uint8_t hash[info->crypto->key_len]; int ret; -#ifdef USE_HOSTCC - if (!info->fdt_blob) - return rsa_verify_openssl(info, region, region_count, sig, sig_len); -#endif - /* * Verify that the checksum-length does not exceed the * rsa-signature-length diff --git a/tools/.gitignore b/tools/.gitignore index 6a5c613f772..0108c567309 100644 --- a/tools/.gitignore +++ b/tools/.gitignore @@ -29,7 +29,6 @@ /mxsboot /ncb /prelink-riscv -/preload_check_sign /printinitialenv /proftool /relocate-rela diff --git a/tools/Kconfig b/tools/Kconfig index 8e272ee99a8..01ff0fcf748 100644 --- a/tools/Kconfig +++ b/tools/Kconfig @@ -9,11 +9,6 @@ config MKIMAGE_DTC_PATH some cases the system dtc may not support all required features and the path to a different version should be given here. -config TOOLS_IMAGE_PRE_LOAD - def_bool y - help - Enable pre-load signature support in the tools builds. - config TOOLS_CRC16 def_bool y help diff --git a/tools/Makefile b/tools/Makefile index e5f5eea47c7..237fa900a24 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -66,7 +66,6 @@ mkenvimage-objs := mkenvimage.o os_support.o generated/lib/crc32.o hostprogs-y += dumpimage mkimage hostprogs-$(CONFIG_TOOLS_LIBCRYPTO) += fit_info fit_check_sign hostprogs-$(CONFIG_TOOLS_LIBCRYPTO) += fdt_add_pubkey -hostprogs-$(CONFIG_TOOLS_LIBCRYPTO) += preload_check_sign ifneq ($(CONFIG_CMD_BOOTEFI_SELFTEST)$(CONFIG_FWU_MDATA_GPT_BLK),) hostprogs-y += file2include @@ -90,8 +89,6 @@ ECDSA_OBJS-$(CONFIG_TOOLS_LIBCRYPTO) := $(addprefix generated/lib/ecdsa/, ecdsa- AES_OBJS-$(CONFIG_TOOLS_LIBCRYPTO) := $(addprefix generated/lib/aes/, \ aes-encrypt.o aes-decrypt.o) -PRELOAD_OBJS-$(CONFIG_TOOLS_LIBCRYPTO) := generated/boot/image-pre-load.o - # Cryptographic helpers and image types that depend on openssl/libcrypto LIBCRYPTO_OBJS-$(CONFIG_TOOLS_LIBCRYPTO) := \ generated/lib/fdt-libcrypto.o \ @@ -161,7 +158,6 @@ fit_info-objs := $(dumpimage-mkimage-objs) fit_info.o fit_check_sign-objs := $(dumpimage-mkimage-objs) fit_check_sign.o fdt_add_pubkey-objs := $(dumpimage-mkimage-objs) fdt_add_pubkey.o file2include-objs := file2include.o -preload_check_sign-objs := $(dumpimage-mkimage-objs) $(PRELOAD_OBJS-y) preload_check_sign.o ifneq ($(CONFIG_MX23)$(CONFIG_MX28)$(CONFIG_TOOLS_LIBCRYPTO),) # Add CFG_MXS into host CFLAGS, so we can check whether or not register @@ -199,7 +195,6 @@ HOSTLDLIBS_dumpimage := $(HOSTLDLIBS_mkimage) HOSTLDLIBS_fit_info := $(HOSTLDLIBS_mkimage) HOSTLDLIBS_fit_check_sign := $(HOSTLDLIBS_mkimage) HOSTLDLIBS_fdt_add_pubkey := $(HOSTLDLIBS_mkimage) -HOSTLDLIBS_preload_check_sign := $(HOSTLDLIBS_mkimage) hostprogs-$(CONFIG_EXYNOS5250) += mkexynosspl hostprogs-$(CONFIG_EXYNOS5420) += mkexynosspl diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index e0685884cef..d2802f67e2d 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -762,16 +762,6 @@ class TestFunctional(unittest.TestCase): return False return True - def _CheckPreload(self, image, key, algo="sha256,rsa2048", - padding="pkcs-1.5"): - try: - tools.run('preload_check_sign', '-k', key, '-a', algo, '-p', - padding, '-f', image) - except: - self.fail('Expected image signed with a pre-load') - return False - return True - def testRun(self): """Test a basic run with valid args""" result = self._RunBinman('-h') @@ -5791,14 +5781,9 @@ fdt fdtmap Extract the devicetree blob from the fdtmap data = self._DoReadFileDtb( '230_pre_load.dts', entry_args=entry_args, extra_indirs=[os.path.join(self._binman_dir, 'test')])[0] - - image_fname = tools.get_output_filename('image.bin') - is_signed = self._CheckPreload(image_fname, self.TestFile("dev.key")) - self.assertEqual(PRE_LOAD_MAGIC, data[:len(PRE_LOAD_MAGIC)]) self.assertEqual(PRE_LOAD_VERSION, data[4:4 + len(PRE_LOAD_VERSION)]) self.assertEqual(PRE_LOAD_HDR_SIZE, data[8:8 + len(PRE_LOAD_HDR_SIZE)]) - self.assertEqual(is_signed, True) def testPreLoadNoKey(self): """Test an image with a pre-load heade0r with missing key""" diff --git a/tools/image-host.c b/tools/image-host.c index a071a244e07..05d8c898209 100644 --- a/tools/image-host.c +++ b/tools/image-host.c @@ -19,11 +19,6 @@ #include #endif -#if CONFIG_IS_ENABLED(IMAGE_PRE_LOAD) -#include -#include -#endif - /** * fit_set_hash_value - set hash value in requested has node * @fit: pointer to the FIT format image header @@ -1402,139 +1397,3 @@ int fit_check_sign(const void *fit, const void *key, return ret; } #endif - -#if CONFIG_IS_ENABLED(IMAGE_PRE_LOAD) -/** - * rsa_verify_openssl() - Verify a signature against some data with openssl API - * - * Verify a RSA PKCS1.5/PSS signature against an expected hash. - * - * @info: Specifies the key and algorithms - * @region: Pointer to the input data - * @region_count: Number of region - * @sig: Signature - * @sig_len: Number of bytes in the signature - * Return: 0 if verified, -ve on error - */ -int rsa_verify_openssl(struct image_sign_info *info, - const struct image_region region[], int region_count, - uint8_t *sig, uint sig_len) -{ - EVP_PKEY *pkey = NULL; - EVP_PKEY_CTX *ckey = NULL; - EVP_MD_CTX *ctx = NULL; - int pad; - int size; - int i; - int ret = 0; - - if (!info) { - fprintf(stderr, "No info provided\n"); - ret = -EINVAL; - goto out; - } - - if (!info->key) { - fprintf(stderr, "No key provided\n"); - ret = -EINVAL; - goto out; - } - - if (!info->checksum) { - fprintf(stderr, "No checksum information\n"); - ret = -EINVAL; - goto out; - } - - if (!info->padding) { - fprintf(stderr, "No padding information\n"); - ret = -EINVAL; - goto out; - } - - if (region_count < 1) { - fprintf(stderr, "Invalid value for region_count: %d\n", region_count); - ret = -EINVAL; - goto out; - } - - pkey = (EVP_PKEY *)info->key; - - ckey = EVP_PKEY_CTX_new(pkey, NULL); - if (!ckey) { - ret = -ENOMEM; - fprintf(stderr, "EVK key context setup failed: %s\n", - ERR_error_string(ERR_get_error(), NULL)); - goto out; - } - - size = EVP_PKEY_size(pkey); - if (size > sig_len) { - fprintf(stderr, "Invalid signature size (%d bytes)\n", - size); - ret = -EINVAL; - goto out; - } - - ctx = EVP_MD_CTX_new(); - if (!ctx) { - ret = -ENOMEM; - fprintf(stderr, "EVP context creation failed: %s\n", - ERR_error_string(ERR_get_error(), NULL)); - goto out; - } - EVP_MD_CTX_init(ctx); - - if (EVP_DigestVerifyInit(ctx, &ckey, - EVP_get_digestbyname(info->checksum->name), - NULL, pkey) <= 0) { - ret = -EINVAL; - fprintf(stderr, "Verifier setup failed: %s\n", - ERR_error_string(ERR_get_error(), NULL)); - goto out; - } - - if (!strcmp(info->padding->name, "pkcs-1.5")) { - pad = RSA_PKCS1_PADDING; - } else if (!strcmp(info->padding->name, "pss")) { - pad = RSA_PKCS1_PSS_PADDING; - } else { - ret = -ENOMSG; - fprintf(stderr, "Unsupported padding: %s\n", - info->padding->name); - goto out; - } - - if (EVP_PKEY_CTX_set_rsa_padding(ckey, pad) <= 0) { - ret = -EINVAL; - fprintf(stderr, "padding setup has failed: %s\n", - ERR_error_string(ERR_get_error(), NULL)); - goto out; - } - - for (i = 0; i < region_count; ++i) { - if (EVP_DigestVerifyUpdate(ctx, region[i].data, - region[i].size) <= 0) { - ret = -EINVAL; - fprintf(stderr, "Hashing data failed: %s\n", - ERR_error_string(ERR_get_error(), NULL)); - goto out; - } - } - - if (EVP_DigestVerifyFinal(ctx, sig, sig_len) <= 0) { - ret = -EINVAL; - fprintf(stderr, "Verifying digest failed: %s\n", - ERR_error_string(ERR_get_error(), NULL)); - goto out; - } -out: - if (ctx) - EVP_MD_CTX_free(ctx); - - if (ret) - fprintf(stderr, "Failed to verify signature\n"); - - return ret; -} -#endif diff --git a/tools/mkimage.h b/tools/mkimage.h index 5d6bcc9301a..15741f250fd 100644 --- a/tools/mkimage.h +++ b/tools/mkimage.h @@ -37,10 +37,6 @@ static inline void *map_sysmem(ulong paddr, unsigned long len) return (void *)(uintptr_t)paddr; } -static inline void unmap_sysmem(const void *vaddr) -{ -} - static inline ulong map_to_sysmem(const void *ptr) { return (ulong)(uintptr_t)ptr; diff --git a/tools/preload_check_sign.c b/tools/preload_check_sign.c deleted file mode 100644 index 63a778203f0..00000000000 --- a/tools/preload_check_sign.c +++ /dev/null @@ -1,161 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0+ -/* - * Check a file including a preload header including a signature - * - * Copyright (c) 2025 Paul HENRYS - * - * Binman makes it possible to generate a preload header signing part or the - * complete file. The tool preload_check_sign allows to verify and authenticate - * a file starting with a preload header. - */ -#include -#include -#include -#include -#include -#include -#include - -extern void image_pre_load_sig_set_info(struct image_sig_info *info); -extern int image_pre_load_sig(ulong addr); - -static void usage(char *cmdname) -{ - fprintf(stderr, "Usage: %s -f file -k PEM key file\n" - " -f ==> set file which should be checked\n" - " -k ==> PEM key file\n" - " -a ==> algo (default: sha256,rsa2048)\n" - " -p ==> padding (default: pkcs-1.5)\n" - " -h ==> help\n", - cmdname); - exit(EXIT_FAILURE); -} - -int main(int argc, char **argv) -{ - int ret = 0; - char cmdname[256]; - char *file = NULL; - char *keyfile = NULL; - int c; - FILE *fp = NULL; - FILE *fp_key = NULL; - size_t bytes; - long filesize; - void *buffer = NULL; - EVP_PKEY *pkey = NULL; - char *algo = "sha256,rsa2048"; - char *padding = "pkcs-1.5"; - struct image_sig_info info = {0}; - - strncpy(cmdname, *argv, sizeof(cmdname) - 1); - cmdname[sizeof(cmdname) - 1] = '\0'; - while ((c = getopt(argc, argv, "f:k:a:p:h")) != -1) - switch (c) { - case 'f': - file = optarg; - break; - case 'k': - keyfile = optarg; - break; - case 'a': - algo = optarg; - break; - case 'p': - padding = optarg; - break; - default: - usage(cmdname); - break; - } - - if (!file) { - fprintf(stderr, "%s: Missing file\n", *argv); - usage(*argv); - } - - if (!keyfile) { - fprintf(stderr, "%s: Missing key file\n", *argv); - usage(*argv); - } - - fp = fopen(file, "r"); - if (!fp) { - fprintf(stderr, "Error opening file: %s\n", file); - ret = EXIT_FAILURE; - goto out; - } - - fseek(fp, 0, SEEK_END); - filesize = ftell(fp); - rewind(fp); - - buffer = malloc(filesize); - if (!buffer) { - fprintf(stderr, "Memory allocation failed"); - ret = EXIT_FAILURE; - goto out; - } - - bytes = fread(buffer, 1, filesize, fp); - if (bytes != filesize) { - fprintf(stderr, "Error reading file\n"); - ret = EXIT_FAILURE; - goto out; - } - - fp_key = fopen(keyfile, "r"); - if (!fp_key) { - fprintf(stderr, "Error opening file: %s\n", keyfile); - ret = EXIT_FAILURE; - goto out; - } - - /* Attempt to read the private key */ - pkey = PEM_read_PrivateKey(fp_key, NULL, NULL, NULL); - if (!pkey) { - /* If private key reading fails, try reading as a public key */ - fseek(fp_key, 0, SEEK_SET); - pkey = PEM_read_PUBKEY(fp_key, NULL, NULL, NULL); - } - if (!pkey) { - fprintf(stderr, "Unable to retrieve the public key: %s\n", - ERR_error_string(ERR_get_error(), NULL)); - ret = EXIT_FAILURE; - goto out; - } - - info.algo_name = algo; - info.padding_name = padding; - info.key = (uint8_t *)pkey; - info.mandatory = 1; - info.sig_size = (EVP_PKEY_get_bits(pkey) + 7) / 8; - if (info.sig_size < 0) { - fprintf(stderr, "Fail to retrieve the signature size: %s\n", - ERR_error_string(ERR_get_error(), NULL)); - ret = EXIT_FAILURE; - goto out; - } - - /* Compute signature information */ - info.sig_info.name = info.algo_name; - info.sig_info.padding = image_get_padding_algo(info.padding_name); - info.sig_info.checksum = image_get_checksum_algo(info.sig_info.name); - info.sig_info.crypto = image_get_crypto_algo(info.sig_info.name); - info.sig_info.key = info.key; - info.sig_info.keylen = info.key_len; - - /* Check the signature */ - image_pre_load_sig_set_info(&info); - ret = image_pre_load_sig((ulong)buffer); -out: - if (fp) - fclose(fp); - if (fp_key) - fclose(fp_key); - if (info.key) - EVP_PKEY_free(pkey); - free(buffer); - - exit(ret); -} -- cgit v1.3.1 From 205bf9763208c8b94cdd4c723253d230f5a5866b Mon Sep 17 00:00:00 2001 From: Maks Mishin Date: Fri, 31 Jan 2025 13:04:46 +0300 Subject: tools: imximage: Fix potential memory leak Dynamic memory, referenced by 'line', is allocated at imximage.c:761 by calling function 'getline' and lost at imximage.c:793. Signed-off-by: Maks Mishin --- tools/imximage.c | 1 + 1 file changed, 1 insertion(+) (limited to 'tools') diff --git a/tools/imximage.c b/tools/imximage.c index 467d9f27d2a..55231caf8f3 100644 --- a/tools/imximage.c +++ b/tools/imximage.c @@ -783,6 +783,7 @@ static uint32_t parse_cfg_file(struct imx_header *imxhdr, char *name) } (*set_dcd_rst)(imxhdr, dcd_len, name, lineno); + free(line); fclose(fd); /* Exit if there is no BOOT_FROM field specifying the flash_offset */ -- cgit v1.3.1 From 410ef0bb34b41f6ec23a167fb9e10f3f9483434b Mon Sep 17 00:00:00 2001 From: Maks Mishin Date: Sun, 2 Feb 2025 20:05:17 +0300 Subject: tools: imx8image: Fix potential memory leak Dynamic memory, referenced by 'line', is allocated at imx8image.c:270 by calling function 'getline' and lost at imx8image.c:294. Signed-off-by: Maks Mishin --- tools/imx8image.c | 1 + 1 file changed, 1 insertion(+) (limited to 'tools') diff --git a/tools/imx8image.c b/tools/imx8image.c index 15510d3e712..0135b190951 100644 --- a/tools/imx8image.c +++ b/tools/imx8image.c @@ -290,6 +290,7 @@ static uint32_t parse_cfg_file(image_t *param_stack, char *name) } } + free(line); fclose(fd); return 0; } -- cgit v1.3.1 From 959f3316592b5f272ccd36be00a860856dc5da5b Mon Sep 17 00:00:00 2001 From: Maks Mishin Date: Sun, 2 Feb 2025 20:10:39 +0300 Subject: tools: imx8mimage: Fix potential memory leak Dynamic memory, referenced by 'line', is allocated at imx8mimage.c:187 by calling function 'getline' and lost at imx8mimage.c:210. Signed-off-by: Maks Mishin --- tools/imx8mimage.c | 1 + 1 file changed, 1 insertion(+) (limited to 'tools') diff --git a/tools/imx8mimage.c b/tools/imx8mimage.c index d60d293e649..0f24ba75c0f 100644 --- a/tools/imx8mimage.c +++ b/tools/imx8mimage.c @@ -206,6 +206,7 @@ static uint32_t parse_cfg_file(char *name) } } + free(line); fclose(fd); return 0; } -- cgit v1.3.1 From 942c8c8e669739d2e8dec67a7ed90158defc93ed Mon Sep 17 00:00:00 2001 From: Paul HENRYS Date: Mon, 24 Feb 2025 22:20:50 +0100 Subject: rsa: Add rsa_verify_openssl() to use openssl for host builds rsa_verify_openssl() is used in lib/rsa/rsa-verify.c to authenticate data when building host tools. Signed-off-by: Paul HENRYS --- include/image.h | 18 +++++++ lib/rsa/rsa-verify.c | 5 ++ tools/image-host.c | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 164 insertions(+) (limited to 'tools') diff --git a/include/image.h b/include/image.h index 07912606f33..c1db8383459 100644 --- a/include/image.h +++ b/include/image.h @@ -1688,6 +1688,24 @@ struct sig_header_s { */ int image_pre_load(ulong addr); +#if defined(USE_HOSTCC) +/** + * rsa_verify_openssl() - Verify a signature against some data with openssl API + * + * Verify a RSA PKCS1.5/PSS signature against an expected hash. + * + * @info: Specifies the key and algorithms + * @region: Pointer to the input data + * @region_count: Number of region + * @sig: Signature + * @sig_len: Number of bytes in the signature + * Return: 0 if verified, -ve on error + */ +int rsa_verify_openssl(struct image_sign_info *info, + const struct image_region region[], int region_count, + uint8_t *sig, uint sig_len); +#endif + /** * fit_image_verify_required_sigs() - Verify signatures marked as 'required' * diff --git a/lib/rsa/rsa-verify.c b/lib/rsa/rsa-verify.c index d3b4f71d6be..b74aaf86e6d 100644 --- a/lib/rsa/rsa-verify.c +++ b/lib/rsa/rsa-verify.c @@ -565,6 +565,11 @@ int rsa_verify(struct image_sign_info *info, uint8_t hash[info->crypto->key_len]; int ret; +#ifdef USE_HOSTCC + if (!info->fdt_blob) + return rsa_verify_openssl(info, region, region_count, sig, sig_len); +#endif + /* * Verify that the checksum-length does not exceed the * rsa-signature-length diff --git a/tools/image-host.c b/tools/image-host.c index e6de34fa059..e19166aeb18 100644 --- a/tools/image-host.c +++ b/tools/image-host.c @@ -19,6 +19,11 @@ #include #endif +#if CONFIG_IS_ENABLED(IMAGE_PRE_LOAD) +#include +#include +#endif + /** * fit_set_hash_value - set hash value in requested has node * @fit: pointer to the FIT format image header @@ -1401,3 +1406,139 @@ int fit_check_sign(const void *fit, const void *key, return ret; } #endif + +#if CONFIG_IS_ENABLED(IMAGE_PRE_LOAD) +/** + * rsa_verify_openssl() - Verify a signature against some data with openssl API + * + * Verify a RSA PKCS1.5/PSS signature against an expected hash. + * + * @info: Specifies the key and algorithms + * @region: Pointer to the input data + * @region_count: Number of region + * @sig: Signature + * @sig_len: Number of bytes in the signature + * Return: 0 if verified, -ve on error + */ +int rsa_verify_openssl(struct image_sign_info *info, + const struct image_region region[], int region_count, + uint8_t *sig, uint sig_len) +{ + EVP_PKEY *pkey = NULL; + EVP_PKEY_CTX *ckey = NULL; + EVP_MD_CTX *ctx = NULL; + int pad; + int size; + int i; + int ret = 0; + + if (!info) { + fprintf(stderr, "No info provided\n"); + ret = -EINVAL; + goto out; + } + + if (!info->key) { + fprintf(stderr, "No key provided\n"); + ret = -EINVAL; + goto out; + } + + if (!info->checksum) { + fprintf(stderr, "No checksum information\n"); + ret = -EINVAL; + goto out; + } + + if (!info->padding) { + fprintf(stderr, "No padding information\n"); + ret = -EINVAL; + goto out; + } + + if (region_count < 1) { + fprintf(stderr, "Invalid value for region_count: %d\n", region_count); + ret = -EINVAL; + goto out; + } + + pkey = (EVP_PKEY *)info->key; + + ckey = EVP_PKEY_CTX_new(pkey, NULL); + if (!ckey) { + ret = -ENOMEM; + fprintf(stderr, "EVK key context setup failed: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + goto out; + } + + size = EVP_PKEY_size(pkey); + if (size > sig_len) { + fprintf(stderr, "Invalid signature size (%d bytes)\n", + size); + ret = -EINVAL; + goto out; + } + + ctx = EVP_MD_CTX_new(); + if (!ctx) { + ret = -ENOMEM; + fprintf(stderr, "EVP context creation failed: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + goto out; + } + EVP_MD_CTX_init(ctx); + + if (EVP_DigestVerifyInit(ctx, &ckey, + EVP_get_digestbyname(info->checksum->name), + NULL, pkey) <= 0) { + ret = -EINVAL; + fprintf(stderr, "Verifier setup failed: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + goto out; + } + + if (!strcmp(info->padding->name, "pkcs-1.5")) { + pad = RSA_PKCS1_PADDING; + } else if (!strcmp(info->padding->name, "pss")) { + pad = RSA_PKCS1_PSS_PADDING; + } else { + ret = -ENOMSG; + fprintf(stderr, "Unsupported padding: %s\n", + info->padding->name); + goto out; + } + + if (EVP_PKEY_CTX_set_rsa_padding(ckey, pad) <= 0) { + ret = -EINVAL; + fprintf(stderr, "padding setup has failed: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + goto out; + } + + for (i=0 ; i < region_count ; ++i) { + if (EVP_DigestVerifyUpdate(ctx, region[i].data, + region[i].size) <= 0) { + ret = -EINVAL; + fprintf(stderr, "Hashing data failed: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + goto out; + } + } + + if (EVP_DigestVerifyFinal(ctx, sig, sig_len) <= 0) { + ret = -EINVAL; + fprintf(stderr, "Verifying digest failed: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + goto out; + } +out: + if (ctx) + EVP_MD_CTX_free(ctx); + + if (ret) + fprintf(stderr, "Failed to verify signature\n"); + + return ret; +} +#endif -- cgit v1.3.1 From 47f9186a3e1cf8a18a94feda362025c40ce690cd Mon Sep 17 00:00:00 2001 From: Paul HENRYS Date: Mon, 24 Feb 2025 22:20:51 +0100 Subject: image: Add an inline declaration of unmap_sysmem() Add an empty inline declaration when compiling tools for a host where unmap_sysmem() is not defined. Signed-off-by: Paul HENRYS --- tools/mkimage.h | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'tools') diff --git a/tools/mkimage.h b/tools/mkimage.h index 15741f250fd..5d6bcc9301a 100644 --- a/tools/mkimage.h +++ b/tools/mkimage.h @@ -37,6 +37,10 @@ static inline void *map_sysmem(ulong paddr, unsigned long len) return (void *)(uintptr_t)paddr; } +static inline void unmap_sysmem(const void *vaddr) +{ +} + static inline ulong map_to_sysmem(const void *ptr) { return (ulong)(uintptr_t)ptr; -- cgit v1.3.1 From 76de7b061f6df1bd8ddc423ef1980489d72da873 Mon Sep 17 00:00:00 2001 From: Paul HENRYS Date: Mon, 24 Feb 2025 22:20:53 +0100 Subject: tools: Add preload_check_sign to authenticate images with a pre-load preload_check_sign is added so that it can be used to authenticate images signed with the pre-load signature supported by binman and U-Boot. It could also be used to test the signature in binman tests signing images with the pre-load. Signed-off-by: Paul HENRYS --- tools/.gitignore | 1 + tools/Kconfig | 5 ++ tools/Makefile | 5 ++ tools/preload_check_sign.c | 160 +++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 171 insertions(+) create mode 100644 tools/preload_check_sign.c (limited to 'tools') diff --git a/tools/.gitignore b/tools/.gitignore index 0108c567309..6a5c613f772 100644 --- a/tools/.gitignore +++ b/tools/.gitignore @@ -29,6 +29,7 @@ /mxsboot /ncb /prelink-riscv +/preload_check_sign /printinitialenv /proftool /relocate-rela diff --git a/tools/Kconfig b/tools/Kconfig index 01ff0fcf748..8e272ee99a8 100644 --- a/tools/Kconfig +++ b/tools/Kconfig @@ -9,6 +9,11 @@ config MKIMAGE_DTC_PATH some cases the system dtc may not support all required features and the path to a different version should be given here. +config TOOLS_IMAGE_PRE_LOAD + def_bool y + help + Enable pre-load signature support in the tools builds. + config TOOLS_CRC16 def_bool y help diff --git a/tools/Makefile b/tools/Makefile index 237fa900a24..e5f5eea47c7 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -66,6 +66,7 @@ mkenvimage-objs := mkenvimage.o os_support.o generated/lib/crc32.o hostprogs-y += dumpimage mkimage hostprogs-$(CONFIG_TOOLS_LIBCRYPTO) += fit_info fit_check_sign hostprogs-$(CONFIG_TOOLS_LIBCRYPTO) += fdt_add_pubkey +hostprogs-$(CONFIG_TOOLS_LIBCRYPTO) += preload_check_sign ifneq ($(CONFIG_CMD_BOOTEFI_SELFTEST)$(CONFIG_FWU_MDATA_GPT_BLK),) hostprogs-y += file2include @@ -89,6 +90,8 @@ ECDSA_OBJS-$(CONFIG_TOOLS_LIBCRYPTO) := $(addprefix generated/lib/ecdsa/, ecdsa- AES_OBJS-$(CONFIG_TOOLS_LIBCRYPTO) := $(addprefix generated/lib/aes/, \ aes-encrypt.o aes-decrypt.o) +PRELOAD_OBJS-$(CONFIG_TOOLS_LIBCRYPTO) := generated/boot/image-pre-load.o + # Cryptographic helpers and image types that depend on openssl/libcrypto LIBCRYPTO_OBJS-$(CONFIG_TOOLS_LIBCRYPTO) := \ generated/lib/fdt-libcrypto.o \ @@ -158,6 +161,7 @@ fit_info-objs := $(dumpimage-mkimage-objs) fit_info.o fit_check_sign-objs := $(dumpimage-mkimage-objs) fit_check_sign.o fdt_add_pubkey-objs := $(dumpimage-mkimage-objs) fdt_add_pubkey.o file2include-objs := file2include.o +preload_check_sign-objs := $(dumpimage-mkimage-objs) $(PRELOAD_OBJS-y) preload_check_sign.o ifneq ($(CONFIG_MX23)$(CONFIG_MX28)$(CONFIG_TOOLS_LIBCRYPTO),) # Add CFG_MXS into host CFLAGS, so we can check whether or not register @@ -195,6 +199,7 @@ HOSTLDLIBS_dumpimage := $(HOSTLDLIBS_mkimage) HOSTLDLIBS_fit_info := $(HOSTLDLIBS_mkimage) HOSTLDLIBS_fit_check_sign := $(HOSTLDLIBS_mkimage) HOSTLDLIBS_fdt_add_pubkey := $(HOSTLDLIBS_mkimage) +HOSTLDLIBS_preload_check_sign := $(HOSTLDLIBS_mkimage) hostprogs-$(CONFIG_EXYNOS5250) += mkexynosspl hostprogs-$(CONFIG_EXYNOS5420) += mkexynosspl diff --git a/tools/preload_check_sign.c b/tools/preload_check_sign.c new file mode 100644 index 00000000000..ebead459273 --- /dev/null +++ b/tools/preload_check_sign.c @@ -0,0 +1,160 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * Check a file including a preload header including a signature + * + * Copyright (c) 2025 Paul HENRYS + * + * Binman makes it possible to generate a preload header signing part or the + * complete file. The tool preload_check_sign allows to verify and authenticate + * a file starting with a preload header. + */ +#include +#include +#include +#include +#include +#include + +extern void image_pre_load_sig_set_info(struct image_sig_info *info); +extern int image_pre_load_sig(ulong addr); + +static void usage(char *cmdname) +{ + fprintf(stderr, "Usage: %s -f file -k PEM key file\n" + " -f ==> set file which should be checked\n" + " -k ==> PEM key file\n" + " -a ==> algo (default: sha256,rsa2048)\n" + " -p ==> padding (default: pkcs-1.5)\n" + " -h ==> help\n", + cmdname); + exit(EXIT_FAILURE); +} + +int main(int argc, char **argv) +{ + int ret = 0; + char cmdname[256]; + char *file = NULL; + char *keyfile = NULL; + int c; + FILE *fp = NULL; + FILE *fp_key = NULL; + size_t bytes; + long filesize; + void *buffer = NULL; + EVP_PKEY *pkey = NULL; + char *algo = "sha256,rsa2048"; + char *padding = "pkcs-1.5"; + struct image_sig_info info = {0}; + + strncpy(cmdname, *argv, sizeof(cmdname) - 1); + cmdname[sizeof(cmdname) - 1] = '\0'; + while ((c = getopt(argc, argv, "f:k:a:p:h")) != -1) + switch (c) { + case 'f': + file = optarg; + break; + case 'k': + keyfile = optarg; + break; + case 'a': + algo = optarg; + break; + case 'p': + padding = optarg; + break; + default: + usage(cmdname); + break; + } + + if (!file) { + fprintf(stderr, "%s: Missing file\n", *argv); + usage(*argv); + } + + if (!keyfile) { + fprintf(stderr, "%s: Missing key file\n", *argv); + usage(*argv); + } + + fp = fopen(file, "r"); + if (!fp) { + fprintf(stderr, "Error opening file: %s\n", file); + ret = EXIT_FAILURE; + goto out; + } + + fseek(fp, 0, SEEK_END); + filesize = ftell(fp); + rewind(fp); + + buffer = malloc(filesize); + if (!buffer) { + fprintf(stderr, "Memory allocation failed"); + ret = EXIT_FAILURE; + goto out; + } + + bytes = fread(buffer, 1, filesize, fp); + if (bytes != filesize) { + fprintf(stderr, "Error reading file\n"); + ret = EXIT_FAILURE; + goto out; + } + + fp_key = fopen(keyfile, "r"); + if (!fp_key) { + fprintf(stderr, "Error opening file: %s\n", keyfile); + ret = EXIT_FAILURE; + goto out; + } + + /* Attempt to read the private key */ + pkey = PEM_read_PrivateKey(fp_key, NULL, NULL, NULL); + if (!pkey) { + /* If private key reading fails, try reading as a public key */ + fseek(fp_key, 0, SEEK_SET); + pkey = PEM_read_PUBKEY(fp_key, NULL, NULL, NULL); + } + if (!pkey) { + fprintf(stderr, "Unable to retrieve the public key: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + ret = EXIT_FAILURE; + goto out; + } + + info.algo_name = algo; + info.padding_name = padding; + info.key = (uint8_t *)pkey; + info.mandatory = 1; + info.sig_size = EVP_PKEY_size(pkey); + if (info.sig_size < 0) { + fprintf(stderr, "Fail to retrieve the signature size: %s\n", + ERR_error_string(ERR_get_error(), NULL)); + ret = EXIT_FAILURE; + goto out; + } + + /* Compute signature information */ + info.sig_info.name = info.algo_name; + info.sig_info.padding = image_get_padding_algo(info.padding_name); + info.sig_info.checksum = image_get_checksum_algo(info.sig_info.name); + info.sig_info.crypto = image_get_crypto_algo(info.sig_info.name); + info.sig_info.key = info.key; + info.sig_info.keylen = info.key_len; + + /* Check the signature */ + image_pre_load_sig_set_info(&info); + ret = image_pre_load_sig((ulong)buffer); +out: + if (fp) + fclose(fp); + if (fp_key) + fclose(fp_key); + if (info.key) + EVP_PKEY_free(pkey); + free(buffer); + + exit(ret); +} -- cgit v1.3.1 From b9b87d01efc496d18bbc17c58c552d54a06ef6ba Mon Sep 17 00:00:00 2001 From: Paul HENRYS Date: Mon, 24 Feb 2025 22:20:55 +0100 Subject: binman: Authenticate the image when testing the preload signature Use preload_check_sign to authenticate the generated image when testing the preload signature in testPreLoad(). Signed-off-by: Paul HENRYS --- tools/binman/ftest.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'tools') diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index a553ca9e564..8cf867fd3fe 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -762,6 +762,16 @@ class TestFunctional(unittest.TestCase): return False return True + def _CheckPreload(self, image, key, algo="sha256,rsa2048", + padding="pkcs-1.5"): + try: + tools.run('preload_check_sign', '-k', key, '-a', algo, '-p', + padding, '-f', image) + except: + self.fail('Expected image signed with a pre-load') + return False + return True + def testRun(self): """Test a basic run with valid args""" result = self._RunBinman('-h') @@ -5781,9 +5791,14 @@ fdt fdtmap Extract the devicetree blob from the fdtmap data = self._DoReadFileDtb( '230_pre_load.dts', entry_args=entry_args, extra_indirs=[os.path.join(self._binman_dir, 'test')])[0] + + image_fname = tools.get_output_filename('image.bin') + is_signed = self._CheckPreload(image_fname, self.TestFile("dev.key")) + self.assertEqual(PRE_LOAD_MAGIC, data[:len(PRE_LOAD_MAGIC)]) self.assertEqual(PRE_LOAD_VERSION, data[4:4 + len(PRE_LOAD_VERSION)]) self.assertEqual(PRE_LOAD_HDR_SIZE, data[8:8 + len(PRE_LOAD_HDR_SIZE)]) + self.assertEqual(is_signed, True) def testPreLoadNoKey(self): """Test an image with a pre-load heade0r with missing key""" -- cgit v1.3.1 From d6900a778a72ddb33b10550503719a13cc59bc18 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 3 Feb 2025 09:26:42 -0700 Subject: u_boot_pylib: Correct case for test_result This should be in capitals and defined at the start of the file. Update it. Signed-off-by: Simon Glass --- tools/binman/ftest.py | 14 +++++++------- tools/buildman/func_test.py | 10 +++++----- tools/u_boot_pylib/command.py | 23 +++++++++++------------ 3 files changed, 23 insertions(+), 24 deletions(-) (limited to 'tools') diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index a553ca9e564..03eb7f814fe 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -303,7 +303,7 @@ class TestFunctional(unittest.TestCase): def setUp(self): # Enable this to turn on debugging output # tout.init(tout.DEBUG) - command.test_result = None + command.TEST_RESULT = None def tearDown(self): """Remove the temporary output directory""" @@ -780,11 +780,11 @@ class TestFunctional(unittest.TestCase): def testFullHelpInternal(self): """Test that the full help is displayed with -H""" try: - command.test_result = command.CommandResult() + command.TEST_RESULT = command.CommandResult() result = self._DoBinman('-H') help_file = os.path.join(self._binman_dir, 'README.rst') finally: - command.test_result = None + command.TEST_RESULT = None def testHelp(self): """Test that the basic help is displayed with -h""" @@ -1872,7 +1872,7 @@ class TestFunctional(unittest.TestCase): def testGbb(self): """Test for the Chromium OS Google Binary Block""" - command.test_result = self._HandleGbbCommand + command.TEST_RESULT = self._HandleGbbCommand entry_args = { 'keydir': 'devkeys', 'bmpblk': 'bmpblk.bin', @@ -1941,7 +1941,7 @@ class TestFunctional(unittest.TestCase): def testVblock(self): """Test for the Chromium OS Verified Boot Block""" self._hash_data = False - command.test_result = self._HandleVblockCommand + command.TEST_RESULT = self._HandleVblockCommand entry_args = { 'keydir': 'devkeys', } @@ -1974,7 +1974,7 @@ class TestFunctional(unittest.TestCase): def testVblockContent(self): """Test that the vblock signs the right data""" self._hash_data = True - command.test_result = self._HandleVblockCommand + command.TEST_RESULT = self._HandleVblockCommand entry_args = { 'keydir': 'devkeys', } @@ -5496,7 +5496,7 @@ fdt fdtmap Extract the devicetree blob from the fdtmap def testFitSubentryUsesBintool(self): """Test that binman FIT subentries can use bintools""" - command.test_result = self._HandleGbbCommand + command.TEST_RESULT = self._HandleGbbCommand entry_args = { 'keydir': 'devkeys', 'bmpblk': 'bmpblk.bin', diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index 4e12c671a3d..51a2366e8da 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -187,7 +187,7 @@ class TestFunctional(unittest.TestCase): self._git_dir = os.path.join(self._base_dir, 'src') self._buildman_pathname = sys.argv[0] self._buildman_dir = os.path.dirname(os.path.realpath(sys.argv[0])) - command.test_result = self._HandleCommand + command.TEST_RESULT = self._HandleCommand bsettings.setup(None) bsettings.add_file(settings_data) self.setupToolchains() @@ -266,7 +266,7 @@ class TestFunctional(unittest.TestCase): return result def testFullHelp(self): - command.test_result = None + command.TEST_RESULT = None result = self._RunBuildman('-H') help_file = os.path.join(self._buildman_dir, 'README.rst') # Remove possible extraneous strings @@ -277,7 +277,7 @@ class TestFunctional(unittest.TestCase): self.assertEqual(0, result.return_code) def testHelp(self): - command.test_result = None + command.TEST_RESULT = None result = self._RunBuildman('-h') help_file = os.path.join(self._buildman_dir, 'README.rst') self.assertTrue(len(result.stdout) > 1000) @@ -286,11 +286,11 @@ class TestFunctional(unittest.TestCase): def testGitSetup(self): """Test gitutils.Setup(), from outside the module itself""" - command.test_result = command.CommandResult(return_code=1) + command.TEST_RESULT = command.CommandResult(return_code=1) gitutil.setup() self.assertEqual(gitutil.use_no_decorate, False) - command.test_result = command.CommandResult(return_code=0) + command.TEST_RESULT = command.CommandResult(return_code=0) gitutil.setup() self.assertEqual(gitutil.use_no_decorate, True) diff --git a/tools/u_boot_pylib/command.py b/tools/u_boot_pylib/command.py index bbe95d86122..103358420dd 100644 --- a/tools/u_boot_pylib/command.py +++ b/tools/u_boot_pylib/command.py @@ -6,6 +6,13 @@ import os from u_boot_pylib import cros_subprocess +# This permits interception of RunPipe for test purposes. If it is set to +# a function, then that function is called with the pipe list being +# executed. Otherwise, it is assumed to be a CommandResult object, and is +# returned as the result for every run_pipe() call. +# When this value is None, commands are executed as normal. +TEST_RESULT = None + """Shell command ease-ups for Python.""" class CommandResult: @@ -32,14 +39,6 @@ class CommandResult: self.combined = self.combined.decode('utf-8') return self - -# This permits interception of RunPipe for test purposes. If it is set to -# a function, then that function is called with the pipe list being -# executed. Otherwise, it is assumed to be a CommandResult object, and is -# returned as the result for every run_pipe() call. -# When this value is None, commands are executed as normal. -test_result = None - def run_pipe(pipe_list, infile=None, outfile=None, capture=False, capture_stderr=False, oneline=False, raise_on_error=True, cwd=None, binary=False, @@ -63,14 +62,14 @@ def run_pipe(pipe_list, infile=None, outfile=None, Returns: CommandResult object """ - if test_result: - if hasattr(test_result, '__call__'): + if TEST_RESULT: + if hasattr(TEST_RESULT, '__call__'): # pylint: disable=E1102 - result = test_result(pipe_list=pipe_list) + result = TEST_RESULT(pipe_list=pipe_list) if result: return result else: - return test_result + return TEST_RESULT # No result: fall through to normal processing result = CommandResult(b'', b'', b'') last_pipe = None -- cgit v1.3.1 From 54ead4be04241f34967a6a591d529ee8ba66f301 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 3 Feb 2025 09:26:43 -0700 Subject: u_boot_pylib: Add an exception-class for errors Throwing an Exception is not very friendly since it is the top-level class of all exceptions. Declare a new class instead. Signed-off-by: Simon Glass --- tools/patman/gitutil.py | 2 +- tools/u_boot_pylib/command.py | 20 ++++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) (limited to 'tools') diff --git a/tools/patman/gitutil.py b/tools/patman/gitutil.py index 10ea5ff39f5..ffe05273b35 100644 --- a/tools/patman/gitutil.py +++ b/tools/patman/gitutil.py @@ -140,7 +140,7 @@ def get_upstream(git_dir, branch): 'branch.%s.remote' % branch) merge = command.output_one_line('git', '--git-dir', git_dir, 'config', 'branch.%s.merge' % branch) - except Exception: + except command.CommandExc: upstream, msg = guess_upstream(git_dir, branch) return upstream, msg diff --git a/tools/u_boot_pylib/command.py b/tools/u_boot_pylib/command.py index 103358420dd..4a9916bd814 100644 --- a/tools/u_boot_pylib/command.py +++ b/tools/u_boot_pylib/command.py @@ -13,6 +13,19 @@ from u_boot_pylib import cros_subprocess # When this value is None, commands are executed as normal. TEST_RESULT = None + +class CommandExc(Exception): + """Reports an exception to the caller""" + def __init__(self, msg, result): + """Set up a new exception object + + Args: + result (CommandResult): Execution result so far + """ + super().__init__(msg) + self.result = result + + """Shell command ease-ups for Python.""" class CommandResult: @@ -61,6 +74,8 @@ def run_pipe(pipe_list, infile=None, outfile=None, kwargs: Additional keyword arguments to cros_subprocess.Popen() Returns: CommandResult object + Raises: + CommandExc if an exception happens """ if TEST_RESULT: if hasattr(TEST_RESULT, '__call__'): @@ -95,7 +110,8 @@ def run_pipe(pipe_list, infile=None, outfile=None, except Exception as err: result.exception = err if raise_on_error: - raise Exception("Error running '%s': %s" % (user_pipestr, str)) + raise CommandExc(f"Error running '{user_pipestr}': {err}", + result) from err result.return_code = 255 return result.to_output(binary) @@ -106,7 +122,7 @@ def run_pipe(pipe_list, infile=None, outfile=None, result.output = result.stdout.rstrip(b'\r\n') result.return_code = last_pipe.wait() if raise_on_error and result.return_code: - raise Exception("Error running '%s'" % user_pipestr) + raise CommandExc(f"Error running '{user_pipestr}'", result) return result.to_output(binary) def output(*cmd, **kwargs): -- cgit v1.3.1 From f8456c91aad8259ab08bdf3654b8ee8c0187a45d Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 3 Feb 2025 09:26:44 -0700 Subject: u_boot_pylib: Fix pylint warnings in command This file has a lot of warnings. Before adding any more features, fix those which are straightforward to resolve. Signed-off-by: Simon Glass --- tools/u_boot_pylib/command.py | 107 +++++++++++++++++++++++++++++++----------- 1 file changed, 80 insertions(+), 27 deletions(-) (limited to 'tools') diff --git a/tools/u_boot_pylib/command.py b/tools/u_boot_pylib/command.py index 4a9916bd814..a98dcedd322 100644 --- a/tools/u_boot_pylib/command.py +++ b/tools/u_boot_pylib/command.py @@ -1,8 +1,11 @@ # SPDX-License-Identifier: GPL-2.0+ -# Copyright (c) 2011 The Chromium OS Authors. -# +""" +Shell command ease-ups for Python -import os +Copyright (c) 2011 The Chromium OS Authors. +""" + +import subprocess from u_boot_pylib import cros_subprocess @@ -26,16 +29,16 @@ class CommandExc(Exception): self.result = result -"""Shell command ease-ups for Python.""" - class CommandResult: """A class which captures the result of executing a command. Members: - stdout: stdout obtained from command, as a string - stderr: stderr obtained from command, as a string - return_code: Return code from command - exception: Exception received, or None if all ok + stdout (bytes): stdout obtained from command, as a string + stderr (bytes): stderr obtained from command, as a string + combined (bytes): stdout and stderr interleaved + return_code (int): Return code from command + exception (Exception): Exception received, or None if all ok + output (str or None): Returns output as a single line if requested """ def __init__(self, stdout='', stderr='', combined='', return_code=0, exception=None): @@ -44,34 +47,46 @@ class CommandResult: self.combined = combined self.return_code = return_code self.exception = exception + self.output = None def to_output(self, binary): + """Converts binary output to its final form + + Args: + binary (bool): True to report binary output, False to use strings + Returns: + self + """ if not binary: self.stdout = self.stdout.decode('utf-8') self.stderr = self.stderr.decode('utf-8') self.combined = self.combined.decode('utf-8') return self -def run_pipe(pipe_list, infile=None, outfile=None, - capture=False, capture_stderr=False, oneline=False, - raise_on_error=True, cwd=None, binary=False, - output_func=None, **kwargs): + +def run_pipe(pipe_list, infile=None, outfile=None, capture=False, + capture_stderr=False, oneline=False, raise_on_error=True, cwd=None, + binary=False, output_func=None, **kwargs): """ Perform a command pipeline, with optional input/output filenames. Args: - pipe_list: List of command lines to execute. Each command line is - piped into the next, and is itself a list of strings. For + pipe_list (list of list): List of command lines to execute. Each command + line is piped into the next, and is itself a list of strings. For example [ ['ls', '.git'] ['wc'] ] will pipe the output of 'ls .git' into 'wc'. - infile: File to provide stdin to the pipeline - outfile: File to store stdout - capture: True to capture output - capture_stderr: True to capture stderr - oneline: True to strip newline chars from output - output_func: Output function to call with each output fragment - (if it returns True the function terminates) - kwargs: Additional keyword arguments to cros_subprocess.Popen() + infile (str): File to provide stdin to the pipeline + outfile (str): File to store stdout + capture (bool): True to capture output + capture_stderr (bool): True to capture stderr + oneline (bool): True to strip newline chars from output + raise_on_error (bool): True to raise on an error, False to return it in + the CommandResult + cwd (str or None): Directory to run the command in + binary (bool): True to report binary output, False to use strings + output_func (function): Output function to call with each output + fragment (if it returns True the function terminates) + **kwargs: Additional keyword arguments to cros_subprocess.Popen() Returns: CommandResult object Raises: @@ -89,7 +104,7 @@ def run_pipe(pipe_list, infile=None, outfile=None, result = CommandResult(b'', b'', b'') last_pipe = None pipeline = list(pipe_list) - user_pipestr = '|'.join([' '.join(pipe) for pipe in pipe_list]) + user_pipestr = '|'.join([' '.join(pipe) for pipe in pipe_list]) kwargs['stdout'] = None kwargs['stderr'] = None while pipeline: @@ -125,28 +140,66 @@ def run_pipe(pipe_list, infile=None, outfile=None, raise CommandExc(f"Error running '{user_pipestr}'", result) return result.to_output(binary) + def output(*cmd, **kwargs): + """Run a command and return its output + + Args: + *cmd (list of str): Command to run + **kwargs (dict of args): Extra arguments to pass in + + Returns: + str: command output + """ kwargs['raise_on_error'] = kwargs.get('raise_on_error', True) return run_pipe([cmd], capture=True, **kwargs).stdout + def output_one_line(*cmd, **kwargs): """Run a command and output it as a single-line string - The command us expected to produce a single line of output + The command is expected to produce a single line of output + + Args: + *cmd (list of str): Command to run + **kwargs (dict of args): Extra arguments to pass in Returns: - String containing output of command + str: output of command with all newlines removed """ raise_on_error = kwargs.pop('raise_on_error', True) result = run_pipe([cmd], capture=True, oneline=True, - raise_on_error=raise_on_error, **kwargs).stdout.strip() + raise_on_error=raise_on_error, **kwargs).stdout.strip() return result + def run(*cmd, **kwargs): + """Run a command + + Note that you must add 'capture' to kwargs to obtain non-empty output + + Args: + *cmd (list of str): Command to run + **kwargs (dict of args): Extra arguments to pass in + + Returns: + str: output of command + """ return run_pipe([cmd], **kwargs).stdout + def run_list(cmd): + """Run a command and return its output + + Args: + cmd (list of str): Command to run + + Returns: + str: output of command + """ return run_pipe([cmd], capture=True).stdout + def stop_all(): + """Stop all subprocesses initiated with cros_subprocess""" cros_subprocess.stay_alive = False -- cgit v1.3.1 From 3d094ce28a22690c3d672988af5f161310822603 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Mon, 3 Feb 2025 09:26:45 -0700 Subject: u_boot_pylib: Add a function to run a single command Add a helper to avoid needing to use a list within a list for this simple case. Update existing users of runpipe() to use this where possible. Signed-off-by: Simon Glass --- tools/binman/ftest.py | 5 ++-- tools/buildman/boards.py | 4 +-- tools/buildman/builder.py | 11 ++++---- tools/buildman/builderthread.py | 31 ++++++++++++----------- tools/buildman/func_test.py | 6 ++--- tools/buildman/toolchain.py | 2 +- tools/patman/gitutil.py | 56 ++++++++++++++++++++--------------------- tools/patman/patchstream.py | 2 +- tools/rmboard.py | 25 +++++++----------- tools/u_boot_pylib/command.py | 15 +++++++++++ tools/u_boot_pylib/tools.py | 2 +- 11 files changed, 84 insertions(+), 75 deletions(-) (limited to 'tools') diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 03eb7f814fe..9f6688ee7a4 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -345,8 +345,9 @@ class TestFunctional(unittest.TestCase): Arguments to pass, as a list of strings kwargs: Arguments to pass to Command.RunPipe() """ - result = command.run_pipe([[self._binman_pathname] + list(args)], - capture=True, capture_stderr=True, raise_on_error=False) + all_args = [self._binman_pathname] + list(args) + result = command.run_one(*all_args, capture=True, capture_stderr=True, + raise_on_error=False) if result.return_code and kwargs.get('raise_on_error', True): raise Exception("Error running '%s': %s" % (' '.join(args), result.stdout + result.stderr)) diff --git a/tools/buildman/boards.py b/tools/buildman/boards.py index e7aa0d85a58..2fe43c3fc89 100644 --- a/tools/buildman/boards.py +++ b/tools/buildman/boards.py @@ -251,9 +251,9 @@ class KconfigScanner: '-undef', '-x', 'assembler-with-cpp', defconfig] - result = command.run_pipe([cmd], capture=True, capture_stderr=True) + stdout = command.output(*cmd, capture_stderr=True) temp = tempfile.NamedTemporaryFile(prefix='buildman-') - tools.write_file(temp.name, result.stdout, False) + tools.write_file(temp.name, stdout, False) fname = temp.name tout.info(f'Processing #include to produce {defconfig}') else: diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index cbf1345281b..3eac17ac212 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -510,7 +510,7 @@ class Builder: stage: Stage that we are at (mrproper, config, oldconfig, build) cwd: Directory where make should be run args: Arguments to pass to make - kwargs: Arguments to pass to command.run_pipe() + kwargs: Arguments to pass to command.run_one() """ def check_output(stream, data): @@ -531,11 +531,12 @@ class Builder: return False self._restarting_config = False - self._terminated = False + self._terminated = False cmd = [self.gnu_make] + list(args) - result = command.run_pipe([cmd], capture=True, capture_stderr=True, - cwd=cwd, raise_on_error=False, infile='/dev/null', - output_func=check_output, **kwargs) + result = command.run_one(*cmd, capture=True, capture_stderr=True, + cwd=cwd, raise_on_error=False, + infile='/dev/null', output_func=check_output, + **kwargs) if self._terminated: # Try to be helpful diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index 29e6cf32af1..7646f2e3e27 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -179,13 +179,12 @@ class BuilderThread(threading.Thread): cwd (str): Working directory to set, or None to leave it alone *args (list of str): Arguments to pass to 'make' **kwargs (dict): A list of keyword arguments to pass to - command.run_pipe() + command.run_one() Returns: CommandResult object """ - return self.builder.do_make(commit, brd, stage, cwd, *args, - **kwargs) + return self.builder.do_make(commit, brd, stage, cwd, *args, **kwargs) def _build_args(self, brd, out_dir, out_rel_dir, work_dir, commit_upto): """Set up arguments to the args list based on the settings @@ -588,9 +587,10 @@ class BuilderThread(threading.Thread): lines = [] for fname in BASE_ELF_FILENAMES: cmd = [f'{self.toolchain.cross}nm', '--size-sort', fname] - nm_result = command.run_pipe([cmd], capture=True, - capture_stderr=True, cwd=result.out_dir, - raise_on_error=False, env=env) + nm_result = command.run_one(*cmd, capture=True, + capture_stderr=True, + cwd=result.out_dir, + raise_on_error=False, env=env) if nm_result.stdout: nm_fname = self.builder.get_func_sizes_file( result.commit_upto, result.brd.target, fname) @@ -598,9 +598,10 @@ class BuilderThread(threading.Thread): print(nm_result.stdout, end=' ', file=outf) cmd = [f'{self.toolchain.cross}objdump', '-h', fname] - dump_result = command.run_pipe([cmd], capture=True, - capture_stderr=True, cwd=result.out_dir, - raise_on_error=False, env=env) + dump_result = command.run_one(*cmd, capture=True, + capture_stderr=True, + cwd=result.out_dir, + raise_on_error=False, env=env) rodata_size = '' if dump_result.stdout: objdump = self.builder.get_objdump_file(result.commit_upto, @@ -613,9 +614,10 @@ class BuilderThread(threading.Thread): rodata_size = fields[2] cmd = [f'{self.toolchain.cross}size', fname] - size_result = command.run_pipe([cmd], capture=True, - capture_stderr=True, cwd=result.out_dir, - raise_on_error=False, env=env) + size_result = command.run_one(*cmd, capture=True, + capture_stderr=True, + cwd=result.out_dir, + raise_on_error=False, env=env) if size_result.stdout: lines.append(size_result.stdout.splitlines()[1] + ' ' + rodata_size) @@ -624,9 +626,8 @@ class BuilderThread(threading.Thread): cmd = [f'{self.toolchain.cross}objcopy', '-O', 'binary', '-j', '.rodata.default_environment', 'env/built-in.o', 'uboot.env'] - command.run_pipe([cmd], capture=True, - capture_stderr=True, cwd=result.out_dir, - raise_on_error=False, env=env) + command.run_one(*cmd, capture=True, capture_stderr=True, + cwd=result.out_dir, raise_on_error=False, env=env) if not work_in_output: copy_files(result.out_dir, build_dir, '', ['uboot.env']) diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index 51a2366e8da..1afc2fb1594 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -232,8 +232,8 @@ class TestFunctional(unittest.TestCase): self._toolchains.Add('gcc', test=False) def _RunBuildman(self, *args): - return command.run_pipe([[self._buildman_pathname] + list(args)], - capture=True, capture_stderr=True) + all_args = [self._buildman_pathname] + list(args) + return command.run_one(*all_args, capture=True, capture_stderr=True) def _RunControl(self, *args, brds=False, clean_dir=False, test_thread_exceptions=False, get_builder=True): @@ -445,7 +445,7 @@ class TestFunctional(unittest.TestCase): stage: Stage that we are at (mrproper, config, build) cwd: Directory where make should be run args: Arguments to pass to make - kwargs: Arguments to pass to command.run_pipe() + kwargs: Arguments to pass to command.run_one() """ self._make_calls += 1 out_dir = '' diff --git a/tools/buildman/toolchain.py b/tools/buildman/toolchain.py index 958f36f9f61..5d051e005da 100644 --- a/tools/buildman/toolchain.py +++ b/tools/buildman/toolchain.py @@ -100,7 +100,7 @@ class Toolchain: else: self.priority = priority if test: - result = command.run_pipe([cmd], capture=True, env=env, + result = command.run_one(*cmd, capture=True, env=env, raise_on_error=False) self.ok = result.return_code == 0 if verbose: diff --git a/tools/patman/gitutil.py b/tools/patman/gitutil.py index ffe05273b35..6d6a7eedecc 100644 --- a/tools/patman/gitutil.py +++ b/tools/patman/gitutil.py @@ -65,9 +65,9 @@ def count_commits_to_branch(branch): rev_range = '%s..%s' % (us, branch) else: rev_range = '@{upstream}..' - pipe = [log_cmd(rev_range, oneline=True)] - result = command.run_pipe(pipe, capture=True, capture_stderr=True, - oneline=True, raise_on_error=False) + cmd = log_cmd(rev_range, oneline=True) + result = command.run_one(*cmd, capture=True, capture_stderr=True, + oneline=True, raise_on_error=False) if result.return_code: raise ValueError('Failed to determine upstream: %s' % result.stderr.strip()) @@ -84,8 +84,7 @@ def name_revision(commit_hash): Return: Name of revision, if any, else None """ - pipe = ['git', 'name-rev', commit_hash] - stdout = command.run_pipe([pipe], capture=True, oneline=True).stdout + stdout = command.output_one_line('git', 'name-rev', commit_hash) # We expect a commit, a space, then a revision name name = stdout.split(' ')[1].strip() @@ -108,9 +107,9 @@ def guess_upstream(git_dir, branch): Name of upstream branch (e.g. 'upstream/master') or None if none Warning/error message, or None if none """ - pipe = [log_cmd(branch, git_dir=git_dir, oneline=True, count=100)] - result = command.run_pipe(pipe, capture=True, capture_stderr=True, - raise_on_error=False) + cmd = log_cmd(branch, git_dir=git_dir, oneline=True, count=100) + result = command.run_one(*cmd, capture=True, capture_stderr=True, + raise_on_error=False) if result.return_code: return None, "Branch '%s' not found" % branch for line in result.stdout.splitlines()[1:]: @@ -183,9 +182,9 @@ def count_commits_in_range(git_dir, range_expr): Number of patches that exist in the supplied range or None if none were found """ - pipe = [log_cmd(range_expr, git_dir=git_dir, oneline=True)] - result = command.run_pipe(pipe, capture=True, capture_stderr=True, - raise_on_error=False) + cmd = log_cmd(range_expr, git_dir=git_dir, oneline=True) + result = command.run_one(*cmd, capture=True, capture_stderr=True, + raise_on_error=False) if result.return_code: return None, "Range '%s' not found or is invalid" % range_expr patch_count = len(result.stdout.splitlines()) @@ -250,9 +249,8 @@ def clone(git_dir, output_dir): Args: commit_hash: Commit hash to check out """ - pipe = ['git', 'clone', git_dir, '.'] - result = command.run_pipe([pipe], capture=True, cwd=output_dir, - capture_stderr=True) + result = command.run_one('git', 'clone', git_dir, '.', capture=True, + cwd=output_dir, capture_stderr=True) if result.return_code != 0: raise OSError('git clone: %s' % result.stderr) @@ -263,13 +261,13 @@ def fetch(git_dir=None, work_tree=None): Args: commit_hash: Commit hash to check out """ - pipe = ['git'] + cmd = ['git'] if git_dir: - pipe.extend(['--git-dir', git_dir]) + cmd.extend(['--git-dir', git_dir]) if work_tree: - pipe.extend(['--work-tree', work_tree]) - pipe.append('fetch') - result = command.run_pipe([pipe], capture=True, capture_stderr=True) + cmd.extend(['--work-tree', work_tree]) + cmd.append('fetch') + result = command.run_one(*cmd, capture=True, capture_stderr=True) if result.return_code != 0: raise OSError('git fetch: %s' % result.stderr) @@ -283,9 +281,9 @@ def check_worktree_is_available(git_dir): Returns: True if git-worktree commands will work, False otherwise. """ - pipe = ['git', '--git-dir', git_dir, 'worktree', 'list'] - result = command.run_pipe([pipe], capture=True, capture_stderr=True, - raise_on_error=False) + result = command.run_one('git', '--git-dir', git_dir, 'worktree', 'list', + capture=True, capture_stderr=True, + raise_on_error=False) return result.return_code == 0 @@ -298,11 +296,11 @@ def add_worktree(git_dir, output_dir, commit_hash=None): commit_hash: Commit hash to checkout """ # We need to pass --detach to avoid creating a new branch - pipe = ['git', '--git-dir', git_dir, 'worktree', 'add', '.', '--detach'] + cmd = ['git', '--git-dir', git_dir, 'worktree', 'add', '.', '--detach'] if commit_hash: - pipe.append(commit_hash) - result = command.run_pipe([pipe], capture=True, cwd=output_dir, - capture_stderr=True) + cmd.append(commit_hash) + result = command.run_one(*cmd, capture=True, cwd=output_dir, + capture_stderr=True) if result.return_code != 0: raise OSError('git worktree add: %s' % result.stderr) @@ -313,8 +311,8 @@ def prune_worktrees(git_dir): Args: git_dir: The repository whose deleted worktrees should be pruned """ - pipe = ['git', '--git-dir', git_dir, 'worktree', 'prune'] - result = command.run_pipe([pipe], capture=True, capture_stderr=True) + result = command.run_one('git', '--git-dir', git_dir, 'worktree', 'prune', + capture=True, capture_stderr=True) if result.return_code != 0: raise OSError('git worktree prune: %s' % result.stderr) @@ -687,7 +685,7 @@ def setup(): if alias_fname: settings.ReadGitAliases(alias_fname) cmd = log_cmd(None, count=0) - use_no_decorate = (command.run_pipe([cmd], raise_on_error=False) + use_no_decorate = (command.run_one(*cmd, raise_on_error=False) .return_code == 0) diff --git a/tools/patman/patchstream.py b/tools/patman/patchstream.py index 4955f6aaab9..940b50ca8d4 100644 --- a/tools/patman/patchstream.py +++ b/tools/patman/patchstream.py @@ -711,7 +711,7 @@ def get_list(commit_range, git_dir=None, count=None): """ params = gitutil.log_cmd(commit_range, reverse=True, count=count, git_dir=git_dir) - return command.run_pipe([params], capture=True).stdout + return command.run_one(*params, capture=True).stdout def get_metadata_for_list(commit_range, git_dir=None, count=None, series=None, allow_overwrite=False): diff --git a/tools/rmboard.py b/tools/rmboard.py index 0c56b149e0f..594fd89b8d7 100755 --- a/tools/rmboard.py +++ b/tools/rmboard.py @@ -43,18 +43,16 @@ def rm_kconfig_include(path): Args: path: Path to search for and remove """ - cmd = ['git', 'grep', path] - stdout = command.run_pipe([cmd], capture=True, raise_on_error=False).stdout + stdout = command.output('git', 'grep', path, raise_on_error=False) if not stdout: return fname = stdout.split(':')[0] print("Fixing up '%s' to remove reference to '%s'" % (fname, path)) - cmd = ['sed', '-i', '\|%s|d' % path, fname] - stdout = command.run_pipe([cmd], capture=True).stdout + stdout = command.run_one('sed', '-i', rf'\|{path}|d', fname, + capture=True).stdout - cmd = ['git', 'add', fname] - stdout = command.run_pipe([cmd], capture=True).stdout + stdout = command.output('git', 'add', fname) def rm_board(board): """Create a commit which removes a single board @@ -68,8 +66,7 @@ def rm_board(board): """ # Find all MAINTAINERS and Kconfig files which mention the board - cmd = ['git', 'grep', '-l', board] - stdout = command.run_pipe([cmd], capture=True).stdout + stdout = command.output('git', 'grep', '-l', board) maintain = [] kconfig = [] for line in stdout.splitlines(): @@ -109,16 +106,14 @@ def rm_board(board): # Search for Kconfig files in the resulting list. Remove any 'source' lines # which reference Kconfig files we want to remove for path in real: - cmd = ['find', path] - stdout = (command.run_pipe([cmd], capture=True, raise_on_error=False). - stdout) + stdout = command.output('find', path, raise_on_error=False) for fname in stdout.splitlines(): if fname.endswith('Kconfig'): rm_kconfig_include(fname) # Remove unwanted files cmd = ['git', 'rm', '-r'] + real - stdout = command.run_pipe([cmd], capture=True).stdout + stdout = command.output(*cmd, capture=True) ## Change the messages as needed msg = '''arm: Remove %s board @@ -131,13 +126,11 @@ Remove it. msg += 'Patch-cc: %s\n' % name # Create the commit - cmd = ['git', 'commit', '-s', '-m', msg] - stdout = command.run_pipe([cmd], capture=True).stdout + stdout = command.output('git', 'commit', '-s', '-m', msg) # Check if the board is mentioned anywhere else. The user will need to deal # with this - cmd = ['git', 'grep', '-il', board] - print(command.run_pipe([cmd], capture=True, raise_on_error=False).stdout) + print(command.output('git', 'grep', '-il', board, raise_on_error=False)) print(' '.join(cmd)) for board in sys.argv[1:]: diff --git a/tools/u_boot_pylib/command.py b/tools/u_boot_pylib/command.py index a98dcedd322..0e247355ef6 100644 --- a/tools/u_boot_pylib/command.py +++ b/tools/u_boot_pylib/command.py @@ -188,6 +188,21 @@ def run(*cmd, **kwargs): return run_pipe([cmd], **kwargs).stdout +def run_one(*cmd, **kwargs): + """Run a single command + + Note that you must add 'capture' to kwargs to obtain non-empty output + + Args: + *cmd (list of str): Command to run + **kwargs (dict of args): Extra arguments to pass in + + Returns: + CommandResult: output of command + """ + return run_pipe([cmd], **kwargs) + + def run_list(cmd): """Run a command and return its output diff --git a/tools/u_boot_pylib/tools.py b/tools/u_boot_pylib/tools.py index 0499a75526f..1afd289eadd 100644 --- a/tools/u_boot_pylib/tools.py +++ b/tools/u_boot_pylib/tools.py @@ -376,7 +376,7 @@ def run_result(name, *args, **kwargs): args = tuple(extra_args) + args name = os.path.expanduser(name) # Expand paths containing ~ all_args = (name,) + args - result = command.run_pipe([all_args], capture=True, capture_stderr=True, + result = command.run_one(*all_args, capture=True, capture_stderr=True, env=env, raise_on_error=False, binary=binary) if result.return_code: if raise_on_error: -- cgit v1.3.1 From 5f2096d2bc58ac265110777d08673a0e4b27cd84 Mon Sep 17 00:00:00 2001 From: Leonard Anderweit Date: Wed, 26 Feb 2025 22:04:59 +0100 Subject: binman: build_from_git: Add argument specifying branch Add optional argument git_branch to build_from_git. The new argument allows specifying which branch of the repo to use. Signed-off-by: Leonard Anderweit Reviewed-by: Simon Glass --- tools/binman/bintool.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'tools') diff --git a/tools/binman/bintool.py b/tools/binman/bintool.py index 3c4ad1adbb9..7280ee4f8cd 100644 --- a/tools/binman/bintool.py +++ b/tools/binman/bintool.py @@ -328,7 +328,8 @@ class Bintool: return result.stdout @classmethod - def build_from_git(cls, git_repo, make_targets, bintool_path, flags=None): + def build_from_git(cls, git_repo, make_targets, bintool_path, + flags=None, git_branch=None): """Build a bintool from a git repo This clones the repo in a temporary directory, builds it with 'make', @@ -341,6 +342,7 @@ class Bintool: bintool_path (str): Relative path of the tool in the repo, after build is complete flags (list of str): Flags or variables to pass to make, or None + git_branch (str): Branch of git repo, or None to use the default Returns: tuple: @@ -350,7 +352,11 @@ class Bintool: """ tmpdir = tempfile.mkdtemp(prefix='binmanf.') print(f"- clone git repo '{git_repo}' to '{tmpdir}'") - tools.run('git', 'clone', '--depth', '1', git_repo, tmpdir) + if git_branch: + tools.run('git', 'clone', '--depth', '1', '--branch', git_branch, + git_repo, tmpdir) + else: + tools.run('git', 'clone', '--depth', '1', git_repo, tmpdir) for target in make_targets: print(f"- build target '{target}'") cmd = ['make', '-C', tmpdir, '-j', f'{multiprocessing.cpu_count()}', -- cgit v1.3.1 From b48bc416206788da990c34ddcf92d5c3816247ea Mon Sep 17 00:00:00 2001 From: Leonard Anderweit Date: Wed, 26 Feb 2025 22:05:00 +0100 Subject: binman: build_from_git: Add optional make path inside git repo Add optional argument make_path to build_from git. The new argument allows specifying the path to a Makefile in case it is not in the root of the git repo. Also adjust the corresponding test. Signed-off-by: Leonard Anderweit Reviewed-by: Simon Glass --- tools/binman/bintool.py | 9 +++++++-- tools/binman/bintool_test.py | 1 + 2 files changed, 8 insertions(+), 2 deletions(-) (limited to 'tools') diff --git a/tools/binman/bintool.py b/tools/binman/bintool.py index 7280ee4f8cd..81872db377f 100644 --- a/tools/binman/bintool.py +++ b/tools/binman/bintool.py @@ -329,7 +329,7 @@ class Bintool: @classmethod def build_from_git(cls, git_repo, make_targets, bintool_path, - flags=None, git_branch=None): + flags=None, git_branch=None, make_path=None): """Build a bintool from a git repo This clones the repo in a temporary directory, builds it with 'make', @@ -343,6 +343,8 @@ class Bintool: build is complete flags (list of str): Flags or variables to pass to make, or None git_branch (str): Branch of git repo, or None to use the default + make_path (str): Relative path inside git repo containing the + Makefile, or None Returns: tuple: @@ -359,7 +361,10 @@ class Bintool: tools.run('git', 'clone', '--depth', '1', git_repo, tmpdir) for target in make_targets: print(f"- build target '{target}'") - cmd = ['make', '-C', tmpdir, '-j', f'{multiprocessing.cpu_count()}', + makedir = tmpdir + if make_path: + makedir = os.path.join(tmpdir, make_path) + cmd = ['make', '-C', makedir, '-j', f'{multiprocessing.cpu_count()}', target] if flags: cmd += flags diff --git a/tools/binman/bintool_test.py b/tools/binman/bintool_test.py index f9b16d4c73b..949d6f4c8a9 100644 --- a/tools/binman/bintool_test.py +++ b/tools/binman/bintool_test.py @@ -303,6 +303,7 @@ class TestBintool(unittest.TestCase): # See Bintool.build_from_git() tmpdir = cmd[2] self.fname = os.path.join(tmpdir, 'pathname') + os.makedirs(os.path.dirname(tmpdir), exist_ok=True) tools.write_file(self.fname, b'hello') expected = b'this is a test' -- cgit v1.3.1 From 326b7ea9823a74a83b3a7ef6ee3f4927eb36987e Mon Sep 17 00:00:00 2001 From: Leonard Anderweit Date: Wed, 26 Feb 2025 22:05:01 +0100 Subject: binman: cst: Build from source Build the imx code singing tool from source instead of relying on the distro to provide the tool. Use the debian/unstable branch because the default branch is outdated. The binary is supposed to be build with docker, work around that by selecting the correct Makefile directly. Also append the description and add a link to documentation. Signed-off-by: Leonard Anderweit Reviewed-by: Simon Glass --- tools/binman/bintools.rst | 8 ++++++++ tools/binman/btool/cst.py | 37 +++++++++++++++++++++---------------- 2 files changed, 29 insertions(+), 16 deletions(-) (limited to 'tools') diff --git a/tools/binman/bintools.rst b/tools/binman/bintools.rst index cd05ad8cb26..9f6cab544a5 100644 --- a/tools/binman/bintools.rst +++ b/tools/binman/bintools.rst @@ -52,6 +52,14 @@ Bintool: cst: Image generation for U-Boot This bintool supports running `cst` with some basic parameters as needed by binman. +cst (imx code signing tool) is used for sigining bootloader binaries for +various i.MX SoCs. + +See `Code Signing Tool Users Guide`_ for more information. + +.. _`Code Signing Tool Users Guide`: + https://community.nxp.com/pwmxy87654/attachments/pwmxy87654/imx-processors/202591/1/CST_UG.pdf + Bintool: fdt_add_pubkey: Add public key to control dtb (spl or u-boot proper) diff --git a/tools/binman/btool/cst.py b/tools/binman/btool/cst.py index 30e78bdbbd9..8a3981adc89 100644 --- a/tools/binman/btool/cst.py +++ b/tools/binman/btool/cst.py @@ -12,6 +12,14 @@ class Bintoolcst(bintool.Bintool): This bintool supports running `cst` with some basic parameters as needed by binman. + + cst (imx code signing tool) is used for sigining bootloader binaries for + various i.MX SoCs. + + See `Code Signing Tool Users Guide`_ for more information. + + .. _`Code Signing Tool Users Guide`: + https://community.nxp.com/pwmxy87654/attachments/pwmxy87654/imx-processors/202591/1/CST_UG.pdf """ def __init__(self, name): super().__init__(name, 'Sign NXP i.MX image') @@ -29,20 +37,17 @@ class Bintoolcst(bintool.Bintool): return self.run_cmd(*args) def fetch(self, method): - """Fetch handler for cst - - This installs cst using the apt utility. - - Args: - method (FETCH_...): Method to use - - Returns: - True if the file was fetched and now installed, None if a method - other than FETCH_BIN was requested - - Raises: - Valuerror: Fetching could not be completed - """ - if method != bintool.FETCH_BIN: + """Build cst from git""" + if method != bintool.FETCH_BUILD: return None - return self.apt_install('imx-code-signing-tool') + + from platform import architecture + arch = 'linux64' if architecture()[0] == '64bit' else 'linux32' + result = self.build_from_git( + 'https://gitlab.apertis.org/pkg/imx-code-signing-tool', + ['all'], + f'code/obj.{arch}/cst', + flags=[f'OSTYPE={arch}', 'ENCRYPTION=yes'], + git_branch='debian/unstable', + make_path=f'code/obj.{arch}/') + return result -- cgit v1.3.1 From a920169b4c7446702f3fbf3a61cc55069ee128f9 Mon Sep 17 00:00:00 2001 From: Tom Rini Date: Wed, 12 Feb 2025 16:24:15 -0600 Subject: Dockerfile: Add missing 'rm -rf /tmp/coreboot-24.08' We had missed removing the coreboot directory once done, fix this. Signed-off-by: Tom Rini --- tools/docker/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile index 85d67848327..569912303fc 100644 --- a/tools/docker/Dockerfile +++ b/tools/docker/Dockerfile @@ -296,7 +296,8 @@ RUN wget -O - https://coreboot.org/releases/coreboot-24.08.tar.xz | tar -C /tmp make olddefconfig && \ make -j $(nproc) && \ sudo mkdir /opt/coreboot && \ - sudo cp build/coreboot.rom build/cbfstool /opt/coreboot/ + sudo cp build/coreboot.rom build/cbfstool /opt/coreboot/ && \ + rm -rf /tmp/coreboot-24.08 # Create our user/group RUN echo uboot ALL=NOPASSWD: ALL > /etc/sudoers.d/uboot -- cgit v1.3.1 From 7a765a37b3d4dead659a061561489ccce1276923 Mon Sep 17 00:00:00 2001 From: Raymond Mao Date: Wed, 26 Feb 2025 06:19:51 -0800 Subject: tools: add HOSTCFLAGS from openssl pkg-config HOSTCFLAGS of some tools components (image-host, rsa-sign and ecdsa-libcrypto) depend on the directory where openssl is installed. Add them via pkg-config. This fixes a potential build failure in tools when openssl in installed in varied directories. Signed-off-by: Raymond Mao Reviewed-by: Peter Robinson --- tools/Makefile | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'tools') diff --git a/tools/Makefile b/tools/Makefile index e5f5eea47c7..53b87d22a80 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -76,6 +76,9 @@ FIT_OBJS-y := fit_common.o fit_image.o image-host.o generated/boot/image-fit.o FIT_SIG_OBJS-$(CONFIG_TOOLS_LIBCRYPTO) := image-sig-host.o generated/boot/image-fit-sig.o FIT_CIPHER_OBJS-$(CONFIG_TOOLS_LIBCRYPTO) := generated/boot/image-cipher.o +HOSTCFLAGS_image-host.o += \ + $(shell pkg-config --cflags libssl libcrypto 2> /dev/null || echo "") + # The following files are synced with upstream DTC. # Use synced versions from scripts/dtc/libfdt/. LIBFDT_OBJS := $(addprefix libfdt/, fdt.o fdt_ro.o fdt_wip.o fdt_sw.o fdt_rw.o \ @@ -84,8 +87,12 @@ LIBFDT_OBJS := $(addprefix libfdt/, fdt.o fdt_ro.o fdt_wip.o fdt_sw.o fdt_rw.o \ RSA_OBJS-$(CONFIG_TOOLS_LIBCRYPTO) := $(addprefix generated/lib/rsa/, \ rsa-sign.o rsa-verify.o \ rsa-mod-exp.o) +HOSTCFLAGS_rsa-sign.o += \ + $(shell pkg-config --cflags libssl libcrypto 2> /dev/null || echo "") ECDSA_OBJS-$(CONFIG_TOOLS_LIBCRYPTO) := $(addprefix generated/lib/ecdsa/, ecdsa-libcrypto.o) +HOSTCFLAGS_ecdsa-libcrypto.o += \ + $(shell pkg-config --cflags libssl libcrypto 2> /dev/null || echo "") AES_OBJS-$(CONFIG_TOOLS_LIBCRYPTO) := $(addprefix generated/lib/aes/, \ aes-encrypt.o aes-decrypt.o) -- cgit v1.3.1 From 3703298e57c8d749b3c06798f5873562754fb84c Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sun, 16 Mar 2025 08:00:18 +0000 Subject: u_boot_pylib: Clean up pylint warnings in gitutil.py This file has about 40 pylint warnings, but no errors. Quite a few of these warnings have been there for a while, but most are coming from newer versions of pylint, where people come up with new warnings. The f-string warning is the most common one: C0209: Formatting a regular string which could be an f-string That feature was not available when the code was written, but it is often more convenient than using % with a list of arguments. This patches reduces the number of warnings in this file, with 7 left remaining. Signed-off-by: Simon Glass --- tools/buildman/func_test.py | 4 +- tools/u_boot_pylib/gitutil.py | 240 ++++++++++++++++++++++-------------------- 2 files changed, 128 insertions(+), 116 deletions(-) (limited to 'tools') diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index d779040c74e..b45eb95a1e6 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -288,11 +288,11 @@ class TestFunctional(unittest.TestCase): """Test gitutils.Setup(), from outside the module itself""" command.TEST_RESULT = command.CommandResult(return_code=1) gitutil.setup() - self.assertEqual(gitutil.use_no_decorate, False) + self.assertEqual(gitutil.USE_NO_DECORATE, False) command.TEST_RESULT = command.CommandResult(return_code=0) gitutil.setup() - self.assertEqual(gitutil.use_no_decorate, True) + self.assertEqual(gitutil.USE_NO_DECORATE, True) def _HandleCommandGitLog(self, args): if args[-1] == '--': diff --git a/tools/u_boot_pylib/gitutil.py b/tools/u_boot_pylib/gitutil.py index 6d6a7eedecc..5e3e98ac9a6 100644 --- a/tools/u_boot_pylib/gitutil.py +++ b/tools/u_boot_pylib/gitutil.py @@ -10,7 +10,7 @@ from u_boot_pylib import command from u_boot_pylib import terminal # True to use --no-decorate - we check this in setup() -use_no_decorate = True +USE_NO_DECORATE = True def log_cmd(commit_range, git_dir=None, oneline=False, reverse=False, @@ -18,11 +18,11 @@ def log_cmd(commit_range, git_dir=None, oneline=False, reverse=False, """Create a command to perform a 'git log' Args: - commit_range: Range expression to use for log, None for none - git_dir: Path to git repository (None to use default) - oneline: True to use --oneline, else False - reverse: True to reverse the log (--reverse) - count: Number of commits to list, or None for no limit + commit_range (str): Range expression to use for log, None for none + git_dir (str): Path to git repository (None to use default) + oneline (bool): True to use --oneline, else False + reverse (bool): True to reverse the log (--reverse) + count (int or None): Number of commits to list, or None for no limit Return: List containing command and arguments to run """ @@ -32,12 +32,12 @@ def log_cmd(commit_range, git_dir=None, oneline=False, reverse=False, cmd += ['--no-pager', 'log', '--no-color'] if oneline: cmd.append('--oneline') - if use_no_decorate: + if USE_NO_DECORATE: cmd.append('--no-decorate') if reverse: cmd.append('--reverse') if count is not None: - cmd.append('-n%d' % count) + cmd.append(f'-n{count}') if commit_range: cmd.append(commit_range) @@ -55,22 +55,22 @@ def count_commits_to_branch(branch): since then. Args: - branch: Branch to count from (None for current branch) + branch (str or None): Branch to count from (None for current branch) Return: Number of patches that exist on top of the branch """ if branch: - us, msg = get_upstream('.git', branch) - rev_range = '%s..%s' % (us, branch) + us, _ = get_upstream('.git', branch) + rev_range = f'{us}..{branch}' else: rev_range = '@{upstream}..' cmd = log_cmd(rev_range, oneline=True) result = command.run_one(*cmd, capture=True, capture_stderr=True, oneline=True, raise_on_error=False) if result.return_code: - raise ValueError('Failed to determine upstream: %s' % - result.stderr.strip()) + raise ValueError( + f'Failed to determine upstream: {result.stderr.strip()}') patch_count = len(result.stdout.splitlines()) return patch_count @@ -79,7 +79,7 @@ def name_revision(commit_hash): """Gets the revision name for a commit Args: - commit_hash: Commit hash to look up + commit_hash (str): Commit hash to look up Return: Name of revision, if any, else None @@ -99,8 +99,8 @@ def guess_upstream(git_dir, branch): 'git name-rev' returns a plain branch name, with no ! or ^ modifiers. Args: - git_dir: Git directory containing repo - branch: Name of branch + git_dir (str): Git directory containing repo + branch (str): Name of branch Returns: Tuple: @@ -111,23 +111,23 @@ def guess_upstream(git_dir, branch): result = command.run_one(*cmd, capture=True, capture_stderr=True, raise_on_error=False) if result.return_code: - return None, "Branch '%s' not found" % branch + return None, f"Branch '{branch}' not found" for line in result.stdout.splitlines()[1:]: commit_hash = line.split(' ')[0] name = name_revision(commit_hash) if '~' not in name and '^' not in name: if name.startswith('remotes/'): name = name[8:] - return name, "Guessing upstream as '%s'" % name - return None, "Cannot find a suitable upstream for branch '%s'" % branch + return name, f"Guessing upstream as '{name}'" + return None, f"Cannot find a suitable upstream for branch '{branch}'" def get_upstream(git_dir, branch): """Returns the name of the upstream for a branch Args: - git_dir: Git directory containing repo - branch: Name of branch + git_dir (str): Git directory containing repo + branch (str): Name of branch Returns: Tuple: @@ -136,31 +136,30 @@ def get_upstream(git_dir, branch): """ try: remote = command.output_one_line('git', '--git-dir', git_dir, 'config', - 'branch.%s.remote' % branch) + f'branch.{branch}.remote') merge = command.output_one_line('git', '--git-dir', git_dir, 'config', - 'branch.%s.merge' % branch) + f'branch.{branch}.merge') except command.CommandExc: upstream, msg = guess_upstream(git_dir, branch) return upstream, msg if remote == '.': return merge, None - elif remote and merge: + if remote and merge: # Drop the initial refs/heads from merge leaf = merge.split('/', maxsplit=2)[2:] - return '%s/%s' % (remote, '/'.join(leaf)), None - else: - raise ValueError("Cannot determine upstream branch for branch " - "'%s' remote='%s', merge='%s'" - % (branch, remote, merge)) + return f'{remote}/{"/".join(leaf)}', None + raise ValueError("Cannot determine upstream branch for branch " + f"'{branch}' remote='{remote}', merge='{merge}'") def get_range_in_branch(git_dir, branch, include_upstream=False): """Returns an expression for the commits in the given branch. Args: - git_dir: Directory containing git repo - branch: Name of branch + git_dir (str): Directory containing git repo + branch (str): Name of branch + include_upstream (bool): Include the upstream commit as well Return: Expression in the form 'upstream..branch' which can be used to access the commits. If the branch does not exist, returns None. @@ -168,7 +167,7 @@ def get_range_in_branch(git_dir, branch, include_upstream=False): upstream, msg = get_upstream(git_dir, branch) if not upstream: return None, msg - rstr = '%s%s..%s' % (upstream, '~' if include_upstream else '', branch) + rstr = f"{upstream}{'~' if include_upstream else ''}..{branch}" return rstr, msg @@ -176,8 +175,8 @@ def count_commits_in_range(git_dir, range_expr): """Returns the number of commits in the given range. Args: - git_dir: Directory containing git repo - range_expr: Range to check + git_dir (str): Directory containing git repo + range_expr (str): Range to check Return: Number of patches that exist in the supplied range or None if none were found @@ -186,7 +185,7 @@ def count_commits_in_range(git_dir, range_expr): result = command.run_one(*cmd, capture=True, capture_stderr=True, raise_on_error=False) if result.return_code: - return None, "Range '%s' not found or is invalid" % range_expr + return None, f"Range '{range_expr}' not found or is invalid" patch_count = len(result.stdout.splitlines()) return patch_count, None @@ -195,8 +194,9 @@ def count_commits_in_branch(git_dir, branch, include_upstream=False): """Returns the number of commits in the given branch. Args: - git_dir: Directory containing git repo - branch: Name of branch + git_dir (str): Directory containing git repo + branch (str): Name of branch + include_upstream (bool): Include the upstream commit as well Return: Number of patches that exist on top of the branch, or None if the branch does not exist. @@ -211,7 +211,7 @@ def count_commits(commit_range): """Returns the number of commits in the given range. Args: - commit_range: Range of commits to count (e.g. 'HEAD..base') + commit_range (str): Range of commits to count (e.g. 'HEAD..base') Return: Number of patches that exist on top of the branch """ @@ -226,7 +226,10 @@ def checkout(commit_hash, git_dir=None, work_tree=None, force=False): """Checkout the selected commit for this build Args: - commit_hash: Commit hash to check out + commit_hash (str): Commit hash to check out + git_dir (str): Directory containing git repo, or None for current dir + work_tree (str): Git worktree to use, or None if none + force (bool): True to force the checkout (git checkout -f) """ pipe = ['git'] if git_dir: @@ -240,26 +243,28 @@ def checkout(commit_hash, git_dir=None, work_tree=None, force=False): result = command.run_pipe([pipe], capture=True, raise_on_error=False, capture_stderr=True) if result.return_code != 0: - raise OSError('git checkout (%s): %s' % (pipe, result.stderr)) + raise OSError(f'git checkout ({pipe}): {result.stderr}') -def clone(git_dir, output_dir): - """Checkout the selected commit for this build +def clone(repo, output_dir): + """Clone a repo Args: - commit_hash: Commit hash to check out + repo (str): Repo to clone (e.g. web address) + output_dir (str): Directory to close into """ - result = command.run_one('git', 'clone', git_dir, '.', capture=True, + result = command.run_one('git', 'clone', repo, '.', capture=True, cwd=output_dir, capture_stderr=True) if result.return_code != 0: - raise OSError('git clone: %s' % result.stderr) + raise OSError(f'git clone: {result.stderr}') def fetch(git_dir=None, work_tree=None): """Fetch from the origin repo Args: - commit_hash: Commit hash to check out + git_dir (str): Directory containing git repo, or None for current dir + work_tree (str or None): Git worktree to use, or None if none """ cmd = ['git'] if git_dir: @@ -269,14 +274,14 @@ def fetch(git_dir=None, work_tree=None): cmd.append('fetch') result = command.run_one(*cmd, capture=True, capture_stderr=True) if result.return_code != 0: - raise OSError('git fetch: %s' % result.stderr) + raise OSError(f'git fetch: {result.stderr}') def check_worktree_is_available(git_dir): """Check if git-worktree functionality is available Args: - git_dir: The repository to test in + git_dir (str): The repository to test in Returns: True if git-worktree commands will work, False otherwise. @@ -291,9 +296,9 @@ def add_worktree(git_dir, output_dir, commit_hash=None): """Create and checkout a new git worktree for this build Args: - git_dir: The repository to checkout the worktree from - output_dir: Path for the new worktree - commit_hash: Commit hash to checkout + git_dir (str): The repository to checkout the worktree from + output_dir (str): Path for the new worktree + commit_hash (str): Commit hash to checkout """ # We need to pass --detach to avoid creating a new branch cmd = ['git', '--git-dir', git_dir, 'worktree', 'add', '.', '--detach'] @@ -302,19 +307,19 @@ def add_worktree(git_dir, output_dir, commit_hash=None): result = command.run_one(*cmd, capture=True, cwd=output_dir, capture_stderr=True) if result.return_code != 0: - raise OSError('git worktree add: %s' % result.stderr) + raise OSError(f'git worktree add: {result.stderr}') def prune_worktrees(git_dir): """Remove administrative files for deleted worktrees Args: - git_dir: The repository whose deleted worktrees should be pruned + git_dir (str): The repository whose deleted worktrees should be pruned """ result = command.run_one('git', '--git-dir', git_dir, 'worktree', 'prune', capture=True, capture_stderr=True) if result.return_code != 0: - raise OSError('git worktree prune: %s' % result.stderr) + raise OSError(f'git worktree prune: {result.stderr}') def create_patches(branch, start, count, ignore_binary, series, signoff=True): @@ -324,11 +329,12 @@ def create_patches(branch, start, count, ignore_binary, series, signoff=True): git format-patch. Args: - branch: Branch to create patches from (None for current branch) - start: Commit to start from: 0=HEAD, 1=next one, etc. - count: number of commits to include - ignore_binary: Don't generate patches for binary files - series: Series object for this series (set of patches) + branch (str): Branch to create patches from (None for current branch) + start (int): Commit to start from: 0=HEAD, 1=next one, etc. + count (int): number of commits to include + ignore_binary (bool): Don't generate patches for binary files + series (Series): Series object for this series (set of patches) + signoff (bool): True to add signoff lines automatically Return: Filename of cover letter (None if none) List of filenames of patch files @@ -342,9 +348,9 @@ def create_patches(branch, start, count, ignore_binary, series, signoff=True): cmd.append('--cover-letter') prefix = series.GetPatchPrefix() if prefix: - cmd += ['--subject-prefix=%s' % prefix] + cmd += [f'--subject-prefix={prefix}'] brname = branch or 'HEAD' - cmd += ['%s~%d..%s~%d' % (brname, start + count, brname, start)] + cmd += [f'{brname}~{start + count}..{brname}~{start}'] stdout = command.run_list(cmd) files = stdout.splitlines() @@ -352,8 +358,7 @@ def create_patches(branch, start, count, ignore_binary, series, signoff=True): # We have an extra file if there is a cover letter if series.get('cover'): return files[0], files[1:] - else: - return None, files + return None, files def build_email_list(in_list, tag=None, alias=None, warn_on_error=True): @@ -367,11 +372,13 @@ def build_email_list(in_list, tag=None, alias=None, warn_on_error=True): command line parameter) then the email address is quoted. Args: - in_list: List of aliases/email addresses - tag: Text to put before each address - alias: Alias dictionary - warn_on_error: True to raise an error when an alias fails to match, - False to just print a message. + in_list (list of str): List of aliases/email addresses + tag (str): Text to put before each address + alias (dict): Alias dictionary: + key: alias + value: list of aliases or email addresses + warn_on_error (bool): True to raise an error when an alias fails to + match, False to just print a message. Returns: List of email addresses @@ -399,7 +406,7 @@ def build_email_list(in_list, tag=None, alias=None, warn_on_error=True): if item not in result: result.append(item) if tag: - return ['%s %s%s%s' % (tag, quote, email, quote) for email in result] + return [f'{tag} {quote}{email}{quote}' for email in result] return result @@ -407,24 +414,23 @@ def check_suppress_cc_config(): """Check if sendemail.suppresscc is configured correctly. Returns: - True if the option is configured correctly, False otherwise. + bool: True if the option is configured correctly, False otherwise. """ suppresscc = command.output_one_line( 'git', 'config', 'sendemail.suppresscc', raise_on_error=False) # Other settings should be fine. - if suppresscc == 'all' or suppresscc == 'cccmd': + if suppresscc in ('all', 'cccmd'): col = terminal.Color() - print((col.build(col.RED, "error") + - ": git config sendemail.suppresscc set to %s\n" - % (suppresscc)) + - " patman needs --cc-cmd to be run to set the cc list.\n" + - " Please run:\n" + - " git config --unset sendemail.suppresscc\n" + - " Or read the man page:\n" + - " git send-email --help\n" + - " and set an option that runs --cc-cmd\n") + print(col.build(col.RED, 'error') + + f': git config sendemail.suppresscc set to {suppresscc}\n' + + ' patman needs --cc-cmd to be run to set the cc list.\n' + + ' Please run:\n' + + ' git config --unset sendemail.suppresscc\n' + + ' Or read the man page:\n' + + ' git send-email --help\n' + + ' and set an option that runs --cc-cmd\n') return False return True @@ -432,24 +438,26 @@ def check_suppress_cc_config(): def email_patches(series, cover_fname, args, dry_run, warn_on_error, cc_fname, self_only=False, alias=None, in_reply_to=None, thread=False, - smtp_server=None, get_maintainer_script=None): + smtp_server=None): """Email a patch series. Args: - series: Series object containing destination info - cover_fname: filename of cover letter - args: list of filenames of patch files - dry_run: Just return the command that would be run - warn_on_error: True to print a warning when an alias fails to match, - False to ignore it. - cc_fname: Filename of Cc file for per-commit Cc - self_only: True to just email to yourself as a test - in_reply_to: If set we'll pass this to git as --in-reply-to. - Should be a message ID that this is in reply to. - thread: True to add --thread to git send-email (make + series (Series): Series object containing destination info + cover_fname (str or None): filename of cover letter + args (list of str): list of filenames of patch files + dry_run (bool): Just return the command that would be run + warn_on_error (bool): True to print a warning when an alias fails to + match, False to ignore it. + cc_fname (str): Filename of Cc file for per-commit Cc + self_only (bool): True to just email to yourself as a test + alias (dict or None): Alias dictionary: (None to use settings default) + key: alias + value: list of aliases or email addresses + in_reply_to (str or None): If set we'll pass this to git as + --in-reply-to - should be a message ID that this is in reply to. + thread (bool): True to add --thread to git send-email (make all patches reply to cover-letter or first patch in series) - smtp_server: SMTP server to use to send patches - get_maintainer_script: File name of script to get maintainers emails + smtp_server (str or None): SMTP server to use to send patches Returns: Git command that was/would be run @@ -500,7 +508,7 @@ send --cc-cmd cc-fname" cover p1 p2' "Series-to: Fred Bloggs \n" "Or do something like this\n" "git config sendemail.to u-boot@lists.denx.de") - return + return None cc = build_email_list(list(set(series.get('cc')) - set(series.get('to'))), '--cc', alias, warn_on_error) if self_only: @@ -509,15 +517,15 @@ send --cc-cmd cc-fname" cover p1 p2' cc = [] cmd = ['git', 'send-email', '--annotate'] if smtp_server: - cmd.append('--smtp-server=%s' % smtp_server) + cmd.append(f'--smtp-server={smtp_server}') if in_reply_to: - cmd.append('--in-reply-to="%s"' % in_reply_to) + cmd.append(f'--in-reply-to="{in_reply_to}"') if thread: cmd.append('--thread') cmd += to cmd += cc - cmd += ['--cc-cmd', '"%s send --cc-cmd %s"' % (sys.argv[0], cc_fname)] + cmd += ['--cc-cmd', f'"{sys.argv[0]} send --cc-cmd {cc_fname}"'] if cover_fname: cmd.append(cover_fname) cmd += args @@ -533,10 +541,13 @@ def lookup_email(lookup_name, alias=None, warn_on_error=True, level=0): TODO: Why not just use git's own alias feature? Args: - lookup_name: Alias or email address to look up - alias: Dictionary containing aliases (None to use settings default) - warn_on_error: True to print a warning when an alias fails to match, - False to ignore it. + lookup_name (str): Alias or email address to look up + alias (dict or None): Alias dictionary: (None to use settings default) + key: alias + value: list of aliases or email addresses + warn_on_error (bool): True to print a warning when an alias fails to + match, False to ignore it. + level (int): Depth of alias stack, used to detect recusion/loops Returns: tuple: @@ -589,16 +600,15 @@ def lookup_email(lookup_name, alias=None, warn_on_error=True, level=0): out_list = [] if level > 10: - msg = "Recursive email alias at '%s'" % lookup_name + msg = f"Recursive email alias at '{lookup_name}'" if warn_on_error: raise OSError(msg) - else: - print(col.build(col.RED, msg)) - return out_list + print(col.build(col.RED, msg)) + return out_list if lookup_name: if lookup_name not in alias: - msg = "Alias '%s' not found" % lookup_name + msg = f"Alias '{lookup_name}' not found" if warn_on_error: print(col.build(col.RED, msg)) return out_list @@ -615,7 +625,7 @@ def get_top_level(): """Return name of top-level directory for this git repo. Returns: - Full path to git top-level directory + str: Full path to git top-level directory This test makes sure that we are running tests in the right subdir @@ -630,7 +640,7 @@ def get_alias_file(): """Gets the name of the git alias file. Returns: - Filename of git alias file, or None if none + str: Filename of git alias file, or None if none """ fname = command.output_one_line('git', 'config', 'sendemail.aliasesfile', raise_on_error=False) @@ -650,7 +660,8 @@ def get_default_user_name(): Returns: User name found in .gitconfig file, or None if none """ - uname = command.output_one_line('git', 'config', '--global', '--includes', 'user.name') + uname = command.output_one_line('git', 'config', '--global', '--includes', + 'user.name') return uname @@ -660,7 +671,8 @@ def get_default_user_email(): Returns: User's email found in .gitconfig file, or None if none """ - uemail = command.output_one_line('git', 'config', '--global', '--includes', 'user.email') + uemail = command.output_one_line('git', 'config', '--global', '--includes', + 'user.email') return uemail @@ -677,15 +689,15 @@ def get_default_subject_prefix(): def setup(): - """Set up git utils, by reading the alias files.""" + """setup() - Set up git utils, by reading the alias files.""" # Check for a git alias file also - global use_no_decorate + global USE_NO_DECORATE alias_fname = get_alias_file() if alias_fname: settings.ReadGitAliases(alias_fname) cmd = log_cmd(None, count=0) - use_no_decorate = (command.run_one(*cmd, raise_on_error=False) + USE_NO_DECORATE = (command.run_one(*cmd, raise_on_error=False) .return_code == 0) -- cgit v1.3.1 From 8d0cc62a60b5b92a010f75fd61d9eb9cb8299567 Mon Sep 17 00:00:00 2001 From: Marek Vasut Date: Mon, 17 Mar 2025 04:12:50 +0100 Subject: test_fs: Add exfat tests Add tests for the exfat filesystem. These tests are largely an extension of the FS_GENERIC tests with the following notable exceptions. The filesystem image for exfat tests is generated using combination of exfatprogs mkfs.exfat and python fattools. The fattols are capable of generating exfat filesystem images too, but this is not used, the fattools are only used as a replacement for dosfstools 'mcopy' and 'mdir', which are used to insert files and directories into existing fatfs images and list existing fatfs images respectively, without the need for superuser access to mount such images. The exfat filesystem has no filesystem specific command, there is only the generic filesystem command interface, therefore check_ubconfig() has to special case exfat and skip check for CONFIG_CMD_EXFAT and instead check for CONFIG_FS_EXFAT. Signed-off-by: Marek Vasut --- test/py/requirements.txt | 1 + test/py/tests/fs_helper.py | 8 ++++++-- test/py/tests/test_fs/conftest.py | 20 +++++++++++--------- test/py/tests/test_fs/fstest_helpers.py | 2 ++ test/py/tests/test_fs/test_ext.py | 20 ++++++++++++++------ tools/docker/Dockerfile | 1 + 6 files changed, 35 insertions(+), 17 deletions(-) (limited to 'tools') diff --git a/test/py/requirements.txt b/test/py/requirements.txt index acfe17dce9f..804a427b351 100644 --- a/test/py/requirements.txt +++ b/test/py/requirements.txt @@ -2,3 +2,4 @@ filelock==3.0.12 pycryptodomex==3.21.0 pytest==6.2.5 pytest-xdist==2.5.0 +FATtools==1.0.42 diff --git a/test/py/tests/fs_helper.py b/test/py/tests/fs_helper.py index 378d5ae0690..94a5b94f479 100644 --- a/test/py/tests/fs_helper.py +++ b/test/py/tests/fs_helper.py @@ -35,7 +35,9 @@ def mk_fs(config, fs_type, size, prefix, src_dir=None, size_gran = 0x100000): else: mkfs_opt = '' - if re.match('fat', fs_type) or fs_type == 'fs_generic': + if fs_type == 'exfat': + fs_lnxtype = 'exfat' + elif re.match('fat', fs_type) or fs_type == 'fs_generic': fs_lnxtype = 'vfat' else: fs_lnxtype = fs_type @@ -43,7 +45,7 @@ def mk_fs(config, fs_type, size, prefix, src_dir=None, size_gran = 0x100000): if src_dir: if fs_lnxtype == 'ext4': mkfs_opt = mkfs_opt + ' -d ' + src_dir - elif fs_lnxtype != 'vfat': + elif fs_lnxtype != 'vfat' and fs_lnxtype != 'exfat': raise ValueError(f'src_dir not implemented for fs {fs_lnxtype}') count = (size + size_gran - 1) // size_gran @@ -64,6 +66,8 @@ def mk_fs(config, fs_type, size, prefix, src_dir=None, size_gran = 0x100000): check_call(f'tune2fs -O ^metadata_csum {fs_img}', shell=True) elif fs_lnxtype == 'vfat' and src_dir: check_call(f'mcopy -i {fs_img} -vsmpQ {src_dir}/* ::/', shell=True) + elif fs_lnxtype == 'exfat' and src_dir: + check_call(f'fattools cp {src_dir}/* {fs_img}', shell=True) return fs_img except CalledProcessError: call(f'rm -f {fs_img}', shell=True) diff --git a/test/py/tests/test_fs/conftest.py b/test/py/tests/test_fs/conftest.py index 691bdf41ede..c73fb4abbcb 100644 --- a/test/py/tests/test_fs/conftest.py +++ b/test/py/tests/test_fs/conftest.py @@ -11,11 +11,11 @@ from fstest_defs import * # pylint: disable=E0611 from tests import fs_helper -supported_fs_basic = ['fat16', 'fat32', 'ext4', 'fs_generic'] -supported_fs_ext = ['fat12', 'fat16', 'fat32', 'fs_generic'] +supported_fs_basic = ['fat16', 'fat32', 'exfat', 'ext4', 'fs_generic'] +supported_fs_ext = ['fat12', 'fat16', 'fat32', 'exfat', 'fs_generic'] supported_fs_fat = ['fat12', 'fat16'] -supported_fs_mkdir = ['fat12', 'fat16', 'fat32', 'fs_generic'] -supported_fs_unlink = ['fat12', 'fat16', 'fat32', 'fs_generic'] +supported_fs_mkdir = ['fat12', 'fat16', 'fat32', 'exfat', 'fs_generic'] +supported_fs_unlink = ['fat12', 'fat16', 'fat32', 'exfat', 'fs_generic'] supported_fs_symlink = ['ext4'] supported_fs_rename = ['fat12', 'fat16', 'fat32'] @@ -117,7 +117,7 @@ def fstype_to_prefix(fs_type): Return: A corresponding command prefix for file system type. """ - if fs_type == 'fs_generic': + if fs_type == 'fs_generic' or fs_type == 'exfat': return '' elif re.match('fat', fs_type): return 'fat' @@ -155,9 +155,11 @@ def check_ubconfig(config, fs_type): Return: Nothing. """ - if not config.buildconfig.get('config_cmd_%s' % fs_type, None): + if fs_type == 'exfat' and not config.buildconfig.get('config_fs_%s' % fs_type, None): + pytest.skip('.config feature "FS_%s" not enabled' % fs_type.upper()) + if fs_type != 'exfat' and not config.buildconfig.get('config_cmd_%s' % fs_type, None): pytest.skip('.config feature "CMD_%s" not enabled' % fs_type.upper()) - if fs_type == 'fs_generic': + if fs_type == 'fs_generic' or fs_type == 'exfat': return if not config.buildconfig.get('config_%s_write' % fs_type, None): pytest.skip('.config feature "%s_WRITE" not enabled' @@ -197,7 +199,7 @@ def fs_obj_basic(request, u_boot_config): """ fs_type = request.param fs_cmd_prefix = fstype_to_prefix(fs_type) - fs_cmd_write = 'save' if fs_type == 'fs_generic' else 'write' + fs_cmd_write = 'save' if fs_type == 'fs_generic' or fs_type == 'exfat' else 'write' fs_img = '' fs_ubtype = fstype_to_ubname(fs_type) @@ -309,7 +311,7 @@ def fs_obj_ext(request, u_boot_config): """ fs_type = request.param fs_cmd_prefix = fstype_to_prefix(fs_type) - fs_cmd_write = 'save' if fs_type == 'fs_generic' else 'write' + fs_cmd_write = 'save' if fs_type == 'fs_generic' or fs_type == 'exfat' else 'write' fs_img = '' fs_ubtype = fstype_to_ubname(fs_type) diff --git a/test/py/tests/test_fs/fstest_helpers.py b/test/py/tests/test_fs/fstest_helpers.py index c1447b4d43e..d25326ee993 100644 --- a/test/py/tests/test_fs/fstest_helpers.py +++ b/test/py/tests/test_fs/fstest_helpers.py @@ -9,6 +9,8 @@ def assert_fs_integrity(fs_type, fs_img): try: if fs_type == 'ext4': check_call('fsck.ext4 -n -f %s' % fs_img, shell=True) + elif fs_type == 'exfat': + check_call('fsck.exfat -n %s' % fs_img, shell=True) elif fs_type in ['fat12', 'fat16', 'fat32']: check_call('fsck.fat -n %s' % fs_img, shell=True) except CalledProcessError: diff --git a/test/py/tests/test_fs/test_ext.py b/test/py/tests/test_fs/test_ext.py index a75cd74dade..41f126e7876 100644 --- a/test/py/tests/test_fs/test_ext.py +++ b/test/py/tests/test_fs/test_ext.py @@ -345,11 +345,19 @@ class TestFsExt(object): '%s%s host 0:0 %x /%s 0' % (fs_cmd_prefix, fs_cmd_write, ADDR, MANGLE_FILE)]) assert('0 bytes written' in ''.join(output)) - # Test Case 12b - Read file system content - output = check_output('mdir -i %s' % fs_img, shell=True).decode() - # Test Case 12c - Check if short filename is not mangled - assert(str2fat(PLAIN_FILE) in ''.join(output)) - # Test Case 12d - Check if long filename is mangled - assert(str2fat(MANGLE_FILE) in ''.join(output)) + if fs_type == 'exfat': + # Test Case 12b - Read file system content + output = check_output('fattools ls %s' % fs_img, shell=True).decode() + # Test Case 12c - Check if short filename is not mangled + assert(PLAIN_FILE in ''.join(output)) + # Test Case 12d - Check if long filename is mangled + assert(MANGLE_FILE in ''.join(output)) + else: + # Test Case 12b - Read file system content + output = check_output('mdir -i %s' % fs_img, shell=True).decode() + # Test Case 12c - Check if short filename is not mangled + assert(str2fat(PLAIN_FILE) in ''.join(output)) + # Test Case 12d - Check if long filename is mangled + assert(str2fat(MANGLE_FILE) in ''.join(output)) assert_fs_integrity(fs_type, fs_img) diff --git a/tools/docker/Dockerfile b/tools/docker/Dockerfile index 569912303fc..80e9247aa60 100644 --- a/tools/docker/Dockerfile +++ b/tools/docker/Dockerfile @@ -74,6 +74,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ e2fsprogs \ efitools \ erofs-utils \ + exfatprogs \ expect \ fakeroot \ flex \ -- cgit v1.3.1 From 42ce9ed7ffef726640fa234966e8c36d39252422 Mon Sep 17 00:00:00 2001 From: Neha Malcom Francis Date: Mon, 17 Mar 2025 10:24:20 +0530 Subject: tools: binman: control.py: Delete template nodes after parsing Dynamically going through the subnode array and deleting leads to templates being skipped from deletion when templates are consecutive in the subnode list. Prevent this from happening by first parsing the DT and then deleting the nodes. Add a testcase as well for this cornercase. Signed-off-by: Neha Malcom Francis --- tools/binman/control.py | 6 +++- tools/binman/ftest.py | 7 +++++ tools/binman/test/346_remove_template.dts | 49 +++++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 1 deletion(-) create mode 100644 tools/binman/test/346_remove_template.dts (limited to 'tools') diff --git a/tools/binman/control.py b/tools/binman/control.py index e73c598298c..81f61e3e152 100644 --- a/tools/binman/control.py +++ b/tools/binman/control.py @@ -522,9 +522,13 @@ def _ProcessTemplates(parent): def _RemoveTemplates(parent): """Remove any templates in the binman description """ + del_nodes = [] for node in parent.subnodes: if node.name.startswith('template'): - node.Delete() + del_nodes.append(node) + + for node in del_nodes: + node.Delete() def PrepareImagesAndDtbs(dtb_fname, select_images, update_fdt, use_expanded, indir): """Prepare the images to be processed and select the device tree diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 1a92a99b511..948fcc02259 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -7990,5 +7990,12 @@ fdt fdtmap Extract the devicetree blob from the fdtmap """Test an image with an FIT with multiple FDT images using NAME""" self.CheckFitFdt('345_fit_fdt_name.dts', use_seq_num=False) + def testRemoveTemplate(self): + """Test whether template is removed""" + TestFunctional._MakeInputFile('my-blob.bin', b'blob') + TestFunctional._MakeInputFile('my-blob2.bin', b'other') + self._DoTestFile('346_remove_template.dts', + force_missing_bintools='openssl',) + if __name__ == "__main__": unittest.main() diff --git a/tools/binman/test/346_remove_template.dts b/tools/binman/test/346_remove_template.dts new file mode 100644 index 00000000000..e05229f3ebc --- /dev/null +++ b/tools/binman/test/346_remove_template.dts @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; +/ { + binman: binman { + multiple-images; + + template_1: template-1 { + section { + phandle1: my-blob.bin { + filename = "my-blob.bin"; + type = "blob-ext"; + }; + }; + }; + template_2: template-2 { + section { + ti-secure { + content = <&phandle2>; + keyfile = "key.pem"; + }; + phandle2: my-blob.bin { + filename = "my-blob.bin"; + type = "blob-ext"; + }; + }; + }; + template_3: template-3 { + section { + phandle3: my-blob.bin { + filename = "my-blob.bin"; + type = "blob-ext"; + }; + }; + }; + + file1 { + insert-template = <&template_1>; + }; + + file2 { + insert-template = <&template_2>; + }; + + file3 { + insert-template = <&template_3>; + }; + }; +}; -- cgit v1.3.1