From 139c464c2ac5027b200ef9b4a66024a9daa39969 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Fri, 10 Feb 2023 13:59:46 -0700 Subject: binman: Avoid requiring a home directory on startup This is needed to download tools, but we may not need to do this. At present binman fails to start if HOME is not set. Use the current directory as a default to avoid this. Signed-off-by: Simon Glass --- tools/binman/bintool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/binman/bintool.py b/tools/binman/bintool.py index 8fda13ff012..f460243e796 100644 --- a/tools/binman/bintool.py +++ b/tools/binman/bintool.py @@ -43,7 +43,7 @@ FETCH_NAMES = { # Status of tool fetching FETCHED, FAIL, PRESENT, STATUS_COUNT = range(4) -DOWNLOAD_DESTDIR = os.path.join(os.getenv('HOME'), 'bin') +DOWNLOAD_DESTDIR = os.path.expanduser('~/bin') class Bintool: """Tool which operates on binaries to help produce entry contents -- cgit v1.3.1 From 9dbb02b9d124e03a141de1244c8b4f4843d58840 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sun, 12 Feb 2023 17:11:15 -0700 Subject: binman: Support marking FMAP areas as preserved Add an entry flag called 'preserve' to indicate that an entry should be preserved by firmware updates. Propagate this to FMAP too. Signed-off-by: Simon Glass --- tools/binman/binman.rst | 8 ++++++++ tools/binman/entries.rst | 5 +++++ tools/binman/entry.py | 7 +++++++ tools/binman/etype/fmap.py | 15 +++++++++++++-- tools/binman/fmap_util.py | 3 +++ tools/binman/ftest.py | 2 +- tools/binman/test/067_fmap.dts | 1 + 7 files changed, 38 insertions(+), 3 deletions(-) (limited to 'tools') diff --git a/tools/binman/binman.rst b/tools/binman/binman.rst index 2bcb7d3886f..8af23fd0fab 100644 --- a/tools/binman/binman.rst +++ b/tools/binman/binman.rst @@ -838,6 +838,14 @@ offset-from-elf: is the symbol to lookup (relative to elf-base-sym) and is an offset to add to that value. +preserve: + Indicates that this entry should be preserved by any firmware updates. This + flag should be checked by the updater when it is deciding which entries to + update. This flag is normally attached to sections but can be attached to + a single entry in a section if the updater supports it. Not that binman + itself has no control over the updater's behaviour, so this is just a + signal. It is not enforced by binman. + Examples of the above options can be found in the tests. See the tools/binman/test directory. diff --git a/tools/binman/entries.rst b/tools/binman/entries.rst index 7a04a613992..19659247cf0 100644 --- a/tools/binman/entries.rst +++ b/tools/binman/entries.rst @@ -887,6 +887,11 @@ before its contents, so that it is possible to reconstruct the hierarchy from the FMAP by using the offset information. This convention does not seem to be documented, but is used in Chromium OS. +To mark an area as preserved, use the normal 'preserved' flag in the entry. +This will result in the corresponding FMAP area having the +FMAP_AREA_PRESERVE flag. This flag does not automatically propagate down to +child entries. + CBFS entries appear as a single entry, i.e. the sub-entries are ignored. diff --git a/tools/binman/entry.py b/tools/binman/entry.py index 5eacc5fa6c4..fd617e4f15f 100644 --- a/tools/binman/entry.py +++ b/tools/binman/entry.py @@ -100,6 +100,10 @@ class Entry(object): appear in the map optional (bool): True if this entry contains an optional external blob overlap (bool): True if this entry overlaps with others + preserve (bool): True if this entry should be preserved when updating + firmware. This means that it will not be changed by the update. + This is just a signal: enforcement of this is up to the updater. + This flag does not automatically propagate down to child entries. """ fake_dir = None @@ -148,6 +152,7 @@ class Entry(object): self.overlap = False self.elf_base_sym = None self.offset_from_elf = None + self.preserve = False @staticmethod def FindEntryClass(etype, expanded): @@ -310,6 +315,8 @@ class Entry(object): self.offset_from_elf = fdt_util.GetPhandleNameOffset(self._node, 'offset-from-elf') + self.preserve = fdt_util.GetBool(self._node, 'preserve') + def GetDefaultFilename(self): return None diff --git a/tools/binman/etype/fmap.py b/tools/binman/etype/fmap.py index 0c576202a48..b35450fec97 100644 --- a/tools/binman/etype/fmap.py +++ b/tools/binman/etype/fmap.py @@ -33,6 +33,11 @@ class Entry_fmap(Entry): from the FMAP by using the offset information. This convention does not seem to be documented, but is used in Chromium OS. + To mark an area as preserved, use the normal 'preserved' flag in the entry. + This will result in the corresponding FMAP area having the + FMAP_AREA_PRESERVE flag. This flag does not automatically propagate down to + child entries. + CBFS entries appear as a single entry, i.e. the sub-entries are ignored. """ def __init__(self, section, etype, node): @@ -48,6 +53,12 @@ class Entry_fmap(Entry): entries = entry.GetEntries() tout.debug("fmap: Add entry '%s' type '%s' (%s subentries)" % (entry.GetPath(), entry.etype, to_hex_size(entries))) + + # Collect any flag (separate lines to ensure code coverage) + flags = 0 + if entry.preserve: + flags = fmap_util.FMAP_AREA_PRESERVE + if entries and entry.etype != 'cbfs': # Create an area for the section, which encompasses all entries # within it @@ -59,7 +70,7 @@ class Entry_fmap(Entry): # Drop @ symbols in name name = entry.name.replace('@', '') areas.append( - fmap_util.FmapArea(pos, entry.size or 0, name, 0)) + fmap_util.FmapArea(pos, entry.size or 0, name, flags)) for subentry in entries.values(): _AddEntries(areas, subentry) else: @@ -67,7 +78,7 @@ class Entry_fmap(Entry): if pos is not None: pos -= entry.section.GetRootSkipAtStart() areas.append(fmap_util.FmapArea(pos or 0, entry.size or 0, - entry.name, 0)) + entry.name, flags)) entries = self.GetImage().GetEntries() areas = [] diff --git a/tools/binman/fmap_util.py b/tools/binman/fmap_util.py index 1ce63d1a832..82e0f74d50f 100644 --- a/tools/binman/fmap_util.py +++ b/tools/binman/fmap_util.py @@ -45,6 +45,9 @@ FMAP_AREA_NAMES = ( 'flags', ) +# Flags supported by areas (bits 2:0 are unused so not included here) +FMAP_AREA_PRESERVE = 1 << 3 # Preserved by any firmware updates + # These are the two data structures supported by flashrom, a header (which # appears once at the start) and an area (which is repeated until the end of # the list of areas) diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 062f54adb0e..df916ed602a 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -1702,7 +1702,7 @@ class TestFunctional(unittest.TestCase): self.assertEqual(b'SECTION0', fentry.name) self.assertEqual(0, fentry.offset) self.assertEqual(16, fentry.size) - self.assertEqual(0, fentry.flags) + self.assertEqual(fmap_util.FMAP_AREA_PRESERVE, fentry.flags) fentry = next(fiter) self.assertEqual(b'RO_U_BOOT', fentry.name) diff --git a/tools/binman/test/067_fmap.dts b/tools/binman/test/067_fmap.dts index 9c0e293ac83..24fa6351ec3 100644 --- a/tools/binman/test/067_fmap.dts +++ b/tools/binman/test/067_fmap.dts @@ -11,6 +11,7 @@ name-prefix = "ro-"; size = <0x10>; pad-byte = <0x21>; + preserve; u-boot { }; -- cgit v1.3.1 From cbe429bc979cd76c365f0f8442ea1eaba7b473ae Mon Sep 17 00:00:00 2001 From: Jonas Karlman Date: Sun, 19 Feb 2023 22:02:03 +0000 Subject: binman: Remove redundant SetAllowFakeBlob from blob-ext entry Entry_blob_ext contains an implementation of SetAllowFakeBlob that is identical to the one in the base Entry class, remove it. Signed-off-by: Jonas Karlman Reviewed-by: Simon Glass --- tools/binman/etype/blob_ext.py | 8 -------- 1 file changed, 8 deletions(-) (limited to 'tools') diff --git a/tools/binman/etype/blob_ext.py b/tools/binman/etype/blob_ext.py index fba6271de2b..d6b0ca17c3f 100644 --- a/tools/binman/etype/blob_ext.py +++ b/tools/binman/etype/blob_ext.py @@ -26,11 +26,3 @@ class Entry_blob_ext(Entry_blob): def __init__(self, section, etype, node): Entry_blob.__init__(self, section, etype, node) self.external = True - - def SetAllowFakeBlob(self, allow_fake): - """Set whether the entry allows to create a fake blob - - Args: - allow_fake_blob: True if allowed, False if not allowed - """ - self.allow_fake = allow_fake -- cgit v1.3.1 From dd4bdad4c1b17fcfc43e3fa56a2c5131fac01c2a Mon Sep 17 00:00:00 2001 From: Jonas Karlman Date: Sun, 19 Feb 2023 22:02:03 +0000 Subject: binman: Fix spelling of nodes in code comments Replace notes with nodes in code comments and docstrings. Signed-off-by: Jonas Karlman Reviewed-by: Simon Glass --- tools/binman/etype/fit.py | 2 +- tools/binman/etype/section.py | 2 +- tools/binman/state.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) (limited to 'tools') diff --git a/tools/binman/etype/fit.py b/tools/binman/etype/fit.py index cd2943533ce..822de798276 100644 --- a/tools/binman/etype/fit.py +++ b/tools/binman/etype/fit.py @@ -823,7 +823,7 @@ class Entry_fit(Entry_section): self.mkimage = self.AddBintool(btools, 'mkimage') def CheckMissing(self, missing_list): - # We must use our private entry list for this since generator notes + # We must use our private entry list for this since generator nodes # which are removed from self._entries will otherwise not show up as # missing for entry in self._priv_entries.values(): diff --git a/tools/binman/etype/section.py b/tools/binman/etype/section.py index 57b91ff726c..8bf5aa437d1 100644 --- a/tools/binman/etype/section.py +++ b/tools/binman/etype/section.py @@ -172,7 +172,7 @@ class Entry_section(Entry): def IsSpecialSubnode(self, node): """Check if a node is a special one used by the section itself - Some notes are used for hashing / signatures and do not add entries to + Some nodes are used for hashing / signatures and do not add entries to the actual section. Returns: diff --git a/tools/binman/state.py b/tools/binman/state.py index 56e5bf8bc10..33563199840 100644 --- a/tools/binman/state.py +++ b/tools/binman/state.py @@ -306,7 +306,7 @@ def GetUpdateNodes(node, for_repack=False): """Yield all the nodes that need to be updated in all device trees The property referenced by this node is added to any device trees which - have the given node. Due to removable of unwanted notes, SPL and TPL may + have the given node. Due to removable of unwanted nodes, SPL and TPL may not have this node. Args: -- cgit v1.3.1 From e389d445c78d454c70688dbf74917af341109959 Mon Sep 17 00:00:00 2001 From: Jonas Karlman Date: Sun, 19 Feb 2023 22:02:04 +0000 Subject: binman: Use correct argument name in docstrings Use correct argument name in docstrings. Signed-off-by: Jonas Karlman Reviewed-by: Simon Glass --- tools/binman/entry.py | 2 +- tools/binman/etype/blob.py | 2 +- tools/binman/etype/section.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'tools') diff --git a/tools/binman/entry.py b/tools/binman/entry.py index fd617e4f15f..11aa8e50d4a 100644 --- a/tools/binman/entry.py +++ b/tools/binman/entry.py @@ -1111,7 +1111,7 @@ features to produce new behaviours. If there are faked blobs, the entries are added to the list Args: - fake_blobs_list: List of Entry objects to be added to + faked_blobs_list: List of Entry objects to be added to """ # This is meaningless for anything other than blobs pass diff --git a/tools/binman/etype/blob.py b/tools/binman/etype/blob.py index c7ddcedffb8..a80741e3633 100644 --- a/tools/binman/etype/blob.py +++ b/tools/binman/etype/blob.py @@ -102,7 +102,7 @@ class Entry_blob(Entry): If there are faked blobs, the entries are added to the list Args: - fake_blobs_list: List of Entry objects to be added to + faked_blobs_list: List of Entry objects to be added to """ if self.faked: faked_blobs_list.append(self) diff --git a/tools/binman/etype/section.py b/tools/binman/etype/section.py index 8bf5aa437d1..d3926f791c7 100644 --- a/tools/binman/etype/section.py +++ b/tools/binman/etype/section.py @@ -885,7 +885,7 @@ class Entry_section(Entry): """Set whether a section allows to create a fake blob Args: - allow_fake_blob: True if allowed, False if not allowed + allow_fake: True if allowed, False if not allowed """ super().SetAllowFakeBlob(allow_fake) for entry in self._entries.values(): @@ -909,7 +909,7 @@ class Entry_section(Entry): If there are faked blobs, the entries are added to the list Args: - fake_blobs_list: List of Entry objects to be added to + faked_blobs_list: List of Entry objects to be added to """ for entry in self._entries.values(): entry.CheckFakedBlobs(faked_blobs_list) -- cgit v1.3.1 From 5a93c1574330b2d6146ab172f99e3f8e20a5402a Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 21 Feb 2023 12:40:26 -0700 Subject: buildman: Add a note about the out-env file This file holds the environment used when doing a build. Add a note about it. Signed-off-by: Simon Glass --- tools/buildman/buildman.rst | 2 ++ 1 file changed, 2 insertions(+) (limited to 'tools') diff --git a/tools/buildman/buildman.rst b/tools/buildman/buildman.rst index 2a83cb7e4f8..9a2d913c785 100644 --- a/tools/buildman/buildman.rst +++ b/tools/buildman/buildman.rst @@ -1108,6 +1108,8 @@ and 'brppt1_spi', removing a trailing semicolon. 'brppt1_nand' gained an a value for 'altbootcmd', but lost one for ' altbootcmd'. The -U option uses the u-boot.env files which are produced by a build. +Internally, buildman writes out an out-env file into the build directory for +later comparison. Building with clang -- cgit v1.3.1 From cd37d5bccf63e75af395dd5e3b5dd21abbd86881 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 21 Feb 2023 12:40:27 -0700 Subject: buildman: Write out the build command used It is sometimes useful to see the exact 'make' command used by buildman for a commit. Add an output file for this. Signed-off-by: Simon Glass --- tools/buildman/builderthread.py | 13 +++++++++++++ tools/buildman/buildman.rst | 8 ++++++++ tools/buildman/func_test.py | 13 +++++++++++++ 3 files changed, 34 insertions(+) (limited to 'tools') diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index 680efae02d7..7ba9a856dd5 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -273,14 +273,19 @@ class BuilderThread(threading.Thread): # If we need to reconfigure, do that now cfg_file = os.path.join(out_dir, '.config') + cmd_list = [] if do_config or adjust_cfg: config_out = '' if self.mrproper: result = self.Make(commit, brd, 'mrproper', cwd, 'mrproper', *args, env=env) config_out += result.combined + cmd_list.append([self.builder.gnu_make, 'mrproper', + *args]) result = self.Make(commit, brd, 'config', cwd, *(args + config_args), env=env) + cmd_list.append([self.builder.gnu_make] + args + + config_args) config_out += result.combined do_config = False # No need to configure next time if adjust_cfg: @@ -290,6 +295,7 @@ class BuilderThread(threading.Thread): args.append('cfg') result = self.Make(commit, brd, 'build', cwd, *args, env=env) + cmd_list.append([self.builder.gnu_make] + args) if (result.return_code == 2 and ('Some images are invalid' in result.stderr)): # This is handled later by the check for output in @@ -303,6 +309,7 @@ class BuilderThread(threading.Thread): result.stderr = result.stderr.replace(src_dir + '/', '') if self.builder.verbose_build: result.stdout = config_out + result.stdout + result.cmd_list = cmd_list else: result.return_code = 1 result.stderr = 'No tool chain for %s\n' % brd.arch @@ -378,6 +385,12 @@ class BuilderThread(threading.Thread): with open(os.path.join(build_dir, 'out-env'), 'wb') as fd: for var in sorted(env.keys()): fd.write(b'%s="%s"' % (var, env[var])) + + with open(os.path.join(build_dir, 'out-cmd'), 'w', + encoding='utf-8') as fd: + for cmd in result.cmd_list: + print(' '.join(cmd), file=fd) + lines = [] for fname in BASE_ELF_FILENAMES: cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] diff --git a/tools/buildman/buildman.rst b/tools/buildman/buildman.rst index 9a2d913c785..11c72141791 100644 --- a/tools/buildman/buildman.rst +++ b/tools/buildman/buildman.rst @@ -1300,6 +1300,14 @@ You should use 'buildman -nv ' instead of greoing the boards.cfg file, since it may be dropped altogether in future. +Checking the command +-------------------- + +Buildman writes out the toolchain information to a `toolchain` file within the +output directory. It also writes the commands used to build U-Boot in an +`out-cmd` file. You can check these if you suspect something strange is +happening. + TODO ---- diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index 559e4edf74b..799c609446e 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -723,3 +723,16 @@ Some images are invalid''' control.get_allow_missing(False, False, 2, True)) self.assertEqual(False, control.get_allow_missing(False, True, 2, True)) + + def testCmdFile(self): + """Test that the -cmd-out file is produced""" + self._RunControl('-o', self._output_dir) + board0_dir = os.path.join(self._output_dir, 'current', 'board0') + self.assertTrue(os.path.exists(os.path.join(board0_dir, 'done'))) + cmd_fname = os.path.join(board0_dir, 'out-cmd') + self.assertTrue(os.path.exists(cmd_fname)) + data = tools.read_file(cmd_fname) + lines = data.splitlines() + self.assertEqual(2, len(lines)) + self.assertRegex(lines[0], b'make O=/.*board0_defconfig') + self.assertRegex(lines[0], b'make O=/.*-s.*') -- cgit v1.3.1 From 93202d72d75ff2e04c14525bc8b585c5ed0d0740 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 21 Feb 2023 12:40:28 -0700 Subject: buildman: Support disabling LTO This cuts down build performance considerably and is not always needed, when checking for build errors, etc. Add a flag to disable it. Signed-off-by: Simon Glass --- tools/buildman/builder.py | 5 ++++- tools/buildman/builderthread.py | 2 ++ tools/buildman/buildman.rst | 14 ++++++++++++++ tools/buildman/cmdline.py | 2 ++ tools/buildman/control.py | 2 +- tools/buildman/func_test.py | 25 +++++++++++++++++++++---- 6 files changed, 44 insertions(+), 6 deletions(-) (limited to 'tools') diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index c2a69027f88..107086cc0e5 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -194,6 +194,7 @@ class Builder: work_in_output: Use the output directory as the work directory and don't write to a separate output directory. thread_exceptions: List of exceptions raised by thread jobs + no_lto (bool): True to set the NO_LTO flag when building Private members: _base_board_dict: Last-summarised Dict of boards @@ -253,7 +254,7 @@ class Builder: config_only=False, squash_config_y=False, warnings_as_errors=False, work_in_output=False, test_thread_exceptions=False, adjust_cfg=None, - allow_missing=False): + allow_missing=False, no_lto=False): """Create a new Builder object Args: @@ -292,6 +293,7 @@ class Builder: C=val to set the value of C (val must have quotes if C is a string Kconfig allow_missing: Run build with BINMAN_ALLOW_MISSING=1 + no_lto (bool): True to set the NO_LTO flag when building """ self.toolchains = toolchains @@ -331,6 +333,7 @@ class Builder: self.adjust_cfg = adjust_cfg self.allow_missing = allow_missing self._ide = False + self.no_lto = no_lto if not self.squash_config_y: self.config_filenames += EXTRA_CONFIG_FILENAMES diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index 7ba9a856dd5..dae3d4ab9ff 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -255,6 +255,8 @@ class BuilderThread(threading.Thread): args.append('KCFLAGS=-Werror') if self.builder.allow_missing: args.append('BINMAN_ALLOW_MISSING=1') + if self.builder.no_lto: + args.append('NO_LTO=1') config_args = ['%s_defconfig' % brd.target] config_out = '' args.extend(self.builder.toolchains.GetMakeArguments(brd)) diff --git a/tools/buildman/buildman.rst b/tools/buildman/buildman.rst index 11c72141791..800b83a89de 100644 --- a/tools/buildman/buildman.rst +++ b/tools/buildman/buildman.rst @@ -1123,6 +1123,20 @@ toolchain. For example: buildman -O clang-7 --board sandbox +Building without LTO +-------------------- + +Link-time optimisation (LTO) is designed to reduce code size by globally +optimising the U-Boot build. Unfortunately this can dramatically slow down +builds. This is particularly noticeable when running a lot of builds. + +Use the -L (--no-lto) flag to disable LTO. + +.. code-block:: bash + + buildman -L --board sandbox + + Doing a simple build -------------------- diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py index c485994e9fe..409013671be 100644 --- a/tools/buildman/cmdline.py +++ b/tools/buildman/cmdline.py @@ -71,6 +71,8 @@ def ParseArgs(): default=False, help="Don't convert y to 1 in configs") parser.add_option('-l', '--list-error-boards', action='store_true', default=False, help='Show a list of boards next to each error/warning') + parser.add_option('-L', '--no-lto', action='store_true', + default=False, help='Disable Link-time Optimisation (LTO) for builds') parser.add_option('--list-tool-chains', action='store_true', default=False, help='List available tool chains (use -v to see probing detail)') parser.add_option('-m', '--mrproper', action='store_true', diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 87e7d0e2012..13a462ae595 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -351,7 +351,7 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None, work_in_output=options.work_in_output, test_thread_exceptions=test_thread_exceptions, adjust_cfg=adjust_cfg, - allow_missing=allow_missing) + allow_missing=allow_missing, no_lto=options.no_lto) builder.force_config_on_failure = not options.quick if make_func: builder.do_make = make_func diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index 799c609446e..6d1557293f0 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -724,15 +724,32 @@ Some images are invalid''' self.assertEqual(False, control.get_allow_missing(False, True, 2, True)) - def testCmdFile(self): - """Test that the -cmd-out file is produced""" - self._RunControl('-o', self._output_dir) + def check_command(self, *extra_args): + """Run a command with the extra arguments and return the commands used + + Args: + extra_args (list of str): List of extra arguments + + Returns: + list of str: Lines returned in the out-cmd file + """ + self._RunControl('-o', self._output_dir, *extra_args) board0_dir = os.path.join(self._output_dir, 'current', 'board0') self.assertTrue(os.path.exists(os.path.join(board0_dir, 'done'))) cmd_fname = os.path.join(board0_dir, 'out-cmd') self.assertTrue(os.path.exists(cmd_fname)) data = tools.read_file(cmd_fname) - lines = data.splitlines() + return data.splitlines() + + def testCmdFile(self): + """Test that the -cmd-out file is produced""" + lines = self.check_command() self.assertEqual(2, len(lines)) self.assertRegex(lines[0], b'make O=/.*board0_defconfig') self.assertRegex(lines[0], b'make O=/.*-s.*') + + def testNoLto(self): + """Test that the --no-lto flag works""" + lines = self.check_command('-L') + self.assertIn(b'NO_LTO=1', lines[0]) + -- cgit v1.3.1 From bfb708ad9987ebddd2cd8f55bf4884e4f2305234 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Tue, 21 Feb 2023 12:40:29 -0700 Subject: buildman: Add a flag for reproducible builds This is quite a useful thing to use when building since it avoids small size changes between commits. Add a -r flag for it. Also undefine CONFIG_LOCALVERSION_AUTO since this appends the git hash to the version string, causing every build to be slightly different. Signed-off-by: Simon Glass --- doc/build/reproducible.rst | 2 ++ tools/buildman/builder.py | 4 +++- tools/buildman/builderthread.py | 2 ++ tools/buildman/buildman.rst | 7 ++++--- tools/buildman/cmdline.py | 2 ++ tools/buildman/control.py | 11 ++++++++++- tools/buildman/func_test.py | 40 +++++++++++++++++++++++++++++++++------- 7 files changed, 56 insertions(+), 12 deletions(-) (limited to 'tools') diff --git a/doc/build/reproducible.rst b/doc/build/reproducible.rst index 5423080633e..8b030f469d7 100644 --- a/doc/build/reproducible.rst +++ b/doc/build/reproducible.rst @@ -23,3 +23,5 @@ This date is shown when we launch U-Boot: ./u-boot -T U-Boot 2023.01 (Jan 01 2023 - 00:00:00 +0000) + +The same effect can be obtained with buildman using the `-r` flag. diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index 107086cc0e5..7b9be887e55 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -195,6 +195,7 @@ class Builder: don't write to a separate output directory. thread_exceptions: List of exceptions raised by thread jobs no_lto (bool): True to set the NO_LTO flag when building + reproducible_builds (bool): True to set SOURCE_DATE_EPOCH=0 for builds Private members: _base_board_dict: Last-summarised Dict of boards @@ -254,7 +255,7 @@ class Builder: config_only=False, squash_config_y=False, warnings_as_errors=False, work_in_output=False, test_thread_exceptions=False, adjust_cfg=None, - allow_missing=False, no_lto=False): + allow_missing=False, no_lto=False, reproducible_builds=False): """Create a new Builder object Args: @@ -334,6 +335,7 @@ class Builder: self.allow_missing = allow_missing self._ide = False self.no_lto = no_lto + self.reproducible_builds = reproducible_builds if not self.squash_config_y: self.config_filenames += EXTRA_CONFIG_FILENAMES diff --git a/tools/buildman/builderthread.py b/tools/buildman/builderthread.py index dae3d4ab9ff..8b88c68e5d2 100644 --- a/tools/buildman/builderthread.py +++ b/tools/buildman/builderthread.py @@ -257,6 +257,8 @@ class BuilderThread(threading.Thread): args.append('BINMAN_ALLOW_MISSING=1') if self.builder.no_lto: args.append('NO_LTO=1') + if self.builder.reproducible_builds: + args.append('SOURCE_DATE_EPOCH=0') config_args = ['%s_defconfig' % brd.target] config_out = '' args.extend(self.builder.toolchains.GetMakeArguments(brd)) diff --git a/tools/buildman/buildman.rst b/tools/buildman/buildman.rst index 800b83a89de..c8b0db3d8b9 100644 --- a/tools/buildman/buildman.rst +++ b/tools/buildman/buildman.rst @@ -1023,14 +1023,15 @@ U-Boot's build system embeds information such as a build timestamp into the final binary. This information varies each time U-Boot is built. This causes various files to be rebuilt even if no source changes are made, which in turn requires that the final U-Boot binary be re-linked. This unnecessary work can -be avoided by turning off the timestamp feature. This can be achieved by -setting the SOURCE_DATE_EPOCH environment variable to 0. +be avoided by turning off the timestamp feature. This can be achieved using +the `-r` flag, which enables reproducible builds by setting +`SOURCE_DATE_EPOCH=0` when building. Combining all of these options together yields the command-line shown below. This will provide the quickest possible feedback regarding the current content of the source tree, thus allowing rapid tested evolution of the code:: - SOURCE_DATE_EPOCH=0 ./tools/buildman/buildman -P tegra + ./tools/buildman/buildman -Pr tegra Checking configuration diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py index 409013671be..da7f1a99f6b 100644 --- a/tools/buildman/cmdline.py +++ b/tools/buildman/cmdline.py @@ -97,6 +97,8 @@ def ParseArgs(): default=False, help="Use full toolchain path in CROSS_COMPILE") parser.add_option('-P', '--per-board-out-dir', action='store_true', default=False, help="Use an O= (output) directory per board rather than per thread") + parser.add_option('-r', '--reproducible-builds', action='store_true', + help='Set SOURCE_DATE_EPOCH=0 to suuport a reproducible build') parser.add_option('-R', '--regen-board-list', action='store_true', help='Force regeneration of the list of boards, like the old boards.cfg file') parser.add_option('-s', '--summary', action='store_true', diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 13a462ae595..c3c53881998 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -338,6 +338,14 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None, shutil.rmtree(output_dir) adjust_cfg = cfgutil.convert_list_to_dict(options.adjust_cfg) + # Drop LOCALVERSION_AUTO since it changes the version string on every commit + if options.reproducible_builds: + # If these are mentioned, leave the local version alone + if 'LOCALVERSION' in adjust_cfg or 'LOCALVERSION_AUTO' in adjust_cfg: + print('Not dropping LOCALVERSION_AUTO for reproducible build') + else: + adjust_cfg['LOCALVERSION_AUTO'] = '~' + builder = Builder(toolchains, output_dir, options.git_dir, options.threads, options.jobs, gnu_make=gnu_make, checkout=True, show_unknown=options.show_unknown, step=options.step, @@ -351,7 +359,8 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None, work_in_output=options.work_in_output, test_thread_exceptions=test_thread_exceptions, adjust_cfg=adjust_cfg, - allow_missing=allow_missing, no_lto=options.no_lto) + allow_missing=allow_missing, no_lto=options.no_lto, + reproducible_builds=options.reproducible_builds) builder.force_config_on_failure = not options.quick if make_func: builder.do_make = make_func diff --git a/tools/buildman/func_test.py b/tools/buildman/func_test.py index 6d1557293f0..cf91c339134 100644 --- a/tools/buildman/func_test.py +++ b/tools/buildman/func_test.py @@ -415,17 +415,19 @@ class TestFunctional(unittest.TestCase): kwargs: Arguments to pass to command.run_pipe() """ self._make_calls += 1 + out_dir = '' + for arg in args: + if arg.startswith('O='): + out_dir = arg[2:] if stage == 'mrproper': return command.CommandResult(return_code=0) elif stage == 'config': + fname = os.path.join(cwd or '', out_dir, '.config') + tools.write_file(fname, b'CONFIG_SOMETHING=1') return command.CommandResult(return_code=0, combined='Test configuration complete') elif stage == 'build': stderr = '' - out_dir = '' - for arg in args: - if arg.startswith('O='): - out_dir = arg[2:] fname = os.path.join(cwd or '', out_dir, 'u-boot') tools.write_file(fname, b'U-Boot') @@ -739,17 +741,41 @@ Some images are invalid''' cmd_fname = os.path.join(board0_dir, 'out-cmd') self.assertTrue(os.path.exists(cmd_fname)) data = tools.read_file(cmd_fname) - return data.splitlines() + + config_fname = os.path.join(board0_dir, '.config') + self.assertTrue(os.path.exists(config_fname)) + cfg_data = tools.read_file(config_fname) + + return data.splitlines(), cfg_data def testCmdFile(self): """Test that the -cmd-out file is produced""" - lines = self.check_command() + lines = self.check_command()[0] self.assertEqual(2, len(lines)) self.assertRegex(lines[0], b'make O=/.*board0_defconfig') self.assertRegex(lines[0], b'make O=/.*-s.*') def testNoLto(self): """Test that the --no-lto flag works""" - lines = self.check_command('-L') + lines = self.check_command('-L')[0] self.assertIn(b'NO_LTO=1', lines[0]) + def testReproducible(self): + """Test that the -r flag works""" + lines, cfg_data = self.check_command('-r') + self.assertIn(b'SOURCE_DATE_EPOCH=0', lines[0]) + + # We should see CONFIG_LOCALVERSION_AUTO unset + self.assertEqual(b'''CONFIG_SOMETHING=1 +# CONFIG_LOCALVERSION_AUTO is not set +''', cfg_data) + + with test_util.capture_sys_output() as (stdout, stderr): + lines, cfg_data = self.check_command('-r', '-a', 'LOCALVERSION') + self.assertIn(b'SOURCE_DATE_EPOCH=0', lines[0]) + + # We should see CONFIG_LOCALVERSION_AUTO unset + self.assertEqual(b'''CONFIG_SOMETHING=1 +CONFIG_LOCALVERSION=y +''', cfg_data) + self.assertIn('Not dropping LOCALVERSION_AUTO', stdout.getvalue()) -- cgit v1.3.1 From 6569cb8e1f556988dcb10495d3eab8b44c652959 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Wed, 22 Feb 2023 12:14:45 -0700 Subject: binman: Correct an 'aot' typo Fix this typo. Signed-off-by: Simon Glass --- tools/binman/bintool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/binman/bintool.py b/tools/binman/bintool.py index f460243e796..bb229688556 100644 --- a/tools/binman/bintool.py +++ b/tools/binman/bintool.py @@ -389,7 +389,7 @@ class Bintool: @classmethod def apt_install(cls, package): - """Install a bintool using the 'aot' tool + """Install a bintool using the 'apt' tool This requires use of servo so may request a password -- cgit v1.3.1 From fbb0e480329e964598e79a7d67dbcdff19e7985d Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Wed, 22 Feb 2023 12:14:46 -0700 Subject: binman: Update bintools documentation This was not regenerated with recent changes. Update it. Signed-off-by: Simon Glass --- tools/binman/bintools.rst | 70 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) (limited to 'tools') diff --git a/tools/binman/bintools.rst b/tools/binman/bintools.rst index edb373ab59b..c30e7eb9ff5 100644 --- a/tools/binman/bintools.rst +++ b/tools/binman/bintools.rst @@ -10,6 +10,20 @@ binaries. It is fairly easy to create new bintools. Just add a new file to the +Bintool: bzip2: Compression/decompression using the bzip2 algorithm +------------------------------------------------------------------- + +This bintool supports running `bzip2` to compress and decompress data, as +used by binman. + +It is also possible to fetch the tool, which uses `apt` to install it. + +Documentation is available via:: + + man bzip2 + + + Bintool: cbfstool: Coreboot filesystem (CBFS) tool -------------------------------------------------- @@ -58,6 +72,20 @@ See `Chromium OS vboot documentation`_ for more information. +Bintool: gzip: Compression/decompression using the gzip algorithm +----------------------------------------------------------------- + +This bintool supports running `gzip` to compress and decompress data, as +used by binman. + +It is also possible to fetch the tool, which uses `apt` to install it. + +Documentation is available via:: + + man gzip + + + Bintool: ifwitool: Handles the 'ifwitool' tool ---------------------------------------------- @@ -101,6 +129,20 @@ Documentation is available via:: +Bintool: lzop: Compression/decompression using the lzop algorithm +----------------------------------------------------------------- + +This bintool supports running `lzop` to compress and decompress data, as +used by binman. + +It is also possible to fetch the tool, which uses `apt` to install it. + +Documentation is available via:: + + man lzop + + + Bintool: mkimage: Image generation for U-Boot --------------------------------------------- @@ -113,3 +155,31 @@ Support is provided for fetching this on Debian-like systems, using apt. +Bintool: xz: Compression/decompression using the xz algorithm +------------------------------------------------------------- + +This bintool supports running `xz` to compress and decompress data, as +used by binman. + +It is also possible to fetch the tool, which uses `apt` to install it. + +Documentation is available via:: + + man xz + + + +Bintool: zstd: Compression/decompression using the zstd algorithm +----------------------------------------------------------------- + +This bintool supports running `zstd` to compress and decompress data, as +used by binman. + +It is also possible to fetch the tool, which uses `apt` to install it. + +Documentation is available via:: + + man zstd + + + -- cgit v1.3.1 From 00f674db2dacfb6c62e274b5f87e13b5c97aee97 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Wed, 22 Feb 2023 12:14:47 -0700 Subject: binman: Move the tools directory into the Bintool class We want to be able to change this directory. Use a class member to hold the value, since changing a constant is not good. Signed-off-by: Simon Glass --- tools/binman/bintool.py | 7 ++++--- tools/binman/bintool_test.py | 4 ++-- 2 files changed, 6 insertions(+), 5 deletions(-) (limited to 'tools') diff --git a/tools/binman/bintool.py b/tools/binman/bintool.py index bb229688556..302161fcb47 100644 --- a/tools/binman/bintool.py +++ b/tools/binman/bintool.py @@ -43,8 +43,6 @@ FETCH_NAMES = { # Status of tool fetching FETCHED, FAIL, PRESENT, STATUS_COUNT = range(4) -DOWNLOAD_DESTDIR = os.path.expanduser('~/bin') - class Bintool: """Tool which operates on binaries to help produce entry contents @@ -53,6 +51,9 @@ class Bintool: # List of bintools to regard as missing missing_list = [] + # Directory to store tools + tooldir = os.path.join(os.getenv('HOME'), 'bin') + def __init__(self, name, desc, version_regex=None, version_args='-V'): self.name = name self.desc = desc @@ -208,7 +209,7 @@ class Bintool: return FAIL if result is not True: fname, tmpdir = result - dest = os.path.join(DOWNLOAD_DESTDIR, self.name) + dest = os.path.join(self.tooldir, self.name) print(f"- writing to '{dest}'") shutil.move(fname, dest) if tmpdir: diff --git a/tools/binman/bintool_test.py b/tools/binman/bintool_test.py index 7efb8391db2..57e866eff93 100644 --- a/tools/binman/bintool_test.py +++ b/tools/binman/bintool_test.py @@ -139,7 +139,7 @@ class TestBintool(unittest.TestCase): dest_fname = os.path.join(destdir, '_testing') self.seq = 0 - with unittest.mock.patch.object(bintool, 'DOWNLOAD_DESTDIR', destdir): + with unittest.mock.patch.object(bintool.Bintool, 'tooldir', destdir): with unittest.mock.patch.object(tools, 'download', side_effect=handle_download): with test_util.capture_sys_output() as (stdout, _): @@ -250,7 +250,7 @@ class TestBintool(unittest.TestCase): btest = Bintool.create('_testing') col = terminal.Color() self.fname = None - with unittest.mock.patch.object(bintool, 'DOWNLOAD_DESTDIR', + with unittest.mock.patch.object(bintool.Bintool, 'tooldir', self._indir): with unittest.mock.patch.object(tools, 'run', side_effect=fake_run): with test_util.capture_sys_output() as (stdout, _): -- cgit v1.3.1 From 932e40d0b52242454be9a7773bd2323e12358b92 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Wed, 22 Feb 2023 12:14:48 -0700 Subject: binman: Use a private directory for bintools At present binman writes tools into the ~/bin directory. This is convenient but some may be concerned about downloading unverified binaries and running them. Place then in a special ~/.binman-tools directory instead. Mention this in the documentation. Signed-off-by: Simon Glass Reviewed-by: Tom Rini --- tools/binman/binman.rst | 2 ++ tools/binman/bintool.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/binman/binman.rst b/tools/binman/binman.rst index 8af23fd0fab..9c2cd3c6d6c 100644 --- a/tools/binman/binman.rst +++ b/tools/binman/binman.rst @@ -1415,6 +1415,8 @@ You can also use `--fetch all` to fetch all tools or `--fetch ` to fetch a particular tool. Some tools are built from source code, in which case you will need to have at least the `build-essential` and `git` packages installed. +Tools are fetched into the `~/.binman-tools` directory. + Bintool Documentation ===================== diff --git a/tools/binman/bintool.py b/tools/binman/bintool.py index 302161fcb47..6ca3d886200 100644 --- a/tools/binman/bintool.py +++ b/tools/binman/bintool.py @@ -52,7 +52,7 @@ class Bintool: missing_list = [] # Directory to store tools - tooldir = os.path.join(os.getenv('HOME'), 'bin') + tooldir = os.path.join(os.getenv('HOME'), '.binman-tools') def __init__(self, name, desc, version_regex=None, version_args='-V'): self.name = name -- cgit v1.3.1 From fe7e9245c53e254526d3bbd6296d658596f41b48 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Wed, 22 Feb 2023 12:14:49 -0700 Subject: binman: Make the tooldir configurable Add a command-line argument for setting the tooldir, so that the default can be overridden. Add this directory to the toolpath automatically. Create the directory if it does not already exist. Put the default in the argument parser instead of the class, so that it is more obvious. Update a few tests that expect the utility name to be provided without any path (e.g. 'futility'), so they can accept a path, e.g. /path/to/futility Update the documentation and add a few tests. Improve the help for --toolpath while we are here. Signed-off-by: Simon Glass --- tools/binman/binman.rst | 19 +++++++++++++++---- tools/binman/bintool.py | 11 +++++++++-- tools/binman/bintool_test.py | 11 ++++++++--- tools/binman/cmdline.py | 6 +++++- tools/binman/control.py | 10 ++++++++-- tools/binman/ftest.py | 21 +++++++++++++++++++-- 6 files changed, 64 insertions(+), 14 deletions(-) (limited to 'tools') diff --git a/tools/binman/binman.rst b/tools/binman/binman.rst index 9c2cd3c6d6c..3ac29ee3568 100644 --- a/tools/binman/binman.rst +++ b/tools/binman/binman.rst @@ -1415,7 +1415,15 @@ You can also use `--fetch all` to fetch all tools or `--fetch ` to fetch a particular tool. Some tools are built from source code, in which case you will need to have at least the `build-essential` and `git` packages installed. -Tools are fetched into the `~/.binman-tools` directory. +Tools are fetched into the `~/.binman-tools` directory. This directory is +automatically added to the toolpath so there is no need to use `--toolpath` to +specify it. If you want to use these tools outside binman, you may want to +add this directory to your `PATH`. For example, if you use bash, add this to +the end of `.bashrc`:: + + PATH="$HOME/.binman-tools:$PATH" + +To select a custom directory, use the `--tooldir` option. Bintool Documentation ===================== @@ -1435,8 +1443,9 @@ Binman commands and arguments Usage:: - binman [-h] [-B BUILD_DIR] [-D] [-H] [--toolpath TOOLPATH] [-T THREADS] - [--test-section-timeout] [-v VERBOSITY] [-V] + binman [-h] [-B BUILD_DIR] [-D] [--tooldir TOOLDIR] [-H] + [--toolpath TOOLPATH] [-T THREADS] [--test-section-timeout] + [-v VERBOSITY] [-V] {build,bintool-docs,entry-docs,ls,extract,replace,test,tool} ... Binman provides the following commands: @@ -1461,11 +1470,13 @@ Options: -D, --debug Enabling debugging (provides a full traceback on error) +--tooldir TOOLDIR Set the directory to store tools + -H, --full-help Display the README file --toolpath TOOLPATH - Add a path to the directories containing tools + Add a path to the list of directories containing tools -T THREADS, --threads THREADS Number of threads to use (0=single-thread). Note that -T0 is useful for diff --git a/tools/binman/bintool.py b/tools/binman/bintool.py index 6ca3d886200..7674dfdf7bd 100644 --- a/tools/binman/bintool.py +++ b/tools/binman/bintool.py @@ -51,8 +51,9 @@ class Bintool: # List of bintools to regard as missing missing_list = [] - # Directory to store tools - tooldir = os.path.join(os.getenv('HOME'), '.binman-tools') + # Directory to store tools. Note that this set up by set_tool_dir() which + # must be called before this class is used. + tooldir = '' def __init__(self, name, desc, version_regex=None, version_args='-V'): self.name = name @@ -113,6 +114,11 @@ class Bintool: obj = cls(name) return obj + @classmethod + def set_tool_dir(cls, pathname): + """Set the path to use to store and find tools""" + cls.tooldir = pathname + def show(self): """Show a line of information about a bintool""" if self.is_present(): @@ -210,6 +216,7 @@ class Bintool: if result is not True: fname, tmpdir = result dest = os.path.join(self.tooldir, self.name) + os.makedirs(self.tooldir, exist_ok=True) print(f"- writing to '{dest}'") shutil.move(fname, dest) if tmpdir: diff --git a/tools/binman/bintool_test.py b/tools/binman/bintool_test.py index 57e866eff93..39e4fb13e92 100644 --- a/tools/binman/bintool_test.py +++ b/tools/binman/bintool_test.py @@ -134,8 +134,10 @@ class TestBintool(unittest.TestCase): dirname = os.path.join(self._indir, 'download_dir') os.mkdir(dirname) fname = os.path.join(dirname, 'downloaded') + + # Rely on bintool to create this directory destdir = os.path.join(self._indir, 'dest_dir') - os.mkdir(destdir) + dest_fname = os.path.join(destdir, '_testing') self.seq = 0 @@ -344,8 +346,11 @@ class TestBintool(unittest.TestCase): def test_failed_command(self): """Check that running a command that does not exist returns None""" - btool = Bintool.create('_testing') - result = btool.run_cmd_result('fred') + destdir = os.path.join(self._indir, 'dest_dir') + os.mkdir(destdir) + with unittest.mock.patch.object(bintool.Bintool, 'tooldir', destdir): + btool = Bintool.create('_testing') + result = btool.run_cmd_result('fred') self.assertIsNone(result) diff --git a/tools/binman/cmdline.py b/tools/binman/cmdline.py index 986d6f1a315..4eed3073dce 100644 --- a/tools/binman/cmdline.py +++ b/tools/binman/cmdline.py @@ -7,6 +7,7 @@ import argparse from argparse import ArgumentParser +import os from binman import state def make_extract_parser(subparsers): @@ -80,8 +81,11 @@ controlled by a description in the board device tree.''' help='Enabling debugging (provides a full traceback on error)') parser.add_argument('-H', '--full-help', action='store_true', default=False, help='Display the README file') + parser.add_argument('--tooldir', type=str, + default=os.path.join(os.getenv('HOME'), '.binman-tools'), + help='Set the directory to store tools') parser.add_argument('--toolpath', type=str, action='append', - help='Add a path to the directories containing tools') + help='Add a path to the list of directories containing tools') parser.add_argument('-T', '--threads', type=int, default=None, help='Number of threads to use (0=single-thread)') parser.add_argument('--test-section-timeout', action='store_true', diff --git a/tools/binman/control.py b/tools/binman/control.py index e64740094f6..abe01b76773 100644 --- a/tools/binman/control.py +++ b/tools/binman/control.py @@ -650,6 +650,14 @@ def Binman(args): from binman.image import Image from binman import state + tool_paths = [] + if args.toolpath: + tool_paths += args.toolpath + if args.tooldir: + tool_paths.append(args.tooldir) + tools.set_tool_paths(tool_paths or None) + bintool.Bintool.set_tool_dir(args.tooldir) + if args.cmd in ['ls', 'extract', 'replace', 'tool']: try: tout.init(args.verbosity) @@ -667,7 +675,6 @@ def Binman(args): allow_resize=not args.fix_size, write_map=args.map) if args.cmd == 'tool': - tools.set_tool_paths(args.toolpath) if args.list: bintool.Bintool.list_all() elif args.fetch: @@ -719,7 +726,6 @@ def Binman(args): try: tools.set_input_dirs(args.indir) tools.prepare_output_dir(args.outdir, args.preserve) - tools.set_tool_paths(args.toolpath) state.SetEntryArgs(args.entry_arg) state.SetThreads(args.threads) diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index df916ed602a..0b3bca90c8c 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -1750,7 +1750,7 @@ class TestFunctional(unittest.TestCase): def _HandleGbbCommand(self, pipe_list): """Fake calls to the futility utility""" - if pipe_list[0][0] == 'futility': + if 'futility' in pipe_list[0][0]: fname = pipe_list[0][-1] # Append our GBB data to the file, which will happen every time the # futility command is called. @@ -1812,7 +1812,7 @@ class TestFunctional(unittest.TestCase): self._hash_data is False, it writes VBLOCK_DATA, else it writes a hash of the input data (here, 'input.vblock'). """ - if pipe_list[0][0] == 'futility': + if 'futility' in pipe_list[0][0]: fname = pipe_list[0][3] with open(fname, 'wb') as fd: if self._hash_data: @@ -6386,6 +6386,23 @@ fdt fdtmap Extract the devicetree blob from the fdtmap self.assertEqual(['u-boot', 'atf-2'], fdt_util.GetStringList(node, 'loadables')) + def testTooldir(self): + """Test that we can specify the tooldir""" + with test_util.capture_sys_output() as (stdout, stderr): + self.assertEqual(0, self._DoBinman('--tooldir', 'fred', + 'tool', '-l')) + self.assertEqual('fred', bintool.Bintool.tooldir) + + # Check that the toolpath is updated correctly + self.assertEqual(['fred'], tools.tool_search_paths) + + # Try with a few toolpaths; the tooldir should be at the end + with test_util.capture_sys_output() as (stdout, stderr): + self.assertEqual(0, self._DoBinman( + '--toolpath', 'mary', '--toolpath', 'anna', '--tooldir', 'fred', + 'tool', '-l')) + self.assertEqual(['mary', 'anna', 'fred'], tools.tool_search_paths) + if __name__ == "__main__": unittest.main() -- cgit v1.3.1 From 7c4027af48fc7178d222ffa9bf5c76853390aa0e Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 23 Feb 2023 18:18:01 -0700 Subject: binman: Avoid unwanted output in testFitFirmwareLoadables() This prints a message about the missing tee-os generated by the test. This is confusing, so suppress it. Signed-off-by: Simon Glass --- tools/binman/ftest.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'tools') diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 0b3bca90c8c..d5cae38526e 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -6353,10 +6353,11 @@ fdt fdtmap Extract the devicetree blob from the fdtmap 'tee-os-path': 'missing.bin', } test_subdir = os.path.join(self._indir, TEST_FDT_SUBDIR) - data = self._DoReadFileDtb( - '276_fit_firmware_loadables.dts', - entry_args=entry_args, - extra_indirs=[test_subdir])[0] + with test_util.capture_sys_output() as (stdout, stderr): + data = self._DoReadFileDtb( + '276_fit_firmware_loadables.dts', + entry_args=entry_args, + extra_indirs=[test_subdir])[0] dtb = fdt.Fdt.FromData(data) dtb.Scan() -- cgit v1.3.1 From 6811fea7051d1111c29a83561f967c22d426b0ad Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 23 Feb 2023 18:18:02 -0700 Subject: Revert "patman: test_util: Print test stdout/stderr within test summaries" Unfortunately this adds a new feature to concurrencytest and it has not made it upstream to the project[1]. Drop it for now so we can use the upstream module. Once it is applied we can bring this functionality back. [1] https://github.com/cgoldberg/concurrencytest This reverts commit ebcaafcded40da8ae6cb4234c2ba9901c7bee644. Signed-off-by: Simon Glass --- tools/concurrencytest/concurrencytest.py | 83 ++------------------------------ tools/patman/test_util.py | 33 +------------ 2 files changed, 4 insertions(+), 112 deletions(-) (limited to 'tools') diff --git a/tools/concurrencytest/concurrencytest.py b/tools/concurrencytest/concurrencytest.py index 1c4f03f37e5..5e88b94f415 100644 --- a/tools/concurrencytest/concurrencytest.py +++ b/tools/concurrencytest/concurrencytest.py @@ -31,7 +31,6 @@ from subunit import ProtocolTestCase, TestProtocolClient from subunit.test_results import AutoTimingTestResultDecorator from testtools import ConcurrentTestSuite, iterate_tests -from testtools.content import TracebackContent, text_content _all__ = [ @@ -44,81 +43,11 @@ _all__ = [ CPU_COUNT = cpu_count() -class BufferingTestProtocolClient(TestProtocolClient): - """A TestProtocolClient which can buffer the test outputs - - This class captures the stdout and stderr output streams of the - tests as it runs them, and includes the output texts in the subunit - stream as additional details. - - Args: - stream: A file-like object to write a subunit stream to - buffer (bool): True to capture test stdout/stderr outputs and - include them in the test details - """ - def __init__(self, stream, buffer=True): - super().__init__(stream) - self.buffer = buffer - - def _addOutcome(self, outcome, test, error=None, details=None, - error_permitted=True): - """Report a test outcome to the subunit stream - - The parent class uses this function as a common implementation - for various methods that report successes, errors, failures, etc. - - This version automatically upgrades the error tracebacks to the - new 'details' format by wrapping them in a Content object, so - that we can include the captured test output in the test result - details. - - Args: - outcome: A string describing the outcome - used as the - event name in the subunit stream. - test: The test case whose outcome is to be reported - error: Standard unittest positional argument form - an - exc_info tuple. - details: New Testing-in-python drafted API; a dict from - string to subunit.Content objects. - error_permitted: If True then one and only one of error or - details must be supplied. If False then error must not - be supplied and details is still optional. - """ - if details is None: - details = {} - - # Parent will raise an exception if error_permitted is False but - # error is not None. We want that exception in that case, so - # don't touch error when error_permitted is explicitly False. - if error_permitted and error is not None: - # Parent class prefers error over details - details['traceback'] = TracebackContent(error, test) - error_permitted = False - error = None - - if self.buffer: - stdout = sys.stdout.getvalue() - if stdout: - details['stdout'] = text_content(stdout) - - stderr = sys.stderr.getvalue() - if stderr: - details['stderr'] = text_content(stderr) - - return super()._addOutcome(outcome, test, error=error, - details=details, error_permitted=error_permitted) - - -def fork_for_tests(concurrency_num=CPU_COUNT, buffer=False): +def fork_for_tests(concurrency_num=CPU_COUNT): """Implementation of `make_tests` used to construct `ConcurrentTestSuite`. :param concurrency_num: number of processes to use. """ - if buffer: - test_protocol_client_class = BufferingTestProtocolClient - else: - test_protocol_client_class = TestProtocolClient - def do_fork(suite): """Take suite and start up multiple runners by forking (Unix only). @@ -147,7 +76,7 @@ def fork_for_tests(concurrency_num=CPU_COUNT, buffer=False): # child actually gets keystrokes for pdb etc). sys.stdin.close() subunit_result = AutoTimingTestResultDecorator( - test_protocol_client_class(stream) + TestProtocolClient(stream) ) process_suite.run(subunit_result) except: @@ -164,13 +93,7 @@ def fork_for_tests(concurrency_num=CPU_COUNT, buffer=False): else: os.close(c2pwrite) stream = os.fdopen(c2pread, 'rb') - # If we don't pass the second argument here, it defaults - # to sys.stdout.buffer down the line. But if we don't - # pass it *now*, it may be resolved after sys.stdout is - # replaced with a StringIO (to capture tests' outputs) - # which doesn't have a buffer attribute and can end up - # occasionally causing a 'broken-runner' error. - test = ProtocolTestCase(stream, sys.stdout.buffer) + test = ProtocolTestCase(stream) result.append(test) return result return do_fork diff --git a/tools/patman/test_util.py b/tools/patman/test_util.py index 0f6d1aa902d..4ee58f9fbb9 100644 --- a/tools/patman/test_util.py +++ b/tools/patman/test_util.py @@ -15,7 +15,6 @@ from patman import command from io import StringIO -buffer_outputs = True use_concurrent = True try: from concurrencytest.concurrencytest import ConcurrentTestSuite @@ -120,7 +119,6 @@ class FullTextTestResult(unittest.TextTestResult): 0: Print nothing 1: Print a dot per test 2: Print test names - 3: Print test names, and buffered outputs for failing tests """ def __init__(self, stream, descriptions, verbosity): self.verbosity = verbosity @@ -140,39 +138,12 @@ class FullTextTestResult(unittest.TextTestResult): self.printErrorList('XFAIL', self.expectedFailures) self.printErrorList('XPASS', unexpected_successes) - def addError(self, test, err): - """Called when an error has occurred.""" - super().addError(test, err) - self._mirrorOutput &= self.verbosity >= 3 - - def addFailure(self, test, err): - """Called when a test has failed.""" - super().addFailure(test, err) - self._mirrorOutput &= self.verbosity >= 3 - - def addSubTest(self, test, subtest, err): - """Called at the end of a subtest.""" - super().addSubTest(test, subtest, err) - self._mirrorOutput &= self.verbosity >= 3 - - def addSuccess(self, test): - """Called when a test has completed successfully""" - super().addSuccess(test) - # Don't print stdout/stderr for successful tests - self._mirrorOutput = False - def addSkip(self, test, reason): """Called when a test is skipped.""" # Add empty line to keep spacing consistent with other results if not reason.endswith('\n'): reason += '\n' super().addSkip(test, reason) - self._mirrorOutput &= self.verbosity >= 3 - - def addExpectedFailure(self, test, err): - """Called when an expected failure/error occurred.""" - super().addExpectedFailure(test, err) - self._mirrorOutput &= self.verbosity >= 3 def run_test_suites(toolname, debug, verbosity, test_preserve_dirs, processes, @@ -208,14 +179,12 @@ def run_test_suites(toolname, debug, verbosity, test_preserve_dirs, processes, runner = unittest.TextTestRunner( stream=sys.stdout, verbosity=(1 if verbosity is None else verbosity), - buffer=False if test_name else buffer_outputs, resultclass=FullTextTestResult, ) if use_concurrent and processes != 1: suite = ConcurrentTestSuite(suite, - fork_for_tests(processes or multiprocessing.cpu_count(), - buffer=False if test_name else buffer_outputs)) + fork_for_tests(processes or multiprocessing.cpu_count())) for module in class_and_module_list: if isinstance(module, str) and (not test_name or test_name == module): -- cgit v1.3.1 From 00290d6a5bdf41dc610d89d763fcb48936285600 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 23 Feb 2023 18:18:03 -0700 Subject: Remove concurrencytest While our version is better, it is tricky to use it when we are trying to package things with pip. Drop it. Somewhat reduced functionality is provided by the upstream version[1], along with a rather annoying message each time it is used[2] [3]. [1] pip install concurrencytest [2] https://github.com/cgoldberg/concurrencytest/issues/12 [3] https://github.com/cgoldberg/concurrencytest/pull/14 Signed-off-by: Simon Glass --- tools/concurrencytest/.gitignore | 1 - tools/concurrencytest/README.md | 74 ---------------- tools/concurrencytest/__init__.py | 0 tools/concurrencytest/concurrencytest.py | 144 ------------------------------- tools/patman/test_util.py | 4 +- 5 files changed, 2 insertions(+), 221 deletions(-) delete mode 100644 tools/concurrencytest/.gitignore delete mode 100644 tools/concurrencytest/README.md delete mode 100644 tools/concurrencytest/__init__.py delete mode 100644 tools/concurrencytest/concurrencytest.py (limited to 'tools') diff --git a/tools/concurrencytest/.gitignore b/tools/concurrencytest/.gitignore deleted file mode 100644 index 0d20b6487c6..00000000000 --- a/tools/concurrencytest/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*.pyc diff --git a/tools/concurrencytest/README.md b/tools/concurrencytest/README.md deleted file mode 100644 index 2d7fe75df53..00000000000 --- a/tools/concurrencytest/README.md +++ /dev/null @@ -1,74 +0,0 @@ -concurrencytest -=============== - -![testing goats](https://raw.github.com/cgoldberg/concurrencytest/master/testing-goats.png "testing goats") - -Python testtools extension for running unittest suites concurrently. - ----- - -Install from PyPI: -``` -pip install concurrencytest -``` - ----- - -Requires: - -* [testtools](https://pypi.python.org/pypi/testtools) : `pip install testtools` -* [python-subunit](https://pypi.python.org/pypi/python-subunit) : `pip install python-subunit` - ----- - -Example: - -```python -import time -import unittest - -from concurrencytest import ConcurrentTestSuite, fork_for_tests - - -class SampleTestCase(unittest.TestCase): - """Dummy tests that sleep for demo.""" - - def test_me_1(self): - time.sleep(0.5) - - def test_me_2(self): - time.sleep(0.5) - - def test_me_3(self): - time.sleep(0.5) - - def test_me_4(self): - time.sleep(0.5) - - -# Load tests from SampleTestCase defined above -suite = unittest.TestLoader().loadTestsFromTestCase(SampleTestCase) -runner = unittest.TextTestRunner() - -# Run tests sequentially -runner.run(suite) - -# Run same tests across 4 processes -suite = unittest.TestLoader().loadTestsFromTestCase(SampleTestCase) -concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests(4)) -runner.run(concurrent_suite) -``` -Output: - -``` -.... ----------------------------------------------------------------------- -Ran 4 tests in 2.003s - -OK -.... ----------------------------------------------------------------------- -Ran 4 tests in 0.504s - -OK -``` diff --git a/tools/concurrencytest/__init__.py b/tools/concurrencytest/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/tools/concurrencytest/concurrencytest.py b/tools/concurrencytest/concurrencytest.py deleted file mode 100644 index 5e88b94f415..00000000000 --- a/tools/concurrencytest/concurrencytest.py +++ /dev/null @@ -1,144 +0,0 @@ -#!/usr/bin/env python -# SPDX-License-Identifier: GPL-2.0+ -# -# Modified by: Corey Goldberg, 2013 -# -# Original code from: -# Bazaar (bzrlib.tests.__init__.py, v2.6, copied Jun 01 2013) -# Copyright (C) 2005-2011 Canonical Ltd - -"""Python testtools extension for running unittest suites concurrently. - -The `testtools` project provides a ConcurrentTestSuite class, but does -not provide a `make_tests` implementation needed to use it. - -This allows you to parallelize a test run across a configurable number -of worker processes. While this can speed up CPU-bound test runs, it is -mainly useful for IO-bound tests that spend most of their time waiting for -data to arrive from someplace else and can benefit from cocncurrency. - -Unix only. -""" - -import os -import sys -import traceback -import unittest -from itertools import cycle -from multiprocessing import cpu_count - -from subunit import ProtocolTestCase, TestProtocolClient -from subunit.test_results import AutoTimingTestResultDecorator - -from testtools import ConcurrentTestSuite, iterate_tests - - -_all__ = [ - 'ConcurrentTestSuite', - 'fork_for_tests', - 'partition_tests', -] - - -CPU_COUNT = cpu_count() - - -def fork_for_tests(concurrency_num=CPU_COUNT): - """Implementation of `make_tests` used to construct `ConcurrentTestSuite`. - - :param concurrency_num: number of processes to use. - """ - def do_fork(suite): - """Take suite and start up multiple runners by forking (Unix only). - - :param suite: TestSuite object. - - :return: An iterable of TestCase-like objects which can each have - run(result) called on them to feed tests to result. - """ - result = [] - test_blocks = partition_tests(suite, concurrency_num) - # Clear the tests from the original suite so it doesn't keep them alive - suite._tests[:] = [] - for process_tests in test_blocks: - process_suite = unittest.TestSuite(process_tests) - # Also clear each split list so new suite has only reference - process_tests[:] = [] - c2pread, c2pwrite = os.pipe() - pid = os.fork() - if pid == 0: - try: - stream = os.fdopen(c2pwrite, 'wb') - os.close(c2pread) - # Leave stderr and stdout open so we can see test noise - # Close stdin so that the child goes away if it decides to - # read from stdin (otherwise its a roulette to see what - # child actually gets keystrokes for pdb etc). - sys.stdin.close() - subunit_result = AutoTimingTestResultDecorator( - TestProtocolClient(stream) - ) - process_suite.run(subunit_result) - except: - # Try and report traceback on stream, but exit with error - # even if stream couldn't be created or something else - # goes wrong. The traceback is formatted to a string and - # written in one go to avoid interleaving lines from - # multiple failing children. - try: - stream.write(traceback.format_exc()) - finally: - os._exit(1) - os._exit(0) - else: - os.close(c2pwrite) - stream = os.fdopen(c2pread, 'rb') - test = ProtocolTestCase(stream) - result.append(test) - return result - return do_fork - - -def partition_tests(suite, count): - """Partition suite into count lists of tests.""" - # This just assigns tests in a round-robin fashion. On one hand this - # splits up blocks of related tests that might run faster if they shared - # resources, but on the other it avoids assigning blocks of slow tests to - # just one partition. So the slowest partition shouldn't be much slower - # than the fastest. - partitions = [list() for _ in range(count)] - tests = iterate_tests(suite) - for partition, test in zip(cycle(partitions), tests): - partition.append(test) - return partitions - - -if __name__ == '__main__': - import time - - class SampleTestCase(unittest.TestCase): - """Dummy tests that sleep for demo.""" - - def test_me_1(self): - time.sleep(0.5) - - def test_me_2(self): - time.sleep(0.5) - - def test_me_3(self): - time.sleep(0.5) - - def test_me_4(self): - time.sleep(0.5) - - # Load tests from SampleTestCase defined above - suite = unittest.TestLoader().loadTestsFromTestCase(SampleTestCase) - runner = unittest.TextTestRunner() - - # Run tests sequentially - runner.run(suite) - - # Run same tests across 4 processes - suite = unittest.TestLoader().loadTestsFromTestCase(SampleTestCase) - concurrent_suite = ConcurrentTestSuite(suite, fork_for_tests(4)) - runner.run(concurrent_suite) diff --git a/tools/patman/test_util.py b/tools/patman/test_util.py index 4ee58f9fbb9..9e0811b61a2 100644 --- a/tools/patman/test_util.py +++ b/tools/patman/test_util.py @@ -17,8 +17,8 @@ from io import StringIO use_concurrent = True try: - from concurrencytest.concurrencytest import ConcurrentTestSuite - from concurrencytest.concurrencytest import fork_for_tests + from concurrencytest import ConcurrentTestSuite + from concurrencytest import fork_for_tests except: use_concurrent = False -- cgit v1.3.1 From 4583c00236efd4ee768ff874f92526c229891a05 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 23 Feb 2023 18:18:04 -0700 Subject: patman: Move library functions into a library directory The patman directory has a number of modules which are used by other tools in U-Boot. This makes it hard to package the tools using pypi since the common files must be copied along with the tool that uses them. To address this, move these files into a new u_boot_pylib library. This can be packaged separately and listed as a dependency of each tool. Signed-off-by: Simon Glass --- scripts/event_dump.py | 2 +- test/run | 1 + tools/binman/bintool.py | 8 +- tools/binman/bintool_test.py | 8 +- tools/binman/btool/lz4.py | 2 +- tools/binman/btool/lzma_alone.py | 2 +- tools/binman/cbfs_util.py | 4 +- tools/binman/cbfs_util_test.py | 4 +- tools/binman/control.py | 6 +- tools/binman/elf.py | 6 +- tools/binman/elf_test.py | 8 +- tools/binman/entry.py | 6 +- tools/binman/entry_test.py | 2 +- tools/binman/etype/_testing.py | 2 +- tools/binman/etype/atf_fip.py | 2 +- tools/binman/etype/blob.py | 4 +- tools/binman/etype/blob_ext.py | 4 +- tools/binman/etype/blob_ext_list.py | 4 +- tools/binman/etype/fdtmap.py | 4 +- tools/binman/etype/files.py | 2 +- tools/binman/etype/fill.py | 2 +- tools/binman/etype/fit.py | 2 +- tools/binman/etype/fmap.py | 6 +- tools/binman/etype/gbb.py | 4 +- tools/binman/etype/intel_ifwi.py | 2 +- tools/binman/etype/mkimage.py | 2 +- tools/binman/etype/null.py | 2 +- tools/binman/etype/pre_load.py | 2 +- tools/binman/etype/section.py | 6 +- tools/binman/etype/text.py | 2 +- tools/binman/etype/u_boot_dtb_with_ucode.py | 2 +- tools/binman/etype/u_boot_elf.py | 2 +- tools/binman/etype/u_boot_env.py | 2 +- tools/binman/etype/u_boot_spl_bss_pad.py | 2 +- tools/binman/etype/u_boot_spl_expanded.py | 2 +- tools/binman/etype/u_boot_tpl_bss_pad.py | 2 +- tools/binman/etype/u_boot_tpl_expanded.py | 2 +- tools/binman/etype/u_boot_tpl_with_ucode_ptr.py | 4 +- tools/binman/etype/u_boot_ucode.py | 2 +- tools/binman/etype/u_boot_vpl_bss_pad.py | 2 +- tools/binman/etype/u_boot_vpl_expanded.py | 2 +- tools/binman/etype/u_boot_with_ucode_ptr.py | 4 +- tools/binman/etype/vblock.py | 2 +- tools/binman/fdt_test.py | 2 +- tools/binman/fip_util.py | 4 +- tools/binman/fip_util_test.py | 4 +- tools/binman/fmap_util.py | 2 +- tools/binman/ftest.py | 8 +- tools/binman/image.py | 4 +- tools/binman/image_test.py | 2 +- tools/binman/main.py | 7 +- tools/binman/state.py | 4 +- tools/buildman/builder.py | 6 +- tools/buildman/builderthread.py | 2 +- tools/buildman/cfgutil.py | 2 +- tools/buildman/control.py | 8 +- tools/buildman/func_test.py | 8 +- tools/buildman/main.py | 4 +- tools/buildman/test.py | 8 +- tools/buildman/toolchain.py | 6 +- tools/dtoc/fdt.py | 2 +- tools/dtoc/fdt_util.py | 4 +- tools/dtoc/main.py | 5 +- tools/dtoc/test_dtoc.py | 4 +- tools/dtoc/test_fdt.py | 7 +- tools/dtoc/test_src_scan.py | 4 +- tools/patman/__init__.py | 7 +- tools/patman/__main__.py | 8 +- tools/patman/checkpatch.py | 4 +- tools/patman/command.py | 139 ------ tools/patman/control.py | 2 +- tools/patman/cros_subprocess.py | 401 ---------------- tools/patman/func_test.py | 6 +- tools/patman/get_maintainer.py | 2 +- tools/patman/gitutil.py | 4 +- tools/patman/patchstream.py | 2 +- tools/patman/series.py | 4 +- tools/patman/status.py | 4 +- tools/patman/terminal.py | 270 ----------- tools/patman/test_settings.py | 2 +- tools/patman/test_util.py | 216 --------- tools/patman/tools.py | 596 ------------------------ tools/patman/tout.py | 179 ------- tools/rmboard.py | 2 +- tools/u_boot_pylib/__init__.py | 4 + tools/u_boot_pylib/__main__.py | 23 + tools/u_boot_pylib/command.py | 139 ++++++ tools/u_boot_pylib/cros_subprocess.py | 401 ++++++++++++++++ tools/u_boot_pylib/terminal.py | 270 +++++++++++ tools/u_boot_pylib/test_util.py | 216 +++++++++ tools/u_boot_pylib/tools.py | 596 ++++++++++++++++++++++++ tools/u_boot_pylib/tout.py | 179 +++++++ tools/u_boot_pylib/u_boot_pylib | 1 + 93 files changed, 1978 insertions(+), 1947 deletions(-) delete mode 100644 tools/patman/command.py delete mode 100644 tools/patman/cros_subprocess.py delete mode 100644 tools/patman/terminal.py delete mode 100644 tools/patman/test_util.py delete mode 100644 tools/patman/tools.py delete mode 100644 tools/patman/tout.py create mode 100644 tools/u_boot_pylib/__init__.py create mode 100755 tools/u_boot_pylib/__main__.py create mode 100644 tools/u_boot_pylib/command.py create mode 100644 tools/u_boot_pylib/cros_subprocess.py create mode 100644 tools/u_boot_pylib/terminal.py create mode 100644 tools/u_boot_pylib/test_util.py create mode 100644 tools/u_boot_pylib/tools.py create mode 100644 tools/u_boot_pylib/tout.py create mode 120000 tools/u_boot_pylib/u_boot_pylib (limited to 'tools') diff --git a/scripts/event_dump.py b/scripts/event_dump.py index d87823f3749..0117457526e 100755 --- a/scripts/event_dump.py +++ b/scripts/event_dump.py @@ -15,7 +15,7 @@ src_path = os.path.dirname(our_path) sys.path.insert(1, os.path.join(our_path, '../tools')) from binman import elf -from patman import tools +from u_boot_pylib import tools # A typical symbol looks like this: # _u_boot_list_2_evspy_info_2_EVT_MISC_INIT_F_3_sandbox_misc_init_f diff --git a/test/run b/test/run index c4ab046ce8f..93b556f6cff 100755 --- a/test/run +++ b/test/run @@ -76,6 +76,7 @@ TOOLS_DIR=build-sandbox_spl/tools run_test "binman" ./tools/binman/binman --toolpath ${TOOLS_DIR} test run_test "patman" ./tools/patman/patman test +run_test "u_boot_pylib" ./tools/u_boot_pylib/u_boot_pylib run_test "buildman" ./tools/buildman/buildman -t ${skip} run_test "fdt" ./tools/dtoc/test_fdt -t diff --git a/tools/binman/bintool.py b/tools/binman/bintool.py index 7674dfdf7bd..81629683df6 100644 --- a/tools/binman/bintool.py +++ b/tools/binman/bintool.py @@ -18,10 +18,10 @@ import shutil import tempfile import urllib.error -from patman import command -from patman import terminal -from patman import tools -from patman import tout +from u_boot_pylib import command +from u_boot_pylib import terminal +from u_boot_pylib import tools +from u_boot_pylib import tout BINMAN_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/tools/binman/bintool_test.py b/tools/binman/bintool_test.py index 39e4fb13e92..f9b16d4c73b 100644 --- a/tools/binman/bintool_test.py +++ b/tools/binman/bintool_test.py @@ -16,10 +16,10 @@ import urllib.error from binman import bintool from binman.bintool import Bintool -from patman import command -from patman import terminal -from patman import test_util -from patman import tools +from u_boot_pylib import command +from u_boot_pylib import terminal +from u_boot_pylib import test_util +from u_boot_pylib import tools # pylint: disable=R0904 class TestBintool(unittest.TestCase): diff --git a/tools/binman/btool/lz4.py b/tools/binman/btool/lz4.py index dc9e37921a6..fd520d13a56 100644 --- a/tools/binman/btool/lz4.py +++ b/tools/binman/btool/lz4.py @@ -60,7 +60,7 @@ import re import tempfile from binman import bintool -from patman import tools +from u_boot_pylib import tools # pylint: disable=C0103 class Bintoollz4(bintool.Bintool): diff --git a/tools/binman/btool/lzma_alone.py b/tools/binman/btool/lzma_alone.py index 52a960fd2fa..1fda2f68c7b 100644 --- a/tools/binman/btool/lzma_alone.py +++ b/tools/binman/btool/lzma_alone.py @@ -37,7 +37,7 @@ import re import tempfile from binman import bintool -from patman import tools +from u_boot_pylib import tools # pylint: disable=C0103 class Bintoollzma_alone(bintool.Bintool): diff --git a/tools/binman/cbfs_util.py b/tools/binman/cbfs_util.py index 7bd3d897981..fc56b40b753 100644 --- a/tools/binman/cbfs_util.py +++ b/tools/binman/cbfs_util.py @@ -22,8 +22,8 @@ import sys from binman import bintool from binman import elf -from patman import command -from patman import tools +from u_boot_pylib import command +from u_boot_pylib import tools # Set to True to enable printing output while working DEBUG = False diff --git a/tools/binman/cbfs_util_test.py b/tools/binman/cbfs_util_test.py index e0f792fd344..ee951d10cf3 100755 --- a/tools/binman/cbfs_util_test.py +++ b/tools/binman/cbfs_util_test.py @@ -20,8 +20,8 @@ from binman import bintool from binman import cbfs_util from binman.cbfs_util import CbfsWriter from binman import elf -from patman import test_util -from patman import tools +from u_boot_pylib import test_util +from u_boot_pylib import tools U_BOOT_DATA = b'1234' U_BOOT_DTB_DATA = b'udtb' diff --git a/tools/binman/control.py b/tools/binman/control.py index abe01b76773..9b183eda736 100644 --- a/tools/binman/control.py +++ b/tools/binman/control.py @@ -12,14 +12,14 @@ import pkg_resources import re import sys -from patman import tools from binman import bintool from binman import cbfs_util -from patman import command from binman import elf from binman import entry -from patman import tout +from u_boot_pylib import command +from u_boot_pylib import tools +from u_boot_pylib import tout # These are imported if needed since they import libfdt state = None diff --git a/tools/binman/elf.py b/tools/binman/elf.py index 3cc8a384495..5816284c32a 100644 --- a/tools/binman/elf.py +++ b/tools/binman/elf.py @@ -13,9 +13,9 @@ import shutil import struct import tempfile -from patman import command -from patman import tools -from patman import tout +from u_boot_pylib import command +from u_boot_pylib import tools +from u_boot_pylib import tout ELF_TOOLS = True try: diff --git a/tools/binman/elf_test.py b/tools/binman/elf_test.py index 8cb55ebb815..c98083961b5 100644 --- a/tools/binman/elf_test.py +++ b/tools/binman/elf_test.py @@ -12,10 +12,10 @@ import tempfile import unittest from binman import elf -from patman import command -from patman import test_util -from patman import tools -from patman import tout +from u_boot_pylib import command +from u_boot_pylib import test_util +from u_boot_pylib import tools +from u_boot_pylib import tout binman_dir = os.path.dirname(os.path.realpath(sys.argv[0])) diff --git a/tools/binman/entry.py b/tools/binman/entry.py index 11aa8e50d4a..f732d40c37c 100644 --- a/tools/binman/entry.py +++ b/tools/binman/entry.py @@ -14,9 +14,9 @@ import time from binman import bintool from binman import elf from dtoc import fdt_util -from patman import tools -from patman.tools import to_hex, to_hex_size -from patman import tout +from u_boot_pylib import tools +from u_boot_pylib.tools import to_hex, to_hex_size +from u_boot_pylib import tout modules = {} diff --git a/tools/binman/entry_test.py b/tools/binman/entry_test.py index a6fbf62731f..ac6582cf86a 100644 --- a/tools/binman/entry_test.py +++ b/tools/binman/entry_test.py @@ -14,7 +14,7 @@ from binman import entry from binman.etype.blob import Entry_blob from dtoc import fdt from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class TestEntry(unittest.TestCase): def setUp(self): diff --git a/tools/binman/etype/_testing.py b/tools/binman/etype/_testing.py index 1c1efb21a44..e092d98ce15 100644 --- a/tools/binman/etype/_testing.py +++ b/tools/binman/etype/_testing.py @@ -9,7 +9,7 @@ from collections import OrderedDict from binman.entry import Entry, EntryArg from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry__testing(Entry): diff --git a/tools/binman/etype/atf_fip.py b/tools/binman/etype/atf_fip.py index 6ecd95b71f9..d5b862040b4 100644 --- a/tools/binman/etype/atf_fip.py +++ b/tools/binman/etype/atf_fip.py @@ -11,7 +11,7 @@ from binman.entry import Entry from binman.etype.section import Entry_section from binman.fip_util import FIP_TYPES, FipReader, FipWriter, UUID_LEN from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_atf_fip(Entry_section): """ARM Trusted Firmware's Firmware Image Package (FIP) diff --git a/tools/binman/etype/blob.py b/tools/binman/etype/blob.py index a80741e3633..064fae50365 100644 --- a/tools/binman/etype/blob.py +++ b/tools/binman/etype/blob.py @@ -8,8 +8,8 @@ from binman.entry import Entry from binman import state from dtoc import fdt_util -from patman import tools -from patman import tout +from u_boot_pylib import tools +from u_boot_pylib import tout class Entry_blob(Entry): """Arbitrary binary blob diff --git a/tools/binman/etype/blob_ext.py b/tools/binman/etype/blob_ext.py index d6b0ca17c3f..ca265307380 100644 --- a/tools/binman/etype/blob_ext.py +++ b/tools/binman/etype/blob_ext.py @@ -9,8 +9,8 @@ import os from binman.etype.blob import Entry_blob from dtoc import fdt_util -from patman import tools -from patman import tout +from u_boot_pylib import tools +from u_boot_pylib import tout class Entry_blob_ext(Entry_blob): """Externally built binary blob diff --git a/tools/binman/etype/blob_ext_list.py b/tools/binman/etype/blob_ext_list.py index f00202e9ebc..1bfcf6733a7 100644 --- a/tools/binman/etype/blob_ext_list.py +++ b/tools/binman/etype/blob_ext_list.py @@ -9,8 +9,8 @@ import os from binman.etype.blob import Entry_blob from dtoc import fdt_util -from patman import tools -from patman import tout +from u_boot_pylib import tools +from u_boot_pylib import tout class Entry_blob_ext_list(Entry_blob): """List of externally built binary blobs diff --git a/tools/binman/etype/fdtmap.py b/tools/binman/etype/fdtmap.py index 33c9d039a91..f1f6217940f 100644 --- a/tools/binman/etype/fdtmap.py +++ b/tools/binman/etype/fdtmap.py @@ -9,8 +9,8 @@ image. """ from binman.entry import Entry -from patman import tools -from patman import tout +from u_boot_pylib import tools +from u_boot_pylib import tout FDTMAP_MAGIC = b'_FDTMAP_' FDTMAP_HDR_LEN = 16 diff --git a/tools/binman/etype/files.py b/tools/binman/etype/files.py index 2081bc727b9..c8757eafab1 100644 --- a/tools/binman/etype/files.py +++ b/tools/binman/etype/files.py @@ -11,7 +11,7 @@ import os from binman.etype.section import Entry_section from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools # This is imported if needed state = None diff --git a/tools/binman/etype/fill.py b/tools/binman/etype/fill.py index c91d0152a8a..7c93d4e2689 100644 --- a/tools/binman/etype/fill.py +++ b/tools/binman/etype/fill.py @@ -5,7 +5,7 @@ from binman.entry import Entry from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_fill(Entry): """An entry which is filled to a particular byte value diff --git a/tools/binman/etype/fit.py b/tools/binman/etype/fit.py index 822de798276..9def4433618 100644 --- a/tools/binman/etype/fit.py +++ b/tools/binman/etype/fit.py @@ -12,7 +12,7 @@ from binman.etype.section import Entry_section from binman import elf from dtoc import fdt_util from dtoc.fdt import Fdt -from patman import tools +from u_boot_pylib import tools # Supported operations, with the fit,operation property OP_GEN_FDT_NODES, OP_SPLIT_ELF = range(2) diff --git a/tools/binman/etype/fmap.py b/tools/binman/etype/fmap.py index b35450fec97..3669d91a0bc 100644 --- a/tools/binman/etype/fmap.py +++ b/tools/binman/etype/fmap.py @@ -7,9 +7,9 @@ from binman.entry import Entry from binman import fmap_util -from patman import tools -from patman.tools import to_hex_size -from patman import tout +from u_boot_pylib import tools +from u_boot_pylib.tools import to_hex_size +from u_boot_pylib import tout class Entry_fmap(Entry): diff --git a/tools/binman/etype/gbb.py b/tools/binman/etype/gbb.py index ba2a362bb59..cca18af6e2f 100644 --- a/tools/binman/etype/gbb.py +++ b/tools/binman/etype/gbb.py @@ -8,11 +8,11 @@ from collections import OrderedDict -from patman import command +from u_boot_pylib import command from binman.entry import Entry, EntryArg from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools # Build GBB flags. # (src/platform/vboot_reference/firmware/include/gbb_header.h) diff --git a/tools/binman/etype/intel_ifwi.py b/tools/binman/etype/intel_ifwi.py index 04fad401eee..6513b97c3e5 100644 --- a/tools/binman/etype/intel_ifwi.py +++ b/tools/binman/etype/intel_ifwi.py @@ -10,7 +10,7 @@ from collections import OrderedDict from binman.entry import Entry from binman.etype.blob_ext import Entry_blob_ext from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_intel_ifwi(Entry_blob_ext): """Intel Integrated Firmware Image (IFWI) file diff --git a/tools/binman/etype/mkimage.py b/tools/binman/etype/mkimage.py index cb264c3cad0..27a0c4bd7c6 100644 --- a/tools/binman/etype/mkimage.py +++ b/tools/binman/etype/mkimage.py @@ -9,7 +9,7 @@ from collections import OrderedDict from binman.entry import Entry from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_mkimage(Entry): """Binary produced by mkimage diff --git a/tools/binman/etype/null.py b/tools/binman/etype/null.py index c10d4824472..263fb5244df 100644 --- a/tools/binman/etype/null.py +++ b/tools/binman/etype/null.py @@ -5,7 +5,7 @@ from binman.entry import Entry from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_null(Entry): """An entry which has no contents of its own diff --git a/tools/binman/etype/pre_load.py b/tools/binman/etype/pre_load.py index b6222811592..bd3545bffc0 100644 --- a/tools/binman/etype/pre_load.py +++ b/tools/binman/etype/pre_load.py @@ -8,7 +8,7 @@ import os import struct from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools from binman.entry import Entry from binman.etype.collection import Entry_collection diff --git a/tools/binman/etype/section.py b/tools/binman/etype/section.py index d3926f791c7..eb733672855 100644 --- a/tools/binman/etype/section.py +++ b/tools/binman/etype/section.py @@ -16,9 +16,9 @@ import sys from binman.entry import Entry from binman import state from dtoc import fdt_util -from patman import tools -from patman import tout -from patman.tools import to_hex_size +from u_boot_pylib import tools +from u_boot_pylib import tout +from u_boot_pylib.tools import to_hex_size class Entry_section(Entry): diff --git a/tools/binman/etype/text.py b/tools/binman/etype/text.py index c55e0233b1e..e4deb4abacc 100644 --- a/tools/binman/etype/text.py +++ b/tools/binman/etype/text.py @@ -7,7 +7,7 @@ from collections import OrderedDict from binman.entry import Entry, EntryArg from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_text(Entry): diff --git a/tools/binman/etype/u_boot_dtb_with_ucode.py b/tools/binman/etype/u_boot_dtb_with_ucode.py index 047d310cdf4..f7225cecc16 100644 --- a/tools/binman/etype/u_boot_dtb_with_ucode.py +++ b/tools/binman/etype/u_boot_dtb_with_ucode.py @@ -7,7 +7,7 @@ from binman.entry import Entry from binman.etype.blob_dtb import Entry_blob_dtb -from patman import tools +from u_boot_pylib import tools # This is imported if needed state = None diff --git a/tools/binman/etype/u_boot_elf.py b/tools/binman/etype/u_boot_elf.py index 3ec774f38ad..f4d86aa176a 100644 --- a/tools/binman/etype/u_boot_elf.py +++ b/tools/binman/etype/u_boot_elf.py @@ -9,7 +9,7 @@ from binman.entry import Entry from binman.etype.blob import Entry_blob from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_u_boot_elf(Entry_blob): """U-Boot ELF image diff --git a/tools/binman/etype/u_boot_env.py b/tools/binman/etype/u_boot_env.py index c38340b256e..c027e93d42c 100644 --- a/tools/binman/etype/u_boot_env.py +++ b/tools/binman/etype/u_boot_env.py @@ -8,7 +8,7 @@ import zlib from binman.etype.blob import Entry_blob from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_u_boot_env(Entry_blob): """An entry which contains a U-Boot environment diff --git a/tools/binman/etype/u_boot_spl_bss_pad.py b/tools/binman/etype/u_boot_spl_bss_pad.py index 680d1983056..1ffeb3911fd 100644 --- a/tools/binman/etype/u_boot_spl_bss_pad.py +++ b/tools/binman/etype/u_boot_spl_bss_pad.py @@ -10,7 +10,7 @@ from binman import elf from binman.entry import Entry from binman.etype.blob import Entry_blob -from patman import tools +from u_boot_pylib import tools class Entry_u_boot_spl_bss_pad(Entry_blob): """U-Boot SPL binary padded with a BSS region diff --git a/tools/binman/etype/u_boot_spl_expanded.py b/tools/binman/etype/u_boot_spl_expanded.py index 319f6708fe6..fcd0dd19ac4 100644 --- a/tools/binman/etype/u_boot_spl_expanded.py +++ b/tools/binman/etype/u_boot_spl_expanded.py @@ -5,7 +5,7 @@ # Entry-type module for expanded U-Boot SPL binary # -from patman import tout +from u_boot_pylib import tout from binman import state from binman.etype.blob_phase import Entry_blob_phase diff --git a/tools/binman/etype/u_boot_tpl_bss_pad.py b/tools/binman/etype/u_boot_tpl_bss_pad.py index 47f4b23f357..29c6a954129 100644 --- a/tools/binman/etype/u_boot_tpl_bss_pad.py +++ b/tools/binman/etype/u_boot_tpl_bss_pad.py @@ -10,7 +10,7 @@ from binman import elf from binman.entry import Entry from binman.etype.blob import Entry_blob -from patman import tools +from u_boot_pylib import tools class Entry_u_boot_tpl_bss_pad(Entry_blob): """U-Boot TPL binary padded with a BSS region diff --git a/tools/binman/etype/u_boot_tpl_expanded.py b/tools/binman/etype/u_boot_tpl_expanded.py index 55fde3c8e66..58db4f37556 100644 --- a/tools/binman/etype/u_boot_tpl_expanded.py +++ b/tools/binman/etype/u_boot_tpl_expanded.py @@ -5,7 +5,7 @@ # Entry-type module for expanded U-Boot TPL binary # -from patman import tout +from u_boot_pylib import tout from binman import state from binman.etype.blob_phase import Entry_blob_phase diff --git a/tools/binman/etype/u_boot_tpl_with_ucode_ptr.py b/tools/binman/etype/u_boot_tpl_with_ucode_ptr.py index c7f3f9dedb5..86f9578b714 100644 --- a/tools/binman/etype/u_boot_tpl_with_ucode_ptr.py +++ b/tools/binman/etype/u_boot_tpl_with_ucode_ptr.py @@ -7,11 +7,11 @@ import struct -from patman import command from binman.entry import Entry from binman.etype.blob import Entry_blob from binman.etype.u_boot_with_ucode_ptr import Entry_u_boot_with_ucode_ptr -from patman import tools +from u_boot_pylib import command +from u_boot_pylib import tools class Entry_u_boot_tpl_with_ucode_ptr(Entry_u_boot_with_ucode_ptr): """U-Boot TPL with embedded microcode pointer diff --git a/tools/binman/etype/u_boot_ucode.py b/tools/binman/etype/u_boot_ucode.py index 6945411cf90..97ed7d7eb14 100644 --- a/tools/binman/etype/u_boot_ucode.py +++ b/tools/binman/etype/u_boot_ucode.py @@ -7,7 +7,7 @@ from binman.entry import Entry from binman.etype.blob import Entry_blob -from patman import tools +from u_boot_pylib import tools class Entry_u_boot_ucode(Entry_blob): """U-Boot microcode block diff --git a/tools/binman/etype/u_boot_vpl_bss_pad.py b/tools/binman/etype/u_boot_vpl_bss_pad.py index b2ce2a31352..bba38ccf9e9 100644 --- a/tools/binman/etype/u_boot_vpl_bss_pad.py +++ b/tools/binman/etype/u_boot_vpl_bss_pad.py @@ -10,7 +10,7 @@ from binman import elf from binman.entry import Entry from binman.etype.blob import Entry_blob -from patman import tools +from u_boot_pylib import tools class Entry_u_boot_vpl_bss_pad(Entry_blob): """U-Boot VPL binary padded with a BSS region diff --git a/tools/binman/etype/u_boot_vpl_expanded.py b/tools/binman/etype/u_boot_vpl_expanded.py index 92c64f0a65e..deff5a3f8c2 100644 --- a/tools/binman/etype/u_boot_vpl_expanded.py +++ b/tools/binman/etype/u_boot_vpl_expanded.py @@ -5,7 +5,7 @@ # Entry-type module for expanded U-Boot VPL binary # -from patman import tout +from u_boot_pylib import tout from binman import state from binman.etype.blob_phase import Entry_blob_phase diff --git a/tools/binman/etype/u_boot_with_ucode_ptr.py b/tools/binman/etype/u_boot_with_ucode_ptr.py index e275698cebe..41731fd0e13 100644 --- a/tools/binman/etype/u_boot_with_ucode_ptr.py +++ b/tools/binman/etype/u_boot_with_ucode_ptr.py @@ -11,8 +11,8 @@ from binman import elf from binman.entry import Entry from binman.etype.blob import Entry_blob from dtoc import fdt_util -from patman import tools -from patman import command +from u_boot_pylib import tools +from u_boot_pylib import command class Entry_u_boot_with_ucode_ptr(Entry_blob): """U-Boot with embedded microcode pointer diff --git a/tools/binman/etype/vblock.py b/tools/binman/etype/vblock.py index 04cb7228aa0..4adb9a4e9bf 100644 --- a/tools/binman/etype/vblock.py +++ b/tools/binman/etype/vblock.py @@ -13,7 +13,7 @@ from binman.entry import EntryArg from binman.etype.collection import Entry_collection from dtoc import fdt_util -from patman import tools +from u_boot_pylib import tools class Entry_vblock(Entry_collection): """An entry which contains a Chromium OS verified boot block diff --git a/tools/binman/fdt_test.py b/tools/binman/fdt_test.py index 94347b1a1e2..7ef87295463 100644 --- a/tools/binman/fdt_test.py +++ b/tools/binman/fdt_test.py @@ -12,7 +12,7 @@ import unittest from dtoc import fdt from dtoc import fdt_util from dtoc.fdt import FdtScan -from patman import tools +from u_boot_pylib import tools class TestFdt(unittest.TestCase): @classmethod diff --git a/tools/binman/fip_util.py b/tools/binman/fip_util.py index 95eee32bc00..b5caab2d37a 100755 --- a/tools/binman/fip_util.py +++ b/tools/binman/fip_util.py @@ -37,8 +37,8 @@ OUR_PATH = os.path.dirname(OUR_FILE) sys.path.insert(2, os.path.join(OUR_PATH, '..')) # pylint: disable=C0413 -from patman import command -from patman import tools +from u_boot_pylib import command +from u_boot_pylib import tools # The TOC header, at the start of the FIP HEADER_FORMAT = ' -# Licensed to PSF under a Contributor Agreement. -# See http://www.python.org/2.4/license for licensing details. - -"""Subprocess execution - -This module holds a subclass of subprocess.Popen with our own required -features, mainly that we get access to the subprocess output while it -is running rather than just at the end. This makes it easier to show -progress information and filter output in real time. -""" - -import errno -import os -import pty -import select -import subprocess -import sys -import unittest - - -# Import these here so the caller does not need to import subprocess also. -PIPE = subprocess.PIPE -STDOUT = subprocess.STDOUT -PIPE_PTY = -3 # Pipe output through a pty -stay_alive = True - - -class Popen(subprocess.Popen): - """Like subprocess.Popen with ptys and incremental output - - This class deals with running a child process and filtering its output on - both stdout and stderr while it is running. We do this so we can monitor - progress, and possibly relay the output to the user if requested. - - The class is similar to subprocess.Popen, the equivalent is something like: - - Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - - But this class has many fewer features, and two enhancement: - - 1. Rather than getting the output data only at the end, this class sends it - to a provided operation as it arrives. - 2. We use pseudo terminals so that the child will hopefully flush its output - to us as soon as it is produced, rather than waiting for the end of a - line. - - Use communicate_filter() to handle output from the subprocess. - - """ - - def __init__(self, args, stdin=None, stdout=PIPE_PTY, stderr=PIPE_PTY, - shell=False, cwd=None, env=None, **kwargs): - """Cut-down constructor - - Args: - args: Program and arguments for subprocess to execute. - stdin: See subprocess.Popen() - stdout: See subprocess.Popen(), except that we support the sentinel - value of cros_subprocess.PIPE_PTY. - stderr: See subprocess.Popen(), except that we support the sentinel - value of cros_subprocess.PIPE_PTY. - shell: See subprocess.Popen() - cwd: Working directory to change to for subprocess, or None if none. - env: Environment to use for this subprocess, or None to inherit parent. - kwargs: No other arguments are supported at the moment. Passing other - arguments will cause a ValueError to be raised. - """ - stdout_pty = None - stderr_pty = None - - if stdout == PIPE_PTY: - stdout_pty = pty.openpty() - stdout = os.fdopen(stdout_pty[1]) - if stderr == PIPE_PTY: - stderr_pty = pty.openpty() - stderr = os.fdopen(stderr_pty[1]) - - super(Popen, self).__init__(args, stdin=stdin, - stdout=stdout, stderr=stderr, shell=shell, cwd=cwd, env=env, - **kwargs) - - # If we're on a PTY, we passed the slave half of the PTY to the subprocess. - # We want to use the master half on our end from now on. Setting this here - # does make some assumptions about the implementation of subprocess, but - # those assumptions are pretty minor. - - # Note that if stderr is STDOUT, then self.stderr will be set to None by - # this constructor. - if stdout_pty is not None: - self.stdout = os.fdopen(stdout_pty[0]) - if stderr_pty is not None: - self.stderr = os.fdopen(stderr_pty[0]) - - # Insist that unit tests exist for other arguments we don't support. - if kwargs: - raise ValueError("Unit tests do not test extra args - please add tests") - - def convert_data(self, data): - """Convert stdout/stderr data to the correct format for output - - Args: - data: Data to convert, or None for '' - - Returns: - Converted data, as bytes - """ - if data is None: - return b'' - return data - - def communicate_filter(self, output, input_buf=''): - """Interact with process: Read data from stdout and stderr. - - This method runs until end-of-file is reached, then waits for the - subprocess to terminate. - - The output function is sent all output from the subprocess and must be - defined like this: - - def output([self,] stream, data) - Args: - stream: the stream the output was received on, which will be - sys.stdout or sys.stderr. - data: a string containing the data - - Returns: - True to terminate the process - - Note: The data read is buffered in memory, so do not use this - method if the data size is large or unlimited. - - Args: - output: Function to call with each fragment of output. - - Returns: - A tuple (stdout, stderr, combined) which is the data received on - stdout, stderr and the combined data (interleaved stdout and stderr). - - Note that the interleaved output will only be sensible if you have - set both stdout and stderr to PIPE or PIPE_PTY. Even then it depends on - the timing of the output in the subprocess. If a subprocess flips - between stdout and stderr quickly in succession, by the time we come to - read the output from each we may see several lines in each, and will read - all the stdout lines, then all the stderr lines. So the interleaving - may not be correct. In this case you might want to pass - stderr=cros_subprocess.STDOUT to the constructor. - - This feature is still useful for subprocesses where stderr is - rarely used and indicates an error. - - Note also that if you set stderr to STDOUT, then stderr will be empty - and the combined output will just be the same as stdout. - """ - - read_set = [] - write_set = [] - stdout = None # Return - stderr = None # Return - - if self.stdin: - # Flush stdio buffer. This might block, if the user has - # been writing to .stdin in an uncontrolled fashion. - self.stdin.flush() - if input_buf: - write_set.append(self.stdin) - else: - self.stdin.close() - if self.stdout: - read_set.append(self.stdout) - stdout = bytearray() - if self.stderr and self.stderr != self.stdout: - read_set.append(self.stderr) - stderr = bytearray() - combined = bytearray() - - stop_now = False - input_offset = 0 - while read_set or write_set: - try: - rlist, wlist, _ = select.select(read_set, write_set, [], 0.2) - except select.error as e: - if e.args[0] == errno.EINTR: - continue - raise - - if not stay_alive: - self.terminate() - - if self.stdin in wlist: - # When select has indicated that the file is writable, - # we can write up to PIPE_BUF bytes without risk - # blocking. POSIX defines PIPE_BUF >= 512 - chunk = input_buf[input_offset : input_offset + 512] - bytes_written = os.write(self.stdin.fileno(), chunk) - input_offset += bytes_written - if input_offset >= len(input_buf): - self.stdin.close() - write_set.remove(self.stdin) - - if self.stdout in rlist: - data = b'' - # We will get an error on read if the pty is closed - try: - data = os.read(self.stdout.fileno(), 1024) - except OSError: - pass - if not len(data): - self.stdout.close() - read_set.remove(self.stdout) - else: - stdout += data - combined += data - if output: - stop_now = output(sys.stdout, data) - if self.stderr in rlist: - data = b'' - # We will get an error on read if the pty is closed - try: - data = os.read(self.stderr.fileno(), 1024) - except OSError: - pass - if not len(data): - self.stderr.close() - read_set.remove(self.stderr) - else: - stderr += data - combined += data - if output: - stop_now = output(sys.stderr, data) - if stop_now: - self.terminate() - - # All data exchanged. Translate lists into strings. - stdout = self.convert_data(stdout) - stderr = self.convert_data(stderr) - combined = self.convert_data(combined) - - self.wait() - return (stdout, stderr, combined) - - -# Just being a unittest.TestCase gives us 14 public methods. Unless we -# disable this, we can only have 6 tests in a TestCase. That's not enough. -# -# pylint: disable=R0904 - -class TestSubprocess(unittest.TestCase): - """Our simple unit test for this module""" - - class MyOperation: - """Provides a operation that we can pass to Popen""" - def __init__(self, input_to_send=None): - """Constructor to set up the operation and possible input. - - Args: - input_to_send: a text string to send when we first get input. We will - add \r\n to the string. - """ - self.stdout_data = '' - self.stderr_data = '' - self.combined_data = '' - self.stdin_pipe = None - self._input_to_send = input_to_send - if input_to_send: - pipe = os.pipe() - self.stdin_read_pipe = pipe[0] - self._stdin_write_pipe = os.fdopen(pipe[1], 'w') - - def output(self, stream, data): - """Output handler for Popen. Stores the data for later comparison""" - if stream == sys.stdout: - self.stdout_data += data - if stream == sys.stderr: - self.stderr_data += data - self.combined_data += data - - # Output the input string if we have one. - if self._input_to_send: - self._stdin_write_pipe.write(self._input_to_send + '\r\n') - self._stdin_write_pipe.flush() - - def _basic_check(self, plist, oper): - """Basic checks that the output looks sane.""" - self.assertEqual(plist[0], oper.stdout_data) - self.assertEqual(plist[1], oper.stderr_data) - self.assertEqual(plist[2], oper.combined_data) - - # The total length of stdout and stderr should equal the combined length - self.assertEqual(len(plist[0]) + len(plist[1]), len(plist[2])) - - def test_simple(self): - """Simple redirection: Get process list""" - oper = TestSubprocess.MyOperation() - plist = Popen(['ps']).communicate_filter(oper.output) - self._basic_check(plist, oper) - - def test_stderr(self): - """Check stdout and stderr""" - oper = TestSubprocess.MyOperation() - cmd = 'echo fred >/dev/stderr && false || echo bad' - plist = Popen([cmd], shell=True).communicate_filter(oper.output) - self._basic_check(plist, oper) - self.assertEqual(plist [0], 'bad\r\n') - self.assertEqual(plist [1], 'fred\r\n') - - def test_shell(self): - """Check with and without shell works""" - oper = TestSubprocess.MyOperation() - cmd = 'echo test >/dev/stderr' - self.assertRaises(OSError, Popen, [cmd], shell=False) - plist = Popen([cmd], shell=True).communicate_filter(oper.output) - self._basic_check(plist, oper) - self.assertEqual(len(plist [0]), 0) - self.assertEqual(plist [1], 'test\r\n') - - def test_list_args(self): - """Check with and without shell works using list arguments""" - oper = TestSubprocess.MyOperation() - cmd = ['echo', 'test', '>/dev/stderr'] - plist = Popen(cmd, shell=False).communicate_filter(oper.output) - self._basic_check(plist, oper) - self.assertEqual(plist [0], ' '.join(cmd[1:]) + '\r\n') - self.assertEqual(len(plist [1]), 0) - - oper = TestSubprocess.MyOperation() - - # this should be interpreted as 'echo' with the other args dropped - cmd = ['echo', 'test', '>/dev/stderr'] - plist = Popen(cmd, shell=True).communicate_filter(oper.output) - self._basic_check(plist, oper) - self.assertEqual(plist [0], '\r\n') - - def test_cwd(self): - """Check we can change directory""" - for shell in (False, True): - oper = TestSubprocess.MyOperation() - plist = Popen('pwd', shell=shell, cwd='/tmp').communicate_filter( - oper.output) - self._basic_check(plist, oper) - self.assertEqual(plist [0], '/tmp\r\n') - - def test_env(self): - """Check we can change environment""" - for add in (False, True): - oper = TestSubprocess.MyOperation() - env = os.environ - if add: - env ['FRED'] = 'fred' - cmd = 'echo $FRED' - plist = Popen(cmd, shell=True, env=env).communicate_filter(oper.output) - self._basic_check(plist, oper) - self.assertEqual(plist [0], add and 'fred\r\n' or '\r\n') - - def test_extra_args(self): - """Check we can't add extra arguments""" - self.assertRaises(ValueError, Popen, 'true', close_fds=False) - - def test_basic_input(self): - """Check that incremental input works - - We set up a subprocess which will prompt for name. When we see this prompt - we send the name as input to the process. It should then print the name - properly to stdout. - """ - oper = TestSubprocess.MyOperation('Flash') - prompt = 'What is your name?: ' - cmd = 'echo -n "%s"; read name; echo Hello $name' % prompt - plist = Popen([cmd], stdin=oper.stdin_read_pipe, - shell=True).communicate_filter(oper.output) - self._basic_check(plist, oper) - self.assertEqual(len(plist [1]), 0) - self.assertEqual(plist [0], prompt + 'Hello Flash\r\r\n') - - def test_isatty(self): - """Check that ptys appear as terminals to the subprocess""" - oper = TestSubprocess.MyOperation() - cmd = ('if [ -t %d ]; then echo "terminal %d" >&%d; ' - 'else echo "not %d" >&%d; fi;') - both_cmds = '' - for fd in (1, 2): - both_cmds += cmd % (fd, fd, fd, fd, fd) - plist = Popen(both_cmds, shell=True).communicate_filter(oper.output) - self._basic_check(plist, oper) - self.assertEqual(plist [0], 'terminal 1\r\n') - self.assertEqual(plist [1], 'terminal 2\r\n') - - # Now try with PIPE and make sure it is not a terminal - oper = TestSubprocess.MyOperation() - plist = Popen(both_cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - shell=True).communicate_filter(oper.output) - self._basic_check(plist, oper) - self.assertEqual(plist [0], 'not 1\n') - self.assertEqual(plist [1], 'not 2\n') - -if __name__ == '__main__': - unittest.main() diff --git a/tools/patman/func_test.py b/tools/patman/func_test.py index c25a47bdeb2..8c2dfbe4528 100644 --- a/tools/patman/func_test.py +++ b/tools/patman/func_test.py @@ -23,9 +23,9 @@ from patman import patchstream from patman.patchstream import PatchStream from patman.series import Series from patman import settings -from patman import terminal -from patman import tools -from patman.test_util import capture_sys_output +from u_boot_pylib import terminal +from u_boot_pylib import tools +from u_boot_pylib.test_util import capture_sys_output import pygit2 from patman import status diff --git a/tools/patman/get_maintainer.py b/tools/patman/get_maintainer.py index f7011be1e49..8df3d124bac 100644 --- a/tools/patman/get_maintainer.py +++ b/tools/patman/get_maintainer.py @@ -7,8 +7,8 @@ import os import shlex import shutil -from patman import command from patman import gitutil +from u_boot_pylib import command def find_get_maintainer(script_file_name): diff --git a/tools/patman/gitutil.py b/tools/patman/gitutil.py index 5e742102c21..6700057359f 100644 --- a/tools/patman/gitutil.py +++ b/tools/patman/gitutil.py @@ -5,9 +5,9 @@ import os import sys -from patman import command from patman import settings -from patman import terminal +from u_boot_pylib import command +from u_boot_pylib import terminal # True to use --no-decorate - we check this in setup() use_no_decorate = True diff --git a/tools/patman/patchstream.py b/tools/patman/patchstream.py index fb6a6036f3b..f91669a9404 100644 --- a/tools/patman/patchstream.py +++ b/tools/patman/patchstream.py @@ -14,10 +14,10 @@ import queue import shutil import tempfile -from patman import command from patman import commit from patman import gitutil from patman.series import Series +from u_boot_pylib import command # Tags that we detect and remove RE_REMOVE = re.compile(r'^BUG=|^TEST=|^BRANCH=|^Review URL:' diff --git a/tools/patman/series.py b/tools/patman/series.py index 2eeeef71dc6..88417acb434 100644 --- a/tools/patman/series.py +++ b/tools/patman/series.py @@ -11,8 +11,8 @@ import os from patman import get_maintainer from patman import gitutil from patman import settings -from patman import terminal -from patman import tools +from u_boot_pylib import terminal +from u_boot_pylib import tools # Series-xxx tags that we understand valid_series = ['to', 'cc', 'version', 'changes', 'prefix', 'notes', 'name', diff --git a/tools/patman/status.py b/tools/patman/status.py index 47ed6d61d4d..5fb436e08ff 100644 --- a/tools/patman/status.py +++ b/tools/patman/status.py @@ -18,8 +18,8 @@ import requests from patman import patchstream from patman.patchstream import PatchStream -from patman import terminal -from patman import tout +from u_boot_pylib import terminal +from u_boot_pylib import tout # Patches which are part of a multi-patch series are shown with a prefix like # [prefix, version, sequence], for example '[RFC, v2, 3/5]'. All but the last diff --git a/tools/patman/terminal.py b/tools/patman/terminal.py deleted file mode 100644 index 40d79f8ac07..00000000000 --- a/tools/patman/terminal.py +++ /dev/null @@ -1,270 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0+ -# Copyright (c) 2011 The Chromium OS Authors. -# - -"""Terminal utilities - -This module handles terminal interaction including ANSI color codes. -""" - -import os -import re -import shutil -import sys - -# Selection of when we want our output to be colored -COLOR_IF_TERMINAL, COLOR_ALWAYS, COLOR_NEVER = range(3) - -# Initially, we are set up to print to the terminal -print_test_mode = False -print_test_list = [] - -# The length of the last line printed without a newline -last_print_len = None - -# credit: -# stackoverflow.com/questions/14693701/how-can-i-remove-the-ansi-escape-sequences-from-a-string-in-python -ansi_escape = re.compile(r'\x1b(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') - -class PrintLine: - """A line of text output - - Members: - text: Text line that was printed - newline: True to output a newline after the text - colour: Text colour to use - """ - def __init__(self, text, colour, newline=True, bright=True): - self.text = text - self.newline = newline - self.colour = colour - self.bright = bright - - def __eq__(self, other): - return (self.text == other.text and - self.newline == other.newline and - self.colour == other.colour and - self.bright == other.bright) - - def __str__(self): - return ("newline=%s, colour=%s, bright=%d, text='%s'" % - (self.newline, self.colour, self.bright, self.text)) - - -def calc_ascii_len(text): - """Calculate the length of a string, ignoring any ANSI sequences - - When displayed on a terminal, ANSI sequences don't take any space, so we - need to ignore them when calculating the length of a string. - - Args: - text: Text to check - - Returns: - Length of text, after skipping ANSI sequences - - >>> col = Color(COLOR_ALWAYS) - >>> text = col.build(Color.RED, 'abc') - >>> len(text) - 14 - >>> calc_ascii_len(text) - 3 - >>> - >>> text += 'def' - >>> calc_ascii_len(text) - 6 - >>> text += col.build(Color.RED, 'abc') - >>> calc_ascii_len(text) - 9 - """ - result = ansi_escape.sub('', text) - return len(result) - -def trim_ascii_len(text, size): - """Trim a string containing ANSI sequences to the given ASCII length - - The string is trimmed with ANSI sequences being ignored for the length - calculation. - - >>> col = Color(COLOR_ALWAYS) - >>> text = col.build(Color.RED, 'abc') - >>> len(text) - 14 - >>> calc_ascii_len(trim_ascii_len(text, 4)) - 3 - >>> calc_ascii_len(trim_ascii_len(text, 2)) - 2 - >>> text += 'def' - >>> calc_ascii_len(trim_ascii_len(text, 4)) - 4 - >>> text += col.build(Color.RED, 'ghi') - >>> calc_ascii_len(trim_ascii_len(text, 7)) - 7 - """ - if calc_ascii_len(text) < size: - return text - pos = 0 - out = '' - left = size - - # Work through each ANSI sequence in turn - for m in ansi_escape.finditer(text): - # Find the text before the sequence and add it to our string, making - # sure it doesn't overflow - before = text[pos:m.start()] - toadd = before[:left] - out += toadd - - # Figure out how much non-ANSI space we have left - left -= len(toadd) - - # Add the ANSI sequence and move to the position immediately after it - out += m.group() - pos = m.start() + len(m.group()) - - # Deal with text after the last ANSI sequence - after = text[pos:] - toadd = after[:left] - out += toadd - - return out - - -def tprint(text='', newline=True, colour=None, limit_to_line=False, bright=True): - """Handle a line of output to the terminal. - - In test mode this is recorded in a list. Otherwise it is output to the - terminal. - - Args: - text: Text to print - newline: True to add a new line at the end of the text - colour: Colour to use for the text - """ - global last_print_len - - if print_test_mode: - print_test_list.append(PrintLine(text, colour, newline, bright)) - else: - if colour: - col = Color() - text = col.build(colour, text, bright=bright) - if newline: - print(text) - last_print_len = None - else: - if limit_to_line: - cols = shutil.get_terminal_size().columns - text = trim_ascii_len(text, cols) - print(text, end='', flush=True) - last_print_len = calc_ascii_len(text) - -def print_clear(): - """Clear a previously line that was printed with no newline""" - global last_print_len - - if last_print_len: - print('\r%s\r' % (' '* last_print_len), end='', flush=True) - last_print_len = None - -def set_print_test_mode(enable=True): - """Go into test mode, where all printing is recorded""" - global print_test_mode - - print_test_mode = enable - get_print_test_lines() - -def get_print_test_lines(): - """Get a list of all lines output through tprint() - - Returns: - A list of PrintLine objects - """ - global print_test_list - - ret = print_test_list - print_test_list = [] - return ret - -def echo_print_test_lines(): - """Print out the text lines collected""" - for line in print_test_list: - if line.colour: - col = Color() - print(col.build(line.colour, line.text), end='') - else: - print(line.text, end='') - if line.newline: - print() - - -class Color(object): - """Conditionally wraps text in ANSI color escape sequences.""" - BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8) - BOLD = -1 - BRIGHT_START = '\033[1;%dm' - NORMAL_START = '\033[22;%dm' - BOLD_START = '\033[1m' - RESET = '\033[0m' - - def __init__(self, colored=COLOR_IF_TERMINAL): - """Create a new Color object, optionally disabling color output. - - Args: - enabled: True if color output should be enabled. If False then this - class will not add color codes at all. - """ - try: - self._enabled = (colored == COLOR_ALWAYS or - (colored == COLOR_IF_TERMINAL and - os.isatty(sys.stdout.fileno()))) - except: - self._enabled = False - - def start(self, color, bright=True): - """Returns a start color code. - - Args: - color: Color to use, .e.g BLACK, RED, etc. - - Returns: - If color is enabled, returns an ANSI sequence to start the given - color, otherwise returns empty string - """ - if self._enabled: - base = self.BRIGHT_START if bright else self.NORMAL_START - return base % (color + 30) - return '' - - def stop(self): - """Returns a stop color code. - - Returns: - If color is enabled, returns an ANSI color reset sequence, - otherwise returns empty string - """ - if self._enabled: - return self.RESET - return '' - - def build(self, color, text, bright=True): - """Returns text with conditionally added color escape sequences. - - Keyword arguments: - color: Text color -- one of the color constants defined in this - class. - text: The text to color. - - Returns: - If self._enabled is False, returns the original text. If it's True, - returns text with color escape sequences based on the value of - color. - """ - if not self._enabled: - return text - if color == self.BOLD: - start = self.BOLD_START - else: - base = self.BRIGHT_START if bright else self.NORMAL_START - start = base % (color + 30) - return start + text + self.RESET diff --git a/tools/patman/test_settings.py b/tools/patman/test_settings.py index c768a2fc641..06b7cbc3ab6 100644 --- a/tools/patman/test_settings.py +++ b/tools/patman/test_settings.py @@ -10,7 +10,7 @@ import sys import tempfile from patman import settings -from patman import tools +from u_boot_pylib import tools @contextlib.contextmanager diff --git a/tools/patman/test_util.py b/tools/patman/test_util.py deleted file mode 100644 index 9e0811b61a2..00000000000 --- a/tools/patman/test_util.py +++ /dev/null @@ -1,216 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0+ -# -# Copyright (c) 2016 Google, Inc -# - -from contextlib import contextmanager -import doctest -import glob -import multiprocessing -import os -import sys -import unittest - -from patman import command - -from io import StringIO - -use_concurrent = True -try: - from concurrencytest import ConcurrentTestSuite - from concurrencytest import fork_for_tests -except: - use_concurrent = False - - -def run_test_coverage(prog, filter_fname, exclude_list, build_dir, required=None, - extra_args=None): - """Run tests and check that we get 100% coverage - - Args: - prog: Program to run (with be passed a '-t' argument to run tests - filter_fname: Normally all *.py files in the program's directory will - be included. If this is not None, then it is used to filter the - list so that only filenames that don't contain filter_fname are - included. - exclude_list: List of file patterns to exclude from the coverage - calculation - build_dir: Build directory, used to locate libfdt.py - required: List of modules which must be in the coverage report - extra_args (str): Extra arguments to pass to the tool before the -t/test - arg - - Raises: - ValueError if the code coverage is not 100% - """ - # This uses the build output from sandbox_spl to get _libfdt.so - path = os.path.dirname(prog) - if filter_fname: - glob_list = glob.glob(os.path.join(path, '*.py')) - glob_list = [fname for fname in glob_list if filter_fname in fname] - else: - glob_list = [] - glob_list += exclude_list - glob_list += ['*libfdt.py', '*site-packages*', '*dist-packages*'] - glob_list += ['*concurrencytest*'] - test_cmd = 'test' if 'binman' in prog or 'patman' in prog else '-t' - prefix = '' - if build_dir: - prefix = 'PYTHONPATH=$PYTHONPATH:%s/sandbox_spl/tools ' % build_dir - cmd = ('%spython3-coverage run ' - '--omit "%s" %s %s %s -P1' % (prefix, ','.join(glob_list), - prog, extra_args or '', test_cmd)) - os.system(cmd) - stdout = command.output('python3-coverage', 'report') - lines = stdout.splitlines() - if required: - # Convert '/path/to/name.py' just the module name 'name' - test_set = set([os.path.splitext(os.path.basename(line.split()[0]))[0] - for line in lines if '/etype/' in line]) - missing_list = required - missing_list.discard('__init__') - missing_list.difference_update(test_set) - if missing_list: - print('Missing tests for %s' % (', '.join(missing_list))) - print(stdout) - ok = False - - coverage = lines[-1].split(' ')[-1] - ok = True - print(coverage) - if coverage != '100%': - print(stdout) - print("To get a report in 'htmlcov/index.html', type: python3-coverage html") - print('Coverage error: %s, but should be 100%%' % coverage) - ok = False - if not ok: - raise ValueError('Test coverage failure') - - -# Use this to suppress stdout/stderr output: -# with capture_sys_output() as (stdout, stderr) -# ...do something... -@contextmanager -def capture_sys_output(): - capture_out, capture_err = StringIO(), StringIO() - old_out, old_err = sys.stdout, sys.stderr - try: - sys.stdout, sys.stderr = capture_out, capture_err - yield capture_out, capture_err - finally: - sys.stdout, sys.stderr = old_out, old_err - - -class FullTextTestResult(unittest.TextTestResult): - """A test result class that can print extended text results to a stream - - This is meant to be used by a TestRunner as a result class. Like - TextTestResult, this prints out the names of tests as they are run, - errors as they occur, and a summary of the results at the end of the - test run. Beyond those, this prints information about skipped tests, - expected failures and unexpected successes. - - Args: - stream: A file-like object to write results to - descriptions (bool): True to print descriptions with test names - verbosity (int): Detail of printed output per test as they run - Test stdout and stderr always get printed when buffering - them is disabled by the test runner. In addition to that, - 0: Print nothing - 1: Print a dot per test - 2: Print test names - """ - def __init__(self, stream, descriptions, verbosity): - self.verbosity = verbosity - super().__init__(stream, descriptions, verbosity) - - def printErrors(self): - "Called by TestRunner after test run to summarize the tests" - # The parent class doesn't keep unexpected successes in the same - # format as the rest. Adapt it to what printErrorList expects. - unexpected_successes = [ - (test, 'Test was expected to fail, but succeeded.\n') - for test in self.unexpectedSuccesses - ] - - super().printErrors() # FAIL and ERROR - self.printErrorList('SKIP', self.skipped) - self.printErrorList('XFAIL', self.expectedFailures) - self.printErrorList('XPASS', unexpected_successes) - - def addSkip(self, test, reason): - """Called when a test is skipped.""" - # Add empty line to keep spacing consistent with other results - if not reason.endswith('\n'): - reason += '\n' - super().addSkip(test, reason) - - -def run_test_suites(toolname, debug, verbosity, test_preserve_dirs, processes, - test_name, toolpath, class_and_module_list): - """Run a series of test suites and collect the results - - Args: - toolname: Name of the tool that ran the tests - debug: True to enable debugging, which shows a full stack trace on error - verbosity: Verbosity level to use (0-4) - test_preserve_dirs: True to preserve the input directory used by tests - so that it can be examined afterwards (only useful for debugging - tests). If a single test is selected (in args[0]) it also preserves - the output directory for this test. Both directories are displayed - on the command line. - processes: Number of processes to use to run tests (None=same as #CPUs) - test_name: Name of test to run, or None for all - toolpath: List of paths to use for tools - class_and_module_list: List of test classes (type class) and module - names (type str) to run - """ - sys.argv = [sys.argv[0]] - if debug: - sys.argv.append('-D') - if verbosity: - sys.argv.append('-v%d' % verbosity) - if toolpath: - for path in toolpath: - sys.argv += ['--toolpath', path] - - suite = unittest.TestSuite() - loader = unittest.TestLoader() - runner = unittest.TextTestRunner( - stream=sys.stdout, - verbosity=(1 if verbosity is None else verbosity), - resultclass=FullTextTestResult, - ) - - if use_concurrent and processes != 1: - suite = ConcurrentTestSuite(suite, - fork_for_tests(processes or multiprocessing.cpu_count())) - - for module in class_and_module_list: - if isinstance(module, str) and (not test_name or test_name == module): - suite.addTests(doctest.DocTestSuite(module)) - - for module in class_and_module_list: - if isinstance(module, str): - continue - # Test the test module about our arguments, if it is interested - if hasattr(module, 'setup_test_args'): - setup_test_args = getattr(module, 'setup_test_args') - setup_test_args(preserve_indir=test_preserve_dirs, - preserve_outdirs=test_preserve_dirs and test_name is not None, - toolpath=toolpath, verbosity=verbosity) - if test_name: - # Since Python v3.5 If an ImportError or AttributeError occurs - # while traversing a name then a synthetic test that raises that - # error when run will be returned. Check that the requested test - # exists, otherwise these errors are included in the results. - if test_name in loader.getTestCaseNames(module): - suite.addTests(loader.loadTestsFromName(test_name, module)) - else: - suite.addTests(loader.loadTestsFromTestCase(module)) - - print(f" Running {toolname} tests ".center(70, "=")) - result = runner.run(suite) - print() - - return result diff --git a/tools/patman/tools.py b/tools/patman/tools.py deleted file mode 100644 index 2ac814d476f..00000000000 --- a/tools/patman/tools.py +++ /dev/null @@ -1,596 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0+ -# -# Copyright (c) 2016 Google, Inc -# - -import glob -import os -import shlex -import shutil -import sys -import tempfile -import urllib.request - -from patman import command -from patman import tout - -# Output directly (generally this is temporary) -outdir = None - -# True to keep the output directory around after exiting -preserve_outdir = False - -# Path to the Chrome OS chroot, if we know it -chroot_path = None - -# Search paths to use for filename(), used to find files -search_paths = [] - -tool_search_paths = [] - -# Tools and the packages that contain them, on debian -packages = { - 'lz4': 'liblz4-tool', - } - -# List of paths to use when looking for an input file -indir = [] - -def prepare_output_dir(dirname, preserve=False): - """Select an output directory, ensuring it exists. - - This either creates a temporary directory or checks that the one supplied - by the user is valid. For a temporary directory, it makes a note to - remove it later if required. - - Args: - dirname: a string, name of the output directory to use to store - intermediate and output files. If is None - create a temporary - directory. - preserve: a Boolean. If outdir above is None and preserve is False, the - created temporary directory will be destroyed on exit. - - Raises: - OSError: If it cannot create the output directory. - """ - global outdir, preserve_outdir - - preserve_outdir = dirname or preserve - if dirname: - outdir = dirname - if not os.path.isdir(outdir): - try: - os.makedirs(outdir) - except OSError as err: - raise ValueError( - f"Cannot make output directory 'outdir': 'err.strerror'") - tout.debug("Using output directory '%s'" % outdir) - else: - outdir = tempfile.mkdtemp(prefix='binman.') - tout.debug("Using temporary directory '%s'" % outdir) - -def _remove_output_dir(): - global outdir - - shutil.rmtree(outdir) - tout.debug("Deleted temporary directory '%s'" % outdir) - outdir = None - -def finalise_output_dir(): - global outdir, preserve_outdir - - """Tidy up: delete output directory if temporary and not preserved.""" - if outdir and not preserve_outdir: - _remove_output_dir() - outdir = None - -def get_output_filename(fname): - """Return a filename within the output directory. - - Args: - fname: Filename to use for new file - - Returns: - The full path of the filename, within the output directory - """ - return os.path.join(outdir, fname) - -def get_output_dir(): - """Return the current output directory - - Returns: - str: The output directory - """ - return outdir - -def _finalise_for_test(): - """Remove the output directory (for use by tests)""" - global outdir - - if outdir: - _remove_output_dir() - outdir = None - -def set_input_dirs(dirname): - """Add a list of input directories, where input files are kept. - - Args: - dirname: a list of paths to input directories to use for obtaining - files needed by binman to place in the image. - """ - global indir - - indir = dirname - tout.debug("Using input directories %s" % indir) - -def get_input_filename(fname, allow_missing=False): - """Return a filename for use as input. - - Args: - fname: Filename to use for new file - allow_missing: True if the filename can be missing - - Returns: - fname, if indir is None; - full path of the filename, within the input directory; - None, if file is missing and allow_missing is True - - Raises: - ValueError if file is missing and allow_missing is False - """ - if not indir or fname[:1] == '/': - return fname - for dirname in indir: - pathname = os.path.join(dirname, fname) - if os.path.exists(pathname): - return pathname - - if allow_missing: - return None - raise ValueError("Filename '%s' not found in input path (%s) (cwd='%s')" % - (fname, ','.join(indir), os.getcwd())) - -def get_input_filename_glob(pattern): - """Return a list of filenames for use as input. - - Args: - pattern: Filename pattern to search for - - Returns: - A list of matching files in all input directories - """ - if not indir: - return glob.glob(pattern) - files = [] - for dirname in indir: - pathname = os.path.join(dirname, pattern) - files += glob.glob(pathname) - return sorted(files) - -def align(pos, align): - if align: - mask = align - 1 - pos = (pos + mask) & ~mask - return pos - -def not_power_of_two(num): - return num and (num & (num - 1)) - -def set_tool_paths(toolpaths): - """Set the path to search for tools - - Args: - toolpaths: List of paths to search for tools executed by run() - """ - global tool_search_paths - - tool_search_paths = toolpaths - -def path_has_file(path_spec, fname): - """Check if a given filename is in the PATH - - Args: - path_spec: Value of PATH variable to check - fname: Filename to check - - Returns: - True if found, False if not - """ - for dir in path_spec.split(':'): - if os.path.exists(os.path.join(dir, fname)): - return True - return False - -def get_host_compile_tool(env, name): - """Get the host-specific version for a compile tool - - This checks the environment variables that specify which version of - the tool should be used (e.g. ${HOSTCC}). - - The following table lists the host-specific versions of the tools - this function resolves to: - - Compile Tool | Host version - --------------+---------------- - as | ${HOSTAS} - ld | ${HOSTLD} - cc | ${HOSTCC} - cpp | ${HOSTCPP} - c++ | ${HOSTCXX} - ar | ${HOSTAR} - nm | ${HOSTNM} - ldr | ${HOSTLDR} - strip | ${HOSTSTRIP} - objcopy | ${HOSTOBJCOPY} - objdump | ${HOSTOBJDUMP} - dtc | ${HOSTDTC} - - Args: - name: Command name to run - - Returns: - host_name: Exact command name to run instead - extra_args: List of extra arguments to pass - """ - host_name = None - extra_args = [] - if name in ('as', 'ld', 'cc', 'cpp', 'ar', 'nm', 'ldr', 'strip', - 'objcopy', 'objdump', 'dtc'): - host_name, *host_args = env.get('HOST' + name.upper(), '').split(' ') - elif name == 'c++': - host_name, *host_args = env.get('HOSTCXX', '').split(' ') - - if host_name: - return host_name, extra_args - return name, [] - -def get_target_compile_tool(name, cross_compile=None): - """Get the target-specific version for a compile tool - - This first checks the environment variables that specify which - version of the tool should be used (e.g. ${CC}). If those aren't - specified, it checks the CROSS_COMPILE variable as a prefix for the - tool with some substitutions (e.g. "${CROSS_COMPILE}gcc" for cc). - - The following table lists the target-specific versions of the tools - this function resolves to: - - Compile Tool | First choice | Second choice - --------------+----------------+---------------------------- - as | ${AS} | ${CROSS_COMPILE}as - ld | ${LD} | ${CROSS_COMPILE}ld.bfd - | | or ${CROSS_COMPILE}ld - cc | ${CC} | ${CROSS_COMPILE}gcc - cpp | ${CPP} | ${CROSS_COMPILE}gcc -E - c++ | ${CXX} | ${CROSS_COMPILE}g++ - ar | ${AR} | ${CROSS_COMPILE}ar - nm | ${NM} | ${CROSS_COMPILE}nm - ldr | ${LDR} | ${CROSS_COMPILE}ldr - strip | ${STRIP} | ${CROSS_COMPILE}strip - objcopy | ${OBJCOPY} | ${CROSS_COMPILE}objcopy - objdump | ${OBJDUMP} | ${CROSS_COMPILE}objdump - dtc | ${DTC} | (no CROSS_COMPILE version) - - Args: - name: Command name to run - - Returns: - target_name: Exact command name to run instead - extra_args: List of extra arguments to pass - """ - env = dict(os.environ) - - target_name = None - extra_args = [] - if name in ('as', 'ld', 'cc', 'cpp', 'ar', 'nm', 'ldr', 'strip', - 'objcopy', 'objdump', 'dtc'): - target_name, *extra_args = env.get(name.upper(), '').split(' ') - elif name == 'c++': - target_name, *extra_args = env.get('CXX', '').split(' ') - - if target_name: - return target_name, extra_args - - if cross_compile is None: - cross_compile = env.get('CROSS_COMPILE', '') - - if name in ('as', 'ar', 'nm', 'ldr', 'strip', 'objcopy', 'objdump'): - target_name = cross_compile + name - elif name == 'ld': - try: - if run(cross_compile + 'ld.bfd', '-v'): - target_name = cross_compile + 'ld.bfd' - except: - target_name = cross_compile + 'ld' - elif name == 'cc': - target_name = cross_compile + 'gcc' - elif name == 'cpp': - target_name = cross_compile + 'gcc' - extra_args = ['-E'] - elif name == 'c++': - target_name = cross_compile + 'g++' - else: - target_name = name - return target_name, extra_args - -def get_env_with_path(): - """Get an updated environment with the PATH variable set correctly - - If there are any search paths set, these need to come first in the PATH so - that these override any other version of the tools. - - Returns: - dict: New environment with PATH updated, or None if there are not search - paths - """ - if tool_search_paths: - env = dict(os.environ) - env['PATH'] = ':'.join(tool_search_paths) + ':' + env['PATH'] - return env - -def run_result(name, *args, **kwargs): - """Run a tool with some arguments - - This runs a 'tool', which is a program used by binman to process files and - perhaps produce some output. Tools can be located on the PATH or in a - search path. - - Args: - name: Command name to run - args: Arguments to the tool - for_host: True to resolve the command to the version for the host - for_target: False to run the command as-is, without resolving it - to the version for the compile target - raise_on_error: Raise an error if the command fails (True by default) - - Returns: - CommandResult object - """ - try: - binary = kwargs.get('binary') - for_host = kwargs.get('for_host', False) - for_target = kwargs.get('for_target', not for_host) - raise_on_error = kwargs.get('raise_on_error', True) - env = get_env_with_path() - if for_target: - name, extra_args = get_target_compile_tool(name) - args = tuple(extra_args) + args - elif for_host: - name, extra_args = get_host_compile_tool(env, name) - args = tuple(extra_args) + args - name = os.path.expanduser(name) # Expand paths containing ~ - all_args = (name,) + args - result = command.run_pipe([all_args], capture=True, capture_stderr=True, - env=env, raise_on_error=False, binary=binary) - if result.return_code: - if raise_on_error: - raise ValueError("Error %d running '%s': %s" % - (result.return_code,' '.join(all_args), - result.stderr or result.stdout)) - return result - except ValueError: - if env and not path_has_file(env['PATH'], name): - msg = "Please install tool '%s'" % name - package = packages.get(name) - if package: - msg += " (e.g. from package '%s')" % package - raise ValueError(msg) - raise - -def tool_find(name): - """Search the current path for a tool - - This uses both PATH and any value from set_tool_paths() to search for a tool - - Args: - name (str): Name of tool to locate - - Returns: - str: Full path to tool if found, else None - """ - name = os.path.expanduser(name) # Expand paths containing ~ - paths = [] - pathvar = os.environ.get('PATH') - if pathvar: - paths = pathvar.split(':') - if tool_search_paths: - paths += tool_search_paths - for path in paths: - fname = os.path.join(path, name) - if os.path.isfile(fname) and os.access(fname, os.X_OK): - return fname - -def run(name, *args, **kwargs): - """Run a tool with some arguments - - This runs a 'tool', which is a program used by binman to process files and - perhaps produce some output. Tools can be located on the PATH or in a - search path. - - Args: - name: Command name to run - args: Arguments to the tool - for_host: True to resolve the command to the version for the host - for_target: False to run the command as-is, without resolving it - to the version for the compile target - - Returns: - CommandResult object - """ - result = run_result(name, *args, **kwargs) - if result is not None: - return result.stdout - -def filename(fname): - """Resolve a file path to an absolute path. - - If fname starts with ##/ and chroot is available, ##/ gets replaced with - the chroot path. If chroot is not available, this file name can not be - resolved, `None' is returned. - - If fname is not prepended with the above prefix, and is not an existing - file, the actual file name is retrieved from the passed in string and the - search_paths directories (if any) are searched to for the file. If found - - the path to the found file is returned, `None' is returned otherwise. - - Args: - fname: a string, the path to resolve. - - Returns: - Absolute path to the file or None if not found. - """ - if fname.startswith('##/'): - if chroot_path: - fname = os.path.join(chroot_path, fname[3:]) - else: - return None - - # Search for a pathname that exists, and return it if found - if fname and not os.path.exists(fname): - for path in search_paths: - pathname = os.path.join(path, os.path.basename(fname)) - if os.path.exists(pathname): - return pathname - - # If not found, just return the standard, unchanged path - return fname - -def read_file(fname, binary=True): - """Read and return the contents of a file. - - Args: - fname: path to filename to read, where ## signifiies the chroot. - - Returns: - data read from file, as a string. - """ - with open(filename(fname), binary and 'rb' or 'r') as fd: - data = fd.read() - #self._out.Info("Read file '%s' size %d (%#0x)" % - #(fname, len(data), len(data))) - return data - -def write_file(fname, data, binary=True): - """Write data into a file. - - Args: - fname: path to filename to write - data: data to write to file, as a string - """ - #self._out.Info("Write file '%s' size %d (%#0x)" % - #(fname, len(data), len(data))) - with open(filename(fname), binary and 'wb' or 'w') as fd: - fd.write(data) - -def get_bytes(byte, size): - """Get a string of bytes of a given size - - Args: - byte: Numeric byte value to use - size: Size of bytes/string to return - - Returns: - A bytes type with 'byte' repeated 'size' times - """ - return bytes([byte]) * size - -def to_bytes(string): - """Convert a str type into a bytes type - - Args: - string: string to convert - - Returns: - A bytes type - """ - return string.encode('utf-8') - -def to_string(bval): - """Convert a bytes type into a str type - - Args: - bval: bytes value to convert - - Returns: - Python 3: A bytes type - Python 2: A string type - """ - return bval.decode('utf-8') - -def to_hex(val): - """Convert an integer value (or None) to a string - - Returns: - hex value, or 'None' if the value is None - """ - return 'None' if val is None else '%#x' % val - -def to_hex_size(val): - """Return the size of an object in hex - - Returns: - hex value of size, or 'None' if the value is None - """ - return 'None' if val is None else '%#x' % len(val) - -def print_full_help(fname): - """Print the full help message for a tool using an appropriate pager. - - Args: - fname: Path to a file containing the full help message - """ - pager = shlex.split(os.getenv('PAGER', '')) - if not pager: - lesspath = shutil.which('less') - pager = [lesspath] if lesspath else None - if not pager: - pager = ['more'] - command.run(*pager, fname) - -def download(url, tmpdir_pattern='.patman'): - """Download a file to a temporary directory - - Args: - url (str): URL to download - tmpdir_pattern (str): pattern to use for the temporary directory - - Returns: - Tuple: - Full path to the downloaded archive file in that directory, - or None if there was an error while downloading - Temporary directory name - """ - print('- downloading: %s' % url) - leaf = url.split('/')[-1] - tmpdir = tempfile.mkdtemp(tmpdir_pattern) - response = urllib.request.urlopen(url) - fname = os.path.join(tmpdir, leaf) - fd = open(fname, 'wb') - meta = response.info() - size = int(meta.get('Content-Length')) - done = 0 - block_size = 1 << 16 - status = '' - - # Read the file in chunks and show progress as we go - while True: - buffer = response.read(block_size) - if not buffer: - print(chr(8) * (len(status) + 1), '\r', end=' ') - break - - done += len(buffer) - fd.write(buffer) - status = r'%10d MiB [%3d%%]' % (done // 1024 // 1024, - done * 100 // size) - status = status + chr(8) * (len(status) + 1) - print(status, end=' ') - sys.stdout.flush() - print('\r', end='') - sys.stdout.flush() - fd.close() - if done != size: - print('Error, failed to download') - os.remove(fname) - fname = None - return fname, tmpdir diff --git a/tools/patman/tout.py b/tools/patman/tout.py deleted file mode 100644 index ff0fd92afcc..00000000000 --- a/tools/patman/tout.py +++ /dev/null @@ -1,179 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0+ -# Copyright (c) 2016 Google, Inc -# -# Terminal output logging. -# - -import sys - -from patman import terminal - -# Output verbosity levels that we support -ERROR, WARNING, NOTICE, INFO, DETAIL, DEBUG = range(6) - -in_progress = False - -""" -This class handles output of progress and other useful information -to the user. It provides for simple verbosity level control and can -output nothing but errors at verbosity zero. - -The idea is that modules set up an Output object early in their years and pass -it around to other modules that need it. This keeps the output under control -of a single class. - -Public properties: - verbose: Verbosity level: 0=silent, 1=progress, 3=full, 4=debug -""" -def __enter__(): - return - -def __exit__(unused1, unused2, unused3): - """Clean up and remove any progress message.""" - clear_progress() - return False - -def user_is_present(): - """This returns True if it is likely that a user is present. - - Sometimes we want to prompt the user, but if no one is there then this - is a waste of time, and may lock a script which should otherwise fail. - - Returns: - True if it thinks the user is there, and False otherwise - """ - return stdout_is_tty and verbose > 0 - -def clear_progress(): - """Clear any active progress message on the terminal.""" - global in_progress - if verbose > 0 and stdout_is_tty and in_progress: - _stdout.write('\r%s\r' % (" " * len (_progress))) - _stdout.flush() - in_progress = False - -def progress(msg, warning=False, trailer='...'): - """Display progress information. - - Args: - msg: Message to display. - warning: True if this is a warning.""" - global in_progress - clear_progress() - if verbose > 0: - _progress = msg + trailer - if stdout_is_tty: - col = _color.YELLOW if warning else _color.GREEN - _stdout.write('\r' + _color.build(col, _progress)) - _stdout.flush() - in_progress = True - else: - _stdout.write(_progress + '\n') - -def _output(level, msg, color=None): - """Output a message to the terminal. - - Args: - level: Verbosity level for this message. It will only be displayed if - this as high as the currently selected level. - msg; Message to display. - error: True if this is an error message, else False. - """ - if verbose >= level: - clear_progress() - if color: - msg = _color.build(color, msg) - if level < NOTICE: - print(msg, file=sys.stderr) - else: - print(msg) - -def do_output(level, msg): - """Output a message to the terminal. - - Args: - level: Verbosity level for this message. It will only be displayed if - this as high as the currently selected level. - msg; Message to display. - """ - _output(level, msg) - -def error(msg): - """Display an error message - - Args: - msg; Message to display. - """ - _output(ERROR, msg, _color.RED) - -def warning(msg): - """Display a warning message - - Args: - msg; Message to display. - """ - _output(WARNING, msg, _color.YELLOW) - -def notice(msg): - """Display an important infomation message - - Args: - msg; Message to display. - """ - _output(NOTICE, msg) - -def info(msg): - """Display an infomation message - - Args: - msg; Message to display. - """ - _output(INFO, msg) - -def detail(msg): - """Display a detailed message - - Args: - msg; Message to display. - """ - _output(DETAIL, msg) - -def debug(msg): - """Display a debug message - - Args: - msg; Message to display. - """ - _output(DEBUG, msg) - -def user_output(msg): - """Display a message regardless of the current output level. - - This is used when the output was specifically requested by the user. - Args: - msg; Message to display. - """ - _output(0, msg) - -def init(_verbose=WARNING, stdout=sys.stdout): - """Initialize a new output object. - - Args: - verbose: Verbosity level (0-4). - stdout: File to use for stdout. - """ - global verbose, _progress, _color, _stdout, stdout_is_tty - - verbose = _verbose - _progress = '' # Our last progress message - _color = terminal.Color() - _stdout = stdout - - # TODO(sjg): Move this into Chromite libraries when we have them - stdout_is_tty = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty() - stderr_is_tty = hasattr(sys.stderr, 'isatty') and sys.stderr.isatty() - -def uninit(): - clear_progress() - -init() diff --git a/tools/rmboard.py b/tools/rmboard.py index ae256321270..0c56b149e0f 100755 --- a/tools/rmboard.py +++ b/tools/rmboard.py @@ -28,7 +28,7 @@ import os import re import sys -from patman import command +from u_boot_pylib import command def rm_kconfig_include(path): """Remove a path from Kconfig files diff --git a/tools/u_boot_pylib/__init__.py b/tools/u_boot_pylib/__init__.py new file mode 100644 index 00000000000..63c88e85ec0 --- /dev/null +++ b/tools/u_boot_pylib/__init__.py @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0+ + +__all__ = ['command', 'cros_subprocess','terminal', 'test_util', 'tools', + 'tout'] diff --git a/tools/u_boot_pylib/__main__.py b/tools/u_boot_pylib/__main__.py new file mode 100755 index 00000000000..8f98d7bd9f8 --- /dev/null +++ b/tools/u_boot_pylib/__main__.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0+ +# +# Copyright 2023 Google LLC +# + +import os +import sys + +if __name__ == "__main__": + # Allow 'from u_boot_pylib import xxx to work' + our_path = os.path.dirname(os.path.realpath(__file__)) + sys.path.append(os.path.join(our_path, '..')) + + # Run tests + from u_boot_pylib import terminal + from u_boot_pylib import test_util + + result = test_util.run_test_suites( + 'u_boot_pylib', False, False, False, None, None, None, + ['terminal']) + + sys.exit(0 if result.wasSuccessful() else 1) diff --git a/tools/u_boot_pylib/command.py b/tools/u_boot_pylib/command.py new file mode 100644 index 00000000000..9bbfc5bdd83 --- /dev/null +++ b/tools/u_boot_pylib/command.py @@ -0,0 +1,139 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2011 The Chromium OS Authors. +# + +import os + +from u_boot_pylib import cros_subprocess + +"""Shell command ease-ups for Python.""" + +class CommandResult: + """A class which captures the result of executing a command. + + Members: + stdout: stdout obtained from command, as a string + stderr: stderr obtained from command, as a string + return_code: Return code from command + exception: Exception received, or None if all ok + """ + def __init__(self, stdout='', stderr='', combined='', return_code=0, + exception=None): + self.stdout = stdout + self.stderr = stderr + self.combined = combined + self.return_code = return_code + self.exception = exception + + def to_output(self, binary): + if not binary: + self.stdout = self.stdout.decode('utf-8') + self.stderr = self.stderr.decode('utf-8') + self.combined = self.combined.decode('utf-8') + return self + + +# This permits interception of RunPipe for test purposes. If it is set to +# a function, then that function is called with the pipe list being +# executed. Otherwise, it is assumed to be a CommandResult object, and is +# returned as the result for every run_pipe() call. +# When this value is None, commands are executed as normal. +test_result = None + +def run_pipe(pipe_list, infile=None, outfile=None, + capture=False, capture_stderr=False, oneline=False, + raise_on_error=True, cwd=None, binary=False, + output_func=None, **kwargs): + """ + Perform a command pipeline, with optional input/output filenames. + + Args: + pipe_list: List of command lines to execute. Each command line is + piped into the next, and is itself a list of strings. For + example [ ['ls', '.git'] ['wc'] ] will pipe the output of + 'ls .git' into 'wc'. + infile: File to provide stdin to the pipeline + outfile: File to store stdout + capture: True to capture output + capture_stderr: True to capture stderr + oneline: True to strip newline chars from output + output_func: Output function to call with each output fragment + (if it returns True the function terminates) + kwargs: Additional keyword arguments to cros_subprocess.Popen() + Returns: + CommandResult object + """ + if test_result: + if hasattr(test_result, '__call__'): + # pylint: disable=E1102 + result = test_result(pipe_list=pipe_list) + if result: + return result + else: + return test_result + # No result: fall through to normal processing + result = CommandResult(b'', b'', b'') + last_pipe = None + pipeline = list(pipe_list) + user_pipestr = '|'.join([' '.join(pipe) for pipe in pipe_list]) + kwargs['stdout'] = None + kwargs['stderr'] = None + while pipeline: + cmd = pipeline.pop(0) + if last_pipe is not None: + kwargs['stdin'] = last_pipe.stdout + elif infile: + kwargs['stdin'] = open(infile, 'rb') + if pipeline or capture: + kwargs['stdout'] = cros_subprocess.PIPE + elif outfile: + kwargs['stdout'] = open(outfile, 'wb') + if capture_stderr: + kwargs['stderr'] = cros_subprocess.PIPE + + try: + last_pipe = cros_subprocess.Popen(cmd, cwd=cwd, **kwargs) + except Exception as err: + result.exception = err + if raise_on_error: + raise Exception("Error running '%s': %s" % (user_pipestr, str)) + result.return_code = 255 + return result.to_output(binary) + + if capture: + result.stdout, result.stderr, result.combined = ( + last_pipe.communicate_filter(output_func)) + if result.stdout and oneline: + result.output = result.stdout.rstrip(b'\r\n') + result.return_code = last_pipe.wait() + else: + result.return_code = os.waitpid(last_pipe.pid, 0)[1] + if raise_on_error and result.return_code: + raise Exception("Error running '%s'" % user_pipestr) + return result.to_output(binary) + +def output(*cmd, **kwargs): + kwargs['raise_on_error'] = kwargs.get('raise_on_error', True) + return run_pipe([cmd], capture=True, **kwargs).stdout + +def output_one_line(*cmd, **kwargs): + """Run a command and output it as a single-line string + + The command us expected to produce a single line of output + + Returns: + String containing output of command + """ + raise_on_error = kwargs.pop('raise_on_error', True) + result = run_pipe([cmd], capture=True, oneline=True, + raise_on_error=raise_on_error, **kwargs).stdout.strip() + return result + +def run(*cmd, **kwargs): + return run_pipe([cmd], **kwargs).stdout + +def run_list(cmd): + return run_pipe([cmd], capture=True).stdout + +def stop_all(): + cros_subprocess.stay_alive = False diff --git a/tools/u_boot_pylib/cros_subprocess.py b/tools/u_boot_pylib/cros_subprocess.py new file mode 100644 index 00000000000..cd614f38a64 --- /dev/null +++ b/tools/u_boot_pylib/cros_subprocess.py @@ -0,0 +1,401 @@ +# Copyright (c) 2012 The Chromium OS Authors. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. +# +# Copyright (c) 2003-2005 by Peter Astrand +# Licensed to PSF under a Contributor Agreement. +# See http://www.python.org/2.4/license for licensing details. + +"""Subprocess execution + +This module holds a subclass of subprocess.Popen with our own required +features, mainly that we get access to the subprocess output while it +is running rather than just at the end. This makes it easier to show +progress information and filter output in real time. +""" + +import errno +import os +import pty +import select +import subprocess +import sys +import unittest + + +# Import these here so the caller does not need to import subprocess also. +PIPE = subprocess.PIPE +STDOUT = subprocess.STDOUT +PIPE_PTY = -3 # Pipe output through a pty +stay_alive = True + + +class Popen(subprocess.Popen): + """Like subprocess.Popen with ptys and incremental output + + This class deals with running a child process and filtering its output on + both stdout and stderr while it is running. We do this so we can monitor + progress, and possibly relay the output to the user if requested. + + The class is similar to subprocess.Popen, the equivalent is something like: + + Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + But this class has many fewer features, and two enhancement: + + 1. Rather than getting the output data only at the end, this class sends it + to a provided operation as it arrives. + 2. We use pseudo terminals so that the child will hopefully flush its output + to us as soon as it is produced, rather than waiting for the end of a + line. + + Use communicate_filter() to handle output from the subprocess. + + """ + + def __init__(self, args, stdin=None, stdout=PIPE_PTY, stderr=PIPE_PTY, + shell=False, cwd=None, env=None, **kwargs): + """Cut-down constructor + + Args: + args: Program and arguments for subprocess to execute. + stdin: See subprocess.Popen() + stdout: See subprocess.Popen(), except that we support the sentinel + value of cros_subprocess.PIPE_PTY. + stderr: See subprocess.Popen(), except that we support the sentinel + value of cros_subprocess.PIPE_PTY. + shell: See subprocess.Popen() + cwd: Working directory to change to for subprocess, or None if none. + env: Environment to use for this subprocess, or None to inherit parent. + kwargs: No other arguments are supported at the moment. Passing other + arguments will cause a ValueError to be raised. + """ + stdout_pty = None + stderr_pty = None + + if stdout == PIPE_PTY: + stdout_pty = pty.openpty() + stdout = os.fdopen(stdout_pty[1]) + if stderr == PIPE_PTY: + stderr_pty = pty.openpty() + stderr = os.fdopen(stderr_pty[1]) + + super(Popen, self).__init__(args, stdin=stdin, + stdout=stdout, stderr=stderr, shell=shell, cwd=cwd, env=env, + **kwargs) + + # If we're on a PTY, we passed the slave half of the PTY to the subprocess. + # We want to use the master half on our end from now on. Setting this here + # does make some assumptions about the implementation of subprocess, but + # those assumptions are pretty minor. + + # Note that if stderr is STDOUT, then self.stderr will be set to None by + # this constructor. + if stdout_pty is not None: + self.stdout = os.fdopen(stdout_pty[0]) + if stderr_pty is not None: + self.stderr = os.fdopen(stderr_pty[0]) + + # Insist that unit tests exist for other arguments we don't support. + if kwargs: + raise ValueError("Unit tests do not test extra args - please add tests") + + def convert_data(self, data): + """Convert stdout/stderr data to the correct format for output + + Args: + data: Data to convert, or None for '' + + Returns: + Converted data, as bytes + """ + if data is None: + return b'' + return data + + def communicate_filter(self, output, input_buf=''): + """Interact with process: Read data from stdout and stderr. + + This method runs until end-of-file is reached, then waits for the + subprocess to terminate. + + The output function is sent all output from the subprocess and must be + defined like this: + + def output([self,] stream, data) + Args: + stream: the stream the output was received on, which will be + sys.stdout or sys.stderr. + data: a string containing the data + + Returns: + True to terminate the process + + Note: The data read is buffered in memory, so do not use this + method if the data size is large or unlimited. + + Args: + output: Function to call with each fragment of output. + + Returns: + A tuple (stdout, stderr, combined) which is the data received on + stdout, stderr and the combined data (interleaved stdout and stderr). + + Note that the interleaved output will only be sensible if you have + set both stdout and stderr to PIPE or PIPE_PTY. Even then it depends on + the timing of the output in the subprocess. If a subprocess flips + between stdout and stderr quickly in succession, by the time we come to + read the output from each we may see several lines in each, and will read + all the stdout lines, then all the stderr lines. So the interleaving + may not be correct. In this case you might want to pass + stderr=cros_subprocess.STDOUT to the constructor. + + This feature is still useful for subprocesses where stderr is + rarely used and indicates an error. + + Note also that if you set stderr to STDOUT, then stderr will be empty + and the combined output will just be the same as stdout. + """ + + read_set = [] + write_set = [] + stdout = None # Return + stderr = None # Return + + if self.stdin: + # Flush stdio buffer. This might block, if the user has + # been writing to .stdin in an uncontrolled fashion. + self.stdin.flush() + if input_buf: + write_set.append(self.stdin) + else: + self.stdin.close() + if self.stdout: + read_set.append(self.stdout) + stdout = bytearray() + if self.stderr and self.stderr != self.stdout: + read_set.append(self.stderr) + stderr = bytearray() + combined = bytearray() + + stop_now = False + input_offset = 0 + while read_set or write_set: + try: + rlist, wlist, _ = select.select(read_set, write_set, [], 0.2) + except select.error as e: + if e.args[0] == errno.EINTR: + continue + raise + + if not stay_alive: + self.terminate() + + if self.stdin in wlist: + # When select has indicated that the file is writable, + # we can write up to PIPE_BUF bytes without risk + # blocking. POSIX defines PIPE_BUF >= 512 + chunk = input_buf[input_offset : input_offset + 512] + bytes_written = os.write(self.stdin.fileno(), chunk) + input_offset += bytes_written + if input_offset >= len(input_buf): + self.stdin.close() + write_set.remove(self.stdin) + + if self.stdout in rlist: + data = b'' + # We will get an error on read if the pty is closed + try: + data = os.read(self.stdout.fileno(), 1024) + except OSError: + pass + if not len(data): + self.stdout.close() + read_set.remove(self.stdout) + else: + stdout += data + combined += data + if output: + stop_now = output(sys.stdout, data) + if self.stderr in rlist: + data = b'' + # We will get an error on read if the pty is closed + try: + data = os.read(self.stderr.fileno(), 1024) + except OSError: + pass + if not len(data): + self.stderr.close() + read_set.remove(self.stderr) + else: + stderr += data + combined += data + if output: + stop_now = output(sys.stderr, data) + if stop_now: + self.terminate() + + # All data exchanged. Translate lists into strings. + stdout = self.convert_data(stdout) + stderr = self.convert_data(stderr) + combined = self.convert_data(combined) + + self.wait() + return (stdout, stderr, combined) + + +# Just being a unittest.TestCase gives us 14 public methods. Unless we +# disable this, we can only have 6 tests in a TestCase. That's not enough. +# +# pylint: disable=R0904 + +class TestSubprocess(unittest.TestCase): + """Our simple unit test for this module""" + + class MyOperation: + """Provides a operation that we can pass to Popen""" + def __init__(self, input_to_send=None): + """Constructor to set up the operation and possible input. + + Args: + input_to_send: a text string to send when we first get input. We will + add \r\n to the string. + """ + self.stdout_data = '' + self.stderr_data = '' + self.combined_data = '' + self.stdin_pipe = None + self._input_to_send = input_to_send + if input_to_send: + pipe = os.pipe() + self.stdin_read_pipe = pipe[0] + self._stdin_write_pipe = os.fdopen(pipe[1], 'w') + + def output(self, stream, data): + """Output handler for Popen. Stores the data for later comparison""" + if stream == sys.stdout: + self.stdout_data += data + if stream == sys.stderr: + self.stderr_data += data + self.combined_data += data + + # Output the input string if we have one. + if self._input_to_send: + self._stdin_write_pipe.write(self._input_to_send + '\r\n') + self._stdin_write_pipe.flush() + + def _basic_check(self, plist, oper): + """Basic checks that the output looks sane.""" + self.assertEqual(plist[0], oper.stdout_data) + self.assertEqual(plist[1], oper.stderr_data) + self.assertEqual(plist[2], oper.combined_data) + + # The total length of stdout and stderr should equal the combined length + self.assertEqual(len(plist[0]) + len(plist[1]), len(plist[2])) + + def test_simple(self): + """Simple redirection: Get process list""" + oper = TestSubprocess.MyOperation() + plist = Popen(['ps']).communicate_filter(oper.output) + self._basic_check(plist, oper) + + def test_stderr(self): + """Check stdout and stderr""" + oper = TestSubprocess.MyOperation() + cmd = 'echo fred >/dev/stderr && false || echo bad' + plist = Popen([cmd], shell=True).communicate_filter(oper.output) + self._basic_check(plist, oper) + self.assertEqual(plist [0], 'bad\r\n') + self.assertEqual(plist [1], 'fred\r\n') + + def test_shell(self): + """Check with and without shell works""" + oper = TestSubprocess.MyOperation() + cmd = 'echo test >/dev/stderr' + self.assertRaises(OSError, Popen, [cmd], shell=False) + plist = Popen([cmd], shell=True).communicate_filter(oper.output) + self._basic_check(plist, oper) + self.assertEqual(len(plist [0]), 0) + self.assertEqual(plist [1], 'test\r\n') + + def test_list_args(self): + """Check with and without shell works using list arguments""" + oper = TestSubprocess.MyOperation() + cmd = ['echo', 'test', '>/dev/stderr'] + plist = Popen(cmd, shell=False).communicate_filter(oper.output) + self._basic_check(plist, oper) + self.assertEqual(plist [0], ' '.join(cmd[1:]) + '\r\n') + self.assertEqual(len(plist [1]), 0) + + oper = TestSubprocess.MyOperation() + + # this should be interpreted as 'echo' with the other args dropped + cmd = ['echo', 'test', '>/dev/stderr'] + plist = Popen(cmd, shell=True).communicate_filter(oper.output) + self._basic_check(plist, oper) + self.assertEqual(plist [0], '\r\n') + + def test_cwd(self): + """Check we can change directory""" + for shell in (False, True): + oper = TestSubprocess.MyOperation() + plist = Popen('pwd', shell=shell, cwd='/tmp').communicate_filter( + oper.output) + self._basic_check(plist, oper) + self.assertEqual(plist [0], '/tmp\r\n') + + def test_env(self): + """Check we can change environment""" + for add in (False, True): + oper = TestSubprocess.MyOperation() + env = os.environ + if add: + env ['FRED'] = 'fred' + cmd = 'echo $FRED' + plist = Popen(cmd, shell=True, env=env).communicate_filter(oper.output) + self._basic_check(plist, oper) + self.assertEqual(plist [0], add and 'fred\r\n' or '\r\n') + + def test_extra_args(self): + """Check we can't add extra arguments""" + self.assertRaises(ValueError, Popen, 'true', close_fds=False) + + def test_basic_input(self): + """Check that incremental input works + + We set up a subprocess which will prompt for name. When we see this prompt + we send the name as input to the process. It should then print the name + properly to stdout. + """ + oper = TestSubprocess.MyOperation('Flash') + prompt = 'What is your name?: ' + cmd = 'echo -n "%s"; read name; echo Hello $name' % prompt + plist = Popen([cmd], stdin=oper.stdin_read_pipe, + shell=True).communicate_filter(oper.output) + self._basic_check(plist, oper) + self.assertEqual(len(plist [1]), 0) + self.assertEqual(plist [0], prompt + 'Hello Flash\r\r\n') + + def test_isatty(self): + """Check that ptys appear as terminals to the subprocess""" + oper = TestSubprocess.MyOperation() + cmd = ('if [ -t %d ]; then echo "terminal %d" >&%d; ' + 'else echo "not %d" >&%d; fi;') + both_cmds = '' + for fd in (1, 2): + both_cmds += cmd % (fd, fd, fd, fd, fd) + plist = Popen(both_cmds, shell=True).communicate_filter(oper.output) + self._basic_check(plist, oper) + self.assertEqual(plist [0], 'terminal 1\r\n') + self.assertEqual(plist [1], 'terminal 2\r\n') + + # Now try with PIPE and make sure it is not a terminal + oper = TestSubprocess.MyOperation() + plist = Popen(both_cmds, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + shell=True).communicate_filter(oper.output) + self._basic_check(plist, oper) + self.assertEqual(plist [0], 'not 1\n') + self.assertEqual(plist [1], 'not 2\n') + +if __name__ == '__main__': + unittest.main() diff --git a/tools/u_boot_pylib/terminal.py b/tools/u_boot_pylib/terminal.py new file mode 100644 index 00000000000..40d79f8ac07 --- /dev/null +++ b/tools/u_boot_pylib/terminal.py @@ -0,0 +1,270 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2011 The Chromium OS Authors. +# + +"""Terminal utilities + +This module handles terminal interaction including ANSI color codes. +""" + +import os +import re +import shutil +import sys + +# Selection of when we want our output to be colored +COLOR_IF_TERMINAL, COLOR_ALWAYS, COLOR_NEVER = range(3) + +# Initially, we are set up to print to the terminal +print_test_mode = False +print_test_list = [] + +# The length of the last line printed without a newline +last_print_len = None + +# credit: +# stackoverflow.com/questions/14693701/how-can-i-remove-the-ansi-escape-sequences-from-a-string-in-python +ansi_escape = re.compile(r'\x1b(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])') + +class PrintLine: + """A line of text output + + Members: + text: Text line that was printed + newline: True to output a newline after the text + colour: Text colour to use + """ + def __init__(self, text, colour, newline=True, bright=True): + self.text = text + self.newline = newline + self.colour = colour + self.bright = bright + + def __eq__(self, other): + return (self.text == other.text and + self.newline == other.newline and + self.colour == other.colour and + self.bright == other.bright) + + def __str__(self): + return ("newline=%s, colour=%s, bright=%d, text='%s'" % + (self.newline, self.colour, self.bright, self.text)) + + +def calc_ascii_len(text): + """Calculate the length of a string, ignoring any ANSI sequences + + When displayed on a terminal, ANSI sequences don't take any space, so we + need to ignore them when calculating the length of a string. + + Args: + text: Text to check + + Returns: + Length of text, after skipping ANSI sequences + + >>> col = Color(COLOR_ALWAYS) + >>> text = col.build(Color.RED, 'abc') + >>> len(text) + 14 + >>> calc_ascii_len(text) + 3 + >>> + >>> text += 'def' + >>> calc_ascii_len(text) + 6 + >>> text += col.build(Color.RED, 'abc') + >>> calc_ascii_len(text) + 9 + """ + result = ansi_escape.sub('', text) + return len(result) + +def trim_ascii_len(text, size): + """Trim a string containing ANSI sequences to the given ASCII length + + The string is trimmed with ANSI sequences being ignored for the length + calculation. + + >>> col = Color(COLOR_ALWAYS) + >>> text = col.build(Color.RED, 'abc') + >>> len(text) + 14 + >>> calc_ascii_len(trim_ascii_len(text, 4)) + 3 + >>> calc_ascii_len(trim_ascii_len(text, 2)) + 2 + >>> text += 'def' + >>> calc_ascii_len(trim_ascii_len(text, 4)) + 4 + >>> text += col.build(Color.RED, 'ghi') + >>> calc_ascii_len(trim_ascii_len(text, 7)) + 7 + """ + if calc_ascii_len(text) < size: + return text + pos = 0 + out = '' + left = size + + # Work through each ANSI sequence in turn + for m in ansi_escape.finditer(text): + # Find the text before the sequence and add it to our string, making + # sure it doesn't overflow + before = text[pos:m.start()] + toadd = before[:left] + out += toadd + + # Figure out how much non-ANSI space we have left + left -= len(toadd) + + # Add the ANSI sequence and move to the position immediately after it + out += m.group() + pos = m.start() + len(m.group()) + + # Deal with text after the last ANSI sequence + after = text[pos:] + toadd = after[:left] + out += toadd + + return out + + +def tprint(text='', newline=True, colour=None, limit_to_line=False, bright=True): + """Handle a line of output to the terminal. + + In test mode this is recorded in a list. Otherwise it is output to the + terminal. + + Args: + text: Text to print + newline: True to add a new line at the end of the text + colour: Colour to use for the text + """ + global last_print_len + + if print_test_mode: + print_test_list.append(PrintLine(text, colour, newline, bright)) + else: + if colour: + col = Color() + text = col.build(colour, text, bright=bright) + if newline: + print(text) + last_print_len = None + else: + if limit_to_line: + cols = shutil.get_terminal_size().columns + text = trim_ascii_len(text, cols) + print(text, end='', flush=True) + last_print_len = calc_ascii_len(text) + +def print_clear(): + """Clear a previously line that was printed with no newline""" + global last_print_len + + if last_print_len: + print('\r%s\r' % (' '* last_print_len), end='', flush=True) + last_print_len = None + +def set_print_test_mode(enable=True): + """Go into test mode, where all printing is recorded""" + global print_test_mode + + print_test_mode = enable + get_print_test_lines() + +def get_print_test_lines(): + """Get a list of all lines output through tprint() + + Returns: + A list of PrintLine objects + """ + global print_test_list + + ret = print_test_list + print_test_list = [] + return ret + +def echo_print_test_lines(): + """Print out the text lines collected""" + for line in print_test_list: + if line.colour: + col = Color() + print(col.build(line.colour, line.text), end='') + else: + print(line.text, end='') + if line.newline: + print() + + +class Color(object): + """Conditionally wraps text in ANSI color escape sequences.""" + BLACK, RED, GREEN, YELLOW, BLUE, MAGENTA, CYAN, WHITE = range(8) + BOLD = -1 + BRIGHT_START = '\033[1;%dm' + NORMAL_START = '\033[22;%dm' + BOLD_START = '\033[1m' + RESET = '\033[0m' + + def __init__(self, colored=COLOR_IF_TERMINAL): + """Create a new Color object, optionally disabling color output. + + Args: + enabled: True if color output should be enabled. If False then this + class will not add color codes at all. + """ + try: + self._enabled = (colored == COLOR_ALWAYS or + (colored == COLOR_IF_TERMINAL and + os.isatty(sys.stdout.fileno()))) + except: + self._enabled = False + + def start(self, color, bright=True): + """Returns a start color code. + + Args: + color: Color to use, .e.g BLACK, RED, etc. + + Returns: + If color is enabled, returns an ANSI sequence to start the given + color, otherwise returns empty string + """ + if self._enabled: + base = self.BRIGHT_START if bright else self.NORMAL_START + return base % (color + 30) + return '' + + def stop(self): + """Returns a stop color code. + + Returns: + If color is enabled, returns an ANSI color reset sequence, + otherwise returns empty string + """ + if self._enabled: + return self.RESET + return '' + + def build(self, color, text, bright=True): + """Returns text with conditionally added color escape sequences. + + Keyword arguments: + color: Text color -- one of the color constants defined in this + class. + text: The text to color. + + Returns: + If self._enabled is False, returns the original text. If it's True, + returns text with color escape sequences based on the value of + color. + """ + if not self._enabled: + return text + if color == self.BOLD: + start = self.BOLD_START + else: + base = self.BRIGHT_START if bright else self.NORMAL_START + start = base % (color + 30) + return start + text + self.RESET diff --git a/tools/u_boot_pylib/test_util.py b/tools/u_boot_pylib/test_util.py new file mode 100644 index 00000000000..e7564e10c99 --- /dev/null +++ b/tools/u_boot_pylib/test_util.py @@ -0,0 +1,216 @@ +# SPDX-License-Identifier: GPL-2.0+ +# +# Copyright (c) 2016 Google, Inc +# + +from contextlib import contextmanager +import doctest +import glob +import multiprocessing +import os +import sys +import unittest + +from u_boot_pylib import command + +from io import StringIO + +use_concurrent = True +try: + from concurrencytest import ConcurrentTestSuite + from concurrencytest import fork_for_tests +except: + use_concurrent = False + + +def run_test_coverage(prog, filter_fname, exclude_list, build_dir, required=None, + extra_args=None): + """Run tests and check that we get 100% coverage + + Args: + prog: Program to run (with be passed a '-t' argument to run tests + filter_fname: Normally all *.py files in the program's directory will + be included. If this is not None, then it is used to filter the + list so that only filenames that don't contain filter_fname are + included. + exclude_list: List of file patterns to exclude from the coverage + calculation + build_dir: Build directory, used to locate libfdt.py + required: List of modules which must be in the coverage report + extra_args (str): Extra arguments to pass to the tool before the -t/test + arg + + Raises: + ValueError if the code coverage is not 100% + """ + # This uses the build output from sandbox_spl to get _libfdt.so + path = os.path.dirname(prog) + if filter_fname: + glob_list = glob.glob(os.path.join(path, '*.py')) + glob_list = [fname for fname in glob_list if filter_fname in fname] + else: + glob_list = [] + glob_list += exclude_list + glob_list += ['*libfdt.py', '*site-packages*', '*dist-packages*'] + glob_list += ['*concurrencytest*'] + test_cmd = 'test' if 'binman' in prog or 'patman' in prog else '-t' + prefix = '' + if build_dir: + prefix = 'PYTHONPATH=$PYTHONPATH:%s/sandbox_spl/tools ' % build_dir + cmd = ('%spython3-coverage run ' + '--omit "%s" %s %s %s -P1' % (prefix, ','.join(glob_list), + prog, extra_args or '', test_cmd)) + os.system(cmd) + stdout = command.output('python3-coverage', 'report') + lines = stdout.splitlines() + if required: + # Convert '/path/to/name.py' just the module name 'name' + test_set = set([os.path.splitext(os.path.basename(line.split()[0]))[0] + for line in lines if '/etype/' in line]) + missing_list = required + missing_list.discard('__init__') + missing_list.difference_update(test_set) + if missing_list: + print('Missing tests for %s' % (', '.join(missing_list))) + print(stdout) + ok = False + + coverage = lines[-1].split(' ')[-1] + ok = True + print(coverage) + if coverage != '100%': + print(stdout) + print("To get a report in 'htmlcov/index.html', type: python3-coverage html") + print('Coverage error: %s, but should be 100%%' % coverage) + ok = False + if not ok: + raise ValueError('Test coverage failure') + + +# Use this to suppress stdout/stderr output: +# with capture_sys_output() as (stdout, stderr) +# ...do something... +@contextmanager +def capture_sys_output(): + capture_out, capture_err = StringIO(), StringIO() + old_out, old_err = sys.stdout, sys.stderr + try: + sys.stdout, sys.stderr = capture_out, capture_err + yield capture_out, capture_err + finally: + sys.stdout, sys.stderr = old_out, old_err + + +class FullTextTestResult(unittest.TextTestResult): + """A test result class that can print extended text results to a stream + + This is meant to be used by a TestRunner as a result class. Like + TextTestResult, this prints out the names of tests as they are run, + errors as they occur, and a summary of the results at the end of the + test run. Beyond those, this prints information about skipped tests, + expected failures and unexpected successes. + + Args: + stream: A file-like object to write results to + descriptions (bool): True to print descriptions with test names + verbosity (int): Detail of printed output per test as they run + Test stdout and stderr always get printed when buffering + them is disabled by the test runner. In addition to that, + 0: Print nothing + 1: Print a dot per test + 2: Print test names + """ + def __init__(self, stream, descriptions, verbosity): + self.verbosity = verbosity + super().__init__(stream, descriptions, verbosity) + + def printErrors(self): + "Called by TestRunner after test run to summarize the tests" + # The parent class doesn't keep unexpected successes in the same + # format as the rest. Adapt it to what printErrorList expects. + unexpected_successes = [ + (test, 'Test was expected to fail, but succeeded.\n') + for test in self.unexpectedSuccesses + ] + + super().printErrors() # FAIL and ERROR + self.printErrorList('SKIP', self.skipped) + self.printErrorList('XFAIL', self.expectedFailures) + self.printErrorList('XPASS', unexpected_successes) + + def addSkip(self, test, reason): + """Called when a test is skipped.""" + # Add empty line to keep spacing consistent with other results + if not reason.endswith('\n'): + reason += '\n' + super().addSkip(test, reason) + + +def run_test_suites(toolname, debug, verbosity, test_preserve_dirs, processes, + test_name, toolpath, class_and_module_list): + """Run a series of test suites and collect the results + + Args: + toolname: Name of the tool that ran the tests + debug: True to enable debugging, which shows a full stack trace on error + verbosity: Verbosity level to use (0-4) + test_preserve_dirs: True to preserve the input directory used by tests + so that it can be examined afterwards (only useful for debugging + tests). If a single test is selected (in args[0]) it also preserves + the output directory for this test. Both directories are displayed + on the command line. + processes: Number of processes to use to run tests (None=same as #CPUs) + test_name: Name of test to run, or None for all + toolpath: List of paths to use for tools + class_and_module_list: List of test classes (type class) and module + names (type str) to run + """ + sys.argv = [sys.argv[0]] + if debug: + sys.argv.append('-D') + if verbosity: + sys.argv.append('-v%d' % verbosity) + if toolpath: + for path in toolpath: + sys.argv += ['--toolpath', path] + + suite = unittest.TestSuite() + loader = unittest.TestLoader() + runner = unittest.TextTestRunner( + stream=sys.stdout, + verbosity=(1 if verbosity is None else verbosity), + resultclass=FullTextTestResult, + ) + + if use_concurrent and processes != 1: + suite = ConcurrentTestSuite(suite, + fork_for_tests(processes or multiprocessing.cpu_count())) + + for module in class_and_module_list: + if isinstance(module, str) and (not test_name or test_name == module): + suite.addTests(doctest.DocTestSuite(module)) + + for module in class_and_module_list: + if isinstance(module, str): + continue + # Test the test module about our arguments, if it is interested + if hasattr(module, 'setup_test_args'): + setup_test_args = getattr(module, 'setup_test_args') + setup_test_args(preserve_indir=test_preserve_dirs, + preserve_outdirs=test_preserve_dirs and test_name is not None, + toolpath=toolpath, verbosity=verbosity) + if test_name: + # Since Python v3.5 If an ImportError or AttributeError occurs + # while traversing a name then a synthetic test that raises that + # error when run will be returned. Check that the requested test + # exists, otherwise these errors are included in the results. + if test_name in loader.getTestCaseNames(module): + suite.addTests(loader.loadTestsFromName(test_name, module)) + else: + suite.addTests(loader.loadTestsFromTestCase(module)) + + print(f" Running {toolname} tests ".center(70, "=")) + result = runner.run(suite) + print() + + return result diff --git a/tools/u_boot_pylib/tools.py b/tools/u_boot_pylib/tools.py new file mode 100644 index 00000000000..187725b5015 --- /dev/null +++ b/tools/u_boot_pylib/tools.py @@ -0,0 +1,596 @@ +# SPDX-License-Identifier: GPL-2.0+ +# +# Copyright (c) 2016 Google, Inc +# + +import glob +import os +import shlex +import shutil +import sys +import tempfile +import urllib.request + +from u_boot_pylib import command +from u_boot_pylib import tout + +# Output directly (generally this is temporary) +outdir = None + +# True to keep the output directory around after exiting +preserve_outdir = False + +# Path to the Chrome OS chroot, if we know it +chroot_path = None + +# Search paths to use for filename(), used to find files +search_paths = [] + +tool_search_paths = [] + +# Tools and the packages that contain them, on debian +packages = { + 'lz4': 'liblz4-tool', + } + +# List of paths to use when looking for an input file +indir = [] + +def prepare_output_dir(dirname, preserve=False): + """Select an output directory, ensuring it exists. + + This either creates a temporary directory or checks that the one supplied + by the user is valid. For a temporary directory, it makes a note to + remove it later if required. + + Args: + dirname: a string, name of the output directory to use to store + intermediate and output files. If is None - create a temporary + directory. + preserve: a Boolean. If outdir above is None and preserve is False, the + created temporary directory will be destroyed on exit. + + Raises: + OSError: If it cannot create the output directory. + """ + global outdir, preserve_outdir + + preserve_outdir = dirname or preserve + if dirname: + outdir = dirname + if not os.path.isdir(outdir): + try: + os.makedirs(outdir) + except OSError as err: + raise ValueError( + f"Cannot make output directory 'outdir': 'err.strerror'") + tout.debug("Using output directory '%s'" % outdir) + else: + outdir = tempfile.mkdtemp(prefix='binman.') + tout.debug("Using temporary directory '%s'" % outdir) + +def _remove_output_dir(): + global outdir + + shutil.rmtree(outdir) + tout.debug("Deleted temporary directory '%s'" % outdir) + outdir = None + +def finalise_output_dir(): + global outdir, preserve_outdir + + """Tidy up: delete output directory if temporary and not preserved.""" + if outdir and not preserve_outdir: + _remove_output_dir() + outdir = None + +def get_output_filename(fname): + """Return a filename within the output directory. + + Args: + fname: Filename to use for new file + + Returns: + The full path of the filename, within the output directory + """ + return os.path.join(outdir, fname) + +def get_output_dir(): + """Return the current output directory + + Returns: + str: The output directory + """ + return outdir + +def _finalise_for_test(): + """Remove the output directory (for use by tests)""" + global outdir + + if outdir: + _remove_output_dir() + outdir = None + +def set_input_dirs(dirname): + """Add a list of input directories, where input files are kept. + + Args: + dirname: a list of paths to input directories to use for obtaining + files needed by binman to place in the image. + """ + global indir + + indir = dirname + tout.debug("Using input directories %s" % indir) + +def get_input_filename(fname, allow_missing=False): + """Return a filename for use as input. + + Args: + fname: Filename to use for new file + allow_missing: True if the filename can be missing + + Returns: + fname, if indir is None; + full path of the filename, within the input directory; + None, if file is missing and allow_missing is True + + Raises: + ValueError if file is missing and allow_missing is False + """ + if not indir or fname[:1] == '/': + return fname + for dirname in indir: + pathname = os.path.join(dirname, fname) + if os.path.exists(pathname): + return pathname + + if allow_missing: + return None + raise ValueError("Filename '%s' not found in input path (%s) (cwd='%s')" % + (fname, ','.join(indir), os.getcwd())) + +def get_input_filename_glob(pattern): + """Return a list of filenames for use as input. + + Args: + pattern: Filename pattern to search for + + Returns: + A list of matching files in all input directories + """ + if not indir: + return glob.glob(pattern) + files = [] + for dirname in indir: + pathname = os.path.join(dirname, pattern) + files += glob.glob(pathname) + return sorted(files) + +def align(pos, align): + if align: + mask = align - 1 + pos = (pos + mask) & ~mask + return pos + +def not_power_of_two(num): + return num and (num & (num - 1)) + +def set_tool_paths(toolpaths): + """Set the path to search for tools + + Args: + toolpaths: List of paths to search for tools executed by run() + """ + global tool_search_paths + + tool_search_paths = toolpaths + +def path_has_file(path_spec, fname): + """Check if a given filename is in the PATH + + Args: + path_spec: Value of PATH variable to check + fname: Filename to check + + Returns: + True if found, False if not + """ + for dir in path_spec.split(':'): + if os.path.exists(os.path.join(dir, fname)): + return True + return False + +def get_host_compile_tool(env, name): + """Get the host-specific version for a compile tool + + This checks the environment variables that specify which version of + the tool should be used (e.g. ${HOSTCC}). + + The following table lists the host-specific versions of the tools + this function resolves to: + + Compile Tool | Host version + --------------+---------------- + as | ${HOSTAS} + ld | ${HOSTLD} + cc | ${HOSTCC} + cpp | ${HOSTCPP} + c++ | ${HOSTCXX} + ar | ${HOSTAR} + nm | ${HOSTNM} + ldr | ${HOSTLDR} + strip | ${HOSTSTRIP} + objcopy | ${HOSTOBJCOPY} + objdump | ${HOSTOBJDUMP} + dtc | ${HOSTDTC} + + Args: + name: Command name to run + + Returns: + host_name: Exact command name to run instead + extra_args: List of extra arguments to pass + """ + host_name = None + extra_args = [] + if name in ('as', 'ld', 'cc', 'cpp', 'ar', 'nm', 'ldr', 'strip', + 'objcopy', 'objdump', 'dtc'): + host_name, *host_args = env.get('HOST' + name.upper(), '').split(' ') + elif name == 'c++': + host_name, *host_args = env.get('HOSTCXX', '').split(' ') + + if host_name: + return host_name, extra_args + return name, [] + +def get_target_compile_tool(name, cross_compile=None): + """Get the target-specific version for a compile tool + + This first checks the environment variables that specify which + version of the tool should be used (e.g. ${CC}). If those aren't + specified, it checks the CROSS_COMPILE variable as a prefix for the + tool with some substitutions (e.g. "${CROSS_COMPILE}gcc" for cc). + + The following table lists the target-specific versions of the tools + this function resolves to: + + Compile Tool | First choice | Second choice + --------------+----------------+---------------------------- + as | ${AS} | ${CROSS_COMPILE}as + ld | ${LD} | ${CROSS_COMPILE}ld.bfd + | | or ${CROSS_COMPILE}ld + cc | ${CC} | ${CROSS_COMPILE}gcc + cpp | ${CPP} | ${CROSS_COMPILE}gcc -E + c++ | ${CXX} | ${CROSS_COMPILE}g++ + ar | ${AR} | ${CROSS_COMPILE}ar + nm | ${NM} | ${CROSS_COMPILE}nm + ldr | ${LDR} | ${CROSS_COMPILE}ldr + strip | ${STRIP} | ${CROSS_COMPILE}strip + objcopy | ${OBJCOPY} | ${CROSS_COMPILE}objcopy + objdump | ${OBJDUMP} | ${CROSS_COMPILE}objdump + dtc | ${DTC} | (no CROSS_COMPILE version) + + Args: + name: Command name to run + + Returns: + target_name: Exact command name to run instead + extra_args: List of extra arguments to pass + """ + env = dict(os.environ) + + target_name = None + extra_args = [] + if name in ('as', 'ld', 'cc', 'cpp', 'ar', 'nm', 'ldr', 'strip', + 'objcopy', 'objdump', 'dtc'): + target_name, *extra_args = env.get(name.upper(), '').split(' ') + elif name == 'c++': + target_name, *extra_args = env.get('CXX', '').split(' ') + + if target_name: + return target_name, extra_args + + if cross_compile is None: + cross_compile = env.get('CROSS_COMPILE', '') + + if name in ('as', 'ar', 'nm', 'ldr', 'strip', 'objcopy', 'objdump'): + target_name = cross_compile + name + elif name == 'ld': + try: + if run(cross_compile + 'ld.bfd', '-v'): + target_name = cross_compile + 'ld.bfd' + except: + target_name = cross_compile + 'ld' + elif name == 'cc': + target_name = cross_compile + 'gcc' + elif name == 'cpp': + target_name = cross_compile + 'gcc' + extra_args = ['-E'] + elif name == 'c++': + target_name = cross_compile + 'g++' + else: + target_name = name + return target_name, extra_args + +def get_env_with_path(): + """Get an updated environment with the PATH variable set correctly + + If there are any search paths set, these need to come first in the PATH so + that these override any other version of the tools. + + Returns: + dict: New environment with PATH updated, or None if there are not search + paths + """ + if tool_search_paths: + env = dict(os.environ) + env['PATH'] = ':'.join(tool_search_paths) + ':' + env['PATH'] + return env + +def run_result(name, *args, **kwargs): + """Run a tool with some arguments + + This runs a 'tool', which is a program used by binman to process files and + perhaps produce some output. Tools can be located on the PATH or in a + search path. + + Args: + name: Command name to run + args: Arguments to the tool + for_host: True to resolve the command to the version for the host + for_target: False to run the command as-is, without resolving it + to the version for the compile target + raise_on_error: Raise an error if the command fails (True by default) + + Returns: + CommandResult object + """ + try: + binary = kwargs.get('binary') + for_host = kwargs.get('for_host', False) + for_target = kwargs.get('for_target', not for_host) + raise_on_error = kwargs.get('raise_on_error', True) + env = get_env_with_path() + if for_target: + name, extra_args = get_target_compile_tool(name) + args = tuple(extra_args) + args + elif for_host: + name, extra_args = get_host_compile_tool(env, name) + args = tuple(extra_args) + args + name = os.path.expanduser(name) # Expand paths containing ~ + all_args = (name,) + args + result = command.run_pipe([all_args], capture=True, capture_stderr=True, + env=env, raise_on_error=False, binary=binary) + if result.return_code: + if raise_on_error: + raise ValueError("Error %d running '%s': %s" % + (result.return_code,' '.join(all_args), + result.stderr or result.stdout)) + return result + except ValueError: + if env and not path_has_file(env['PATH'], name): + msg = "Please install tool '%s'" % name + package = packages.get(name) + if package: + msg += " (e.g. from package '%s')" % package + raise ValueError(msg) + raise + +def tool_find(name): + """Search the current path for a tool + + This uses both PATH and any value from set_tool_paths() to search for a tool + + Args: + name (str): Name of tool to locate + + Returns: + str: Full path to tool if found, else None + """ + name = os.path.expanduser(name) # Expand paths containing ~ + paths = [] + pathvar = os.environ.get('PATH') + if pathvar: + paths = pathvar.split(':') + if tool_search_paths: + paths += tool_search_paths + for path in paths: + fname = os.path.join(path, name) + if os.path.isfile(fname) and os.access(fname, os.X_OK): + return fname + +def run(name, *args, **kwargs): + """Run a tool with some arguments + + This runs a 'tool', which is a program used by binman to process files and + perhaps produce some output. Tools can be located on the PATH or in a + search path. + + Args: + name: Command name to run + args: Arguments to the tool + for_host: True to resolve the command to the version for the host + for_target: False to run the command as-is, without resolving it + to the version for the compile target + + Returns: + CommandResult object + """ + result = run_result(name, *args, **kwargs) + if result is not None: + return result.stdout + +def filename(fname): + """Resolve a file path to an absolute path. + + If fname starts with ##/ and chroot is available, ##/ gets replaced with + the chroot path. If chroot is not available, this file name can not be + resolved, `None' is returned. + + If fname is not prepended with the above prefix, and is not an existing + file, the actual file name is retrieved from the passed in string and the + search_paths directories (if any) are searched to for the file. If found - + the path to the found file is returned, `None' is returned otherwise. + + Args: + fname: a string, the path to resolve. + + Returns: + Absolute path to the file or None if not found. + """ + if fname.startswith('##/'): + if chroot_path: + fname = os.path.join(chroot_path, fname[3:]) + else: + return None + + # Search for a pathname that exists, and return it if found + if fname and not os.path.exists(fname): + for path in search_paths: + pathname = os.path.join(path, os.path.basename(fname)) + if os.path.exists(pathname): + return pathname + + # If not found, just return the standard, unchanged path + return fname + +def read_file(fname, binary=True): + """Read and return the contents of a file. + + Args: + fname: path to filename to read, where ## signifiies the chroot. + + Returns: + data read from file, as a string. + """ + with open(filename(fname), binary and 'rb' or 'r') as fd: + data = fd.read() + #self._out.Info("Read file '%s' size %d (%#0x)" % + #(fname, len(data), len(data))) + return data + +def write_file(fname, data, binary=True): + """Write data into a file. + + Args: + fname: path to filename to write + data: data to write to file, as a string + """ + #self._out.Info("Write file '%s' size %d (%#0x)" % + #(fname, len(data), len(data))) + with open(filename(fname), binary and 'wb' or 'w') as fd: + fd.write(data) + +def get_bytes(byte, size): + """Get a string of bytes of a given size + + Args: + byte: Numeric byte value to use + size: Size of bytes/string to return + + Returns: + A bytes type with 'byte' repeated 'size' times + """ + return bytes([byte]) * size + +def to_bytes(string): + """Convert a str type into a bytes type + + Args: + string: string to convert + + Returns: + A bytes type + """ + return string.encode('utf-8') + +def to_string(bval): + """Convert a bytes type into a str type + + Args: + bval: bytes value to convert + + Returns: + Python 3: A bytes type + Python 2: A string type + """ + return bval.decode('utf-8') + +def to_hex(val): + """Convert an integer value (or None) to a string + + Returns: + hex value, or 'None' if the value is None + """ + return 'None' if val is None else '%#x' % val + +def to_hex_size(val): + """Return the size of an object in hex + + Returns: + hex value of size, or 'None' if the value is None + """ + return 'None' if val is None else '%#x' % len(val) + +def print_full_help(fname): + """Print the full help message for a tool using an appropriate pager. + + Args: + fname: Path to a file containing the full help message + """ + pager = shlex.split(os.getenv('PAGER', '')) + if not pager: + lesspath = shutil.which('less') + pager = [lesspath] if lesspath else None + if not pager: + pager = ['more'] + command.run(*pager, fname) + +def download(url, tmpdir_pattern='.patman'): + """Download a file to a temporary directory + + Args: + url (str): URL to download + tmpdir_pattern (str): pattern to use for the temporary directory + + Returns: + Tuple: + Full path to the downloaded archive file in that directory, + or None if there was an error while downloading + Temporary directory name + """ + print('- downloading: %s' % url) + leaf = url.split('/')[-1] + tmpdir = tempfile.mkdtemp(tmpdir_pattern) + response = urllib.request.urlopen(url) + fname = os.path.join(tmpdir, leaf) + fd = open(fname, 'wb') + meta = response.info() + size = int(meta.get('Content-Length')) + done = 0 + block_size = 1 << 16 + status = '' + + # Read the file in chunks and show progress as we go + while True: + buffer = response.read(block_size) + if not buffer: + print(chr(8) * (len(status) + 1), '\r', end=' ') + break + + done += len(buffer) + fd.write(buffer) + status = r'%10d MiB [%3d%%]' % (done // 1024 // 1024, + done * 100 // size) + status = status + chr(8) * (len(status) + 1) + print(status, end=' ') + sys.stdout.flush() + print('\r', end='') + sys.stdout.flush() + fd.close() + if done != size: + print('Error, failed to download') + os.remove(fname) + fname = None + return fname, tmpdir diff --git a/tools/u_boot_pylib/tout.py b/tools/u_boot_pylib/tout.py new file mode 100644 index 00000000000..6bd2806f88f --- /dev/null +++ b/tools/u_boot_pylib/tout.py @@ -0,0 +1,179 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (c) 2016 Google, Inc +# +# Terminal output logging. +# + +import sys + +from u_boot_pylib import terminal + +# Output verbosity levels that we support +ERROR, WARNING, NOTICE, INFO, DETAIL, DEBUG = range(6) + +in_progress = False + +""" +This class handles output of progress and other useful information +to the user. It provides for simple verbosity level control and can +output nothing but errors at verbosity zero. + +The idea is that modules set up an Output object early in their years and pass +it around to other modules that need it. This keeps the output under control +of a single class. + +Public properties: + verbose: Verbosity level: 0=silent, 1=progress, 3=full, 4=debug +""" +def __enter__(): + return + +def __exit__(unused1, unused2, unused3): + """Clean up and remove any progress message.""" + clear_progress() + return False + +def user_is_present(): + """This returns True if it is likely that a user is present. + + Sometimes we want to prompt the user, but if no one is there then this + is a waste of time, and may lock a script which should otherwise fail. + + Returns: + True if it thinks the user is there, and False otherwise + """ + return stdout_is_tty and verbose > 0 + +def clear_progress(): + """Clear any active progress message on the terminal.""" + global in_progress + if verbose > 0 and stdout_is_tty and in_progress: + _stdout.write('\r%s\r' % (" " * len (_progress))) + _stdout.flush() + in_progress = False + +def progress(msg, warning=False, trailer='...'): + """Display progress information. + + Args: + msg: Message to display. + warning: True if this is a warning.""" + global in_progress + clear_progress() + if verbose > 0: + _progress = msg + trailer + if stdout_is_tty: + col = _color.YELLOW if warning else _color.GREEN + _stdout.write('\r' + _color.build(col, _progress)) + _stdout.flush() + in_progress = True + else: + _stdout.write(_progress + '\n') + +def _output(level, msg, color=None): + """Output a message to the terminal. + + Args: + level: Verbosity level for this message. It will only be displayed if + this as high as the currently selected level. + msg; Message to display. + error: True if this is an error message, else False. + """ + if verbose >= level: + clear_progress() + if color: + msg = _color.build(color, msg) + if level < NOTICE: + print(msg, file=sys.stderr) + else: + print(msg) + +def do_output(level, msg): + """Output a message to the terminal. + + Args: + level: Verbosity level for this message. It will only be displayed if + this as high as the currently selected level. + msg; Message to display. + """ + _output(level, msg) + +def error(msg): + """Display an error message + + Args: + msg; Message to display. + """ + _output(ERROR, msg, _color.RED) + +def warning(msg): + """Display a warning message + + Args: + msg; Message to display. + """ + _output(WARNING, msg, _color.YELLOW) + +def notice(msg): + """Display an important infomation message + + Args: + msg; Message to display. + """ + _output(NOTICE, msg) + +def info(msg): + """Display an infomation message + + Args: + msg; Message to display. + """ + _output(INFO, msg) + +def detail(msg): + """Display a detailed message + + Args: + msg; Message to display. + """ + _output(DETAIL, msg) + +def debug(msg): + """Display a debug message + + Args: + msg; Message to display. + """ + _output(DEBUG, msg) + +def user_output(msg): + """Display a message regardless of the current output level. + + This is used when the output was specifically requested by the user. + Args: + msg; Message to display. + """ + _output(0, msg) + +def init(_verbose=WARNING, stdout=sys.stdout): + """Initialize a new output object. + + Args: + verbose: Verbosity level (0-4). + stdout: File to use for stdout. + """ + global verbose, _progress, _color, _stdout, stdout_is_tty + + verbose = _verbose + _progress = '' # Our last progress message + _color = terminal.Color() + _stdout = stdout + + # TODO(sjg): Move this into Chromite libraries when we have them + stdout_is_tty = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty() + stderr_is_tty = hasattr(sys.stderr, 'isatty') and sys.stderr.isatty() + +def uninit(): + clear_progress() + +init() diff --git a/tools/u_boot_pylib/u_boot_pylib b/tools/u_boot_pylib/u_boot_pylib new file mode 120000 index 00000000000..5a427d19424 --- /dev/null +++ b/tools/u_boot_pylib/u_boot_pylib @@ -0,0 +1 @@ +__main__.py \ No newline at end of file -- cgit v1.3.1 From 75554dfac298f86038124b2fb133fd40cbbaa29c Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 23 Feb 2023 18:18:06 -0700 Subject: patman: Add support for building a u_boot_tools PyPi package Create the necessary files to build this new package. Signed-off-by: Simon Glass --- Makefile | 18 +- tools/u_boot_pylib/LICENSE | 339 ++++++++++++++++++++++++++++++++++++++ tools/u_boot_pylib/README.rst | 15 ++ tools/u_boot_pylib/pyproject.toml | 22 +++ 4 files changed, 393 insertions(+), 1 deletion(-) create mode 100644 tools/u_boot_pylib/LICENSE create mode 100644 tools/u_boot_pylib/README.rst create mode 100644 tools/u_boot_pylib/pyproject.toml (limited to 'tools') diff --git a/Makefile b/Makefile index e5750615886..21d606b10e6 100644 --- a/Makefile +++ b/Makefile @@ -522,7 +522,7 @@ env_h := include/generated/environment.h no-dot-config-targets := clean clobber mrproper distclean \ help %docs check% coccicheck \ ubootversion backup tests check pcheck qcheck tcheck \ - pylint pylint_err + pylint pylint_err _pip pip pip_test pip_release config-targets := 0 mixed-targets := 0 @@ -2272,6 +2272,17 @@ backup: F=`basename $(srctree)` ; cd .. ; \ gtar --force-local -zcvf `LC_ALL=C date "+$$F-%Y-%m-%d-%T.tar.gz"` $$F +PHONY += _pip pip pip_release + +pip_release: PIP_ARGS="--real" +pip_test: PIP_ARGS="" +pip: PIP_ARGS="-n" + +pip pip_test pip_release: _pip + +_pip: + scripts/make_pip.sh u_boot_pylib ${PIP_ARGS} + help: @echo 'Cleaning targets:' @echo ' clean - Remove most generated files but keep the config' @@ -2305,6 +2316,11 @@ help: @echo " cfg - Don't build, just create the .cfg files" @echo " envtools - Build only the target-side environment tools" @echo '' + @echo 'PyPi / pip targets:' + @echo ' pip - Check building of PyPi packages' + @echo ' pip_test - Build PyPi pakages and upload to test server' + @echo ' pip_release - Build PyPi pakages and upload to release server' + @echo '' @echo 'Static analysers' @echo ' checkstack - Generate a list of stack hogs' @echo ' coccicheck - Execute static code analysis with Coccinelle' diff --git a/tools/u_boot_pylib/LICENSE b/tools/u_boot_pylib/LICENSE new file mode 100644 index 00000000000..d159169d105 --- /dev/null +++ b/tools/u_boot_pylib/LICENSE @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/tools/u_boot_pylib/README.rst b/tools/u_boot_pylib/README.rst new file mode 100644 index 00000000000..93858f5571d --- /dev/null +++ b/tools/u_boot_pylib/README.rst @@ -0,0 +1,15 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +# U-Boot Python Library +===================== + +This is a Python library used by various U-Boot tools, including patman, +buildman and binman. + +The module can be installed with pip:: + + pip install u_boot_pylib + +or via setup.py:: + + ./setup.py install [--user] diff --git a/tools/u_boot_pylib/pyproject.toml b/tools/u_boot_pylib/pyproject.toml new file mode 100644 index 00000000000..3f33caf6f8d --- /dev/null +++ b/tools/u_boot_pylib/pyproject.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "u_boot_pylib" +version = "0.0.2" +authors = [ + { name="Simon Glass", email="sjg@chromium.org" }, +] +description = "U-Boot python library" +readme = "README.md" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)", + "Operating System :: OS Independent", +] + +[project.urls] +"Homepage" = "https://u-boot.readthedocs.io" +"Bug Tracker" = "https://source.denx.de/groups/u-boot/-/issues" -- cgit v1.3.1 From a545dc1db91bfe9ba04d39c27070cabdbafa6111 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 23 Feb 2023 18:18:07 -0700 Subject: patman: Avoid importing test_checkpatch before it is needed Tests are not packaged with patman so this file will not be accessible when installing with pip. Move the import later in the file, when we know the file is present. Signed-off-by: Simon Glass --- tools/patman/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/patman/__main__.py b/tools/patman/__main__.py index 30632559bb6..48ffbc8eadf 100755 --- a/tools/patman/__main__.py +++ b/tools/patman/__main__.py @@ -24,7 +24,6 @@ from patman import func_test from patman import gitutil from patman import project from patman import settings -from patman import test_checkpatch from u_boot_pylib import terminal from u_boot_pylib import test_util from u_boot_pylib import tools @@ -146,6 +145,7 @@ if not args.debug: # Run our meagre tests if args.cmd == 'test': from patman import func_test + from patman import test_checkpatch result = test_util.run_test_suites( 'patman', False, False, False, None, None, None, -- cgit v1.3.1 From 30eb11ae0483f95f85b483a44387163478bb14d8 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 23 Feb 2023 18:18:08 -0700 Subject: patman: Add support for building a patman PyPi package Create the necessary files to build this new package. Signed-off-by: Simon Glass --- Makefile | 1 + tools/patman/pyproject.toml | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 tools/patman/pyproject.toml (limited to 'tools') diff --git a/Makefile b/Makefile index 21d606b10e6..21e58cae57b 100644 --- a/Makefile +++ b/Makefile @@ -2282,6 +2282,7 @@ pip pip_test pip_release: _pip _pip: scripts/make_pip.sh u_boot_pylib ${PIP_ARGS} + scripts/make_pip.sh patman ${PIP_ARGS} help: @echo 'Cleaning targets:' diff --git a/tools/patman/pyproject.toml b/tools/patman/pyproject.toml new file mode 100644 index 00000000000..c5dc7c7e276 --- /dev/null +++ b/tools/patman/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "patch-manager" +version = "0.0.2" +authors = [ + { name="Simon Glass", email="sjg@chromium.org" }, +] +dependencies = ["u_boot_pylib"] +description = "Patman patch manager" +readme = "README.rst" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)", + "Operating System :: OS Independent", +] + +[project.urls] +"Homepage" = "https://u-boot.readthedocs.io/en/latest/develop/patman.html" +"Bug Tracker" = "https://source.denx.de/groups/u-boot/-/issues" + +[project.scripts] +patman = "patman.__main__:run_patman" + +[tool.setuptools.package-data] +patman = ["*.rst"] -- cgit v1.3.1 From 793aa1761929bc95fe88ef8d0f9747833d185f36 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 23 Feb 2023 18:18:09 -0700 Subject: buildman: Move the main code into a function Put this code into a function so it is easy for it be run when packaged. Signed-off-by: Simon Glass --- tools/buildman/main.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) (limited to 'tools') diff --git a/tools/buildman/main.py b/tools/buildman/main.py index 6076ba5d63d..5e1f68d8235 100755 --- a/tools/buildman/main.py +++ b/tools/buildman/main.py @@ -46,17 +46,22 @@ def RunTests(skip_net_tests, verboose, args): return (0 if result.wasSuccessful() else 1) -options, args = cmdline.ParseArgs() +def run_buildman(): + options, args = cmdline.ParseArgs() -if not options.debug: - sys.tracebacklimit = 0 + if not options.debug: + sys.tracebacklimit = 0 -# Run our meagre tests -if options.test: - RunTests(options.skip_net_tests, options.verbose, args) + # Run our meagre tests + if cmdline.HAS_TESTS and options.test: + RunTests(options.skip_net_tests, options.verbose, args) -# Build selected commits for selected boards -else: - bsettings.Setup(options.config_file) - ret_code = control.DoBuildman(options, args) - sys.exit(ret_code) + # Build selected commits for selected boards + else: + bsettings.Setup(options.config_file) + ret_code = control.DoBuildman(options, args) + sys.exit(ret_code) + + +if __name__ == "__main__": + run_buildman() -- cgit v1.3.1 From 5cfb73b5902d46f656657c765c61e84980d9989c Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 23 Feb 2023 18:18:10 -0700 Subject: buildman: Hide the test options unless test code is available It doesn't make much sense to expose tests when buildman is running outside of the U-Boot git checkout. Hide the option in this case Signed-off-by: Simon Glass --- tools/buildman/cmdline.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) (limited to 'tools') diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py index da7f1a99f6b..a9cda249572 100644 --- a/tools/buildman/cmdline.py +++ b/tools/buildman/cmdline.py @@ -3,6 +3,11 @@ # from optparse import OptionParser +import os +import pathlib + +BUILDMAN_DIR = pathlib.Path(__file__).parent +HAS_TESTS = os.path.exists(BUILDMAN_DIR / "test.py") def ParseArgs(): """Parse command line arguments from sys.argv[] @@ -105,12 +110,13 @@ def ParseArgs(): default=False, help='Show a build summary') parser.add_option('-S', '--show-sizes', action='store_true', default=False, help='Show image size variation in summary') - parser.add_option('--skip-net-tests', action='store_true', default=False, - help='Skip tests which need the network') parser.add_option('--step', type='int', default=1, help='Only build every n commits (0=just first and last)') - parser.add_option('-t', '--test', action='store_true', dest='test', - default=False, help='run tests') + if HAS_TESTS: + parser.add_option('--skip-net-tests', action='store_true', default=False, + help='Skip tests which need the network') + parser.add_option('-t', '--test', action='store_true', dest='test', + default=False, help='run tests') parser.add_option('-T', '--threads', type='int', default=None, help='Number of builder threads to use (0=single-thread)') -- cgit v1.3.1 From 8dd7be7e28cbee524e32f294c366403f2585f0b1 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 23 Feb 2023 18:18:11 -0700 Subject: buildman: Fix use of a type as a variable Using 'str' as a variable makes it impossible to use it as a type in the same function. Fix this by using a different name. Signed-off-by: Simon Glass --- tools/buildman/control.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'tools') diff --git a/tools/buildman/control.py b/tools/buildman/control.py index f765fe3653b..aacfb2fc0db 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -261,9 +261,9 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None, count += 1 # Build upstream commit also if not count: - str = ("No commits found to process in branch '%s': " + msg = ("No commits found to process in branch '%s': " "set branch's upstream or use -c flag" % options.branch) - sys.exit(col.build(col.RED, str)) + sys.exit(col.build(col.RED, msg)) if options.work_in_output: if len(selected) != 1: sys.exit(col.build(col.RED, -- cgit v1.3.1 From d85f7909f8fed1e06fe56e03c52d5ff6d8dfbdcf Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 23 Feb 2023 18:18:12 -0700 Subject: buildman: Use importlib to find the help Use this function so that the help can be found even when buildman is running from a package. Signed-off-by: Simon Glass --- tools/buildman/control.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'tools') diff --git a/tools/buildman/control.py b/tools/buildman/control.py index aacfb2fc0db..35f44c0cf3d 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -3,6 +3,7 @@ # import multiprocessing +import importlib.resources import os import shutil import subprocess @@ -152,9 +153,8 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None, global builder if options.full_help: - tools.print_full_help( - os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), - 'README.rst')) + with importlib.resources.path('buildman', 'README.rst') as readme: + tools.print_full_help(str(readme)) return 0 gitutil.setup() -- cgit v1.3.1 From 0de2ffe717766c16f66775abf95d623f15489e83 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 23 Feb 2023 18:18:13 -0700 Subject: buildman: Add support for building a buildman PyPi package Create the necessary files to build this new package. It is not actually clear whether this is useful, since buildman has no purpose outside U-Boot. It is included for completeness, since adding this later would be more trouble. Move the main program into a function so that it can easily be called by the PyPi-created script. Signed-off-by: Simon Glass --- Makefile | 1 + tools/buildman/pyproject.toml | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 tools/buildman/pyproject.toml (limited to 'tools') diff --git a/Makefile b/Makefile index 21e58cae57b..f4ffa9e14ba 100644 --- a/Makefile +++ b/Makefile @@ -2283,6 +2283,7 @@ pip pip_test pip_release: _pip _pip: scripts/make_pip.sh u_boot_pylib ${PIP_ARGS} scripts/make_pip.sh patman ${PIP_ARGS} + scripts/make_pip.sh buildman ${PIP_ARGS} help: @echo 'Cleaning targets:' diff --git a/tools/buildman/pyproject.toml b/tools/buildman/pyproject.toml new file mode 100644 index 00000000000..4d75e772ee1 --- /dev/null +++ b/tools/buildman/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "buildman" +version = "0.0.2" +authors = [ + { name="Simon Glass", email="sjg@chromium.org" }, +] +dependencies = ["u_boot_pylib", "patch-manager"] +description = "Buildman build tool for U-Boot" +readme = "README.rst" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)", + "Operating System :: OS Independent", +] + +[project.urls] +"Homepage" = "https://u-boot.readthedocs.io/en/latest/build/buildman.html" +"Bug Tracker" = "https://source.denx.de/groups/u-boot/-/issues" + +[project.scripts] +buildman = "buildman.main:run_buildman" + +[tool.setuptools.package-data] +buildman = ["*.rst"] -- cgit v1.3.1 From ab9272b8042db7e3f26b495635c20cbbe439ce42 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 23 Feb 2023 18:18:14 -0700 Subject: dtoc: Hide the test options unless test code is available It doesn't make much sense to expose tests when dtoc is running outside of the U-Boot git checkout. Hide the option in this case. Signed-off-by: Simon Glass --- tools/dtoc/main.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) (limited to 'tools') diff --git a/tools/dtoc/main.py b/tools/dtoc/main.py index fc9207d1b63..91521661f46 100755 --- a/tools/dtoc/main.py +++ b/tools/dtoc/main.py @@ -23,6 +23,7 @@ see doc/driver-model/of-plat.rst from argparse import ArgumentParser import os +import pathlib import sys # Bring in the patman libraries @@ -37,6 +38,9 @@ sys.path.insert(0, os.path.join(our_path, from dtoc import dtb_platdata from u_boot_pylib import test_util +DTOC_DIR = pathlib.Path(__file__).parent +HAVE_TESTS = (DTOC_DIR / 'test_dtoc.py').exists() + def run_tests(processes, args): """Run all the test we have for dtoc @@ -93,19 +97,22 @@ parser.add_argument('-p', '--phase', type=str, help='set phase of U-Boot this invocation is for (spl/tpl)') parser.add_argument('-P', '--processes', type=int, help='set number of processes to use for running tests') -parser.add_argument('-t', '--test', action='store_true', dest='test', - default=False, help='run tests') -parser.add_argument('-T', '--test-coverage', action='store_true', - default=False, help='run tests and check for 100%% coverage') +if HAVE_TESTS: + parser.add_argument('-t', '--test', action='store_true', dest='test', + default=False, help='run tests') + parser.add_argument( + '-T', '--test-coverage', action='store_true', + default=False, help='run tests and check for 100%% coverage') + parser.add_argument('files', nargs='*') args = parser.parse_args() # Run our meagre tests -if args.test: +if HAVE_TESTS and args.test: ret_code = run_tests(args.processes, args) sys.exit(ret_code) -elif args.test_coverage: +elif HAVE_TESTS and args.test_coverage: RunTestCoverage() else: -- cgit v1.3.1 From b3f5474077d60a1886f4ebb329387c108eca22d4 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 23 Feb 2023 18:18:15 -0700 Subject: dtoc: Move the main code into a function Put this code into a function so it is easy for it be run when packaged. Signed-off-by: Simon Glass --- tools/dtoc/main.py | 100 ++++++++++++++++++++++++++++------------------------- 1 file changed, 52 insertions(+), 48 deletions(-) (limited to 'tools') diff --git a/tools/dtoc/main.py b/tools/dtoc/main.py index 91521661f46..6c91450410e 100755 --- a/tools/dtoc/main.py +++ b/tools/dtoc/main.py @@ -65,58 +65,62 @@ def run_tests(processes, args): return (0 if result.wasSuccessful() else 1) -def RunTestCoverage(): +def RunTestCoverage(build_dir): """Run the tests and check that we get 100% coverage""" sys.argv = [sys.argv[0]] test_util.run_test_coverage('tools/dtoc/dtoc', '/main.py', ['tools/patman/*.py', 'tools/u_boot_pylib/*','*/fdt*', '*test*'], - args.build_dir) - - -if __name__ != '__main__': - sys.exit(1) - -epilog = '''Generate C code from devicetree files. See of-plat.rst for details''' - -parser = ArgumentParser(epilog=epilog) -parser.add_argument('-B', '--build-dir', type=str, default='b', - help='Directory containing the build output') -parser.add_argument('-c', '--c-output-dir', action='store', - help='Select output directory for C files') -parser.add_argument('-C', '--h-output-dir', action='store', - help='Select output directory for H files (defaults to --c-output-di)') -parser.add_argument('-d', '--dtb-file', action='store', - help='Specify the .dtb input file') -parser.add_argument('-i', '--instantiate', action='store_true', default=False, - help='Instantiate devices to avoid needing device_bind()') -parser.add_argument('--include-disabled', action='store_true', - help='Include disabled nodes') -parser.add_argument('-o', '--output', action='store', - help='Select output filename') -parser.add_argument('-p', '--phase', type=str, - help='set phase of U-Boot this invocation is for (spl/tpl)') -parser.add_argument('-P', '--processes', type=int, - help='set number of processes to use for running tests') -if HAVE_TESTS: - parser.add_argument('-t', '--test', action='store_true', dest='test', - default=False, help='run tests') - parser.add_argument( - '-T', '--test-coverage', action='store_true', - default=False, help='run tests and check for 100%% coverage') - -parser.add_argument('files', nargs='*') -args = parser.parse_args() + build_dir) -# Run our meagre tests -if HAVE_TESTS and args.test: - ret_code = run_tests(args.processes, args) - sys.exit(ret_code) -elif HAVE_TESTS and args.test_coverage: - RunTestCoverage() +def run_dtoc(): + epilog = 'Generate C code from devicetree files. See of-plat.rst for details' -else: - dtb_platdata.run_steps(args.files, args.dtb_file, args.include_disabled, - args.output, - [args.c_output_dir, args.h_output_dir], - args.phase, instantiate=args.instantiate) + parser = ArgumentParser(epilog=epilog) + parser.add_argument('-B', '--build-dir', type=str, default='b', + help='Directory containing the build output') + parser.add_argument('-c', '--c-output-dir', action='store', + help='Select output directory for C files') + parser.add_argument( + '-C', '--h-output-dir', action='store', + help='Select output directory for H files (defaults to --c-output-di)') + parser.add_argument('-d', '--dtb-file', action='store', + help='Specify the .dtb input file') + parser.add_argument( + '-i', '--instantiate', action='store_true', default=False, + help='Instantiate devices to avoid needing device_bind()') + parser.add_argument('--include-disabled', action='store_true', + help='Include disabled nodes') + parser.add_argument('-o', '--output', action='store', + help='Select output filename') + parser.add_argument( + '-p', '--phase', type=str, + help='set phase of U-Boot this invocation is for (spl/tpl)') + parser.add_argument('-P', '--processes', type=int, + help='set number of processes to use for running tests') + if HAVE_TESTS: + parser.add_argument('-t', '--test', action='store_true', dest='test', + default=False, help='run tests') + parser.add_argument( + '-T', '--test-coverage', action='store_true', + default=False, help='run tests and check for 100%% coverage') + parser.add_argument('files', nargs='*') + args = parser.parse_args() + + # Run our meagre tests + if HAVE_TESTS and args.test: + ret_code = run_tests(args.processes, args) + sys.exit(ret_code) + + elif HAVE_TESTS and args.test_coverage: + RunTestCoverage(args.build_dir) + + else: + dtb_platdata.run_steps(args.files, args.dtb_file, args.include_disabled, + args.output, + [args.c_output_dir, args.h_output_dir], + args.phase, instantiate=args.instantiate) + + +if __name__ == '__main__': + run_dtoc() -- cgit v1.3.1 From 1688d6ca0e0c665c909ec6cd30c2aa53445548f2 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 23 Feb 2023 18:18:16 -0700 Subject: dtoc: Use pathlib to find the test directory Update this so that the directory being used is declared at the top of the file. Use pathlib as it seems to be more modern. Signed-off-by: Simon Glass --- tools/dtoc/test_dtoc.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'tools') diff --git a/tools/dtoc/test_dtoc.py b/tools/dtoc/test_dtoc.py index 0f544f9f54a..597c93e8a87 100755 --- a/tools/dtoc/test_dtoc.py +++ b/tools/dtoc/test_dtoc.py @@ -13,6 +13,7 @@ import collections import copy import glob import os +import pathlib import struct import unittest @@ -28,7 +29,8 @@ from dtoc.src_scan import get_compat_name from u_boot_pylib import test_util from u_boot_pylib import tools -OUR_PATH = os.path.dirname(os.path.realpath(__file__)) +DTOC_DIR = pathlib.Path(__file__).parent +TEST_DATA_DIR = DTOC_DIR / 'test/' HEADER = '''/* @@ -91,7 +93,7 @@ def get_dtb_file(dts_fname, capture_stderr=False): Returns: str: Filename of compiled file in output directory """ - return fdt_util.EnsureCompiled(os.path.join(OUR_PATH, 'test', dts_fname), + return fdt_util.EnsureCompiled(str(TEST_DATA_DIR / dts_fname), capture_stderr=capture_stderr) -- cgit v1.3.1 From 77b3ccb89ae373d054533f5f4a4192a4ba020405 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 23 Feb 2023 18:18:17 -0700 Subject: dtoc: Add support for building a dtoc PyPi package Create the necessary files to build this new package. This is needed for binman. Move the main program into a function so that it can easily be called by the PyPi-created script. Signed-off-by: Simon Glass --- Makefile | 1 + tools/dtoc/README.rst | 15 +++++++++++++++ tools/dtoc/pyproject.toml | 26 ++++++++++++++++++++++++++ 3 files changed, 42 insertions(+) create mode 100644 tools/dtoc/README.rst create mode 100644 tools/dtoc/pyproject.toml (limited to 'tools') diff --git a/Makefile b/Makefile index f4ffa9e14ba..bf6abe5f6aa 100644 --- a/Makefile +++ b/Makefile @@ -2284,6 +2284,7 @@ _pip: scripts/make_pip.sh u_boot_pylib ${PIP_ARGS} scripts/make_pip.sh patman ${PIP_ARGS} scripts/make_pip.sh buildman ${PIP_ARGS} + scripts/make_pip.sh dtoc ${PIP_ARGS} help: @echo 'Cleaning targets:' diff --git a/tools/dtoc/README.rst b/tools/dtoc/README.rst new file mode 100644 index 00000000000..92b39759ed1 --- /dev/null +++ b/tools/dtoc/README.rst @@ -0,0 +1,15 @@ +.. SPDX-License-Identifier: GPL-2.0+ + +Devicetree-to-C generator +========================= + +This is a Python program and associated utilities, which supports converting +devicetree files into C code. It generates header files containing struct +definitions, as well as C files containing the data. It does not require any +modification of the devicetree files. + +Some high-level libraries are provided for working with devicetree. These may +be useful in other projects. + +This package also includes some U-Boot-specific features, such as creating +`struct udevice` and `struct uclass` entries for devicetree nodes. diff --git a/tools/dtoc/pyproject.toml b/tools/dtoc/pyproject.toml new file mode 100644 index 00000000000..77fe4da2158 --- /dev/null +++ b/tools/dtoc/pyproject.toml @@ -0,0 +1,26 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "dtoc" +version = "0.0.2" +authors = [ + { name="Simon Glass", email="sjg@chromium.org" }, +] +dependencies = ["pylibfdt", "u_boot_pylib"] +description = "Devicetree-to-C generator" +readme = "README.rst" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)", + "Operating System :: OS Independent", +] + +[project.urls] +"Homepage" = "https://u-boot.readthedocs.io/en/latest/develop/driver-model/of-plat.html" +"Bug Tracker" = "https://source.denx.de/groups/u-boot/-/issues" + +[project.scripts] +dtoc = "dtoc.main:run_dtoc" -- cgit v1.3.1 From 952a61adb4537843855e2c35a57646c25abc6128 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 23 Feb 2023 18:18:18 -0700 Subject: binman: Move the main code into a function Put this code into a function so it is easy for it be run when packaged. Signed-off-by: Simon Glass --- tools/binman/main.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'tools') diff --git a/tools/binman/main.py b/tools/binman/main.py index eef17b7b55f..92d2431aea7 100755 --- a/tools/binman/main.py +++ b/tools/binman/main.py @@ -85,7 +85,7 @@ def RunTests(debug, verbosity, processes, test_preserve_dirs, args, toolpath): return (0 if result.wasSuccessful() else 1) -def RunTestCoverage(toolpath): +def RunTestCoverage(toolpath, build_dir): """Run the tests and check that we get 100% coverage""" glob_list = control.GetEntryModules(False) all_set = set([os.path.splitext(os.path.basename(item))[0] @@ -97,7 +97,7 @@ def RunTestCoverage(toolpath): test_util.run_test_coverage('tools/binman/binman', None, ['*test*', '*main.py', 'tools/patman/*', 'tools/dtoc/*', 'tools/u_boot_pylib/*'], - args.build_dir, all_set, extra_args or None) + build_dir, all_set, extra_args or None) def RunBinman(args): """Main entry point to binman once arguments are parsed @@ -117,7 +117,7 @@ def RunBinman(args): if args.cmd == 'test': if args.test_coverage: - RunTestCoverage(args.toolpath) + RunTestCoverage(args.toolpath, args.build_dir) else: ret_code = RunTests(args.debug, args.verbosity, args.processes, args.test_preserve_dirs, args.tests, @@ -141,8 +141,12 @@ def RunBinman(args): return ret_code -if __name__ == "__main__": +def start_binman(): args = cmdline.ParseArgs(sys.argv[1:]) ret_code = RunBinman(args) sys.exit(ret_code) + + +if __name__ == "__main__": + start_binman() -- cgit v1.3.1 From 94bb5cd2f9960baa724976a44303067dd4654b2e Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 23 Feb 2023 18:18:19 -0700 Subject: binman: Hide the 'test' command unless test code is available It doesn't make much sense to expose tests when dtoc is running outside of the U-Boot git checkout. Hide the option in this case. Fix a long line while we are here. Signed-off-by: Simon Glass --- tools/binman/cmdline.py | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) (limited to 'tools') diff --git a/tools/binman/cmdline.py b/tools/binman/cmdline.py index 4eed3073dce..54926adb09f 100644 --- a/tools/binman/cmdline.py +++ b/tools/binman/cmdline.py @@ -9,6 +9,11 @@ import argparse from argparse import ArgumentParser import os from binman import state +import os +import pathlib + +BINMAN_DIR = pathlib.Path(__file__).parent +HAS_TESTS = (BINMAN_DIR / "ftest.py").exists() def make_extract_parser(subparsers): """make_extract_parser: Make a subparser for the 'extract' command @@ -167,23 +172,26 @@ controlled by a description in the board device tree.''' replace_parser.add_argument('paths', type=str, nargs='*', help='Paths within file to replace (wildcard)') - test_parser = subparsers.add_parser('test', help='Run tests') - test_parser.add_argument('-P', '--processes', type=int, - help='set number of processes to use for running tests') - test_parser.add_argument('-T', '--test-coverage', action='store_true', - default=False, help='run tests and check for 100%% coverage') - test_parser.add_argument('-X', '--test-preserve-dirs', action='store_true', - help='Preserve and display test-created input directories; also ' - 'preserve the output directory if a single test is run (pass test ' - 'name at the end of the command line') - test_parser.add_argument('tests', nargs='*', - help='Test names to run (omit for all)') + if HAS_TESTS: + test_parser = subparsers.add_parser('test', help='Run tests') + test_parser.add_argument('-P', '--processes', type=int, + help='set number of processes to use for running tests') + test_parser.add_argument('-T', '--test-coverage', action='store_true', + default=False, help='run tests and check for 100%% coverage') + test_parser.add_argument( + '-X', '--test-preserve-dirs', action='store_true', + help='Preserve and display test-created input directories; also ' + 'preserve the output directory if a single test is run (pass ' + 'test name at the end of the command line') + test_parser.add_argument('tests', nargs='*', + help='Test names to run (omit for all)') tool_parser = subparsers.add_parser('tool', help='Check bintools') tool_parser.add_argument('-l', '--list', action='store_true', help='List all known bintools') - tool_parser.add_argument('-f', '--fetch', action='store_true', - help='fetch a bintool from a known location (or: all/missing)') + tool_parser.add_argument( + '-f', '--fetch', action='store_true', + help='fetch a bintool from a known location (or: all/missing)') tool_parser.add_argument('bintools', type=str, nargs='*') return parser.parse_args(argv) -- cgit v1.3.1 From 8de6adbf4a92ae49af7fb48bbb6d2cf4ae0b8f60 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 23 Feb 2023 18:18:20 -0700 Subject: binman: Use importlib to find the help Use this function so that the help can be found even when binman is running from a package. Signed-off-by: Simon Glass --- tools/binman/control.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'tools') diff --git a/tools/binman/control.py b/tools/binman/control.py index 9b183eda736..2900538ffc0 100644 --- a/tools/binman/control.py +++ b/tools/binman/control.py @@ -7,6 +7,7 @@ from collections import OrderedDict import glob +import importlib.resources import os import pkg_resources import re @@ -641,9 +642,8 @@ def Binman(args): global state if args.full_help: - tools.print_full_help( - os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), 'README.rst') - ) + with importlib.resources.path('binman', 'README.rst') as readme: + tools.print_full_help(str(readme)) return 0 # Put these here so that we can import this module without libfdt -- cgit v1.3.1 From ada5e2f97814ea778eabdb5ab56a0505ed0d3f5c Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 23 Feb 2023 18:18:21 -0700 Subject: binman: Add support for building a binmanu PyPi package Create the necessary files to build this new package. It is not actually clear whether this is useful, since buildman has no purpose outside U-Boot. Move the main program into a function so that it can easily be called by the PyPi-created script. Signed-off-by: Simon Glass --- Makefile | 1 + tools/binman/pyproject.toml | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 tools/binman/pyproject.toml (limited to 'tools') diff --git a/Makefile b/Makefile index bf6abe5f6aa..c00683bdc95 100644 --- a/Makefile +++ b/Makefile @@ -2285,6 +2285,7 @@ _pip: scripts/make_pip.sh patman ${PIP_ARGS} scripts/make_pip.sh buildman ${PIP_ARGS} scripts/make_pip.sh dtoc ${PIP_ARGS} + scripts/make_pip.sh binman ${PIP_ARGS} help: @echo 'Cleaning targets:' diff --git a/tools/binman/pyproject.toml b/tools/binman/pyproject.toml new file mode 100644 index 00000000000..b4b54fbaee6 --- /dev/null +++ b/tools/binman/pyproject.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "binary-manager" +version = "0.0.2" +authors = [ + { name="Simon Glass", email="sjg@chromium.org" }, +] +dependencies = ["pylibfdt", "u_boot_pylib", "dtoc"] +description = "Binman firmware-packaging tool" +readme = "README.rst" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)", + "Operating System :: OS Independent", +] + +[project.urls] +"Homepage" = "https://u-boot.readthedocs.io/en/latest/develop/package/index.html" +"Bug Tracker" = "https://source.denx.de/groups/u-boot/-/issues" + +[project.scripts] +binman = "binman.main:start_binman" + +[tool.setuptools.package-data] +patman = ["*.rst"] -- cgit v1.3.1 From 6608acb29d25f354d4c9574b126616c582fcc1bc Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 23 Feb 2023 18:18:23 -0700 Subject: doc: Add notes on how to install patman and binman These can be installed with 'pip' now. Add the details for those who are interested. Signed-off-by: Simon Glass --- tools/binman/binman.rst | 13 +++++++++++++ tools/patman/patman.rst | 12 ++++++++++++ 2 files changed, 25 insertions(+) (limited to 'tools') diff --git a/tools/binman/binman.rst b/tools/binman/binman.rst index 3ac29ee3568..a729b5322f1 100644 --- a/tools/binman/binman.rst +++ b/tools/binman/binman.rst @@ -95,6 +95,19 @@ Binman uses the following terms: - binary - an input binary that goes into the image +Installation +------------ + +You can install binman using:: + + pip install binary-manager + +The name is chosen since binman conflicts with an existing package. + +If you are using binman within the U-Boot tree, it may be easiest to add a +symlink from your local `~/.bin` directory to `/path/to/tools/binman/binman`. + + Relationship to FIT ------------------- diff --git a/tools/patman/patman.rst b/tools/patman/patman.rst index 6113962fb4f..038b651ee87 100644 --- a/tools/patman/patman.rst +++ b/tools/patman/patman.rst @@ -41,6 +41,18 @@ In Linux and U-Boot this will also call get_maintainer.pl on each of your patches automatically (unless you use -m to disable this). +Installation +------------ + +You can install patman using:: + + pip install patch-manager + +The name is chosen since patman conflicts with an existing package. + +If you are using patman within the U-Boot tree, it may be easiest to add a +symlink from your local `~/.bin` directory to `/path/to/tools/patman/patman`. + How to use this tool -------------------- -- cgit v1.3.1 From 98bc0b48bdf02694a75ef02be09118777d8a8c1e Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Wed, 8 Mar 2023 10:52:52 -0800 Subject: patman: Drop an incorrect comment about git am Patman does not do this anymore, as of this commit: 7428dc14b0f ("patman: Remove the -a option") Drop the comment. Signed-off-by: Simon Glass Reviewed-by: Douglas Anderson --- tools/patman/control.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/patman/control.py b/tools/patman/control.py index d1bcea0c9a7..916ddf8fcff 100644 --- a/tools/patman/control.py +++ b/tools/patman/control.py @@ -85,7 +85,7 @@ def check_patches(series, patch_files, run_checkpatch, verbose, use_tree): # Do a few checks on the series series.DoChecks() - # Check the patches, and run them through 'git am' just to be sure + # Check the patches if run_checkpatch: ok = checkpatch.check_patches(verbose, patch_files, use_tree) else: -- cgit v1.3.1 From c524cd61397f11f106ea1e43cb31f8ce2bae2ebb Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Wed, 8 Mar 2023 10:52:53 -0800 Subject: patman: Refactor MakeCcFile() into two functions This function is quite long. Moving the handling of a commit into a separate function. This will make it easier to do the work in parallel. Update function comments while we are here. Signed-off-by: Simon Glass Reviewed-by: Douglas Anderson --- tools/patman/series.py | 81 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 57 insertions(+), 24 deletions(-) (limited to 'tools') diff --git a/tools/patman/series.py b/tools/patman/series.py index 88417acb434..e6f61e1fe23 100644 --- a/tools/patman/series.py +++ b/tools/patman/series.py @@ -234,6 +234,49 @@ class Series(dict): str = 'Change log exists, but no version is set' print(col.build(col.RED, str)) + def GetCcForCommit(self, commit, process_tags, warn_on_error, + add_maintainers, limit, get_maintainer_script, + all_skips): + """Get the email CCs to use with a particular commit + + Uses subject tags and get_maintainers.pl script to find people to cc + on a patch + + Args: + commit (Commit): Commit to process + process_tags (bool): Process tags as if they were aliases + warn_on_error (bool): True to print a warning when an alias fails to + match, False to ignore it. + add_maintainers (bool or list of str): Either: + True/False to call the get_maintainers to CC maintainers + List of maintainers to include (for testing) + limit (int): Limit the length of the Cc list (None if no limit) + get_maintainer_script (str): The file name of the get_maintainer.pl + script (or compatible). + all_skips (set of str): Updated to include the set of bouncing email + addresses that were dropped from the output. This is essentially + a return value from this function. + + Returns: + list of str: List of email addresses to cc + """ + cc = [] + if process_tags: + cc += gitutil.build_email_list(commit.tags, + warn_on_error=warn_on_error) + cc += gitutil.build_email_list(commit.cc_list, + warn_on_error=warn_on_error) + if type(add_maintainers) == type(cc): + cc += add_maintainers + elif add_maintainers: + cc += get_maintainer.get_maintainer(get_maintainer_script, + commit.patch) + all_skips |= set(cc) & set(settings.bounces) + cc = list(set(cc) - set(settings.bounces)) + if limit is not None: + cc = cc[:limit] + return cc + def MakeCcFile(self, process_tags, cover_fname, warn_on_error, add_maintainers, limit, get_maintainer_script): """Make a cc file for us to use for per-commit Cc automation @@ -241,15 +284,15 @@ class Series(dict): Also stores in self._generated_cc to make ShowActions() faster. Args: - process_tags: Process tags as if they were aliases - cover_fname: If non-None the name of the cover letter. - warn_on_error: True to print a warning when an alias fails to match, - False to ignore it. - add_maintainers: Either: + process_tags (bool): Process tags as if they were aliases + cover_fname (str): If non-None the name of the cover letter. + warn_on_error (bool): True to print a warning when an alias fails to + match, False to ignore it. + add_maintainers (bool or list of str): Either: True/False to call the get_maintainers to CC maintainers List of maintainers to include (for testing) - limit: Limit the length of the Cc list (None if no limit) - get_maintainer_script: The file name of the get_maintainer.pl + limit (int): Limit the length of the Cc list (None if no limit) + get_maintainer_script (str): The file name of the get_maintainer.pl script (or compatible). Return: Filename of temp file created @@ -259,28 +302,18 @@ class Series(dict): fname = '/tmp/patman.%d' % os.getpid() fd = open(fname, 'w', encoding='utf-8') all_ccs = [] + all_skips = set() for commit in self.commits: - cc = [] - if process_tags: - cc += gitutil.build_email_list(commit.tags, - warn_on_error=warn_on_error) - cc += gitutil.build_email_list(commit.cc_list, - warn_on_error=warn_on_error) - if type(add_maintainers) == type(cc): - cc += add_maintainers - elif add_maintainers: - - cc += get_maintainer.get_maintainer(get_maintainer_script, - commit.patch) - for x in set(cc) & set(settings.bounces): - print(col.build(col.YELLOW, 'Skipping "%s"' % x)) - cc = list(set(cc) - set(settings.bounces)) - if limit is not None: - cc = cc[:limit] + cc = self.GetCcForCommit(commit, process_tags, warn_on_error, + add_maintainers, limit, + get_maintainer_script, all_skips) all_ccs += cc print(commit.patch, '\0'.join(sorted(set(cc))), file=fd) self._generated_cc[commit.patch] = cc + for x in sorted(all_skips): + print(col.build(col.YELLOW, f'Skipping "{x}"')) + if cover_fname: cover_cc = gitutil.build_email_list(self.get('cover_cc', '')) cover_cc = list(set(cover_cc + all_ccs)) -- cgit v1.3.1 From 27409e35d5fa56f1b51b6e0704081133e3881058 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Wed, 8 Mar 2023 10:52:54 -0800 Subject: patman: Run get_maintainer.pl in parallel This script can take ages on some series. Try to limit the time by using threads. If a few stubborn patches remain, show progress so the user has some idea what is going on. Signed-off-by: Simon Glass Reviewed-by: Douglas Anderson --- tools/patman/func_test.py | 2 ++ tools/patman/series.py | 33 ++++++++++++++++++++++++++++++--- 2 files changed, 32 insertions(+), 3 deletions(-) (limited to 'tools') diff --git a/tools/patman/func_test.py b/tools/patman/func_test.py index 8c2dfbe4528..42ac4ed77b7 100644 --- a/tools/patman/func_test.py +++ b/tools/patman/func_test.py @@ -240,6 +240,8 @@ class TestFunctional(unittest.TestCase): self.assertEqual('Change log missing for v3', next(lines)) self.assertEqual('Change log for unknown version v4', next(lines)) self.assertEqual("Alias 'pci' not found", next(lines)) + while next(lines) != 'Cc processing complete': + pass self.assertIn('Dry run', next(lines)) self.assertEqual('', next(lines)) self.assertIn('Send a total of %d patches' % count, next(lines)) diff --git a/tools/patman/series.py b/tools/patman/series.py index e6f61e1fe23..6866e1dbd08 100644 --- a/tools/patman/series.py +++ b/tools/patman/series.py @@ -5,8 +5,11 @@ from __future__ import print_function import collections +import concurrent.futures import itertools import os +import sys +import time from patman import get_maintainer from patman import gitutil @@ -303,10 +306,34 @@ class Series(dict): fd = open(fname, 'w', encoding='utf-8') all_ccs = [] all_skips = set() + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor: + for i, commit in enumerate(self.commits): + commit.seq = i + commit.future = executor.submit( + self.GetCcForCommit, commit, process_tags, warn_on_error, + add_maintainers, limit, get_maintainer_script, all_skips) + + # Show progress any commits that are taking forever + lastlen = 0 + while True: + left = [commit for commit in self.commits + if not commit.future.done()] + if not left: + break + names = ', '.join(f'{c.seq + 1}:{c.subject}' + for c in left[:2]) + out = f'\r{len(left)} remaining: {names}'[:79] + spaces = ' ' * (lastlen - len(out)) + if lastlen: # Don't print anything the first time + print(out, spaces, end='') + sys.stdout.flush() + lastlen = len(out) + time.sleep(.25) + print(f'\rdone{" " * lastlen}\r', end='') + print('Cc processing complete') + for commit in self.commits: - cc = self.GetCcForCommit(commit, process_tags, warn_on_error, - add_maintainers, limit, - get_maintainer_script, all_skips) + cc = commit.future.result() all_ccs += cc print(commit.patch, '\0'.join(sorted(set(cc))), file=fd) self._generated_cc[commit.patch] = cc -- cgit v1.3.1 From 00d54ae8f4ea17af90dee294f326a156a00cb4ba Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Wed, 8 Mar 2023 10:52:55 -0800 Subject: patman: Check patches in parallel For large series this can take a while. Run checkpatch in parallel to try to reduce the time. The checkpatch information is still reported in sequential order, so a very slow patch at the start can still slow things down. But overall this gives good results. Signed-off-by: Simon Glass Reviewed-by: Douglas Anderson --- tools/patman/checkpatch.py | 46 ++++++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 20 deletions(-) (limited to 'tools') diff --git a/tools/patman/checkpatch.py b/tools/patman/checkpatch.py index c1dec323f36..e03cac115e4 100644 --- a/tools/patman/checkpatch.py +++ b/tools/patman/checkpatch.py @@ -3,6 +3,7 @@ # import collections +import concurrent.futures import os import re import sys @@ -244,26 +245,31 @@ def check_patches(verbose, args, use_tree): error_count, warning_count, check_count = 0, 0, 0 col = terminal.Color() - for fname in args: - result = check_patch(fname, verbose, use_tree=use_tree) - if not result.ok: - error_count += result.errors - warning_count += result.warnings - check_count += result.checks - print('%d errors, %d warnings, %d checks for %s:' % (result.errors, - result.warnings, result.checks, col.build(col.BLUE, fname))) - if (len(result.problems) != result.errors + result.warnings + - result.checks): - print("Internal error: some problems lost") - # Python seems to get confused by this - # pylint: disable=E1133 - for item in result.problems: - sys.stderr.write( - get_warning_msg(col, item.get('type', ''), - item.get('file', ''), - item.get('line', 0), item.get('msg', 'message'))) - print - #print(stdout) + with concurrent.futures.ThreadPoolExecutor(max_workers=16) as executor: + futures = [] + for fname in args: + f = executor.submit(check_patch, fname, verbose, use_tree=use_tree) + futures.append(f) + + for fname, f in zip(args, futures): + result = f.result() + if not result.ok: + error_count += result.errors + warning_count += result.warnings + check_count += result.checks + print('%d errors, %d warnings, %d checks for %s:' % (result.errors, + result.warnings, result.checks, col.build(col.BLUE, fname))) + if (len(result.problems) != result.errors + result.warnings + + result.checks): + print("Internal error: some problems lost") + # Python seems to get confused by this + # pylint: disable=E1133 + for item in result.problems: + sys.stderr.write( + get_warning_msg(col, item.get('type', ''), + item.get('file', ''), + item.get('line', 0), item.get('msg', 'message'))) + print if error_count or warning_count or check_count: str = 'checkpatch.pl found %d error(s), %d warning(s), %d checks(s)' color = col.GREEN -- cgit v1.3.1 From bd0a548ad4a155fec29473d4cc8e135832926973 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 2 Mar 2023 06:11:44 -0700 Subject: buildman: Correct CROSS_COMPILE output for sandbox At present, 'buildman -A sandbox' adds the path containing the toolchain at present. We can assume that this is in the path and we don't want to set CROSS_COMPILE=/bin/ so change this to align with what MakeEnvironment() does. Signed-off-by: Simon Glass --- tools/buildman/toolchain.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'tools') diff --git a/tools/buildman/toolchain.py b/tools/buildman/toolchain.py index 688f2e26872..8f9130bdcdf 100644 --- a/tools/buildman/toolchain.py +++ b/tools/buildman/toolchain.py @@ -156,9 +156,8 @@ class Toolchain: Returns: Value of that environment variable or arguments """ - wrapper = self.GetWrapper() if which == VAR_CROSS_COMPILE: - return wrapper + os.path.join(self.path, self.cross) + return self.GetWrapper() + self.cross elif which == VAR_PATH: return self.path elif which == VAR_ARCH: -- cgit v1.3.1 From e00197f92d8485a1285903227f6ee5058ee423df Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 2 Mar 2023 17:02:42 -0700 Subject: binman: Allow preserving the output dir when replacing Add these flags for the 'replace' subcommand too, to aid debugging. Signed-off-by: Simon Glass 44 2023 -0700 --- tools/binman/binman.rst | 6 ++++++ tools/binman/cmdline.py | 16 ++++++++++------ tools/binman/control.py | 5 ++++- 3 files changed, 20 insertions(+), 7 deletions(-) (limited to 'tools') diff --git a/tools/binman/binman.rst b/tools/binman/binman.rst index a729b5322f1..86a4b95e57d 100644 --- a/tools/binman/binman.rst +++ b/tools/binman/binman.rst @@ -1697,6 +1697,12 @@ Options: -m, --map Output a map file for the updated image +-O OUTDIR, --outdir OUTDIR + Path to directory to use for intermediate and output files + +-p, --preserve + Preserve temporary output directory even if option -O is not given + This replaces one or more entries in an existing image. See `Replacing files in an image`_. diff --git a/tools/binman/cmdline.py b/tools/binman/cmdline.py index 54926adb09f..1b7bbe80cda 100644 --- a/tools/binman/cmdline.py +++ b/tools/binman/cmdline.py @@ -73,6 +73,14 @@ def ParseArgs(argv): options provides access to the options (e.g. option.debug) args is a list of string arguments """ + def _AddPreserve(pars): + pars.add_argument('-O', '--outdir', type=str, + action='store', help='Path to directory to use for intermediate ' + 'and output files') + pars.add_argument('-p', '--preserve', action='store_true',\ + help='Preserve temporary output directory even if option -O is not ' + 'given') + if '-H' in argv: argv.append('build') @@ -127,12 +135,7 @@ controlled by a description in the board device tree.''' build_parser.add_argument('-n', '--no-expanded', action='store_true', help="Don't use 'expanded' versions of entries where available; " "normally 'u-boot' becomes 'u-boot-expanded', for example") - build_parser.add_argument('-O', '--outdir', type=str, - action='store', help='Path to directory to use for intermediate and ' - 'output files') - build_parser.add_argument('-p', '--preserve', action='store_true',\ - help='Preserve temporary output directory even if option -O is not ' - 'given') + _AddPreserve(build_parser) build_parser.add_argument('-u', '--update-fdt', action='store_true', default=False, help='Update the binman node with offset/size info') build_parser.add_argument('--update-fdt-in-elf', type=str, @@ -169,6 +172,7 @@ controlled by a description in the board device tree.''' help='Path to directory to use for input files') replace_parser.add_argument('-m', '--map', action='store_true', default=False, help='Output a map file for the updated image') + _AddPreserve(replace_parser) replace_parser.add_argument('paths', type=str, nargs='*', help='Paths within file to replace (wildcard)') diff --git a/tools/binman/control.py b/tools/binman/control.py index 2900538ffc0..bd40f074c87 100644 --- a/tools/binman/control.py +++ b/tools/binman/control.py @@ -661,7 +661,10 @@ def Binman(args): if args.cmd in ['ls', 'extract', 'replace', 'tool']: try: tout.init(args.verbosity) - tools.prepare_output_dir(None) + if args.cmd == 'replace': + tools.prepare_output_dir(args.outdir, args.preserve) + else: + tools.prepare_output_dir(None) if args.cmd == 'ls': ListEntries(args.image, args.paths) -- cgit v1.3.1 From 033828cf34e11776913298385efac406de89dd08 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 2 Mar 2023 17:02:43 -0700 Subject: binman: Handle missing bintools correctly in fit At present these are handled as if they are allowed to be missing, but this is only true if the -M flag is provided. Fix this and add a test. Signed-off-by: Simon Glass --- tools/binman/etype/fit.py | 2 ++ tools/binman/ftest.py | 10 +++++++++- 2 files changed, 11 insertions(+), 1 deletion(-) (limited to 'tools') diff --git a/tools/binman/etype/fit.py b/tools/binman/etype/fit.py index 9def4433618..39aa792b10b 100644 --- a/tools/binman/etype/fit.py +++ b/tools/binman/etype/fit.py @@ -453,6 +453,8 @@ class Entry_fit(Entry_section): args.update({'align': fdt_util.fdt32_to_cpu(align.value)}) if self.mkimage.run(reset_timestamp=True, output_fname=output_fname, **args) is None: + if not self.GetAllowMissing(): + self.Raise("Missing tool: 'mkimage'") # Bintool is missing; just use empty data as the output self.record_missing_bintool(self.mkimage) return tools.get_bytes(0, 1024) diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 59085804c24..934c8dd26f6 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -3999,9 +3999,17 @@ class TestFunctional(unittest.TestCase): self.assertEqual(expected, data[image_pos:image_pos+size]) def testFitMissing(self): + """Test that binman complains if mkimage is missing""" + with self.assertRaises(ValueError) as e: + self._DoTestFile('162_fit_external.dts', + force_missing_bintools='mkimage') + self.assertIn("Node '/binman/fit': Missing tool: 'mkimage'", + str(e.exception)) + + def testFitMissingOK(self): """Test that binman still produces a FIT image if mkimage is missing""" with test_util.capture_sys_output() as (_, stderr): - self._DoTestFile('162_fit_external.dts', + self._DoTestFile('162_fit_external.dts', allow_missing=True, force_missing_bintools='mkimage') err = stderr.getvalue() self.assertRegex(err, "Image 'image'.*missing bintools.*: mkimage") -- cgit v1.3.1 From 7caa372a5e41cadd3903165fb26a4b1e0268edbc Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 2 Mar 2023 17:02:44 -0700 Subject: binman: Support updating section contents Implement this feature since it is useful for updating FITs within an image. Signed-off-by: Simon Glass --- tools/binman/binman.rst | 16 +++ tools/binman/control.py | 2 + tools/binman/entry.py | 22 ++++ tools/binman/etype/atf_fip.py | 2 +- tools/binman/etype/cbfs.py | 2 +- tools/binman/etype/fit.py | 5 + tools/binman/etype/section.py | 30 ++++-- tools/binman/ftest.py | 137 ++++++++++++++++++++++++- tools/binman/test/277_replace_fit_sibling.dts | 61 +++++++++++ tools/binman/test/278_replace_section_deep.dts | 25 +++++ 10 files changed, 287 insertions(+), 15 deletions(-) create mode 100644 tools/binman/test/277_replace_fit_sibling.dts create mode 100644 tools/binman/test/278_replace_section_deep.dts (limited to 'tools') diff --git a/tools/binman/binman.rst b/tools/binman/binman.rst index 86a4b95e57d..e65fbff783e 100644 --- a/tools/binman/binman.rst +++ b/tools/binman/binman.rst @@ -1347,6 +1347,22 @@ You can also replace just a selection of entries:: $ binman replace -i image.bin "*u-boot*" -I indir +It is possible to replace whole sections as well, but in that case any +information about entries within the section may become outdated. This is +because Binman cannot know whether things have moved around or resized within +the section, once you have updated its data. + +Technical note: With 'allow-repack', Binman writes information about the +original offset and size properties of each entry, if any were specified, in +the 'orig-offset' and 'orig-size' properties. This allows Binman to distinguish +between an entry which ended up being packed at an offset (or assigned a size) +and an entry which had a particular offset / size requested in the Binman +configuration. Where are particular offset / size was requested, this is treated +as set in stone, so Binman will ensure it doesn't change. Without this feature, +repacking an entry might cause it to disobey the original constraints provided +when it was created. + + Repacking an image involves .. _`BinmanLogging`: diff --git a/tools/binman/control.py b/tools/binman/control.py index bd40f074c87..2f2b4893b7e 100644 --- a/tools/binman/control.py +++ b/tools/binman/control.py @@ -403,6 +403,8 @@ def ReplaceEntries(image_fname, input_fname, indir, entry_paths, image_fname = os.path.abspath(image_fname) image = Image.FromFile(image_fname) + image.mark_build_done() + # Replace an entry from a single file, as a special case if input_fname: if not entry_paths: diff --git a/tools/binman/entry.py b/tools/binman/entry.py index f732d40c37c..b10a43333ef 100644 --- a/tools/binman/entry.py +++ b/tools/binman/entry.py @@ -104,6 +104,10 @@ class Entry(object): firmware. This means that it will not be changed by the update. This is just a signal: enforcement of this is up to the updater. This flag does not automatically propagate down to child entries. + build_done (bool): Indicates that the entry data has been built and does + not need to be done again. This is only used with 'binman replace', + to stop sections from being rebuilt if their entries have not been + replaced """ fake_dir = None @@ -153,6 +157,7 @@ class Entry(object): self.elf_base_sym = None self.offset_from_elf = None self.preserve = False + self.build_done = False @staticmethod def FindEntryClass(etype, expanded): @@ -1013,6 +1018,7 @@ features to produce new behaviours. else: self.contents_size = self.pre_reset_size ok = self.ProcessContentsUpdate(data) + self.build_done = False self.Detail('WriteData: size=%x, ok=%s' % (len(data), ok)) section_ok = self.section.WriteChildData(self) return ok and section_ok @@ -1034,6 +1040,14 @@ features to produce new behaviours. True if the section could be updated successfully, False if the data is such that the section could not update """ + self.build_done = False + entry = self.section + + # Now we must rebuild all sections above this one + while entry and entry != entry.section: + self.build_done = False + entry = entry.section + return True def GetSiblingOrder(self): @@ -1356,3 +1370,11 @@ features to produce new behaviours. val = elf.GetSymbolOffset(entry.elf_fname, sym_name, entry.elf_base_sym) return val + offset + + def mark_build_done(self): + """Mark an entry as already built""" + self.build_done = True + entries = self.GetEntries() + if entries: + for entry in entries.values(): + entry.mark_build_done() diff --git a/tools/binman/etype/atf_fip.py b/tools/binman/etype/atf_fip.py index d5b862040b4..73a3f85b9f4 100644 --- a/tools/binman/etype/atf_fip.py +++ b/tools/binman/etype/atf_fip.py @@ -270,4 +270,4 @@ class Entry_atf_fip(Entry_section): # Recreate the data structure, leaving the data for this child alone, # so that child.data is used to pack into the FIP. self.ObtainContents(skip_entry=child) - return True + return super().WriteChildData(child) diff --git a/tools/binman/etype/cbfs.py b/tools/binman/etype/cbfs.py index 832f8d038f0..575aa624f6c 100644 --- a/tools/binman/etype/cbfs.py +++ b/tools/binman/etype/cbfs.py @@ -295,7 +295,7 @@ class Entry_cbfs(Entry): # Recreate the data structure, leaving the data for this child alone, # so that child.data is used to pack into the FIP. self.ObtainContents(skip_entry=child) - return True + return super().WriteChildData(child) def AddBintools(self, btools): super().AddBintools(btools) diff --git a/tools/binman/etype/fit.py b/tools/binman/etype/fit.py index 39aa792b10b..03fe88e7a6c 100644 --- a/tools/binman/etype/fit.py +++ b/tools/binman/etype/fit.py @@ -777,6 +777,8 @@ class Entry_fit(Entry_section): Args: image_pos (int): Position of this entry in the image """ + if self.build_done: + return super().SetImagePos(image_pos) # If mkimage is missing we'll have empty data, @@ -830,3 +832,6 @@ class Entry_fit(Entry_section): # missing for entry in self._priv_entries.values(): entry.CheckMissing(missing_list) + + def CheckEntries(self): + pass diff --git a/tools/binman/etype/section.py b/tools/binman/etype/section.py index eb733672855..c36edd13508 100644 --- a/tools/binman/etype/section.py +++ b/tools/binman/etype/section.py @@ -397,10 +397,13 @@ class Entry_section(Entry): This excludes any padding. If the section is compressed, the compressed data is returned """ - data = self.BuildSectionData(required) - if data is None: - return None - self.SetContents(data) + if not self.build_done: + data = self.BuildSectionData(required) + if data is None: + return None + self.SetContents(data) + else: + data = self.data if self._filename: tools.write_file(tools.get_output_filename(self._filename), data) return data @@ -427,8 +430,11 @@ class Entry_section(Entry): self._SortEntries() self._extend_entries() - data = self.BuildSectionData(True) - self.SetContents(data) + if self.build_done: + self.size = None + else: + data = self.BuildSectionData(True) + self.SetContents(data) self.CheckSize() @@ -810,6 +816,9 @@ class Entry_section(Entry): def LoadData(self, decomp=True): for entry in self._entries.values(): entry.LoadData(decomp) + data = self.ReadData(decomp) + self.contents_size = len(data) + self.ProcessContentsUpdate(data) self.Detail('Loaded data') def GetImage(self): @@ -866,10 +875,15 @@ class Entry_section(Entry): return data def WriteData(self, data, decomp=True): - self.Raise("Replacing sections is not implemented yet") + ok = super().WriteData(data, decomp) + + # The section contents are now fixed and cannot be rebuilt from the + # containing entries. + self.mark_build_done() + return ok def WriteChildData(self, child): - return True + return super().WriteChildData(child) def SetAllowMissing(self, allow_missing): """Set whether a section allows missing external blobs diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 934c8dd26f6..76445969201 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -5819,13 +5819,61 @@ fdt fdtmap Extract the devicetree blob from the fdtmap self.assertEqual(expected_fdtmap, fdtmap) def testReplaceSectionSimple(self): - """Test replacing a simple section with arbitrary data""" + """Test replacing a simple section with same-sized data""" new_data = b'w' * len(COMPRESS_DATA + U_BOOT_DATA) - with self.assertRaises(ValueError) as exc: - self._RunReplaceCmd('section', new_data, - dts='241_replace_section_simple.dts') + data, expected_fdtmap, image = self._RunReplaceCmd('section', + new_data, dts='241_replace_section_simple.dts') + self.assertEqual(new_data, data) + + entries = image.GetEntries() + self.assertIn('section', entries) + entry = entries['section'] + self.assertEqual(len(new_data), entry.size) + + def testReplaceSectionLarger(self): + """Test replacing a simple section with larger data""" + new_data = b'w' * (len(COMPRESS_DATA + U_BOOT_DATA) + 1) + data, expected_fdtmap, image = self._RunReplaceCmd('section', + new_data, dts='241_replace_section_simple.dts') + self.assertEqual(new_data, data) + + entries = image.GetEntries() + self.assertIn('section', entries) + entry = entries['section'] + self.assertEqual(len(new_data), entry.size) + fentry = entries['fdtmap'] + self.assertEqual(entry.offset + entry.size, fentry.offset) + + def testReplaceSectionSmaller(self): + """Test replacing a simple section with smaller data""" + new_data = b'w' * (len(COMPRESS_DATA + U_BOOT_DATA) - 1) + b'\0' + data, expected_fdtmap, image = self._RunReplaceCmd('section', + new_data, dts='241_replace_section_simple.dts') + self.assertEqual(new_data, data) + + # The new size is the same as the old, just with a pad byte at the end + entries = image.GetEntries() + self.assertIn('section', entries) + entry = entries['section'] + self.assertEqual(len(new_data), entry.size) + + def testReplaceSectionSmallerAllow(self): + """Test failing to replace a simple section with smaller data""" + new_data = b'w' * (len(COMPRESS_DATA + U_BOOT_DATA) - 1) + try: + state.SetAllowEntryContraction(True) + with self.assertRaises(ValueError) as exc: + self._RunReplaceCmd('section', new_data, + dts='241_replace_section_simple.dts') + finally: + state.SetAllowEntryContraction(False) + + # Since we have no information about the position of things within the + # section, we cannot adjust the position of /section-u-boot so it ends + # up outside the section self.assertIn( - "Node '/section': Replacing sections is not implemented yet", + "Node '/section/u-boot': Offset 0x24 (36) size 0x4 (4) is outside " + "the section '/section' starting at 0x0 (0) of size 0x27 (39)", str(exc.exception)) def testMkimageImagename(self): @@ -6412,6 +6460,85 @@ fdt fdtmap Extract the devicetree blob from the fdtmap 'tool', '-l')) self.assertEqual(['mary', 'anna', 'fred'], tools.tool_search_paths) + def testReplaceSectionEntry(self): + """Test replacing an entry in a section""" + expect_data = b'w' * len(U_BOOT_DATA + COMPRESS_DATA) + entry_data, expected_fdtmap, image = self._RunReplaceCmd('section/blob', + expect_data, dts='241_replace_section_simple.dts') + self.assertEqual(expect_data, entry_data) + + entries = image.GetEntries() + self.assertIn('section', entries) + section = entries['section'] + + sect_entries = section.GetEntries() + self.assertIn('blob', sect_entries) + entry = sect_entries['blob'] + self.assertEqual(len(expect_data), entry.size) + + fname = tools.get_output_filename('image-updated.bin') + data = tools.read_file(fname) + + new_blob_data = data[entry.image_pos:entry.image_pos + len(expect_data)] + self.assertEqual(expect_data, new_blob_data) + + self.assertEqual(U_BOOT_DATA, + data[entry.image_pos + len(expect_data):] + [:len(U_BOOT_DATA)]) + + def testReplaceSectionDeep(self): + """Test replacing an entry in two levels of sections""" + expect_data = b'w' * len(U_BOOT_DATA + COMPRESS_DATA) + entry_data, expected_fdtmap, image = self._RunReplaceCmd( + 'section/section/blob', expect_data, + dts='278_replace_section_deep.dts') + self.assertEqual(expect_data, entry_data) + + entries = image.GetEntries() + self.assertIn('section', entries) + section = entries['section'] + + subentries = section.GetEntries() + self.assertIn('section', subentries) + section = subentries['section'] + + sect_entries = section.GetEntries() + self.assertIn('blob', sect_entries) + entry = sect_entries['blob'] + self.assertEqual(len(expect_data), entry.size) + + fname = tools.get_output_filename('image-updated.bin') + data = tools.read_file(fname) + + new_blob_data = data[entry.image_pos:entry.image_pos + len(expect_data)] + self.assertEqual(expect_data, new_blob_data) + + self.assertEqual(U_BOOT_DATA, + data[entry.image_pos + len(expect_data):] + [:len(U_BOOT_DATA)]) + + def testReplaceFitSibling(self): + """Test an image with a FIT inside where we replace its sibling""" + fname = TestFunctional._MakeInputFile('once', b'available once') + self._DoReadFileRealDtb('277_replace_fit_sibling.dts') + os.remove(fname) + + try: + tmpdir, updated_fname = self._SetupImageInTmpdir() + + fname = os.path.join(tmpdir, 'update-blob') + expected = b'w' * (len(COMPRESS_DATA + U_BOOT_DATA) + 1) + tools.write_file(fname, expected) + + self._DoBinman('replace', '-i', updated_fname, 'blob', '-f', fname) + data = tools.read_file(updated_fname) + start = len(U_BOOT_DTB_DATA) + self.assertEqual(expected, data[start:start + len(expected)]) + map_fname = os.path.join(tmpdir, 'image-updated.map') + self.assertFalse(os.path.exists(map_fname)) + finally: + shutil.rmtree(tmpdir) + if __name__ == "__main__": unittest.main() diff --git a/tools/binman/test/277_replace_fit_sibling.dts b/tools/binman/test/277_replace_fit_sibling.dts new file mode 100644 index 00000000000..fc941a80816 --- /dev/null +++ b/tools/binman/test/277_replace_fit_sibling.dts @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: GPL-2.0+ +/dts-v1/; + +/ { + binman { + allow-repack; + + u-boot { + }; + + blob { + filename = "compress"; + }; + + fit { + description = "test-desc"; + #address-cells = <1>; + + images { + kernel { + description = "Vanilla Linux kernel"; + type = "kernel"; + arch = "ppc"; + os = "linux"; + compression = "gzip"; + load = <00000000>; + entry = <00000000>; + hash-1 { + algo = "crc32"; + }; + blob-ext { + filename = "once"; + }; + }; + fdt-1 { + description = "Flattened Device Tree blob"; + type = "flat_dt"; + arch = "ppc"; + compression = "none"; + hash-1 { + algo = "crc32"; + }; + u-boot-spl-dtb { + }; + }; + }; + + configurations { + default = "conf-1"; + conf-1 { + description = "Boot Linux kernel with FDT blob"; + kernel = "kernel"; + fdt = "fdt-1"; + }; + }; + }; + + fdtmap { + }; + }; +}; diff --git a/tools/binman/test/278_replace_section_deep.dts b/tools/binman/test/278_replace_section_deep.dts new file mode 100644 index 00000000000..fba2d7dcf28 --- /dev/null +++ b/tools/binman/test/278_replace_section_deep.dts @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0+ +/dts-v1/; + +/ { + binman { + allow-repack; + + u-boot-dtb { + }; + + section { + section { + blob { + filename = "compress"; + }; + }; + + u-boot { + }; + }; + + fdtmap { + }; + }; +}; -- cgit v1.3.1 From 953d4177afa0bee0ba0db4b81036d3197595b997 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Thu, 2 Mar 2023 17:02:45 -0700 Subject: binman: Support generation of x509 certificates And a new entry type which supports generation of x509 certificates. This uses a new 'openssl' btool with just one operation so far. Signed-off-by: Simon Glass --- tools/binman/btool/openssl.py | 94 +++++++++++++++++++++++++++++++++++++ tools/binman/entries.rst | 18 +++++++ tools/binman/etype/x509_cert.py | 92 ++++++++++++++++++++++++++++++++++++ tools/binman/ftest.py | 26 ++++++++++ tools/binman/test/279_x509_cert.dts | 19 ++++++++ tools/binman/test/key.key | 52 ++++++++++++++++++++ tools/binman/test/key.pem | 32 +++++++++++++ 7 files changed, 333 insertions(+) create mode 100644 tools/binman/btool/openssl.py create mode 100644 tools/binman/etype/x509_cert.py create mode 100644 tools/binman/test/279_x509_cert.dts create mode 100644 tools/binman/test/key.key create mode 100644 tools/binman/test/key.pem (limited to 'tools') diff --git a/tools/binman/btool/openssl.py b/tools/binman/btool/openssl.py new file mode 100644 index 00000000000..3a4dbdd6d73 --- /dev/null +++ b/tools/binman/btool/openssl.py @@ -0,0 +1,94 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright 2022 Google LLC +# +"""Bintool implementation for openssl + +openssl provides a number of features useful for signing images + +Documentation is at https://www.coreboot.org/CBFS + +Source code is at https://www.openssl.org/ +""" + +import hashlib + +from binman import bintool +from u_boot_pylib import tools + +class Bintoolopenssl(bintool.Bintool): + """openssl tool + + This bintool supports creating new openssl certificates. + + It also supports fetching a binary openssl + + Documentation about openssl is at https://www.openssl.org/ + """ + def __init__(self, name): + super().__init__( + name, 'openssl cryptography toolkit', + version_regex=r'OpenSSL (.*) \(', version_args='version') + + def x509_cert(self, cert_fname, input_fname, key_fname, cn, revision, + config_fname): + """Create a certificate + + Args: + cert_fname (str): Filename of certificate to create + input_fname (str): Filename containing data to sign + key_fname (str): Filename of .pem file + cn (str): Common name + revision (int): Revision number + config_fname (str): Filename to write fconfig into + + Returns: + str: Tool output + """ + indata = tools.read_file(input_fname) + hashval = hashlib.sha512(indata).hexdigest() + with open(config_fname, 'w', encoding='utf-8') as outf: + print(f'''[ req ] +distinguished_name = req_distinguished_name +x509_extensions = v3_ca +prompt = no +dirstring_type = nobmp + +[ req_distinguished_name ] +CN = {cert_fname} + +[ v3_ca ] +basicConstraints = CA:true +1.3.6.1.4.1.294.1.3 = ASN1:SEQUENCE:swrv +1.3.6.1.4.1.294.1.34 = ASN1:SEQUENCE:sysfw_image_integrity + +[ swrv ] +swrv = INTEGER:{revision} + +[ sysfw_image_integrity ] +shaType = OID:2.16.840.1.101.3.4.2.3 +shaValue = FORMAT:HEX,OCT:{hashval} +imageSize = INTEGER:{len(indata)} +''', file=outf) + args = ['req', '-new', '-x509', '-key', key_fname, '-nodes', + '-outform', 'DER', '-out', cert_fname, '-config', config_fname, + '-sha512'] + return self.run_cmd(*args) + + def fetch(self, method): + """Fetch handler for openssl + + This installs the openssl package using the apt utility. + + Args: + method (FETCH_...): Method to use + + Returns: + True if the file was fetched and now installed, None if a method + other than FETCH_BIN was requested + + Raises: + Valuerror: Fetching could not be completed + """ + if method != bintool.FETCH_BIN: + return None + return self.apt_install('openssl') diff --git a/tools/binman/entries.rst b/tools/binman/entries.rst index 19659247cf0..9a52b225a93 100644 --- a/tools/binman/entries.rst +++ b/tools/binman/entries.rst @@ -2276,6 +2276,24 @@ and kernel are genuine. +.. _etype_x509_cert: + +Entry: x509-cert: An entry which contains an X509 certificate +------------------------------------------------------------- + +Properties / Entry arguments: + - content: List of phandles to entries to sign + +Output files: + - input. - input file passed to openssl + - cert. - output file generated by openssl (which is + used as the entry contents) + +openssl signs the provided data, writing the signature in this entry. This +allows verification that the data is genuine + + + .. _etype_x86_reset16: Entry: x86-reset16: x86 16-bit reset code for U-Boot diff --git a/tools/binman/etype/x509_cert.py b/tools/binman/etype/x509_cert.py new file mode 100644 index 00000000000..f80a6ec2d12 --- /dev/null +++ b/tools/binman/etype/x509_cert.py @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright 2023 Google LLC +# Written by Simon Glass +# + +# Support for an X509 certificate, used to sign a set of entries + +from collections import OrderedDict +import os + +from binman.entry import EntryArg +from binman.etype.collection import Entry_collection + +from dtoc import fdt_util +from u_boot_pylib import tools + +class Entry_x509_cert(Entry_collection): + """An entry which contains an X509 certificate + + Properties / Entry arguments: + - content: List of phandles to entries to sign + + Output files: + - input. - input file passed to openssl + - cert. - output file generated by openssl (which is + used as the entry contents) + + openssl signs the provided data, writing the signature in this entry. This + allows verification that the data is genuine + """ + def __init__(self, section, etype, node): + super().__init__(section, etype, node) + self.openssl = None + + def ReadNode(self): + super().ReadNode() + self._cert_ca = fdt_util.GetString(self._node, 'cert-ca') + self._cert_rev = fdt_util.GetInt(self._node, 'cert-revision-int', 0) + self.key_fname = self.GetEntryArgsOrProps([ + EntryArg('keyfile', str)], required=True)[0] + + def GetCertificate(self, required): + """Get the contents of this entry + + Args: + required: True if the data must be present, False if it is OK to + return None + + Returns: + bytes content of the entry, which is the signed vblock for the + provided data + """ + # Join up the data files to be signed + input_data = self.GetContents(required) + if input_data is None: + return None + + uniq = self.GetUniqueName() + output_fname = tools.get_output_filename('cert.%s' % uniq) + input_fname = tools.get_output_filename('input.%s' % uniq) + config_fname = tools.get_output_filename('config.%s' % uniq) + tools.write_file(input_fname, input_data) + stdout = self.openssl.x509_cert( + cert_fname=output_fname, + input_fname=input_fname, + key_fname=self.key_fname, + cn=self._cert_ca, + revision=self._cert_rev, + config_fname=config_fname) + if stdout is not None: + data = tools.read_file(output_fname) + else: + # Bintool is missing; just use 4KB of zero data + self.record_missing_bintool(self.openssl) + data = tools.get_bytes(0, 4096) + return data + + def ObtainContents(self): + data = self.GetCertificate(False) + if data is None: + return False + self.SetContents(data) + return True + + def ProcessContents(self): + # The blob may have changed due to WriteSymbols() + data = self.GetCertificate(True) + return self.ProcessContentsUpdate(data) + + def AddBintools(self, btools): + super().AddBintools(btools) + self.openssl = self.AddBintool(btools, 'openssl') diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 76445969201..f1e14c6b3dc 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -6539,6 +6539,32 @@ fdt fdtmap Extract the devicetree blob from the fdtmap finally: shutil.rmtree(tmpdir) + def testX509Cert(self): + """Test creating an X509 certificate""" + keyfile = self.TestFile('key.key') + entry_args = { + 'keyfile': keyfile, + } + data = self._DoReadFileDtb('279_x509_cert.dts', + entry_args=entry_args)[0] + cert = data[:-4] + self.assertEqual(U_BOOT_DATA, data[-4:]) + + # TODO: verify the signature + + def testX509CertMissing(self): + """Test that binman still produces an image if openssl is missing""" + keyfile = self.TestFile('key.key') + entry_args = { + 'keyfile': 'keyfile', + } + with test_util.capture_sys_output() as (_, stderr): + self._DoTestFile('279_x509_cert.dts', + force_missing_bintools='openssl', + entry_args=entry_args) + err = stderr.getvalue() + self.assertRegex(err, "Image 'image'.*missing bintools.*: openssl") + if __name__ == "__main__": unittest.main() diff --git a/tools/binman/test/279_x509_cert.dts b/tools/binman/test/279_x509_cert.dts new file mode 100644 index 00000000000..71238172717 --- /dev/null +++ b/tools/binman/test/279_x509_cert.dts @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + x509-cert { + cert-ca = "IOT2050 Firmware Signature"; + cert-revision-int = <0>; + content = <&u_boot>; + }; + + u_boot: u-boot { + }; + }; +}; diff --git a/tools/binman/test/key.key b/tools/binman/test/key.key new file mode 100644 index 00000000000..9de3be14da8 --- /dev/null +++ b/tools/binman/test/key.key @@ -0,0 +1,52 @@ +-----BEGIN PRIVATE KEY----- +MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQCSDLMHq1Jw3U+G +H2wutSGrT4Xhs5Yy7uhR/rDOiuKTW3zkVdfSIliye3Nnwrl/nNUFzEJ+4t/AiDaJ +Qk5KddTAJnOkw5SYBvFsTDhMR4HH6AyfzaaVl+AAGOg4LXwZzGYKncgOY5u6ZyMB +SzHxozJmmoqYaCIi4Iv2VZRZw1YPBoT6sv38RQSET5ci/g+89Sfb85ZPHPu6PLlz +ZTufG+yzAhIDsIvNpt2YlCnQ1TqoZxXsztxN1bKIP68xvlAQHSAB8+x4y0tYPE1I +UT1DK22FMgz5iyBp6ksFaqI06fITtJjPKG13z8sXXgb4/rJ5I0lhsn1ySsHQ0zLw +/CX4La2/VMA0Bw6GLFRhu/rOycqKfmwLm25bExV8xL6lwFohxbzBQgYr93ujGFyQ +AXBDOphvZcdXP3CHAcEViVRjrsBWNz8wyf7X8h2FIU16kAd30WuspjmnGuvRZ6Gn +SNDVO2tbEKvwkg6liYWy4IXtWcvooMtkhYyvFudcxRPgxEUTQ00biYfJ59ukqD7I +hyT7pq1bZDCVnAt6dUUPWZutrbBacsyITs01hyiPxvAAQ7XRoInmW1DLqHZ+gCJU +YJ0TaiAI8AmnjypMWRUo19l0zIgPdva8EJ+mz+kKFsZszo1nwuxQL7oUSUCb0hfB +2k3WxNthBi3QpspUKPtKweIg9ITtIwIDAQABAoICAA9dS6ZGZTVfauLKwnhFcOXT +R1vfpzDzhjg+CX6pCL4E1WY2C67dEySvrQvg5d/hcV2bR/GOT4izK72T3qWhsMCI +KwlN0/+MV3CTsiaALUyJAm77VQeOwy9vb1qdml0ibie2wpmU7AiXmgykSvxHNWGq +52KyLckqgz7mcOVikdah0nKHSwXzgs6iit1RCfnQdqGChjELdQX6Jm5X24ZZCzUn +xhpiQ8reP5iyGZYRIIsf0SQo/O8pSI9h173tbgHL9paOATYR+Pqu2Vh+x2meE3b8 +NXY5Jy9NSRgoSCk15VQiXyMH90Av+YcbSrN+I+tvhWREQUM5Txt3ZHgKprntoEYE +XLHAr9cvmIzLNeApt2z/g4t80xFBpIvTG3+SV/rthmq0KGCLW2kPkdujOiIwdNFF +6fJ6ikphKAbx2NgUY+6AM5AoOh5QPMqvCdsPwO21YG1WoxmiUpNTaYMlR1fDofr/ +A/z2bFH4SiJPkHXRT2KBiJh4ZZWNzP6hOqGy+jreOpWh5IAyn7cKx6t3I28Q9df0 +tK/1PLgR8WWu6G4uHtF5lKL+LgqFCTbSu9JtLQVQntD7Qyd98sF5o23QQWyA19uU +TVGxtkVaP1y7v+gtC+xMTW9MbGIeJiqMZuZ3xXJVvUNg1/2BDd+VAfPCOq6xGHC7 +s9MFqwUsLCAFFebXC8oVAoIBAQDKGc/o21Ags2t61IJaJjU7YwrsRywhZR+vUz5F +xtqH4jt9AkdWpDkKbO7xNMQ2OFdnobq5mkM+iW6Jvc1fi4gm1HDyP296nPKZdFrJ +UgGfTxOhxFLp7gsJ2F0GX5eDJYvqUTBeYB3wrQkCc+t7fLg2oS+gKGIIn2CP07Mx +Bist3eCcDvL6QIxYS43u+ptTyAItyUYn8KwvCxlIEfjxowsxfhRWuU+Mr4A4hfGB +64xSI1YU1AYZLMucOtK/mmlscfO8isdcyfea0GJn4VLRnNvAKL5g627IdErWHs3u +KgYWAXtVKzHrf4hO8dpVgIzO69wAsqZEvKYGmTJhfyvBN9DdAoIBAQC5AA7s2XOX +raVymhPwEy4I/2w9NuMFmTavOREBp/gA9uaWBdqAWn1rRJiJ5plgdcnOBFPSGBnc +thkuWBRqkklQ0YPKhNBT48CZGBN7VUsvyTZD1+IXLW1TmY5UGT0p6/dAYkoJHnvX +TAHl1tfmeHxVCJWV6Shf5LfJJwsAiykxzetkzmeaycy2s9GKCnkc2uFxyhKnfM0/ +SLwTuXQIJvHuErTYA4jjVOG9EGYW2/uKScPBLpB1YTliAUIvByDy6suCN5pVZGT8 +xVLTYec9lXjhfyhysOAjhD3w77Jh7Exft91fEK50k2ZkqYYnh+mYZcnR52msVSBS +3YL9kK/9dNX/AoIBABcEaZFzqOSQiqUqns31nApvdUcDtBr5kWo+aNE5nJntQiky +oT1U5soxLeV6xP4H3KyI1uNcllwA+v3lCAbhtVf2ygZNAz1LsrWXct+K33RtZSb/ +XRIXclpksfOP34moNQ8yv/d/qulGS8hju2YNBk3yfaIX91JUFINM8ROcSD6pDnO3 +oCSwRUupDzkwgZBBLz5Xtg3Gc1XIRdDXeyrKDvRMD7Tw1gaH1mqZlq/dS9XvAFbO +7wLe/zGD4YzA4VDgiYnnpF0FA5Y2NX7vQqds3fo8qbIQHkXmOL+6Mmn1j0viT1Gb +4cuYcsXK9brXMTI/2oaZ0iXx9la6C+reuPUAjmECggEBAInEvlips0hgW1ZV4cUm +M2El/dA0YKoZqDyjDcQi9zCYra1JXKe7O603XzVK0iugbBGM7XMG2bOgtG3r0ABx +QkH6VN/rOk1OzW31HQT6xswmVs/9I/TIsqLQNsrwJLlkbTO4PpQ97FGv27Xy4cNT +NJwKkYMbKCMJa8hT2ACmoZ3iUIs4nrUJ1Pa2QLRBCmJvqfYYWv35lcur+cvijsNH +ZWE68wvuzfEllBo87RnW5qLcPfhOGewf5CDU+RmWgHYGXllx2PAAnKgUtpKOVStq +daPQEyoeCDzKzWnwxvHfjBy4CxYxkQllf5o1GJ+1ukLwgnRbljltB25OYa89IaJp +cLcCggEAa5vbegzMKYPjR3zcVjnvhRsLXQi1vMtbUqOQ5wYMwGIef4v3QHNoF7EA +aNpWQ/qgCTQUzl3qoQCkRiVmVBBr60Fs5y7sfA92eBxQIV5hxJftH3vmiKqeWeqm +ila9DNw84MNAIqI2u6R3K/ur9fkSswDr3nzvFjuheW5V/M/6zAUtJZXr4iUih929 +uhf2dn6pSLR+epJ5023CVaI2zwz+U6PDEATKy9HjeKab3tQMHxQkT/5IWcLqrVTs +0rMobIgONzQqYDi2sO05YvgNBxvX3pUvqNlthcOtauT8BoE6wxLYm7ZcWYLPn15A +wR0+2mDpx+HDyu76q3M+KxXG2U8sJg== +-----END PRIVATE KEY----- diff --git a/tools/binman/test/key.pem b/tools/binman/test/key.pem new file mode 100644 index 00000000000..7a7b84a8bba --- /dev/null +++ b/tools/binman/test/key.pem @@ -0,0 +1,32 @@ +-----BEGIN CERTIFICATE----- +MIIFcTCCA1kCFB/17qhcvpyKhG+jfS2c0qG1yjruMA0GCSqGSIb3DQEBCwUAMHUx +CzAJBgNVBAYTAk5aMRMwEQYDVQQIDApDYW50ZXJidXJ5MRUwEwYDVQQHDAxDaHJp +c3RjaHVyY2gxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEXMBUG +A1UEAwwOTXkgQ29tbW9uIE5hbWUwHhcNMjMwMjEzMDM1MzMzWhcNMjQwMjEzMDM1 +MzMzWjB1MQswCQYDVQQGEwJOWjETMBEGA1UECAwKQ2FudGVyYnVyeTEVMBMGA1UE +BwwMQ2hyaXN0Y2h1cmNoMSEwHwYDVQQKDBhJbnRlcm5ldCBXaWRnaXRzIFB0eSBM +dGQxFzAVBgNVBAMMDk15IENvbW1vbiBOYW1lMIICIjANBgkqhkiG9w0BAQEFAAOC +Ag8AMIICCgKCAgEAkgyzB6tScN1Phh9sLrUhq0+F4bOWMu7oUf6wzorik1t85FXX +0iJYsntzZ8K5f5zVBcxCfuLfwIg2iUJOSnXUwCZzpMOUmAbxbEw4TEeBx+gMn82m +lZfgABjoOC18GcxmCp3IDmObumcjAUsx8aMyZpqKmGgiIuCL9lWUWcNWDwaE+rL9 +/EUEhE+XIv4PvPUn2/OWTxz7ujy5c2U7nxvsswISA7CLzabdmJQp0NU6qGcV7M7c +TdWyiD+vMb5QEB0gAfPseMtLWDxNSFE9QytthTIM+YsgaepLBWqiNOnyE7SYzyht +d8/LF14G+P6yeSNJYbJ9ckrB0NMy8Pwl+C2tv1TANAcOhixUYbv6zsnKin5sC5tu +WxMVfMS+pcBaIcW8wUIGK/d7oxhckAFwQzqYb2XHVz9whwHBFYlUY67AVjc/MMn+ +1/IdhSFNepAHd9FrrKY5pxrr0Wehp0jQ1TtrWxCr8JIOpYmFsuCF7VnL6KDLZIWM +rxbnXMUT4MRFE0NNG4mHyefbpKg+yIck+6atW2QwlZwLenVFD1mbra2wWnLMiE7N +NYcoj8bwAEO10aCJ5ltQy6h2foAiVGCdE2ogCPAJp48qTFkVKNfZdMyID3b2vBCf +ps/pChbGbM6NZ8LsUC+6FElAm9IXwdpN1sTbYQYt0KbKVCj7SsHiIPSE7SMCAwEA +ATANBgkqhkiG9w0BAQsFAAOCAgEAJAJoia6Vq4vXP/0bCgW3o9TOMmFYhI/xPxoh +Gd7was9R7BOrMGO+/3E7DZtjycZYL0r9nOtr9S/BBreuZ4vkk/PSoGaSnG8ST4jC +Ajk7ew/32RGOgA/oIzgKj1SPkBtvW+x+76sjUkGKsxmABBUhycIY7K0U8McTTfJ7 +gJ164VXmdG7qFMWmRy4Ry9QGXkDsbMSOZ485X7zbphjK5OZXEujP7GMUgg1lP479 +NqC1g+1m/A3PIB767lVYA7APQsrckHdRqOTkK9TYRQ3mvyE2wruhqE6lx8G/UyFh +RZjZ3lh2bx07UWIlyMabnGDMrM4FCnesqVyVAc8VAbkdXkeJI9r6DdFw+dzIY0P1 +il+MlYpZNwRyNv2W5SCPilyuhuPOSrSnsSHx64puCIvwG/4xA30Jw8nviJuyGSef +7uE+W7SD9E/hQHi/S9KRsYVoo7a6X9ADiwNsRNzVnuqc7K3mv/C5E9s6uFTNoObe +fUBA7pL3Fmvc5pYatxTFI85ajBpe/la6AA+7HX/8PXEphmp6GhFCcfsq+DL03vTM +DqIJL1i/JXggwqvvdcfaSeMDIOIzO89yUGGwwuj9rqMeEY99qDtljgy1EljjrB5i +0j4Jg4O0OEd2KIOD7nz4do1tLNlRcpysDZeXIiwAI7Dd3wWMsgpOQxs0zqWyqDVq +mCKa5Tw= +-----END CERTIFICATE----- -- cgit v1.3.1