From dfe1db4030edd11275e0c9f8fa020b12ecf90d8c Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 13 Aug 2022 11:40:48 -0600 Subject: binman: Allow the image name to be the data file Some image types use the -n parameter to pass in the data file. Add support for this, with a new property. Signed-off-by: Simon Glass --- tools/binman/test/235_mkimage_name.dts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 tools/binman/test/235_mkimage_name.dts (limited to 'tools/binman/test') diff --git a/tools/binman/test/235_mkimage_name.dts b/tools/binman/test/235_mkimage_name.dts new file mode 100644 index 00000000000..fbc82f1f8d6 --- /dev/null +++ b/tools/binman/test/235_mkimage_name.dts @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + mkimage { + args = "-T script"; + data-to-imagename; + + u-boot-spl { + }; + }; + }; +}; -- cgit v1.3.1 From 9db9e932c79b653e81d5e11241b8f20e8fd61bdd Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 13 Aug 2022 11:40:49 -0600 Subject: binman: Allow passing entries using -n Also control over what goes in the file passed with -n using a separate imagename subnode. This can include a section or any other entry type. Signed-off-by: Simon Glass --- tools/binman/entries.rst | 18 ++++++++++ tools/binman/etype/mkimage.py | 39 +++++++++++++++++++++- tools/binman/ftest.py | 34 +++++++++++++++++++ tools/binman/test/236_mkimage_image.dts | 21 ++++++++++++ tools/binman/test/237_mkimage_image_no_content.dts | 22 ++++++++++++ tools/binman/test/238_mkimage_image_bad.dts | 22 ++++++++++++ 6 files changed, 155 insertions(+), 1 deletion(-) create mode 100644 tools/binman/test/236_mkimage_image.dts create mode 100644 tools/binman/test/237_mkimage_image_no_content.dts create mode 100644 tools/binman/test/238_mkimage_image_bad.dts (limited to 'tools/binman/test') diff --git a/tools/binman/entries.rst b/tools/binman/entries.rst index 9844b2efff9..3ee7b051688 100644 --- a/tools/binman/entries.rst +++ b/tools/binman/entries.rst @@ -1224,6 +1224,24 @@ That will pass the data to mkimage both as the data file (with -d) and as the image name (with -n). +If need to pass different data in with -n, then use an imagename subnode:: + + mkimage { + args = "-T imximage"; + + imagename { + blob { + filename = "spl/u-boot-spl.cfgout" + }; + }; + + u-boot-spl { + }; + }; + +This will pass in u-boot-spl as the input data and the .cfgout file as the +-n data. + .. _etype_opensbi: diff --git a/tools/binman/etype/mkimage.py b/tools/binman/etype/mkimage.py index 53622546dc0..437fcdacfd7 100644 --- a/tools/binman/etype/mkimage.py +++ b/tools/binman/etype/mkimage.py @@ -75,10 +75,29 @@ class Entry_mkimage(Entry): That will pass the data to mkimage both as the data file (with -d) and as the image name (with -n). In both cases, a filename is passed as the argument, with the actual data being in that file. + + If need to pass different data in with -n, then use an `imagename` subnode:: + + mkimage { + args = "-T imximage"; + + imagename { + blob { + filename = "spl/u-boot-spl.cfgout" + }; + }; + + u-boot-spl { + }; + }; + + This will pass in u-boot-spl as the input data and the .cfgout file as the + -n data. """ def __init__(self, section, etype, node): super().__init__(section, etype, node) self._mkimage_entries = OrderedDict() + self._imagename = None self.align_default = None def ReadNode(self): @@ -86,6 +105,8 @@ class Entry_mkimage(Entry): self._args = fdt_util.GetArgs(self._node, 'args') self._data_to_imagename = fdt_util.GetBool(self._node, 'data-to-imagename') + if self._data_to_imagename and self._node.FindNode('imagename'): + self.Raise('Cannot use both imagename node and data-to-imagename') self.ReadEntries() def ReadEntries(self): @@ -93,7 +114,10 @@ class Entry_mkimage(Entry): for node in self._node.subnodes: entry = Entry.Create(self, node) entry.ReadNode() - self._mkimage_entries[entry.name] = entry + if entry.name == 'imagename': + self._imagename = entry + else: + self._mkimage_entries[entry.name] = entry def ObtainContents(self): # Use a non-zero size for any fake files to keep mkimage happy @@ -102,11 +126,18 @@ class Entry_mkimage(Entry): self._mkimage_entries.values(), 'mkimage', 1024) if data is None: return False + if self._imagename: + image_data, imagename_fname, _ = self.collect_contents_to_file( + [self._imagename], 'mkimage-n', 1024) + if image_data is None: + return False output_fname = tools.get_output_filename('mkimage-out.%s' % uniq) args = ['-d', input_fname] if self._data_to_imagename: args += ['-n', input_fname] + elif self._imagename: + args += ['-n', imagename_fname] args += self._args + [output_fname] if self.mkimage.run_cmd(*args) is not None: self.SetContents(tools.read_file(output_fname)) @@ -126,6 +157,8 @@ class Entry_mkimage(Entry): self.allow_missing = allow_missing for entry in self._mkimage_entries.values(): entry.SetAllowMissing(allow_missing) + if self._imagename: + self._imagename.SetAllowMissing(allow_missing) def SetAllowFakeBlob(self, allow_fake): """Set whether the sub nodes allows to create a fake blob @@ -135,6 +168,8 @@ class Entry_mkimage(Entry): """ for entry in self._mkimage_entries.values(): entry.SetAllowFakeBlob(allow_fake) + if self._imagename: + self._imagename.SetAllowFakeBlob(allow_fake) def CheckFakedBlobs(self, faked_blobs_list): """Check if any entries in this section have faked external blobs @@ -146,6 +181,8 @@ class Entry_mkimage(Entry): """ for entry in self._mkimage_entries.values(): entry.CheckFakedBlobs(faked_blobs_list) + if self._imagename: + self._imagename.CheckFakedBlobs(faked_blobs_list) def AddBintools(self, btools): self.mkimage = self.AddBintool(btools, 'mkimage') diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index e88eedff51b..9b10fd8698d 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -5739,6 +5739,40 @@ fdt fdtmap Extract the devicetree blob from the fdtmap # Check that the image name is set to the temporary filename used self.assertEqual(expect.encode('utf-8')[:0x20], name) + def testMkimageImage(self): + """Test using mkimage with -n holding the data too""" + data = self._DoReadFile('236_mkimage_image.dts') + + # Check that the data appears in the file somewhere + self.assertIn(U_BOOT_SPL_DATA, data) + + # Get struct image_header -> ih_name + name = data[0x20:0x40] + + # Build the filename that we expect to be placed in there, by virtue of + # the -n paraameter + expect = os.path.join(tools.get_output_dir(), 'mkimage-n.mkimage') + + # Check that the image name is set to the temporary filename used + self.assertEqual(expect.encode('utf-8')[:0x20], name) + + # Check the corect data is in the imagename file + self.assertEqual(U_BOOT_DATA, tools.read_file(expect)) + + def testMkimageImageNoContent(self): + """Test using mkimage with -n and no data""" + with self.assertRaises(ValueError) as exc: + self._DoReadFile('237_mkimage_image_no_content.dts') + self.assertIn('Could not complete processing of contents', + str(exc.exception)) + + def testMkimageImageBad(self): + """Test using mkimage with imagename node and data-to-imagename""" + with self.assertRaises(ValueError) as exc: + self._DoReadFile('238_mkimage_image_bad.dts') + self.assertIn('Cannot use both imagename node and data-to-imagename', + str(exc.exception)) + if __name__ == "__main__": unittest.main() diff --git a/tools/binman/test/236_mkimage_image.dts b/tools/binman/test/236_mkimage_image.dts new file mode 100644 index 00000000000..6b8f4a4a401 --- /dev/null +++ b/tools/binman/test/236_mkimage_image.dts @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + mkimage { + args = "-T script"; + + imagename { + type = "u-boot"; + }; + + u-boot-spl { + }; + }; + }; +}; diff --git a/tools/binman/test/237_mkimage_image_no_content.dts b/tools/binman/test/237_mkimage_image_no_content.dts new file mode 100644 index 00000000000..7306c06af45 --- /dev/null +++ b/tools/binman/test/237_mkimage_image_no_content.dts @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + mkimage { + args = "-T script"; + + imagename { + type = "_testing"; + return-unknown-contents; + }; + + u-boot-spl { + }; + }; + }; +}; diff --git a/tools/binman/test/238_mkimage_image_bad.dts b/tools/binman/test/238_mkimage_image_bad.dts new file mode 100644 index 00000000000..54d2c99d628 --- /dev/null +++ b/tools/binman/test/238_mkimage_image_bad.dts @@ -0,0 +1,22 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + mkimage { + args = "-T script"; + data-to-imagename; + + imagename { + type = "u-boot"; + }; + + u-boot-spl { + }; + }; + }; +}; -- cgit v1.3.1 From d626e825f5d45ca543999340428a5a809024b0db Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 13 Aug 2022 11:40:50 -0600 Subject: binman: Allow collection to use entries from other sections At present the collections etype only works with entries in the same section. This can be limiting, since in some cases the data may be inside a subsection, e.g. if there are alignment constraints. Add a function to find the entries in an etype and have it search recursively. Make use of this for mkimage also. Signed-off-by: Simon Glass --- tools/binman/entries.rst | 3 +++ tools/binman/entry.py | 23 +++++++++++++++++++++++ tools/binman/etype/collection.py | 3 +++ tools/binman/etype/mkimage.py | 7 +++++++ tools/binman/etype/section.py | 8 ++++---- tools/binman/ftest.py | 14 ++++++++++++++ tools/binman/test/239_collection_other.dts | 29 +++++++++++++++++++++++++++++ tools/binman/test/240_mkimage_coll.dts | 27 +++++++++++++++++++++++++++ 8 files changed, 110 insertions(+), 4 deletions(-) create mode 100644 tools/binman/test/239_collection_other.dts create mode 100644 tools/binman/test/240_mkimage_coll.dts (limited to 'tools/binman/test') diff --git a/tools/binman/entries.rst b/tools/binman/entries.rst index 3ee7b051688..4dda34ca01f 100644 --- a/tools/binman/entries.rst +++ b/tools/binman/entries.rst @@ -447,6 +447,9 @@ listed entries are combined to form this entry. This serves as a useful base class for entry types which need to process data from elsewhere in the image, not necessarily child entries. +The entries can generally be anywhere in the same image, even if they are in +a different section from this entry. + .. _etype_cros_ec_rw: diff --git a/tools/binman/entry.py b/tools/binman/entry.py index d159d90e4c5..d413c91f181 100644 --- a/tools/binman/entry.py +++ b/tools/binman/entry.py @@ -679,6 +679,7 @@ class Entry(object): self.WriteMapLine(fd, indent, self.name, self.offset, self.size, self.image_pos) + # pylint: disable=assignment-from-none def GetEntries(self): """Return a list of entries contained by this entry @@ -688,6 +689,28 @@ class Entry(object): """ return None + def FindEntryByNode(self, find_node): + """Find a node in an entry, searching all subentries + + This does a recursive search. + + Args: + find_node (fdt.Node): Node to find + + Returns: + Entry: entry, if found, else None + """ + entries = self.GetEntries() + if entries: + for entry in entries.values(): + if entry._node == find_node: + return entry + found = entry.FindEntryByNode(find_node) + if found: + return found + + return None + def GetArg(self, name, datatype=str): """Get the value of an entry argument or device-tree-node property diff --git a/tools/binman/etype/collection.py b/tools/binman/etype/collection.py index 442b40b48b3..c532aafe3e7 100644 --- a/tools/binman/etype/collection.py +++ b/tools/binman/etype/collection.py @@ -21,6 +21,9 @@ class Entry_collection(Entry): listed entries are combined to form this entry. This serves as a useful base class for entry types which need to process data from elsewhere in the image, not necessarily child entries. + + The entries can generally be anywhere in the same image, even if they are in + a different section from this entry. """ def __init__(self, section, etype, node): super().__init__(section, etype, node) diff --git a/tools/binman/etype/mkimage.py b/tools/binman/etype/mkimage.py index 437fcdacfd7..d298776741e 100644 --- a/tools/binman/etype/mkimage.py +++ b/tools/binman/etype/mkimage.py @@ -148,6 +148,13 @@ class Entry_mkimage(Entry): return True + def GetEntries(self): + # Make a copy so we don't change the original + entries = OrderedDict(self._mkimage_entries) + if self._imagename: + entries['imagename'] = self._imagename + return entries + def SetAllowMissing(self, allow_missing): """Set whether a section allows missing external blobs diff --git a/tools/binman/etype/section.py b/tools/binman/etype/section.py index bd67238b919..5c326a75e8c 100644 --- a/tools/binman/etype/section.py +++ b/tools/binman/etype/section.py @@ -506,10 +506,10 @@ class Entry_section(Entry): node = self._node.GetFdt().LookupPhandle(phandle) if not node: source_entry.Raise("Cannot find node for phandle %d" % phandle) - for entry in self._entries.values(): - if entry._node == node: - return entry.GetData(required) - source_entry.Raise("Cannot find entry for node '%s'" % node.name) + entry = self.FindEntryByNode(node) + if not entry: + source_entry.Raise("Cannot find entry for node '%s'" % node.name) + return entry.GetData(required) def LookupSymbol(self, sym_name, optional, msg, base_addr, entries=None): """Look up a symbol in an ELF file diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index 9b10fd8698d..737dbcc2466 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -5773,6 +5773,20 @@ fdt fdtmap Extract the devicetree blob from the fdtmap self.assertIn('Cannot use both imagename node and data-to-imagename', str(exc.exception)) + def testCollectionOther(self): + """Test a collection where the data comes from another section""" + data = self._DoReadFile('239_collection_other.dts') + self.assertEqual(U_BOOT_NODTB_DATA + U_BOOT_DTB_DATA + + tools.get_bytes(0xff, 2) + U_BOOT_NODTB_DATA + + tools.get_bytes(0xfe, 3) + U_BOOT_DTB_DATA, + data) + + def testMkimageCollection(self): + """Test using a collection referring to an entry in a mkimage entry""" + data = self._DoReadFile('240_mkimage_coll.dts') + expect = U_BOOT_SPL_DATA + U_BOOT_DATA + self.assertEqual(expect, data[:len(expect)]) + if __name__ == "__main__": unittest.main() diff --git a/tools/binman/test/239_collection_other.dts b/tools/binman/test/239_collection_other.dts new file mode 100644 index 00000000000..09de20e5bca --- /dev/null +++ b/tools/binman/test/239_collection_other.dts @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + collection { + content = <&u_boot_nodtb &dtb>; + }; + section { + fill { + size = <2>; + fill-byte = [ff]; + }; + u_boot_nodtb: u-boot-nodtb { + }; + fill2 { + type = "fill"; + size = <3>; + fill-byte = [fe]; + }; + }; + dtb: u-boot-dtb { + }; + }; +}; diff --git a/tools/binman/test/240_mkimage_coll.dts b/tools/binman/test/240_mkimage_coll.dts new file mode 100644 index 00000000000..30860118860 --- /dev/null +++ b/tools/binman/test/240_mkimage_coll.dts @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + collection { + content = <&spl &u_boot>; + }; + mkimage { + args = "-T script"; + + spl: u-boot-spl { + }; + + imagename { + type = "section"; + + u_boot: u-boot { + }; + }; + }; + }; +}; -- cgit v1.3.1 From 6aa8000e7400c3861d7ede0ee4e206e19085116e Mon Sep 17 00:00:00 2001 From: Stefan Herbrechtsmeier Date: Fri, 19 Aug 2022 16:25:25 +0200 Subject: binman: Add length header attribute to dtb entry Add an optional length header attribute to the device tree blob entry class based on the compressed data header from the utilities to compress and decompress data. If needed the header could be enabled with the following attribute beside the compress attribute: prepend = "length"; The header was introduced as part of commit eb0f4a4cb402 ("binman: Support replacing data in a cbfs") to allow device tree entries to be larger than the compressed contents. Regarding the commit "this is necessary to cope with a compressed device tree being updated in such a way that it shrinks after the entry size is already set (an obscure case)". This case need to be fixed without influence any compressed data by itself. Signed-off-by: Stefan Herbrechtsmeier Reviewed-by: Simon Glass --- tools/binman/entries.rst | 3 ++ tools/binman/etype/blob_dtb.py | 27 +++++++++++++++ tools/binman/ftest.py | 39 ++++++++++++++++++++++ .../test/235_compress_dtb_prepend_invalid.dts | 17 ++++++++++ .../test/236_compress_dtb_prepend_length.dts | 19 +++++++++++ 5 files changed, 105 insertions(+) create mode 100644 tools/binman/test/235_compress_dtb_prepend_invalid.dts create mode 100644 tools/binman/test/236_compress_dtb_prepend_length.dts (limited to 'tools/binman/test') diff --git a/tools/binman/entries.rst b/tools/binman/entries.rst index 4dda34ca01f..b3613d7cbd0 100644 --- a/tools/binman/entries.rst +++ b/tools/binman/entries.rst @@ -216,6 +216,9 @@ This is a blob containing a device tree. The contents of the blob are obtained from the list of available device-tree files, managed by the 'state' module. +Additional Properties / Entry arguments: + - prepend: Header type to use: + length: 32-bit length header .. _etype_blob_ext: diff --git a/tools/binman/etype/blob_dtb.py b/tools/binman/etype/blob_dtb.py index 4159e3032a1..5a6a4547485 100644 --- a/tools/binman/etype/blob_dtb.py +++ b/tools/binman/etype/blob_dtb.py @@ -7,6 +7,8 @@ from binman.entry import Entry from binman.etype.blob import Entry_blob +from dtoc import fdt_util +import struct # This is imported if needed state = None @@ -17,6 +19,9 @@ class Entry_blob_dtb(Entry_blob): This is a blob containing a device tree. The contents of the blob are obtained from the list of available device-tree files, managed by the 'state' module. + + Additional attributes: + prepend: Header used (e.g. 'length') """ def __init__(self, section, etype, node): # Put this here to allow entry-docs and help to work without libfdt @@ -24,6 +29,14 @@ class Entry_blob_dtb(Entry_blob): from binman import state super().__init__(section, etype, node) + self.prepend = None + + def ReadNode(self): + super().ReadNode() + self.prepend = fdt_util.GetString(self._node, 'prepend') + if self.prepend and self.prepend not in ['length']: + self.Raise("Invalid prepend in '%s': '%s'" % + (self._node.name, self.prepend)) def ObtainContents(self): """Get the device-tree from the list held by the 'state' module""" @@ -58,3 +71,17 @@ class Entry_blob_dtb(Entry_blob): # will still return the old contents state.UpdateFdtContents(self.GetFdtEtype(), data) return ok + + def CompressData(self, indata): + data = super().CompressData(indata) + if self.prepend == 'length': + hdr = struct.pack('; + #size-cells = <1>; + + binman { + u-boot { + }; + u-boot-dtb { + compress = "lz4"; + prepend = "invalid"; + }; + }; +}; diff --git a/tools/binman/test/236_compress_dtb_prepend_length.dts b/tools/binman/test/236_compress_dtb_prepend_length.dts new file mode 100644 index 00000000000..1570233637a --- /dev/null +++ b/tools/binman/test/236_compress_dtb_prepend_length.dts @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + u-boot { + }; + u-boot-dtb { + compress = "lz4"; + prepend = "length"; + }; + fdtmap { + }; + }; +}; -- cgit v1.3.1 From ec7d27d3a83023af37e4fd42f67ec328d27b20c7 Mon Sep 17 00:00:00 2001 From: Stefan Herbrechtsmeier Date: Fri, 19 Aug 2022 16:25:30 +0200 Subject: binman: Move compression bintool management into entry class Move management of the bintool to compress and decompress data into the entry class and add the bintool to the list of required bintools. Signed-off-by: Stefan Herbrechtsmeier Reviewed-by: Simon Glass --- tools/binman/comp_util.py | 69 -------------------------- tools/binman/entry.py | 23 +++++++-- tools/binman/ftest.py | 16 +++--- tools/binman/test/237_compress_dtb_invalid.dts | 16 ++++++ 4 files changed, 41 insertions(+), 83 deletions(-) delete mode 100644 tools/binman/comp_util.py create mode 100644 tools/binman/test/237_compress_dtb_invalid.dts (limited to 'tools/binman/test') diff --git a/tools/binman/comp_util.py b/tools/binman/comp_util.py deleted file mode 100644 index 269bbf79756..00000000000 --- a/tools/binman/comp_util.py +++ /dev/null @@ -1,69 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0+ -# Copyright 2022 Google LLC -# -"""Utilities to compress and decompress data""" - -import tempfile - -from binman import bintool -from patman import tools - -LZ4 = bintool.Bintool.create('lz4') -HAVE_LZ4 = LZ4.is_present() - -LZMA_ALONE = bintool.Bintool.create('lzma_alone') -HAVE_LZMA_ALONE = LZMA_ALONE.is_present() - - -def compress(indata, algo): - """Compress some data using a given algorithm - - Note that for lzma this uses an old version of the algorithm, not that - provided by xz. - - This requires 'lz4' and 'lzma_alone' tools. It also requires an output - directory to be previously set up, by calling PrepareOutputDir(). - - Args: - indata (bytes): Input data to compress - algo (str): Algorithm to use ('none', 'lz4' or 'lzma') - - Returns: - bytes: Compressed data - """ - if algo == 'none': - return indata - if algo == 'lz4': - data = LZ4.compress(indata) - # cbfstool uses a very old version of lzma - elif algo == 'lzma': - data = LZMA_ALONE.compress(indata) - else: - raise ValueError("Unknown algorithm '%s'" % algo) - return data - -def decompress(indata, algo): - """Decompress some data using a given algorithm - - Note that for lzma this uses an old version of the algorithm, not that - provided by xz. - - This requires 'lz4' and 'lzma_alone' tools. It also requires an output - directory to be previously set up, by calling PrepareOutputDir(). - - Args: - indata (bytes): Input data to decompress - algo (str): Algorithm to use ('none', 'lz4' or 'lzma') - - Returns: - (bytes) Compressed data - """ - if algo == 'none': - return indata - if algo == 'lz4': - data = LZ4.decompress(indata) - elif algo == 'lzma': - data = LZMA_ALONE.decompress(indata) - else: - raise ValueError("Unknown algorithm '%s'" % algo) - return data diff --git a/tools/binman/entry.py b/tools/binman/entry.py index f448adbcfe0..af8e277995b 100644 --- a/tools/binman/entry.py +++ b/tools/binman/entry.py @@ -12,7 +12,6 @@ import sys import time from binman import bintool -from binman import comp_util from dtoc import fdt_util from patman import tools from patman.tools import to_hex, to_hex_size @@ -83,6 +82,7 @@ class Entry(object): missing_bintools: List of missing bintools for this entry update_hash: True if this entry's "hash" subnode should be updated with a hash of the entry contents + comp_bintool: Bintools used for compress and decompress data fake_fname: Fake filename, if one was created, else None required_props (dict of str): Properties which must be present. This can be added to by subclasses @@ -124,6 +124,7 @@ class Entry(object): self.update_hash = True self.fake_fname = None self.required_props = [] + self.comp_bintool = None @staticmethod def FindEntryClass(etype, expanded): @@ -1117,7 +1118,9 @@ features to produce new behaviours. self.uncomp_data = indata if self.compress != 'none': self.uncomp_size = len(indata) - data = comp_util.compress(indata, self.compress) + data = self.comp_bintool.compress(indata) + else: + data = indata return data def DecompressData(self, indata): @@ -1129,9 +1132,11 @@ features to produce new behaviours. Returns: Decompressed data """ - data = comp_util.decompress(indata, self.compress) if self.compress != 'none': + data = self.comp_bintool.decompress(indata) self.uncomp_size = len(data) + else: + data = indata self.uncomp_data = data return data @@ -1172,8 +1177,18 @@ features to produce new behaviours. Args: btools (dict of Bintool): + + Raise: + ValueError if compression algorithm is not supported """ - pass + algo = self.compress + if algo != 'none': + algos = ['lz4', 'lzma'] + if algo not in algos: + raise ValueError("Unknown algorithm '%s'" % algo) + names = {'lzma': 'lzma_alone'} + name = names.get(self.compress, self.compress) + self.comp_bintool = self.AddBintool(btools, name) @classmethod def AddBintool(self, tools, name): diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index b1a929242af..4be84f6e17f 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -23,7 +23,6 @@ import urllib.error from binman import bintool from binman import cbfs_util from binman import cmdline -from binman import comp_util from binman import control from binman import elf from binman import elf_test @@ -5229,15 +5228,6 @@ fdt fdtmap Extract the devicetree blob from the fdtmap self._DoBinman(*args) self.assertIn('failed to fetch with all methods', stdout.getvalue()) - def testInvalidCompress(self): - with self.assertRaises(ValueError) as e: - comp_util.compress(b'', 'invalid') - self.assertIn("Unknown algorithm 'invalid'", str(e.exception)) - - with self.assertRaises(ValueError) as e: - comp_util.decompress(b'1234', 'invalid') - self.assertIn("Unknown algorithm 'invalid'", str(e.exception)) - def testBintoolDocs(self): """Test for creation of bintool documentation""" with test_util.capture_sys_output() as (stdout, stderr): @@ -5858,6 +5848,12 @@ fdt fdtmap Extract the devicetree blob from the fdtmap orig2 = self._decompress(comp_data) self.assertEqual(orig, orig2) + def testInvalidCompress(self): + """Test that invalid compress algorithm is detected""" + with self.assertRaises(ValueError) as e: + self._DoTestFile('237_compress_dtb_invalid.dts') + self.assertIn("Unknown algorithm 'invalid'", str(e.exception)) + if __name__ == "__main__": unittest.main() diff --git a/tools/binman/test/237_compress_dtb_invalid.dts b/tools/binman/test/237_compress_dtb_invalid.dts new file mode 100644 index 00000000000..228139060bc --- /dev/null +++ b/tools/binman/test/237_compress_dtb_invalid.dts @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + u-boot { + }; + u-boot-dtb { + compress = "invalid"; + }; + }; +}; -- cgit v1.3.1 From cd15b640b0b8d5a7ba5f1c0587e4f9c767e2d8fb Mon Sep 17 00:00:00 2001 From: Stefan Herbrechtsmeier Date: Fri, 19 Aug 2022 16:25:38 +0200 Subject: binman: Add zstd bintool Add zstd bintool to binman to support on-the-fly compression. Signed-off-by: Stefan Herbrechtsmeier Reviewed-by: Simon Glass --- tools/binman/btool/zstd.py | 30 +++++++++++++++++++++++++++++ tools/binman/entry.py | 2 +- tools/binman/etype/blob_dtb.py | 4 ++++ tools/binman/ftest.py | 12 ++++++++++-- tools/binman/test/238_compress_dtb_zstd.dts | 16 +++++++++++++++ 5 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 tools/binman/btool/zstd.py create mode 100644 tools/binman/test/238_compress_dtb_zstd.dts (limited to 'tools/binman/test') diff --git a/tools/binman/btool/zstd.py b/tools/binman/btool/zstd.py new file mode 100644 index 00000000000..299bd371269 --- /dev/null +++ b/tools/binman/btool/zstd.py @@ -0,0 +1,30 @@ +# SPDX-License-Identifier: GPL-2.0+ +# Copyright (C) 2022 Weidmüller Interface GmbH & Co. KG +# Stefan Herbrechtsmeier +# +"""Bintool implementation for zstd + +zstd allows compression and decompression of files. + +Documentation is available via:: + + man zstd +""" + +from binman import bintool + +# pylint: disable=C0103 +class Bintoolzstd(bintool.BintoolPacker): + """Compression/decompression using the zstd algorithm + + This bintool supports running `zstd` to compress and decompress data, as + used by binman. + + It is also possible to fetch the tool, which uses `apt` to install it. + + Documentation is available via:: + + man zstd + """ + def __init__(self, name): + super().__init__(name) diff --git a/tools/binman/entry.py b/tools/binman/entry.py index 770130b1aa0..63ec5cea3b2 100644 --- a/tools/binman/entry.py +++ b/tools/binman/entry.py @@ -1191,7 +1191,7 @@ features to produce new behaviours. """ algo = self.compress if algo != 'none': - algos = ['bzip2', 'gzip', 'lz4', 'lzma', 'lzo', 'xz'] + algos = ['bzip2', 'gzip', 'lz4', 'lzma', 'lzo', 'xz', 'zstd'] if algo not in algos: raise ValueError("Unknown algorithm '%s'" % algo) names = {'lzma': 'lzma_alone', 'lzo': 'lzop'} diff --git a/tools/binman/etype/blob_dtb.py b/tools/binman/etype/blob_dtb.py index 5a6a4547485..6a3fbc47753 100644 --- a/tools/binman/etype/blob_dtb.py +++ b/tools/binman/etype/blob_dtb.py @@ -47,6 +47,10 @@ class Entry_blob_dtb(Entry_blob): def ProcessContents(self): """Re-read the DTB contents so that we get any calculated properties""" _, indata = state.GetFdtContents(self.GetFdtEtype()) + + if self.compress == 'zstd' and self.prepend != 'length': + self.Raise('The zstd compression requires a length header') + data = self.CompressData(indata) return self.ProcessContentsUpdate(data) diff --git a/tools/binman/ftest.py b/tools/binman/ftest.py index bfca9c1355d..0b1774046f7 100644 --- a/tools/binman/ftest.py +++ b/tools/binman/ftest.py @@ -107,7 +107,7 @@ BASE_DTB_PROPS = ['offset', 'size', 'image-pos'] REPACK_DTB_PROPS = ['orig-offset', 'orig-size'] # Supported compression bintools -COMP_BINTOOLS = ['bzip2', 'gzip', 'lz4', 'lzma_alone', 'lzop', 'xz'] +COMP_BINTOOLS = ['bzip2', 'gzip', 'lz4', 'lzma_alone', 'lzop', 'xz', 'zstd'] class TestFunctional(unittest.TestCase): """Functional tests for binman @@ -5881,7 +5881,8 @@ fdt fdtmap Extract the devicetree blob from the fdtmap def testCompUtilPadding(self): """Test padding of compression algorithms""" - for bintool in self.comp_bintools.values(): + # Skip zstd because it doesn't support padding + for bintool in [v for k,v in self.comp_bintools.items() if k != 'zstd']: self._CheckBintool(bintool) data = bintool.compress(COMPRESS_DATA) self.assertNotEqual(COMPRESS_DATA, data) @@ -5889,6 +5890,13 @@ fdt fdtmap Extract the devicetree blob from the fdtmap orig = bintool.decompress(data) self.assertEquals(COMPRESS_DATA, orig) + def testCompressDtbZstd(self): + """Test that zstd compress of device-tree files failed""" + with self.assertRaises(ValueError) as e: + self._DoTestFile('238_compress_dtb_zstd.dts') + self.assertIn("Node '/binman/u-boot-dtb': The zstd compression " + "requires a length header", str(e.exception)) + if __name__ == "__main__": unittest.main() diff --git a/tools/binman/test/238_compress_dtb_zstd.dts b/tools/binman/test/238_compress_dtb_zstd.dts new file mode 100644 index 00000000000..90cf85d1e2c --- /dev/null +++ b/tools/binman/test/238_compress_dtb_zstd.dts @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0+ + +/dts-v1/; + +/ { + #address-cells = <1>; + #size-cells = <1>; + + binman { + u-boot { + }; + u-boot-dtb { + compress = "zstd"; + }; + }; +}; -- cgit v1.3.1