From bd7ebbf17b3b8c28fcf8c797a6e29bc78e220157 Mon Sep 17 00:00:00 2001 From: Bernard Xiong Date: Sun, 13 Jan 2019 14:24:00 +0800 Subject: [DFS][romfs] fix the mkrom issue when file/dir size zero --- components/dfs/filesystems/romfs/dfs_romfs.h | 8 +- components/dfs/filesystems/romfs/mkromfs.py | 252 ------------------------- tools/mkromfs.py | 264 +++++++++++++++++++++++++++ 3 files changed, 268 insertions(+), 256 deletions(-) delete mode 100644 components/dfs/filesystems/romfs/mkromfs.py create mode 100644 tools/mkromfs.py diff --git a/components/dfs/filesystems/romfs/dfs_romfs.h b/components/dfs/filesystems/romfs/dfs_romfs.h index 74b256c8a6..05d2906449 100644 --- a/components/dfs/filesystems/romfs/dfs_romfs.h +++ b/components/dfs/filesystems/romfs/dfs_romfs.h @@ -17,11 +17,11 @@ struct romfs_dirent { - rt_uint32_t type; /* dirent type */ + rt_uint32_t type; /* dirent type */ - const char *name; /* dirent name */ - const rt_uint8_t *data; /* file date ptr */ - rt_size_t size; /* file size */ + const char *name; /* dirent name */ + const rt_uint8_t *data; /* file date ptr */ + rt_size_t size; /* file size */ }; int dfs_romfs_init(void); diff --git a/components/dfs/filesystems/romfs/mkromfs.py b/components/dfs/filesystems/romfs/mkromfs.py deleted file mode 100644 index f3c12876d2..0000000000 --- a/components/dfs/filesystems/romfs/mkromfs.py +++ /dev/null @@ -1,252 +0,0 @@ -#!/usr/bin/env python - -import sys -import os - -import struct -from collections import namedtuple -import StringIO - -import argparse -parser = argparse.ArgumentParser() -parser.add_argument('rootdir', type=str, help='the path to rootfs') -parser.add_argument('output', type=argparse.FileType('wb'), nargs='?', help='output file name') -parser.add_argument('--dump', action='store_true', help='dump the fs hierarchy') -parser.add_argument('--binary', action='store_true', help='output binary file') -parser.add_argument('--addr', default='0', help='set the base address of the binary file, default to 0.') - -class File(object): - def __init__(self, name): - self._name = name - self._data = open(name, 'rb').read() - - @property - def name(self): - return self._name - - @property - def c_name(self): - return '_' + self._name.replace('.', '_') - - @property - def bin_name(self): - # Pad to 4 bytes boundary with \0 - pad_len = 4 - bn = self._name + '\0' * (pad_len - len(self._name) % pad_len) - return bn - - def c_data(self, prefix=''): - '''Get the C code represent of the file content.''' - head = 'static const rt_uint8_t %s[] = {\n' % \ - (prefix + self.c_name) - tail = '\n};' - return head + ','.join(('0x%02x' % ord(i) for i in self._data)) + tail - - @property - def entry_size(self): - return len(self._data) - - def bin_data(self, base_addr=0x0): - return bytes(self._data) - - def dump(self, indent=0): - print('%s%s' % (' ' * indent, self._name)) - -class Folder(object): - bin_fmt = struct.Struct('IIII') - bin_item = namedtuple('dirent', 'type, name, data, size') - - def __init__(self, name): - self._name = name - self._children = [] - - @property - def name(self): - return self._name - - @property - def c_name(self): - # add _ to avoid conflict with C key words. - return '_' + self._name - - @property - def bin_name(self): - # Pad to 4 bytes boundary with \0 - pad_len = 4 - bn = self._name + '\0' * (pad_len - len(self._name) % pad_len) - return bn - - def walk(self): - # os.listdir will return unicode list if the argument is unicode. - # TODO: take care of the unicode names - for ent in os.listdir(u'.'): - if os.path.isdir(ent): - cwd = os.getcwdu() - d = Folder(ent) - # depth-first - os.chdir(os.path.join(cwd, ent)) - d.walk() - # restore the cwd - os.chdir(cwd) - self._children.append(d) - else: - self._children.append(File(ent)) - - def sort(self): - def _sort(x, y): - if x.name == y.name: - return 0 - elif x.name > y.name: - return 1 - else: - return -1 - self._children.sort(cmp=_sort) - - # sort recursively - for c in self._children: - if isinstance(c, Folder): - c.sort() - - def dump(self, indent=0): - print('%s%s' % (' ' * indent, self._name)) - for c in self._children: - c.dump(indent + 1) - - def c_data(self, prefix=''): - '''get the C code represent of the folder. - - It is recursive.''' - # make the current dirent - # static is good. Only root dirent is global visible. - dhead = 'static const struct romfs_dirent %s[] = {\n' % (prefix + self.c_name) - dtail = '\n};' - body_fmt = ' {{{type}, "{name}", (rt_uint8_t *){data}, sizeof({data})/sizeof({data}[0])}}' - # prefix of children - cpf = prefix+self.c_name - body_li = [] - payload_li = [] - for c in self._children: - if isinstance(c, File): - tp = 'ROMFS_DIRENT_FILE' - elif isinstance(c, Folder): - tp = 'ROMFS_DIRENT_DIR' - else: - assert False, 'Unkown instance:%s' % str(c) - body_li.append(body_fmt.format(type=tp, - name=c.name, - data=cpf+c.c_name)) - payload_li.append(c.c_data(prefix=cpf)) - - # All the data we need is defined in payload so we should append the - # dirent to it. It also meet the depth-first policy in this code. - payload_li.append(dhead + ',\n'.join(body_li) + dtail) - - return '\n\n'.join(payload_li) - - @property - def entry_size(self): - return len(self._children) - - def bin_data(self, base_addr=0x0): - '''Return StringIO object''' - # The binary layout is different from the C code layout. We put the - # dirent before the payload in this mode. But the idea is still simple: - # Depth-First. - - #{ - # rt_uint32_t type; - # const char *name; - # const rt_uint8_t *data; - # rt_size_t size; - #} - d_li = [] - # payload base - p_base = base_addr + self.bin_fmt.size * self.entry_size - # the length to record how many data is in - v_len = p_base - # payload - p_li = [] - for c in self._children: - if isinstance(c, File): - # ROMFS_DIRENT_FILE - tp = 0 - elif isinstance(c, Folder): - # ROMFS_DIRENT_DIR - tp = 1 - else: - assert False, 'Unkown instance:%s' % str(c) - - name = bytes(c.bin_name) - name_addr = v_len - v_len += len(name) - - data = c.bin_data(base_addr=v_len) - data_addr = v_len - # pad the data to 4 bytes boundary - pad_len = 4 - if len(data) % pad_len != 0: - data += '\0' * (pad_len - len(data) % pad_len) - v_len += len(data) - - d_li.append(self.bin_fmt.pack(*self.bin_item( - type=tp, - name=name_addr, - data=data_addr, - size=c.entry_size))) - - p_li.extend((name, data)) - - return bytes().join(d_li) + bytes().join(p_li) - -def get_c_data(tree): - # Handle the root dirent specially. - root_dirent_fmt = '''/* Generated by mkromfs. Edit with caution. */ -#include -#include - -{data} - -const struct romfs_dirent {name} = {{ - ROMFS_DIRENT_DIR, "/", (rt_uint8_t *){rootdirent}, sizeof({rootdirent})/sizeof({rootdirent}[0]) -}}; -''' - - return root_dirent_fmt.format(name='romfs_root', - rootdirent=tree.c_name, - data=tree.c_data()) - -def get_bin_data(tree, base_addr): - v_len = base_addr + Folder.bin_fmt.size - name = bytes('/\0\0\0') - name_addr = v_len - v_len += len(name) - data_addr = v_len - # root entry - data = Folder.bin_fmt.pack(*Folder.bin_item(type=1, - name=name_addr, - data=data_addr, - size=tree.entry_size)) - return data + name + tree.bin_data(v_len) - -if __name__ == '__main__': - args = parser.parse_args() - - os.chdir(args.rootdir) - - tree = Folder('romfs_root') - tree.walk() - tree.sort() - - if args.dump: - tree.dump() - - if args.binary: - data = get_bin_data(tree, int(args.addr, 16)) - else: - data = get_c_data(tree) - - output = args.output - if not output: - output = sys.stdout - - output.write(data) diff --git a/tools/mkromfs.py b/tools/mkromfs.py new file mode 100644 index 0000000000..5371106987 --- /dev/null +++ b/tools/mkromfs.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python + +import sys +import os + +import struct +from collections import namedtuple +import StringIO + +import argparse +parser = argparse.ArgumentParser() +parser.add_argument('rootdir', type=str, help='the path to rootfs') +parser.add_argument('output', type=argparse.FileType('wb'), nargs='?', help='output file name') +parser.add_argument('--dump', action='store_true', help='dump the fs hierarchy') +parser.add_argument('--binary', action='store_true', help='output binary file') +parser.add_argument('--addr', default='0', help='set the base address of the binary file, default to 0.') + +class File(object): + def __init__(self, name): + self._name = name + self._data = open(name, 'rb').read() + + @property + def name(self): + return self._name + + @property + def c_name(self): + return '_' + self._name.replace('.', '_') + + @property + def bin_name(self): + # Pad to 4 bytes boundary with \0 + pad_len = 4 + bn = self._name + '\0' * (pad_len - len(self._name) % pad_len) + return bn + + def c_data(self, prefix=''): + '''Get the C code represent of the file content.''' + head = 'static const rt_uint8_t %s[] = {\n' % \ + (prefix + self.c_name) + tail = '\n};' + + if self.entry_size == 0: + return '' + + return head + ','.join(('0x%02x' % ord(i) for i in self._data)) + tail + + @property + def entry_size(self): + return len(self._data) + + def bin_data(self, base_addr=0x0): + return bytes(self._data) + + def dump(self, indent=0): + print('%s%s' % (' ' * indent, self._name)) + +class Folder(object): + bin_fmt = struct.Struct('IIII') + bin_item = namedtuple('dirent', 'type, name, data, size') + + def __init__(self, name): + self._name = name + self._children = [] + + @property + def name(self): + return self._name + + @property + def c_name(self): + # add _ to avoid conflict with C key words. + return '_' + self._name + + @property + def bin_name(self): + # Pad to 4 bytes boundary with \0 + pad_len = 4 + bn = self._name + '\0' * (pad_len - len(self._name) % pad_len) + return bn + + def walk(self): + # os.listdir will return unicode list if the argument is unicode. + # TODO: take care of the unicode names + for ent in os.listdir(u'.'): + if os.path.isdir(ent): + cwd = os.getcwdu() + d = Folder(ent) + # depth-first + os.chdir(os.path.join(cwd, ent)) + d.walk() + # restore the cwd + os.chdir(cwd) + self._children.append(d) + else: + self._children.append(File(ent)) + + def sort(self): + def _sort(x, y): + if x.name == y.name: + return 0 + elif x.name > y.name: + return 1 + else: + return -1 + self._children.sort(cmp=_sort) + + # sort recursively + for c in self._children: + if isinstance(c, Folder): + c.sort() + + def dump(self, indent=0): + print('%s%s' % (' ' * indent, self._name)) + for c in self._children: + c.dump(indent + 1) + + def c_data(self, prefix=''): + '''get the C code represent of the folder. + + It is recursive.''' + # make the current dirent + # static is good. Only root dirent is global visible. + if self.entry_size == 0: + return '' + + dhead = 'static const struct romfs_dirent %s[] = {\n' % (prefix + self.c_name) + dtail = '\n};' + body_fmt = ' {{{type}, "{name}", (rt_uint8_t *){data}, sizeof({data})/sizeof({data}[0])}}' + body_fmt0= ' {{{type}, "{name}", RT_NULL, 0}}' + # prefix of children + cpf = prefix+self.c_name + body_li = [] + payload_li = [] + for c in self._children: + entry_size = c.entry_size + if isinstance(c, File): + tp = 'ROMFS_DIRENT_FILE' + elif isinstance(c, Folder): + tp = 'ROMFS_DIRENT_DIR' + else: + assert False, 'Unkown instance:%s' % str(c) + if entry_size == 0: + body_li.append(body_fmt0.format(type=tp, name = c.name)) + else: + body_li.append(body_fmt.format(type=tp, + name=c.name, + data=cpf+c.c_name)) + payload_li.append(c.c_data(prefix=cpf)) + + # All the data we need is defined in payload so we should append the + # dirent to it. It also meet the depth-first policy in this code. + payload_li.append(dhead + ',\n'.join(body_li) + dtail) + + return '\n\n'.join(payload_li) + + @property + def entry_size(self): + return len(self._children) + + def bin_data(self, base_addr=0x0): + '''Return StringIO object''' + # The binary layout is different from the C code layout. We put the + # dirent before the payload in this mode. But the idea is still simple: + # Depth-First. + + #{ + # rt_uint32_t type; + # const char *name; + # const rt_uint8_t *data; + # rt_size_t size; + #} + d_li = [] + # payload base + p_base = base_addr + self.bin_fmt.size * self.entry_size + # the length to record how many data is in + v_len = p_base + # payload + p_li = [] + for c in self._children: + if isinstance(c, File): + # ROMFS_DIRENT_FILE + tp = 0 + elif isinstance(c, Folder): + # ROMFS_DIRENT_DIR + tp = 1 + else: + assert False, 'Unkown instance:%s' % str(c) + + name = bytes(c.bin_name) + name_addr = v_len + v_len += len(name) + + data = c.bin_data(base_addr=v_len) + data_addr = v_len + # pad the data to 4 bytes boundary + pad_len = 4 + if len(data) % pad_len != 0: + data += '\0' * (pad_len - len(data) % pad_len) + v_len += len(data) + + d_li.append(self.bin_fmt.pack(*self.bin_item( + type=tp, + name=name_addr, + data=data_addr, + size=c.entry_size))) + + p_li.extend((name, data)) + + return bytes().join(d_li) + bytes().join(p_li) + +def get_c_data(tree): + # Handle the root dirent specially. + root_dirent_fmt = '''/* Generated by mkromfs. Edit with caution. */ +#include +#include + +{data} + +const struct romfs_dirent {name} = {{ + ROMFS_DIRENT_DIR, "/", (rt_uint8_t *){rootdirent}, sizeof({rootdirent})/sizeof({rootdirent}[0]) +}}; +''' + + return root_dirent_fmt.format(name='romfs_root', + rootdirent=tree.c_name, + data=tree.c_data()) + +def get_bin_data(tree, base_addr): + v_len = base_addr + Folder.bin_fmt.size + name = bytes('/\0\0\0') + name_addr = v_len + v_len += len(name) + data_addr = v_len + # root entry + data = Folder.bin_fmt.pack(*Folder.bin_item(type=1, + name=name_addr, + data=data_addr, + size=tree.entry_size)) + return data + name + tree.bin_data(v_len) + +if __name__ == '__main__': + args = parser.parse_args() + + os.chdir(args.rootdir) + + tree = Folder('romfs_root') + tree.walk() + tree.sort() + + if args.dump: + tree.dump() + + if args.binary: + data = get_bin_data(tree, int(args.addr, 16)) + else: + data = get_c_data(tree) + + output = args.output + if not output: + output = sys.stdout + + output.write(data) -- cgit v1.3.1 From 9af2d2935136ef46e9bc455777165934d9df96b3 Mon Sep 17 00:00:00 2001 From: Bernard Xiong Date: Sun, 13 Jan 2019 14:31:55 +0800 Subject: [DFS][romfs] code cleanup --- components/dfs/filesystems/romfs/dfs_romfs.h | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/components/dfs/filesystems/romfs/dfs_romfs.h b/components/dfs/filesystems/romfs/dfs_romfs.h index 05d2906449..2004e55bc6 100644 --- a/components/dfs/filesystems/romfs/dfs_romfs.h +++ b/components/dfs/filesystems/romfs/dfs_romfs.h @@ -1,10 +1,11 @@ /* - * Copyright (c) 2006-2018, RT-Thread Development Team + * Copyright (c) 2006-2019, RT-Thread Development Team * * SPDX-License-Identifier: Apache-2.0 * * Change Logs: * Date Author Notes + * 2019/01/13 Bernard code cleanup */ #ifndef __DFS_ROMFS_H__ @@ -17,11 +18,11 @@ struct romfs_dirent { - rt_uint32_t type; /* dirent type */ + rt_uint32_t type; /* dirent type */ - const char *name; /* dirent name */ - const rt_uint8_t *data; /* file date ptr */ - rt_size_t size; /* file size */ + const char *name; /* dirent name */ + const rt_uint8_t *data; /* file date ptr */ + rt_size_t size; /* file size */ }; int dfs_romfs_init(void); -- cgit v1.3.1 From b998c4ed8dbf12e0744b7ffee95cb9c1ebcbc108 Mon Sep 17 00:00:00 2001 From: Bernard Xiong Date: Sun, 13 Jan 2019 14:33:24 +0800 Subject: [DFS][romfs] code cleanup --- components/dfs/filesystems/romfs/dfs_romfs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/dfs/filesystems/romfs/dfs_romfs.h b/components/dfs/filesystems/romfs/dfs_romfs.h index 2004e55bc6..efb6d44caf 100644 --- a/components/dfs/filesystems/romfs/dfs_romfs.h +++ b/components/dfs/filesystems/romfs/dfs_romfs.h @@ -18,7 +18,7 @@ struct romfs_dirent { - rt_uint32_t type; /* dirent type */ + rt_uint32_t type; /* dirent type */ const char *name; /* dirent name */ const rt_uint8_t *data; /* file date ptr */ -- cgit v1.3.1