diff options
| -rw-r--r-- | tools/buildman/README | 21 | ||||
| -rw-r--r-- | tools/buildman/boards.py | 402 | ||||
| -rw-r--r-- | tools/buildman/cmdline.py | 2 | ||||
| -rw-r--r-- | tools/buildman/control.py | 16 |
4 files changed, 426 insertions, 15 deletions
diff --git a/tools/buildman/README b/tools/buildman/README index c67a074cb50..0666bc7d564 100644 --- a/tools/buildman/README +++ b/tools/buildman/README @@ -128,10 +128,10 @@ Selecting which boards to build =============================== Buildman lets you build all boards, or a subset. Specify the subset by passing -command-line arguments that list the desired board name, architecture name, -SOC name, or anything else in the boards.cfg file. Multiple arguments are -allowed. Each argument will be interpreted as a regular expression, so -behaviour is a superset of exact or substring matching. Examples are: +command-line arguments that list the desired build target, architecture, +CPU, board name, vendor, SoC or options. Multiple arguments are allowed. Each +argument will be interpreted as a regular expression, so behaviour is a superset +of exact or substring matching. Examples are: * 'tegra20' All boards with a Tegra20 SoC * 'tegra' All boards with any Tegra Soc (Tegra20, Tegra30, Tegra114...) @@ -1054,7 +1054,6 @@ between one commit and the next. For example: $ buildman -b squash brppt1 -sU -boards.cfg is up to date. Nothing to do. Summary of 2 commits for 3 boards (3 threads, 3 jobs per thread) 01: Migrate bootlimit to Kconfig 02: Squashed commit of the following: @@ -1309,6 +1308,18 @@ Some options you might like are: break anything. But note this does not check bisectability! +Using boards.cfg +================ + +This file is no-longer needed by buildman but it is still generated in the +working directory. This helps avoid a delay on every build, since scanning all +the Kconfig files takes a few seconds. Use the -R flag to force regeneration +of the file - in that case buildman exits after writing the file. + +You should use 'buildman -nv <criteria>' instead of greoing the boards.cfg file, +since it may be dropped altogether in future. + + TODO ==== diff --git a/tools/buildman/boards.py b/tools/buildman/boards.py index ec143f9e0f5..c18914253e4 100644 --- a/tools/buildman/boards.py +++ b/tools/buildman/boards.py @@ -1,12 +1,93 @@ # SPDX-License-Identifier: GPL-2.0+ # Copyright (c) 2012 The Chromium OS Authors. +# Author: Simon Glass <[email protected]> +# Author: Masahiro Yamada <[email protected]> """Maintains a list of boards and allows them to be selected""" from collections import OrderedDict +import errno +import fnmatch +import glob +import multiprocessing +import os import re +import sys +import tempfile +import time from buildman import board +from buildman import kconfiglib + + +### constant variables ### +OUTPUT_FILE = 'boards.cfg' +CONFIG_DIR = 'configs' +SLEEP_TIME = 0.03 +COMMENT_BLOCK = '''# +# List of boards +# Automatically generated by %s: don't edit +# +# Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers + +''' % __file__ + + +def try_remove(f): + """Remove a file ignoring 'No such file or directory' error.""" + try: + os.remove(f) + except OSError as exception: + # Ignore 'No such file or directory' error + if exception.errno != errno.ENOENT: + raise + + +def output_is_new(output): + """Check if the output file is up to date. + + Returns: + True if the given output file exists and is newer than any of + *_defconfig, MAINTAINERS and Kconfig*. False otherwise. + """ + try: + ctime = os.path.getctime(output) + except OSError as exception: + if exception.errno == errno.ENOENT: + # return False on 'No such file or directory' error + return False + else: + raise + + for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR): + for filename in fnmatch.filter(filenames, '*_defconfig'): + if fnmatch.fnmatch(filename, '.*'): + continue + filepath = os.path.join(dirpath, filename) + if ctime < os.path.getctime(filepath): + return False + + for (dirpath, dirnames, filenames) in os.walk('.'): + for filename in filenames: + if (fnmatch.fnmatch(filename, '*~') or + not fnmatch.fnmatch(filename, 'Kconfig*') and + not filename == 'MAINTAINERS'): + continue + filepath = os.path.join(dirpath, filename) + if ctime < os.path.getctime(filepath): + return False + + # Detect a board that has been removed since the current board database + # was generated + with open(output, encoding="utf-8") as f: + for line in f: + if line[0] == '#' or line == '\n': + continue + defconfig = line.split()[6] + '_defconfig' + if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)): + return False + + return True class Expr: @@ -76,6 +157,190 @@ class Term: return True +class KconfigScanner: + + """Kconfig scanner.""" + + ### constant variable only used in this class ### + _SYMBOL_TABLE = { + 'arch' : 'SYS_ARCH', + 'cpu' : 'SYS_CPU', + 'soc' : 'SYS_SOC', + 'vendor' : 'SYS_VENDOR', + 'board' : 'SYS_BOARD', + 'config' : 'SYS_CONFIG_NAME', + 'options' : 'SYS_EXTRA_OPTIONS' + } + + def __init__(self): + """Scan all the Kconfig files and create a Kconfig object.""" + # Define environment variables referenced from Kconfig + os.environ['srctree'] = os.getcwd() + os.environ['UBOOTVERSION'] = 'dummy' + os.environ['KCONFIG_OBJDIR'] = '' + self._conf = kconfiglib.Kconfig(warn=False) + + def __del__(self): + """Delete a leftover temporary file before exit. + + The scan() method of this class creates a temporay file and deletes + it on success. If scan() method throws an exception on the way, + the temporary file might be left over. In that case, it should be + deleted in this destructor. + """ + if hasattr(self, '_tmpfile') and self._tmpfile: + try_remove(self._tmpfile) + + def scan(self, defconfig): + """Load a defconfig file to obtain board parameters. + + Arguments: + defconfig: path to the defconfig file to be processed + + Returns: + A dictionary of board parameters. It has a form of: + { + 'arch': <arch_name>, + 'cpu': <cpu_name>, + 'soc': <soc_name>, + 'vendor': <vendor_name>, + 'board': <board_name>, + 'target': <target_name>, + 'config': <config_header_name>, + 'options': <extra_options> + } + """ + # strip special prefixes and save it in a temporary file + fd, self._tmpfile = tempfile.mkstemp() + with os.fdopen(fd, 'w') as f: + for line in open(defconfig): + colon = line.find(':CONFIG_') + if colon == -1: + f.write(line) + else: + f.write(line[colon + 1:]) + + self._conf.load_config(self._tmpfile) + try_remove(self._tmpfile) + self._tmpfile = None + + params = {} + + # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc. + # Set '-' if the value is empty. + for key, symbol in list(self._SYMBOL_TABLE.items()): + value = self._conf.syms.get(symbol).str_value + if value: + params[key] = value + else: + params[key] = '-' + + defconfig = os.path.basename(defconfig) + params['target'], match, rear = defconfig.partition('_defconfig') + assert match and not rear, '%s : invalid defconfig' % defconfig + + # fix-up for aarch64 + if params['arch'] == 'arm' and params['cpu'] == 'armv8': + params['arch'] = 'aarch64' + + # fix-up options field. It should have the form: + # <config name>[:comma separated config options] + if params['options'] != '-': + params['options'] = params['config'] + ':' + \ + params['options'].replace(r'\"', '"') + elif params['config'] != params['target']: + params['options'] = params['config'] + + return params + + +class MaintainersDatabase: + + """The database of board status and maintainers.""" + + def __init__(self): + """Create an empty database.""" + self.database = {} + + def get_status(self, target): + """Return the status of the given board. + + The board status is generally either 'Active' or 'Orphan'. + Display a warning message and return '-' if status information + is not found. + + Returns: + 'Active', 'Orphan' or '-'. + """ + if not target in self.database: + print("WARNING: no status info for '%s'" % target, file=sys.stderr) + return '-' + + tmp = self.database[target][0] + if tmp.startswith('Maintained'): + return 'Active' + elif tmp.startswith('Supported'): + return 'Active' + elif tmp.startswith('Orphan'): + return 'Orphan' + else: + print(("WARNING: %s: unknown status for '%s'" % + (tmp, target)), file=sys.stderr) + return '-' + + def get_maintainers(self, target): + """Return the maintainers of the given board. + + Returns: + Maintainers of the board. If the board has two or more maintainers, + they are separated with colons. + """ + if not target in self.database: + print("WARNING: no maintainers for '%s'" % target, file=sys.stderr) + return '' + + return ':'.join(self.database[target][1]) + + def parse_file(self, file): + """Parse a MAINTAINERS file. + + Parse a MAINTAINERS file and accumulates board status and + maintainers information. + + Arguments: + file: MAINTAINERS file to be parsed + """ + targets = [] + maintainers = [] + status = '-' + for line in open(file, encoding="utf-8"): + # Check also commented maintainers + if line[:3] == '#M:': + line = line[1:] + tag, rest = line[:2], line[2:].strip() + if tag == 'M:': + maintainers.append(rest) + elif tag == 'F:': + # expand wildcard and filter by 'configs/*_defconfig' + for f in glob.glob(rest): + front, match, rear = f.partition('configs/') + if not front and match: + front, match, rear = rear.rpartition('_defconfig') + if match and not rear: + targets.append(front) + elif tag == 'S:': + status = rest + elif line == '\n': + for target in targets: + self.database[target] = (status, maintainers) + targets = [] + maintainers = [] + status = '-' + if targets: + for target in targets: + self.database[target] = (status, maintainers) + + class Boards: """Manage a list of boards.""" def __init__(self): @@ -288,3 +553,140 @@ class Boards: warnings.append(f"Boards not found: {', '.join(remaining)}\n") return result, warnings + + def scan_defconfigs_for_multiprocess(self, queue, defconfigs): + """Scan defconfig files and queue their board parameters + + This function is intended to be passed to + multiprocessing.Process() constructor. + + Arguments: + queue: An instance of multiprocessing.Queue(). + The resulting board parameters are written into it. + defconfigs: A sequence of defconfig files to be scanned. + """ + kconf_scanner = KconfigScanner() + for defconfig in defconfigs: + queue.put(kconf_scanner.scan(defconfig)) + + def read_queues(self, queues, params_list): + """Read the queues and append the data to the paramers list""" + for q in queues: + while not q.empty(): + params_list.append(q.get()) + + def scan_defconfigs(self, jobs=1): + """Collect board parameters for all defconfig files. + + This function invokes multiple processes for faster processing. + + Arguments: + jobs: The number of jobs to run simultaneously + """ + all_defconfigs = [] + for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR): + for filename in fnmatch.filter(filenames, '*_defconfig'): + if fnmatch.fnmatch(filename, '.*'): + continue + all_defconfigs.append(os.path.join(dirpath, filename)) + + total_boards = len(all_defconfigs) + processes = [] + queues = [] + for i in range(jobs): + defconfigs = all_defconfigs[total_boards * i // jobs : + total_boards * (i + 1) // jobs] + q = multiprocessing.Queue(maxsize=-1) + p = multiprocessing.Process( + target=self.scan_defconfigs_for_multiprocess, + args=(q, defconfigs)) + p.start() + processes.append(p) + queues.append(q) + + # The resulting data should be accumulated to this list + params_list = [] + + # Data in the queues should be retrieved preriodically. + # Otherwise, the queues would become full and subprocesses would get stuck. + while any([p.is_alive() for p in processes]): + self.read_queues(queues, params_list) + # sleep for a while until the queues are filled + time.sleep(SLEEP_TIME) + + # Joining subprocesses just in case + # (All subprocesses should already have been finished) + for p in processes: + p.join() + + # retrieve leftover data + self.read_queues(queues, params_list) + + return params_list + + def insert_maintainers_info(self, params_list): + """Add Status and Maintainers information to the board parameters list. + + Arguments: + params_list: A list of the board parameters + """ + database = MaintainersDatabase() + for (dirpath, dirnames, filenames) in os.walk('.'): + if 'MAINTAINERS' in filenames: + database.parse_file(os.path.join(dirpath, 'MAINTAINERS')) + + for i, params in enumerate(params_list): + target = params['target'] + params['status'] = database.get_status(target) + params['maintainers'] = database.get_maintainers(target) + params_list[i] = params + + def format_and_output(self, params_list, output): + """Write board parameters into a file. + + Columnate the board parameters, sort lines alphabetically, + and then write them to a file. + + Arguments: + params_list: The list of board parameters + output: The path to the output file + """ + FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target', + 'options', 'maintainers') + + # First, decide the width of each column + max_length = dict([ (f, 0) for f in FIELDS]) + for params in params_list: + for f in FIELDS: + max_length[f] = max(max_length[f], len(params[f])) + + output_lines = [] + for params in params_list: + line = '' + for f in FIELDS: + # insert two spaces between fields like column -t would + line += ' ' + params[f].ljust(max_length[f]) + output_lines.append(line.strip()) + + # ignore case when sorting + output_lines.sort(key=str.lower) + + with open(output, 'w', encoding="utf-8") as f: + f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n') + + def ensure_board_list(self, output, jobs=1, force=False, quiet=False): + """Generate a board database file if needed. + + Arguments: + output: The name of the output file + jobs: The number of jobs to run simultaneously + force: Force to generate the output even if it is new + quiet: True to avoid printing a message if nothing needs doing + """ + if not force and output_is_new(output): + if not quiet: + print("%s is up to date. Nothing to do." % output) + return + params_list = self.scan_defconfigs(jobs) + self.insert_maintainers_info(params_list) + self.format_and_output(params_list, output) diff --git a/tools/buildman/cmdline.py b/tools/buildman/cmdline.py index cef2068c983..b29c1eb5ee7 100644 --- a/tools/buildman/cmdline.py +++ b/tools/buildman/cmdline.py @@ -89,6 +89,8 @@ def ParseArgs(): default=False, help="Use full toolchain path in CROSS_COMPILE") parser.add_option('-P', '--per-board-out-dir', action='store_true', default=False, help="Use an O= (output) directory per board rather than per thread") + parser.add_option('-R', '--regen-board-list', action='store_true', + help='Force regeneration of the list of boards, like the old boards.cfg file') parser.add_option('-s', '--summary', action='store_true', default=False, help='Show a build summary') parser.add_option('-S', '--show-sizes', action='store_true', diff --git a/tools/buildman/control.py b/tools/buildman/control.py index 8d3e781d51a..79ce2f6978a 100644 --- a/tools/buildman/control.py +++ b/tools/buildman/control.py @@ -186,18 +186,14 @@ def DoBuildman(options, args, toolchains=None, make_func=None, brds=None, if not os.path.exists(options.output_dir): os.makedirs(options.output_dir) board_file = os.path.join(options.output_dir, 'boards.cfg') - our_path = os.path.dirname(os.path.realpath(__file__)) - genboardscfg = os.path.join(our_path, '../genboardscfg.py') - if not os.path.exists(genboardscfg): - genboardscfg = os.path.join(options.git, 'tools/genboardscfg.py') - status = subprocess.call([genboardscfg, '-q', '-o', board_file]) - if status != 0: - # Older versions don't support -q - status = subprocess.call([genboardscfg, '-o', board_file]) - if status != 0: - sys.exit("Failed to generate boards.cfg") brds = boards.Boards() + brds.ensure_board_list(board_file, + options.threads or multiprocessing.cpu_count(), + force=options.regen_board_list, + quiet=not options.verbose) + if options.regen_board_list: + return 0 brds.read_boards(board_file) exclude = [] |