From ea4d6dead3728e5e7e6a2f070ca240de832bce92 Mon Sep 17 00:00:00 2001 From: Simon Glass Date: Sat, 23 Sep 2023 13:44:14 -0600 Subject: moveconfig: Rename the tool to qconfig This does not move configs anymore, but queries them, based on a database it can build. Rename the tool to better reflect its purpose. Signed-off-by: Simon Glass --- tools/moveconfig.py | 1624 --------------------------------------------------- tools/qconfig.py | 1624 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 1624 insertions(+), 1624 deletions(-) delete mode 100755 tools/moveconfig.py create mode 100755 tools/qconfig.py diff --git a/tools/moveconfig.py b/tools/moveconfig.py deleted file mode 100755 index ed75d3277d6..00000000000 --- a/tools/moveconfig.py +++ /dev/null @@ -1,1624 +0,0 @@ -#!/usr/bin/env python3 -# SPDX-License-Identifier: GPL-2.0+ -# -# Author: Masahiro Yamada -# - -""" -Move config options from headers to defconfig files. - -See doc/develop/moveconfig.rst for documentation. -""" - -from argparse import ArgumentParser -import collections -from contextlib import ExitStack -import doctest -import filecmp -import fnmatch -import glob -import multiprocessing -import os -import queue -import re -import shutil -import subprocess -import sys -import tempfile -import threading -import time -import unittest - -import asteval -from buildman import bsettings -from buildman import kconfiglib -from buildman import toolchain -from u_boot_pylib import terminal - -SHOW_GNU_MAKE = 'scripts/show-gnu-make' -SLEEP_TIME=0.03 - -STATE_IDLE = 0 -STATE_DEFCONFIG = 1 -STATE_AUTOCONF = 2 -STATE_SAVEDEFCONFIG = 3 - -AUTO_CONF_PATH = 'include/config/auto.conf' -CONFIG_DATABASE = 'moveconfig.db' -FAILED_LIST = 'moveconfig.failed' - -CONFIG_LEN = len('CONFIG_') - -SIZES = { - 'SZ_1': 0x00000001, 'SZ_2': 0x00000002, - 'SZ_4': 0x00000004, 'SZ_8': 0x00000008, - 'SZ_16': 0x00000010, 'SZ_32': 0x00000020, - 'SZ_64': 0x00000040, 'SZ_128': 0x00000080, - 'SZ_256': 0x00000100, 'SZ_512': 0x00000200, - 'SZ_1K': 0x00000400, 'SZ_2K': 0x00000800, - 'SZ_4K': 0x00001000, 'SZ_8K': 0x00002000, - 'SZ_16K': 0x00004000, 'SZ_32K': 0x00008000, - 'SZ_64K': 0x00010000, 'SZ_128K': 0x00020000, - 'SZ_256K': 0x00040000, 'SZ_512K': 0x00080000, - 'SZ_1M': 0x00100000, 'SZ_2M': 0x00200000, - 'SZ_4M': 0x00400000, 'SZ_8M': 0x00800000, - 'SZ_16M': 0x01000000, 'SZ_32M': 0x02000000, - 'SZ_64M': 0x04000000, 'SZ_128M': 0x08000000, - 'SZ_256M': 0x10000000, 'SZ_512M': 0x20000000, - 'SZ_1G': 0x40000000, 'SZ_2G': 0x80000000, - 'SZ_4G': 0x100000000 -} - -RE_REMOVE_DEFCONFIG = re.compile(r'(.*)_defconfig') - -# CONFIG symbols present in the build system (from Linux) but not actually used -# in U-Boot; KCONFIG symbols -IGNORE_SYMS = ['DEBUG_SECTION_MISMATCH', 'FTRACE_MCOUNT_RECORD', 'GCOV_KERNEL', - 'GCOV_PROFILE_ALL', 'KALLSYMS', 'KASAN', 'MODVERSIONS', 'SHELL', - 'TPL_BUILD', 'VPL_BUILD', 'IS_ENABLED', 'FOO', 'IF_ENABLED_INT', - 'IS_ENABLED_', 'IS_ENABLED_1', 'IS_ENABLED_2', 'IS_ENABLED_3', - 'SPL_', 'TPL_', 'SPL_FOO', 'TPL_FOO', 'TOOLS_FOO', - 'ACME', 'SPL_ACME', 'TPL_ACME', 'TRACE_BRANCH_PROFILING', - 'VAL', '_UNDEFINED', 'SPL_BUILD', ] - -SPL_PREFIXES = ['SPL_', 'TPL_', 'VPL_', 'TOOLS_'] - -### helper functions ### -def check_top_directory(): - """Exit if we are not at the top of source directory.""" - for fname in 'README', 'Licenses': - if not os.path.exists(fname): - sys.exit('Please run at the top of source directory.') - -def check_clean_directory(): - """Exit if the source tree is not clean.""" - for fname in '.config', 'include/config': - if os.path.exists(fname): - sys.exit("source tree is not clean, please run 'make mrproper'") - -def get_make_cmd(): - """Get the command name of GNU Make. - - U-Boot needs GNU Make for building, but the command name is not - necessarily "make". (for example, "gmake" on FreeBSD). - Returns the most appropriate command name on your system. - """ - with subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) as proc: - ret = proc.communicate() - if proc.returncode: - sys.exit('GNU Make not found') - return ret[0].rstrip() - -def get_matched_defconfig(line): - """Get the defconfig files that match a pattern - - Args: - line (str): Path or filename to match, e.g. 'configs/snow_defconfig' or - 'k2*_defconfig'. If no directory is provided, 'configs/' is - prepended - - Returns: - list of str: a list of matching defconfig files - """ - dirname = os.path.dirname(line) - if dirname: - pattern = line - else: - pattern = os.path.join('configs', line) - return glob.glob(pattern) + glob.glob(pattern + '_defconfig') - -def get_matched_defconfigs(defconfigs_file): - """Get all the defconfig files that match the patterns in a file. - - Args: - defconfigs_file (str): File containing a list of defconfigs to process, - or '-' to read the list from stdin - - Returns: - list of str: A list of paths to defconfig files, with no duplicates - """ - defconfigs = [] - with ExitStack() as stack: - if defconfigs_file == '-': - inf = sys.stdin - defconfigs_file = 'stdin' - else: - inf = stack.enter_context(open(defconfigs_file, encoding='utf-8')) - for i, line in enumerate(inf): - line = line.strip() - if not line: - continue # skip blank lines silently - if ' ' in line: - line = line.split(' ')[0] # handle 'git log' input - matched = get_matched_defconfig(line) - if not matched: - print(f"warning: {defconfigs_file}:{i + 1}: no defconfig matched '{line}'", - file=sys.stderr) - - defconfigs += matched - - # use set() to drop multiple matching - return [defconfig[len('configs') + 1:] for defconfig in set(defconfigs)] - -def get_all_defconfigs(): - """Get all the defconfig files under the configs/ directory. - - Returns: - list of str: List of paths to defconfig files - """ - defconfigs = [] - for (dirpath, _, filenames) in os.walk('configs'): - dirpath = dirpath[len('configs') + 1:] - for filename in fnmatch.filter(filenames, '*_defconfig'): - defconfigs.append(os.path.join(dirpath, filename)) - - return defconfigs - -def write_file(fname, data): - """Write data to a file - - Args: - fname (str): Filename to write to - data (list of str): Lines to write (with or without trailing newline); - or str to write - """ - with open(fname, 'w', encoding='utf-8') as out: - if isinstance(data, list): - for line in data: - print(line.rstrip('\n'), file=out) - else: - out.write(data) - -def read_file(fname, as_lines=True, skip_unicode=False): - """Read a file and return the contents - - Args: - fname (str): Filename to read from - as_lines (bool): Return file contents as a list of lines - skip_unicode (bool): True to report unicode errors and continue - - Returns: - iter of str: List of ;ines from the file with newline removed; str if - as_lines is False with newlines intact; or None if a unicode error - occurred - - Raises: - UnicodeDecodeError: Unicode error occurred when reading - """ - with open(fname, encoding='utf-8') as inf: - try: - if as_lines: - return [line.rstrip('\n') for line in inf.readlines()] - return inf.read() - except UnicodeDecodeError as exc: - if not skip_unicode: - raise - print(f"Failed on file '{fname}: {exc}") - return None - -def try_expand(line): - """If value looks like an expression, try expanding it - Otherwise just return the existing value - """ - if line.find('=') == -1: - return line - - try: - aeval = asteval.Interpreter( usersyms=SIZES, minimal=True ) - cfg, val = re.split("=", line) - val= val.strip('\"') - if re.search(r'[*+-/]|<<|SZ_+|\(([^\)]+)\)', val): - newval = hex(aeval(val)) - print(f'\tExpanded expression {val} to {newval}') - return cfg+'='+newval - except: - print(f'\tFailed to expand expression in {line}') - - return line - - -### classes ### -class Progress: - - """Progress Indicator""" - - def __init__(self, col, total): - """Create a new progress indicator. - - Args: - color_enabled (bool): True for colour output - total (int): A number of defconfig files to process. - """ - self.col = col - self.current = 0 - self.good = 0 - self.total = total - - def inc(self, success): - """Increment the number of processed defconfig files. - - Args: - success (bool): True if processing succeeded - """ - self.good += success - self.current += 1 - - def show(self): - """Display the progress.""" - if self.current != self.total: - line = self.col.build(self.col.GREEN, f'{self.good:5d}') - line += self.col.build(self.col.RED, - f'{self.current - self.good:5d}') - line += self.col.build(self.col.MAGENTA, - f'/{self.total - self.current}') - print(f'{line} \r', end='') - sys.stdout.flush() - - -class KconfigScanner: - """Kconfig scanner.""" - - def __init__(self): - """Scan all the Kconfig files and create a Config object.""" - # Define environment variables referenced from Kconfig - os.environ['srctree'] = os.getcwd() - os.environ['UBOOTVERSION'] = 'dummy' - os.environ['KCONFIG_OBJDIR'] = '' - os.environ['CC'] = 'gcc' - self.conf = kconfiglib.Kconfig() - - -class KconfigParser: - - """A parser of .config and include/autoconf.mk.""" - - re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') - re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') - - def __init__(self, args, build_dir): - """Create a new parser. - - Args: - args (Namespace): program arguments - build_dir: Build directory. - """ - self.args = args - self.dotconfig = os.path.join(build_dir, '.config') - self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk') - self.spl_autoconf = os.path.join(build_dir, 'spl', 'include', - 'autoconf.mk') - self.config_autoconf = os.path.join(build_dir, AUTO_CONF_PATH) - self.defconfig = os.path.join(build_dir, 'defconfig') - - def get_arch(self): - """Parse .config file and return the architecture. - - Returns: - Architecture name (e.g. 'arm'). - """ - arch = '' - cpu = '' - for line in read_file(self.dotconfig): - m_arch = self.re_arch.match(line) - if m_arch: - arch = m_arch.group(1) - continue - m_cpu = self.re_cpu.match(line) - if m_cpu: - cpu = m_cpu.group(1) - - if not arch: - return None - - # fix-up for aarch64 - if arch == 'arm' and cpu == 'armv8': - arch = 'aarch64' - - return arch - - -class DatabaseThread(threading.Thread): - """This thread processes results from Slot threads. - - It collects the data in the master config directary. There is only one - result thread, and this helps to serialise the build output. - """ - def __init__(self, config_db, db_queue): - """Set up a new result thread - - Args: - builder: Builder which will be sent each result - """ - threading.Thread.__init__(self) - self.config_db = config_db - self.db_queue= db_queue - - def run(self): - """Called to start up the result thread. - - We collect the next result job and pass it on to the build. - """ - while True: - defconfig, configs = self.db_queue.get() - self.config_db[defconfig] = configs - self.db_queue.task_done() - - -class Slot: - - """A slot to store a subprocess. - - Each instance of this class handles one subprocess. - This class is useful to control multiple threads - for faster processing. - """ - - def __init__(self, toolchains, args, progress, devnull, make_cmd, - reference_src_dir, db_queue, col): - """Create a new process slot. - - Args: - toolchains: Toolchains object containing toolchains. - args: Program arguments - progress: A progress indicator. - devnull: A file object of '/dev/null'. - make_cmd: command name of GNU Make. - reference_src_dir: Determine the true starting config state from this - source tree. - db_queue: output queue to write config info for the database - col (terminal.Color): Colour object - """ - self.toolchains = toolchains - self.args = args - self.progress = progress - self.build_dir = tempfile.mkdtemp() - self.devnull = devnull - self.make_cmd = (make_cmd, 'O=' + self.build_dir) - self.reference_src_dir = reference_src_dir - self.db_queue = db_queue - self.col = col - self.parser = KconfigParser(args, self.build_dir) - self.state = STATE_IDLE - self.failed_boards = set() - self.defconfig = None - self.log = [] - self.current_src_dir = None - self.proc = None - - def __del__(self): - """Delete the working directory - - This function makes sure the temporary directory is cleaned away - even if Python suddenly dies due to error. It should be done in here - because it is guaranteed the destructor is always invoked when the - instance of the class gets unreferenced. - - If the subprocess is still running, wait until it finishes. - """ - if self.state != STATE_IDLE: - while self.proc.poll() is None: - pass - shutil.rmtree(self.build_dir) - - def add(self, defconfig): - """Assign a new subprocess for defconfig and add it to the slot. - - If the slot is vacant, create a new subprocess for processing the - given defconfig and add it to the slot. Just returns False if - the slot is occupied (i.e. the current subprocess is still running). - - Args: - defconfig (str): defconfig name. - - Returns: - Return True on success or False on failure - """ - if self.state != STATE_IDLE: - return False - - self.defconfig = defconfig - self.log = [] - self.current_src_dir = self.reference_src_dir - self.do_defconfig() - return True - - def poll(self): - """Check the status of the subprocess and handle it as needed. - - Returns True if the slot is vacant (i.e. in idle state). - If the configuration is successfully finished, assign a new - subprocess to build include/autoconf.mk. - If include/autoconf.mk is generated, invoke the parser to - parse the .config and the include/autoconf.mk, moving - config options to the .config as needed. - If the .config was updated, run "make savedefconfig" to sync - it, update the original defconfig, and then set the slot back - to the idle state. - - Returns: - Return True if the subprocess is terminated, False otherwise - """ - if self.state == STATE_IDLE: - return True - - if self.proc.poll() is None: - return False - - if self.proc.poll() != 0: - self.handle_error() - elif self.state == STATE_DEFCONFIG: - if self.reference_src_dir and not self.current_src_dir: - self.do_savedefconfig() - else: - self.do_autoconf() - elif self.state == STATE_AUTOCONF: - if self.current_src_dir: - self.current_src_dir = None - self.do_defconfig() - elif self.args.build_db: - self.do_build_db() - else: - self.do_savedefconfig() - elif self.state == STATE_SAVEDEFCONFIG: - self.update_defconfig() - else: - sys.exit('Internal Error. This should not happen.') - - return self.state == STATE_IDLE - - def handle_error(self): - """Handle error cases.""" - - self.log.append(self.col.build(self.col.RED, 'Failed to process', - bright=True)) - if self.args.verbose: - for line in self.proc.stderr.read().decode().splitlines(): - self.log.append(self.col.build(self.col.CYAN, line, True)) - self.finish(False) - - def do_defconfig(self): - """Run 'make _defconfig' to create the .config file.""" - - cmd = list(self.make_cmd) - cmd.append(self.defconfig) - self.proc = subprocess.Popen(cmd, stdout=self.devnull, - stderr=subprocess.PIPE, - cwd=self.current_src_dir) - self.state = STATE_DEFCONFIG - - def do_autoconf(self): - """Run 'make AUTO_CONF_PATH'.""" - - arch = self.parser.get_arch() - try: - tchain = self.toolchains.Select(arch) - except ValueError: - self.log.append(self.col.build( - self.col.YELLOW, - f"Tool chain for '{arch}' is missing: do nothing")) - self.finish(False) - return - env = tchain.MakeEnvironment(False) - - cmd = list(self.make_cmd) - cmd.append('KCONFIG_IGNORE_DUPLICATES=1') - cmd.append(AUTO_CONF_PATH) - self.proc = subprocess.Popen(cmd, stdout=self.devnull, env=env, - stderr=subprocess.PIPE, - cwd=self.current_src_dir) - self.state = STATE_AUTOCONF - - def do_build_db(self): - """Add the board to the database""" - configs = {} - for line in read_file(os.path.join(self.build_dir, AUTO_CONF_PATH)): - if line.startswith('CONFIG'): - config, value = line.split('=', 1) - configs[config] = value.rstrip() - self.db_queue.put([self.defconfig, configs]) - self.finish(True) - - def do_savedefconfig(self): - """Update the .config and run 'make savedefconfig'.""" - if not self.args.force_sync: - self.finish(True) - return - - cmd = list(self.make_cmd) - cmd.append('savedefconfig') - self.proc = subprocess.Popen(cmd, stdout=self.devnull, - stderr=subprocess.PIPE) - self.state = STATE_SAVEDEFCONFIG - - def update_defconfig(self): - """Update the input defconfig and go back to the idle state.""" - orig_defconfig = os.path.join('configs', self.defconfig) - new_defconfig = os.path.join(self.build_dir, 'defconfig') - updated = not filecmp.cmp(orig_defconfig, new_defconfig) - - if updated: - self.log.append( - self.col.build(self.col.BLUE, 'defconfig updated', bright=True)) - - if not self.args.dry_run and updated: - shutil.move(new_defconfig, orig_defconfig) - self.finish(True) - - def finish(self, success): - """Display log along with progress and go to the idle state. - - Args: - success (bool): Should be True when the defconfig was processed - successfully, or False when it fails. - """ - # output at least 30 characters to hide the "* defconfigs out of *". - name = self.defconfig[:-len('_defconfig')] - if self.log: - - # Put the first log line on the first line - log = name.ljust(20) + ' ' + self.log[0] - - if len(self.log) > 1: - log += '\n' + '\n'.join([' ' + s for s in self.log[1:]]) - # Some threads are running in parallel. - # Print log atomically to not mix up logs from different threads. - print(log, file=(sys.stdout if success else sys.stderr)) - - if not success: - if self.args.exit_on_error: - sys.exit('Exit on error.') - # If --exit-on-error flag is not set, skip this board and continue. - # Record the failed board. - self.failed_boards.add(name) - - self.progress.inc(success) - self.progress.show() - self.state = STATE_IDLE - - def get_failed_boards(self): - """Returns a set of failed boards (defconfigs) in this slot. - """ - return self.failed_boards - -class Slots: - - """Controller of the array of subprocess slots.""" - - def __init__(self, toolchains, args, progress, reference_src_dir, db_queue, - col): - """Create a new slots controller. - - Args: - toolchains (Toolchains): Toolchains object containing toolchains - args (Namespace): Program arguments - progress (Progress): A progress indicator. - reference_src_dir (str): Determine the true starting config state - from this source tree (None for none) - db_queue (Queue): output queue to write config info for the database - col (terminal.Color): Colour object - """ - self.args = args - self.slots = [] - self.progress = progress - self.col = col - devnull = subprocess.DEVNULL - make_cmd = get_make_cmd() - for _ in range(args.jobs): - self.slots.append(Slot(toolchains, args, progress, devnull, - make_cmd, reference_src_dir, db_queue, col)) - - def add(self, defconfig): - """Add a new subprocess if a vacant slot is found. - - Args: - defconfig (str): defconfig name to be put into. - - Returns: - Return True on success or False on failure - """ - for slot in self.slots: - if slot.add(defconfig): - return True - return False - - def available(self): - """Check if there is a vacant slot. - - Returns: - Return True if at lease one vacant slot is found, False otherwise. - """ - for slot in self.slots: - if slot.poll(): - return True - return False - - def empty(self): - """Check if all slots are vacant. - - Returns: - Return True if all the slots are vacant, False otherwise. - """ - ret = True - for slot in self.slots: - if not slot.poll(): - ret = False - return ret - - def write_failed_boards(self): - """Show the results of processing""" - boards = set() - - for slot in self.slots: - boards |= slot.get_failed_boards() - - if boards: - boards = '\n'.join(sorted(boards)) + '\n' - write_file(FAILED_LIST, boards) - - -class ReferenceSource: - - """Reference source against which original configs should be parsed.""" - - def __init__(self, commit): - """Create a reference source directory based on a specified commit. - - Args: - commit: commit to git-clone - """ - self.src_dir = tempfile.mkdtemp() - print('Cloning git repo to a separate work directory...') - subprocess.check_output(['git', 'clone', os.getcwd(), '.'], - cwd=self.src_dir) - rev = subprocess.check_output(['git', 'rev-parse', '--short', - commit]).strip() - print(f"Checkout '{rev}' to build the original autoconf.mk.") - subprocess.check_output(['git', 'checkout', commit], - stderr=subprocess.STDOUT, cwd=self.src_dir) - - def __del__(self): - """Delete the reference source directory - - This function makes sure the temporary directory is cleaned away - even if Python suddenly dies due to error. It should be done in here - because it is guaranteed the destructor is always invoked when the - instance of the class gets unreferenced. - """ - shutil.rmtree(self.src_dir) - - def get_dir(self): - """Return the absolute path to the reference source directory.""" - - return self.src_dir - -def move_config(toolchains, args, db_queue, col): - """Build database or sync config options to defconfig files. - - Args: - toolchains (Toolchains): Toolchains to use - args (Namespace): Program arguments - db_queue (Queue): Queue for database updates - col (terminal.Color): Colour object - - Returns: - Progress: Progress indicator - """ - if args.git_ref: - reference_src = ReferenceSource(args.git_ref) - reference_src_dir = reference_src.get_dir() - else: - reference_src_dir = None - - if args.defconfigs: - defconfigs = get_matched_defconfigs(args.defconfigs) - else: - defconfigs = get_all_defconfigs() - - progress = Progress(col, len(defconfigs)) - slots = Slots(toolchains, args, progress, reference_src_dir, db_queue, col) - - # Main loop to process defconfig files: - # Add a new subprocess into a vacant slot. - # Sleep if there is no available slot. - for defconfig in defconfigs: - while not slots.add(defconfig): - while not slots.available(): - # No available slot: sleep for a while - time.sleep(SLEEP_TIME) - - # wait until all the subprocesses finish - while not slots.empty(): - time.sleep(SLEEP_TIME) - - slots.write_failed_boards() - return progress - -def find_kconfig_rules(kconf, config, imply_config): - """Check whether a config has a 'select' or 'imply' keyword - - Args: - kconf (Kconfiglib.Kconfig): Kconfig object - config (str): Name of config to check (without CONFIG_ prefix) - imply_config (str): Implying config (without CONFIG_ prefix) which may - or may not have an 'imply' for 'config') - - Returns: - Symbol object for 'config' if found, else None - """ - sym = kconf.syms.get(imply_config) - if sym: - for sel, _ in (sym.selects + sym.implies): - if sel.name == config: - return sym - return None - -def check_imply_rule(kconf, config, imply_config): - """Check if we can add an 'imply' option - - This finds imply_config in the Kconfig and looks to see if it is possible - to add an 'imply' for 'config' to that part of the Kconfig. - - Args: - kconf (Kconfiglib.Kconfig): Kconfig object - config (str): Name of config to check (without CONFIG_ prefix) - imply_config (str): Implying config (without CONFIG_ prefix) which may - or may not have an 'imply' for 'config') - - Returns: - tuple: - str: filename of Kconfig file containing imply_config, or None if - none - int: line number within the Kconfig file, or 0 if none - str: message indicating the result - """ - sym = kconf.syms.get(imply_config) - if not sym: - return 'cannot find sym' - nodes = sym.nodes - if len(nodes) != 1: - return f'{len(nodes)} locations' - node = nodes[0] - fname, linenum = node.filename, node.linenr - cwd = os.getcwd() - if cwd and fname.startswith(cwd): - fname = fname[len(cwd) + 1:] - file_line = f' at {fname}:{linenum}' - data = read_file(fname) - if data[linenum - 1] != f'config {imply_config}': - return None, 0, f'bad sym format {data[linenum]}{file_line})' - return fname, linenum, f'adding{file_line}' - -def add_imply_rule(config, fname, linenum): - """Add a new 'imply' option to a Kconfig - - Args: - config (str): config option to add an imply for (without CONFIG_ prefix) - fname (str): Kconfig filename to update - linenum (int): Line number to place the 'imply' before - - Returns: - Message indicating the result - """ - file_line = f' at {fname}:{linenum}' - data = read_file(fname) - linenum -= 1 - - for offset, line in enumerate(data[linenum:]): - if line.strip().startswith('help') or not line: - data.insert(linenum + offset, f'\timply {config}') - write_file(fname, data) - return f'added{file_line}' - - return 'could not insert%s' - -(IMPLY_MIN_2, IMPLY_TARGET, IMPLY_CMD, IMPLY_NON_ARCH_BOARD) = ( - 1, 2, 4, 8) - -IMPLY_FLAGS = { - 'min2': [IMPLY_MIN_2, 'Show options which imply >2 boards (normally >5)'], - 'target': [IMPLY_TARGET, 'Allow CONFIG_TARGET_... options to imply'], - 'cmd': [IMPLY_CMD, 'Allow CONFIG_CMD_... to imply'], - 'non-arch-board': [ - IMPLY_NON_ARCH_BOARD, - 'Allow Kconfig options outside arch/ and /board/ to imply'], -} - - -def read_database(): - """Read in the config database - - Returns: - tuple: - set of all config options seen (each a str) - set of all defconfigs seen (each a str) - dict of configs for each defconfig: - key: defconfig name, e.g. "MPC8548CDS_legacy_defconfig" - value: dict: - key: CONFIG option - value: Value of option - dict of defconfigs for each config: - key: CONFIG option - value: set of boards using that option - - """ - configs = {} - - # key is defconfig name, value is dict of (CONFIG_xxx, value) - config_db = {} - - # Set of all config options we have seen - all_configs = set() - - # Set of all defconfigs we have seen - all_defconfigs = set() - - defconfig_db = collections.defaultdict(set) - for line in read_file(CONFIG_DATABASE): - line = line.rstrip() - if not line: # Separator between defconfigs - config_db[defconfig] = configs - all_defconfigs.add(defconfig) - configs = {} - elif line[0] == ' ': # CONFIG line - config, value = line.strip().split('=', 1) - configs[config] = value - defconfig_db[config].add(defconfig) - all_configs.add(config) - else: # New defconfig - defconfig = line - - return all_configs, all_defconfigs, config_db, defconfig_db - - -def do_imply_config(config_list, add_imply, imply_flags, skip_added, - check_kconfig=True, find_superset=False): - """Find CONFIG options which imply those in the list - - Some CONFIG options can be implied by others and this can help to reduce - the size of the defconfig files. For example, CONFIG_X86 implies - CONFIG_CMD_IRQ, so we can put 'imply CMD_IRQ' under 'config X86' and - all x86 boards will have that option, avoiding adding CONFIG_CMD_IRQ to - each of the x86 defconfig files. - - This function uses the moveconfig database to find such options. It - displays a list of things that could possibly imply those in the list. - The algorithm ignores any that start with CONFIG_TARGET since these - typically refer to only a few defconfigs (often one). It also does not - display a config with less than 5 defconfigs. - - The algorithm works using sets. For each target config in config_list: - - Get the set 'defconfigs' which use that target config - - For each config (from a list of all configs): - - Get the set 'imply_defconfig' of defconfigs which use that config - - - - If imply_defconfigs contains anything not in defconfigs then - this config does not imply the target config - - Params: - config_list: List of CONFIG options to check (each a string) - add_imply: Automatically add an 'imply' for each config. - imply_flags: Flags which control which implying configs are allowed - (IMPLY_...) - skip_added: Don't show options which already have an imply added. - check_kconfig: Check if implied symbols already have an 'imply' or - 'select' for the target config, and show this information if so. - find_superset: True to look for configs which are a superset of those - already found. So for example if CONFIG_EXYNOS5 implies an option, - but CONFIG_EXYNOS covers a larger set of defconfigs and also - implies that option, this will drop the former in favour of the - latter. In practice this option has not proved very used. - - Note the terminoloy: - config - a CONFIG_XXX options (a string, e.g. 'CONFIG_CMD_EEPROM') - defconfig - a defconfig file (a string, e.g. 'configs/snow_defconfig') - """ - kconf = KconfigScanner().conf if check_kconfig else None - if add_imply and add_imply != 'all': - add_imply = add_imply.split(',') - - all_configs, all_defconfigs, _, defconfig_db = read_database() - - # Work through each target config option in turn, independently - for config in config_list: - defconfigs = defconfig_db.get(config) - if not defconfigs: - print(f'{config} not found in any defconfig') - continue - - # Get the set of defconfigs without this one (since a config cannot - # imply itself) - non_defconfigs = all_defconfigs - defconfigs - num_defconfigs = len(defconfigs) - print(f'{config} found in {num_defconfigs}/{len(all_configs)} defconfigs') - - # This will hold the results: key=config, value=defconfigs containing it - imply_configs = {} - rest_configs = all_configs - set([config]) - - # Look at every possible config, except the target one - for imply_config in rest_configs: - if 'ERRATUM' in imply_config: - continue - if not imply_flags & IMPLY_CMD: - if 'CONFIG_CMD' in imply_config: - continue - if not imply_flags & IMPLY_TARGET: - if 'CONFIG_TARGET' in imply_config: - continue - - # Find set of defconfigs that have this config - imply_defconfig = defconfig_db[imply_config] - - # Get the intersection of this with defconfigs containing the - # target config - common_defconfigs = imply_defconfig & defconfigs - - # Get the set of defconfigs containing this config which DO NOT - # also contain the taret config. If this set is non-empty it means - # that this config affects other defconfigs as well as (possibly) - # the ones affected by the target config. This means it implies - # things we don't want to imply. - not_common_defconfigs = imply_defconfig & non_defconfigs - if not_common_defconfigs: - continue - - # If there are common defconfigs, imply_config may be useful - if common_defconfigs: - skip = False - if find_superset: - for prev in list(imply_configs.keys()): - prev_count = len(imply_configs[prev]) - count = len(common_defconfigs) - if (prev_count > count and - (imply_configs[prev] & common_defconfigs == - common_defconfigs)): - # skip imply_config because prev is a superset - skip = True - break - if count > prev_count: - # delete prev because imply_config is a superset - del imply_configs[prev] - if not skip: - imply_configs[imply_config] = common_defconfigs - - # Now we have a dict imply_configs of configs which imply each config - # The value of each dict item is the set of defconfigs containing that - # config. Rank them so that we print the configs that imply the largest - # number of defconfigs first. - ranked_iconfigs = sorted(imply_configs, - key=lambda k: len(imply_configs[k]), reverse=True) - kconfig_info = '' - cwd = os.getcwd() - add_list = collections.defaultdict(list) - for iconfig in ranked_iconfigs: - num_common = len(imply_configs[iconfig]) - - # Don't bother if there are less than 5 defconfigs affected. - if num_common < (2 if imply_flags & IMPLY_MIN_2 else 5): - continue - missing = defconfigs - imply_configs[iconfig] - missing_str = ', '.join(missing) if missing else 'all' - missing_str = '' - show = True - if kconf: - sym = find_kconfig_rules(kconf, config[CONFIG_LEN:], - iconfig[CONFIG_LEN:]) - kconfig_info = '' - if sym: - nodes = sym.nodes - if len(nodes) == 1: - fname, linenum = nodes[0].filename, nodes[0].linenr - if cwd and fname.startswith(cwd): - fname = fname[len(cwd) + 1:] - kconfig_info = f'{fname}:{linenum}' - if skip_added: - show = False - else: - sym = kconf.syms.get(iconfig[CONFIG_LEN:]) - fname = '' - if sym: - nodes = sym.nodes - if len(nodes) == 1: - fname, linenum = nodes[0].filename, nodes[0].linenr - if cwd and fname.startswith(cwd): - fname = fname[len(cwd) + 1:] - in_arch_board = not sym or (fname.startswith('arch') or - fname.startswith('board')) - if (not in_arch_board and - not imply_flags & IMPLY_NON_ARCH_BOARD): - continue - - if add_imply and (add_imply == 'all' or - iconfig in add_imply): - fname, linenum, kconfig_info = (check_imply_rule(kconf, - config[CONFIG_LEN:], iconfig[CONFIG_LEN:])) - if fname: - add_list[fname].append(linenum) - - if show and kconfig_info != 'skip': - print(f'{num_common:5d} : ' - f'{iconfig.ljust(30):-30s}{kconfig_info:-25s} {missing_str}') - - # Having collected a list of things to add, now we add them. We process - # each file from the largest line number to the smallest so that - # earlier additions do not affect our line numbers. E.g. if we added an - # imply at line 20 it would change the position of each line after - # that. - for fname, linenums in add_list.items(): - for linenum in sorted(linenums, reverse=True): - add_imply_rule(config[CONFIG_LEN:], fname, linenum) - -def defconfig_matches(configs, re_match): - """Check if any CONFIG option matches a regex - - The match must be complete, i.e. from the start to end of the CONFIG option. - - Args: - configs (dict): Dict of CONFIG options: - key: CONFIG option - value: Value of option - re_match (re.Pattern): Match to check - - Returns: - bool: True if any CONFIG matches the regex - """ - for cfg in configs: - if re_match.fullmatch(cfg): - return True - return False - -def do_find_config(config_list): - """Find boards with a given combination of CONFIGs - - Params: - config_list: List of CONFIG options to check (each a regex consisting - of a config option, with or without a CONFIG_ prefix. If an option - is preceded by a tilde (~) then it must be false, otherwise it must - be true) - """ - _, all_defconfigs, config_db, _ = read_database() - - # Start with all defconfigs - out = all_defconfigs - - # Work through each config in turn - for item in config_list: - # Get the real config name and whether we want this config or not - cfg = item - want = True - if cfg[0] == '~': - want = False - cfg = cfg[1:] - - # Search everything that is still in the running. If it has a config - # that we want, or doesn't have one that we don't, add it into the - # running for the next stage - in_list = out - out = set() - re_match = re.compile(cfg) - for defc in in_list: - has_cfg = defconfig_matches(config_db[defc], re_match) - if has_cfg == want: - out.add(defc) - print(f'{len(out)} matches') - print(' '.join(item.split('_defconfig')[0] for item in out)) - - -def prefix_config(cfg): - """Prefix a config with CONFIG_ if needed - - This handles ~ operator, which indicates that the CONFIG should be disabled - - >>> prefix_config('FRED') - 'CONFIG_FRED' - >>> prefix_config('CONFIG_FRED') - 'CONFIG_FRED' - >>> prefix_config('~FRED') - '~CONFIG_FRED' - >>> prefix_config('~CONFIG_FRED') - '~CONFIG_FRED' - >>> prefix_config('A123') - 'CONFIG_A123' - """ - oper = '' - if cfg[0] == '~': - oper = cfg[0] - cfg = cfg[1:] - if not cfg.startswith('CONFIG_'): - cfg = 'CONFIG_' + cfg - return oper + cfg - - -RE_MK_CONFIGS = re.compile(r'CONFIG_(\$\(SPL_(?:TPL_)?\))?([A-Za-z0-9_]*)') -RE_IFDEF = re.compile(r'(ifdef|ifndef)') -RE_C_CONFIGS = re.compile(r'CONFIG_([A-Za-z0-9_]*)') -RE_CONFIG_IS = re.compile(r'CONFIG_IS_ENABLED\(([A-Za-z0-9_]*)\)') - -class ConfigUse: - def __init__(self, cfg, is_spl, fname, rest): - self.cfg = cfg - self.is_spl = is_spl - self.fname = fname - self.rest = rest - - def __hash__(self): - return hash((self.cfg, self.is_spl)) - -def scan_makefiles(fnames): - """Scan Makefiles looking for Kconfig options - - Looks for uses of CONFIG options in Makefiles - - Args: - fnames (list of tuple): - str: Makefile filename where the option was found - str: Line of the Makefile - - Returns: - tuple: - dict: all_uses - key (ConfigUse): object - value (list of str): matching lines - dict: Uses by filename - key (str): filename - value (set of ConfigUse): uses in that filename - - >>> RE_MK_CONFIGS.search('CONFIG_FRED').groups() - (None, 'FRED') - >>> RE_MK_CONFIGS.search('CONFIG_$(SPL_)MARY').groups() - ('$(SPL_)', 'MARY') - >>> RE_MK_CONFIGS.search('CONFIG_$(SPL_TPL_)MARY').groups() - ('$(SPL_TPL_)', 'MARY') - """ - all_uses = collections.defaultdict(list) - fname_uses = {} - for fname, rest in fnames: - m_iter = RE_MK_CONFIGS.finditer(rest) - for mat in m_iter: - real_opt = mat.group(2) - if real_opt == '': - continue - is_spl = False - if mat.group(1): - is_spl = True - use = ConfigUse(real_opt, is_spl, fname, rest) - if fname not in fname_uses: - fname_uses[fname] = set() - fname_uses[fname].add(use) - all_uses[use].append(rest) - return all_uses, fname_uses - - -def scan_src_files(fnames): - """Scan source files (other than Makefiles) looking for Kconfig options - - Looks for uses of CONFIG options - - Args: - fnames (list of tuple): - str: Makefile filename where the option was found - str: Line of the Makefile - - Returns: - tuple: - dict: all_uses - key (ConfigUse): object - value (list of str): matching lines - dict: Uses by filename - key (str): filename - value (set of ConfigUse): uses in that filename - - >>> RE_C_CONFIGS.search('CONFIG_FRED').groups() - ('FRED',) - >>> RE_CONFIG_IS.search('CONFIG_IS_ENABLED(MARY)').groups() - ('MARY',) - >>> RE_CONFIG_IS.search('#if CONFIG_IS_ENABLED(OF_PLATDATA)').groups() - ('OF_PLATDATA',) - """ - fname = None - rest = None - - def add_uses(m_iter, is_spl): - for mat in m_iter: - real_opt = mat.group(1) - if real_opt == '': - continue - use = ConfigUse(real_opt, is_spl, fname, rest) - if fname not in fname_uses: - fname_uses[fname] = set() - fname_uses[fname].add(use) - all_uses[use].append(rest) - - all_uses = collections.defaultdict(list) - fname_uses = {} - for fname, rest in fnames: - m_iter = RE_C_CONFIGS.finditer(rest) - add_uses(m_iter, False) - - m_iter2 = RE_CONFIG_IS.finditer(rest) - add_uses(m_iter2, True) - - return all_uses, fname_uses - - -MODE_NORMAL, MODE_SPL, MODE_PROPER = range(3) - -def do_scan_source(path, do_update): - """Scan the source tree for Kconfig inconsistencies - - Args: - path (str): Path to source tree - do_update (bool) : True to write to scripts/kconf_... files - """ - def is_not_proper(name): - for prefix in SPL_PREFIXES: - if name.startswith(prefix): - return name[len(prefix):] - return False - - def check_not_found(all_uses, spl_mode): - """Check for Kconfig options mentioned in the source but not in Kconfig - - Args: - all_uses (dict): - key (ConfigUse): object - value (list of str): matching lines - spl_mode (int): If MODE_SPL, look at source code which implies - an SPL_ option, but for which there is none; - for MOD_PROPER, look at source code which implies a Proper - option (i.e. use of CONFIG_IS_ENABLED() or $(SPL_) or - $(SPL_TPL_) but for which there none; - if MODE_NORMAL, ignore SPL - - Returns: - dict: - key (str): CONFIG name (without 'CONFIG_' prefix - value (list of ConfigUse): List of uses of this CONFIG - """ - # Make sure we know about all the options - not_found = collections.defaultdict(list) - for use, _ in all_uses.items(): - name = use.cfg - if name in IGNORE_SYMS: - continue - check = True - - if spl_mode == MODE_SPL: - check = use.is_spl - - # If it is an SPL symbol, try prepending all SPL_ prefixes to - # find at least one SPL symbol - if use.is_spl: - for prefix in SPL_PREFIXES: - try_name = prefix + name - sym = kconf.syms.get(try_name) - if sym: - break - if not sym: - not_found[f'SPL_{name}'].append(use) - continue - elif spl_mode == MODE_PROPER: - # Try to find the Proper version of this symbol, i.e. without - # the SPL_ prefix - proper_name = is_not_proper(name) - if proper_name: - name = proper_name - elif not use.is_spl: - check = False - else: # MODE_NORMAL - sym = kconf.syms.get(name) - if not sym: - proper_name = is_not_proper(name) - if proper_name: - name = proper_name - sym = kconf.syms.get(name) - if not sym: - for prefix in SPL_PREFIXES: - try_name = prefix + name - sym = kconf.syms.get(try_name) - if sym: - break - if not sym: - not_found[name].append(use) - continue - - sym = kconf.syms.get(name) - if not sym and check: - not_found[name].append(use) - return not_found - - def show_uses(uses): - """Show a list of uses along with their filename and code snippet - - Args: - uses (dict): - key (str): CONFIG name (without 'CONFIG_' prefix - value (list of ConfigUse): List of uses of this CONFIG - """ - for name in sorted(uses): - print(f'{name}: ', end='') - for i, use in enumerate(uses[name]): - print(f'{" " if i else ""}{use.fname}: {use.rest.strip()}') - - - print('Scanning Kconfig') - kconf = KconfigScanner().conf - print(f'Scanning source in {path}') - args = ['git', 'grep', '-E', r'IS_ENABLED|\bCONFIG'] - with subprocess.Popen(args, stdout=subprocess.PIPE) as proc: - out, _ = proc.communicate() - lines = out.splitlines() - re_fname = re.compile('^([^:]*):(.*)') - src_list = [] - mk_list = [] - for line in lines: - linestr = line.decode('utf-8') - m_fname = re_fname.search(linestr) - if not m_fname: - continue - fname, rest = m_fname.groups() - dirname, leaf = os.path.split(fname) - root, ext = os.path.splitext(leaf) - if ext == '.autoconf': - pass - elif ext in ['.c', '.h', '.S', '.lds', '.dts', '.dtsi', '.asl', '.cfg', - '.env', '.tmpl']: - src_list.append([fname, rest]) - elif 'Makefile' in root or ext == '.mk': - mk_list.append([fname, rest]) - elif ext in ['.yml', '.sh', '.py', '.awk', '.pl', '.rst', '', '.sed']: - pass - elif 'Kconfig' in root or 'Kbuild' in root: - pass - elif 'README' in root: - pass - elif dirname in ['configs']: - pass - elif dirname.startswith('doc') or dirname.startswith('scripts/kconfig'): - pass - else: - print(f'Not sure how to handle file {fname}') - - # Scan the Makefiles - all_uses, _ = scan_makefiles(mk_list) - - spl_not_found = set() - proper_not_found = set() - - # Make sure we know about all the options - print('\nCONFIG options present in Makefiles but not Kconfig:') - not_found = check_not_found(all_uses, MODE_NORMAL) - show_uses(not_found) - - print('\nCONFIG options present in Makefiles but not Kconfig (SPL):') - not_found = check_not_found(all_uses, MODE_SPL) - show_uses(not_found) - spl_not_found |= {is_not_proper(key) or key for key in not_found.keys()} - - print('\nCONFIG options used as Proper in Makefiles but without a non-SPL_ variant:') - not_found = check_not_found(all_uses, MODE_PROPER) - show_uses(not_found) - proper_not_found |= {not_found.keys()} - - # Scan the source code - all_uses, _ = scan_src_files(src_list) - - # Make sure we know about all the options - print('\nCONFIG options present in source but not Kconfig:') - not_found = check_not_found(all_uses, MODE_NORMAL) - show_uses(not_found) - - print('\nCONFIG options present in source but not Kconfig (SPL):') - not_found = check_not_found(all_uses, MODE_SPL) - show_uses(not_found) - spl_not_found |= {is_not_proper(key) or key for key in not_found.keys()} - - print('\nCONFIG options used as Proper in source but without a non-SPL_ variant:') - not_found = check_not_found(all_uses, MODE_PROPER) - show_uses(not_found) - proper_not_found |= {not_found.keys()} - - print('\nCONFIG options used as SPL but without an SPL_ variant:') - for item in sorted(spl_not_found): - print(f' {item}') - - print('\nCONFIG options used as Proper but without a non-SPL_ variant:') - for item in sorted(proper_not_found): - print(f' {item}') - - # Write out the updated information - if do_update: - with open(os.path.join(path, 'scripts', 'conf_nospl'), 'w', - encoding='utf-8') as out: - print('# These options should not be enabled in SPL builds\n', - file=out) - for item in sorted(spl_not_found): - print(item, file=out) - with open(os.path.join(path, 'scripts', 'conf_noproper'), 'w', - encoding='utf-8') as out: - print('# These options should not be enabled in Proper builds\n', - file=out) - for item in sorted(proper_not_found): - print(item, file=out) - - -def main(): - try: - cpu_count = multiprocessing.cpu_count() - except NotImplementedError: - cpu_count = 1 - - epilog = '''Move config options from headers to defconfig files. See -doc/develop/moveconfig.rst for documentation.''' - - parser = ArgumentParser(epilog=epilog) - # Add arguments here - parser.add_argument('-a', '--add-imply', type=str, default='', - help='comma-separated list of CONFIG options to add ' - "an 'imply' statement to for the CONFIG in -i") - parser.add_argument('-A', '--skip-added', action='store_true', default=False, - help="don't show options which are already marked as " - 'implying others') - parser.add_argument('-b', '--build-db', action='store_true', default=False, - help='build a CONFIG database') - parser.add_argument('-C', '--commit', action='store_true', default=False, - help='Create a git commit for the operation') - parser.add_argument('--nocolour', action='store_true', default=False, - help="don't display the log in colour") - parser.add_argument('-d', '--defconfigs', type=str, - help='a file containing a list of defconfigs to move, ' - "one per line (for example 'snow_defconfig') " - "or '-' to read from stdin") - parser.add_argument('-e', '--exit-on-error', action='store_true', - default=False, - help='exit immediately on any error') - parser.add_argument('-f', '--find', action='store_true', default=False, - help='Find boards with a given config combination') - parser.add_argument('-i', '--imply', action='store_true', default=False, - help='find options which imply others') - parser.add_argument('-I', '--imply-flags', type=str, default='', - help="control the -i option ('help' for help") - parser.add_argument('-j', '--jobs', type=int, default=cpu_count, - help='the number of jobs to run simultaneously') - parser.add_argument('-n', '--dry-run', action='store_true', default=False, - help='perform a trial run (show log with no changes)') - parser.add_argument('-r', '--git-ref', type=str, - help='the git ref to clone for building the autoconf.mk') - parser.add_argument('-s', '--force-sync', action='store_true', default=False, - help='force sync by savedefconfig') - parser.add_argument('-S', '--spl', action='store_true', default=False, - help='parse config options defined for SPL build') - parser.add_argument('--scan-source', action='store_true', default=False, - help='scan source for uses of CONFIG options') - parser.add_argument('-t', '--test', action='store_true', default=False, - help='run unit tests') - parser.add_argument('-y', '--yes', action='store_true', default=False, - help="respond 'yes' to any prompts") - parser.add_argument('-u', '--update', action='store_true', default=False, - help="update scripts/ files (use with --scan-source)") - parser.add_argument('-v', '--verbose', action='store_true', default=False, - help='show any build errors as boards are built') - parser.add_argument('configs', nargs='*') - - args = parser.parse_args() - - if args.test: - sys.argv = [sys.argv[0]] - fail, _ = doctest.testmod() - if fail: - return 1 - unittest.main() - - col = terminal.Color(terminal.COLOR_NEVER if args.nocolour - else terminal.COLOR_IF_TERMINAL) - - if args.scan_source: - do_scan_source(os.getcwd(), args.update) - return 0 - - if not any((args.force_sync, args.build_db, args.imply, args.find)): - parser.print_usage() - sys.exit(1) - - # prefix the option name with CONFIG_ if missing - configs = [prefix_config(cfg) for cfg in args.configs] - - check_top_directory() - - if args.imply: - imply_flags = 0 - if args.imply_flags == 'all': - imply_flags = -1 - - elif args.imply_flags: - for flag in args.imply_flags.split(','): - bad = flag not in IMPLY_FLAGS - if bad: - print(f"Invalid flag '{flag}'") - if flag == 'help' or bad: - print("Imply flags: (separate with ',')") - for name, info in IMPLY_FLAGS.items(): - print(f' {name:-15s}: {info[1]}') - parser.print_usage() - sys.exit(1) - imply_flags |= IMPLY_FLAGS[flag][0] - - do_imply_config(configs, args.add_imply, imply_flags, args.skip_added) - return 0 - - if args.find: - do_find_config(configs) - return 0 - - # We are either building the database or forcing a sync of defconfigs - config_db = {} - db_queue = queue.Queue() - dbt = DatabaseThread(config_db, db_queue) - dbt.daemon = True - dbt.start() - - check_clean_directory() - bsettings.setup('') - toolchains = toolchain.Toolchains() - toolchains.GetSettings() - toolchains.Scan(verbose=False) - progress = move_config(toolchains, args, db_queue, col) - db_queue.join() - - if args.commit: - subprocess.call(['git', 'add', '-u']) - if configs: - msg = 'Convert %s %sto Kconfig' % (configs[0], - 'et al ' if len(configs) > 1 else '') - msg += ('\n\nThis converts the following to Kconfig:\n %s\n' % - '\n '.join(configs)) - else: - msg = 'configs: Resync with savedefconfig' - msg += '\n\nRsync all defconfig files using moveconfig.py' - subprocess.call(['git', 'commit', '-s', '-m', msg]) - - failed = progress.total - progress.good - failure = f'{failed} failed, ' if failed else '' - if args.build_db: - with open(CONFIG_DATABASE, 'w', encoding='utf-8') as outf: - for defconfig, configs in config_db.items(): - outf.write(f'{defconfig}\n') - for config in sorted(configs.keys()): - outf.write(f' {config}={configs[config]}\n') - outf.write('\n') - print(col.build( - col.RED if failed else col.GREEN, - f'{failure}{len(config_db)} boards written to {CONFIG_DATABASE}')) - else: - if failed: - print(col.build(col.RED, f'{failure}see {FAILED_LIST}', True)) - else: - # Add enough spaces to overwrite the progress indicator - print(col.build( - col.GREEN, f'{progress.total} processed ', bright=True)) - - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/tools/qconfig.py b/tools/qconfig.py new file mode 100755 index 00000000000..dd55a696fda --- /dev/null +++ b/tools/qconfig.py @@ -0,0 +1,1624 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0+ +# +# Author: Masahiro Yamada +# + +""" +Build and query a Kconfig database for boards. + +See doc/develop/moveconfig.rst for documentation. +""" + +from argparse import ArgumentParser +import collections +from contextlib import ExitStack +import doctest +import filecmp +import fnmatch +import glob +import multiprocessing +import os +import queue +import re +import shutil +import subprocess +import sys +import tempfile +import threading +import time +import unittest + +import asteval +from buildman import bsettings +from buildman import kconfiglib +from buildman import toolchain +from u_boot_pylib import terminal + +SHOW_GNU_MAKE = 'scripts/show-gnu-make' +SLEEP_TIME=0.03 + +STATE_IDLE = 0 +STATE_DEFCONFIG = 1 +STATE_AUTOCONF = 2 +STATE_SAVEDEFCONFIG = 3 + +AUTO_CONF_PATH = 'include/config/auto.conf' +CONFIG_DATABASE = 'moveconfig.db' +FAILED_LIST = 'moveconfig.failed' + +CONFIG_LEN = len('CONFIG_') + +SIZES = { + 'SZ_1': 0x00000001, 'SZ_2': 0x00000002, + 'SZ_4': 0x00000004, 'SZ_8': 0x00000008, + 'SZ_16': 0x00000010, 'SZ_32': 0x00000020, + 'SZ_64': 0x00000040, 'SZ_128': 0x00000080, + 'SZ_256': 0x00000100, 'SZ_512': 0x00000200, + 'SZ_1K': 0x00000400, 'SZ_2K': 0x00000800, + 'SZ_4K': 0x00001000, 'SZ_8K': 0x00002000, + 'SZ_16K': 0x00004000, 'SZ_32K': 0x00008000, + 'SZ_64K': 0x00010000, 'SZ_128K': 0x00020000, + 'SZ_256K': 0x00040000, 'SZ_512K': 0x00080000, + 'SZ_1M': 0x00100000, 'SZ_2M': 0x00200000, + 'SZ_4M': 0x00400000, 'SZ_8M': 0x00800000, + 'SZ_16M': 0x01000000, 'SZ_32M': 0x02000000, + 'SZ_64M': 0x04000000, 'SZ_128M': 0x08000000, + 'SZ_256M': 0x10000000, 'SZ_512M': 0x20000000, + 'SZ_1G': 0x40000000, 'SZ_2G': 0x80000000, + 'SZ_4G': 0x100000000 +} + +RE_REMOVE_DEFCONFIG = re.compile(r'(.*)_defconfig') + +# CONFIG symbols present in the build system (from Linux) but not actually used +# in U-Boot; KCONFIG symbols +IGNORE_SYMS = ['DEBUG_SECTION_MISMATCH', 'FTRACE_MCOUNT_RECORD', 'GCOV_KERNEL', + 'GCOV_PROFILE_ALL', 'KALLSYMS', 'KASAN', 'MODVERSIONS', 'SHELL', + 'TPL_BUILD', 'VPL_BUILD', 'IS_ENABLED', 'FOO', 'IF_ENABLED_INT', + 'IS_ENABLED_', 'IS_ENABLED_1', 'IS_ENABLED_2', 'IS_ENABLED_3', + 'SPL_', 'TPL_', 'SPL_FOO', 'TPL_FOO', 'TOOLS_FOO', + 'ACME', 'SPL_ACME', 'TPL_ACME', 'TRACE_BRANCH_PROFILING', + 'VAL', '_UNDEFINED', 'SPL_BUILD', ] + +SPL_PREFIXES = ['SPL_', 'TPL_', 'VPL_', 'TOOLS_'] + +### helper functions ### +def check_top_directory(): + """Exit if we are not at the top of source directory.""" + for fname in 'README', 'Licenses': + if not os.path.exists(fname): + sys.exit('Please run at the top of source directory.') + +def check_clean_directory(): + """Exit if the source tree is not clean.""" + for fname in '.config', 'include/config': + if os.path.exists(fname): + sys.exit("source tree is not clean, please run 'make mrproper'") + +def get_make_cmd(): + """Get the command name of GNU Make. + + U-Boot needs GNU Make for building, but the command name is not + necessarily "make". (for example, "gmake" on FreeBSD). + Returns the most appropriate command name on your system. + """ + with subprocess.Popen([SHOW_GNU_MAKE], stdout=subprocess.PIPE) as proc: + ret = proc.communicate() + if proc.returncode: + sys.exit('GNU Make not found') + return ret[0].rstrip() + +def get_matched_defconfig(line): + """Get the defconfig files that match a pattern + + Args: + line (str): Path or filename to match, e.g. 'configs/snow_defconfig' or + 'k2*_defconfig'. If no directory is provided, 'configs/' is + prepended + + Returns: + list of str: a list of matching defconfig files + """ + dirname = os.path.dirname(line) + if dirname: + pattern = line + else: + pattern = os.path.join('configs', line) + return glob.glob(pattern) + glob.glob(pattern + '_defconfig') + +def get_matched_defconfigs(defconfigs_file): + """Get all the defconfig files that match the patterns in a file. + + Args: + defconfigs_file (str): File containing a list of defconfigs to process, + or '-' to read the list from stdin + + Returns: + list of str: A list of paths to defconfig files, with no duplicates + """ + defconfigs = [] + with ExitStack() as stack: + if defconfigs_file == '-': + inf = sys.stdin + defconfigs_file = 'stdin' + else: + inf = stack.enter_context(open(defconfigs_file, encoding='utf-8')) + for i, line in enumerate(inf): + line = line.strip() + if not line: + continue # skip blank lines silently + if ' ' in line: + line = line.split(' ')[0] # handle 'git log' input + matched = get_matched_defconfig(line) + if not matched: + print(f"warning: {defconfigs_file}:{i + 1}: no defconfig matched '{line}'", + file=sys.stderr) + + defconfigs += matched + + # use set() to drop multiple matching + return [defconfig[len('configs') + 1:] for defconfig in set(defconfigs)] + +def get_all_defconfigs(): + """Get all the defconfig files under the configs/ directory. + + Returns: + list of str: List of paths to defconfig files + """ + defconfigs = [] + for (dirpath, _, filenames) in os.walk('configs'): + dirpath = dirpath[len('configs') + 1:] + for filename in fnmatch.filter(filenames, '*_defconfig'): + defconfigs.append(os.path.join(dirpath, filename)) + + return defconfigs + +def write_file(fname, data): + """Write data to a file + + Args: + fname (str): Filename to write to + data (list of str): Lines to write (with or without trailing newline); + or str to write + """ + with open(fname, 'w', encoding='utf-8') as out: + if isinstance(data, list): + for line in data: + print(line.rstrip('\n'), file=out) + else: + out.write(data) + +def read_file(fname, as_lines=True, skip_unicode=False): + """Read a file and return the contents + + Args: + fname (str): Filename to read from + as_lines (bool): Return file contents as a list of lines + skip_unicode (bool): True to report unicode errors and continue + + Returns: + iter of str: List of ;ines from the file with newline removed; str if + as_lines is False with newlines intact; or None if a unicode error + occurred + + Raises: + UnicodeDecodeError: Unicode error occurred when reading + """ + with open(fname, encoding='utf-8') as inf: + try: + if as_lines: + return [line.rstrip('\n') for line in inf.readlines()] + return inf.read() + except UnicodeDecodeError as exc: + if not skip_unicode: + raise + print(f"Failed on file '{fname}: {exc}") + return None + +def try_expand(line): + """If value looks like an expression, try expanding it + Otherwise just return the existing value + """ + if line.find('=') == -1: + return line + + try: + aeval = asteval.Interpreter( usersyms=SIZES, minimal=True ) + cfg, val = re.split("=", line) + val= val.strip('\"') + if re.search(r'[*+-/]|<<|SZ_+|\(([^\)]+)\)', val): + newval = hex(aeval(val)) + print(f'\tExpanded expression {val} to {newval}') + return cfg+'='+newval + except: + print(f'\tFailed to expand expression in {line}') + + return line + + +### classes ### +class Progress: + + """Progress Indicator""" + + def __init__(self, col, total): + """Create a new progress indicator. + + Args: + color_enabled (bool): True for colour output + total (int): A number of defconfig files to process. + """ + self.col = col + self.current = 0 + self.good = 0 + self.total = total + + def inc(self, success): + """Increment the number of processed defconfig files. + + Args: + success (bool): True if processing succeeded + """ + self.good += success + self.current += 1 + + def show(self): + """Display the progress.""" + if self.current != self.total: + line = self.col.build(self.col.GREEN, f'{self.good:5d}') + line += self.col.build(self.col.RED, + f'{self.current - self.good:5d}') + line += self.col.build(self.col.MAGENTA, + f'/{self.total - self.current}') + print(f'{line} \r', end='') + sys.stdout.flush() + + +class KconfigScanner: + """Kconfig scanner.""" + + def __init__(self): + """Scan all the Kconfig files and create a Config object.""" + # Define environment variables referenced from Kconfig + os.environ['srctree'] = os.getcwd() + os.environ['UBOOTVERSION'] = 'dummy' + os.environ['KCONFIG_OBJDIR'] = '' + os.environ['CC'] = 'gcc' + self.conf = kconfiglib.Kconfig() + + +class KconfigParser: + + """A parser of .config and include/autoconf.mk.""" + + re_arch = re.compile(r'CONFIG_SYS_ARCH="(.*)"') + re_cpu = re.compile(r'CONFIG_SYS_CPU="(.*)"') + + def __init__(self, args, build_dir): + """Create a new parser. + + Args: + args (Namespace): program arguments + build_dir: Build directory. + """ + self.args = args + self.dotconfig = os.path.join(build_dir, '.config') + self.autoconf = os.path.join(build_dir, 'include', 'autoconf.mk') + self.spl_autoconf = os.path.join(build_dir, 'spl', 'include', + 'autoconf.mk') + self.config_autoconf = os.path.join(build_dir, AUTO_CONF_PATH) + self.defconfig = os.path.join(build_dir, 'defconfig') + + def get_arch(self): + """Parse .config file and return the architecture. + + Returns: + Architecture name (e.g. 'arm'). + """ + arch = '' + cpu = '' + for line in read_file(self.dotconfig): + m_arch = self.re_arch.match(line) + if m_arch: + arch = m_arch.group(1) + continue + m_cpu = self.re_cpu.match(line) + if m_cpu: + cpu = m_cpu.group(1) + + if not arch: + return None + + # fix-up for aarch64 + if arch == 'arm' and cpu == 'armv8': + arch = 'aarch64' + + return arch + + +class DatabaseThread(threading.Thread): + """This thread processes results from Slot threads. + + It collects the data in the master config directary. There is only one + result thread, and this helps to serialise the build output. + """ + def __init__(self, config_db, db_queue): + """Set up a new result thread + + Args: + builder: Builder which will be sent each result + """ + threading.Thread.__init__(self) + self.config_db = config_db + self.db_queue= db_queue + + def run(self): + """Called to start up the result thread. + + We collect the next result job and pass it on to the build. + """ + while True: + defconfig, configs = self.db_queue.get() + self.config_db[defconfig] = configs + self.db_queue.task_done() + + +class Slot: + + """A slot to store a subprocess. + + Each instance of this class handles one subprocess. + This class is useful to control multiple threads + for faster processing. + """ + + def __init__(self, toolchains, args, progress, devnull, make_cmd, + reference_src_dir, db_queue, col): + """Create a new process slot. + + Args: + toolchains: Toolchains object containing toolchains. + args: Program arguments + progress: A progress indicator. + devnull: A file object of '/dev/null'. + make_cmd: command name of GNU Make. + reference_src_dir: Determine the true starting config state from this + source tree. + db_queue: output queue to write config info for the database + col (terminal.Color): Colour object + """ + self.toolchains = toolchains + self.args = args + self.progress = progress + self.build_dir = tempfile.mkdtemp() + self.devnull = devnull + self.make_cmd = (make_cmd, 'O=' + self.build_dir) + self.reference_src_dir = reference_src_dir + self.db_queue = db_queue + self.col = col + self.parser = KconfigParser(args, self.build_dir) + self.state = STATE_IDLE + self.failed_boards = set() + self.defconfig = None + self.log = [] + self.current_src_dir = None + self.proc = None + + def __del__(self): + """Delete the working directory + + This function makes sure the temporary directory is cleaned away + even if Python suddenly dies due to error. It should be done in here + because it is guaranteed the destructor is always invoked when the + instance of the class gets unreferenced. + + If the subprocess is still running, wait until it finishes. + """ + if self.state != STATE_IDLE: + while self.proc.poll() is None: + pass + shutil.rmtree(self.build_dir) + + def add(self, defconfig): + """Assign a new subprocess for defconfig and add it to the slot. + + If the slot is vacant, create a new subprocess for processing the + given defconfig and add it to the slot. Just returns False if + the slot is occupied (i.e. the current subprocess is still running). + + Args: + defconfig (str): defconfig name. + + Returns: + Return True on success or False on failure + """ + if self.state != STATE_IDLE: + return False + + self.defconfig = defconfig + self.log = [] + self.current_src_dir = self.reference_src_dir + self.do_defconfig() + return True + + def poll(self): + """Check the status of the subprocess and handle it as needed. + + Returns True if the slot is vacant (i.e. in idle state). + If the configuration is successfully finished, assign a new + subprocess to build include/autoconf.mk. + If include/autoconf.mk is generated, invoke the parser to + parse the .config and the include/autoconf.mk, moving + config options to the .config as needed. + If the .config was updated, run "make savedefconfig" to sync + it, update the original defconfig, and then set the slot back + to the idle state. + + Returns: + Return True if the subprocess is terminated, False otherwise + """ + if self.state == STATE_IDLE: + return True + + if self.proc.poll() is None: + return False + + if self.proc.poll() != 0: + self.handle_error() + elif self.state == STATE_DEFCONFIG: + if self.reference_src_dir and not self.current_src_dir: + self.do_savedefconfig() + else: + self.do_autoconf() + elif self.state == STATE_AUTOCONF: + if self.current_src_dir: + self.current_src_dir = None + self.do_defconfig() + elif self.args.build_db: + self.do_build_db() + else: + self.do_savedefconfig() + elif self.state == STATE_SAVEDEFCONFIG: + self.update_defconfig() + else: + sys.exit('Internal Error. This should not happen.') + + return self.state == STATE_IDLE + + def handle_error(self): + """Handle error cases.""" + + self.log.append(self.col.build(self.col.RED, 'Failed to process', + bright=True)) + if self.args.verbose: + for line in self.proc.stderr.read().decode().splitlines(): + self.log.append(self.col.build(self.col.CYAN, line, True)) + self.finish(False) + + def do_defconfig(self): + """Run 'make _defconfig' to create the .config file.""" + + cmd = list(self.make_cmd) + cmd.append(self.defconfig) + self.proc = subprocess.Popen(cmd, stdout=self.devnull, + stderr=subprocess.PIPE, + cwd=self.current_src_dir) + self.state = STATE_DEFCONFIG + + def do_autoconf(self): + """Run 'make AUTO_CONF_PATH'.""" + + arch = self.parser.get_arch() + try: + tchain = self.toolchains.Select(arch) + except ValueError: + self.log.append(self.col.build( + self.col.YELLOW, + f"Tool chain for '{arch}' is missing: do nothing")) + self.finish(False) + return + env = tchain.MakeEnvironment(False) + + cmd = list(self.make_cmd) + cmd.append('KCONFIG_IGNORE_DUPLICATES=1') + cmd.append(AUTO_CONF_PATH) + self.proc = subprocess.Popen(cmd, stdout=self.devnull, env=env, + stderr=subprocess.PIPE, + cwd=self.current_src_dir) + self.state = STATE_AUTOCONF + + def do_build_db(self): + """Add the board to the database""" + configs = {} + for line in read_file(os.path.join(self.build_dir, AUTO_CONF_PATH)): + if line.startswith('CONFIG'): + config, value = line.split('=', 1) + configs[config] = value.rstrip() + self.db_queue.put([self.defconfig, configs]) + self.finish(True) + + def do_savedefconfig(self): + """Update the .config and run 'make savedefconfig'.""" + if not self.args.force_sync: + self.finish(True) + return + + cmd = list(self.make_cmd) + cmd.append('savedefconfig') + self.proc = subprocess.Popen(cmd, stdout=self.devnull, + stderr=subprocess.PIPE) + self.state = STATE_SAVEDEFCONFIG + + def update_defconfig(self): + """Update the input defconfig and go back to the idle state.""" + orig_defconfig = os.path.join('configs', self.defconfig) + new_defconfig = os.path.join(self.build_dir, 'defconfig') + updated = not filecmp.cmp(orig_defconfig, new_defconfig) + + if updated: + self.log.append( + self.col.build(self.col.BLUE, 'defconfig updated', bright=True)) + + if not self.args.dry_run and updated: + shutil.move(new_defconfig, orig_defconfig) + self.finish(True) + + def finish(self, success): + """Display log along with progress and go to the idle state. + + Args: + success (bool): Should be True when the defconfig was processed + successfully, or False when it fails. + """ + # output at least 30 characters to hide the "* defconfigs out of *". + name = self.defconfig[:-len('_defconfig')] + if self.log: + + # Put the first log line on the first line + log = name.ljust(20) + ' ' + self.log[0] + + if len(self.log) > 1: + log += '\n' + '\n'.join([' ' + s for s in self.log[1:]]) + # Some threads are running in parallel. + # Print log atomically to not mix up logs from different threads. + print(log, file=(sys.stdout if success else sys.stderr)) + + if not success: + if self.args.exit_on_error: + sys.exit('Exit on error.') + # If --exit-on-error flag is not set, skip this board and continue. + # Record the failed board. + self.failed_boards.add(name) + + self.progress.inc(success) + self.progress.show() + self.state = STATE_IDLE + + def get_failed_boards(self): + """Returns a set of failed boards (defconfigs) in this slot. + """ + return self.failed_boards + +class Slots: + + """Controller of the array of subprocess slots.""" + + def __init__(self, toolchains, args, progress, reference_src_dir, db_queue, + col): + """Create a new slots controller. + + Args: + toolchains (Toolchains): Toolchains object containing toolchains + args (Namespace): Program arguments + progress (Progress): A progress indicator. + reference_src_dir (str): Determine the true starting config state + from this source tree (None for none) + db_queue (Queue): output queue to write config info for the database + col (terminal.Color): Colour object + """ + self.args = args + self.slots = [] + self.progress = progress + self.col = col + devnull = subprocess.DEVNULL + make_cmd = get_make_cmd() + for _ in range(args.jobs): + self.slots.append(Slot(toolchains, args, progress, devnull, + make_cmd, reference_src_dir, db_queue, col)) + + def add(self, defconfig): + """Add a new subprocess if a vacant slot is found. + + Args: + defconfig (str): defconfig name to be put into. + + Returns: + Return True on success or False on failure + """ + for slot in self.slots: + if slot.add(defconfig): + return True + return False + + def available(self): + """Check if there is a vacant slot. + + Returns: + Return True if at lease one vacant slot is found, False otherwise. + """ + for slot in self.slots: + if slot.poll(): + return True + return False + + def empty(self): + """Check if all slots are vacant. + + Returns: + Return True if all the slots are vacant, False otherwise. + """ + ret = True + for slot in self.slots: + if not slot.poll(): + ret = False + return ret + + def write_failed_boards(self): + """Show the results of processing""" + boards = set() + + for slot in self.slots: + boards |= slot.get_failed_boards() + + if boards: + boards = '\n'.join(sorted(boards)) + '\n' + write_file(FAILED_LIST, boards) + + +class ReferenceSource: + + """Reference source against which original configs should be parsed.""" + + def __init__(self, commit): + """Create a reference source directory based on a specified commit. + + Args: + commit: commit to git-clone + """ + self.src_dir = tempfile.mkdtemp() + print('Cloning git repo to a separate work directory...') + subprocess.check_output(['git', 'clone', os.getcwd(), '.'], + cwd=self.src_dir) + rev = subprocess.check_output(['git', 'rev-parse', '--short', + commit]).strip() + print(f"Checkout '{rev}' to build the original autoconf.mk.") + subprocess.check_output(['git', 'checkout', commit], + stderr=subprocess.STDOUT, cwd=self.src_dir) + + def __del__(self): + """Delete the reference source directory + + This function makes sure the temporary directory is cleaned away + even if Python suddenly dies due to error. It should be done in here + because it is guaranteed the destructor is always invoked when the + instance of the class gets unreferenced. + """ + shutil.rmtree(self.src_dir) + + def get_dir(self): + """Return the absolute path to the reference source directory.""" + + return self.src_dir + +def move_config(toolchains, args, db_queue, col): + """Build database or sync config options to defconfig files. + + Args: + toolchains (Toolchains): Toolchains to use + args (Namespace): Program arguments + db_queue (Queue): Queue for database updates + col (terminal.Color): Colour object + + Returns: + Progress: Progress indicator + """ + if args.git_ref: + reference_src = ReferenceSource(args.git_ref) + reference_src_dir = reference_src.get_dir() + else: + reference_src_dir = None + + if args.defconfigs: + defconfigs = get_matched_defconfigs(args.defconfigs) + else: + defconfigs = get_all_defconfigs() + + progress = Progress(col, len(defconfigs)) + slots = Slots(toolchains, args, progress, reference_src_dir, db_queue, col) + + # Main loop to process defconfig files: + # Add a new subprocess into a vacant slot. + # Sleep if there is no available slot. + for defconfig in defconfigs: + while not slots.add(defconfig): + while not slots.available(): + # No available slot: sleep for a while + time.sleep(SLEEP_TIME) + + # wait until all the subprocesses finish + while not slots.empty(): + time.sleep(SLEEP_TIME) + + slots.write_failed_boards() + return progress + +def find_kconfig_rules(kconf, config, imply_config): + """Check whether a config has a 'select' or 'imply' keyword + + Args: + kconf (Kconfiglib.Kconfig): Kconfig object + config (str): Name of config to check (without CONFIG_ prefix) + imply_config (str): Implying config (without CONFIG_ prefix) which may + or may not have an 'imply' for 'config') + + Returns: + Symbol object for 'config' if found, else None + """ + sym = kconf.syms.get(imply_config) + if sym: + for sel, _ in (sym.selects + sym.implies): + if sel.name == config: + return sym + return None + +def check_imply_rule(kconf, config, imply_config): + """Check if we can add an 'imply' option + + This finds imply_config in the Kconfig and looks to see if it is possible + to add an 'imply' for 'config' to that part of the Kconfig. + + Args: + kconf (Kconfiglib.Kconfig): Kconfig object + config (str): Name of config to check (without CONFIG_ prefix) + imply_config (str): Implying config (without CONFIG_ prefix) which may + or may not have an 'imply' for 'config') + + Returns: + tuple: + str: filename of Kconfig file containing imply_config, or None if + none + int: line number within the Kconfig file, or 0 if none + str: message indicating the result + """ + sym = kconf.syms.get(imply_config) + if not sym: + return 'cannot find sym' + nodes = sym.nodes + if len(nodes) != 1: + return f'{len(nodes)} locations' + node = nodes[0] + fname, linenum = node.filename, node.linenr + cwd = os.getcwd() + if cwd and fname.startswith(cwd): + fname = fname[len(cwd) + 1:] + file_line = f' at {fname}:{linenum}' + data = read_file(fname) + if data[linenum - 1] != f'config {imply_config}': + return None, 0, f'bad sym format {data[linenum]}{file_line})' + return fname, linenum, f'adding{file_line}' + +def add_imply_rule(config, fname, linenum): + """Add a new 'imply' option to a Kconfig + + Args: + config (str): config option to add an imply for (without CONFIG_ prefix) + fname (str): Kconfig filename to update + linenum (int): Line number to place the 'imply' before + + Returns: + Message indicating the result + """ + file_line = f' at {fname}:{linenum}' + data = read_file(fname) + linenum -= 1 + + for offset, line in enumerate(data[linenum:]): + if line.strip().startswith('help') or not line: + data.insert(linenum + offset, f'\timply {config}') + write_file(fname, data) + return f'added{file_line}' + + return 'could not insert%s' + +(IMPLY_MIN_2, IMPLY_TARGET, IMPLY_CMD, IMPLY_NON_ARCH_BOARD) = ( + 1, 2, 4, 8) + +IMPLY_FLAGS = { + 'min2': [IMPLY_MIN_2, 'Show options which imply >2 boards (normally >5)'], + 'target': [IMPLY_TARGET, 'Allow CONFIG_TARGET_... options to imply'], + 'cmd': [IMPLY_CMD, 'Allow CONFIG_CMD_... to imply'], + 'non-arch-board': [ + IMPLY_NON_ARCH_BOARD, + 'Allow Kconfig options outside arch/ and /board/ to imply'], +} + + +def read_database(): + """Read in the config database + + Returns: + tuple: + set of all config options seen (each a str) + set of all defconfigs seen (each a str) + dict of configs for each defconfig: + key: defconfig name, e.g. "MPC8548CDS_legacy_defconfig" + value: dict: + key: CONFIG option + value: Value of option + dict of defconfigs for each config: + key: CONFIG option + value: set of boards using that option + + """ + configs = {} + + # key is defconfig name, value is dict of (CONFIG_xxx, value) + config_db = {} + + # Set of all config options we have seen + all_configs = set() + + # Set of all defconfigs we have seen + all_defconfigs = set() + + defconfig_db = collections.defaultdict(set) + for line in read_file(CONFIG_DATABASE): + line = line.rstrip() + if not line: # Separator between defconfigs + config_db[defconfig] = configs + all_defconfigs.add(defconfig) + configs = {} + elif line[0] == ' ': # CONFIG line + config, value = line.strip().split('=', 1) + configs[config] = value + defconfig_db[config].add(defconfig) + all_configs.add(config) + else: # New defconfig + defconfig = line + + return all_configs, all_defconfigs, config_db, defconfig_db + + +def do_imply_config(config_list, add_imply, imply_flags, skip_added, + check_kconfig=True, find_superset=False): + """Find CONFIG options which imply those in the list + + Some CONFIG options can be implied by others and this can help to reduce + the size of the defconfig files. For example, CONFIG_X86 implies + CONFIG_CMD_IRQ, so we can put 'imply CMD_IRQ' under 'config X86' and + all x86 boards will have that option, avoiding adding CONFIG_CMD_IRQ to + each of the x86 defconfig files. + + This function uses the qconfig database to find such options. It + displays a list of things that could possibly imply those in the list. + The algorithm ignores any that start with CONFIG_TARGET since these + typically refer to only a few defconfigs (often one). It also does not + display a config with less than 5 defconfigs. + + The algorithm works using sets. For each target config in config_list: + - Get the set 'defconfigs' which use that target config + - For each config (from a list of all configs): + - Get the set 'imply_defconfig' of defconfigs which use that config + - + - If imply_defconfigs contains anything not in defconfigs then + this config does not imply the target config + + Params: + config_list: List of CONFIG options to check (each a string) + add_imply: Automatically add an 'imply' for each config. + imply_flags: Flags which control which implying configs are allowed + (IMPLY_...) + skip_added: Don't show options which already have an imply added. + check_kconfig: Check if implied symbols already have an 'imply' or + 'select' for the target config, and show this information if so. + find_superset: True to look for configs which are a superset of those + already found. So for example if CONFIG_EXYNOS5 implies an option, + but CONFIG_EXYNOS covers a larger set of defconfigs and also + implies that option, this will drop the former in favour of the + latter. In practice this option has not proved very used. + + Note the terminoloy: + config - a CONFIG_XXX options (a string, e.g. 'CONFIG_CMD_EEPROM') + defconfig - a defconfig file (a string, e.g. 'configs/snow_defconfig') + """ + kconf = KconfigScanner().conf if check_kconfig else None + if add_imply and add_imply != 'all': + add_imply = add_imply.split(',') + + all_configs, all_defconfigs, _, defconfig_db = read_database() + + # Work through each target config option in turn, independently + for config in config_list: + defconfigs = defconfig_db.get(config) + if not defconfigs: + print(f'{config} not found in any defconfig') + continue + + # Get the set of defconfigs without this one (since a config cannot + # imply itself) + non_defconfigs = all_defconfigs - defconfigs + num_defconfigs = len(defconfigs) + print(f'{config} found in {num_defconfigs}/{len(all_configs)} defconfigs') + + # This will hold the results: key=config, value=defconfigs containing it + imply_configs = {} + rest_configs = all_configs - set([config]) + + # Look at every possible config, except the target one + for imply_config in rest_configs: + if 'ERRATUM' in imply_config: + continue + if not imply_flags & IMPLY_CMD: + if 'CONFIG_CMD' in imply_config: + continue + if not imply_flags & IMPLY_TARGET: + if 'CONFIG_TARGET' in imply_config: + continue + + # Find set of defconfigs that have this config + imply_defconfig = defconfig_db[imply_config] + + # Get the intersection of this with defconfigs containing the + # target config + common_defconfigs = imply_defconfig & defconfigs + + # Get the set of defconfigs containing this config which DO NOT + # also contain the taret config. If this set is non-empty it means + # that this config affects other defconfigs as well as (possibly) + # the ones affected by the target config. This means it implies + # things we don't want to imply. + not_common_defconfigs = imply_defconfig & non_defconfigs + if not_common_defconfigs: + continue + + # If there are common defconfigs, imply_config may be useful + if common_defconfigs: + skip = False + if find_superset: + for prev in list(imply_configs.keys()): + prev_count = len(imply_configs[prev]) + count = len(common_defconfigs) + if (prev_count > count and + (imply_configs[prev] & common_defconfigs == + common_defconfigs)): + # skip imply_config because prev is a superset + skip = True + break + if count > prev_count: + # delete prev because imply_config is a superset + del imply_configs[prev] + if not skip: + imply_configs[imply_config] = common_defconfigs + + # Now we have a dict imply_configs of configs which imply each config + # The value of each dict item is the set of defconfigs containing that + # config. Rank them so that we print the configs that imply the largest + # number of defconfigs first. + ranked_iconfigs = sorted(imply_configs, + key=lambda k: len(imply_configs[k]), reverse=True) + kconfig_info = '' + cwd = os.getcwd() + add_list = collections.defaultdict(list) + for iconfig in ranked_iconfigs: + num_common = len(imply_configs[iconfig]) + + # Don't bother if there are less than 5 defconfigs affected. + if num_common < (2 if imply_flags & IMPLY_MIN_2 else 5): + continue + missing = defconfigs - imply_configs[iconfig] + missing_str = ', '.join(missing) if missing else 'all' + missing_str = '' + show = True + if kconf: + sym = find_kconfig_rules(kconf, config[CONFIG_LEN:], + iconfig[CONFIG_LEN:]) + kconfig_info = '' + if sym: + nodes = sym.nodes + if len(nodes) == 1: + fname, linenum = nodes[0].filename, nodes[0].linenr + if cwd and fname.startswith(cwd): + fname = fname[len(cwd) + 1:] + kconfig_info = f'{fname}:{linenum}' + if skip_added: + show = False + else: + sym = kconf.syms.get(iconfig[CONFIG_LEN:]) + fname = '' + if sym: + nodes = sym.nodes + if len(nodes) == 1: + fname, linenum = nodes[0].filename, nodes[0].linenr + if cwd and fname.startswith(cwd): + fname = fname[len(cwd) + 1:] + in_arch_board = not sym or (fname.startswith('arch') or + fname.startswith('board')) + if (not in_arch_board and + not imply_flags & IMPLY_NON_ARCH_BOARD): + continue + + if add_imply and (add_imply == 'all' or + iconfig in add_imply): + fname, linenum, kconfig_info = (check_imply_rule(kconf, + config[CONFIG_LEN:], iconfig[CONFIG_LEN:])) + if fname: + add_list[fname].append(linenum) + + if show and kconfig_info != 'skip': + print(f'{num_common:5d} : ' + f'{iconfig.ljust(30):-30s}{kconfig_info:-25s} {missing_str}') + + # Having collected a list of things to add, now we add them. We process + # each file from the largest line number to the smallest so that + # earlier additions do not affect our line numbers. E.g. if we added an + # imply at line 20 it would change the position of each line after + # that. + for fname, linenums in add_list.items(): + for linenum in sorted(linenums, reverse=True): + add_imply_rule(config[CONFIG_LEN:], fname, linenum) + +def defconfig_matches(configs, re_match): + """Check if any CONFIG option matches a regex + + The match must be complete, i.e. from the start to end of the CONFIG option. + + Args: + configs (dict): Dict of CONFIG options: + key: CONFIG option + value: Value of option + re_match (re.Pattern): Match to check + + Returns: + bool: True if any CONFIG matches the regex + """ + for cfg in configs: + if re_match.fullmatch(cfg): + return True + return False + +def do_find_config(config_list): + """Find boards with a given combination of CONFIGs + + Params: + config_list: List of CONFIG options to check (each a regex consisting + of a config option, with or without a CONFIG_ prefix. If an option + is preceded by a tilde (~) then it must be false, otherwise it must + be true) + """ + _, all_defconfigs, config_db, _ = read_database() + + # Start with all defconfigs + out = all_defconfigs + + # Work through each config in turn + for item in config_list: + # Get the real config name and whether we want this config or not + cfg = item + want = True + if cfg[0] == '~': + want = False + cfg = cfg[1:] + + # Search everything that is still in the running. If it has a config + # that we want, or doesn't have one that we don't, add it into the + # running for the next stage + in_list = out + out = set() + re_match = re.compile(cfg) + for defc in in_list: + has_cfg = defconfig_matches(config_db[defc], re_match) + if has_cfg == want: + out.add(defc) + print(f'{len(out)} matches') + print(' '.join(item.split('_defconfig')[0] for item in out)) + + +def prefix_config(cfg): + """Prefix a config with CONFIG_ if needed + + This handles ~ operator, which indicates that the CONFIG should be disabled + + >>> prefix_config('FRED') + 'CONFIG_FRED' + >>> prefix_config('CONFIG_FRED') + 'CONFIG_FRED' + >>> prefix_config('~FRED') + '~CONFIG_FRED' + >>> prefix_config('~CONFIG_FRED') + '~CONFIG_FRED' + >>> prefix_config('A123') + 'CONFIG_A123' + """ + oper = '' + if cfg[0] == '~': + oper = cfg[0] + cfg = cfg[1:] + if not cfg.startswith('CONFIG_'): + cfg = 'CONFIG_' + cfg + return oper + cfg + + +RE_MK_CONFIGS = re.compile(r'CONFIG_(\$\(SPL_(?:TPL_)?\))?([A-Za-z0-9_]*)') +RE_IFDEF = re.compile(r'(ifdef|ifndef)') +RE_C_CONFIGS = re.compile(r'CONFIG_([A-Za-z0-9_]*)') +RE_CONFIG_IS = re.compile(r'CONFIG_IS_ENABLED\(([A-Za-z0-9_]*)\)') + +class ConfigUse: + def __init__(self, cfg, is_spl, fname, rest): + self.cfg = cfg + self.is_spl = is_spl + self.fname = fname + self.rest = rest + + def __hash__(self): + return hash((self.cfg, self.is_spl)) + +def scan_makefiles(fnames): + """Scan Makefiles looking for Kconfig options + + Looks for uses of CONFIG options in Makefiles + + Args: + fnames (list of tuple): + str: Makefile filename where the option was found + str: Line of the Makefile + + Returns: + tuple: + dict: all_uses + key (ConfigUse): object + value (list of str): matching lines + dict: Uses by filename + key (str): filename + value (set of ConfigUse): uses in that filename + + >>> RE_MK_CONFIGS.search('CONFIG_FRED').groups() + (None, 'FRED') + >>> RE_MK_CONFIGS.search('CONFIG_$(SPL_)MARY').groups() + ('$(SPL_)', 'MARY') + >>> RE_MK_CONFIGS.search('CONFIG_$(SPL_TPL_)MARY').groups() + ('$(SPL_TPL_)', 'MARY') + """ + all_uses = collections.defaultdict(list) + fname_uses = {} + for fname, rest in fnames: + m_iter = RE_MK_CONFIGS.finditer(rest) + for mat in m_iter: + real_opt = mat.group(2) + if real_opt == '': + continue + is_spl = False + if mat.group(1): + is_spl = True + use = ConfigUse(real_opt, is_spl, fname, rest) + if fname not in fname_uses: + fname_uses[fname] = set() + fname_uses[fname].add(use) + all_uses[use].append(rest) + return all_uses, fname_uses + + +def scan_src_files(fnames): + """Scan source files (other than Makefiles) looking for Kconfig options + + Looks for uses of CONFIG options + + Args: + fnames (list of tuple): + str: Makefile filename where the option was found + str: Line of the Makefile + + Returns: + tuple: + dict: all_uses + key (ConfigUse): object + value (list of str): matching lines + dict: Uses by filename + key (str): filename + value (set of ConfigUse): uses in that filename + + >>> RE_C_CONFIGS.search('CONFIG_FRED').groups() + ('FRED',) + >>> RE_CONFIG_IS.search('CONFIG_IS_ENABLED(MARY)').groups() + ('MARY',) + >>> RE_CONFIG_IS.search('#if CONFIG_IS_ENABLED(OF_PLATDATA)').groups() + ('OF_PLATDATA',) + """ + fname = None + rest = None + + def add_uses(m_iter, is_spl): + for mat in m_iter: + real_opt = mat.group(1) + if real_opt == '': + continue + use = ConfigUse(real_opt, is_spl, fname, rest) + if fname not in fname_uses: + fname_uses[fname] = set() + fname_uses[fname].add(use) + all_uses[use].append(rest) + + all_uses = collections.defaultdict(list) + fname_uses = {} + for fname, rest in fnames: + m_iter = RE_C_CONFIGS.finditer(rest) + add_uses(m_iter, False) + + m_iter2 = RE_CONFIG_IS.finditer(rest) + add_uses(m_iter2, True) + + return all_uses, fname_uses + + +MODE_NORMAL, MODE_SPL, MODE_PROPER = range(3) + +def do_scan_source(path, do_update): + """Scan the source tree for Kconfig inconsistencies + + Args: + path (str): Path to source tree + do_update (bool) : True to write to scripts/kconf_... files + """ + def is_not_proper(name): + for prefix in SPL_PREFIXES: + if name.startswith(prefix): + return name[len(prefix):] + return False + + def check_not_found(all_uses, spl_mode): + """Check for Kconfig options mentioned in the source but not in Kconfig + + Args: + all_uses (dict): + key (ConfigUse): object + value (list of str): matching lines + spl_mode (int): If MODE_SPL, look at source code which implies + an SPL_ option, but for which there is none; + for MOD_PROPER, look at source code which implies a Proper + option (i.e. use of CONFIG_IS_ENABLED() or $(SPL_) or + $(SPL_TPL_) but for which there none; + if MODE_NORMAL, ignore SPL + + Returns: + dict: + key (str): CONFIG name (without 'CONFIG_' prefix + value (list of ConfigUse): List of uses of this CONFIG + """ + # Make sure we know about all the options + not_found = collections.defaultdict(list) + for use, _ in all_uses.items(): + name = use.cfg + if name in IGNORE_SYMS: + continue + check = True + + if spl_mode == MODE_SPL: + check = use.is_spl + + # If it is an SPL symbol, try prepending all SPL_ prefixes to + # find at least one SPL symbol + if use.is_spl: + for prefix in SPL_PREFIXES: + try_name = prefix + name + sym = kconf.syms.get(try_name) + if sym: + break + if not sym: + not_found[f'SPL_{name}'].append(use) + continue + elif spl_mode == MODE_PROPER: + # Try to find the Proper version of this symbol, i.e. without + # the SPL_ prefix + proper_name = is_not_proper(name) + if proper_name: + name = proper_name + elif not use.is_spl: + check = False + else: # MODE_NORMAL + sym = kconf.syms.get(name) + if not sym: + proper_name = is_not_proper(name) + if proper_name: + name = proper_name + sym = kconf.syms.get(name) + if not sym: + for prefix in SPL_PREFIXES: + try_name = prefix + name + sym = kconf.syms.get(try_name) + if sym: + break + if not sym: + not_found[name].append(use) + continue + + sym = kconf.syms.get(name) + if not sym and check: + not_found[name].append(use) + return not_found + + def show_uses(uses): + """Show a list of uses along with their filename and code snippet + + Args: + uses (dict): + key (str): CONFIG name (without 'CONFIG_' prefix + value (list of ConfigUse): List of uses of this CONFIG + """ + for name in sorted(uses): + print(f'{name}: ', end='') + for i, use in enumerate(uses[name]): + print(f'{" " if i else ""}{use.fname}: {use.rest.strip()}') + + + print('Scanning Kconfig') + kconf = KconfigScanner().conf + print(f'Scanning source in {path}') + args = ['git', 'grep', '-E', r'IS_ENABLED|\bCONFIG'] + with subprocess.Popen(args, stdout=subprocess.PIPE) as proc: + out, _ = proc.communicate() + lines = out.splitlines() + re_fname = re.compile('^([^:]*):(.*)') + src_list = [] + mk_list = [] + for line in lines: + linestr = line.decode('utf-8') + m_fname = re_fname.search(linestr) + if not m_fname: + continue + fname, rest = m_fname.groups() + dirname, leaf = os.path.split(fname) + root, ext = os.path.splitext(leaf) + if ext == '.autoconf': + pass + elif ext in ['.c', '.h', '.S', '.lds', '.dts', '.dtsi', '.asl', '.cfg', + '.env', '.tmpl']: + src_list.append([fname, rest]) + elif 'Makefile' in root or ext == '.mk': + mk_list.append([fname, rest]) + elif ext in ['.yml', '.sh', '.py', '.awk', '.pl', '.rst', '', '.sed']: + pass + elif 'Kconfig' in root or 'Kbuild' in root: + pass + elif 'README' in root: + pass + elif dirname in ['configs']: + pass + elif dirname.startswith('doc') or dirname.startswith('scripts/kconfig'): + pass + else: + print(f'Not sure how to handle file {fname}') + + # Scan the Makefiles + all_uses, _ = scan_makefiles(mk_list) + + spl_not_found = set() + proper_not_found = set() + + # Make sure we know about all the options + print('\nCONFIG options present in Makefiles but not Kconfig:') + not_found = check_not_found(all_uses, MODE_NORMAL) + show_uses(not_found) + + print('\nCONFIG options present in Makefiles but not Kconfig (SPL):') + not_found = check_not_found(all_uses, MODE_SPL) + show_uses(not_found) + spl_not_found |= {is_not_proper(key) or key for key in not_found.keys()} + + print('\nCONFIG options used as Proper in Makefiles but without a non-SPL_ variant:') + not_found = check_not_found(all_uses, MODE_PROPER) + show_uses(not_found) + proper_not_found |= {not_found.keys()} + + # Scan the source code + all_uses, _ = scan_src_files(src_list) + + # Make sure we know about all the options + print('\nCONFIG options present in source but not Kconfig:') + not_found = check_not_found(all_uses, MODE_NORMAL) + show_uses(not_found) + + print('\nCONFIG options present in source but not Kconfig (SPL):') + not_found = check_not_found(all_uses, MODE_SPL) + show_uses(not_found) + spl_not_found |= {is_not_proper(key) or key for key in not_found.keys()} + + print('\nCONFIG options used as Proper in source but without a non-SPL_ variant:') + not_found = check_not_found(all_uses, MODE_PROPER) + show_uses(not_found) + proper_not_found |= {not_found.keys()} + + print('\nCONFIG options used as SPL but without an SPL_ variant:') + for item in sorted(spl_not_found): + print(f' {item}') + + print('\nCONFIG options used as Proper but without a non-SPL_ variant:') + for item in sorted(proper_not_found): + print(f' {item}') + + # Write out the updated information + if do_update: + with open(os.path.join(path, 'scripts', 'conf_nospl'), 'w', + encoding='utf-8') as out: + print('# These options should not be enabled in SPL builds\n', + file=out) + for item in sorted(spl_not_found): + print(item, file=out) + with open(os.path.join(path, 'scripts', 'conf_noproper'), 'w', + encoding='utf-8') as out: + print('# These options should not be enabled in Proper builds\n', + file=out) + for item in sorted(proper_not_found): + print(item, file=out) + + +def main(): + try: + cpu_count = multiprocessing.cpu_count() + except NotImplementedError: + cpu_count = 1 + + epilog = '''Move config options from headers to defconfig files. See +doc/develop/moveconfig.rst for documentation.''' + + parser = ArgumentParser(epilog=epilog) + # Add arguments here + parser.add_argument('-a', '--add-imply', type=str, default='', + help='comma-separated list of CONFIG options to add ' + "an 'imply' statement to for the CONFIG in -i") + parser.add_argument('-A', '--skip-added', action='store_true', default=False, + help="don't show options which are already marked as " + 'implying others') + parser.add_argument('-b', '--build-db', action='store_true', default=False, + help='build a CONFIG database') + parser.add_argument('-C', '--commit', action='store_true', default=False, + help='Create a git commit for the operation') + parser.add_argument('--nocolour', action='store_true', default=False, + help="don't display the log in colour") + parser.add_argument('-d', '--defconfigs', type=str, + help='a file containing a list of defconfigs to move, ' + "one per line (for example 'snow_defconfig') " + "or '-' to read from stdin") + parser.add_argument('-e', '--exit-on-error', action='store_true', + default=False, + help='exit immediately on any error') + parser.add_argument('-f', '--find', action='store_true', default=False, + help='Find boards with a given config combination') + parser.add_argument('-i', '--imply', action='store_true', default=False, + help='find options which imply others') + parser.add_argument('-I', '--imply-flags', type=str, default='', + help="control the -i option ('help' for help") + parser.add_argument('-j', '--jobs', type=int, default=cpu_count, + help='the number of jobs to run simultaneously') + parser.add_argument('-n', '--dry-run', action='store_true', default=False, + help='perform a trial run (show log with no changes)') + parser.add_argument('-r', '--git-ref', type=str, + help='the git ref to clone for building the autoconf.mk') + parser.add_argument('-s', '--force-sync', action='store_true', default=False, + help='force sync by savedefconfig') + parser.add_argument('-S', '--spl', action='store_true', default=False, + help='parse config options defined for SPL build') + parser.add_argument('--scan-source', action='store_true', default=False, + help='scan source for uses of CONFIG options') + parser.add_argument('-t', '--test', action='store_true', default=False, + help='run unit tests') + parser.add_argument('-y', '--yes', action='store_true', default=False, + help="respond 'yes' to any prompts") + parser.add_argument('-u', '--update', action='store_true', default=False, + help="update scripts/ files (use with --scan-source)") + parser.add_argument('-v', '--verbose', action='store_true', default=False, + help='show any build errors as boards are built') + parser.add_argument('configs', nargs='*') + + args = parser.parse_args() + + if args.test: + sys.argv = [sys.argv[0]] + fail, _ = doctest.testmod() + if fail: + return 1 + unittest.main() + + col = terminal.Color(terminal.COLOR_NEVER if args.nocolour + else terminal.COLOR_IF_TERMINAL) + + if args.scan_source: + do_scan_source(os.getcwd(), args.update) + return 0 + + if not any((args.force_sync, args.build_db, args.imply, args.find)): + parser.print_usage() + sys.exit(1) + + # prefix the option name with CONFIG_ if missing + configs = [prefix_config(cfg) for cfg in args.configs] + + check_top_directory() + + if args.imply: + imply_flags = 0 + if args.imply_flags == 'all': + imply_flags = -1 + + elif args.imply_flags: + for flag in args.imply_flags.split(','): + bad = flag not in IMPLY_FLAGS + if bad: + print(f"Invalid flag '{flag}'") + if flag == 'help' or bad: + print("Imply flags: (separate with ',')") + for name, info in IMPLY_FLAGS.items(): + print(f' {name:-15s}: {info[1]}') + parser.print_usage() + sys.exit(1) + imply_flags |= IMPLY_FLAGS[flag][0] + + do_imply_config(configs, args.add_imply, imply_flags, args.skip_added) + return 0 + + if args.find: + do_find_config(configs) + return 0 + + # We are either building the database or forcing a sync of defconfigs + config_db = {} + db_queue = queue.Queue() + dbt = DatabaseThread(config_db, db_queue) + dbt.daemon = True + dbt.start() + + check_clean_directory() + bsettings.setup('') + toolchains = toolchain.Toolchains() + toolchains.GetSettings() + toolchains.Scan(verbose=False) + progress = move_config(toolchains, args, db_queue, col) + db_queue.join() + + if args.commit: + subprocess.call(['git', 'add', '-u']) + if configs: + msg = 'Convert %s %sto Kconfig' % (configs[0], + 'et al ' if len(configs) > 1 else '') + msg += ('\n\nThis converts the following to Kconfig:\n %s\n' % + '\n '.join(configs)) + else: + msg = 'configs: Resync with savedefconfig' + msg += '\n\nRsync all defconfig files using moveconfig.py' + subprocess.call(['git', 'commit', '-s', '-m', msg]) + + failed = progress.total - progress.good + failure = f'{failed} failed, ' if failed else '' + if args.build_db: + with open(CONFIG_DATABASE, 'w', encoding='utf-8') as outf: + for defconfig, configs in config_db.items(): + outf.write(f'{defconfig}\n') + for config in sorted(configs.keys()): + outf.write(f' {config}={configs[config]}\n') + outf.write('\n') + print(col.build( + col.RED if failed else col.GREEN, + f'{failure}{len(config_db)} boards written to {CONFIG_DATABASE}')) + else: + if failed: + print(col.build(col.RED, f'{failure}see {FAILED_LIST}', True)) + else: + # Add enough spaces to overwrite the progress indicator + print(col.build( + col.GREEN, f'{progress.total} processed ', bright=True)) + + return 0 + + +if __name__ == '__main__': + sys.exit(main()) -- cgit v1.2.3