From ebf4c0f24e7c8366d6d986808c36d208752190fe Mon Sep 17 00:00:00 2001 From: Meco Man <920369182@qq.com> Date: Sat, 13 May 2023 22:35:28 -0400 Subject: [tools/ci]move file_check.py into ci folder --- .github/workflows/file_check.yml | 2 +- tools/ci/file_check.py | 280 +++++++++++++++++++++++++++++++++++++++ tools/file_check.py | 280 --------------------------------------- 3 files changed, 281 insertions(+), 281 deletions(-) create mode 100644 tools/ci/file_check.py delete mode 100644 tools/file_check.py diff --git a/.github/workflows/file_check.yml b/.github/workflows/file_check.yml index a7c706d9c9..8cd2f7423c 100644 --- a/.github/workflows/file_check.yml +++ b/.github/workflows/file_check.yml @@ -17,4 +17,4 @@ jobs: shell: bash run: | pip install click chardet PyYaml - python tools/file_check.py check 'https://github.com/RT-Thread/rt-thread' 'master' + python tools/ci/file_check.py check 'https://github.com/RT-Thread/rt-thread' 'master' diff --git a/tools/ci/file_check.py b/tools/ci/file_check.py new file mode 100644 index 0000000000..fbb72786ad --- /dev/null +++ b/tools/ci/file_check.py @@ -0,0 +1,280 @@ +# +# Copyright (c) 2006-2022, RT-Thread Development Team +# +# SPDX-License-Identifier: Apache-2.0 +# +# Change Logs: +# Date Author Notes +# 2021-04-01 LiuKang the first version +# + +import os +import re +import sys +import click +import yaml +import chardet +import logging +import datetime + + +def init_logger(): + log_format = "[%(filename)s %(lineno)d %(levelname)s] %(message)s " + date_format = '%Y-%m-%d %H:%M:%S %a ' + logging.basicConfig(level=logging.INFO, + format=log_format, + datefmt=date_format, + ) + + +class CheckOut: + def __init__(self, rtt_repo, rtt_branch): + self.root = os.getcwd() + self.rtt_repo = rtt_repo + self.rtt_branch = rtt_branch + + def __exclude_file(self, file_path): + dir_number = file_path.split('/') + ignore_path = file_path + + # gets the file path depth. + for i in dir_number: + # current directory. + dir_name = os.path.dirname(ignore_path) + ignore_path = dir_name + # judge the ignore file exists in the current directory. + ignore_file_path = os.path.join(dir_name, ".ignore_format.yml") + if not os.path.exists(ignore_file_path): + continue + try: + with open(ignore_file_path) as f: + ignore_config = yaml.safe_load(f.read()) + file_ignore = ignore_config.get("file_path", []) + dir_ignore = ignore_config.get("dir_path", []) + except Exception as e: + logging.error(e) + continue + logging.debug("ignore file path: {}".format(ignore_file_path)) + logging.debug("file_ignore: {}".format(file_ignore)) + logging.debug("dir_ignore: {}".format(dir_ignore)) + try: + # judge file_path in the ignore file. + for file in file_ignore: + if file is not None: + file_real_path = os.path.join(dir_name, file) + if file_real_path == file_path: + logging.info("ignore file path: {}".format(file_real_path)) + return 0 + + file_dir_path = os.path.dirname(file_path) + for _dir in dir_ignore: + if _dir is not None: + dir_real_path = os.path.join(dir_name, _dir) + if file_dir_path.startswith(dir_real_path): + logging.info("ignore dir path: {}".format(dir_real_path)) + return 0 + except Exception as e: + logging.error(e) + continue + + return 1 + + def get_new_file(self): + file_list = list() + try: + os.system('git remote add rtt_repo {}'.format(self.rtt_repo)) + os.system('git fetch rtt_repo') + os.system('git merge rtt_repo/{}'.format(self.rtt_branch)) + os.system('git reset rtt_repo/{} --soft'.format(self.rtt_branch)) + os.system('git status > git.txt') + except Exception as e: + logging.error(e) + return None + try: + with open('git.txt', 'r') as f: + file_lines = f.readlines() + except Exception as e: + logging.error(e) + return None + file_path = '' + for line in file_lines: + if 'new file' in line: + file_path = line.split('new file:')[1].strip() + logging.info('new file -> {}'.format(file_path)) + elif 'deleted' in line: + logging.info('deleted file -> {}'.format(line.split('deleted:')[1].strip())) + elif 'modified' in line: + file_path = line.split('modified:')[1].strip() + logging.info('modified file -> {}'.format(file_path)) + else: + continue + + result = self.__exclude_file(file_path) + if result != 0: + file_list.append(file_path) + + return file_list + + +class FormatCheck: + def __init__(self, file_list): + self.file_list = file_list + + def __check_file(self, file_lines, file_path): + line_num = 1 + check_result = True + for line in file_lines: + # check line start + line_start = line.replace(' ', '') + # find tab + if line_start.startswith('\t'): + logging.error("{} line[{}]: please use space replace tab at the start of this line.".format(file_path, line_num)) + check_result = False + # check line end + lin_end = line.split('\n')[0] + if lin_end.endswith(' ') or lin_end.endswith('\t'): + logging.error("{} line[{}]: please delete extra space at the end of this line.".format(file_path, line_num)) + check_result = False + line_num += 1 + + return check_result + + def check(self): + logging.info("Start to check files format.") + if len(self.file_list) == 0: + logging.warning("There are no files to check format.") + return True + encoding_check_result = True + format_check_fail_files = 0 + for file_path in self.file_list: + code = '' + if file_path.endswith(".c") or file_path.endswith(".h"): + try: + with open(file_path, 'rb') as f: + file = f.read() + # get file encoding + code = chardet.detect(file)['encoding'] + except Exception as e: + logging.error(e) + else: + continue + + if code != 'utf-8' and code != 'ascii': + logging.error("[{0}]: encoding not utf-8, please format it.".format(file_path)) + encoding_check_result = False + else: + logging.info('[{0}]: encoding check success.'.format(file_path)) + + with open(file_path, 'r', encoding = "utf-8") as f: + file_lines = f.readlines() + if not self.__check_file(file_lines, file_path): + format_check_fail_files += 1 + + if (not encoding_check_result) or (format_check_fail_files != 0): + logging.error("files format check fail.") + return False + + logging.info("files format check success.") + + return True + + +class LicenseCheck: + def __init__(self, file_list): + self.file_list = file_list + + def check(self): + current_year = datetime.date.today().year + logging.info("current year: {}".format(current_year)) + if len(self.file_list) == 0: + logging.warning("There are no files to check license.") + return 0 + logging.info("Start to check files license.") + check_result = True + for file_path in self.file_list: + if file_path.endswith(".c") or file_path.endswith(".h"): + try: + with open(file_path, 'r') as f: + file = f.readlines() + except Exception as e: + logging.error(e) + else: + continue + + if 'Copyright' in file[1] and 'SPDX-License-Identifier: Apache-2.0' in file[3]: + try: + license_year = re.search(r'2006-\d{4}', file[1]).group() + true_year = '2006-{}'.format(current_year) + if license_year != true_year: + logging.warning("[{0}]: license year: {} is not true: {}, please update.".format(file_path, + license_year, + true_year)) + + else: + logging.info("[{0}]: license check success.".format(file_path)) + except Exception as e: + logging.error(e) + + else: + logging.error("[{0}]: license check fail.".format(file_path)) + check_result = False + + return check_result + + +@click.group() +@click.pass_context +def cli(ctx): + pass + + +@cli.command() +@click.option( + '--license', + "check_license", + required=False, + type=click.BOOL, + flag_value=True, + help="Enable File license check.", +) +@click.argument( + 'repo', + nargs=1, + type=click.STRING, + default='https://github.com/RT-Thread/rt-thread', +) +@click.argument( + 'branch', + nargs=1, + type=click.STRING, + default='master', +) +def check(check_license, repo, branch): + """ + check files license and format. + """ + init_logger() + # get modified files list + checkout = CheckOut(repo, branch) + file_list = checkout.get_new_file() + if file_list is None: + logging.error("checkout files fail") + sys.exit(1) + + # check modified files format + format_check = FormatCheck(file_list) + format_check_result = format_check.check() + license_check_result = True + if check_license: + license_check = LicenseCheck(file_list) + license_check_result = license_check.check() + + if not format_check_result or not license_check_result: + logging.error("file format check or license check fail.") + sys.exit(1) + logging.info("check success.") + sys.exit(0) + + +if __name__ == '__main__': + cli() diff --git a/tools/file_check.py b/tools/file_check.py deleted file mode 100644 index fbb72786ad..0000000000 --- a/tools/file_check.py +++ /dev/null @@ -1,280 +0,0 @@ -# -# Copyright (c) 2006-2022, RT-Thread Development Team -# -# SPDX-License-Identifier: Apache-2.0 -# -# Change Logs: -# Date Author Notes -# 2021-04-01 LiuKang the first version -# - -import os -import re -import sys -import click -import yaml -import chardet -import logging -import datetime - - -def init_logger(): - log_format = "[%(filename)s %(lineno)d %(levelname)s] %(message)s " - date_format = '%Y-%m-%d %H:%M:%S %a ' - logging.basicConfig(level=logging.INFO, - format=log_format, - datefmt=date_format, - ) - - -class CheckOut: - def __init__(self, rtt_repo, rtt_branch): - self.root = os.getcwd() - self.rtt_repo = rtt_repo - self.rtt_branch = rtt_branch - - def __exclude_file(self, file_path): - dir_number = file_path.split('/') - ignore_path = file_path - - # gets the file path depth. - for i in dir_number: - # current directory. - dir_name = os.path.dirname(ignore_path) - ignore_path = dir_name - # judge the ignore file exists in the current directory. - ignore_file_path = os.path.join(dir_name, ".ignore_format.yml") - if not os.path.exists(ignore_file_path): - continue - try: - with open(ignore_file_path) as f: - ignore_config = yaml.safe_load(f.read()) - file_ignore = ignore_config.get("file_path", []) - dir_ignore = ignore_config.get("dir_path", []) - except Exception as e: - logging.error(e) - continue - logging.debug("ignore file path: {}".format(ignore_file_path)) - logging.debug("file_ignore: {}".format(file_ignore)) - logging.debug("dir_ignore: {}".format(dir_ignore)) - try: - # judge file_path in the ignore file. - for file in file_ignore: - if file is not None: - file_real_path = os.path.join(dir_name, file) - if file_real_path == file_path: - logging.info("ignore file path: {}".format(file_real_path)) - return 0 - - file_dir_path = os.path.dirname(file_path) - for _dir in dir_ignore: - if _dir is not None: - dir_real_path = os.path.join(dir_name, _dir) - if file_dir_path.startswith(dir_real_path): - logging.info("ignore dir path: {}".format(dir_real_path)) - return 0 - except Exception as e: - logging.error(e) - continue - - return 1 - - def get_new_file(self): - file_list = list() - try: - os.system('git remote add rtt_repo {}'.format(self.rtt_repo)) - os.system('git fetch rtt_repo') - os.system('git merge rtt_repo/{}'.format(self.rtt_branch)) - os.system('git reset rtt_repo/{} --soft'.format(self.rtt_branch)) - os.system('git status > git.txt') - except Exception as e: - logging.error(e) - return None - try: - with open('git.txt', 'r') as f: - file_lines = f.readlines() - except Exception as e: - logging.error(e) - return None - file_path = '' - for line in file_lines: - if 'new file' in line: - file_path = line.split('new file:')[1].strip() - logging.info('new file -> {}'.format(file_path)) - elif 'deleted' in line: - logging.info('deleted file -> {}'.format(line.split('deleted:')[1].strip())) - elif 'modified' in line: - file_path = line.split('modified:')[1].strip() - logging.info('modified file -> {}'.format(file_path)) - else: - continue - - result = self.__exclude_file(file_path) - if result != 0: - file_list.append(file_path) - - return file_list - - -class FormatCheck: - def __init__(self, file_list): - self.file_list = file_list - - def __check_file(self, file_lines, file_path): - line_num = 1 - check_result = True - for line in file_lines: - # check line start - line_start = line.replace(' ', '') - # find tab - if line_start.startswith('\t'): - logging.error("{} line[{}]: please use space replace tab at the start of this line.".format(file_path, line_num)) - check_result = False - # check line end - lin_end = line.split('\n')[0] - if lin_end.endswith(' ') or lin_end.endswith('\t'): - logging.error("{} line[{}]: please delete extra space at the end of this line.".format(file_path, line_num)) - check_result = False - line_num += 1 - - return check_result - - def check(self): - logging.info("Start to check files format.") - if len(self.file_list) == 0: - logging.warning("There are no files to check format.") - return True - encoding_check_result = True - format_check_fail_files = 0 - for file_path in self.file_list: - code = '' - if file_path.endswith(".c") or file_path.endswith(".h"): - try: - with open(file_path, 'rb') as f: - file = f.read() - # get file encoding - code = chardet.detect(file)['encoding'] - except Exception as e: - logging.error(e) - else: - continue - - if code != 'utf-8' and code != 'ascii': - logging.error("[{0}]: encoding not utf-8, please format it.".format(file_path)) - encoding_check_result = False - else: - logging.info('[{0}]: encoding check success.'.format(file_path)) - - with open(file_path, 'r', encoding = "utf-8") as f: - file_lines = f.readlines() - if not self.__check_file(file_lines, file_path): - format_check_fail_files += 1 - - if (not encoding_check_result) or (format_check_fail_files != 0): - logging.error("files format check fail.") - return False - - logging.info("files format check success.") - - return True - - -class LicenseCheck: - def __init__(self, file_list): - self.file_list = file_list - - def check(self): - current_year = datetime.date.today().year - logging.info("current year: {}".format(current_year)) - if len(self.file_list) == 0: - logging.warning("There are no files to check license.") - return 0 - logging.info("Start to check files license.") - check_result = True - for file_path in self.file_list: - if file_path.endswith(".c") or file_path.endswith(".h"): - try: - with open(file_path, 'r') as f: - file = f.readlines() - except Exception as e: - logging.error(e) - else: - continue - - if 'Copyright' in file[1] and 'SPDX-License-Identifier: Apache-2.0' in file[3]: - try: - license_year = re.search(r'2006-\d{4}', file[1]).group() - true_year = '2006-{}'.format(current_year) - if license_year != true_year: - logging.warning("[{0}]: license year: {} is not true: {}, please update.".format(file_path, - license_year, - true_year)) - - else: - logging.info("[{0}]: license check success.".format(file_path)) - except Exception as e: - logging.error(e) - - else: - logging.error("[{0}]: license check fail.".format(file_path)) - check_result = False - - return check_result - - -@click.group() -@click.pass_context -def cli(ctx): - pass - - -@cli.command() -@click.option( - '--license', - "check_license", - required=False, - type=click.BOOL, - flag_value=True, - help="Enable File license check.", -) -@click.argument( - 'repo', - nargs=1, - type=click.STRING, - default='https://github.com/RT-Thread/rt-thread', -) -@click.argument( - 'branch', - nargs=1, - type=click.STRING, - default='master', -) -def check(check_license, repo, branch): - """ - check files license and format. - """ - init_logger() - # get modified files list - checkout = CheckOut(repo, branch) - file_list = checkout.get_new_file() - if file_list is None: - logging.error("checkout files fail") - sys.exit(1) - - # check modified files format - format_check = FormatCheck(file_list) - format_check_result = format_check.check() - license_check_result = True - if check_license: - license_check = LicenseCheck(file_list) - license_check_result = license_check.check() - - if not format_check_result or not license_check_result: - logging.error("file format check or license check fail.") - sys.exit(1) - logging.info("check success.") - sys.exit(0) - - -if __name__ == '__main__': - cli() -- cgit v1.3.1