Page MenuHomePhabricator
Differential D5217 Diff 16164 contrib/devtools/copyright_header.py

Changeset View

Standalone View

View Options

contrib/devtools/copyright_header.py

#!/usr/bin/env python3 #!/usr/bin/env python3
# Copyright (c) 2016 The Bitcoin Core developers # Copyright (c) 2016 The Bitcoin Core developers
# Copyright (c) 2017 The Bitcoin developers # Copyright (c) 2017 The Bitcoin developers
# Distributed under the MIT software license, see the accompanying # Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php. # file COPYING or http://www.opensource.org/licenses/mit-license.php.
import re import re
import fnmatch import fnmatch
import sys import sys
import subprocess import subprocess
import datetime import datetime
import os import os
################################################################################ ##########################################################################
# file filtering # file filtering
################################################################################ ##########################################################################
EXCLUDE = [ EXCLUDE = [
# libsecp256k1: # libsecp256k1:
'src/secp256k1/include/secp256k1.h', 'src/secp256k1/include/secp256k1.h',
'src/secp256k1/include/secp256k1_ecdh.h', 'src/secp256k1/include/secp256k1_ecdh.h',
'src/secp256k1/include/secp256k1_recovery.h', 'src/secp256k1/include/secp256k1_recovery.h',
'src/secp256k1/include/secp256k1_schnorr.h', 'src/secp256k1/include/secp256k1_schnorr.h',
'src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.c', 'src/secp256k1/src/java/org_bitcoin_NativeSecp256k1.c',
Show All 16 LinesEXCLUDE_COMPILED = re.compile(
'|'.join([fnmatch.translate(m) for m in EXCLUDE])) '|'.join([fnmatch.translate(m) for m in EXCLUDE]))
INCLUDE = ['*.h', '*.cpp', '*.cc', '*.c', '*.py'] INCLUDE = ['*.h', '*.cpp', '*.cc', '*.c', '*.py']
INCLUDE_COMPILED = re.compile( INCLUDE_COMPILED = re.compile(
'|'.join([fnmatch.translate(m) for m in INCLUDE])) '|'.join([fnmatch.translate(m) for m in INCLUDE]))
def applies_to_file(filename): def applies_to_file(filename):
return ((EXCLUDE_COMPILED.match(filename) is None) and return ((EXCLUDE_COMPILED.match(filename) is None)
(INCLUDE_COMPILED.match(filename) is not None)) and (INCLUDE_COMPILED.match(filename) is not None))
################################################################################ ##########################################################################
# obtain list of files in repo according to INCLUDE and EXCLUDE # obtain list of files in repo according to INCLUDE and EXCLUDE
################################################################################ ##########################################################################
GIT_LS_CMD = 'git ls-files' GIT_LS_CMD = 'git ls-files'
def call_git_ls(): def call_git_ls():
out = subprocess.check_output(GIT_LS_CMD.split(' ')) out = subprocess.check_output(GIT_LS_CMD.split(' '))
return [f for f in out.decode("utf-8").split('\n') if f != ''] return [f for f in out.decode("utf-8").split('\n') if f != '']
def get_filenames_to_examine(): def get_filenames_to_examine():
filenames = call_git_ls() filenames = call_git_ls()
return sorted([filename for filename in filenames if return sorted([filename for filename in filenames if
applies_to_file(filename)]) applies_to_file(filename)])
################################################################################ ##########################################################################
# define and compile regexes for the patterns we are looking for # define and compile regexes for the patterns we are looking for
################################################################################ ##########################################################################
COPYRIGHT_WITH_C = r'Copyright \(c\)' COPYRIGHT_WITH_C = r'Copyright \(c\)'
COPYRIGHT_WITHOUT_C = 'Copyright' COPYRIGHT_WITHOUT_C = 'Copyright'
ANY_COPYRIGHT_STYLE = '({}|{})'.format(COPYRIGHT_WITH_C, COPYRIGHT_WITHOUT_C) ANY_COPYRIGHT_STYLE = '({}|{})'.format(COPYRIGHT_WITH_C, COPYRIGHT_WITHOUT_C)
YEAR = "20[0-9][0-9]" YEAR = "20[0-9][0-9]"
YEAR_RANGE = '({})(-{})?'.format(YEAR, YEAR) YEAR_RANGE = '({})(-{})?'.format(YEAR, YEAR)
▲ Show 20 Lines • Show All 45 Lines • ▼ Show 20 Linesfor holder_name in EXPECTED_HOLDER_NAMES:
DOMINANT_STYLE_COMPILED[holder_name] = ( DOMINANT_STYLE_COMPILED[holder_name] = (
compile_copyright_regex(COPYRIGHT_WITH_C, YEAR_RANGE, holder_name)) compile_copyright_regex(COPYRIGHT_WITH_C, YEAR_RANGE, holder_name))
YEAR_LIST_STYLE_COMPILED[holder_name] = ( YEAR_LIST_STYLE_COMPILED[holder_name] = (
compile_copyright_regex(COPYRIGHT_WITH_C, YEAR_LIST, holder_name)) compile_copyright_regex(COPYRIGHT_WITH_C, YEAR_LIST, holder_name))
WITHOUT_C_STYLE_COMPILED[holder_name] = ( WITHOUT_C_STYLE_COMPILED[holder_name] = (
compile_copyright_regex(COPYRIGHT_WITHOUT_C, ANY_YEAR_STYLE, compile_copyright_regex(COPYRIGHT_WITHOUT_C, ANY_YEAR_STYLE,
holder_name)) holder_name))
################################################################################ ##########################################################################
# search file contents for copyright message of particular category # search file contents for copyright message of particular category
################################################################################ ##########################################################################
def get_count_of_copyrights_of_any_style_any_holder(contents): def get_count_of_copyrights_of_any_style_any_holder(contents):
return len(ANY_COPYRIGHT_COMPILED.findall(contents)) return len(ANY_COPYRIGHT_COMPILED.findall(contents))
def file_has_dominant_style_copyright_for_holder(contents, holder_name): def file_has_dominant_style_copyright_for_holder(contents, holder_name):
match = DOMINANT_STYLE_COMPILED[holder_name].search(contents) match = DOMINANT_STYLE_COMPILED[holder_name].search(contents)
return match is not None return match is not None
def file_has_year_list_style_copyright_for_holder(contents, holder_name): def file_has_year_list_style_copyright_for_holder(contents, holder_name):
match = YEAR_LIST_STYLE_COMPILED[holder_name].search(contents) match = YEAR_LIST_STYLE_COMPILED[holder_name].search(contents)
return match is not None return match is not None
def file_has_without_c_style_copyright_for_holder(contents, holder_name): def file_has_without_c_style_copyright_for_holder(contents, holder_name):
match = WITHOUT_C_STYLE_COMPILED[holder_name].search(contents) match = WITHOUT_C_STYLE_COMPILED[holder_name].search(contents)
return match is not None return match is not None
################################################################################ ##########################################################################
# get file info # get file info
################################################################################ ##########################################################################
def read_file(filename): def read_file(filename):
return open(os.path.abspath(filename), 'r', encoding="utf8").read() return open(os.path.abspath(filename), 'r', encoding="utf8").read()
def gather_file_info(filename): def gather_file_info(filename):
info = {} info = {}
Show All 16 Lines for holder_name in EXPECTED_HOLDER_NAMES:
file_has_without_c_style_copyright_for_holder(c, holder_name)) file_has_without_c_style_copyright_for_holder(c, holder_name))
info['dominant_style'][holder_name] = has_dominant_style info['dominant_style'][holder_name] = has_dominant_style
info['year_list_style'][holder_name] = has_year_list_style info['year_list_style'][holder_name] = has_year_list_style
info['without_c_style'][holder_name] = has_without_c_style info['without_c_style'][holder_name] = has_without_c_style
if has_dominant_style or has_year_list_style or has_without_c_style: if has_dominant_style or has_year_list_style or has_without_c_style:
info['classified_copyrights'] = info['classified_copyrights'] + 1 info['classified_copyrights'] = info['classified_copyrights'] + 1
return info return info
################################################################################ ##########################################################################
# report execution # report execution
################################################################################ ##########################################################################
SEPARATOR = '-'.join(['' for _ in range(80)]) SEPARATOR = '-'.join(['' for _ in range(80)])
def print_filenames(filenames, verbose): def print_filenames(filenames, verbose):
if not verbose: if not verbose:
return return
▲ Show 20 Lines • Show All 79 Lines • ▼ Show 20 Lines
def exec_report(base_directory, verbose): def exec_report(base_directory, verbose):
original_cwd = os.getcwd() original_cwd = os.getcwd()
os.chdir(base_directory) os.chdir(base_directory)
filenames = get_filenames_to_examine() filenames = get_filenames_to_examine()
file_infos = [gather_file_info(f) for f in filenames] file_infos = [gather_file_info(f) for f in filenames]
print_report(file_infos, verbose) print_report(file_infos, verbose)
os.chdir(original_cwd) os.chdir(original_cwd)
################################################################################ ##########################################################################
# report cmd # report cmd
################################################################################ ##########################################################################
REPORT_USAGE = """ REPORT_USAGE = """
Produces a report of all copyright header notices found inside the source files Produces a report of all copyright header notices found inside the source files
of a repository. of a repository.
Usage: Usage:
$ ./copyright_header.py report <base_directory> [verbose] $ ./copyright_header.py report <base_directory> [verbose]
Show All 16 Lines if len(argv) == 3:
verbose = False verbose = False
elif argv[3] == 'verbose': elif argv[3] == 'verbose':
verbose = True verbose = True
else: else:
sys.exit("*** unknown argument: {}".format(argv[2])) sys.exit("*** unknown argument: {}".format(argv[2]))
exec_report(base_directory, verbose) exec_report(base_directory, verbose)
################################################################################ ##########################################################################
# query git for year of last change # query git for year of last change
################################################################################ ##########################################################################
GIT_LOG_CMD = "git log --pretty=format:%ai {}" GIT_LOG_CMD = "git log --pretty=format:%ai {}"
def call_git_log(filename): def call_git_log(filename):
out = subprocess.check_output((GIT_LOG_CMD.format(filename)).split(' ')) out = subprocess.check_output((GIT_LOG_CMD.format(filename)).split(' '))
return out.decode("utf-8").split('\n') return out.decode("utf-8").split('\n')
def get_git_change_years(filename): def get_git_change_years(filename):
git_log_lines = call_git_log(filename) git_log_lines = call_git_log(filename)
if len(git_log_lines) == 0: if len(git_log_lines) == 0:
return [datetime.date.today().year] return [datetime.date.today().year]
# timestamp is in ISO 8601 format. e.g. "2016-09-05 14:25:32 -0600" # timestamp is in ISO 8601 format. e.g. "2016-09-05 14:25:32 -0600"
return [line.split(' ')[0].split('-')[0] for line in git_log_lines] return [line.split(' ')[0].split('-')[0] for line in git_log_lines]
def get_most_recent_git_change_year(filename): def get_most_recent_git_change_year(filename):
return max(get_git_change_years(filename)) return max(get_git_change_years(filename))
################################################################################ ##########################################################################
# read and write to file # read and write to file
################################################################################ ##########################################################################
def read_file_lines(filename): def read_file_lines(filename):
f = open(os.path.abspath(filename), 'r', encoding="utf8") f = open(os.path.abspath(filename), 'r', encoding="utf8")
file_lines = f.readlines() file_lines = f.readlines()
f.close() f.close()
return file_lines return file_lines
def write_file_lines(filename, file_lines): def write_file_lines(filename, file_lines):
f = open(os.path.abspath(filename), 'w', encoding="utf8") f = open(os.path.abspath(filename), 'w', encoding="utf8")
f.write(''.join(file_lines)) f.write(''.join(file_lines))
f.close() f.close()
################################################################################ ##########################################################################
# update header years execution # update header years execution
################################################################################ ##########################################################################
COPYRIGHT = r'Copyright \(c\)' COPYRIGHT = r'Copyright \(c\)'
YEAR = "20[0-9][0-9]" YEAR = "20[0-9][0-9]"
YEAR_RANGE = '({})(-{})?'.format(YEAR, YEAR) YEAR_RANGE = '({})(-{})?'.format(YEAR, YEAR)
HOLDER = 'The Bitcoin developers' HOLDER = 'The Bitcoin developers'
UPDATEABLE_LINE_COMPILED = re.compile( UPDATEABLE_LINE_COMPILED = re.compile(
' '.join([COPYRIGHT, YEAR_RANGE, HOLDER])) ' '.join([COPYRIGHT, YEAR_RANGE, HOLDER]))
▲ Show 20 Lines • Show All 42 Lines • ▼ Show 20 Linesdef create_updated_copyright_line(line, last_git_change_year):
before_copyright = copyright_split[0] before_copyright = copyright_split[0]
after_copyright = copyright_split[1] after_copyright = copyright_split[1]
space_split = after_copyright.split(' ') space_split = after_copyright.split(' ')
year_range = space_split[0] year_range = space_split[0]
start_year, end_year = parse_year_range(year_range) start_year, end_year = parse_year_range(year_range)
if end_year == last_git_change_year: if end_year == last_git_change_year:
return line return line
return (before_copyright + copyright_splitter + return (before_copyright + copyright_splitter
year_range_to_str(start_year, last_git_change_year) + ' ' + + year_range_to_str(start_year, last_git_change_year) + ' '
' '.join(space_split[1:])) + ' '.join(space_split[1:]))
def update_updatable_copyright(filename): def update_updatable_copyright(filename):
file_lines = read_file_lines(filename) file_lines = read_file_lines(filename)
index, line = get_updatable_copyright_line(file_lines) index, line = get_updatable_copyright_line(file_lines)
if not line: if not line:
print_file_action_message(filename, "No updatable copyright.") print_file_action_message(filename, "No updatable copyright.")
return return
Show All 11 Lines
def exec_update_header_year(base_directory): def exec_update_header_year(base_directory):
original_cwd = os.getcwd() original_cwd = os.getcwd()
os.chdir(base_directory) os.chdir(base_directory)
for filename in get_filenames_to_examine(): for filename in get_filenames_to_examine():
update_updatable_copyright(filename) update_updatable_copyright(filename)
os.chdir(original_cwd) os.chdir(original_cwd)
################################################################################ ##########################################################################
# update cmd # update cmd
################################################################################ ##########################################################################
UPDATE_USAGE = """ UPDATE_USAGE = """
Updates all the copyright headers of "The Bitcoin developers" which were Updates all the copyright headers of "The Bitcoin developers" which were
changed in a year more recent than is listed. For example: changed in a year more recent than is listed. For example:
// Copyright (c) <firstYear>-<lastYear> The Bitcoin developers // Copyright (c) <firstYear>-<lastYear> The Bitcoin developers
Show All 29 Linesdef update_cmd(argv):
if len(argv) != 3: if len(argv) != 3:
sys.exit(UPDATE_USAGE) sys.exit(UPDATE_USAGE)
base_directory = argv[2] base_directory = argv[2]
if not os.path.exists(base_directory): if not os.path.exists(base_directory):
sys.exit("*** bad base_directory: {}".format(base_directory)) sys.exit("*** bad base_directory: {}".format(base_directory))
exec_update_header_year(base_directory) exec_update_header_year(base_directory)
################################################################################ ##########################################################################
# inserted copyright header format # inserted copyright header format
################################################################################ ##########################################################################
def get_header_lines(header, start_year, end_year): def get_header_lines(header, start_year, end_year):
lines = header.split('\n')[1:-1] lines = header.split('\n')[1:-1]
lines[0] = lines[0].format(year_range_to_str(start_year, end_year)) lines[0] = lines[0].format(year_range_to_str(start_year, end_year))
return [line + '\n' for line in lines] return [line + '\n' for line in lines]
Show All 13 Lines
# Distributed under the MIT software license, see the accompanying # Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php. # file COPYING or http://www.opensource.org/licenses/mit-license.php.
''' '''
def get_python_header_lines_to_insert(start_year, end_year): def get_python_header_lines_to_insert(start_year, end_year):
return reversed(get_header_lines(PYTHON_HEADER, start_year, end_year)) return reversed(get_header_lines(PYTHON_HEADER, start_year, end_year))
################################################################################ ##########################################################################
# query git for year of last change # query git for year of last change
################################################################################ ##########################################################################
def get_git_change_year_range(filename): def get_git_change_year_range(filename):
years = get_git_change_years(filename) years = get_git_change_years(filename)
return min(years), max(years) return min(years), max(years)
################################################################################ ##########################################################################
# check for existing ABC copyright # check for existing ABC copyright
################################################################################ ##########################################################################
def file_already_has_bitcoin_copyright(file_lines): def file_already_has_bitcoin_copyright(file_lines):
index, _ = get_updatable_copyright_line(file_lines) index, _ = get_updatable_copyright_line(file_lines)
return index is not None return index is not None
################################################################################ ##########################################################################
# insert header execution # insert header execution
################################################################################ ##########################################################################
def file_has_hashbang(file_lines): def file_has_hashbang(file_lines):
if len(file_lines) < 1: if len(file_lines) < 1:
return False return False
if len(file_lines[0]) <= 2: if len(file_lines[0]) <= 2:
return False return False
return file_lines[0][:2] == '#!' return file_lines[0][:2] == '#!'
Show All 31 Lines if file_already_has_bitcoin_copyright(file_lines):
sys.exit('*** {} already has a copyright by The Bitcoin developers'.format( sys.exit('*** {} already has a copyright by The Bitcoin developers'.format(
filename)) filename))
start_year, end_year = get_git_change_year_range(filename) start_year, end_year = get_git_change_year_range(filename)
if style == 'python': if style == 'python':
insert_python_header(filename, file_lines, start_year, end_year) insert_python_header(filename, file_lines, start_year, end_year)
else: else:
insert_cpp_header(filename, file_lines, start_year, end_year) insert_cpp_header(filename, file_lines, start_year, end_year)
################################################################################ ##########################################################################
# insert cmd # insert cmd
################################################################################ ##########################################################################
INSERT_USAGE = """ INSERT_USAGE = """
Inserts a copyright header for "The Bitcoin developers" at the top of the Inserts a copyright header for "The Bitcoin developers" at the top of the
file in either Python or C++ style as determined by the file extension. If the file in either Python or C++ style as determined by the file extension. If the
file is a Python file and it has a '#!' starting the first line, the header is file is a Python file and it has a '#!' starting the first line, the header is
inserted in the line below it. inserted in the line below it.
Show All 29 Lines if extension not in ['.h', '.cpp', '.cc', '.c', '.py']:
sys.exit("*** cannot insert for file extension {}".format(extension)) sys.exit("*** cannot insert for file extension {}".format(extension))
if extension == '.py': if extension == '.py':
style = 'python' style = 'python'
else: else:
style = 'cpp' style = 'cpp'
exec_insert_header(filename, style) exec_insert_header(filename, style)
################################################################################ ##########################################################################
# UI # UI
################################################################################ ##########################################################################
USAGE = """ USAGE = """
copyright_header.py - utilities for managing copyright headers of 'The Bitcoin copyright_header.py - utilities for managing copyright headers of 'The Bitcoin
developers' in repository source files. developers' in repository source files.
Usage: Usage:
$ ./copyright_header <subcommand> $ ./copyright_header <subcommand>
Show All 23 Lines