https://github.com/torvalds/linux
Revision 28557cc106e6d2aa8b8c5c7687ea9f8055ff3911 authored by Sebastian Andrzej Siewior on 28 June 2018, 06:26:05 UTC, committed by Linus Torvalds on 28 June 2018, 18:16:44 UTC
Revert commit c7f26ccfb2c3 ("mm/vmstat.c: fix vmstat_update() preemption BUG"). Steven saw a "using smp_processor_id() in preemptible" message and added a preempt_disable() section around it to keep it quiet. This is not the right thing to do it does not fix the real problem. vmstat_update() is invoked by a kworker on a specific CPU. This worker it bound to this CPU. The name of the worker was "kworker/1:1" so it should have been a worker which was bound to CPU1. A worker which can run on any CPU would have a `u' before the first digit. smp_processor_id() can be used in a preempt-enabled region as long as the task is bound to a single CPU which is the case here. If it could run on an arbitrary CPU then this is the problem we have an should seek to resolve. Not only this smp_processor_id() must not be migrated to another CPU but also refresh_cpu_vm_stats() which might access wrong per-CPU variables. Not to mention that other code relies on the fact that such a worker runs on one specific CPU only. Therefore revert that commit and we should look instead what broke the affinity mask of the kworker. Link: http://lkml.kernel.org/r/20180504104451.20278-1-bigeasy@linutronix.de Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de> Cc: Steven J. Hill <steven.hill@cavium.com> Cc: Tejun Heo <htejun@gmail.com> Cc: Vlastimil Babka <vbabka@suse.cz> Cc: Thomas Gleixner <tglx@linutronix.de> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
1 parent 4bb6e96
Tip revision: 28557cc106e6d2aa8b8c5c7687ea9f8055ff3911 authored by Sebastian Andrzej Siewior on 28 June 2018, 06:26:05 UTC
Revert mm/vmstat.c: fix vmstat_update() preemption BUG
Revert mm/vmstat.c: fix vmstat_update() preemption BUG
Tip revision: 28557cc
checkkconfigsymbols.py
#!/usr/bin/env python3
"""Find Kconfig symbols that are referenced but not defined."""
# (c) 2014-2017 Valentin Rothberg <valentinrothberg@gmail.com>
# (c) 2014 Stefan Hengelein <stefan.hengelein@fau.de>
#
# Licensed under the terms of the GNU GPL License version 2
import argparse
import difflib
import os
import re
import signal
import subprocess
import sys
from multiprocessing import Pool, cpu_count
# regex expressions
OPERATORS = r"&|\(|\)|\||\!"
SYMBOL = r"(?:\w*[A-Z0-9]\w*){2,}"
DEF = r"^\s*(?:menu){,1}config\s+(" + SYMBOL + r")\s*"
EXPR = r"(?:" + OPERATORS + r"|\s|" + SYMBOL + r")+"
DEFAULT = r"default\s+.*?(?:if\s.+){,1}"
STMT = r"^\s*(?:if|select|imply|depends\s+on|(?:" + DEFAULT + r"))\s+" + EXPR
SOURCE_SYMBOL = r"(?:\W|\b)+[D]{,1}CONFIG_(" + SYMBOL + r")"
# regex objects
REGEX_FILE_KCONFIG = re.compile(r".*Kconfig[\.\w+\-]*$")
REGEX_SYMBOL = re.compile(r'(?!\B)' + SYMBOL + r'(?!\B)')
REGEX_SOURCE_SYMBOL = re.compile(SOURCE_SYMBOL)
REGEX_KCONFIG_DEF = re.compile(DEF)
REGEX_KCONFIG_EXPR = re.compile(EXPR)
REGEX_KCONFIG_STMT = re.compile(STMT)
REGEX_KCONFIG_HELP = re.compile(r"^\s+(help|---help---)\s*$")
REGEX_FILTER_SYMBOLS = re.compile(r"[A-Za-z0-9]$")
REGEX_NUMERIC = re.compile(r"0[xX][0-9a-fA-F]+|[0-9]+")
REGEX_QUOTES = re.compile("(\"(.*?)\")")
def parse_options():
"""The user interface of this module."""
usage = "Run this tool to detect Kconfig symbols that are referenced but " \
"not defined in Kconfig. If no option is specified, " \
"checkkconfigsymbols defaults to check your current tree. " \
"Please note that specifying commits will 'git reset --hard\' " \
"your current tree! You may save uncommitted changes to avoid " \
"losing data."
parser = argparse.ArgumentParser(description=usage)
parser.add_argument('-c', '--commit', dest='commit', action='store',
default="",
help="check if the specified commit (hash) introduces "
"undefined Kconfig symbols")
parser.add_argument('-d', '--diff', dest='diff', action='store',
default="",
help="diff undefined symbols between two commits "
"(e.g., -d commmit1..commit2)")
parser.add_argument('-f', '--find', dest='find', action='store_true',
default=False,
help="find and show commits that may cause symbols to be "
"missing (required to run with --diff)")
parser.add_argument('-i', '--ignore', dest='ignore', action='store',
default="",
help="ignore files matching this Python regex "
"(e.g., -i '.*defconfig')")
parser.add_argument('-s', '--sim', dest='sim', action='store', default="",
help="print a list of max. 10 string-similar symbols")
parser.add_argument('--force', dest='force', action='store_true',
default=False,
help="reset current Git tree even when it's dirty")
parser.add_argument('--no-color', dest='color', action='store_false',
default=True,
help="don't print colored output (default when not "
"outputting to a terminal)")
args = parser.parse_args()
if args.commit and args.diff:
sys.exit("Please specify only one option at once.")
if args.diff and not re.match(r"^[\w\-\.\^]+\.\.[\w\-\.\^]+$", args.diff):
sys.exit("Please specify valid input in the following format: "
"\'commit1..commit2\'")
if args.commit or args.diff:
if not args.force and tree_is_dirty():
sys.exit("The current Git tree is dirty (see 'git status'). "
"Running this script may\ndelete important data since it "
"calls 'git reset --hard' for some performance\nreasons. "
" Please run this script in a clean Git tree or pass "
"'--force' if you\nwant to ignore this warning and "
"continue.")
if args.commit:
args.find = False
if args.ignore:
try:
re.match(args.ignore, "this/is/just/a/test.c")
except:
sys.exit("Please specify a valid Python regex.")
return args
def main():
"""Main function of this module."""
args = parse_options()
global COLOR
COLOR = args.color and sys.stdout.isatty()
if args.sim and not args.commit and not args.diff:
sims = find_sims(args.sim, args.ignore)
if sims:
print("%s: %s" % (yel("Similar symbols"), ', '.join(sims)))
else:
print("%s: no similar symbols found" % yel("Similar symbols"))
sys.exit(0)
# dictionary of (un)defined symbols
defined = {}
undefined = {}
if args.commit or args.diff:
head = get_head()
# get commit range
commit_a = None
commit_b = None
if args.commit:
commit_a = args.commit + "~"
commit_b = args.commit
elif args.diff:
split = args.diff.split("..")
commit_a = split[0]
commit_b = split[1]
undefined_a = {}
undefined_b = {}
# get undefined items before the commit
reset(commit_a)
undefined_a, _ = check_symbols(args.ignore)
# get undefined items for the commit
reset(commit_b)
undefined_b, defined = check_symbols(args.ignore)
# report cases that are present for the commit but not before
for symbol in sorted(undefined_b):
# symbol has not been undefined before
if symbol not in undefined_a:
files = sorted(undefined_b.get(symbol))
undefined[symbol] = files
# check if there are new files that reference the undefined symbol
else:
files = sorted(undefined_b.get(symbol) -
undefined_a.get(symbol))
if files:
undefined[symbol] = files
# reset to head
reset(head)
# default to check the entire tree
else:
undefined, defined = check_symbols(args.ignore)
# now print the output
for symbol in sorted(undefined):
print(red(symbol))
files = sorted(undefined.get(symbol))
print("%s: %s" % (yel("Referencing files"), ", ".join(files)))
sims = find_sims(symbol, args.ignore, defined)
sims_out = yel("Similar symbols")
if sims:
print("%s: %s" % (sims_out, ', '.join(sims)))
else:
print("%s: %s" % (sims_out, "no similar symbols found"))
if args.find:
print("%s:" % yel("Commits changing symbol"))
commits = find_commits(symbol, args.diff)
if commits:
for commit in commits:
commit = commit.split(" ", 1)
print("\t- %s (\"%s\")" % (yel(commit[0]), commit[1]))
else:
print("\t- no commit found")
print() # new line
def reset(commit):
"""Reset current git tree to %commit."""
execute(["git", "reset", "--hard", commit])
def yel(string):
"""
Color %string yellow.
"""
return "\033[33m%s\033[0m" % string if COLOR else string
def red(string):
"""
Color %string red.
"""
return "\033[31m%s\033[0m" % string if COLOR else string
def execute(cmd):
"""Execute %cmd and return stdout. Exit in case of error."""
try:
stdout = subprocess.check_output(cmd, stderr=subprocess.STDOUT, shell=False)
stdout = stdout.decode(errors='replace')
except subprocess.CalledProcessError as fail:
exit(fail)
return stdout
def find_commits(symbol, diff):
"""Find commits changing %symbol in the given range of %diff."""
commits = execute(["git", "log", "--pretty=oneline",
"--abbrev-commit", "-G",
symbol, diff])
return [x for x in commits.split("\n") if x]
def tree_is_dirty():
"""Return true if the current working tree is dirty (i.e., if any file has
been added, deleted, modified, renamed or copied but not committed)."""
stdout = execute(["git", "status", "--porcelain"])
for line in stdout:
if re.findall(r"[URMADC]{1}", line[:2]):
return True
return False
def get_head():
"""Return commit hash of current HEAD."""
stdout = execute(["git", "rev-parse", "HEAD"])
return stdout.strip('\n')
def partition(lst, size):
"""Partition list @lst into eveni-sized lists of size @size."""
return [lst[i::size] for i in range(size)]
def init_worker():
"""Set signal handler to ignore SIGINT."""
signal.signal(signal.SIGINT, signal.SIG_IGN)
def find_sims(symbol, ignore, defined=[]):
"""Return a list of max. ten Kconfig symbols that are string-similar to
@symbol."""
if defined:
return difflib.get_close_matches(symbol, set(defined), 10)
pool = Pool(cpu_count(), init_worker)
kfiles = []
for gitfile in get_files():
if REGEX_FILE_KCONFIG.match(gitfile):
kfiles.append(gitfile)
arglist = []
for part in partition(kfiles, cpu_count()):
arglist.append((part, ignore))
for res in pool.map(parse_kconfig_files, arglist):
defined.extend(res[0])
return difflib.get_close_matches(symbol, set(defined), 10)
def get_files():
"""Return a list of all files in the current git directory."""
# use 'git ls-files' to get the worklist
stdout = execute(["git", "ls-files"])
if len(stdout) > 0 and stdout[-1] == "\n":
stdout = stdout[:-1]
files = []
for gitfile in stdout.rsplit("\n"):
if ".git" in gitfile or "ChangeLog" in gitfile or \
".log" in gitfile or os.path.isdir(gitfile) or \
gitfile.startswith("tools/"):
continue
files.append(gitfile)
return files
def check_symbols(ignore):
"""Find undefined Kconfig symbols and return a dict with the symbol as key
and a list of referencing files as value. Files matching %ignore are not
checked for undefined symbols."""
pool = Pool(cpu_count(), init_worker)
try:
return check_symbols_helper(pool, ignore)
except KeyboardInterrupt:
pool.terminate()
pool.join()
sys.exit(1)
def check_symbols_helper(pool, ignore):
"""Helper method for check_symbols(). Used to catch keyboard interrupts in
check_symbols() in order to properly terminate running worker processes."""
source_files = []
kconfig_files = []
defined_symbols = []
referenced_symbols = dict() # {file: [symbols]}
for gitfile in get_files():
if REGEX_FILE_KCONFIG.match(gitfile):
kconfig_files.append(gitfile)
else:
if ignore and not re.match(ignore, gitfile):
continue
# add source files that do not match the ignore pattern
source_files.append(gitfile)
# parse source files
arglist = partition(source_files, cpu_count())
for res in pool.map(parse_source_files, arglist):
referenced_symbols.update(res)
# parse kconfig files
arglist = []
for part in partition(kconfig_files, cpu_count()):
arglist.append((part, ignore))
for res in pool.map(parse_kconfig_files, arglist):
defined_symbols.extend(res[0])
referenced_symbols.update(res[1])
defined_symbols = set(defined_symbols)
# inverse mapping of referenced_symbols to dict(symbol: [files])
inv_map = dict()
for _file, symbols in referenced_symbols.items():
for symbol in symbols:
inv_map[symbol] = inv_map.get(symbol, set())
inv_map[symbol].add(_file)
referenced_symbols = inv_map
undefined = {} # {symbol: [files]}
for symbol in sorted(referenced_symbols):
# filter some false positives
if symbol == "FOO" or symbol == "BAR" or \
symbol == "FOO_BAR" or symbol == "XXX":
continue
if symbol not in defined_symbols:
if symbol.endswith("_MODULE"):
# avoid false positives for kernel modules
if symbol[:-len("_MODULE")] in defined_symbols:
continue
undefined[symbol] = referenced_symbols.get(symbol)
return undefined, defined_symbols
def parse_source_files(source_files):
"""Parse each source file in @source_files and return dictionary with source
files as keys and lists of references Kconfig symbols as values."""
referenced_symbols = dict()
for sfile in source_files:
referenced_symbols[sfile] = parse_source_file(sfile)
return referenced_symbols
def parse_source_file(sfile):
"""Parse @sfile and return a list of referenced Kconfig symbols."""
lines = []
references = []
if not os.path.exists(sfile):
return references
with open(sfile, "r", encoding='utf-8', errors='replace') as stream:
lines = stream.readlines()
for line in lines:
if "CONFIG_" not in line:
continue
symbols = REGEX_SOURCE_SYMBOL.findall(line)
for symbol in symbols:
if not REGEX_FILTER_SYMBOLS.search(symbol):
continue
references.append(symbol)
return references
def get_symbols_in_line(line):
"""Return mentioned Kconfig symbols in @line."""
return REGEX_SYMBOL.findall(line)
def parse_kconfig_files(args):
"""Parse kconfig files and return tuple of defined and references Kconfig
symbols. Note, @args is a tuple of a list of files and the @ignore
pattern."""
kconfig_files = args[0]
ignore = args[1]
defined_symbols = []
referenced_symbols = dict()
for kfile in kconfig_files:
defined, references = parse_kconfig_file(kfile)
defined_symbols.extend(defined)
if ignore and re.match(ignore, kfile):
# do not collect references for files that match the ignore pattern
continue
referenced_symbols[kfile] = references
return (defined_symbols, referenced_symbols)
def parse_kconfig_file(kfile):
"""Parse @kfile and update symbol definitions and references."""
lines = []
defined = []
references = []
skip = False
if not os.path.exists(kfile):
return defined, references
with open(kfile, "r", encoding='utf-8', errors='replace') as stream:
lines = stream.readlines()
for i in range(len(lines)):
line = lines[i]
line = line.strip('\n')
line = line.split("#")[0] # ignore comments
if REGEX_KCONFIG_DEF.match(line):
symbol_def = REGEX_KCONFIG_DEF.findall(line)
defined.append(symbol_def[0])
skip = False
elif REGEX_KCONFIG_HELP.match(line):
skip = True
elif skip:
# ignore content of help messages
pass
elif REGEX_KCONFIG_STMT.match(line):
line = REGEX_QUOTES.sub("", line)
symbols = get_symbols_in_line(line)
# multi-line statements
while line.endswith("\\"):
i += 1
line = lines[i]
line = line.strip('\n')
symbols.extend(get_symbols_in_line(line))
for symbol in set(symbols):
if REGEX_NUMERIC.match(symbol):
# ignore numeric values
continue
references.append(symbol)
return defined, references
if __name__ == "__main__":
main()
Computing file changes ...