From 790ca8b0b5a343090b148d9c0c1ddb2d1a012952 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 10 Feb 2025 11:17:58 +0100 Subject: scripts/documentation-file-ref-check: don't check perl/python scripts Such scripts may have regular expressions, which would make the parser confusing. Also, they shouldn't hardcode filenames there, so skipping them is OK. While here, also don't check references on extensions used for file backup and patch rej/orig. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/712bfc8412ee5ad8ab43dd21a8c30fc858eff5a6.1739182025.git.mchehab+huawei@kernel.org --- scripts/documentation-file-ref-check | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/documentation-file-ref-check b/scripts/documentation-file-ref-check index 68083f2f1122..408b1dbe7884 100755 --- a/scripts/documentation-file-ref-check +++ b/scripts/documentation-file-ref-check @@ -92,7 +92,7 @@ while () { next if ($f =~ m,^Next/,); # Makefiles and scripts contain nasty expressions to parse docs - next if ($f =~ m/Makefile/ || $f =~ m/\.sh$/); + next if ($f =~ m/Makefile/ || $f =~ m/\.(sh|py|pl|~|rej|org|orig)$/); # It doesn't make sense to parse hidden files next if ($f =~ m#/\.#); -- cgit v1.2.3 From 484e9aa6efaf96a7a5b5fe3216f24973166fbfe3 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 10 Feb 2025 11:17:59 +0100 Subject: scripts/get_abi.py: add a Python tool to generate ReST output The get_abi.pl script is requiring some care, but it seems that the number of changes on it since when I originally wrote it was not too high. Maintaining perl scripts without using classes requires a higher efforted than on python, due to global variables management. Also, it sounds easier to find python developer those days than perl ones. As a plus, using a Python class to handle ABI allows a better integration with Sphinx extensions, allowing, for instance, to let automarkup to generate cross-references for ABI symbols. With that in mind, rewrite the core of get_abi.pl in Python, using classes, to help producing documentation. This will allow a better integration in the future with the Sphinx ABI extension. The algorithms used there are the same as the ones in Perl, with a couple of cleanups to remove redundant variables and to help with cross-reference generation. While doing that, remove some code that were important in the past, where ABI files weren't using ReST format. Some minor improvements were added like using a fixed seed when generating ABI keys for duplicated names, making its results reproductible. The end script is a little bit faster than the original one (tested on a machine with ssd disks). That's probably because we're now using only pre-compiled regular expressions, and it is using string replacement methods instead of regex where possible. The new version is a little bit more conservative when converting text to cross-references to avoid adding them into literal blocks. To ensure that the ReST output is parsing all variables and files properly, the end result was compared using diff with the one produced by the perl script and showed no regressions. There are minor improvements at the results, as it now properly groups What on some special cases. It also better escape some XREF names. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/71a894211a8b69664711144d9c4f8a0e73d1ae3c.1739182025.git.mchehab+huawei@kernel.org --- scripts/get_abi.py | 118 ++++++++++ scripts/lib/abi/abi_parser.py | 512 ++++++++++++++++++++++++++++++++++++++++++ scripts/lib/abi/helpers.py | 28 +++ 3 files changed, 658 insertions(+) create mode 100755 scripts/get_abi.py create mode 100644 scripts/lib/abi/abi_parser.py create mode 100644 scripts/lib/abi/helpers.py (limited to 'scripts') diff --git a/scripts/get_abi.py b/scripts/get_abi.py new file mode 100755 index 000000000000..bb17c54feeff --- /dev/null +++ b/scripts/get_abi.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +# pylint: disable=R0903 +# Copyright(c) 2025: Mauro Carvalho Chehab . +# SPDX-License-Identifier: GPL-2.0 + +""" +Parse ABI documentation and produce results from it. +""" + +import argparse +import logging +import os +import sys + +# Import Python modules + +LIB_DIR = "lib/abi" +SRC_DIR = os.path.dirname(os.path.realpath(__file__)) + +sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR)) + +from abi_parser import AbiParser # pylint: disable=C0413 +from helpers import ABI_DIR, DEBUG_HELP # pylint: disable=C0413 + +# Command line classes + + +REST_DESC = """ +Produce output in ReST format. + +The output is done on two sections: + +- Symbols: show all parsed symbols in alphabetic order; +- Files: cross reference the content of each file with the symbols on it. +""" + +class AbiRest: + """Initialize an argparse subparser for rest output""" + + def __init__(self, subparsers): + """Initialize argparse subparsers""" + + parser = subparsers.add_parser("rest", + formatter_class=argparse.RawTextHelpFormatter, + description=REST_DESC) + + parser.add_argument("--enable-lineno", action="store_true", + help="enable lineno") + parser.add_argument("--raw", action="store_true", + help="output text as contained in the ABI files. " + "It not used, output will contain dynamically" + " generated cross references when possible.") + parser.add_argument("--no-file", action="store_true", + help="Don't the files section") + parser.add_argument("--show-hints", help="Show-hints") + + parser.set_defaults(func=self.run) + + def run(self, args): + """Run subparser""" + + parser = AbiParser(args.dir, debug=args.debug) + parser.parse_abi() + parser.check_issues() + parser.print_data(args.enable_lineno, args.raw, not args.no_file) + + +class AbiValidate: + """Initialize an argparse subparser for ABI validation""" + + def __init__(self, subparsers): + """Initialize argparse subparsers""" + + parser = subparsers.add_parser("validate", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description="list events") + + parser.set_defaults(func=self.run) + + def run(self, args): + """Run subparser""" + + parser = AbiParser(args.dir, debug=args.debug) + parser.parse_abi() + parser.check_issues() + + +def main(): + """Main program""" + + parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) + + parser.add_argument("-d", "--debug", type=int, default=0, help="debug level") + parser.add_argument("-D", "--dir", default=ABI_DIR, help=DEBUG_HELP) + + subparsers = parser.add_subparsers() + + AbiRest(subparsers) + AbiValidate(subparsers) + + args = parser.parse_args() + + if args.debug: + level = logging.DEBUG + else: + level = logging.INFO + + logging.basicConfig(level=level, format="[%(levelname)s] %(message)s") + + if "func" in args: + args.func(args) + else: + sys.exit(f"Please specify a valid command for {sys.argv[0]}") + + +# Call main method +if __name__ == "__main__": + main() diff --git a/scripts/lib/abi/abi_parser.py b/scripts/lib/abi/abi_parser.py new file mode 100644 index 000000000000..b3fa70eee412 --- /dev/null +++ b/scripts/lib/abi/abi_parser.py @@ -0,0 +1,512 @@ +#!/usr/bin/env python3 +# pylint: disable=R0902,R0903,R0911,R0912,R0913,R0914,R0915,R0917,C0302 +# Copyright(c) 2025: Mauro Carvalho Chehab . +# SPDX-License-Identifier: GPL-2.0 + +""" +Parse ABI documentation and produce results from it. +""" + +from argparse import Namespace +import logging +import os +import re + +from glob import glob +from pprint import pformat +from random import randrange, seed + +# Import Python modules + +from helpers import AbiDebug, ABI_DIR + + +class AbiParser: + """Main class to parse ABI files""" + + TAGS = r"(what|where|date|kernelversion|contact|description|users)" + XREF = r"(?:^|\s|\()(\/(?:sys|config|proc|dev|kvd)\/[^,.:;\)\s]+)(?:[,.:;\)\s]|\Z)" + + def __init__(self, directory, logger=None, + enable_lineno=False, show_warnings=True, debug=0): + """Stores arguments for the class and initialize class vars""" + + self.directory = directory + self.enable_lineno = enable_lineno + self.show_warnings = show_warnings + self.debug = debug + + if not logger: + self.log = logging.getLogger("get_abi") + else: + self.log = logger + + self.data = {} + self.what_symbols = {} + self.file_refs = {} + self.what_refs = {} + + # Regular expressions used on parser + self.re_tag = re.compile(r"(\S+)(:\s*)(.*)", re.I) + self.re_valid = re.compile(self.TAGS) + self.re_start_spc = re.compile(r"(\s*)(\S.*)") + self.re_whitespace = re.compile(r"^\s+") + + # Regular used on print + self.re_what = re.compile(r"(\/?(?:[\w\-]+\/?){1,2})") + self.re_escape = re.compile(r"([\.\x01-\x08\x0e-\x1f\x21-\x2f\x3a-\x40\x7b-\xff])") + self.re_unprintable = re.compile(r"([\x00-\x2f\x3a-\x40\x5b-\x60\x7b-\xff]+)") + self.re_title_mark = re.compile(r"\n[\-\*\=\^\~]+\n") + self.re_doc = re.compile(r"Documentation/(?!devicetree)(\S+)\.rst") + self.re_abi = re.compile(r"(Documentation/ABI/)([\w\/\-]+)") + self.re_xref_node = re.compile(self.XREF) + + def warn(self, fdata, msg, extra=None): + """Displays a parse error if warning is enabled""" + + if not self.show_warnings: + return + + msg = f"{fdata.fname}:{fdata.ln}: {msg}" + if extra: + msg += "\n\t\t" + extra + + self.log.warning(msg) + + def add_symbol(self, what, fname, ln=None, xref=None): + """Create a reference table describing where each 'what' is located""" + + if what not in self.what_symbols: + self.what_symbols[what] = {"file": {}} + + if fname not in self.what_symbols[what]["file"]: + self.what_symbols[what]["file"][fname] = [] + + if ln and ln not in self.what_symbols[what]["file"][fname]: + self.what_symbols[what]["file"][fname].append(ln) + + if xref: + self.what_symbols[what]["xref"] = xref + + def _parse_line(self, fdata, line): + """Parse a single line of an ABI file""" + + new_what = False + new_tag = False + content = None + + match = self.re_tag.match(line) + if match: + new = match.group(1).lower() + sep = match.group(2) + content = match.group(3) + + match = self.re_valid.search(new) + if match: + new_tag = match.group(1) + else: + if fdata.tag == "description": + # New "tag" is actually part of description. + # Don't consider it a tag + new_tag = False + elif fdata.tag != "": + self.warn(fdata, f"tag '{fdata.tag}' is invalid", line) + + if new_tag: + # "where" is Invalid, but was a common mistake. Warn if found + if new_tag == "where": + self.warn(fdata, "tag 'Where' is invalid. Should be 'What:' instead") + new_tag = "what" + + if new_tag == "what": + fdata.space = None + + if content not in self.what_symbols: + self.add_symbol(what=content, fname=fdata.fname, ln=fdata.ln) + + if fdata.tag == "what": + fdata.what.append(content.strip("\n")) + else: + if fdata.key: + if "description" not in self.data.get(fdata.key, {}): + self.warn(fdata, f"{fdata.key} doesn't have a description") + + for w in fdata.what: + self.add_symbol(what=w, fname=fdata.fname, + ln=fdata.what_ln, xref=fdata.key) + + fdata.label = content + new_what = True + + key = "abi_" + content.lower() + fdata.key = self.re_unprintable.sub("_", key).strip("_") + + # Avoid duplicated keys but using a defined seed, to make + # the namespace identical if there aren't changes at the + # ABI symbols + seed(42) + + while fdata.key in self.data: + char = randrange(0, 51) + ord("A") + if char > ord("Z"): + char += ord("a") - ord("Z") - 1 + + fdata.key += chr(char) + + if fdata.key and fdata.key not in self.data: + self.data[fdata.key] = { + "what": [content], + "file": [fdata.file_ref], + "line_no": fdata.ln, + } + + fdata.what = self.data[fdata.key]["what"] + + self.what_refs[content] = fdata.key + fdata.tag = new_tag + fdata.what_ln = fdata.ln + + if fdata.nametag["what"]: + t = (content, fdata.key) + if t not in fdata.nametag["symbols"]: + fdata.nametag["symbols"].append(t) + + return + + if fdata.tag and new_tag: + fdata.tag = new_tag + + if new_what: + fdata.label = "" + + self.data[fdata.key]["type"] = fdata.ftype + + if "description" in self.data[fdata.key]: + self.data[fdata.key]["description"] += "\n\n" + + if fdata.file_ref not in self.data[fdata.key]["file"]: + self.data[fdata.key]["file"].append(fdata.file_ref) + + if self.debug == AbiDebug.WHAT_PARSING: + self.log.debug("what: %s", fdata.what) + + if not fdata.what: + self.warn(fdata, "'What:' should come first:", line) + return + + if new_tag == "description": + fdata.space = None + + if content: + sep = sep.replace(":", " ") + + c = " " * len(new_tag) + sep + content + c = c.expandtabs() + + match = self.re_start_spc.match(c) + if match: + # Preserve initial spaces for the first line + fdata.space = match.group(1) + content = match.group(2) + "\n" + + self.data[fdata.key][fdata.tag] = content + + return + + # Store any contents before tags at the database + if not fdata.tag and "what" in fdata.nametag: + fdata.nametag["description"] += line + return + + if fdata.tag == "description": + content = line.expandtabs() + + if self.re_whitespace.sub("", content) == "": + self.data[fdata.key][fdata.tag] += "\n" + return + + if fdata.space is None: + match = self.re_start_spc.match(content) + if match: + # Preserve initial spaces for the first line + fdata.space = match.group(1) + + content = match.group(2) + "\n" + else: + if content.startswith(fdata.space): + content = content[len(fdata.space):] + + else: + fdata.space = "" + + if fdata.tag == "what": + w = content.strip("\n") + if w: + self.data[fdata.key][fdata.tag].append(w) + else: + self.data[fdata.key][fdata.tag] += content + return + + content = line.strip() + if fdata.tag: + if fdata.tag == "what": + w = content.strip("\n") + if w: + self.data[fdata.key][fdata.tag].append(w) + else: + self.data[fdata.key][fdata.tag] += "\n" + content.rstrip("\n") + return + + # Everything else is error + if content: + self.warn(fdata, "Unexpected content", line) + + def parse_file(self, fname, path, basename): + """Parse a single file""" + + ref = f"abi_file_{path}_{basename}" + ref = self.re_unprintable.sub("_", ref).strip("_") + + # Store per-file state into a namespace variable. This will be used + # by the per-line parser state machine and by the warning function. + fdata = Namespace + + fdata.fname = fname + fdata.name = basename + + pos = fname.find(ABI_DIR) + if pos > 0: + f = fname[pos:] + else: + f = fname + + fdata.file_ref = (f, ref) + self.file_refs[f] = ref + + fdata.ln = 0 + fdata.what_ln = 0 + fdata.tag = "" + fdata.label = "" + fdata.what = [] + fdata.key = None + fdata.xrefs = None + fdata.space = None + fdata.ftype = path.split("/")[0] + + fdata.nametag = {} + fdata.nametag["what"] = [f"File {path}/{basename}"] + fdata.nametag["type"] = "File" + fdata.nametag["file"] = [fdata.file_ref] + fdata.nametag["line_no"] = 1 + fdata.nametag["description"] = "" + fdata.nametag["symbols"] = [] + + self.data[ref] = fdata.nametag + + if self.debug & AbiDebug.WHAT_OPEN: + self.log.debug("Opening file %s", fname) + + with open(fname, "r", encoding="utf8", errors="backslashreplace") as fp: + for line in fp: + fdata.ln += 1 + + self._parse_line(fdata, line) + + if "description" in fdata.nametag: + fdata.nametag["description"] = fdata.nametag["description"].lstrip("\n") + + if fdata.key: + if "description" not in self.data.get(fdata.key, {}): + self.warn(fdata, f"{fdata.key} doesn't have a description") + + for w in fdata.what: + self.add_symbol(what=w, fname=fname, xref=fdata.key) + + def parse_abi(self): + """Parse documentation ABI""" + + ignore_suffixes = ("rej", "org", "orig", "bak", "~") + re_abi = re.compile(r".*" + ABI_DIR) + + for fname in glob(os.path.join(self.directory, "**"), recursive=True): + if os.path.isdir(fname): + continue + + basename = os.path.basename(fname) + + if basename == "README": + continue + if basename.startswith(".") or basename.endswith(ignore_suffixes): + continue + + path = re_abi.sub("", os.path.dirname(fname)) + + self.parse_file(fname, path, basename) + + if self.debug & AbiDebug.DUMP_ABI_STRUCTS: + self.log.debug(pformat(self.data)) + + def print_desc_txt(self, desc): + """Print description as found inside ABI files""" + + desc = desc.strip(" \t\n") + + print(desc + "\n") + + def print_desc_rst(self, desc): + """Enrich ReST output by creating cross-references""" + + # Remove title markups from the description + # Having titles inside ABI files will only work if extra + # care would be taken in order to strictly follow the same + # level order for each markup. + desc = self.re_title_mark.sub("\n\n", "\n" + desc) + desc = desc.rstrip(" \t\n").lstrip("\n") + + # Python's regex performance for non-compiled expressions is a lot + # than Perl, as Perl automatically caches them at their + # first usage. Here, we'll need to do the same, as otherwise the + # performance penalty is be high + + new_desc = "" + for d in desc.split("\n"): + if d == "": + new_desc += "\n" + continue + + # Use cross-references for doc files where needed + d = self.re_doc.sub(r":doc:`/\1`", d) + + # Use cross-references for ABI generated docs where needed + matches = self.re_abi.findall(d) + for m in matches: + abi = m[0] + m[1] + + xref = self.file_refs.get(abi) + if not xref: + # This may happen if ABI is on a separate directory, + # like parsing ABI testing and symbol is at stable. + # The proper solution is to move this part of the code + # for it to be inside sphinx/kernel_abi.py + self.log.info("Didn't find ABI reference for '%s'", abi) + else: + new = self.re_escape.sub(r"\\\1", m[1]) + d = re.sub(fr"\b{abi}\b", f":ref:`{new} <{xref}>`", d) + + # Seek for cross reference symbols like /sys/... + # Need to be careful to avoid doing it on a code block + if d[0] not in [" ", "\t"]: + matches = self.re_xref_node.findall(d) + for m in matches: + # Finding ABI here is more complex due to wildcards + xref = self.what_refs.get(m) + if xref: + new = self.re_escape.sub(r"\\\1", m) + d = re.sub(fr"\b{m}\b", f":ref:`{new} <{xref}>`", d) + + new_desc += d + "\n" + + print(new_desc + "\n") + + def print_data(self, enable_lineno, output_in_txt, show_file=False): + """Print ABI at stdout""" + + part = None + for key, v in sorted(self.data.items(), + key=lambda x: (x[1].get("type", ""), + x[1].get("what"))): + + wtype = v.get("type", "Var") + file_ref = v.get("file") + names = v.get("what", [""]) + + if not show_file and wtype == "File": + continue + + if enable_lineno: + ln = v.get("line_no", 1) + print(f".. LINENO {file_ref[0][0]}#{ln}\n") + + if wtype != "File": + cur_part = names[0] + if cur_part.find("/") >= 0: + match = self.re_what.match(cur_part) + if match: + symbol = match.group(1).rstrip("/") + cur_part = "Symbols under " + symbol + + if cur_part and cur_part != part: + part = cur_part + print(f"{part}\n{"-" * len(part)}\n") + + print(f".. _{key}:\n") + + max_len = 0 + for i in range(0, len(names)): # pylint: disable=C0200 + names[i] = "**" + self.re_escape.sub(r"\\\1", names[i]) + "**" + + max_len = max(max_len, len(names[i])) + + print("+-" + "-" * max_len + "-+") + for name in names: + print(f"| {name}" + " " * (max_len - len(name)) + " |") + print("+-" + "-" * max_len + "-+") + print() + + for ref in file_ref: + if wtype == "File": + print(f".. _{ref[1]}:\n") + else: + base = os.path.basename(ref[0]) + print(f"Defined on file :ref:`{base} <{ref[1]}>`\n") + + if wtype == "File": + print(f"{names[0]}\n{"-" * len(names[0])}\n") + + desc = v.get("description") + if not desc and wtype != "File": + print(f"DESCRIPTION MISSING for {names[0]}\n") + + if desc: + if output_in_txt: + self.print_desc_txt(desc) + else: + self.print_desc_rst(desc) + + symbols = v.get("symbols") + if symbols: + print("Has the following ABI:\n") + + for w, label in symbols: + # Escape special chars from content + content = self.re_escape.sub(r"\\\1", w) + + print(f"- :ref:`{content} <{label}>`\n") + + users = v.get("users") + if users and users.strip(" \t\n"): + print(f"Users:\n\t{users.strip("\n").replace('\n', '\n\t')}\n") + + def check_issues(self): + """Warn about duplicated ABI entries""" + + for what, v in self.what_symbols.items(): + files = v.get("file") + if not files: + # Should never happen if the parser works properly + self.log.warning("%s doesn't have a file associated", what) + continue + + if len(files) == 1: + continue + + f = [] + for fname, lines in sorted(files.items()): + if not lines: + f.append(f"{fname}") + elif len(lines) == 1: + f.append(f"{fname}:{lines[0]}") + else: + f.append(f"{fname} lines {", ".join(str(x) for x in lines)}") + + self.log.warning("%s is defined %d times: %s", what, len(f), "; ".join(f)) diff --git a/scripts/lib/abi/helpers.py b/scripts/lib/abi/helpers.py new file mode 100644 index 000000000000..84a253ed5058 --- /dev/null +++ b/scripts/lib/abi/helpers.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 +# Copyright(c) 2025: Mauro Carvalho Chehab . +# pylint: disable=R0903 +# SPDX-License-Identifier: GPL-2.0 + +""" +Helper classes for ABI parser +""" + +ABI_DIR = "Documentation/ABI/" + + +class AbiDebug: + """Debug levels""" + + WHAT_PARSING = 1 + WHAT_OPEN = 2 + DUMP_ABI_STRUCTS = 4 + + +DEBUG_HELP = """ +Print debug information according with the level(s), +which is given by the following bitmask: + +1 - enable debug parsing logic +2 - enable debug messages on file open +4 - enable debug for ABI parse data +""" -- cgit v1.2.3 From 6b48bea16848dd7c771411db3dcc01b3bc4dd4c2 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 10 Feb 2025 11:18:00 +0100 Subject: scripts/get_abi.py: add support for symbol search Add support for searching symbols from Documentation/ABI using regular expressions to match the symbols' names. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/21b2c48657dde112d5417dcd7e0aa7cd383b9a0a.1739182025.git.mchehab+huawei@kernel.org --- scripts/get_abi.py | 24 ++++++++++++++++++++ scripts/lib/abi/abi_parser.py | 52 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) (limited to 'scripts') diff --git a/scripts/get_abi.py b/scripts/get_abi.py index bb17c54feeff..30439f21fdd0 100755 --- a/scripts/get_abi.py +++ b/scripts/get_abi.py @@ -85,6 +85,29 @@ class AbiValidate: parser.check_issues() +class AbiSearch: + """Initialize an argparse subparser for ABI search""" + + def __init__(self, subparsers): + """Initialize argparse subparsers""" + + parser = subparsers.add_parser("search", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + description="Search ABI using a regular expression") + + parser.add_argument("expression", + help="Case-insensitive search pattern for the ABI symbol") + + parser.set_defaults(func=self.run) + + def run(self, args): + """Run subparser""" + + parser = AbiParser(args.dir, debug=args.debug) + parser.parse_abi() + parser.search_symbols(args.expression) + + def main(): """Main program""" @@ -97,6 +120,7 @@ def main(): AbiRest(subparsers) AbiValidate(subparsers) + AbiSearch(subparsers) args = parser.parse_args() diff --git a/scripts/lib/abi/abi_parser.py b/scripts/lib/abi/abi_parser.py index b3fa70eee412..bea7f1a76165 100644 --- a/scripts/lib/abi/abi_parser.py +++ b/scripts/lib/abi/abi_parser.py @@ -510,3 +510,55 @@ class AbiParser: f.append(f"{fname} lines {", ".join(str(x) for x in lines)}") self.log.warning("%s is defined %d times: %s", what, len(f), "; ".join(f)) + + def search_symbols(self, expr): + """ Searches for ABI symbols """ + + regex = re.compile(expr, re.I) + + found_keys = 0 + for t in sorted(self.data.items(), key=lambda x: [0]): + v = t[1] + + wtype = v.get("type", "") + if wtype == "File": + continue + + for what in v.get("what", [""]): + if regex.search(what): + found_keys += 1 + + kernelversion = v.get("kernelversion", "").strip(" \t\n") + date = v.get("date", "").strip(" \t\n") + contact = v.get("contact", "").strip(" \t\n") + users = v.get("users", "").strip(" \t\n") + desc = v.get("description", "").strip(" \t\n") + + files = [] + for f in v.get("file", ()): + files.append(f[0]) + + what = str(found_keys) + ". " + what + title_tag = "-" * len(what) + + print(f"\n{what}\n{title_tag}\n") + + if kernelversion: + print(f"Kernel version:\t\t{kernelversion}") + + if date: + print(f"Date:\t\t\t{date}") + + if contact: + print(f"Contact:\t\t{contact}") + + if users: + print(f"Users:\t\t\t{users}") + + print(f"Defined on file{'s'[:len(files) ^ 1]}:\t{", ".join(files)}") + + if desc: + print(f"\n{desc.strip("\n")}\n") + + if not found_keys: + print(f"Regular expression /{expr}/ not found.") -- cgit v1.2.3 From c67c3fbdd917884e38a366c38717c9f769075c15 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 10 Feb 2025 11:18:02 +0100 Subject: scripts/lib/abi/abi_parser.py: optimize parse_abi() function Instead of using glob, use a recursive function to parse all files. Such change reduces the total excecution time by 15% with my SSD disks. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/190dd358897017ed82c56f1e263192215ffbae43.1739182025.git.mchehab+huawei@kernel.org --- scripts/lib/abi/abi_parser.py | 49 ++++++++++++++++++++++++++++++------------- 1 file changed, 34 insertions(+), 15 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/abi/abi_parser.py b/scripts/lib/abi/abi_parser.py index bea7f1a76165..6052a8aec443 100644 --- a/scripts/lib/abi/abi_parser.py +++ b/scripts/lib/abi/abi_parser.py @@ -12,7 +12,6 @@ import logging import os import re -from glob import glob from pprint import pformat from random import randrange, seed @@ -46,7 +45,11 @@ class AbiParser: self.file_refs = {} self.what_refs = {} + # Ignore files that contain such suffixes + self.ignore_suffixes = (".rej", ".org", ".orig", ".bak", "~") + # Regular expressions used on parser + self.re_abi_dir = re.compile(r"(.*)" + ABI_DIR) self.re_tag = re.compile(r"(\S+)(:\s*)(.*)", re.I) self.re_valid = re.compile(self.TAGS) self.re_start_spc = re.compile(r"(\s*)(\S.*)") @@ -322,26 +325,42 @@ class AbiParser: for w in fdata.what: self.add_symbol(what=w, fname=fname, xref=fdata.key) - def parse_abi(self): - """Parse documentation ABI""" + def _parse_abi(self, root=None): + """Internal function to parse documentation ABI recursively""" - ignore_suffixes = ("rej", "org", "orig", "bak", "~") - re_abi = re.compile(r".*" + ABI_DIR) + if not root: + root = self.directory - for fname in glob(os.path.join(self.directory, "**"), recursive=True): - if os.path.isdir(fname): - continue + with os.scandir(root) as obj: + for entry in obj: + name = os.path.join(root, entry.name) - basename = os.path.basename(fname) + if entry.is_dir(): + self.parse_abi(name) + continue - if basename == "README": - continue - if basename.startswith(".") or basename.endswith(ignore_suffixes): - continue + if not entry.is_file(): + continue + + basename = os.path.basename(name) - path = re_abi.sub("", os.path.dirname(fname)) + if basename == "README": + continue + + if basename.startswith("."): + continue + + if basename.endswith(self.ignore_suffixes): + continue + + path = self.re_abi_dir.sub("", os.path.dirname(name)) + + self.parse_file(name, path, basename) + + def parse_abi(self, root=None): + """Parse documentation ABI""" - self.parse_file(fname, path, basename) + self._parse_abi(root) if self.debug & AbiDebug.DUMP_ABI_STRUCTS: self.log.debug(pformat(self.data)) -- cgit v1.2.3 From 9bec7870c64c00983773cfddab8d6a037f7767f3 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 10 Feb 2025 11:18:03 +0100 Subject: scripts/lib/abi/abi_parser.py: use an interactor for ReST output Instead of printing all results line per line, use an interactor to return each variable as a separate message. This won't change much when using it via command line, but it will help Sphinx integration by providing an interactor that could be used there to handle ABI symbol by symbol. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/e3c94b8cdfd5e955aa19a703921f364a89089634.1739182025.git.mchehab+huawei@kernel.org --- scripts/get_abi.py | 3 ++- scripts/lib/abi/abi_parser.py | 48 +++++++++++++++++++++++-------------------- 2 files changed, 28 insertions(+), 23 deletions(-) (limited to 'scripts') diff --git a/scripts/get_abi.py b/scripts/get_abi.py index 30439f21fdd0..93b973bc07ed 100755 --- a/scripts/get_abi.py +++ b/scripts/get_abi.py @@ -62,8 +62,9 @@ class AbiRest: parser = AbiParser(args.dir, debug=args.debug) parser.parse_abi() parser.check_issues() - parser.print_data(args.enable_lineno, args.raw, not args.no_file) + for msg in parser.doc(args.enable_lineno, args.raw, not args.no_file): + print(msg) class AbiValidate: """Initialize an argparse subparser for ABI validation""" diff --git a/scripts/lib/abi/abi_parser.py b/scripts/lib/abi/abi_parser.py index 6052a8aec443..960e27161c26 100644 --- a/scripts/lib/abi/abi_parser.py +++ b/scripts/lib/abi/abi_parser.py @@ -336,7 +336,7 @@ class AbiParser: name = os.path.join(root, entry.name) if entry.is_dir(): - self.parse_abi(name) + self._parse_abi(name) continue if not entry.is_file(): @@ -365,14 +365,14 @@ class AbiParser: if self.debug & AbiDebug.DUMP_ABI_STRUCTS: self.log.debug(pformat(self.data)) - def print_desc_txt(self, desc): + def desc_txt(self, desc): """Print description as found inside ABI files""" desc = desc.strip(" \t\n") - print(desc + "\n") + return desc + "\n\n" - def print_desc_rst(self, desc): + def desc_rst(self, desc): """Enrich ReST output by creating cross-references""" # Remove title markups from the description @@ -425,9 +425,9 @@ class AbiParser: new_desc += d + "\n" - print(new_desc + "\n") + return new_desc + "\n\n" - def print_data(self, enable_lineno, output_in_txt, show_file=False): + def doc(self, enable_lineno, output_in_txt, show_file=False): """Print ABI at stdout""" part = None @@ -442,9 +442,11 @@ class AbiParser: if not show_file and wtype == "File": continue + msg = "" + if enable_lineno: ln = v.get("line_no", 1) - print(f".. LINENO {file_ref[0][0]}#{ln}\n") + msg += f".. LINENO {file_ref[0][0]}#{ln}\n\n" if wtype != "File": cur_part = names[0] @@ -456,9 +458,9 @@ class AbiParser: if cur_part and cur_part != part: part = cur_part - print(f"{part}\n{"-" * len(part)}\n") + msg += f"{part}\n{"-" * len(part)}\n\n" - print(f".. _{key}:\n") + msg += f".. _{key}:\n\n" max_len = 0 for i in range(0, len(names)): # pylint: disable=C0200 @@ -466,45 +468,47 @@ class AbiParser: max_len = max(max_len, len(names[i])) - print("+-" + "-" * max_len + "-+") + msg += "+-" + "-" * max_len + "-+\n" for name in names: - print(f"| {name}" + " " * (max_len - len(name)) + " |") - print("+-" + "-" * max_len + "-+") - print() + msg += f"| {name}" + " " * (max_len - len(name)) + " |\n" + msg += "+-" + "-" * max_len + "-+\n" + msg += "\n" for ref in file_ref: if wtype == "File": - print(f".. _{ref[1]}:\n") + msg += f".. _{ref[1]}:\n\n" else: base = os.path.basename(ref[0]) - print(f"Defined on file :ref:`{base} <{ref[1]}>`\n") + msg += f"Defined on file :ref:`{base} <{ref[1]}>`\n\n" if wtype == "File": - print(f"{names[0]}\n{"-" * len(names[0])}\n") + msg += f"{names[0]}\n{"-" * len(names[0])}\n\n" desc = v.get("description") if not desc and wtype != "File": - print(f"DESCRIPTION MISSING for {names[0]}\n") + msg += f"DESCRIPTION MISSING for {names[0]}\n\n" if desc: if output_in_txt: - self.print_desc_txt(desc) + msg += self.desc_txt(desc) else: - self.print_desc_rst(desc) + msg += self.desc_rst(desc) symbols = v.get("symbols") if symbols: - print("Has the following ABI:\n") + msg += "Has the following ABI:\n\n" for w, label in symbols: # Escape special chars from content content = self.re_escape.sub(r"\\\1", w) - print(f"- :ref:`{content} <{label}>`\n") + msg += f"- :ref:`{content} <{label}>`\n\n" users = v.get("users") if users and users.strip(" \t\n"): - print(f"Users:\n\t{users.strip("\n").replace('\n', '\n\t')}\n") + msg += f"Users:\n\t{users.strip("\n").replace('\n', '\n\t')}\n\n" + + yield msg def check_issues(self): """Warn about duplicated ABI entries""" -- cgit v1.2.3 From ee34f8300c8940758dc69f80107d9f5873c08f17 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 10 Feb 2025 11:18:04 +0100 Subject: docs: sphinx/kernel_abi: use AbiParser directly Instead of running get_abi.py script, import AbiParser class and handle messages directly there using an interactor. This shold save some memory, as there's no need to exec python inside the Sphinx python extension. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/8dbc244dcda97112c1b694e2512a5d600e62873b.1739182025.git.mchehab+huawei@kernel.org --- Documentation/sphinx/kernel_abi.py | 29 ++++++++++++++++------------- scripts/lib/abi/abi_parser.py | 2 +- 2 files changed, 17 insertions(+), 14 deletions(-) (limited to 'scripts') diff --git a/Documentation/sphinx/kernel_abi.py b/Documentation/sphinx/kernel_abi.py index f314b888d3de..f7b22abebcf4 100644 --- a/Documentation/sphinx/kernel_abi.py +++ b/Documentation/sphinx/kernel_abi.py @@ -34,7 +34,6 @@ u""" import os import re -import subprocess import sys from docutils import nodes @@ -43,6 +42,11 @@ from docutils.parsers.rst import directives, Directive from sphinx.util.docutils import switch_source_input from sphinx.util import logging +srctree = os.path.abspath(os.environ["srctree"]) +sys.path.insert(0, os.path.join(srctree, "scripts/lib/abi")) + +from abi_parser import AbiParser + __version__ = "1.0" @@ -66,7 +70,7 @@ class KernelCmd(Directive): logger = logging.getLogger('kernel_abi') option_spec = { - "debug" : directives.flag, + "debug": directives.flag, } def run(self): @@ -74,20 +78,19 @@ class KernelCmd(Directive): if not doc.settings.file_insertion_enabled: raise self.warning("docutils: file insertion disabled") - srctree = os.path.abspath(os.environ["srctree"]) + path = os.path.join(srctree, "Documentation", self.arguments[0]) + parser = AbiParser(path, logger=self.logger) + parser.parse_abi() + parser.check_issues() - args = [ - os.path.join(srctree, 'scripts/get_abi.py'), - '-D', os.path.join(srctree, 'Documentation', self.arguments[0]), - 'rest', - '--enable-lineno', - ] + msg = "" + for m in parser.doc(enable_lineno=True, show_file=True): + msg += m - lines = subprocess.check_output(args, cwd=os.path.dirname(doc.current_source)).decode('utf-8') - nodeList = self.nestedParse(lines, self.arguments[0]) - return nodeList + node = self.nested_parse(msg, self.arguments[0]) + return node - def nestedParse(self, lines, fname): + def nested_parse(self, lines, fname): env = self.state.document.settings.env content = ViewList() node = nodes.section() diff --git a/scripts/lib/abi/abi_parser.py b/scripts/lib/abi/abi_parser.py index 960e27161c26..57c125fd40a5 100644 --- a/scripts/lib/abi/abi_parser.py +++ b/scripts/lib/abi/abi_parser.py @@ -427,7 +427,7 @@ class AbiParser: return new_desc + "\n\n" - def doc(self, enable_lineno, output_in_txt, show_file=False): + def doc(self, enable_lineno, output_in_txt=False, show_file=False): """Print ABI at stdout""" part = None -- cgit v1.2.3 From aea5e52dce74f679b91c66caad91d587d5504f6c Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 10 Feb 2025 11:18:05 +0100 Subject: docs: sphinx/kernel_abi: reduce buffer usage for ABI messages Instead of producing a big message with all ABI contents and then parse as a whole, simplify the code by handling each ABI symbol in separate. As an additional benefit, there's no need to place file/line nubers inlined at the data and use a regex to convert them. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/15be22955e3c6df49d7256c8fd24f62b397ad0ff.1739182025.git.mchehab+huawei@kernel.org --- Documentation/sphinx/kernel_abi.py | 82 +++++++++++++++++++------------------- scripts/get_abi.py | 7 +++- scripts/lib/abi/abi_parser.py | 10 ++--- 3 files changed, 51 insertions(+), 48 deletions(-) (limited to 'scripts') diff --git a/Documentation/sphinx/kernel_abi.py b/Documentation/sphinx/kernel_abi.py index f7b22abebcf4..742ebd35454f 100644 --- a/Documentation/sphinx/kernel_abi.py +++ b/Documentation/sphinx/kernel_abi.py @@ -68,6 +68,7 @@ class KernelCmd(Directive): has_content = False final_argument_whitespace = True logger = logging.getLogger('kernel_abi') + parser = None option_spec = { "debug": directives.flag, @@ -79,59 +80,60 @@ class KernelCmd(Directive): raise self.warning("docutils: file insertion disabled") path = os.path.join(srctree, "Documentation", self.arguments[0]) - parser = AbiParser(path, logger=self.logger) - parser.parse_abi() - parser.check_issues() + self.parser = AbiParser(path, logger=self.logger) + self.parser.parse_abi() + self.parser.check_issues() - msg = "" - for m in parser.doc(enable_lineno=True, show_file=True): - msg += m - - node = self.nested_parse(msg, self.arguments[0]) + node = self.nested_parse(None, self.arguments[0]) return node - def nested_parse(self, lines, fname): + def nested_parse(self, data, fname): env = self.state.document.settings.env content = ViewList() node = nodes.section() - if "debug" in self.options: - code_block = "\n\n.. code-block:: rst\n :linenos:\n" - for line in lines.split("\n"): - code_block += "\n " + line - lines = code_block + "\n\n" - - line_regex = re.compile(r"^\.\. LINENO (\S+)\#([0-9]+)$") - ln = 0 - n = 0 - f = fname - - for line in lines.split("\n"): - n = n + 1 - match = line_regex.search(line) - if match: - new_f = match.group(1) - - # Sphinx parser is lazy: it stops parsing contents in the - # middle, if it is too big. So, handle it per input file - if new_f != f and content: - self.do_parse(content, node) - content = ViewList() + if data is not None: + # Handles the .rst file + for line in data.split("\n"): + content.append(line, fname, 0) + + self.do_parse(content, node) + else: + # Handles the ABI parser content, symbol by symbol + + old_f = fname + n = 0 + for msg, f, ln in self.parser.doc(): + msg_list = msg.split("\n") + if "debug" in self.options: + lines = [ + "", "", ".. code-block:: rst", + " :linenos:", "" + ] + for m in msg_list: + lines.append(" " + m) + else: + lines = msg_list + + for line in lines: + # sphinx counts lines from 0 + content.append(line, f, ln - 1) + n += 1 + + if f != old_f: # Add the file to Sphinx build dependencies env.note_dependency(os.path.abspath(f)) - f = new_f + old_f = f - # sphinx counts lines from 0 - ln = int(match.group(2)) - 1 - else: - content.append(line, f, ln) - - self.logger.info("%s: parsed %i lines" % (fname, n)) + # Sphinx doesn't like to parse big messages. So, let's + # add content symbol by symbol + if content: + self.do_parse(content, node) + content = ViewList() - if content: - self.do_parse(content, node) + self.logger.info("%s: parsed %i lines" % (fname, n)) return node.children diff --git a/scripts/get_abi.py b/scripts/get_abi.py index 93b973bc07ed..19f78d6aa407 100755 --- a/scripts/get_abi.py +++ b/scripts/get_abi.py @@ -63,8 +63,11 @@ class AbiRest: parser.parse_abi() parser.check_issues() - for msg in parser.doc(args.enable_lineno, args.raw, not args.no_file): - print(msg) + for t in parser.doc(args.raw, not args.no_file): + if args.enable_lineno: + print (f".. LINENO {t[1]}#{t[2]}\n\n") + + print(t[0]) class AbiValidate: """Initialize an argparse subparser for ABI validation""" diff --git a/scripts/lib/abi/abi_parser.py b/scripts/lib/abi/abi_parser.py index 57c125fd40a5..1db6c54fc65a 100644 --- a/scripts/lib/abi/abi_parser.py +++ b/scripts/lib/abi/abi_parser.py @@ -427,7 +427,7 @@ class AbiParser: return new_desc + "\n\n" - def doc(self, enable_lineno, output_in_txt=False, show_file=False): + def doc(self, output_in_txt=False, show_file=True): """Print ABI at stdout""" part = None @@ -444,10 +444,6 @@ class AbiParser: msg = "" - if enable_lineno: - ln = v.get("line_no", 1) - msg += f".. LINENO {file_ref[0][0]}#{ln}\n\n" - if wtype != "File": cur_part = names[0] if cur_part.find("/") >= 0: @@ -508,7 +504,9 @@ class AbiParser: if users and users.strip(" \t\n"): msg += f"Users:\n\t{users.strip("\n").replace('\n', '\n\t')}\n\n" - yield msg + ln = v.get("line_no", 1) + + yield (msg, file_ref[0][0], ln) def check_issues(self): """Warn about duplicated ABI entries""" -- cgit v1.2.3 From 2a21d80dfb4135b4766d8ff3231a3ea1c19bcc83 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 10 Feb 2025 11:18:07 +0100 Subject: scripts/get_abi.pl: Add filtering capabilities to rest output This way, Sphinx ABI extension can parse symbols only once, while keep displaying results in separate files. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/41e108e816e46434aa596e5c0d25d227cb9f0fe5.1739182025.git.mchehab+huawei@kernel.org --- scripts/lib/abi/abi_parser.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/abi/abi_parser.py b/scripts/lib/abi/abi_parser.py index 1db6c54fc65a..b20d5c9d920e 100644 --- a/scripts/lib/abi/abi_parser.py +++ b/scripts/lib/abi/abi_parser.py @@ -160,6 +160,7 @@ class AbiParser: self.data[fdata.key] = { "what": [content], "file": [fdata.file_ref], + "path": fdata.ftype, "line_no": fdata.ln, } @@ -182,8 +183,6 @@ class AbiParser: if new_what: fdata.label = "" - self.data[fdata.key]["type"] = fdata.ftype - if "description" in self.data[fdata.key]: self.data[fdata.key]["description"] += "\n\n" @@ -299,6 +298,7 @@ class AbiParser: fdata.nametag = {} fdata.nametag["what"] = [f"File {path}/{basename}"] fdata.nametag["type"] = "File" + fdata.nametag["path"] = fdata.ftype fdata.nametag["file"] = [fdata.file_ref] fdata.nametag["line_no"] = 1 fdata.nametag["description"] = "" @@ -427,7 +427,8 @@ class AbiParser: return new_desc + "\n\n" - def doc(self, output_in_txt=False, show_file=True): + def doc(self, output_in_txt=False, show_symbols=True, show_file=True, + filter_path=None): """Print ABI at stdout""" part = None @@ -435,12 +436,20 @@ class AbiParser: key=lambda x: (x[1].get("type", ""), x[1].get("what"))): - wtype = v.get("type", "Var") + wtype = v.get("type", "Symbol") file_ref = v.get("file") names = v.get("what", [""]) - if not show_file and wtype == "File": - continue + if wtype == "File": + if not show_file: + continue + else: + if not show_symbols: + continue + + if filter_path: + if v.get("path") != filter_path: + continue msg = "" -- cgit v1.2.3 From 98a4324a8b7bbe433483c90524026be0ccc9ffa8 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 10 Feb 2025 11:18:08 +0100 Subject: scripts/get_abi.pl: add support to parse ABI README file The Documentation/ABI/README file is currently outside the documentation tree. Yet, it may still provide some useful information. Add it to the documentation parsing. As a plus, this avoids a warning when detecting missing cross-references. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/f1285dedfe4d0eb0f0af34f6a68bee6fde36dd7d.1739182025.git.mchehab+huawei@kernel.org --- scripts/lib/abi/abi_parser.py | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/abi/abi_parser.py b/scripts/lib/abi/abi_parser.py index b20d5c9d920e..6fac461d794c 100644 --- a/scripts/lib/abi/abi_parser.py +++ b/scripts/lib/abi/abi_parser.py @@ -263,6 +263,16 @@ class AbiParser: if content: self.warn(fdata, "Unexpected content", line) + def parse_readme(self, nametag, fname): + """Parse ABI README file""" + + with open(fname, "r", encoding="utf8", errors="backslashreplace") as fp: + nametag["description"] = "```\n" + for line in fp: + nametag["description"] += " " + line + + nametag["description"] += "```\n" + def parse_file(self, fname, path, basename): """Parse a single file""" @@ -309,6 +319,10 @@ class AbiParser: if self.debug & AbiDebug.WHAT_OPEN: self.log.debug("Opening file %s", fname) + if basename == "README": + self.parse_readme(fdata.nametag, fname) + return + with open(fname, "r", encoding="utf8", errors="backslashreplace") as fp: for line in fp: fdata.ln += 1 @@ -344,9 +358,6 @@ class AbiParser: basename = os.path.basename(name) - if basename == "README": - continue - if basename.startswith("."): continue @@ -448,8 +459,12 @@ class AbiParser: continue if filter_path: - if v.get("path") != filter_path: - continue + if filter_path == "README": + if not names[0].endswith("README"): + continue + else: + if v.get("path") != filter_path: + continue msg = "" -- cgit v1.2.3 From 5d7871d77f6d62406b3d459a58810c1ddb8904c2 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 10 Feb 2025 11:18:09 +0100 Subject: docs: sphinx/kernel_abi: parse ABI files only once Right now, the logic parses ABI files on 4 steps, one for each directory. While this is fine in principle, by doing that, not all symbol cross-references will be created. Change the logic to do the parsing only once in order to get a global dictionary to be used when creating ABI cross-references. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/5205c53838b6ea25f4cdd4cc1e3d17c0141e75a6.1739182025.git.mchehab+huawei@kernel.org --- Documentation/admin-guide/abi-obsolete.rst | 2 +- Documentation/admin-guide/abi-removed.rst | 2 +- Documentation/admin-guide/abi-stable.rst | 2 +- Documentation/admin-guide/abi-testing.rst | 2 +- Documentation/sphinx/kernel_abi.py | 115 ++++++++++++++++------------- scripts/lib/abi/abi_parser.py | 22 +++--- 6 files changed, 81 insertions(+), 64 deletions(-) (limited to 'scripts') diff --git a/Documentation/admin-guide/abi-obsolete.rst b/Documentation/admin-guide/abi-obsolete.rst index 6d4d9ab7b8c3..bdef91d2cea4 100644 --- a/Documentation/admin-guide/abi-obsolete.rst +++ b/Documentation/admin-guide/abi-obsolete.rst @@ -9,4 +9,4 @@ marked to be removed at some later point in time. The description of the interface will document the reason why it is obsolete and when it can be expected to be removed. -.. kernel-abi:: ABI/obsolete +.. kernel-abi:: obsolete diff --git a/Documentation/admin-guide/abi-removed.rst b/Documentation/admin-guide/abi-removed.rst index 9fc78af6f077..bea0608b8442 100644 --- a/Documentation/admin-guide/abi-removed.rst +++ b/Documentation/admin-guide/abi-removed.rst @@ -3,4 +3,4 @@ ABI removed symbols =================== -.. kernel-abi:: ABI/removed +.. kernel-abi:: removed diff --git a/Documentation/admin-guide/abi-stable.rst b/Documentation/admin-guide/abi-stable.rst index c47c2a295865..33637c0d4fd5 100644 --- a/Documentation/admin-guide/abi-stable.rst +++ b/Documentation/admin-guide/abi-stable.rst @@ -12,4 +12,4 @@ for at least 2 years. Most interfaces (like syscalls) are expected to never change and always be available. -.. kernel-abi:: ABI/stable +.. kernel-abi:: stable diff --git a/Documentation/admin-guide/abi-testing.rst b/Documentation/admin-guide/abi-testing.rst index 40b31985e587..55054985a8ff 100644 --- a/Documentation/admin-guide/abi-testing.rst +++ b/Documentation/admin-guide/abi-testing.rst @@ -18,4 +18,4 @@ Programs that use these interfaces are strongly encouraged to add their name to the description of these interfaces, so that the kernel developers can easily notify them if any changes occur. -.. kernel-abi:: ABI/testing +.. kernel-abi:: testing diff --git a/Documentation/sphinx/kernel_abi.py b/Documentation/sphinx/kernel_abi.py index 0a4057183208..964f586de171 100644 --- a/Documentation/sphinx/kernel_abi.py +++ b/Documentation/sphinx/kernel_abi.py @@ -49,6 +49,13 @@ from abi_parser import AbiParser __version__ = "1.0" +logger = logging.getLogger('kernel_abi') +path = os.path.join(srctree, "Documentation/ABI") + +# Parse ABI symbols only once +kernel_abi = AbiParser(path, logger=logger) +kernel_abi.parse_abi() +kernel_abi.check_issues() def setup(app): @@ -64,14 +71,15 @@ class KernelCmd(Directive): u"""KernelABI (``kernel-abi``) directive""" required_arguments = 1 - optional_arguments = 2 + optional_arguments = 3 has_content = False final_argument_whitespace = True - logger = logging.getLogger('kernel_abi') parser = None option_spec = { "debug": directives.flag, + "no-symbols": directives.flag, + "no-files": directives.flag, } def run(self): @@ -79,62 +87,67 @@ class KernelCmd(Directive): if not doc.settings.file_insertion_enabled: raise self.warning("docutils: file insertion disabled") - path = os.path.join(srctree, "Documentation", self.arguments[0]) - self.parser = AbiParser(path, logger=self.logger) - self.parser.parse_abi() - self.parser.check_issues() - - node = self.nested_parse(None, self.arguments[0]) - return node - - def nested_parse(self, data, fname): env = self.state.document.settings.env content = ViewList() node = nodes.section() - if data is not None: - # Handles the .rst file - for line in data.split("\n"): - content.append(line, fname, 0) + abi_type = self.arguments[0] - self.do_parse(content, node) + if "no-symbols" in self.options: + show_symbols = False + else: + show_symbols = True + if "no-files" in self.options: + show_file = False + else: + show_file = True + + tab_width = self.options.get('tab-width', + self.state.document.settings.tab_width) + + old_f = None + n = 0 + n_sym = 0 + for msg, f, ln in kernel_abi.doc(show_file=show_file, + show_symbols=show_symbols, + filter_path=abi_type): + n_sym += 1 + msg_list = statemachine.string2lines(msg, tab_width, + convert_whitespace=True) + if "debug" in self.options: + lines = [ + "", "", ".. code-block:: rst", + " :linenos:", "" + ] + for m in msg_list: + lines.append(" " + m) + else: + lines = msg_list + + for line in lines: + # sphinx counts lines from 0 + content.append(line, f, ln - 1) + n += 1 + + if f != old_f: + # Add the file to Sphinx build dependencies + env.note_dependency(os.path.abspath(f)) + + old_f = f + + # Sphinx doesn't like to parse big messages. So, let's + # add content symbol by symbol + if content: + self.do_parse(content, node) + content = ViewList() + + if show_symbols and not show_file: + logger.verbose("%s ABI: %i symbols (%i ReST lines)" % (abi_type, n_sym, n)) + elif not show_symbols and show_file: + logger.verbose("%s ABI: %i files (%i ReST lines)" % (abi_type, n_sym, n)) else: - # Handles the ABI parser content, symbol by symbol - - old_f = fname - n = 0 - for msg, f, ln in self.parser.doc(): - msg_list = statemachine.string2lines(msg, tab_width, - convert_whitespace=True) - if "debug" in self.options: - lines = [ - "", "", ".. code-block:: rst", - " :linenos:", "" - ] - for m in msg_list: - lines.append(" " + m) - else: - lines = msg_list - - for line in lines: - # sphinx counts lines from 0 - content.append(line, f, ln - 1) - n += 1 - - if f != old_f: - # Add the file to Sphinx build dependencies - env.note_dependency(os.path.abspath(f)) - - old_f = f - - # Sphinx doesn't like to parse big messages. So, let's - # add content symbol by symbol - if content: - self.do_parse(content, node) - content = ViewList() - - self.logger.info("%s: parsed %i lines" % (fname, n)) + logger.verbose("%s ABI: %i data (%i ReST lines)" % (abi_type, n_sym, n)) return node.children diff --git a/scripts/lib/abi/abi_parser.py b/scripts/lib/abi/abi_parser.py index 6fac461d794c..87d1b9e14bb3 100644 --- a/scripts/lib/abi/abi_parser.py +++ b/scripts/lib/abi/abi_parser.py @@ -266,12 +266,20 @@ class AbiParser: def parse_readme(self, nametag, fname): """Parse ABI README file""" + nametag["what"] = ["ABI file contents"] + nametag["path"] = "README" with open(fname, "r", encoding="utf8", errors="backslashreplace") as fp: - nametag["description"] = "```\n" for line in fp: - nametag["description"] += " " + line + match = self.re_tag.match(line) + if match: + new = match.group(1).lower() + + match = self.re_valid.search(new) + if match: + nametag["description"] += "\n:" + line + continue - nametag["description"] += "```\n" + nametag["description"] += line def parse_file(self, fname, path, basename): """Parse a single file""" @@ -459,12 +467,8 @@ class AbiParser: continue if filter_path: - if filter_path == "README": - if not names[0].endswith("README"): - continue - else: - if v.get("path") != filter_path: - continue + if v.get("path") != filter_path: + continue msg = "" -- cgit v1.2.3 From c940816968da6ef9a9462b7c070cc333d609a16c Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 10 Feb 2025 11:18:11 +0100 Subject: docs: sphinx/automarkup: add cross-references for ABI Now that all ABI files are handled together, we can add a feature at automarkup for it to generate cross-references for ABI symbols. The cross-reference logic can produce references for all existing files, except for README (as this is not parsed). For symbols, they need to be an exact match of what it is described at the docs, which is not always true due to wildcards. If symbols at /sys /proc and /config are identical, a cross-reference will be used. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/0b97a51b68b1c20127ad4a6a55658557fe0848d0.1739182025.git.mchehab+huawei@kernel.org --- Documentation/sphinx/automarkup.py | 45 ++++++++++++++++++++++++++++++++++++++ scripts/lib/abi/abi_parser.py | 11 ++++++++++ 2 files changed, 56 insertions(+) (limited to 'scripts') diff --git a/Documentation/sphinx/automarkup.py b/Documentation/sphinx/automarkup.py index a413f8dd5115..7d91c39b4ca6 100644 --- a/Documentation/sphinx/automarkup.py +++ b/Documentation/sphinx/automarkup.py @@ -11,6 +11,8 @@ from sphinx.errors import NoUri import re from itertools import chain +from kernel_abi import kernel_abi + # # Python 2 lacks re.ASCII... # @@ -48,6 +50,8 @@ RE_typedef = re.compile(r'\b(typedef)\s+([a-zA-Z_]\w+)', flags=ascii_p3) # an optional extension # RE_doc = re.compile(r'(\bDocumentation/)?((\.\./)*[\w\-/]+)\.(rst|txt)') +RE_abi_file = re.compile(r'(\bDocumentation/ABI/[\w\-/]+)') +RE_abi_symbol = re.compile(r'(\b/(sys|config|proc)/[\w\-/]+)') RE_namespace = re.compile(r'^\s*..\s*c:namespace::\s*(\S+)\s*$') @@ -84,10 +88,14 @@ def markup_refs(docname, app, node): # Associate each regex with the function that will markup its matches # markup_func_sphinx2 = {RE_doc: markup_doc_ref, + RE_abi_file: markup_abi_ref, + RE_abi_symbol: markup_abi_ref, RE_function: markup_c_ref, RE_generic_type: markup_c_ref} markup_func_sphinx3 = {RE_doc: markup_doc_ref, + RE_abi_file: markup_abi_ref, + RE_abi_symbol: markup_abi_ref, RE_function: markup_func_ref_sphinx3, RE_struct: markup_c_ref, RE_union: markup_c_ref, @@ -270,6 +278,43 @@ def markup_doc_ref(docname, app, match): else: return nodes.Text(match.group(0)) +# +# Try to replace a documentation reference of the form Documentation/ABI/... +# with a cross reference to that page +# +def markup_abi_ref(docname, app, match): + stddom = app.env.domains['std'] + # + # Go through the dance of getting an xref out of the std domain + # + fname = match.group(1) + target = kernel_abi.xref(fname) + + # Kernel ABI doesn't describe such file or symbol + if not target: + return nodes.Text(match.group(0)) + + pxref = addnodes.pending_xref('', refdomain = 'std', reftype = 'ref', + reftarget = target, modname = None, + classname = None, refexplicit = False) + + # + # XXX The Latex builder will throw NoUri exceptions here, + # work around that by ignoring them. + # + try: + xref = stddom.resolve_xref(app.env, docname, app.builder, 'ref', + target, pxref, None) + except NoUri: + xref = None + # + # Return the xref if we got it; otherwise just return the plain text. + # + if xref: + return xref + else: + return nodes.Text(match.group(0)) + def get_c_namespace(app, docname): source = app.env.doc2path(docname) with open(source) as f: diff --git a/scripts/lib/abi/abi_parser.py b/scripts/lib/abi/abi_parser.py index 87d1b9e14bb3..3b1ab4c0bdd7 100644 --- a/scripts/lib/abi/abi_parser.py +++ b/scripts/lib/abi/abi_parser.py @@ -391,6 +391,17 @@ class AbiParser: return desc + "\n\n" + def xref(self, fname): + """ + Converts a Documentation/ABI + basename into a ReST cross-reference + """ + + xref = self.file_refs.get(fname) + if not xref: + return None + else: + return xref + def desc_rst(self, desc): """Enrich ReST output by creating cross-references""" -- cgit v1.2.3 From dc525a7650d70668c4d54cf03b4cba793b72cb5a Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 10 Feb 2025 11:18:13 +0100 Subject: scripts/lib/abi/abi_parser.py: Rename title name for ABI files This makes them look better when generating cross-references. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/e44574cb2796861d6acbce839068ed3ef385d16c.1739182025.git.mchehab+huawei@kernel.org --- scripts/lib/abi/abi_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'scripts') diff --git a/scripts/lib/abi/abi_parser.py b/scripts/lib/abi/abi_parser.py index 3b1ab4c0bdd7..0c3837e52afa 100644 --- a/scripts/lib/abi/abi_parser.py +++ b/scripts/lib/abi/abi_parser.py @@ -314,7 +314,7 @@ class AbiParser: fdata.ftype = path.split("/")[0] fdata.nametag = {} - fdata.nametag["what"] = [f"File {path}/{basename}"] + fdata.nametag["what"] = [f"ABI file {path}/{basename}"] fdata.nametag["type"] = "File" fdata.nametag["path"] = fdata.ftype fdata.nametag["file"] = [fdata.file_ref] -- cgit v1.2.3 From 6649b4217089c5d17dc210946baf9c9537c7fb5d Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 10 Feb 2025 11:18:14 +0100 Subject: scripts/lib/abi/abi_parser.py: make it backward-compatible with Python 3.6 Despite being introduced on Python 3.6, the original implementation was too limited: it doesn't accept anything but the argument. Even on python 3.10.12, support was still limited, as more complex operations cause SyntaxError: Exception occurred: File ".../linux/Documentation/sphinx/kernel_abi.py", line 48, in from get_abi import AbiParser File ".../linux/scripts/lib/abi/abi_parser.py", line 525 msg += f"{part}\n{"-" * len(part)}\n\n" ^ SyntaxError: f-string: expecting '}' Replace f-strings by normal string concatenation when it doesn't work on Python 3.6. Reported-by: Akira Yokosawa Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/41d2f85df134a46db46fed73a0f9697a3d2ae9ba.1739182025.git.mchehab+huawei@kernel.org --- scripts/lib/abi/abi_parser.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'scripts') diff --git a/scripts/lib/abi/abi_parser.py b/scripts/lib/abi/abi_parser.py index 0c3837e52afa..f08de6d3bf7c 100644 --- a/scripts/lib/abi/abi_parser.py +++ b/scripts/lib/abi/abi_parser.py @@ -493,7 +493,7 @@ class AbiParser: if cur_part and cur_part != part: part = cur_part - msg += f"{part}\n{"-" * len(part)}\n\n" + msg += part + "\n"+ "-" * len(part) +"\n\n" msg += f".. _{key}:\n\n" @@ -517,7 +517,7 @@ class AbiParser: msg += f"Defined on file :ref:`{base} <{ref[1]}>`\n\n" if wtype == "File": - msg += f"{names[0]}\n{"-" * len(names[0])}\n\n" + msg += names[0] +"\n" + "-" * len(names[0]) +"\n\n" desc = v.get("description") if not desc and wtype != "File": @@ -541,7 +541,8 @@ class AbiParser: users = v.get("users") if users and users.strip(" \t\n"): - msg += f"Users:\n\t{users.strip("\n").replace('\n', '\n\t')}\n\n" + users = users.strip("\n").replace('\n', '\n\t') + msg += f"Users:\n\t{users}\n\n" ln = v.get("line_no", 1) @@ -567,7 +568,9 @@ class AbiParser: elif len(lines) == 1: f.append(f"{fname}:{lines[0]}") else: - f.append(f"{fname} lines {", ".join(str(x) for x in lines)}") + m = fname + "lines " + m += ", ".join(str(x) for x in lines) + f.append(m) self.log.warning("%s is defined %d times: %s", what, len(f), "; ".join(f)) @@ -615,10 +618,11 @@ class AbiParser: if users: print(f"Users:\t\t\t{users}") - print(f"Defined on file{'s'[:len(files) ^ 1]}:\t{", ".join(files)}") + print("Defined on file(s):\t" + ", ".join(files)) if desc: - print(f"\n{desc.strip("\n")}\n") + desc = desc.strip("\n") + print(f"\n{desc}\n") if not found_keys: print(f"Regular expression /{expr}/ not found.") -- cgit v1.2.3 From 0d5fd96880d9135a4b35fb5523896b21b13dde78 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 10 Feb 2025 11:18:15 +0100 Subject: scripts/get_abi.py: add support for undefined ABIs The undefined logic is complex and has lots of magic on it. Implement it, using the same algorithm we have at get_abi.pl. Yet, some tweaks to optimize performance and to make the code simpler were added here: - at the perl version, the tree graph had loops, so we had to use BFS to traverse it. On this version, the graph is a tree, so, it simplifies the what group for sysfs aliases; - the logic which splits regular expressions into subgroups was re-written to make it faster; - it may optionally use multiple processes to search for symbol matches; - it has some additional debug levels. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/1529c255845d117696d5af57d8dc05554663afdf.1739182025.git.mchehab+huawei@kernel.org --- scripts/get_abi.py | 68 +++++++ scripts/lib/abi/abi_regex.py | 234 +++++++++++++++++++++++ scripts/lib/abi/helpers.py | 16 +- scripts/lib/abi/system_symbols.py | 378 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 693 insertions(+), 3 deletions(-) create mode 100644 scripts/lib/abi/abi_regex.py create mode 100644 scripts/lib/abi/system_symbols.py (limited to 'scripts') diff --git a/scripts/get_abi.py b/scripts/get_abi.py index 19f78d6aa407..7ce4748a46d2 100755 --- a/scripts/get_abi.py +++ b/scripts/get_abi.py @@ -20,7 +20,9 @@ SRC_DIR = os.path.dirname(os.path.realpath(__file__)) sys.path.insert(0, os.path.join(SRC_DIR, LIB_DIR)) from abi_parser import AbiParser # pylint: disable=C0413 +from abi_regex import AbiRegex # pylint: disable=C0413 from helpers import ABI_DIR, DEBUG_HELP # pylint: disable=C0413 +from system_symbols import SystemSymbols # pylint: disable=C0413 # Command line classes @@ -111,6 +113,71 @@ class AbiSearch: parser.parse_abi() parser.search_symbols(args.expression) +UNDEFINED_DESC=""" +Check undefined ABIs on local machine. + +Read sysfs devnodes and check if the devnodes there are defined inside +ABI documentation. + +The search logic tries to minimize the number of regular expressions to +search per each symbol. + +By default, it runs on a single CPU, as Python support for CPU threads +is still experimental, and multi-process runs on Python is very slow. + +On experimental tests, if the number of ABI symbols to search per devnode +is contained on a limit of ~150 regular expressions, using a single CPU +is a lot faster than using multiple processes. However, if the number of +regular expressions to check is at the order of ~30000, using multiple +CPUs speeds up the check. +""" + +class AbiUndefined: + """ + Initialize an argparse subparser for logic to check undefined ABI at + the current machine's sysfs + """ + + def __init__(self, subparsers): + """Initialize argparse subparsers""" + + parser = subparsers.add_parser("undefined", + formatter_class=argparse.RawTextHelpFormatter, + description=UNDEFINED_DESC) + + parser.add_argument("-S", "--sysfs-dir", default="/sys", + help="directory where sysfs is mounted") + parser.add_argument("-s", "--search-string", + help="search string regular expression to limit symbol search") + parser.add_argument("-H", "--show-hints", action="store_true", + help="Hints about definitions for missing ABI symbols.") + parser.add_argument("-j", "--jobs", "--max-workers", type=int, default=1, + help="If bigger than one, enables multiprocessing.") + parser.add_argument("-c", "--max-chunk-size", type=int, default=50, + help="Maximum number of chunk size") + parser.add_argument("-f", "--found", action="store_true", + help="Also show found items. " + "Helpful to debug the parser."), + parser.add_argument("-d", "--dry-run", action="store_true", + help="Don't actually search for undefined. " + "Helpful to debug the parser."), + + parser.set_defaults(func=self.run) + + def run(self, args): + """Run subparser""" + + abi = AbiRegex(args.dir, debug=args.debug, + search_string=args.search_string) + + abi_symbols = SystemSymbols(abi=abi, hints=args.show_hints, + sysfs=args.sysfs_dir) + + abi_symbols.check_undefined_symbols(dry_run=args.dry_run, + found=args.found, + max_workers=args.jobs, + chunk_size=args.max_chunk_size) + def main(): """Main program""" @@ -125,6 +192,7 @@ def main(): AbiRest(subparsers) AbiValidate(subparsers) AbiSearch(subparsers) + AbiUndefined(subparsers) args = parser.parse_args() diff --git a/scripts/lib/abi/abi_regex.py b/scripts/lib/abi/abi_regex.py new file mode 100644 index 000000000000..8a57846cbc69 --- /dev/null +++ b/scripts/lib/abi/abi_regex.py @@ -0,0 +1,234 @@ +#!/usr/bin/env python3 +# xxpylint: disable=R0903 +# Copyright(c) 2025: Mauro Carvalho Chehab . +# SPDX-License-Identifier: GPL-2.0 + +""" +Convert ABI what into regular expressions +""" + +import re +import sys + +from pprint import pformat + +from abi_parser import AbiParser +from helpers import AbiDebug + +class AbiRegex(AbiParser): + """Extends AbiParser to search ABI nodes with regular expressions""" + + # Escape only ASCII visible characters + escape_symbols = r"([\x21-\x29\x2b-\x2d\x3a-\x40\x5c\x60\x7b-\x7e])" + leave_others = "others" + + # Tuples with regular expressions to be compiled and replacement data + re_whats = [ + # Drop escape characters that might exist + (re.compile("\\\\"), ""), + + # Temporarily escape dot characters + (re.compile(r"\."), "\xf6"), + + # Temporarily change [0-9]+ type of patterns + (re.compile(r"\[0\-9\]\+"), "\xff"), + + # Temporarily change [\d+-\d+] type of patterns + (re.compile(r"\[0\-\d+\]"), "\xff"), + (re.compile(r"\[0:\d+\]"), "\xff"), + (re.compile(r"\[(\d+)\]"), "\xf4\\\\d+\xf5"), + + # Temporarily change [0-9] type of patterns + (re.compile(r"\[(\d)\-(\d)\]"), "\xf4\1-\2\xf5"), + + # Handle multiple option patterns + (re.compile(r"[\{\<\[]([\w_]+)(?:[,|]+([\w_]+)){1,}[\}\>\]]"), r"(\1|\2)"), + + # Handle wildcards + (re.compile(r"([^\/])\*"), "\\1\\\\w\xf7"), + (re.compile(r"/\*/"), "/.*/"), + (re.compile(r"/\xf6\xf6\xf6"), "/.*"), + (re.compile(r"\<[^\>]+\>"), "\\\\w\xf7"), + (re.compile(r"\{[^\}]+\}"), "\\\\w\xf7"), + (re.compile(r"\[[^\]]+\]"), "\\\\w\xf7"), + + (re.compile(r"XX+"), "\\\\w\xf7"), + (re.compile(r"([^A-Z])[XYZ]([^A-Z])"), "\\1\\\\w\xf7\\2"), + (re.compile(r"([^A-Z])[XYZ]$"), "\\1\\\\w\xf7"), + (re.compile(r"_[AB]_"), "_\\\\w\xf7_"), + + # Recover [0-9] type of patterns + (re.compile(r"\xf4"), "["), + (re.compile(r"\xf5"), "]"), + + # Remove duplicated spaces + (re.compile(r"\s+"), r" "), + + # Special case: drop comparison as in: + # What: foo = + # (this happens on a few IIO definitions) + (re.compile(r"\s*\=.*$"), ""), + + # Escape all other symbols + (re.compile(escape_symbols), r"\\\1"), + (re.compile(r"\\\\"), r"\\"), + (re.compile(r"\\([\[\]\(\)\|])"), r"\1"), + (re.compile(r"(\d+)\\(-\d+)"), r"\1\2"), + + (re.compile(r"\xff"), r"\\d+"), + + # Special case: IIO ABI which a parenthesis. + (re.compile(r"sqrt(.*)"), r"sqrt(.*)"), + + # Simplify regexes with multiple .* + (re.compile(r"(?:\.\*){2,}"), ""), + + # Recover dot characters + (re.compile(r"\xf6"), "\\."), + # Recover plus characters + (re.compile(r"\xf7"), "+"), + ] + re_has_num = re.compile(r"\\d") + + # Symbol name after escape_chars that are considered a devnode basename + re_symbol_name = re.compile(r"(\w|\\[\.\-\:])+$") + + # List of popular group names to be skipped to minimize regex group size + # Use AbiDebug.SUBGROUP_SIZE to detect those + skip_names = set(["devices", "hwmon"]) + + def regex_append(self, what, new): + """ + Get a search group for a subset of regular expressions. + + As ABI may have thousands of symbols, using a for to search all + regular expressions is at least O(n^2). When there are wildcards, + the complexity increases substantially, eventually becoming exponential. + + To avoid spending too much time on them, use a logic to split + them into groups. The smaller the group, the better, as it would + mean that searches will be confined to a small number of regular + expressions. + + The conversion to a regex subset is tricky, as we need something + that can be easily obtained from the sysfs symbol and from the + regular expression. So, we need to discard nodes that have + wildcards. + + If it can't obtain a subgroup, place the regular expression inside + a special group (self.leave_others). + """ + + search_group = None + + for search_group in reversed(new.split("/")): + if not search_group or search_group in self.skip_names: + continue + if self.re_symbol_name.match(search_group): + break + + if not search_group: + search_group = self.leave_others + + if self.debug & AbiDebug.SUBGROUP_MAP: + self.log.debug("%s: mapped as %s", what, search_group) + + try: + if search_group not in self.regex_group: + self.regex_group[search_group] = [] + + self.regex_group[search_group].append(re.compile(new)) + if self.search_string: + if what.find(self.search_string) >= 0: + print(f"What: {what}") + except re.PatternError: + self.log.warning("Ignoring '%s' as it produced an invalid regex:\n" + " '%s'", what, new) + + def get_regexes(self, what): + """ + Given an ABI devnode, return a list of all regular expressions that + may match it, based on the sub-groups created by regex_append() + """ + + re_list = [] + + patches = what.split("/") + patches.reverse() + patches.append(self.leave_others) + + for search_group in patches: + if search_group in self.regex_group: + re_list += self.regex_group[search_group] + + return re_list + + def __init__(self, *args, **kwargs): + """ + Override init method to get verbose argument + """ + + self.regex_group = None + self.search_string = None + self.re_string = None + + if "search_string" in kwargs: + self.search_string = kwargs.get("search_string") + del kwargs["search_string"] + + if self.search_string: + + try: + self.re_string = re.compile(self.search_string) + except re.PatternError as e: + msg = f"{self.search_string} is not a valid regular expression" + raise ValueError(msg) from e + + super().__init__(*args, **kwargs) + + def parse_abi(self, *args, **kwargs): + + super().parse_abi(*args, **kwargs) + + self.regex_group = {} + + print("Converting ABI What fields into regexes...", file=sys.stderr) + + for t in sorted(self.data.items(), key=lambda x: x[0]): + v = t[1] + if v.get("type") == "File": + continue + + v["regex"] = [] + + for what in v.get("what", []): + if not what.startswith("/sys"): + continue + + new = what + for r, s in self.re_whats: + try: + new = r.sub(s, new) + except re.PatternError as e: + # Help debugging troubles with new regexes + raise re.PatternError(f"{e}\nwhile re.sub('{r.pattern}', {s}, str)") from e + + v["regex"].append(new) + + if self.debug & AbiDebug.REGEX: + self.log.debug("%-90s <== %s", new, what) + + # Store regex into a subgroup to speedup searches + self.regex_append(what, new) + + if self.debug & AbiDebug.SUBGROUP_DICT: + self.log.debug("%s", pformat(self.regex_group)) + + if self.debug & AbiDebug.SUBGROUP_SIZE: + biggestd_keys = sorted(self.regex_group.keys(), + key= lambda k: len(self.regex_group[k]), + reverse=True) + + print("Top regex subgroups:", file=sys.stderr) + for k in biggestd_keys[:10]: + print(f"{k} has {len(self.regex_group[k])} elements", file=sys.stderr) diff --git a/scripts/lib/abi/helpers.py b/scripts/lib/abi/helpers.py index 84a253ed5058..639b23e4ca33 100644 --- a/scripts/lib/abi/helpers.py +++ b/scripts/lib/abi/helpers.py @@ -16,13 +16,23 @@ class AbiDebug: WHAT_PARSING = 1 WHAT_OPEN = 2 DUMP_ABI_STRUCTS = 4 + UNDEFINED = 8 + REGEX = 16 + SUBGROUP_MAP = 32 + SUBGROUP_DICT = 64 + SUBGROUP_SIZE = 128 + GRAPH = 256 DEBUG_HELP = """ -Print debug information according with the level(s), -which is given by the following bitmask: - 1 - enable debug parsing logic 2 - enable debug messages on file open 4 - enable debug for ABI parse data +8 - enable extra debug information to identify troubles + with ABI symbols found at the local machine that + weren't found on ABI documentation (used only for + undefined subcommand) +16 - enable debug for what to regex conversion +32 - enable debug for symbol regex subgroups +64 - enable debug for sysfs graph tree variable """ diff --git a/scripts/lib/abi/system_symbols.py b/scripts/lib/abi/system_symbols.py new file mode 100644 index 000000000000..f15c94a6e33c --- /dev/null +++ b/scripts/lib/abi/system_symbols.py @@ -0,0 +1,378 @@ +#!/usr/bin/env python3 +# pylint: disable=R0902,R0912,R0914,R0915,R1702 +# Copyright(c) 2025: Mauro Carvalho Chehab . +# SPDX-License-Identifier: GPL-2.0 + +""" +Parse ABI documentation and produce results from it. +""" + +import os +import re +import sys + +from concurrent import futures +from datetime import datetime +from random import shuffle + +from helpers import AbiDebug + +class SystemSymbols: + """Stores arguments for the class and initialize class vars""" + + def graph_add_file(self, path, link=None): + """ + add a file path to the sysfs graph stored at self.root + """ + + if path in self.files: + return + + name = "" + ref = self.root + for edge in path.split("/"): + name += edge + "/" + if edge not in ref: + ref[edge] = {"__name": [name.rstrip("/")]} + + ref = ref[edge] + + if link and link not in ref["__name"]: + ref["__name"].append(link.rstrip("/")) + + self.files.add(path) + + def print_graph(self, root_prefix="", root=None, level=0): + """Prints a reference tree graph using UTF-8 characters""" + + if not root: + root = self.root + level = 0 + + # Prevent endless traverse + if level > 5: + return + + if level > 0: + prefix = "├──" + last_prefix = "└──" + else: + prefix = "" + last_prefix = "" + + items = list(root.items()) + + names = root.get("__name", []) + for k, edge in items: + if k == "__name": + continue + + if not k: + k = "/" + + if len(names) > 1: + k += " links: " + ",".join(names[1:]) + + if edge == items[-1][1]: + print(root_prefix + last_prefix + k) + p = root_prefix + if level > 0: + p += " " + self.print_graph(p, edge, level + 1) + else: + print(root_prefix + prefix + k) + p = root_prefix + "│ " + self.print_graph(p, edge, level + 1) + + def _walk(self, root): + """ + Walk through sysfs to get all devnodes that aren't ignored. + + By default, uses /sys as sysfs mounting point. If another + directory is used, it replaces them to /sys at the patches. + """ + + with os.scandir(root) as obj: + for entry in obj: + path = os.path.join(root, entry.name) + if self.sysfs: + p = path.replace(self.sysfs, "/sys", count=1) + else: + p = path + + if self.re_ignore.search(p): + return + + # Handle link first to avoid directory recursion + if entry.is_symlink(): + real = os.path.realpath(path) + if not self.sysfs: + self.aliases[path] = real + else: + real = real.replace(self.sysfs, "/sys", count=1) + + # Add absfile location to graph if it doesn't exist + if not self.re_ignore.search(real): + # Add link to the graph + self.graph_add_file(real, p) + + elif entry.is_file(): + self.graph_add_file(p) + + elif entry.is_dir(): + self._walk(path) + + def __init__(self, abi, sysfs="/sys", hints=False): + """ + Initialize internal variables and get a list of all files inside + sysfs that can currently be parsed. + + Please notice that there are several entries on sysfs that aren't + documented as ABI. Ignore those. + + The real paths will be stored under self.files. Aliases will be + stored in separate, as self.aliases. + """ + + self.abi = abi + self.log = abi.log + + if sysfs != "/sys": + self.sysfs = sysfs.rstrip("/") + else: + self.sysfs = None + + self.hints = hints + + self.root = {} + self.aliases = {} + self.files = set() + + dont_walk = [ + # Those require root access and aren't documented at ABI + f"^{sysfs}/kernel/debug", + f"^{sysfs}/kernel/tracing", + f"^{sysfs}/fs/pstore", + f"^{sysfs}/fs/bpf", + f"^{sysfs}/fs/fuse", + + # This is not documented at ABI + f"^{sysfs}/module", + + f"^{sysfs}/fs/cgroup", # this is big and has zero docs under ABI + f"^{sysfs}/firmware", # documented elsewhere: ACPI, DT bindings + "sections|notes", # aren't actually part of ABI + + # kernel-parameters.txt - not easy to parse + "parameters", + ] + + self.re_ignore = re.compile("|".join(dont_walk)) + + print(f"Reading {sysfs} directory contents...", file=sys.stderr) + self._walk(sysfs) + + def check_file(self, refs, found): + """Check missing ABI symbols for a given sysfs file""" + + res_list = [] + + try: + for names in refs: + fname = names[0] + + res = { + "found": False, + "fname": fname, + "msg": "", + } + res_list.append(res) + + re_what = self.abi.get_regexes(fname) + if not re_what: + self.abi.log.warning(f"missing rules for {fname}") + continue + + for name in names: + for r in re_what: + if self.abi.debug & AbiDebug.UNDEFINED: + self.log.debug("check if %s matches '%s'", name, r.pattern) + if r.match(name): + res["found"] = True + if found: + res["msg"] += f" {fname}: regex:\n\t" + continue + + if self.hints and not res["found"]: + res["msg"] += f" {fname} not found. Tested regexes:\n" + for r in re_what: + res["msg"] += " " + r.pattern + "\n" + + except KeyboardInterrupt: + pass + + return res_list + + def _ref_interactor(self, root): + """Recursive function to interact over the sysfs tree""" + + for k, v in root.items(): + if isinstance(v, dict): + yield from self._ref_interactor(v) + + if root == self.root or k == "__name": + continue + + if self.abi.re_string: + fname = v["__name"][0] + if self.abi.re_string.search(fname): + yield v + else: + yield v + + + def get_fileref(self, all_refs, chunk_size): + """Interactor to group refs into chunks""" + + n = 0 + refs = [] + + for ref in all_refs: + refs.append(ref) + + n += 1 + if n >= chunk_size: + yield refs + n = 0 + refs = [] + + yield refs + + def check_undefined_symbols(self, max_workers=None, chunk_size=50, + found=None, dry_run=None): + """Seach ABI for sysfs symbols missing documentation""" + + self.abi.parse_abi() + + if self.abi.debug & AbiDebug.GRAPH: + self.print_graph() + + all_refs = [] + for ref in self._ref_interactor(self.root): + all_refs.append(ref["__name"]) + + if dry_run: + print("Would check", file=sys.stderr) + for ref in all_refs: + print(", ".join(ref)) + + return + + print("Starting to search symbols (it may take several minutes):", + file=sys.stderr) + start = datetime.now() + old_elapsed = None + + # Python doesn't support multithreading due to limitations on its + # global lock (GIL). While Python 3.13 finally made GIL optional, + # there are still issues related to it. Also, we want to have + # backward compatibility with older versions of Python. + # + # So, use instead multiprocess. However, Python is very slow passing + # data from/to multiple processes. Also, it may consume lots of memory + # if the data to be shared is not small. So, we need to group workload + # in chunks that are big enough to generate performance gains while + # not being so big that would cause out-of-memory. + + num_refs = len(all_refs) + print(f"Number of references to parse: {num_refs}", file=sys.stderr) + + if not max_workers: + max_workers = os.cpu_count() + elif max_workers > os.cpu_count(): + max_workers = os.cpu_count() + + max_workers = max(max_workers, 1) + + max_chunk_size = int((num_refs + max_workers - 1) / max_workers) + chunk_size = min(chunk_size, max_chunk_size) + chunk_size = max(1, chunk_size) + + if max_workers > 1: + executor = futures.ProcessPoolExecutor + + # Place references in a random order. This may help improving + # performance, by mixing complex/simple expressions when creating + # chunks + shuffle(all_refs) + else: + # Python has a high overhead with processes. When there's just + # one worker, it is faster to not create a new process. + # Yet, User still deserves to have a progress print. So, use + # python's "thread", which is actually a single process, using + # an internal schedule to switch between tasks. No performance + # gains for non-IO tasks, but still it can be quickly interrupted + # from time to time to display progress. + executor = futures.ThreadPoolExecutor + + not_found = [] + f_list = [] + with executor(max_workers=max_workers) as exe: + for refs in self.get_fileref(all_refs, chunk_size): + if refs: + try: + f_list.append(exe.submit(self.check_file, refs, found)) + + except KeyboardInterrupt: + return + + total = len(f_list) + + if not total: + if self.abi.re_string: + print(f"No ABI symbol matches {self.abi.search_string}") + else: + self.abi.log.warning("No ABI symbols found") + return + + print(f"{len(f_list):6d} jobs queued on {max_workers} workers", + file=sys.stderr) + + while f_list: + try: + t = futures.wait(f_list, timeout=1, + return_when=futures.FIRST_COMPLETED) + + done = t[0] + + for fut in done: + res_list = fut.result() + + for res in res_list: + if not res["found"]: + not_found.append(res["fname"]) + if res["msg"]: + print(res["msg"]) + + f_list.remove(fut) + except KeyboardInterrupt: + return + + except RuntimeError as e: + self.abi.log.warning(f"Future: {e}") + break + + if sys.stderr.isatty(): + elapsed = str(datetime.now() - start).split(".", maxsplit=1)[0] + if len(f_list) < total: + elapsed += f" ({total - len(f_list)}/{total} jobs completed). " + if elapsed != old_elapsed: + print(elapsed + "\r", end="", flush=True, + file=sys.stderr) + old_elapsed = elapsed + + elapsed = str(datetime.now() - start).split(".", maxsplit=1)[0] + print(elapsed, file=sys.stderr) + + for f in sorted(not_found): + print(f"{f} not found.") -- cgit v1.2.3 From 1c7e66bc5d20ac7779130e146d70066b3af4711c Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 10 Feb 2025 11:18:16 +0100 Subject: scripts/get_abi.pl: drop now obsoleted script As all functionalities of it were migrated to get_abi.py, drop the now obsoleted script. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/698ec258b36b63ccde5f7da1af9c97cf8df51050.1739182025.git.mchehab+huawei@kernel.org --- scripts/get_abi.pl | 1103 ---------------------------------------------------- 1 file changed, 1103 deletions(-) delete mode 100755 scripts/get_abi.pl (limited to 'scripts') diff --git a/scripts/get_abi.pl b/scripts/get_abi.pl deleted file mode 100755 index de1c0354b50c..000000000000 --- a/scripts/get_abi.pl +++ /dev/null @@ -1,1103 +0,0 @@ -#!/usr/bin/env perl -# SPDX-License-Identifier: GPL-2.0 - -BEGIN { $Pod::Usage::Formatter = 'Pod::Text::Termcap'; } - -use strict; -use warnings; -use utf8; -use Pod::Usage qw(pod2usage); -use Getopt::Long; -use File::Find; -use IO::Handle; -use Fcntl ':mode'; -use Cwd 'abs_path'; -use Data::Dumper; - -my $help = 0; -my $hint = 0; -my $man = 0; -my $debug = 0; -my $enable_lineno = 0; -my $show_warnings = 1; -my $prefix="Documentation/ABI"; -my $sysfs_prefix="/sys"; -my $search_string; - -# Debug options -my $dbg_what_parsing = 1; -my $dbg_what_open = 2; -my $dbg_dump_abi_structs = 4; -my $dbg_undefined = 8; - -$Data::Dumper::Indent = 1; -$Data::Dumper::Terse = 1; - -# -# If true, assumes that the description is formatted with ReST -# -my $description_is_rst = 1; - -GetOptions( - "debug=i" => \$debug, - "enable-lineno" => \$enable_lineno, - "rst-source!" => \$description_is_rst, - "dir=s" => \$prefix, - 'help|?' => \$help, - "show-hints" => \$hint, - "search-string=s" => \$search_string, - man => \$man -) or pod2usage(2); - -pod2usage(1) if $help; -pod2usage(-exitstatus => 0, -noperldoc, -verbose => 2) if $man; - -pod2usage(2) if (scalar @ARGV < 1 || @ARGV > 2); - -my ($cmd, $arg) = @ARGV; - -pod2usage(2) if ($cmd ne "search" && $cmd ne "rest" && $cmd ne "validate" && $cmd ne "undefined"); -pod2usage(2) if ($cmd eq "search" && !$arg); - -require Data::Dumper if ($debug & $dbg_dump_abi_structs); - -my %data; -my %symbols; - -# -# Displays an error message, printing file name and line -# -sub parse_error($$$$) { - my ($file, $ln, $msg, $data) = @_; - - return if (!$show_warnings); - - $data =~ s/\s+$/\n/; - - print STDERR "Warning: file $file#$ln:\n\t$msg"; - - if ($data ne "") { - print STDERR ". Line\n\t\t$data"; - } else { - print STDERR "\n"; - } -} - -# -# Parse an ABI file, storing its contents at %data -# -sub parse_abi { - my $file = $File::Find::name; - - my $mode = (stat($file))[2]; - return if ($mode & S_IFDIR); - return if ($file =~ m,/README,); - return if ($file =~ m,/\.,); - return if ($file =~ m,\.(rej|org|orig|bak)$,); - - my $name = $file; - $name =~ s,.*/,,; - - my $fn = $file; - $fn =~ s,.*Documentation/ABI/,,; - - my $nametag = "File $fn"; - $data{$nametag}->{what} = "File $name"; - $data{$nametag}->{type} = "File"; - $data{$nametag}->{file} = $name; - $data{$nametag}->{filepath} = $file; - $data{$nametag}->{is_file} = 1; - $data{$nametag}->{line_no} = 1; - - my $type = $file; - $type =~ s,.*/(.*)/.*,$1,; - - my $what; - my $new_what; - my $tag = ""; - my $ln; - my $xrefs; - my $space; - my @labels; - my $label = ""; - - print STDERR "Opening $file\n" if ($debug & $dbg_what_open); - open IN, $file; - while() { - $ln++; - if (m/^(\S+)(:\s*)(.*)/i) { - my $new_tag = lc($1); - my $sep = $2; - my $content = $3; - - if (!($new_tag =~ m/(what|where|date|kernelversion|contact|description|users)/)) { - if ($tag eq "description") { - # New "tag" is actually part of - # description. Don't consider it a tag - $new_tag = ""; - } elsif ($tag ne "") { - parse_error($file, $ln, "tag '$tag' is invalid", $_); - } - } - - # Invalid, but it is a common mistake - if ($new_tag eq "where") { - parse_error($file, $ln, "tag 'Where' is invalid. Should be 'What:' instead", ""); - $new_tag = "what"; - } - - if ($new_tag =~ m/what/) { - $space = ""; - $content =~ s/[,.;]$//; - - push @{$symbols{$content}->{file}}, " $file:" . ($ln - 1); - - if ($tag =~ m/what/) { - $what .= "\xac" . $content; - } else { - if ($what) { - parse_error($file, $ln, "What '$what' doesn't have a description", "") if (!$data{$what}->{description}); - - foreach my $w(split /\xac/, $what) { - $symbols{$w}->{xref} = $what; - }; - } - - $what = $content; - $label = $content; - $new_what = 1; - } - push @labels, [($content, $label)]; - $tag = $new_tag; - - push @{$data{$nametag}->{symbols}}, $content if ($data{$nametag}->{what}); - next; - } - - if ($tag ne "" && $new_tag) { - $tag = $new_tag; - - if ($new_what) { - @{$data{$what}->{label_list}} = @labels if ($data{$nametag}->{what}); - @labels = (); - $label = ""; - $new_what = 0; - - $data{$what}->{type} = $type; - if (!defined($data{$what}->{file})) { - $data{$what}->{file} = $name; - $data{$what}->{filepath} = $file; - } else { - $data{$what}->{description} .= "\n\n" if (defined($data{$what}->{description})); - if ($name ne $data{$what}->{file}) { - $data{$what}->{file} .= " " . $name; - $data{$what}->{filepath} .= " " . $file; - } - } - print STDERR "\twhat: $what\n" if ($debug & $dbg_what_parsing); - $data{$what}->{line_no} = $ln; - } else { - $data{$what}->{line_no} = $ln if (!defined($data{$what}->{line_no})); - } - - if (!$what) { - parse_error($file, $ln, "'What:' should come first:", $_); - next; - } - if ($new_tag eq "description") { - $sep =~ s,:, ,; - $content = ' ' x length($new_tag) . $sep . $content; - while ($content =~ s/\t+/' ' x (length($&) * 8 - length($`) % 8)/e) {} - if ($content =~ m/^(\s*)(\S.*)$/) { - # Preserve initial spaces for the first line - $space = $1; - $content = "$2\n"; - $data{$what}->{$tag} .= $content; - } else { - undef($space); - } - - } else { - $data{$what}->{$tag} = $content; - } - next; - } - } - - # Store any contents before tags at the database - if (!$tag && $data{$nametag}->{what}) { - $data{$nametag}->{description} .= $_; - next; - } - - if ($tag eq "description") { - my $content = $_; - while ($content =~ s/\t+/' ' x (length($&) * 8 - length($`) % 8)/e) {} - if (m/^\s*\n/) { - $data{$what}->{$tag} .= "\n"; - next; - } - - if (!defined($space)) { - # Preserve initial spaces for the first line - if ($content =~ m/^(\s*)(\S.*)$/) { - $space = $1; - $content = "$2\n"; - } - } else { - $space = "" if (!($content =~ s/^($space)//)); - } - $data{$what}->{$tag} .= $content; - - next; - } - if (m/^\s*(.*)/) { - $data{$what}->{$tag} .= "\n$1"; - $data{$what}->{$tag} =~ s/\n+$//; - next; - } - - # Everything else is error - parse_error($file, $ln, "Unexpected content", $_); - } - $data{$nametag}->{description} =~ s/^\n+// if ($data{$nametag}->{description}); - if ($what) { - parse_error($file, $ln, "What '$what' doesn't have a description", "") if (!$data{$what}->{description}); - - foreach my $w(split /\xac/,$what) { - $symbols{$w}->{xref} = $what; - }; - } - close IN; -} - -sub create_labels { - my %labels; - - foreach my $what (keys %data) { - next if ($data{$what}->{file} eq "File"); - - foreach my $p (@{$data{$what}->{label_list}}) { - my ($content, $label) = @{$p}; - $label = "abi_" . $label . " "; - $label =~ tr/A-Z/a-z/; - - # Convert special chars to "_" - $label =~s/([\x00-\x2f\x3a-\x40\x5b-\x60\x7b-\xff])/_/g; - $label =~ s,_+,_,g; - $label =~ s,_$,,; - - # Avoid duplicated labels - while (defined($labels{$label})) { - my @chars = ("A".."Z", "a".."z"); - $label .= $chars[rand @chars]; - } - $labels{$label} = 1; - - $data{$what}->{label} = $label; - - # only one label is enough - last; - } - } -} - -# -# Outputs the book on ReST format -# - -# \b doesn't work well with paths. So, we need to define something else: -# Boundaries are punct characters, spaces and end-of-line -my $start = qr {(^|\s|\() }x; -my $bondary = qr { ([,.:;\)\s]|\z) }x; -my $xref_match = qr { $start(\/(sys|config|proc|dev|kvd)\/[^,.:;\)\s]+)$bondary }x; -my $symbols = qr { ([\x01-\x08\x0e-\x1f\x21-\x2f\x3a-\x40\x7b-\xff]) }x; - -sub output_rest { - create_labels(); - - my $part = ""; - - foreach my $what (sort { - ($data{$a}->{type} eq "File") cmp ($data{$b}->{type} eq "File") || - $a cmp $b - } keys %data) { - my $type = $data{$what}->{type}; - - my @file = split / /, $data{$what}->{file}; - my @filepath = split / /, $data{$what}->{filepath}; - - if ($enable_lineno) { - printf ".. LINENO %s%s#%s\n\n", - $prefix, $file[0], - $data{$what}->{line_no}; - } - - my $w = $what; - - if ($type ne "File") { - my $cur_part = $what; - if ($what =~ '/') { - if ($what =~ m#^(\/?(?:[\w\-]+\/?){1,2})#) { - $cur_part = "Symbols under $1"; - $cur_part =~ s,/$,,; - } - } - - if ($cur_part ne "" && $part ne $cur_part) { - $part = $cur_part; - my $bar = $part; - $bar =~ s/./-/g; - print "$part\n$bar\n\n"; - } - - printf ".. _%s:\n\n", $data{$what}->{label}; - - my @names = split /\xac/,$w; - my $len = 0; - - foreach my $name (@names) { - $name =~ s/$symbols/\\$1/g; - $name = "**$name**"; - $len = length($name) if (length($name) > $len); - } - - print "+-" . "-" x $len . "-+\n"; - foreach my $name (@names) { - printf "| %s", $name . " " x ($len - length($name)) . " |\n"; - print "+-" . "-" x $len . "-+\n"; - } - - print "\n"; - } - - for (my $i = 0; $i < scalar(@filepath); $i++) { - my $path = $filepath[$i]; - my $f = $file[$i]; - - $path =~ s,.*/(.*/.*),$1,;; - $path =~ s,[/\-],_,g;; - my $fileref = "abi_file_".$path; - - if ($type eq "File") { - print ".. _$fileref:\n\n"; - } else { - print "Defined on file :ref:`$f <$fileref>`\n\n"; - } - } - - if ($type eq "File") { - my $bar = $w; - $bar =~ s/./-/g; - print "$w\n$bar\n\n"; - } - - my $desc = ""; - $desc = $data{$what}->{description} if (defined($data{$what}->{description})); - $desc =~ s/\s+$/\n/; - - if (!($desc =~ /^\s*$/)) { - if ($description_is_rst) { - # Remove title markups from the description - # Having titles inside ABI files will only work if extra - # care would be taken in order to strictly follow the same - # level order for each markup. - $desc =~ s/\n[\-\*\=\^\~]+\n/\n\n/g; - - # Enrich text by creating cross-references - - my $new_desc = ""; - my $init_indent = -1; - my $literal_indent = -1; - - open(my $fh, "+<", \$desc); - while (my $d = <$fh>) { - my $indent = $d =~ m/^(\s+)/; - my $spaces = length($indent); - $init_indent = $indent if ($init_indent < 0); - if ($literal_indent >= 0) { - if ($spaces > $literal_indent) { - $new_desc .= $d; - next; - } else { - $literal_indent = -1; - } - } else { - if ($d =~ /()::$/ && !($d =~ /^\s*\.\./)) { - $literal_indent = $spaces; - } - } - - $d =~ s,Documentation/(?!devicetree)(\S+)\.rst,:doc:`/$1`,g; - - my @matches = $d =~ m,Documentation/ABI/([\w\/\-]+),g; - foreach my $f (@matches) { - my $xref = $f; - my $path = $f; - $path =~ s,.*/(.*/.*),$1,;; - $path =~ s,[/\-],_,g;; - $xref .= " "; - $d =~ s,\bDocumentation/ABI/$f\b,:ref:`$xref`,g; - } - - # Seek for cross reference symbols like /sys/... - @matches = $d =~ m/$xref_match/g; - - foreach my $s (@matches) { - next if (!($s =~ m,/,)); - if (defined($data{$s}) && defined($data{$s}->{label})) { - my $xref = $s; - - $xref =~ s/$symbols/\\$1/g; - $xref = ":ref:`$xref <" . $data{$s}->{label} . ">`"; - - $d =~ s,$start$s$bondary,$1$xref$2,g; - } - } - $new_desc .= $d; - } - close $fh; - - - print "$new_desc\n\n"; - } else { - $desc =~ s/^\s+//; - - # Remove title markups from the description, as they won't work - $desc =~ s/\n[\-\*\=\^\~]+\n/\n\n/g; - - if ($desc =~ m/\:\n/ || $desc =~ m/\n[\t ]+/ || $desc =~ m/[\x00-\x08\x0b-\x1f\x7b-\xff]/) { - # put everything inside a code block - $desc =~ s/\n/\n /g; - - print "::\n\n"; - print " $desc\n\n"; - } else { - # Escape any special chars from description - $desc =~s/([\x00-\x08\x0b-\x1f\x21-\x2a\x2d\x2f\x3c-\x40\x5c\x5e-\x60\x7b-\xff])/\\$1/g; - print "$desc\n\n"; - } - } - } else { - print "DESCRIPTION MISSING for $what\n\n" if (!$data{$what}->{is_file}); - } - - if ($data{$what}->{symbols}) { - printf "Has the following ABI:\n\n"; - - foreach my $content(@{$data{$what}->{symbols}}) { - my $label = $data{$symbols{$content}->{xref}}->{label}; - - # Escape special chars from content - $content =~s/([\x00-\x1f\x21-\x2f\x3a-\x40\x7b-\xff])/\\$1/g; - - print "- :ref:`$content <$label>`\n\n"; - } - } - - if (defined($data{$what}->{users})) { - my $users = $data{$what}->{users}; - - $users =~ s/\n/\n\t/g; - printf "Users:\n\t%s\n\n", $users if ($users ne ""); - } - - } -} - -# -# Searches for ABI symbols -# -sub search_symbols { - foreach my $what (sort keys %data) { - next if (!($what =~ m/($arg)/)); - - my $type = $data{$what}->{type}; - next if ($type eq "File"); - - my $file = $data{$what}->{filepath}; - - $what =~ s/\xac/, /g; - my $bar = $what; - $bar =~ s/./-/g; - - print "\n$what\n$bar\n\n"; - - my $kernelversion = $data{$what}->{kernelversion} if (defined($data{$what}->{kernelversion})); - my $contact = $data{$what}->{contact} if (defined($data{$what}->{contact})); - my $users = $data{$what}->{users} if (defined($data{$what}->{users})); - my $date = $data{$what}->{date} if (defined($data{$what}->{date})); - my $desc = $data{$what}->{description} if (defined($data{$what}->{description})); - - $kernelversion =~ s/^\s+// if ($kernelversion); - $contact =~ s/^\s+// if ($contact); - if ($users) { - $users =~ s/^\s+//; - $users =~ s/\n//g; - } - $date =~ s/^\s+// if ($date); - $desc =~ s/^\s+// if ($desc); - - printf "Kernel version:\t\t%s\n", $kernelversion if ($kernelversion); - printf "Date:\t\t\t%s\n", $date if ($date); - printf "Contact:\t\t%s\n", $contact if ($contact); - printf "Users:\t\t\t%s\n", $users if ($users); - print "Defined on file(s):\t$file\n\n"; - print "Description:\n\n$desc"; - } -} - -# Exclude /sys/kernel/debug and /sys/kernel/tracing from the search path -sub dont_parse_special_attributes { - if (($File::Find::dir =~ m,^/sys/kernel,)) { - return grep {!/(debug|tracing)/ } @_; - } - - if (($File::Find::dir =~ m,^/sys/fs,)) { - return grep {!/(pstore|bpf|fuse)/ } @_; - } - - return @_ -} - -my %leaf; -my %aliases; -my @files; -my %root; - -sub graph_add_file { - my $file = shift; - my $type = shift; - - my $dir = $file; - $dir =~ s,^(.*/).*,$1,; - $file =~ s,.*/,,; - - my $name; - my $file_ref = \%root; - foreach my $edge(split "/", $dir) { - $name .= "$edge/"; - if (!defined ${$file_ref}{$edge}) { - ${$file_ref}{$edge} = { }; - } - $file_ref = \%{$$file_ref{$edge}}; - ${$file_ref}{"__name"} = [ $name ]; - } - $name .= "$file"; - ${$file_ref}{$file} = { - "__name" => [ $name ] - }; - - return \%{$$file_ref{$file}}; -} - -sub graph_add_link { - my $file = shift; - my $link = shift; - - # Traverse graph to find the reference - my $file_ref = \%root; - foreach my $edge(split "/", $file) { - $file_ref = \%{$$file_ref{$edge}} || die "Missing node!"; - } - - # do a BFS - - my @queue; - my %seen; - my $st; - - push @queue, $file_ref; - $seen{$start}++; - - while (@queue) { - my $v = shift @queue; - my @child = keys(%{$v}); - - foreach my $c(@child) { - next if $seen{$$v{$c}}; - next if ($c eq "__name"); - - if (!defined($$v{$c}{"__name"})) { - printf STDERR "Error: Couldn't find a non-empty name on a children of $file/.*: "; - print STDERR Dumper(%{$v}); - exit; - } - - # Add new name - my $name = @{$$v{$c}{"__name"}}[0]; - if ($name =~ s#^$file/#$link/#) { - push @{$$v{$c}{"__name"}}, $name; - } - # Add child to the queue and mark as seen - push @queue, $$v{$c}; - $seen{$c}++; - } - } -} - -my $escape_symbols = qr { ([\x01-\x08\x0e-\x1f\x21-\x29\x2b-\x2d\x3a-\x40\x7b-\xfe]) }x; -sub parse_existing_sysfs { - my $file = $File::Find::name; - - my $mode = (lstat($file))[2]; - my $abs_file = abs_path($file); - - my @tmp; - push @tmp, $file; - push @tmp, $abs_file if ($abs_file ne $file); - - foreach my $f(@tmp) { - # Ignore cgroup, as this is big and has zero docs under ABI - return if ($f =~ m#^/sys/fs/cgroup/#); - - # Ignore firmware as it is documented elsewhere - # Either ACPI or under Documentation/devicetree/bindings/ - return if ($f =~ m#^/sys/firmware/#); - - # Ignore some sysfs nodes that aren't actually part of ABI - return if ($f =~ m#/sections|notes/#); - - # Would need to check at - # Documentation/admin-guide/kernel-parameters.txt, but this - # is not easily parseable. - return if ($f =~ m#/parameters/#); - } - - if (S_ISLNK($mode)) { - $aliases{$file} = $abs_file; - return; - } - - return if (S_ISDIR($mode)); - - # Trivial: file is defined exactly the same way at ABI What: - return if (defined($data{$file})); - return if (defined($data{$abs_file})); - - push @files, graph_add_file($abs_file, "file"); -} - -sub get_leave($) -{ - my $what = shift; - my $leave; - - my $l = $what; - my $stop = 1; - - $leave = $l; - $leave =~ s,/$,,; - $leave =~ s,.*/,,; - $leave =~ s/[\(\)]//g; - - # $leave is used to improve search performance at - # check_undefined_symbols, as the algorithm there can seek - # for a small number of "what". It also allows giving a - # hint about a leave with the same name somewhere else. - # However, there are a few occurences where the leave is - # either a wildcard or a number. Just group such cases - # altogether. - if ($leave =~ m/\.\*/ || $leave eq "" || $leave =~ /\\d/) { - $leave = "others"; - } - - return $leave; -} - -my @not_found; - -sub check_file($$) -{ - my $file_ref = shift; - my $names_ref = shift; - my @names = @{$names_ref}; - my $file = $names[0]; - - my $found_string; - - my $leave = get_leave($file); - if (!defined($leaf{$leave})) { - $leave = "others"; - } - my @expr = @{$leaf{$leave}->{expr}}; - die ("\rmissing rules for $leave") if (!defined($leaf{$leave})); - - my $path = $file; - $path =~ s,(.*/).*,$1,; - - if ($search_string) { - return if (!($file =~ m#$search_string#)); - $found_string = 1; - } - - for (my $i = 0; $i < @names; $i++) { - if ($found_string && $hint) { - if (!$i) { - print STDERR "--> $names[$i]\n"; - } else { - print STDERR " $names[$i]\n"; - } - } - foreach my $re (@expr) { - print STDERR "$names[$i] =~ /^$re\$/\n" if ($debug && $dbg_undefined); - if ($names[$i] =~ $re) { - return; - } - } - } - - if ($leave ne "others") { - my @expr = @{$leaf{"others"}->{expr}}; - for (my $i = 0; $i < @names; $i++) { - foreach my $re (@expr) { - print STDERR "$names[$i] =~ /^$re\$/\n" if ($debug && $dbg_undefined); - if ($names[$i] =~ $re) { - return; - } - } - } - } - - push @not_found, $file if (!$search_string || $found_string); - - if ($hint && (!$search_string || $found_string)) { - my $what = $leaf{$leave}->{what}; - $what =~ s/\xac/\n\t/g; - if ($leave ne "others") { - print STDERR "\r more likely regexes:\n\t$what\n"; - } else { - print STDERR "\r tested regexes:\n\t$what\n"; - } - } -} - -sub check_undefined_symbols { - my $num_files = scalar @files; - my $next_i = 0; - my $start_time = times; - - @files = sort @files; - - my $last_time = $start_time; - - # When either debug or hint is enabled, there's no sense showing - # progress, as the progress will be overriden. - if ($hint || ($debug && $dbg_undefined)) { - $next_i = $num_files; - } - - my $is_console; - $is_console = 1 if (-t STDERR); - - for (my $i = 0; $i < $num_files; $i++) { - my $file_ref = $files[$i]; - my @names = @{$$file_ref{"__name"}}; - - check_file($file_ref, \@names); - - my $cur_time = times; - - if ($i == $next_i || $cur_time > $last_time + 1) { - my $percent = $i * 100 / $num_files; - - my $tm = $cur_time - $start_time; - my $time = sprintf "%d:%02d", int($tm), 60 * ($tm - int($tm)); - - printf STDERR "\33[2K\r", if ($is_console); - printf STDERR "%s: processing sysfs files... %i%%: $names[0]", $time, $percent; - printf STDERR "\n", if (!$is_console); - STDERR->flush(); - - $next_i = int (($percent + 1) * $num_files / 100); - $last_time = $cur_time; - } - } - - my $cur_time = times; - my $tm = $cur_time - $start_time; - my $time = sprintf "%d:%02d", int($tm), 60 * ($tm - int($tm)); - - printf STDERR "\33[2K\r", if ($is_console); - printf STDERR "%s: processing sysfs files... done\n", $time; - - foreach my $file (@not_found) { - print "$file not found.\n"; - } -} - -sub undefined_symbols { - print STDERR "Reading $sysfs_prefix directory contents..."; - find({ - wanted =>\&parse_existing_sysfs, - preprocess =>\&dont_parse_special_attributes, - no_chdir => 1 - }, $sysfs_prefix); - print STDERR "done.\n"; - - $leaf{"others"}->{what} = ""; - - print STDERR "Converting ABI What fields into regexes..."; - foreach my $w (sort keys %data) { - foreach my $what (split /\xac/,$w) { - next if (!($what =~ m/^$sysfs_prefix/)); - - # Convert what into regular expressions - - # Escape dot characters - $what =~ s/\./\xf6/g; - - # Temporarily change [0-9]+ type of patterns - $what =~ s/\[0\-9\]\+/\xff/g; - - # Temporarily change [\d+-\d+] type of patterns - $what =~ s/\[0\-\d+\]/\xff/g; - $what =~ s/\[(\d+)\]/\xf4$1\xf5/g; - - # Temporarily change [0-9] type of patterns - $what =~ s/\[(\d)\-(\d)\]/\xf4$1-$2\xf5/g; - - # Handle multiple option patterns - $what =~ s/[\{\<\[]([\w_]+)(?:[,|]+([\w_]+)){1,}[\}\>\]]/($1|$2)/g; - - # Handle wildcards - $what =~ s,\*,.*,g; - $what =~ s,/\xf6..,/.*,g; - $what =~ s/\<[^\>]+\>/.*/g; - $what =~ s/\{[^\}]+\}/.*/g; - $what =~ s/\[[^\]]+\]/.*/g; - - $what =~ s/[XYZ]/.*/g; - - # Recover [0-9] type of patterns - $what =~ s/\xf4/[/g; - $what =~ s/\xf5/]/g; - - # Remove duplicated spaces - $what =~ s/\s+/ /g; - - # Special case: this ABI has a parenthesis on it - $what =~ s/sqrt\(x^2\+y^2\+z^2\)/sqrt\(x^2\+y^2\+z^2\)/; - - # Special case: drop comparition as in: - # What: foo = - # (this happens on a few IIO definitions) - $what =~ s,\s*\=.*$,,; - - # Escape all other symbols - $what =~ s/$escape_symbols/\\$1/g; - $what =~ s/\\\\/\\/g; - $what =~ s/\\([\[\]\(\)\|])/$1/g; - $what =~ s/(\d+)\\(-\d+)/$1$2/g; - - $what =~ s/\xff/\\d+/g; - - # Special case: IIO ABI which a parenthesis. - $what =~ s/sqrt(.*)/sqrt\(.*\)/; - - # Simplify regexes with multiple .* - $what =~ s#(?:\.\*){2,}##g; -# $what =~ s#\.\*/\.\*#.*#g; - - # Recover dot characters - $what =~ s/\xf6/\./g; - - my $leave = get_leave($what); - - my $added = 0; - foreach my $l (split /\|/, $leave) { - if (defined($leaf{$l})) { - next if ($leaf{$l}->{what} =~ m/\b$what\b/); - $leaf{$l}->{what} .= "\xac" . $what; - $added = 1; - } else { - $leaf{$l}->{what} = $what; - $added = 1; - } - } - if ($search_string && $added) { - print STDERR "What: $what\n" if ($what =~ m#$search_string#); - } - - } - } - # Compile regexes - foreach my $l (sort keys %leaf) { - my @expr; - foreach my $w(sort split /\xac/, $leaf{$l}->{what}) { - push @expr, qr /^$w$/; - } - $leaf{$l}->{expr} = \@expr; - } - - # Take links into account - foreach my $link (sort keys %aliases) { - my $abs_file = $aliases{$link}; - graph_add_link($abs_file, $link); - } - print STDERR "done.\n"; - - check_undefined_symbols; -} - -# Ensure that the prefix will always end with a slash -# While this is not needed for find, it makes the patch nicer -# with --enable-lineno -$prefix =~ s,/?$,/,; - -if ($cmd eq "undefined" || $cmd eq "search") { - $show_warnings = 0; -} -# -# Parses all ABI files located at $prefix dir -# -find({wanted =>\&parse_abi, no_chdir => 1}, $prefix); - -print STDERR Data::Dumper->Dump([\%data], [qw(*data)]) if ($debug & $dbg_dump_abi_structs); - -# -# Handles the command -# -if ($cmd eq "undefined") { - undefined_symbols; -} elsif ($cmd eq "search") { - search_symbols; -} else { - if ($cmd eq "rest") { - output_rest; - } - - # Warn about duplicated ABI entries - foreach my $what(sort keys %symbols) { - my @files = @{$symbols{$what}->{file}}; - - next if (scalar(@files) == 1); - - printf STDERR "Warning: $what is defined %d times: @files\n", - scalar(@files); - } -} - -__END__ - -=head1 NAME - -get_abi.pl - parse the Linux ABI files and produce a ReST book. - -=head1 SYNOPSIS - -B [--debug ] [--enable-lineno] [--man] [--help] - [--(no-)rst-source] [--dir=] [--show-hints] - [--search-string ] - [] - -Where B can be: - -=over 8 - -B I - search for I inside ABI - -B - output the ABI in ReST markup language - -B - validate the ABI contents - -B - existing symbols at the system that aren't - defined at Documentation/ABI - -=back - -=head1 OPTIONS - -=over 8 - -=item B<--dir> - -Changes the location of the ABI search. By default, it uses -the Documentation/ABI directory. - -=item B<--rst-source> and B<--no-rst-source> - -The input file may be using ReST syntax or not. Those two options allow -selecting between a rst-compliant source ABI (B<--rst-source>), or a -plain text that may be violating ReST spec, so it requres some escaping -logic (B<--no-rst-source>). - -=item B<--enable-lineno> - -Enable output of .. LINENO lines. - -=item B<--debug> I - -Print debug information according with the level, which is given by the -following bitmask: - - - 1: Debug parsing What entries from ABI files; - - 2: Shows what files are opened from ABI files; - - 4: Dump the structs used to store the contents of the ABI files. - -=item B<--show-hints> - -Show hints about possible definitions for the missing ABI symbols. -Used only when B. - -=item B<--search-string> I - -Show only occurences that match a search string. -Used only when B. - -=item B<--help> - -Prints a brief help message and exits. - -=item B<--man> - -Prints the manual page and exits. - -=back - -=head1 DESCRIPTION - -Parse the Linux ABI files from ABI DIR (usually located at Documentation/ABI), -allowing to search for ABI symbols or to produce a ReST book containing -the Linux ABI documentation. - -=head1 EXAMPLES - -Search for all stable symbols with the word "usb": - -=over 8 - -$ scripts/get_abi.pl search usb --dir Documentation/ABI/stable - -=back - -Search for all symbols that match the regex expression "usb.*cap": - -=over 8 - -$ scripts/get_abi.pl search usb.*cap - -=back - -Output all obsoleted symbols in ReST format - -=over 8 - -$ scripts/get_abi.pl rest --dir Documentation/ABI/obsolete - -=back - -=head1 BUGS - -Report bugs to Mauro Carvalho Chehab - -=head1 COPYRIGHT - -Copyright (c) 2016-2021 by Mauro Carvalho Chehab . - -License GPLv2: GNU GPL version 2 . - -This is free software: you are free to change and redistribute it. -There is NO WARRANTY, to the extent permitted by law. - -=cut -- cgit v1.2.3 From c4a16820d90199409c9bf01c4f794e1e9e8d8fd8 Mon Sep 17 00:00:00 2001 From: Christian Brauner Date: Tue, 28 Jan 2025 11:33:41 +0100 Subject: fs: add open_tree_attr() Add open_tree_attr() which allow to atomically create a detached mount tree and set mount options on it. If OPEN_TREE_CLONE is used this will allow the creation of a detached mount with a new set of mount options without it ever being exposed to userspace without that set of mount options applied. Link: https://lore.kernel.org/r/20250128-work-mnt_idmap-update-v2-v1-3-c25feb0d2eb3@kernel.org Reviewed-by: "Seth Forshee (DigitalOcean)" Signed-off-by: Christian Brauner --- arch/alpha/kernel/syscalls/syscall.tbl | 1 + arch/arm/tools/syscall.tbl | 1 + arch/arm64/tools/syscall_32.tbl | 1 + arch/m68k/kernel/syscalls/syscall.tbl | 1 + arch/microblaze/kernel/syscalls/syscall.tbl | 1 + arch/mips/kernel/syscalls/syscall_n32.tbl | 1 + arch/mips/kernel/syscalls/syscall_n64.tbl | 1 + arch/mips/kernel/syscalls/syscall_o32.tbl | 1 + arch/parisc/kernel/syscalls/syscall.tbl | 1 + arch/powerpc/kernel/syscalls/syscall.tbl | 1 + arch/s390/kernel/syscalls/syscall.tbl | 1 + arch/sh/kernel/syscalls/syscall.tbl | 1 + arch/sparc/kernel/syscalls/syscall.tbl | 1 + arch/x86/entry/syscalls/syscall_32.tbl | 1 + arch/x86/entry/syscalls/syscall_64.tbl | 1 + arch/xtensa/kernel/syscalls/syscall.tbl | 1 + fs/namespace.c | 39 +++++++++++++++++++++++++++++ include/linux/syscalls.h | 4 +++ include/uapi/asm-generic/unistd.h | 4 ++- scripts/syscall.tbl | 1 + 20 files changed, 63 insertions(+), 1 deletion(-) (limited to 'scripts') diff --git a/arch/alpha/kernel/syscalls/syscall.tbl b/arch/alpha/kernel/syscalls/syscall.tbl index c59d53d6d3f3..2dd6340de6b4 100644 --- a/arch/alpha/kernel/syscalls/syscall.tbl +++ b/arch/alpha/kernel/syscalls/syscall.tbl @@ -506,3 +506,4 @@ 574 common getxattrat sys_getxattrat 575 common listxattrat sys_listxattrat 576 common removexattrat sys_removexattrat +577 common open_tree_attr sys_open_tree_attr diff --git a/arch/arm/tools/syscall.tbl b/arch/arm/tools/syscall.tbl index 49eeb2ad8dbd..27c1d5ebcd91 100644 --- a/arch/arm/tools/syscall.tbl +++ b/arch/arm/tools/syscall.tbl @@ -481,3 +481,4 @@ 464 common getxattrat sys_getxattrat 465 common listxattrat sys_listxattrat 466 common removexattrat sys_removexattrat +467 common open_tree_attr sys_open_tree_attr diff --git a/arch/arm64/tools/syscall_32.tbl b/arch/arm64/tools/syscall_32.tbl index 69a829912a05..0765b3a8d6d6 100644 --- a/arch/arm64/tools/syscall_32.tbl +++ b/arch/arm64/tools/syscall_32.tbl @@ -478,3 +478,4 @@ 464 common getxattrat sys_getxattrat 465 common listxattrat sys_listxattrat 466 common removexattrat sys_removexattrat +467 common open_tree_attr sys_open_tree_attr diff --git a/arch/m68k/kernel/syscalls/syscall.tbl b/arch/m68k/kernel/syscalls/syscall.tbl index f5ed71f1910d..9fe47112c586 100644 --- a/arch/m68k/kernel/syscalls/syscall.tbl +++ b/arch/m68k/kernel/syscalls/syscall.tbl @@ -466,3 +466,4 @@ 464 common getxattrat sys_getxattrat 465 common listxattrat sys_listxattrat 466 common removexattrat sys_removexattrat +467 common open_tree_attr sys_open_tree_attr diff --git a/arch/microblaze/kernel/syscalls/syscall.tbl b/arch/microblaze/kernel/syscalls/syscall.tbl index 680f568b77f2..7b6e97828e55 100644 --- a/arch/microblaze/kernel/syscalls/syscall.tbl +++ b/arch/microblaze/kernel/syscalls/syscall.tbl @@ -472,3 +472,4 @@ 464 common getxattrat sys_getxattrat 465 common listxattrat sys_listxattrat 466 common removexattrat sys_removexattrat +467 common open_tree_attr sys_open_tree_attr diff --git a/arch/mips/kernel/syscalls/syscall_n32.tbl b/arch/mips/kernel/syscalls/syscall_n32.tbl index 0b9b7e25b69a..aa70e371bb54 100644 --- a/arch/mips/kernel/syscalls/syscall_n32.tbl +++ b/arch/mips/kernel/syscalls/syscall_n32.tbl @@ -405,3 +405,4 @@ 464 n32 getxattrat sys_getxattrat 465 n32 listxattrat sys_listxattrat 466 n32 removexattrat sys_removexattrat +467 n32 open_tree_attr sys_open_tree_attr diff --git a/arch/mips/kernel/syscalls/syscall_n64.tbl b/arch/mips/kernel/syscalls/syscall_n64.tbl index c844cd5cda62..1e8c44c7b614 100644 --- a/arch/mips/kernel/syscalls/syscall_n64.tbl +++ b/arch/mips/kernel/syscalls/syscall_n64.tbl @@ -381,3 +381,4 @@ 464 n64 getxattrat sys_getxattrat 465 n64 listxattrat sys_listxattrat 466 n64 removexattrat sys_removexattrat +467 n64 open_tree_attr sys_open_tree_attr diff --git a/arch/mips/kernel/syscalls/syscall_o32.tbl b/arch/mips/kernel/syscalls/syscall_o32.tbl index 349b8aad1159..114a5a1a6230 100644 --- a/arch/mips/kernel/syscalls/syscall_o32.tbl +++ b/arch/mips/kernel/syscalls/syscall_o32.tbl @@ -454,3 +454,4 @@ 464 o32 getxattrat sys_getxattrat 465 o32 listxattrat sys_listxattrat 466 o32 removexattrat sys_removexattrat +467 o32 open_tree_attr sys_open_tree_attr diff --git a/arch/parisc/kernel/syscalls/syscall.tbl b/arch/parisc/kernel/syscalls/syscall.tbl index d9fc94c86965..94df3cb957e9 100644 --- a/arch/parisc/kernel/syscalls/syscall.tbl +++ b/arch/parisc/kernel/syscalls/syscall.tbl @@ -465,3 +465,4 @@ 464 common getxattrat sys_getxattrat 465 common listxattrat sys_listxattrat 466 common removexattrat sys_removexattrat +467 common open_tree_attr sys_open_tree_attr diff --git a/arch/powerpc/kernel/syscalls/syscall.tbl b/arch/powerpc/kernel/syscalls/syscall.tbl index d8b4ab78bef0..9a084bdb8926 100644 --- a/arch/powerpc/kernel/syscalls/syscall.tbl +++ b/arch/powerpc/kernel/syscalls/syscall.tbl @@ -557,3 +557,4 @@ 464 common getxattrat sys_getxattrat 465 common listxattrat sys_listxattrat 466 common removexattrat sys_removexattrat +467 common open_tree_attr sys_open_tree_attr diff --git a/arch/s390/kernel/syscalls/syscall.tbl b/arch/s390/kernel/syscalls/syscall.tbl index e9115b4d8b63..a4569b96ef06 100644 --- a/arch/s390/kernel/syscalls/syscall.tbl +++ b/arch/s390/kernel/syscalls/syscall.tbl @@ -469,3 +469,4 @@ 464 common getxattrat sys_getxattrat sys_getxattrat 465 common listxattrat sys_listxattrat sys_listxattrat 466 common removexattrat sys_removexattrat sys_removexattrat +467 common open_tree_attr sys_open_tree_attr sys_open_tree_attr diff --git a/arch/sh/kernel/syscalls/syscall.tbl b/arch/sh/kernel/syscalls/syscall.tbl index c8cad33bf250..52a7652fcff6 100644 --- a/arch/sh/kernel/syscalls/syscall.tbl +++ b/arch/sh/kernel/syscalls/syscall.tbl @@ -470,3 +470,4 @@ 464 common getxattrat sys_getxattrat 465 common listxattrat sys_listxattrat 466 common removexattrat sys_removexattrat +467 common open_tree_attr sys_open_tree_attr diff --git a/arch/sparc/kernel/syscalls/syscall.tbl b/arch/sparc/kernel/syscalls/syscall.tbl index 727f99d333b3..83e45eb6c095 100644 --- a/arch/sparc/kernel/syscalls/syscall.tbl +++ b/arch/sparc/kernel/syscalls/syscall.tbl @@ -512,3 +512,4 @@ 464 common getxattrat sys_getxattrat 465 common listxattrat sys_listxattrat 466 common removexattrat sys_removexattrat +467 common open_tree_attr sys_open_tree_attr diff --git a/arch/x86/entry/syscalls/syscall_32.tbl b/arch/x86/entry/syscalls/syscall_32.tbl index 4d0fb2fba7e2..3f0ec87d5db4 100644 --- a/arch/x86/entry/syscalls/syscall_32.tbl +++ b/arch/x86/entry/syscalls/syscall_32.tbl @@ -472,3 +472,4 @@ 464 i386 getxattrat sys_getxattrat 465 i386 listxattrat sys_listxattrat 466 i386 removexattrat sys_removexattrat +467 i386 open_tree_attr sys_open_tree_attr diff --git a/arch/x86/entry/syscalls/syscall_64.tbl b/arch/x86/entry/syscalls/syscall_64.tbl index 5eb708bff1c7..cfb5ca41e30d 100644 --- a/arch/x86/entry/syscalls/syscall_64.tbl +++ b/arch/x86/entry/syscalls/syscall_64.tbl @@ -390,6 +390,7 @@ 464 common getxattrat sys_getxattrat 465 common listxattrat sys_listxattrat 466 common removexattrat sys_removexattrat +467 common open_tree_attr sys_open_tree_attr # # Due to a historical design error, certain syscalls are numbered differently diff --git a/arch/xtensa/kernel/syscalls/syscall.tbl b/arch/xtensa/kernel/syscalls/syscall.tbl index 37effc1b134e..f657a77314f8 100644 --- a/arch/xtensa/kernel/syscalls/syscall.tbl +++ b/arch/xtensa/kernel/syscalls/syscall.tbl @@ -437,3 +437,4 @@ 464 common getxattrat sys_getxattrat 465 common listxattrat sys_listxattrat 466 common removexattrat sys_removexattrat +467 common open_tree_attr sys_open_tree_attr diff --git a/fs/namespace.c b/fs/namespace.c index d2ef1d69839b..ac4ad746c770 100644 --- a/fs/namespace.c +++ b/fs/namespace.c @@ -4995,6 +4995,45 @@ SYSCALL_DEFINE5(mount_setattr, int, dfd, const char __user *, path, return err; } +SYSCALL_DEFINE5(open_tree_attr, int, dfd, const char __user *, filename, + unsigned, flags, struct mount_attr __user *, uattr, + size_t, usize) +{ + struct file __free(fput) *file = NULL; + int fd; + + if (!uattr && usize) + return -EINVAL; + + file = vfs_open_tree(dfd, filename, flags); + if (IS_ERR(file)) + return PTR_ERR(file); + + if (uattr) { + int ret; + struct mount_kattr kattr = { + .recurse = !!(flags & AT_RECURSIVE), + }; + + ret = copy_mount_setattr(uattr, usize, &kattr); + if (ret) + return ret; + + ret = do_mount_setattr(&file->f_path, &kattr); + if (ret) + return ret; + + finish_mount_kattr(&kattr); + } + + fd = get_unused_fd_flags(flags & O_CLOEXEC); + if (fd < 0) + return fd; + + fd_install(fd, no_free_ptr(file)); + return fd; +} + int show_path(struct seq_file *m, struct dentry *root) { if (root->d_sb->s_op->show_path) diff --git a/include/linux/syscalls.h b/include/linux/syscalls.h index c6333204d451..079ea1d09d85 100644 --- a/include/linux/syscalls.h +++ b/include/linux/syscalls.h @@ -951,6 +951,10 @@ asmlinkage long sys_statx(int dfd, const char __user *path, unsigned flags, asmlinkage long sys_rseq(struct rseq __user *rseq, uint32_t rseq_len, int flags, uint32_t sig); asmlinkage long sys_open_tree(int dfd, const char __user *path, unsigned flags); +asmlinkage long sys_open_tree_attr(int dfd, const char __user *path, + unsigned flags, + struct mount_attr __user *uattr, + size_t usize); asmlinkage long sys_move_mount(int from_dfd, const char __user *from_path, int to_dfd, const char __user *to_path, unsigned int ms_flags); diff --git a/include/uapi/asm-generic/unistd.h b/include/uapi/asm-generic/unistd.h index 88dc393c2bca..2892a45023af 100644 --- a/include/uapi/asm-generic/unistd.h +++ b/include/uapi/asm-generic/unistd.h @@ -849,9 +849,11 @@ __SYSCALL(__NR_getxattrat, sys_getxattrat) __SYSCALL(__NR_listxattrat, sys_listxattrat) #define __NR_removexattrat 466 __SYSCALL(__NR_removexattrat, sys_removexattrat) +#define __NR_open_tree_attr 467 +__SYSCALL(__NR_open_tree_attr, sys_open_tree_attr) #undef __NR_syscalls -#define __NR_syscalls 467 +#define __NR_syscalls 468 /* * 32 bit systems traditionally used different diff --git a/scripts/syscall.tbl b/scripts/syscall.tbl index ebbdb3c42e9f..580b4e246aec 100644 --- a/scripts/syscall.tbl +++ b/scripts/syscall.tbl @@ -407,3 +407,4 @@ 464 common getxattrat sys_getxattrat 465 common listxattrat sys_listxattrat 466 common removexattrat sys_removexattrat +467 common open_tree_attr sys_open_tree_attr -- cgit v1.2.3 From ba561b485709b6160e56d1fe32a2717fffb332cc Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Wed, 12 Feb 2025 07:02:52 +0100 Subject: scripts/kernel-doc: remove an obscure logic from kernel-doc Kernel-doc has an obscure logic that uses an external file to map files via a .tmp_filelist.txt file stored at the current directory. The rationale for such code predates git time, as it was added on Kernel v2.4.5.5, with the following description: # 26/05/2001 - Support for separate source and object trees. # Return error code. # Keith Owens from commit 396a6123577d ("v2.4.5.4 -> v2.4.5.5") at the historic tree: https://git.kernel.org/pub/scm/linux/kernel/git/history/history.git/ Support for separate source and object trees is now done on a different way via make O=. There's no logic to create such file, so it sounds to me that this is just dead code. So, drop it. Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/fd3b28dec36ba1668325d6770d4c4754414337fc.1739340170.git.mchehab+huawei@kernel.org Signed-off-by: Jonathan Corbet --- scripts/kernel-doc | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) (limited to 'scripts') diff --git a/scripts/kernel-doc b/scripts/kernel-doc index e57c5e989a0a..70da9a3369c6 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -179,7 +179,7 @@ my ($function, %function_table, %parametertypes, $declaration_purpose); my %nosymbol_table = (); my $declaration_start_line; my ($type, $declaration_name, $return_type); -my ($newsection, $newcontents, $prototype, $brcount, %source_map); +my ($newsection, $newcontents, $prototype, $brcount); if (defined($ENV{'KBUILD_VERBOSE'}) && $ENV{'KBUILD_VERBOSE'} =~ '1') { $verbose = 1; @@ -2005,10 +2005,6 @@ sub map_filename($) { $file = $orig_file; } - if (defined($source_map{$file})) { - $file = $source_map{$file}; - } - return $file; } @@ -2403,19 +2399,6 @@ for (my $k = 0; $k < @highlights; $k++) { $dohighlight .= "\$contents =~ s:$pattern:$result:gs;\n"; } -# Read the file that maps relative names to absolute names for -# separate source and object directories and for shadow trees. -if (open(SOURCE_MAP, "<.tmp_filelist.txt")) { - my ($relname, $absname); - while() { - chop(); - ($relname, $absname) = (split())[0..1]; - $relname =~ s:^/+::; - $source_map{$relname} = $absname; - } - close(SOURCE_MAP); -} - if ($output_selection == OUTPUT_EXPORTED || $output_selection == OUTPUT_INTERNAL) { -- cgit v1.2.3 From de61d6515baece4610e401d9d7c18cac6bd77198 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 11 Feb 2025 06:57:57 +0100 Subject: docs: ABI: move README contents to the top The ABI documentation looks a little bit better if it starts with the contents of the README is placed at the beginning. Move it to the beginning of the ABI chapter. While here, improve the README text and change the title that will be shown at the html/pdf output to be coherent with both ABI file contents and with the generated documentation output. Suggested-by: Jonathan Corbet Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/20250211055809.1898623-1-mchehab+huawei@kernel.org Signed-off-by: Jonathan Corbet --- Documentation/ABI/README | 3 ++- Documentation/admin-guide/abi-readme-file.rst | 6 ------ Documentation/admin-guide/abi.rst | 3 ++- scripts/lib/abi/abi_parser.py | 2 +- 4 files changed, 5 insertions(+), 9 deletions(-) delete mode 100644 Documentation/admin-guide/abi-readme-file.rst (limited to 'scripts') diff --git a/Documentation/ABI/README b/Documentation/ABI/README index 8bac9cb09a6d..ef0e6d11e919 100644 --- a/Documentation/ABI/README +++ b/Documentation/ABI/README @@ -1,4 +1,5 @@ -This directory attempts to document the ABI between the Linux kernel and +This part of the documentation inside Documentation/ABI directory +attempts to document the ABI between the Linux kernel and userspace, and the relative stability of these interfaces. Due to the everchanging nature of Linux, and the differing maturity levels, these interfaces should be used by userspace programs in different ways. diff --git a/Documentation/admin-guide/abi-readme-file.rst b/Documentation/admin-guide/abi-readme-file.rst deleted file mode 100644 index 6172e4ccbda2..000000000000 --- a/Documentation/admin-guide/abi-readme-file.rst +++ /dev/null @@ -1,6 +0,0 @@ -.. SPDX-License-Identifier: GPL-2.0 - -ABI README -========== - -.. kernel-abi:: README diff --git a/Documentation/admin-guide/abi.rst b/Documentation/admin-guide/abi.rst index 15a2dcb1388c..c6039359e585 100644 --- a/Documentation/admin-guide/abi.rst +++ b/Documentation/admin-guide/abi.rst @@ -4,6 +4,8 @@ Linux ABI description ===================== +.. kernel-abi:: README + ABI symbols ----------- @@ -21,7 +23,6 @@ ABI files .. toctree:: :maxdepth: 2 - abi-readme-file abi-stable-files abi-testing-files abi-obsolete-files diff --git a/scripts/lib/abi/abi_parser.py b/scripts/lib/abi/abi_parser.py index f08de6d3bf7c..66a738013ce1 100644 --- a/scripts/lib/abi/abi_parser.py +++ b/scripts/lib/abi/abi_parser.py @@ -266,7 +266,7 @@ class AbiParser: def parse_readme(self, nametag, fname): """Parse ABI README file""" - nametag["what"] = ["ABI file contents"] + nametag["what"] = ["Introduction"] nametag["path"] = "README" with open(fname, "r", encoding="utf8", errors="backslashreplace") as fp: for line in fp: -- cgit v1.2.3 From 089e06c3f113a8641a6cf502a34284a7c0ca1630 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Tue, 11 Feb 2025 07:19:04 +0100 Subject: scripts/kernel-doc: drop Sphinx version check As the current minimal supported Sphinx version is 3.4.3, drop support for older versions. Signed-off-by: Mauro Carvalho Chehab Reviewed-by: Kees Cook Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/0d002e7550476a68547ee53ad06cfd8fdcaf7c3a.1739254187.git.mchehab+huawei@kernel.org --- Documentation/sphinx/cdomain.py | 3 - Documentation/sphinx/kerneldoc.py | 5 -- scripts/kernel-doc | 129 +++++--------------------------------- 3 files changed, 16 insertions(+), 121 deletions(-) (limited to 'scripts') diff --git a/Documentation/sphinx/cdomain.py b/Documentation/sphinx/cdomain.py index 6596fd00663f..e8ea80d4324c 100644 --- a/Documentation/sphinx/cdomain.py +++ b/Documentation/sphinx/cdomain.py @@ -45,9 +45,6 @@ import re __version__ = '1.1' -# Get Sphinx version -major, minor, patch = sphinx.version_info[:3] - # Namespace to be prepended to the full name namespace = None diff --git a/Documentation/sphinx/kerneldoc.py b/Documentation/sphinx/kerneldoc.py index be5b8fbf373f..39ddae6ae7dd 100644 --- a/Documentation/sphinx/kerneldoc.py +++ b/Documentation/sphinx/kerneldoc.py @@ -62,11 +62,6 @@ class KernelDocDirective(Directive): env = self.state.document.settings.env cmd = [env.config.kerneldoc_bin, '-rst', '-enable-lineno'] - # Pass the version string to kernel-doc, as it needs to use a different - # dialect, depending what the C domain supports for each specific - # Sphinx versions - cmd += ['-sphinx-version', sphinx.__version__] - filename = env.config.kerneldoc_srctree + '/' + self.arguments[0] export_file_patterns = [] diff --git a/scripts/kernel-doc b/scripts/kernel-doc index 70da9a3369c6..2c77b914d017 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -26,7 +26,7 @@ kernel-doc - Print formatted kernel documentation to stdout kernel-doc [-h] [-v] [-Werror] [-Wall] [-Wreturn] [-Wshort-desc[ription]] [-Wcontents-before-sections] [ -man | - -rst [-sphinx-version VERSION] [-enable-lineno] | + -rst [-enable-lineno] | -none ] [ @@ -130,7 +130,6 @@ if ($#ARGV == -1) { } my $kernelversion; -my ($sphinx_major, $sphinx_minor, $sphinx_patch); my $dohighlight = ""; @@ -347,23 +346,6 @@ while ($ARGV[0] =~ m/^--?(.*)/) { $enable_lineno = 1; } elsif ($cmd eq 'show-not-found') { $show_not_found = 1; # A no-op but don't fail - } elsif ($cmd eq "sphinx-version") { - my $ver_string = shift @ARGV; - if ($ver_string =~ m/^(\d+)(\.\d+)?(\.\d+)?/) { - $sphinx_major = $1; - if (defined($2)) { - $sphinx_minor = substr($2,1); - } else { - $sphinx_minor = 0; - } - if (defined($3)) { - $sphinx_patch = substr($3,1) - } else { - $sphinx_patch = 0; - } - } else { - die "Sphinx version should either major.minor or major.minor.patch format\n"; - } } else { # Unknown argument pod2usage( @@ -387,8 +369,6 @@ while ($ARGV[0] =~ m/^--?(.*)/) { # continue execution near EOF; -# The C domain dialect changed on Sphinx 3. So, we need to check the -# version in order to produce the right tags. sub findprog($) { foreach(split(/:/, $ENV{PATH})) { @@ -396,42 +376,6 @@ sub findprog($) } } -sub get_sphinx_version() -{ - my $ver; - - my $cmd = "sphinx-build"; - if (!findprog($cmd)) { - my $cmd = "sphinx-build3"; - if (!findprog($cmd)) { - $sphinx_major = 1; - $sphinx_minor = 2; - $sphinx_patch = 0; - printf STDERR "Warning: Sphinx version not found. Using default (Sphinx version %d.%d.%d)\n", - $sphinx_major, $sphinx_minor, $sphinx_patch; - return; - } - } - - open IN, "$cmd --version 2>&1 |"; - while () { - if (m/^\s*sphinx-build\s+([\d]+)\.([\d\.]+)(\+\/[\da-f]+)?$/) { - $sphinx_major = $1; - $sphinx_minor = $2; - $sphinx_patch = $3; - last; - } - # Sphinx 1.2.x uses a different format - if (m/^\s*Sphinx.*\s+([\d]+)\.([\d\.]+)$/) { - $sphinx_major = $1; - $sphinx_minor = $2; - $sphinx_patch = $3; - last; - } - } - close IN; -} - # get kernel version from env sub get_kernel_version() { my $version = 'unknown kernel version'; @@ -859,9 +803,10 @@ sub output_function_rst(%) { $signature .= ")"; } - if ($sphinx_major < 3) { + if ($args{'typedef'} || $args{'functiontype'} eq "") { + print ".. c:macro:: ". $args{'function'} . "\n\n"; + if ($args{'typedef'}) { - print ".. c:type:: ". $args{'function'} . "\n\n"; print_lineno($declaration_start_line); print " **Typedef**: "; $lineprefix = ""; @@ -869,25 +814,10 @@ sub output_function_rst(%) { print "\n\n**Syntax**\n\n"; print " ``$signature``\n\n"; } else { - print ".. c:function:: $signature\n\n"; + print "``$signature``\n\n"; } } else { - if ($args{'typedef'} || $args{'functiontype'} eq "") { - print ".. c:macro:: ". $args{'function'} . "\n\n"; - - if ($args{'typedef'}) { - print_lineno($declaration_start_line); - print " **Typedef**: "; - $lineprefix = ""; - output_highlight_rst($args{'purpose'}); - print "\n\n**Syntax**\n\n"; - print " ``$signature``\n\n"; - } else { - print "``$signature``\n\n"; - } - } else { - print ".. c:function:: $signature\n\n"; - } + print ".. c:function:: $signature\n\n"; } if (!$args{'typedef'}) { @@ -955,13 +885,9 @@ sub output_enum_rst(%) { my $count; my $outer; - if ($sphinx_major < 3) { - my $name = "enum " . $args{'enum'}; - print "\n\n.. c:type:: " . $name . "\n\n"; - } else { - my $name = $args{'enum'}; - print "\n\n.. c:enum:: " . $name . "\n\n"; - } + my $name = $args{'enum'}; + print "\n\n.. c:enum:: " . $name . "\n\n"; + print_lineno($declaration_start_line); $lineprefix = " "; output_highlight_rst($args{'purpose'}); @@ -992,11 +918,8 @@ sub output_typedef_rst(%) { my $oldprefix = $lineprefix; my $name; - if ($sphinx_major < 3) { - $name = "typedef " . $args{'typedef'}; - } else { - $name = $args{'typedef'}; - } + $name = $args{'typedef'}; + print "\n\n.. c:type:: " . $name . "\n\n"; print_lineno($declaration_start_line); $lineprefix = " "; @@ -1012,17 +935,13 @@ sub output_struct_rst(%) { my ($parameter); my $oldprefix = $lineprefix; - if ($sphinx_major < 3) { - my $name = $args{'type'} . " " . $args{'struct'}; - print "\n\n.. c:type:: " . $name . "\n\n"; + my $name = $args{'struct'}; + if ($args{'type'} eq 'union') { + print "\n\n.. c:union:: " . $name . "\n\n"; } else { - my $name = $args{'struct'}; - if ($args{'type'} eq 'union') { - print "\n\n.. c:union:: " . $name . "\n\n"; - } else { - print "\n\n.. c:struct:: " . $name . "\n\n"; - } + print "\n\n.. c:struct:: " . $name . "\n\n"; } + print_lineno($declaration_start_line); $lineprefix = " "; output_highlight_rst($args{'purpose'}); @@ -2383,11 +2302,6 @@ sub process_file($) { close IN_FILE; } - -if ($output_mode eq "rst") { - get_sphinx_version() if (!$sphinx_major); -} - $kernelversion = get_kernel_version(); # generate a sequence of code that will splice in highlighting information @@ -2454,17 +2368,6 @@ Do not output documentation, only warnings. =head3 reStructuredText only -=over 8 - -=item -sphinx-version VERSION - -Use the ReST C domain dialect compatible with a specific Sphinx Version. - -If not specified, kernel-doc will auto-detect using the sphinx-build version -found on PATH. - -=back - =head2 Output selection (mutually exclusive): =over 8 -- cgit v1.2.3 From b9609ecba35ecf7fa85d9bcd519242009174e6a2 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 24 Feb 2025 10:08:11 +0100 Subject: scripts/kernel-doc: don't add not needed new lines This helps comparing kernel-doc output with the new .py version of it. Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/6b036ef7d746f26d7d0044626b04d1f0880a2188.1740387599.git.mchehab+huawei@kernel.org Signed-off-by: Jonathan Corbet --- scripts/kernel-doc | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'scripts') diff --git a/scripts/kernel-doc b/scripts/kernel-doc index 2c77b914d017..d59552e1a31d 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -760,6 +760,10 @@ sub output_highlight_rst { if ($block) { $output .= highlight_block($block); } + + $output =~ s/^\n+//g; + $output =~ s/\n+$//g; + foreach $line (split "\n", $output) { print $lineprefix . $line . "\n"; } -- cgit v1.2.3 From 19b100b0116d703b9529f7bbbf797428de51816a Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Mon, 24 Feb 2025 10:08:12 +0100 Subject: scripts/kernel-doc: drop dead code for Wcontents_before_sections There is a warning about contents before sections, which doesn't work, since in_doc_sect variable is always true at the point it is checked. Drop the dead code. Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/174a15607fd057c736dc9123c53d0835ce20e68b.1740387599.git.mchehab+huawei@kernel.org Signed-off-by: Jonathan Corbet --- scripts/kernel-doc | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'scripts') diff --git a/scripts/kernel-doc b/scripts/kernel-doc index d59552e1a31d..af6cf408b96d 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -137,7 +137,6 @@ my $verbose = 0; my $Werror = 0; my $Wreturn = 0; my $Wshort_desc = 0; -my $Wcontents_before_sections = 0; my $output_mode = "rst"; my $output_preformatted = 0; my $no_doc_sections = 0; @@ -223,7 +222,6 @@ use constant { STATE_INLINE => 7, # gathering doc outside main block }; my $state; -my $in_doc_sect; my $leading_space; # Inline documentation state @@ -332,12 +330,9 @@ while ($ARGV[0] =~ m/^--?(.*)/) { $Wreturn = 1; } elsif ($cmd eq "Wshort-desc" or $cmd eq "Wshort-description") { $Wshort_desc = 1; - } elsif ($cmd eq "Wcontents-before-sections") { - $Wcontents_before_sections = 1; } elsif ($cmd eq "Wall") { $Wreturn = 1; $Wshort_desc = 1; - $Wcontents_before_sections = 1; } elsif (($cmd eq "h") || ($cmd eq "help")) { pod2usage(-exitval => 0, -verbose => 2); } elsif ($cmd eq 'no-doc-sections') { @@ -1963,7 +1958,6 @@ sub process_export_file($) { sub process_normal() { if (/$doc_start/o) { $state = STATE_NAME; # next line is always the function name - $in_doc_sect = 0; $declaration_start_line = $. + 1; } } @@ -2068,7 +2062,6 @@ sub process_body($$) { } if (/$doc_sect/i) { # case insensitive for supported section names - $in_doc_sect = 1; $newsection = $1; $newcontents = $2; @@ -2085,14 +2078,10 @@ sub process_body($$) { } if (($contents ne "") && ($contents ne "\n")) { - if (!$in_doc_sect && $Wcontents_before_sections) { - emit_warning("${file}:$.", "contents before sections\n"); - } dump_section($file, $section, $contents); $section = $section_default; } - $in_doc_sect = 1; $state = STATE_BODY; $contents = $newcontents; $new_start_line = $.; -- cgit v1.2.3 From a3aac126ca3a71b6612a817ef24db325618fd902 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Tue, 4 Mar 2025 08:21:29 -0800 Subject: kbuild: clang: Support building UM with SUBARCH=i386 The UM builds distinguish i386 from x86_64 via SUBARCH, but we don't support building i386 directly with Clang. To make SUBARCH work for i386 UM, we need to explicitly test for it. This lets me run i386 KUnit tests with Clang: $ ./tools/testing/kunit/kunit.py run \ --make_options LLVM=1 \ --make_options SUBARCH=i386 ... Fixes: c7500c1b53bf ("um: Allow builds with Clang") Reviewed-by: Nathan Chancellor Link: https://lore.kernel.org/r/20250304162124.it.785-kees@kernel.org Tested-by: David Gow Signed-off-by: Kees Cook --- scripts/Makefile.clang | 2 ++ 1 file changed, 2 insertions(+) (limited to 'scripts') diff --git a/scripts/Makefile.clang b/scripts/Makefile.clang index 2435efae67f5..b67636b28c35 100644 --- a/scripts/Makefile.clang +++ b/scripts/Makefile.clang @@ -12,6 +12,8 @@ CLANG_TARGET_FLAGS_riscv := riscv64-linux-gnu CLANG_TARGET_FLAGS_s390 := s390x-linux-gnu CLANG_TARGET_FLAGS_sparc := sparc64-linux-gnu CLANG_TARGET_FLAGS_x86 := x86_64-linux-gnu +# This is only for i386 UM builds, which need the 32-bit target not -m32 +CLANG_TARGET_FLAGS_i386 := i386-linux-gnu CLANG_TARGET_FLAGS_um := $(CLANG_TARGET_FLAGS_$(SUBARCH)) CLANG_TARGET_FLAGS := $(CLANG_TARGET_FLAGS_$(SRCARCH)) -- cgit v1.2.3 From ed2b548f1017586c44f50654ef9febb42d491f31 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 6 Mar 2025 20:19:09 -0800 Subject: ubsan/overflow: Rework integer overflow sanitizer option to turn on everything Since we're going to approach integer overflow mitigation a type at a time, we need to enable all of the associated sanitizers, and then opt into types one at a time. Rename the existing "signed wrap" sanitizer to just the entire topic area: "integer wrap". Enable the implicit integer truncation sanitizers, with required callbacks and tests. Notably, this requires features (currently) only available in Clang, so we can depend on the cc-option tests to determine availability instead of doing version tests. Link: https://lore.kernel.org/r/20250307041914.937329-1-kees@kernel.org Signed-off-by: Kees Cook --- include/linux/compiler_types.h | 2 +- kernel/configs/hardening.config | 2 +- lib/Kconfig.ubsan | 23 +++++++++++------------ lib/test_ubsan.c | 18 ++++++++++++++---- lib/ubsan.c | 28 ++++++++++++++++++++++++++-- lib/ubsan.h | 8 ++++++++ scripts/Makefile.lib | 4 ++-- scripts/Makefile.ubsan | 8 ++++++-- 8 files changed, 69 insertions(+), 24 deletions(-) (limited to 'scripts') diff --git a/include/linux/compiler_types.h b/include/linux/compiler_types.h index f59393464ea7..4ad3e900bc3d 100644 --- a/include/linux/compiler_types.h +++ b/include/linux/compiler_types.h @@ -360,7 +360,7 @@ struct ftrace_likely_data { #endif /* Do not trap wrapping arithmetic within an annotated function. */ -#ifdef CONFIG_UBSAN_SIGNED_WRAP +#ifdef CONFIG_UBSAN_INTEGER_WRAP # define __signed_wrap __attribute__((no_sanitize("signed-integer-overflow"))) #else # define __signed_wrap diff --git a/kernel/configs/hardening.config b/kernel/configs/hardening.config index 3fabb8f55ef6..dd7c32fb5ac1 100644 --- a/kernel/configs/hardening.config +++ b/kernel/configs/hardening.config @@ -46,7 +46,7 @@ CONFIG_UBSAN_BOUNDS=y # CONFIG_UBSAN_SHIFT is not set # CONFIG_UBSAN_DIV_ZERO is not set # CONFIG_UBSAN_UNREACHABLE is not set -# CONFIG_UBSAN_SIGNED_WRAP is not set +# CONFIG_UBSAN_INTEGER_WRAP is not set # CONFIG_UBSAN_BOOL is not set # CONFIG_UBSAN_ENUM is not set # CONFIG_UBSAN_ALIGNMENT is not set diff --git a/lib/Kconfig.ubsan b/lib/Kconfig.ubsan index 1d4aa7a83b3a..63e5622010e0 100644 --- a/lib/Kconfig.ubsan +++ b/lib/Kconfig.ubsan @@ -116,21 +116,20 @@ config UBSAN_UNREACHABLE This option enables -fsanitize=unreachable which checks for control flow reaching an expected-to-be-unreachable position. -config UBSAN_SIGNED_WRAP - bool "Perform checking for signed arithmetic wrap-around" +config UBSAN_INTEGER_WRAP + bool "Perform checking for integer arithmetic wrap-around" default UBSAN depends on !COMPILE_TEST - # The no_sanitize attribute was introduced in GCC with version 8. - depends on !CC_IS_GCC || GCC_VERSION >= 80000 depends on $(cc-option,-fsanitize=signed-integer-overflow) - help - This option enables -fsanitize=signed-integer-overflow which checks - for wrap-around of any arithmetic operations with signed integers. - This currently performs nearly no instrumentation due to the - kernel's use of -fno-strict-overflow which converts all would-be - arithmetic undefined behavior into wrap-around arithmetic. Future - sanitizer versions will allow for wrap-around checking (rather than - exclusively undefined behavior). + depends on $(cc-option,-fsanitize=unsigned-integer-overflow) + depends on $(cc-option,-fsanitize=implicit-signed-integer-truncation) + depends on $(cc-option,-fsanitize=implicit-unsigned-integer-truncation) + help + This option enables all of the sanitizers involved in integer overflow + (wrap-around) mitigation: signed-integer-overflow, unsigned-integer-overflow, + implicit-signed-integer-truncation, and implicit-unsigned-integer-truncation. + This is currently limited only to the size_t type while testing and + compiler development continues. config UBSAN_BOOL bool "Perform checking for non-boolean values used as boolean" diff --git a/lib/test_ubsan.c b/lib/test_ubsan.c index 5d7b10e98610..8772e5edaa4f 100644 --- a/lib/test_ubsan.c +++ b/lib/test_ubsan.c @@ -15,7 +15,7 @@ static void test_ubsan_add_overflow(void) { volatile int val = INT_MAX; - UBSAN_TEST(CONFIG_UBSAN_SIGNED_WRAP); + UBSAN_TEST(CONFIG_UBSAN_INTEGER_WRAP); val += 2; } @@ -24,7 +24,7 @@ static void test_ubsan_sub_overflow(void) volatile int val = INT_MIN; volatile int val2 = 2; - UBSAN_TEST(CONFIG_UBSAN_SIGNED_WRAP); + UBSAN_TEST(CONFIG_UBSAN_INTEGER_WRAP); val -= val2; } @@ -32,7 +32,7 @@ static void test_ubsan_mul_overflow(void) { volatile int val = INT_MAX / 2; - UBSAN_TEST(CONFIG_UBSAN_SIGNED_WRAP); + UBSAN_TEST(CONFIG_UBSAN_INTEGER_WRAP); val *= 3; } @@ -40,7 +40,7 @@ static void test_ubsan_negate_overflow(void) { volatile int val = INT_MIN; - UBSAN_TEST(CONFIG_UBSAN_SIGNED_WRAP); + UBSAN_TEST(CONFIG_UBSAN_INTEGER_WRAP); val = -val; } @@ -53,6 +53,15 @@ static void test_ubsan_divrem_overflow(void) val /= val2; } +static void test_ubsan_truncate_signed(void) +{ + volatile long val = LONG_MAX; + volatile int val2 = 0; + + UBSAN_TEST(CONFIG_UBSAN_INTEGER_WRAP); + val2 = val; +} + static void test_ubsan_shift_out_of_bounds(void) { volatile int neg = -1, wrap = 4; @@ -127,6 +136,7 @@ static const test_ubsan_fp test_ubsan_array[] = { test_ubsan_sub_overflow, test_ubsan_mul_overflow, test_ubsan_negate_overflow, + test_ubsan_truncate_signed, test_ubsan_shift_out_of_bounds, test_ubsan_out_of_bounds, test_ubsan_load_invalid_value, diff --git a/lib/ubsan.c b/lib/ubsan.c index a1c983d148f1..cdc1d31c3821 100644 --- a/lib/ubsan.c +++ b/lib/ubsan.c @@ -44,7 +44,7 @@ const char *report_ubsan_failure(struct pt_regs *regs, u32 check_type) case ubsan_shift_out_of_bounds: return "UBSAN: shift out of bounds"; #endif -#if defined(CONFIG_UBSAN_DIV_ZERO) || defined(CONFIG_UBSAN_SIGNED_WRAP) +#if defined(CONFIG_UBSAN_DIV_ZERO) || defined(CONFIG_UBSAN_INTEGER_WRAP) /* * SanitizerKind::IntegerDivideByZero and * SanitizerKind::SignedIntegerOverflow emit @@ -79,7 +79,7 @@ const char *report_ubsan_failure(struct pt_regs *regs, u32 check_type) case ubsan_type_mismatch: return "UBSAN: type mismatch"; #endif -#ifdef CONFIG_UBSAN_SIGNED_WRAP +#ifdef CONFIG_UBSAN_INTEGER_WRAP /* * SanitizerKind::SignedIntegerOverflow emits * SanitizerHandler::AddOverflow, SanitizerHandler::SubOverflow, @@ -303,6 +303,30 @@ void __ubsan_handle_negate_overflow(void *_data, void *old_val) } EXPORT_SYMBOL(__ubsan_handle_negate_overflow); +void __ubsan_handle_implicit_conversion(void *_data, void *from_val, void *to_val) +{ + struct implicit_conversion_data *data = _data; + char from_val_str[VALUE_LENGTH]; + char to_val_str[VALUE_LENGTH]; + + if (suppress_report(&data->location)) + return; + + val_to_string(from_val_str, sizeof(from_val_str), data->from_type, from_val); + val_to_string(to_val_str, sizeof(to_val_str), data->to_type, to_val); + + ubsan_prologue(&data->location, "implicit-conversion"); + + pr_err("cannot represent %s value %s during %s %s, truncated to %s\n", + data->from_type->type_name, + from_val_str, + type_check_kinds[data->type_check_kind], + data->to_type->type_name, + to_val_str); + + ubsan_epilogue(); +} +EXPORT_SYMBOL(__ubsan_handle_implicit_conversion); void __ubsan_handle_divrem_overflow(void *_data, void *lhs, void *rhs) { diff --git a/lib/ubsan.h b/lib/ubsan.h index 07e37d4429b4..b37e22374e77 100644 --- a/lib/ubsan.h +++ b/lib/ubsan.h @@ -62,6 +62,13 @@ struct overflow_data { struct type_descriptor *type; }; +struct implicit_conversion_data { + struct source_location location; + struct type_descriptor *from_type; + struct type_descriptor *to_type; + unsigned char type_check_kind; +}; + struct type_mismatch_data { struct source_location location; struct type_descriptor *type; @@ -142,6 +149,7 @@ void ubsan_linkage __ubsan_handle_sub_overflow(void *data, void *lhs, void *rhs) void ubsan_linkage __ubsan_handle_mul_overflow(void *data, void *lhs, void *rhs); void ubsan_linkage __ubsan_handle_negate_overflow(void *_data, void *old_val); void ubsan_linkage __ubsan_handle_divrem_overflow(void *_data, void *lhs, void *rhs); +void ubsan_linkage __ubsan_handle_implicit_conversion(void *_data, void *lhs, void *rhs); void ubsan_linkage __ubsan_handle_type_mismatch(struct type_mismatch_data *data, void *ptr); void ubsan_linkage __ubsan_handle_type_mismatch_v1(void *_data, void *ptr); void ubsan_linkage __ubsan_handle_out_of_bounds(void *_data, void *index); diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index cad20f0e66ee..981d14ef9db2 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -166,8 +166,8 @@ _c_flags += $(if $(patsubst n%,, \ $(UBSAN_SANITIZE_$(target-stem).o)$(UBSAN_SANITIZE)$(is-kernel-object)), \ $(CFLAGS_UBSAN)) _c_flags += $(if $(patsubst n%,, \ - $(UBSAN_SIGNED_WRAP_$(target-stem).o)$(UBSAN_SANITIZE_$(target-stem).o)$(UBSAN_SIGNED_WRAP)$(UBSAN_SANITIZE)$(is-kernel-object)), \ - $(CFLAGS_UBSAN_SIGNED_WRAP)) + $(UBSAN_INTEGER_WRAP_$(target-stem).o)$(UBSAN_SANITIZE_$(target-stem).o)$(UBSAN_INTEGER_WRAP)$(UBSAN_SANITIZE)$(is-kernel-object)), \ + $(CFLAGS_UBSAN_INTEGER_WRAP)) endif ifeq ($(CONFIG_KCOV),y) diff --git a/scripts/Makefile.ubsan b/scripts/Makefile.ubsan index b2d3b273b802..4fad9afed24c 100644 --- a/scripts/Makefile.ubsan +++ b/scripts/Makefile.ubsan @@ -14,5 +14,9 @@ ubsan-cflags-$(CONFIG_UBSAN_TRAP) += $(call cc-option,-fsanitize-trap=undefined export CFLAGS_UBSAN := $(ubsan-cflags-y) -ubsan-signed-wrap-cflags-$(CONFIG_UBSAN_SIGNED_WRAP) += -fsanitize=signed-integer-overflow -export CFLAGS_UBSAN_SIGNED_WRAP := $(ubsan-signed-wrap-cflags-y) +ubsan-integer-wrap-cflags-$(CONFIG_UBSAN_INTEGER_WRAP) += \ + -fsanitize=signed-integer-overflow \ + -fsanitize=unsigned-integer-overflow \ + -fsanitize=implicit-signed-integer-truncation \ + -fsanitize=implicit-unsigned-integer-truncation +export CFLAGS_UBSAN_INTEGER_WRAP := $(ubsan-integer-wrap-cflags-y) -- cgit v1.2.3 From 272a767063a6856cd1e18bb951d2be4f047b9858 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 6 Mar 2025 20:19:10 -0800 Subject: ubsan/overflow: Enable pattern exclusions To make integer wrap-around mitigation actually useful, the associated sanitizers must not instrument cases where the wrap-around is explicitly defined (e.g. "-2UL"), being tested for (e.g. "if (a + b < a)"), or where it has no impact on code flow (e.g. "while (var--)"). Enable pattern exclusions for the integer wrap sanitizers. Reviewed-by: Justin Stitt Link: https://lore.kernel.org/r/20250307041914.937329-2-kees@kernel.org Signed-off-by: Kees Cook --- lib/Kconfig.ubsan | 1 + scripts/Makefile.ubsan | 1 + 2 files changed, 2 insertions(+) (limited to 'scripts') diff --git a/lib/Kconfig.ubsan b/lib/Kconfig.ubsan index 63e5622010e0..888c2e72c586 100644 --- a/lib/Kconfig.ubsan +++ b/lib/Kconfig.ubsan @@ -120,6 +120,7 @@ config UBSAN_INTEGER_WRAP bool "Perform checking for integer arithmetic wrap-around" default UBSAN depends on !COMPILE_TEST + depends on $(cc-option,-fsanitize-undefined-ignore-overflow-pattern=all) depends on $(cc-option,-fsanitize=signed-integer-overflow) depends on $(cc-option,-fsanitize=unsigned-integer-overflow) depends on $(cc-option,-fsanitize=implicit-signed-integer-truncation) diff --git a/scripts/Makefile.ubsan b/scripts/Makefile.ubsan index 4fad9afed24c..233379c193a7 100644 --- a/scripts/Makefile.ubsan +++ b/scripts/Makefile.ubsan @@ -15,6 +15,7 @@ ubsan-cflags-$(CONFIG_UBSAN_TRAP) += $(call cc-option,-fsanitize-trap=undefined export CFLAGS_UBSAN := $(ubsan-cflags-y) ubsan-integer-wrap-cflags-$(CONFIG_UBSAN_INTEGER_WRAP) += \ + -fsanitize-undefined-ignore-overflow-pattern=all \ -fsanitize=signed-integer-overflow \ -fsanitize=unsigned-integer-overflow \ -fsanitize=implicit-signed-integer-truncation \ -- cgit v1.2.3 From 47f4af43e7c0cf702d6a6321542f0c0d9c4216e3 Mon Sep 17 00:00:00 2001 From: Kees Cook Date: Thu, 6 Mar 2025 20:19:11 -0800 Subject: ubsan/overflow: Enable ignorelist parsing and add type filter Limit integer wrap-around mitigation to only the "size_t" type (for now). Notably this covers all special functions/builtins that return "size_t", like sizeof(). This remains an experimental feature and is likely to be replaced with type annotations. Reviewed-by: Justin Stitt Link: https://lore.kernel.org/r/20250307041914.937329-3-kees@kernel.org Signed-off-by: Kees Cook --- lib/Kconfig.ubsan | 1 + scripts/Makefile.ubsan | 3 ++- scripts/integer-wrap-ignore.scl | 3 +++ 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 scripts/integer-wrap-ignore.scl (limited to 'scripts') diff --git a/lib/Kconfig.ubsan b/lib/Kconfig.ubsan index 888c2e72c586..4216b3a4ff21 100644 --- a/lib/Kconfig.ubsan +++ b/lib/Kconfig.ubsan @@ -125,6 +125,7 @@ config UBSAN_INTEGER_WRAP depends on $(cc-option,-fsanitize=unsigned-integer-overflow) depends on $(cc-option,-fsanitize=implicit-signed-integer-truncation) depends on $(cc-option,-fsanitize=implicit-unsigned-integer-truncation) + depends on $(cc-option,-fsanitize-ignorelist=/dev/null) help This option enables all of the sanitizers involved in integer overflow (wrap-around) mitigation: signed-integer-overflow, unsigned-integer-overflow, diff --git a/scripts/Makefile.ubsan b/scripts/Makefile.ubsan index 233379c193a7..9e35198edbf0 100644 --- a/scripts/Makefile.ubsan +++ b/scripts/Makefile.ubsan @@ -19,5 +19,6 @@ ubsan-integer-wrap-cflags-$(CONFIG_UBSAN_INTEGER_WRAP) += \ -fsanitize=signed-integer-overflow \ -fsanitize=unsigned-integer-overflow \ -fsanitize=implicit-signed-integer-truncation \ - -fsanitize=implicit-unsigned-integer-truncation + -fsanitize=implicit-unsigned-integer-truncation \ + -fsanitize-ignorelist=$(srctree)/scripts/integer-wrap-ignore.scl export CFLAGS_UBSAN_INTEGER_WRAP := $(ubsan-integer-wrap-cflags-y) diff --git a/scripts/integer-wrap-ignore.scl b/scripts/integer-wrap-ignore.scl new file mode 100644 index 000000000000..431c3053a4a2 --- /dev/null +++ b/scripts/integer-wrap-ignore.scl @@ -0,0 +1,3 @@ +[{unsigned-integer-overflow,signed-integer-overflow,implicit-signed-integer-truncation,implicit-unsigned-integer-truncation}] +type:* +type:size_t=sanitize -- cgit v1.2.3 From 78da89c6398335c3de7c09e319d8e82126f18126 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Wed, 12 Mar 2025 16:52:19 +0100 Subject: scripts: get_feat.pl: substitute s390x with s390 Both get_feat.pl and list-arch.sh use uname -m to get the machine hardware name to figure out the current architecture if no architecture is specified with a command line option. This doesn't work for s390, since for 64 bit kernels the hardware name is s390x, while the architecture name within the kernel, as well as in all feature files is s390. Therefore substitute s390x with s390 similar to what is already done for x86_64 and i386. Signed-off-by: Heiko Carstens Signed-off-by: Jonathan Corbet Link: https://lore.kernel.org/r/20250312155219.3597768-1-hca@linux.ibm.com --- Documentation/features/list-arch.sh | 2 +- scripts/get_feat.pl | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'scripts') diff --git a/Documentation/features/list-arch.sh b/Documentation/features/list-arch.sh index e73aa35848f0..ac8ff7f6f859 100755 --- a/Documentation/features/list-arch.sh +++ b/Documentation/features/list-arch.sh @@ -6,6 +6,6 @@ # (If no arguments are given then it will print the host architecture's status.) # -ARCH=${1:-$(uname -m | sed 's/x86_64/x86/' | sed 's/i386/x86/')} +ARCH=${1:-$(uname -m | sed 's/x86_64/x86/' | sed 's/i386/x86/' | sed 's/s390x/s390/')} $(dirname $0)/../../scripts/get_feat.pl list --arch $ARCH diff --git a/scripts/get_feat.pl b/scripts/get_feat.pl index 5c5397eeb237..40fb28c8424e 100755 --- a/scripts/get_feat.pl +++ b/scripts/get_feat.pl @@ -512,13 +512,13 @@ print STDERR Data::Dumper->Dump([\%data], [qw(*data)]) if ($debug); # Handles the command # if ($cmd eq "current") { - $arch = qx(uname -m | sed 's/x86_64/x86/' | sed 's/i386/x86/'); + $arch = qx(uname -m | sed 's/x86_64/x86/' | sed 's/i386/x86/' | sed 's/s390x/s390/'); $arch =~s/\s+$//; } if ($cmd eq "ls" or $cmd eq "list") { if (!$arch) { - $arch = qx(uname -m | sed 's/x86_64/x86/' | sed 's/i386/x86/'); + $arch = qx(uname -m | sed 's/x86_64/x86/' | sed 's/i386/x86/' | sed 's/s390x/s390/'); $arch =~s/\s+$//; } -- cgit v1.2.3 From 36799069b48198e5ce92d99310060c4aecb4b3e3 Mon Sep 17 00:00:00 2001 From: Josh Poimboeuf Date: Fri, 14 Mar 2025 12:29:11 -0700 Subject: objtool: Add CONFIG_OBJTOOL_WERROR Objtool warnings can be indicative of crashes, broken live patching, or even boot failures. Ignoring them is not recommended. Add CONFIG_OBJTOOL_WERROR to upgrade objtool warnings to errors by enabling the objtool --Werror option. Also set --backtrace to print the branches leading up to the warning, which can help considerably when debugging certain warnings. To avoid breaking bots too badly for now, make it the default for real world builds only (!COMPILE_TEST). Co-developed-by: Brendan Jackman Signed-off-by: Josh Poimboeuf Signed-off-by: Peter Zijlstra (Intel) Link: https://lore.kernel.org/r/3e7c109313ff15da6c80788965cc7450115b0196.1741975349.git.jpoimboe@kernel.org --- lib/Kconfig.debug | 11 +++++++++++ scripts/Makefile.lib | 1 + 2 files changed, 12 insertions(+) (limited to 'scripts') diff --git a/lib/Kconfig.debug b/lib/Kconfig.debug index 35796c290ca3..a9709a6db30f 100644 --- a/lib/Kconfig.debug +++ b/lib/Kconfig.debug @@ -545,6 +545,17 @@ config FRAME_POINTER config OBJTOOL bool +config OBJTOOL_WERROR + bool "Upgrade objtool warnings to errors" + depends on OBJTOOL && !COMPILE_TEST + help + Fail the build on objtool warnings. + + Objtool warnings can indicate kernel instability, including boot + failures. This option is highly recommended. + + If unsure, say Y. + config STACK_VALIDATION bool "Compile-time stack metadata validation" depends on HAVE_STACK_VALIDATION && UNWINDER_FRAME_POINTER diff --git a/scripts/Makefile.lib b/scripts/Makefile.lib index cad20f0e66ee..99e281966ba3 100644 --- a/scripts/Makefile.lib +++ b/scripts/Makefile.lib @@ -277,6 +277,7 @@ objtool-args-$(CONFIG_HAVE_STATIC_CALL_INLINE) += --static-call objtool-args-$(CONFIG_HAVE_UACCESS_VALIDATION) += --uaccess objtool-args-$(CONFIG_GCOV_KERNEL) += --no-unreachable objtool-args-$(CONFIG_PREFIX_SYMBOLS) += --prefix=$(CONFIG_FUNCTION_PADDING_BYTES) +objtool-args-$(CONFIG_OBJTOOL_WERROR) += --Werror --backtrace objtool-args = $(objtool-args-y) \ $(if $(delay-objtool), --link) \ -- cgit v1.2.3