From b2d231f4a77800661b3fb812d997841a548c6526 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Wed, 18 Mar 2026 10:11:02 +0100 Subject: docs: kdoc_re: better represent long regular expressions The Sphinx output from autodoc doesn't automatically break long lines, except on spaces. Change KernRe __repr__() to break the pattern on multiple strings, each one with a maximum limit of 60 characters. With that, documentation output for KernRe should now be displayable, even on long strings. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Message-ID: <60c264a9d277fed655b1a62df2195562c8596090.1773823995.git.mchehab+huawei@kernel.org> --- tools/lib/python/kdoc/kdoc_re.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'tools/lib/python') diff --git a/tools/lib/python/kdoc/kdoc_re.py b/tools/lib/python/kdoc/kdoc_re.py index 6f3ae28859ea..28292efe25a2 100644 --- a/tools/lib/python/kdoc/kdoc_re.py +++ b/tools/lib/python/kdoc/kdoc_re.py @@ -70,10 +70,15 @@ class KernRe: flags_name = " | ".join(flags) + max_len = 60 + pattern = "" + for pos in range(0, len(self.regex.pattern), max_len): + pattern += '"' + self.regex.pattern[pos:max_len + pos] + '" ' + if flags_name: - return f'KernRe("{self.regex.pattern}", {flags_name})' + return f'KernRe({pattern}, {flags_name})' else: - return f'KernRe("{self.regex.pattern}")' + return f'KernRe({pattern})' def __add__(self, other): """ -- cgit v1.2.3 From 8c0b7c0d3c0e640b3ebb7f1f648ea322e56c227a Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Wed, 18 Mar 2026 10:11:03 +0100 Subject: docs: kdoc: add c_lex to generated documentation Do some fixes at groups() description for it to be parsed by Sphinx and add it to the documentation. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Message-ID: <799178cf30dd4022fdb1d029ba998a458e037b52.1773823995.git.mchehab+huawei@kernel.org> --- Documentation/tools/kdoc_ancillary.rst | 9 +++++++++ tools/lib/python/kdoc/c_lex.py | 9 +++++---- 2 files changed, 14 insertions(+), 4 deletions(-) (limited to 'tools/lib/python') diff --git a/Documentation/tools/kdoc_ancillary.rst b/Documentation/tools/kdoc_ancillary.rst index 3950d0a3f104..85f3806a431a 100644 --- a/Documentation/tools/kdoc_ancillary.rst +++ b/Documentation/tools/kdoc_ancillary.rst @@ -21,6 +21,15 @@ Regular expression class handler :undoc-members: +C tokenizer +=========== + +.. automodule:: lib.python.kdoc.c_lex + :members: + :show-inheritance: + :undoc-members: + + Chinese, Japanese and Korean variable fonts handler =================================================== diff --git a/tools/lib/python/kdoc/c_lex.py b/tools/lib/python/kdoc/c_lex.py index b6d58bd470a9..e01b154f458e 100644 --- a/tools/lib/python/kdoc/c_lex.py +++ b/tools/lib/python/kdoc/c_lex.py @@ -336,13 +336,14 @@ class CTokenArgs: self.sub_tokeninzer = CTokenizer(sub_str) def groups(self, new_tokenizer): - """ + r""" Create replacement arguments for backrefs like: - ``\0``, ``\1``, ``\2``, ...``\n`` + ``\0``, ``\1``, ``\2``, ... ``\{number}`` - It also accepts a ``+`` character to the highest backref. When used, - it means in practice to ignore delimins after it, being greedy. + It also accepts a ``+`` character to the highest backref, like + ``\4+``. When used, the backref will be greedy, picking all other + arguments afterwards. The logic is smart enough to only go up to the maximum required argument, even if there are more. -- cgit v1.2.3 From e0ebee442d56c11df023b7c2d32edc3b0765b2f3 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Wed, 18 Mar 2026 10:11:04 +0100 Subject: docs: kdoc_files: use a class to group config parameters Instead of abusing argparse.Namespace, define a class to store configuration parameters and logger. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Message-ID: --- tools/lib/python/kdoc/kdoc_files.py | 45 +++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 14 deletions(-) (limited to 'tools/lib/python') diff --git a/tools/lib/python/kdoc/kdoc_files.py b/tools/lib/python/kdoc/kdoc_files.py index 8c2059623949..1c5cb9e5f0e8 100644 --- a/tools/lib/python/kdoc/kdoc_files.py +++ b/tools/lib/python/kdoc/kdoc_files.py @@ -9,7 +9,6 @@ Classes for navigating through the files that kernel-doc needs to handle to generate documentation. """ -import argparse import logging import os import re @@ -87,6 +86,28 @@ class GlobSourceFiles: file_not_found_cb(fname) +class KdocConfig(): + """ + Stores all configuration attributes that kdoc_parser and kdoc_output + needs. + """ + def __init__(self, verbose=False, werror=False, wreturn=False, + wshort_desc=False, wcontents_before_sections=False, + logger=None): + + self.verbose = verbose + self.werror = werror + self.wreturn = wreturn + self.wshort_desc = wshort_desc + self.wcontents_before_sections = wcontents_before_sections + + if logger: + self.log = logger + else: + self.log = logging.getLogger(__file__) + + self.warning = self.log.warning + class KernelFiles(): """ Parse kernel-doc tags on multiple kernel source files. @@ -224,29 +245,25 @@ class KernelFiles(): if kdoc_werror: werror = kdoc_werror + if not logger: + logger = logging.getLogger("kernel-doc") + else: + logger = logger + # Some variables are global to the parser logic as a whole as they are # used to send control configuration to KernelDoc class. As such, # those variables are read-only inside the KernelDoc. - self.config = argparse.Namespace + self.config = KdocConfig(verbose, werror, wreturn, wshort_desc, + wcontents_before_sections, logger) - self.config.verbose = verbose - self.config.werror = werror - self.config.wreturn = wreturn - self.config.wshort_desc = wshort_desc - self.config.wcontents_before_sections = wcontents_before_sections + # Override log warning, as we want to count errors + self.config.warning = self.warning if xforms: self.xforms = xforms else: self.xforms = CTransforms() - if not logger: - self.config.log = logging.getLogger("kernel-doc") - else: - self.config.log = logger - - self.config.warning = self.warning - self.config.src_tree = os.environ.get("SRCTREE", None) # Initialize variables that are internal to KernelFiles -- cgit v1.2.3 From 9ab2ca3dd127194a55bd9789c031e800fd19c254 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Wed, 18 Mar 2026 10:11:05 +0100 Subject: docs: kdoc_files: move output symbols logic to kdoc_output When writing unittests for kdoc_output, it became clear that the logic with handles a series of KdocItem symbols from a single file belons to kdoc_output, and not to kdoc_files. Move the code to it. While here, also ensure that self.config will be placed together with set.out_style. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Message-ID: <4ebc26e37a0b544c50d50b8077760f147fa6a535.1773823995.git.mchehab+huawei@kernel.org> --- tools/lib/python/kdoc/kdoc_files.py | 18 ++---------------- tools/lib/python/kdoc/kdoc_output.py | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 16 deletions(-) (limited to 'tools/lib/python') diff --git a/tools/lib/python/kdoc/kdoc_files.py b/tools/lib/python/kdoc/kdoc_files.py index 1c5cb9e5f0e8..58f4ee08e226 100644 --- a/tools/lib/python/kdoc/kdoc_files.py +++ b/tools/lib/python/kdoc/kdoc_files.py @@ -269,6 +269,7 @@ class KernelFiles(): # Initialize variables that are internal to KernelFiles self.out_style = out_style + self.out_style.set_config(self.config) self.errors = 0 self.results = {} @@ -311,8 +312,6 @@ class KernelFiles(): returning kernel-doc markups on each interaction. """ - self.out_style.set_config(self.config) - if not filenames: filenames = sorted(self.results.keys()) @@ -336,25 +335,12 @@ class KernelFiles(): function_table, enable_lineno, no_doc_sections) - msg = "" if fname not in self.results: self.config.log.warning("No kernel-doc for file %s", fname) continue symbols = self.results[fname] - self.out_style.set_symbols(symbols) - - for arg in symbols: - m = self.out_msg(fname, arg.name, arg) - - if m is None: - ln = arg.get("ln", 0) - dtype = arg.get('type', "") - - self.config.log.warning("%s:%d Can't handle %s", - fname, ln, dtype) - else: - msg += m + msg = self.out_style.output_symbols(fname, symbols) if msg: yield fname, msg diff --git a/tools/lib/python/kdoc/kdoc_output.py b/tools/lib/python/kdoc/kdoc_output.py index 08539dd92cbb..73d71cbeabb5 100644 --- a/tools/lib/python/kdoc/kdoc_output.py +++ b/tools/lib/python/kdoc/kdoc_output.py @@ -222,6 +222,27 @@ class OutputFormat: return None + def output_symbols(self, fname, symbols): + """ + Handles a set of KdocItem symbols. + """ + self.set_symbols(symbols) + + msg = "" + for arg in symbols: + m = self.msg(fname, arg.name, arg) + + if m is None: + ln = arg.get("ln", 0) + dtype = arg.get('type', "") + + self.config.log.warning("%s:%d Can't handle %s", + fname, ln, dtype) + else: + msg += m + + return msg + # Virtual methods to be overridden by inherited classes # At the base class, those do nothing. def set_symbols(self, symbols): -- cgit v1.2.3 From 01c41b99c66ff26a102edbc4f9dff9c74692723e Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Wed, 18 Mar 2026 10:11:06 +0100 Subject: docs: kdoc_item: fix initial value for parameterdesc_start_lines Ensure that parameterdesc_start_lines is a dict at init time, as this is how it will be set later on at the parser. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Message-ID: <1b4ea24dd4cd82e6711e9be80168684427d74c30.1773823995.git.mchehab+huawei@kernel.org> --- tools/lib/python/kdoc/kdoc_item.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tools/lib/python') diff --git a/tools/lib/python/kdoc/kdoc_item.py b/tools/lib/python/kdoc/kdoc_item.py index 2b8a93f79716..c0585cdbcbd1 100644 --- a/tools/lib/python/kdoc/kdoc_item.py +++ b/tools/lib/python/kdoc/kdoc_item.py @@ -22,7 +22,7 @@ class KdocItem: self.sections = {} self.sections_start_lines = {} self.parameterlist = [] - self.parameterdesc_start_lines = [] + self.parameterdesc_start_lines = {} self.parameterdescs = {} self.parametertypes = {} # -- cgit v1.2.3 From 99364ba7f8dca5c1c2d08fe37c5835b86be141f4 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Wed, 18 Mar 2026 10:11:07 +0100 Subject: docs: kdoc_item: add support to generate a KdocItem from a dict When reading the contents on a KdocItem using YAML, the data will be imported into a dict. Add a method to create a new KdocItem from a dict to allow converting such input into a real KdocItem. While here, address an issue that, if the class is initialized with an internal parameter outside the 4 initial arguments, it would end being added inside other_stuff, which breaks initializing it from a dict. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Message-ID: --- tools/lib/python/kdoc/kdoc_item.py | 35 ++++++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) (limited to 'tools/lib/python') diff --git a/tools/lib/python/kdoc/kdoc_item.py b/tools/lib/python/kdoc/kdoc_item.py index c0585cdbcbd1..5f41790efacb 100644 --- a/tools/lib/python/kdoc/kdoc_item.py +++ b/tools/lib/python/kdoc/kdoc_item.py @@ -25,12 +25,31 @@ class KdocItem: self.parameterdesc_start_lines = {} self.parameterdescs = {} self.parametertypes = {} + + self.warnings = [] + # # Just save everything else into our own dict so that the output # side can grab it directly as before. As we move things into more # structured data, this will, hopefully, fade away. # - self.other_stuff = other_stuff + known_keys = { + 'declaration_start_line', + 'sections', + 'sections_start_lines', + 'parameterlist', + 'parameterdesc_start_lines', + 'parameterdescs', + 'parametertypes', + 'warnings', + } + + self.other_stuff = {} + for k, v in other_stuff.items(): + if k in known_keys: + setattr(self, k, v) # real attribute + else: + self.other_stuff[k] = v def get(self, key, default = None): """ @@ -41,6 +60,20 @@ class KdocItem: def __getitem__(self, key): return self.get(key) + @classmethod + def from_dict(cls, d): + """Create a KdocItem from a plain dict.""" + + cp = d.copy() + name = cp.pop('name', None) + fname = cp.pop('fname', None) + type = cp.pop('type', None) + start_line = cp.pop('start_line', 1) + other_stuff = cp.pop('other_stuff', {}) + + # Everything that’s left goes straight to __init__ + return cls(name, fname, type, start_line, **cp, **other_stuff) + # # Tracking of section and parameter information. # -- cgit v1.2.3 From e394855fcc897f73f23c364a3a596b54cc879e4c Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Wed, 18 Mar 2026 10:11:08 +0100 Subject: docs: kdoc_item: fix a typo on sections_start_lines Currently, there are 15 occurrences of section?_start_lines, with 10 using the plural way. This is an issue, as, while kdoc_output works with KdocItem, the term doesn't match its init value. The variable sections_start_lines stores multiple sections, so placing it in plural is its correct way. So, ensure that, on all parts of kdoc, this will be referred as sections_start_lines. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Message-ID: --- tools/lib/python/kdoc/kdoc_item.py | 2 +- tools/lib/python/kdoc/kdoc_output.py | 2 +- tools/lib/python/kdoc/kdoc_parser.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) (limited to 'tools/lib/python') diff --git a/tools/lib/python/kdoc/kdoc_item.py b/tools/lib/python/kdoc/kdoc_item.py index 5f41790efacb..fe08cac861c2 100644 --- a/tools/lib/python/kdoc/kdoc_item.py +++ b/tools/lib/python/kdoc/kdoc_item.py @@ -82,7 +82,7 @@ class KdocItem: Set sections and start lines. """ self.sections = sections - self.section_start_lines = start_lines + self.sections_start_lines = start_lines def set_params(self, names, descs, types, starts): """ diff --git a/tools/lib/python/kdoc/kdoc_output.py b/tools/lib/python/kdoc/kdoc_output.py index 73d71cbeabb5..1b54117dbe19 100644 --- a/tools/lib/python/kdoc/kdoc_output.py +++ b/tools/lib/python/kdoc/kdoc_output.py @@ -389,7 +389,7 @@ class RestFormat(OutputFormat): else: self.data += f'{self.lineprefix}**{section}**\n\n' - self.print_lineno(args.section_start_lines.get(section, 0)) + self.print_lineno(args.sections_start_lines.get(section, 0)) self.output_highlight(text) self.data += "\n" self.data += "\n" diff --git a/tools/lib/python/kdoc/kdoc_parser.py b/tools/lib/python/kdoc/kdoc_parser.py index f6c4ee3b18c9..35658a7e72d5 100644 --- a/tools/lib/python/kdoc/kdoc_parser.py +++ b/tools/lib/python/kdoc/kdoc_parser.py @@ -140,7 +140,7 @@ class KernelEntry: self.parametertypes = {} self.parameterdesc_start_lines = {} - self.section_start_lines = {} + self.sections_start_lines = {} self.sections = {} self.anon_struct_union = False @@ -220,7 +220,7 @@ class KernelEntry: self.sections[name] += '\n' + contents else: self.sections[name] = contents - self.section_start_lines[name] = self.new_start_line + self.sections_start_lines[name] = self.new_start_line self.new_start_line = 0 # self.config.log.debug("Section: %s : %s", name, pformat(vars(self))) @@ -316,7 +316,7 @@ class KernelDoc: for section in ["Description", "Return"]: if section in sections and not sections[section].rstrip(): del sections[section] - item.set_sections(sections, self.entry.section_start_lines) + item.set_sections(sections, self.entry.sections_start_lines) item.set_params(self.entry.parameterlist, self.entry.parameterdescs, self.entry.parametertypes, self.entry.parameterdesc_start_lines) -- cgit v1.2.3 From b37b3cbbb1f1a99bc8b95d9f00fcf887c27f4770 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Wed, 18 Mar 2026 10:11:13 +0100 Subject: docs: add a new file to write kernel-doc output to a YAML file Storing kernel-doc output is helpful to allow debugging problems on it and to preparate unit tests. Add a class to store such contents at the same format as defined at kdoc-test-schema.yaml. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Message-ID: <5d084ca1a91f6a620534a1135d1b8183d934319a.1773823995.git.mchehab+huawei@kernel.org> --- tools/lib/python/kdoc/kdoc_yaml_file.py | 155 ++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 tools/lib/python/kdoc/kdoc_yaml_file.py (limited to 'tools/lib/python') diff --git a/tools/lib/python/kdoc/kdoc_yaml_file.py b/tools/lib/python/kdoc/kdoc_yaml_file.py new file mode 100644 index 000000000000..db131503c3f6 --- /dev/null +++ b/tools/lib/python/kdoc/kdoc_yaml_file.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2026: Mauro Carvalho Chehab . + +import os + +from kdoc.kdoc_output import ManFormat, RestFormat + + +class KDocTestFile(): + """ + Handles the logic needed to store kernel‑doc output inside a YAML file. + Useful for unit tests and regression tests. + """ + + def __init__(self, config, yaml_file, yaml_content): + # + # Bail out early if yaml is not available + # + try: + import yaml + except ImportError: + sys.exit("Warning: yaml package not available. Aborting it.") + + self.config = config + self.test_file = os.path.expanduser(yaml_file) + self.yaml_content = yaml_content + + self.tests = [] + + out_dir = os.path.dirname(self.test_file) + if out_dir and not os.path.isdir(out_dir): + sys.exit(f"Directory {out_dir} doesn't exist.") + + self.out_style = [] + + if "man" in self.yaml_content: + out_style = ManFormat() + out_style.set_config(self.config) + + self.out_style.append(out_style) + + if "rst" in self.yaml_content: + out_style = RestFormat() + out_style.set_config(self.config) + + self.out_style.append(out_style) + + def set_filter(self, export, internal, symbol, nosymbol, + function_table, enable_lineno, no_doc_sections): + """ + Set filters at the output classes. + """ + for out_style in self.out_style: + out_style.set_filter(export, internal, symbol, + nosymbol, function_table, + enable_lineno, no_doc_sections) + + @staticmethod + def get_kdoc_item(arg, start_line=1): + + d = vars(arg) + + declaration_start_line = d.get("declaration_start_line") + if not declaration_start_line: + return d + + d["declaration_start_line"] = start_line + + parameterdesc_start_lines = d.get("parameterdesc_start_lines") + if parameterdesc_start_lines: + for key in parameterdesc_start_lines: + ln = parameterdesc_start_lines[key] + ln += start_line - declaration_start_line + + parameterdesc_start_lines[key] = ln + + sections_start_lines = d.get("sections_start_lines") + if sections_start_lines: + for key in sections_start_lines: + ln = sections_start_lines[key] + ln += start_line - declaration_start_line + + sections_start_lines[key] = ln + + return d + + def output_symbols(self, fname, symbols, source): + """ + Store source, symbols and output strings at self.tests. + """ + + # + # KdocItem needs to be converted into dicts + # + kdoc_item = [] + expected = [] + + if not symbols and not source: + return + + if not source or len(symbols) != len(source): + print(f"Warning: lengths are different. Ignoring {fname}") + + # Folding without line numbers is too hard. + # The right thing to do here to proceed would be to delete + # not-handled source blocks, as len(source) should be bigger + # than len(symbols) + return + + base_name = "test_" + fname.replace(".", "_").replace("/", "_") + expected_dict = {} + start_line=1 + + for i in range(0, len(symbols)): + arg = symbols[i] + + if "KdocItem" in self.yaml_content: + msg = self.get_kdoc_item(arg) + + expected_dict["kdoc_item"] = msg + + for out_style in self.out_style: + if isinstance(out_style, ManFormat): + key = "man" + else: + key = "rst" + + expected_dict[key]= out_style.output_symbols(fname, [arg]) + + name = f"{base_name}_{i:03d}" + + test = { + "name": name, + "description": f"{fname} line {source[i]["ln"]}", + "fname": fname, + "source": source[i]["data"], + "expected": [expected_dict] + } + + self.tests.append(test) + + expected_dict = {} + + def write(self): + """ + Output the content of self.tests to self.test_file. + """ + import yaml + + data = {"tests": self.tests} + + with open(self.test_file, "w", encoding="utf-8") as fp: + yaml.safe_dump(data, fp, sort_keys=False, default_style="|", + default_flow_style=False, allow_unicode=True) -- cgit v1.2.3 From 01d6d7bf9672f1aeabbffaa3fbfb8017223ff878 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Wed, 18 Mar 2026 10:11:14 +0100 Subject: docs: kernel-doc: add support to store output on a YAML file Add a command line parameter and library support to optionally store: - KdocItem intermediate format after parsing; - man pages output; - rst output. inside a YAML file. Signed-off-by: Mauro Carvalho Chehab Signed-off-by: Jonathan Corbet Message-ID: --- tools/docs/kernel-doc | 48 ++++++++++++++++++++++++++++++------ tools/lib/python/kdoc/kdoc_files.py | 47 ++++++++++++++++++++++++++++++----- tools/lib/python/kdoc/kdoc_parser.py | 27 +++++++++++++++++++- 3 files changed, 107 insertions(+), 15 deletions(-) (limited to 'tools/lib/python') diff --git a/tools/docs/kernel-doc b/tools/docs/kernel-doc index 3a932f95bdf5..d9192c3f1645 100755 --- a/tools/docs/kernel-doc +++ b/tools/docs/kernel-doc @@ -240,11 +240,9 @@ def main(): help=EXPORT_FILE_DESC) # - # Output format mutually-exclusive group + # Output format # - out_group = parser.add_argument_group("Output format selection (mutually exclusive)") - - out_fmt = out_group.add_mutually_exclusive_group() + out_fmt = parser.add_argument_group("Output format selection (mutually exclusive)") out_fmt.add_argument("-m", "-man", "--man", action="store_true", help="Output troff manual page format.") @@ -253,6 +251,12 @@ def main(): out_fmt.add_argument("-N", "-none", "--none", action="store_true", help="Do not output documentation, only warnings.") + out_fmt.add_argument("-y", "--yaml-file", "--yaml", + help="Stores kernel-doc output on a yaml file.") + out_fmt.add_argument("-k", "--kdoc-item", "--kdoc", action="store_true", + help="Store KdocItem inside yaml file. Ued together with --yaml.") + + # # Output selection mutually-exclusive group # @@ -323,14 +327,42 @@ def main(): from kdoc.kdoc_files import KernelFiles # pylint: disable=C0415 from kdoc.kdoc_output import RestFormat, ManFormat # pylint: disable=C0415 - if args.man: - out_style = ManFormat(modulename=args.modulename) - elif args.none: + yaml_content = set() + if args.yaml_file: out_style = None + + if args.man: + yaml_content |= {"man"} + + if args.rst: + yaml_content |= {"rst"} + + if args.kdoc_item or not yaml_content: + yaml_content |= {"KdocItem"} + else: - out_style = RestFormat() + n_outputs = 0 + + if args.man: + out_style = ManFormat(modulename=args.modulename) + n_outputs += 1 + + if args.none: + out_style = None + n_outputs += 1 + + if args.rst or n_outputs == 0: + n_outputs += 1 + out_style = RestFormat() + + if n_outputs > 1: + parser.error("Those arguments are muttually exclusive: --man, --rst, --none, except when generating a YAML file.") + + elif not n_outputs: + out_style = RestFormat() kfiles = KernelFiles(verbose=args.verbose, + yaml_file=args.yaml_file, yaml_content=yaml_content, out_style=out_style, werror=args.werror, wreturn=args.wreturn, wshort_desc=args.wshort_desc, wcontents_before_sections=args.wcontents_before_sections) diff --git a/tools/lib/python/kdoc/kdoc_files.py b/tools/lib/python/kdoc/kdoc_files.py index 58f4ee08e226..5a299ed44d62 100644 --- a/tools/lib/python/kdoc/kdoc_files.py +++ b/tools/lib/python/kdoc/kdoc_files.py @@ -16,6 +16,7 @@ import re from kdoc.kdoc_parser import KernelDoc from kdoc.xforms_lists import CTransforms from kdoc.kdoc_output import OutputFormat +from kdoc.kdoc_yaml_file import KDocTestFile class GlobSourceFiles: @@ -152,6 +153,12 @@ class KernelFiles(): If not specified, defaults to use: ``logging.getLogger("kernel-doc")`` + ``yaml_file`` + If defined, stores the output inside a YAML file. + + ``yaml_content`` + Defines what will be inside the YAML file. + Note: There are two type of parsers defined here: @@ -181,7 +188,12 @@ class KernelFiles(): if fname in self.files: return - doc = KernelDoc(self.config, fname, self.xforms) + if self.test_file: + store_src = True + else: + store_src = False + + doc = KernelDoc(self.config, fname, self.xforms, store_src=store_src) export_table, entries = doc.parse_kdoc() self.export_table[fname] = export_table @@ -191,6 +203,10 @@ class KernelFiles(): self.results[fname] = entries + source = doc.get_source() + if source: + self.source[fname] = source + def process_export_file(self, fname): """ Parses ``EXPORT_SYMBOL*`` macros from a single Kernel source file. @@ -220,7 +236,7 @@ class KernelFiles(): def __init__(self, verbose=False, out_style=None, xforms=None, werror=False, wreturn=False, wshort_desc=False, wcontents_before_sections=False, - logger=None): + yaml_file=None, yaml_content=None, logger=None): """ Initialize startup variables and parse all files. """ @@ -259,6 +275,11 @@ class KernelFiles(): # Override log warning, as we want to count errors self.config.warning = self.warning + if yaml_file: + self.test_file = KDocTestFile(self.config, yaml_file, yaml_content) + else: + self.test_file = None + if xforms: self.xforms = xforms else: @@ -273,6 +294,7 @@ class KernelFiles(): self.errors = 0 self.results = {} + self.source = {} self.files = set() self.export_files = set() @@ -331,16 +353,29 @@ class KernelFiles(): for s in symbol: function_table.add(s) - self.out_style.set_filter(export, internal, symbol, nosymbol, - function_table, enable_lineno, - no_doc_sections) - if fname not in self.results: self.config.log.warning("No kernel-doc for file %s", fname) continue symbols = self.results[fname] + if self.test_file: + self.test_file.set_filter(export, internal, symbol, nosymbol, + function_table, enable_lineno, + no_doc_sections) + + self.test_file.output_symbols(fname, symbols, + self.source.get(fname)) + + continue + + self.out_style.set_filter(export, internal, symbol, nosymbol, + function_table, enable_lineno, + no_doc_sections) + msg = self.out_style.output_symbols(fname, symbols) if msg: yield fname, msg + + if self.test_file: + self.test_file.write() diff --git a/tools/lib/python/kdoc/kdoc_parser.py b/tools/lib/python/kdoc/kdoc_parser.py index 35658a7e72d5..a10e64589d76 100644 --- a/tools/lib/python/kdoc/kdoc_parser.py +++ b/tools/lib/python/kdoc/kdoc_parser.py @@ -246,12 +246,13 @@ class KernelDoc: #: String to write when a parameter is not described. undescribed = "-- undescribed --" - def __init__(self, config, fname, xforms): + def __init__(self, config, fname, xforms, store_src=False): """Initialize internal variables""" self.fname = fname self.config = config self.xforms = xforms + self.store_src = store_src tokenizer_set_log(self.config.log, f"{self.fname}: CMatch: ") @@ -264,6 +265,9 @@ class KernelDoc: # Place all potential outputs into an array self.entries = [] + # When store_src is true, the kernel-doc source content is stored here + self.source = None + # # We need Python 3.7 for its "dicts remember the insertion # order" guarantee @@ -1592,6 +1596,15 @@ class KernelDoc: state.DOCBLOCK: process_docblock, } + def get_source(self): + """ + Return the file content of the lines handled by kernel-doc at the + latest parse_kdoc() run. + + Returns none if KernelDoc() was not initialized with store_src, + """ + return self.source + def parse_kdoc(self): """ Open and process each line of a C source file. @@ -1605,6 +1618,8 @@ class KernelDoc: prev = "" prev_ln = None export_table = set() + self.source = [] + self.state = state.NORMAL try: with open(self.fname, "r", encoding="utf8", @@ -1631,6 +1646,8 @@ class KernelDoc: ln, state.name[self.state], line) + prev_state = self.state + # This is an optimization over the original script. # There, when export_file was used for the same file, # it was read twice. Here, we use the already-existing @@ -1641,6 +1658,14 @@ class KernelDoc: # Hand this line to the appropriate state handler self.state_actions[self.state](self, ln, line) + if self.store_src and prev_state != self.state or self.state != state.NORMAL: + if self.state == state.NAME: + # A "/**" was detected. Add a new source element + self.source.append({"ln": ln, "data": line + "\n"}) + else: + # Append to the existing one + self.source[-1]["data"] += line + "\n" + self.emit_unused_warnings() except OSError: -- cgit v1.2.3