mirror of
git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
synced 2025-09-04 20:19:47 +08:00
9bec7870c6
Instead of printing all results line per line, use an interactor to return each variable as a separate message. This won't change much when using it via command line, but it will help Sphinx integration by providing an interactor that could be used there to handle ABI symbol by symbol. Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org> Signed-off-by: Jonathan Corbet <corbet@lwn.net> Link: https://lore.kernel.org/r/e3c94b8cdfd5e955aa19a703921f364a89089634.1739182025.git.mchehab+huawei@kernel.org
588 lines
20 KiB
Python
588 lines
20 KiB
Python
#!/usr/bin/env python3
|
|
# pylint: disable=R0902,R0903,R0911,R0912,R0913,R0914,R0915,R0917,C0302
|
|
# Copyright(c) 2025: Mauro Carvalho Chehab <mchehab@kernel.org>.
|
|
# SPDX-License-Identifier: GPL-2.0
|
|
|
|
"""
|
|
Parse ABI documentation and produce results from it.
|
|
"""
|
|
|
|
from argparse import Namespace
|
|
import logging
|
|
import os
|
|
import re
|
|
|
|
from pprint import pformat
|
|
from random import randrange, seed
|
|
|
|
# Import Python modules
|
|
|
|
from helpers import AbiDebug, ABI_DIR
|
|
|
|
|
|
class AbiParser:
|
|
"""Main class to parse ABI files"""
|
|
|
|
TAGS = r"(what|where|date|kernelversion|contact|description|users)"
|
|
XREF = r"(?:^|\s|\()(\/(?:sys|config|proc|dev|kvd)\/[^,.:;\)\s]+)(?:[,.:;\)\s]|\Z)"
|
|
|
|
def __init__(self, directory, logger=None,
|
|
enable_lineno=False, show_warnings=True, debug=0):
|
|
"""Stores arguments for the class and initialize class vars"""
|
|
|
|
self.directory = directory
|
|
self.enable_lineno = enable_lineno
|
|
self.show_warnings = show_warnings
|
|
self.debug = debug
|
|
|
|
if not logger:
|
|
self.log = logging.getLogger("get_abi")
|
|
else:
|
|
self.log = logger
|
|
|
|
self.data = {}
|
|
self.what_symbols = {}
|
|
self.file_refs = {}
|
|
self.what_refs = {}
|
|
|
|
# Ignore files that contain such suffixes
|
|
self.ignore_suffixes = (".rej", ".org", ".orig", ".bak", "~")
|
|
|
|
# Regular expressions used on parser
|
|
self.re_abi_dir = re.compile(r"(.*)" + ABI_DIR)
|
|
self.re_tag = re.compile(r"(\S+)(:\s*)(.*)", re.I)
|
|
self.re_valid = re.compile(self.TAGS)
|
|
self.re_start_spc = re.compile(r"(\s*)(\S.*)")
|
|
self.re_whitespace = re.compile(r"^\s+")
|
|
|
|
# Regular used on print
|
|
self.re_what = re.compile(r"(\/?(?:[\w\-]+\/?){1,2})")
|
|
self.re_escape = re.compile(r"([\.\x01-\x08\x0e-\x1f\x21-\x2f\x3a-\x40\x7b-\xff])")
|
|
self.re_unprintable = re.compile(r"([\x00-\x2f\x3a-\x40\x5b-\x60\x7b-\xff]+)")
|
|
self.re_title_mark = re.compile(r"\n[\-\*\=\^\~]+\n")
|
|
self.re_doc = re.compile(r"Documentation/(?!devicetree)(\S+)\.rst")
|
|
self.re_abi = re.compile(r"(Documentation/ABI/)([\w\/\-]+)")
|
|
self.re_xref_node = re.compile(self.XREF)
|
|
|
|
def warn(self, fdata, msg, extra=None):
|
|
"""Displays a parse error if warning is enabled"""
|
|
|
|
if not self.show_warnings:
|
|
return
|
|
|
|
msg = f"{fdata.fname}:{fdata.ln}: {msg}"
|
|
if extra:
|
|
msg += "\n\t\t" + extra
|
|
|
|
self.log.warning(msg)
|
|
|
|
def add_symbol(self, what, fname, ln=None, xref=None):
|
|
"""Create a reference table describing where each 'what' is located"""
|
|
|
|
if what not in self.what_symbols:
|
|
self.what_symbols[what] = {"file": {}}
|
|
|
|
if fname not in self.what_symbols[what]["file"]:
|
|
self.what_symbols[what]["file"][fname] = []
|
|
|
|
if ln and ln not in self.what_symbols[what]["file"][fname]:
|
|
self.what_symbols[what]["file"][fname].append(ln)
|
|
|
|
if xref:
|
|
self.what_symbols[what]["xref"] = xref
|
|
|
|
def _parse_line(self, fdata, line):
|
|
"""Parse a single line of an ABI file"""
|
|
|
|
new_what = False
|
|
new_tag = False
|
|
content = None
|
|
|
|
match = self.re_tag.match(line)
|
|
if match:
|
|
new = match.group(1).lower()
|
|
sep = match.group(2)
|
|
content = match.group(3)
|
|
|
|
match = self.re_valid.search(new)
|
|
if match:
|
|
new_tag = match.group(1)
|
|
else:
|
|
if fdata.tag == "description":
|
|
# New "tag" is actually part of description.
|
|
# Don't consider it a tag
|
|
new_tag = False
|
|
elif fdata.tag != "":
|
|
self.warn(fdata, f"tag '{fdata.tag}' is invalid", line)
|
|
|
|
if new_tag:
|
|
# "where" is Invalid, but was a common mistake. Warn if found
|
|
if new_tag == "where":
|
|
self.warn(fdata, "tag 'Where' is invalid. Should be 'What:' instead")
|
|
new_tag = "what"
|
|
|
|
if new_tag == "what":
|
|
fdata.space = None
|
|
|
|
if content not in self.what_symbols:
|
|
self.add_symbol(what=content, fname=fdata.fname, ln=fdata.ln)
|
|
|
|
if fdata.tag == "what":
|
|
fdata.what.append(content.strip("\n"))
|
|
else:
|
|
if fdata.key:
|
|
if "description" not in self.data.get(fdata.key, {}):
|
|
self.warn(fdata, f"{fdata.key} doesn't have a description")
|
|
|
|
for w in fdata.what:
|
|
self.add_symbol(what=w, fname=fdata.fname,
|
|
ln=fdata.what_ln, xref=fdata.key)
|
|
|
|
fdata.label = content
|
|
new_what = True
|
|
|
|
key = "abi_" + content.lower()
|
|
fdata.key = self.re_unprintable.sub("_", key).strip("_")
|
|
|
|
# Avoid duplicated keys but using a defined seed, to make
|
|
# the namespace identical if there aren't changes at the
|
|
# ABI symbols
|
|
seed(42)
|
|
|
|
while fdata.key in self.data:
|
|
char = randrange(0, 51) + ord("A")
|
|
if char > ord("Z"):
|
|
char += ord("a") - ord("Z") - 1
|
|
|
|
fdata.key += chr(char)
|
|
|
|
if fdata.key and fdata.key not in self.data:
|
|
self.data[fdata.key] = {
|
|
"what": [content],
|
|
"file": [fdata.file_ref],
|
|
"line_no": fdata.ln,
|
|
}
|
|
|
|
fdata.what = self.data[fdata.key]["what"]
|
|
|
|
self.what_refs[content] = fdata.key
|
|
fdata.tag = new_tag
|
|
fdata.what_ln = fdata.ln
|
|
|
|
if fdata.nametag["what"]:
|
|
t = (content, fdata.key)
|
|
if t not in fdata.nametag["symbols"]:
|
|
fdata.nametag["symbols"].append(t)
|
|
|
|
return
|
|
|
|
if fdata.tag and new_tag:
|
|
fdata.tag = new_tag
|
|
|
|
if new_what:
|
|
fdata.label = ""
|
|
|
|
self.data[fdata.key]["type"] = fdata.ftype
|
|
|
|
if "description" in self.data[fdata.key]:
|
|
self.data[fdata.key]["description"] += "\n\n"
|
|
|
|
if fdata.file_ref not in self.data[fdata.key]["file"]:
|
|
self.data[fdata.key]["file"].append(fdata.file_ref)
|
|
|
|
if self.debug == AbiDebug.WHAT_PARSING:
|
|
self.log.debug("what: %s", fdata.what)
|
|
|
|
if not fdata.what:
|
|
self.warn(fdata, "'What:' should come first:", line)
|
|
return
|
|
|
|
if new_tag == "description":
|
|
fdata.space = None
|
|
|
|
if content:
|
|
sep = sep.replace(":", " ")
|
|
|
|
c = " " * len(new_tag) + sep + content
|
|
c = c.expandtabs()
|
|
|
|
match = self.re_start_spc.match(c)
|
|
if match:
|
|
# Preserve initial spaces for the first line
|
|
fdata.space = match.group(1)
|
|
content = match.group(2) + "\n"
|
|
|
|
self.data[fdata.key][fdata.tag] = content
|
|
|
|
return
|
|
|
|
# Store any contents before tags at the database
|
|
if not fdata.tag and "what" in fdata.nametag:
|
|
fdata.nametag["description"] += line
|
|
return
|
|
|
|
if fdata.tag == "description":
|
|
content = line.expandtabs()
|
|
|
|
if self.re_whitespace.sub("", content) == "":
|
|
self.data[fdata.key][fdata.tag] += "\n"
|
|
return
|
|
|
|
if fdata.space is None:
|
|
match = self.re_start_spc.match(content)
|
|
if match:
|
|
# Preserve initial spaces for the first line
|
|
fdata.space = match.group(1)
|
|
|
|
content = match.group(2) + "\n"
|
|
else:
|
|
if content.startswith(fdata.space):
|
|
content = content[len(fdata.space):]
|
|
|
|
else:
|
|
fdata.space = ""
|
|
|
|
if fdata.tag == "what":
|
|
w = content.strip("\n")
|
|
if w:
|
|
self.data[fdata.key][fdata.tag].append(w)
|
|
else:
|
|
self.data[fdata.key][fdata.tag] += content
|
|
return
|
|
|
|
content = line.strip()
|
|
if fdata.tag:
|
|
if fdata.tag == "what":
|
|
w = content.strip("\n")
|
|
if w:
|
|
self.data[fdata.key][fdata.tag].append(w)
|
|
else:
|
|
self.data[fdata.key][fdata.tag] += "\n" + content.rstrip("\n")
|
|
return
|
|
|
|
# Everything else is error
|
|
if content:
|
|
self.warn(fdata, "Unexpected content", line)
|
|
|
|
def parse_file(self, fname, path, basename):
|
|
"""Parse a single file"""
|
|
|
|
ref = f"abi_file_{path}_{basename}"
|
|
ref = self.re_unprintable.sub("_", ref).strip("_")
|
|
|
|
# Store per-file state into a namespace variable. This will be used
|
|
# by the per-line parser state machine and by the warning function.
|
|
fdata = Namespace
|
|
|
|
fdata.fname = fname
|
|
fdata.name = basename
|
|
|
|
pos = fname.find(ABI_DIR)
|
|
if pos > 0:
|
|
f = fname[pos:]
|
|
else:
|
|
f = fname
|
|
|
|
fdata.file_ref = (f, ref)
|
|
self.file_refs[f] = ref
|
|
|
|
fdata.ln = 0
|
|
fdata.what_ln = 0
|
|
fdata.tag = ""
|
|
fdata.label = ""
|
|
fdata.what = []
|
|
fdata.key = None
|
|
fdata.xrefs = None
|
|
fdata.space = None
|
|
fdata.ftype = path.split("/")[0]
|
|
|
|
fdata.nametag = {}
|
|
fdata.nametag["what"] = [f"File {path}/{basename}"]
|
|
fdata.nametag["type"] = "File"
|
|
fdata.nametag["file"] = [fdata.file_ref]
|
|
fdata.nametag["line_no"] = 1
|
|
fdata.nametag["description"] = ""
|
|
fdata.nametag["symbols"] = []
|
|
|
|
self.data[ref] = fdata.nametag
|
|
|
|
if self.debug & AbiDebug.WHAT_OPEN:
|
|
self.log.debug("Opening file %s", fname)
|
|
|
|
with open(fname, "r", encoding="utf8", errors="backslashreplace") as fp:
|
|
for line in fp:
|
|
fdata.ln += 1
|
|
|
|
self._parse_line(fdata, line)
|
|
|
|
if "description" in fdata.nametag:
|
|
fdata.nametag["description"] = fdata.nametag["description"].lstrip("\n")
|
|
|
|
if fdata.key:
|
|
if "description" not in self.data.get(fdata.key, {}):
|
|
self.warn(fdata, f"{fdata.key} doesn't have a description")
|
|
|
|
for w in fdata.what:
|
|
self.add_symbol(what=w, fname=fname, xref=fdata.key)
|
|
|
|
def _parse_abi(self, root=None):
|
|
"""Internal function to parse documentation ABI recursively"""
|
|
|
|
if not root:
|
|
root = self.directory
|
|
|
|
with os.scandir(root) as obj:
|
|
for entry in obj:
|
|
name = os.path.join(root, entry.name)
|
|
|
|
if entry.is_dir():
|
|
self._parse_abi(name)
|
|
continue
|
|
|
|
if not entry.is_file():
|
|
continue
|
|
|
|
basename = os.path.basename(name)
|
|
|
|
if basename == "README":
|
|
continue
|
|
|
|
if basename.startswith("."):
|
|
continue
|
|
|
|
if basename.endswith(self.ignore_suffixes):
|
|
continue
|
|
|
|
path = self.re_abi_dir.sub("", os.path.dirname(name))
|
|
|
|
self.parse_file(name, path, basename)
|
|
|
|
def parse_abi(self, root=None):
|
|
"""Parse documentation ABI"""
|
|
|
|
self._parse_abi(root)
|
|
|
|
if self.debug & AbiDebug.DUMP_ABI_STRUCTS:
|
|
self.log.debug(pformat(self.data))
|
|
|
|
def desc_txt(self, desc):
|
|
"""Print description as found inside ABI files"""
|
|
|
|
desc = desc.strip(" \t\n")
|
|
|
|
return desc + "\n\n"
|
|
|
|
def desc_rst(self, desc):
|
|
"""Enrich ReST output by creating cross-references"""
|
|
|
|
# Remove title markups from the description
|
|
# Having titles inside ABI files will only work if extra
|
|
# care would be taken in order to strictly follow the same
|
|
# level order for each markup.
|
|
desc = self.re_title_mark.sub("\n\n", "\n" + desc)
|
|
desc = desc.rstrip(" \t\n").lstrip("\n")
|
|
|
|
# Python's regex performance for non-compiled expressions is a lot
|
|
# than Perl, as Perl automatically caches them at their
|
|
# first usage. Here, we'll need to do the same, as otherwise the
|
|
# performance penalty is be high
|
|
|
|
new_desc = ""
|
|
for d in desc.split("\n"):
|
|
if d == "":
|
|
new_desc += "\n"
|
|
continue
|
|
|
|
# Use cross-references for doc files where needed
|
|
d = self.re_doc.sub(r":doc:`/\1`", d)
|
|
|
|
# Use cross-references for ABI generated docs where needed
|
|
matches = self.re_abi.findall(d)
|
|
for m in matches:
|
|
abi = m[0] + m[1]
|
|
|
|
xref = self.file_refs.get(abi)
|
|
if not xref:
|
|
# This may happen if ABI is on a separate directory,
|
|
# like parsing ABI testing and symbol is at stable.
|
|
# The proper solution is to move this part of the code
|
|
# for it to be inside sphinx/kernel_abi.py
|
|
self.log.info("Didn't find ABI reference for '%s'", abi)
|
|
else:
|
|
new = self.re_escape.sub(r"\\\1", m[1])
|
|
d = re.sub(fr"\b{abi}\b", f":ref:`{new} <{xref}>`", d)
|
|
|
|
# Seek for cross reference symbols like /sys/...
|
|
# Need to be careful to avoid doing it on a code block
|
|
if d[0] not in [" ", "\t"]:
|
|
matches = self.re_xref_node.findall(d)
|
|
for m in matches:
|
|
# Finding ABI here is more complex due to wildcards
|
|
xref = self.what_refs.get(m)
|
|
if xref:
|
|
new = self.re_escape.sub(r"\\\1", m)
|
|
d = re.sub(fr"\b{m}\b", f":ref:`{new} <{xref}>`", d)
|
|
|
|
new_desc += d + "\n"
|
|
|
|
return new_desc + "\n\n"
|
|
|
|
def doc(self, enable_lineno, output_in_txt, show_file=False):
|
|
"""Print ABI at stdout"""
|
|
|
|
part = None
|
|
for key, v in sorted(self.data.items(),
|
|
key=lambda x: (x[1].get("type", ""),
|
|
x[1].get("what"))):
|
|
|
|
wtype = v.get("type", "Var")
|
|
file_ref = v.get("file")
|
|
names = v.get("what", [""])
|
|
|
|
if not show_file and wtype == "File":
|
|
continue
|
|
|
|
msg = ""
|
|
|
|
if enable_lineno:
|
|
ln = v.get("line_no", 1)
|
|
msg += f".. LINENO {file_ref[0][0]}#{ln}\n\n"
|
|
|
|
if wtype != "File":
|
|
cur_part = names[0]
|
|
if cur_part.find("/") >= 0:
|
|
match = self.re_what.match(cur_part)
|
|
if match:
|
|
symbol = match.group(1).rstrip("/")
|
|
cur_part = "Symbols under " + symbol
|
|
|
|
if cur_part and cur_part != part:
|
|
part = cur_part
|
|
msg += f"{part}\n{"-" * len(part)}\n\n"
|
|
|
|
msg += f".. _{key}:\n\n"
|
|
|
|
max_len = 0
|
|
for i in range(0, len(names)): # pylint: disable=C0200
|
|
names[i] = "**" + self.re_escape.sub(r"\\\1", names[i]) + "**"
|
|
|
|
max_len = max(max_len, len(names[i]))
|
|
|
|
msg += "+-" + "-" * max_len + "-+\n"
|
|
for name in names:
|
|
msg += f"| {name}" + " " * (max_len - len(name)) + " |\n"
|
|
msg += "+-" + "-" * max_len + "-+\n"
|
|
msg += "\n"
|
|
|
|
for ref in file_ref:
|
|
if wtype == "File":
|
|
msg += f".. _{ref[1]}:\n\n"
|
|
else:
|
|
base = os.path.basename(ref[0])
|
|
msg += f"Defined on file :ref:`{base} <{ref[1]}>`\n\n"
|
|
|
|
if wtype == "File":
|
|
msg += f"{names[0]}\n{"-" * len(names[0])}\n\n"
|
|
|
|
desc = v.get("description")
|
|
if not desc and wtype != "File":
|
|
msg += f"DESCRIPTION MISSING for {names[0]}\n\n"
|
|
|
|
if desc:
|
|
if output_in_txt:
|
|
msg += self.desc_txt(desc)
|
|
else:
|
|
msg += self.desc_rst(desc)
|
|
|
|
symbols = v.get("symbols")
|
|
if symbols:
|
|
msg += "Has the following ABI:\n\n"
|
|
|
|
for w, label in symbols:
|
|
# Escape special chars from content
|
|
content = self.re_escape.sub(r"\\\1", w)
|
|
|
|
msg += f"- :ref:`{content} <{label}>`\n\n"
|
|
|
|
users = v.get("users")
|
|
if users and users.strip(" \t\n"):
|
|
msg += f"Users:\n\t{users.strip("\n").replace('\n', '\n\t')}\n\n"
|
|
|
|
yield msg
|
|
|
|
def check_issues(self):
|
|
"""Warn about duplicated ABI entries"""
|
|
|
|
for what, v in self.what_symbols.items():
|
|
files = v.get("file")
|
|
if not files:
|
|
# Should never happen if the parser works properly
|
|
self.log.warning("%s doesn't have a file associated", what)
|
|
continue
|
|
|
|
if len(files) == 1:
|
|
continue
|
|
|
|
f = []
|
|
for fname, lines in sorted(files.items()):
|
|
if not lines:
|
|
f.append(f"{fname}")
|
|
elif len(lines) == 1:
|
|
f.append(f"{fname}:{lines[0]}")
|
|
else:
|
|
f.append(f"{fname} lines {", ".join(str(x) for x in lines)}")
|
|
|
|
self.log.warning("%s is defined %d times: %s", what, len(f), "; ".join(f))
|
|
|
|
def search_symbols(self, expr):
|
|
""" Searches for ABI symbols """
|
|
|
|
regex = re.compile(expr, re.I)
|
|
|
|
found_keys = 0
|
|
for t in sorted(self.data.items(), key=lambda x: [0]):
|
|
v = t[1]
|
|
|
|
wtype = v.get("type", "")
|
|
if wtype == "File":
|
|
continue
|
|
|
|
for what in v.get("what", [""]):
|
|
if regex.search(what):
|
|
found_keys += 1
|
|
|
|
kernelversion = v.get("kernelversion", "").strip(" \t\n")
|
|
date = v.get("date", "").strip(" \t\n")
|
|
contact = v.get("contact", "").strip(" \t\n")
|
|
users = v.get("users", "").strip(" \t\n")
|
|
desc = v.get("description", "").strip(" \t\n")
|
|
|
|
files = []
|
|
for f in v.get("file", ()):
|
|
files.append(f[0])
|
|
|
|
what = str(found_keys) + ". " + what
|
|
title_tag = "-" * len(what)
|
|
|
|
print(f"\n{what}\n{title_tag}\n")
|
|
|
|
if kernelversion:
|
|
print(f"Kernel version:\t\t{kernelversion}")
|
|
|
|
if date:
|
|
print(f"Date:\t\t\t{date}")
|
|
|
|
if contact:
|
|
print(f"Contact:\t\t{contact}")
|
|
|
|
if users:
|
|
print(f"Users:\t\t\t{users}")
|
|
|
|
print(f"Defined on file{'s'[:len(files) ^ 1]}:\t{", ".join(files)}")
|
|
|
|
if desc:
|
|
print(f"\n{desc.strip("\n")}\n")
|
|
|
|
if not found_keys:
|
|
print(f"Regular expression /{expr}/ not found.")
|