mirror of
https://github.com/torvalds/linux.git
synced 2025-08-16 06:31:34 +02:00
bb4c5c50ae
The script now have lots or arguments. Better organize and name them, for it to be a little bit more intuitive. Signed-off-by: Mauro Carvalho Chehab <mchehab+huawei@kernel.org> Signed-off-by: Jonathan Corbet <corbet@lwn.net> Link: https://lore.kernel.org/r/acf5e1db38ca6a713c44ceca9db5cdd7d3079c92.1750571906.git.mchehab+huawei@kernel.org
513 lines
16 KiB
Python
Executable file
513 lines
16 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
# SPDX-License-Identifier: GPL-2.0
|
|
# Copyright(c) 2025: Mauro Carvalho Chehab <mchehab+huawei@kernel.org>
|
|
#
|
|
# pylint: disable=R0903,R0912,R0913,R0914,R0917,C0301
|
|
|
|
"""
|
|
Install minimal supported requirements for different Sphinx versions
|
|
and optionally test the build.
|
|
"""
|
|
|
|
import argparse
|
|
import asyncio
|
|
import os.path
|
|
import shutil
|
|
import sys
|
|
import time
|
|
import subprocess
|
|
|
|
# Minimal python version supported by the building system.
|
|
|
|
PYTHON = os.path.basename(sys.executable)
|
|
|
|
min_python_bin = None
|
|
|
|
for i in range(9, 13):
|
|
p = f"python3.{i}"
|
|
if shutil.which(p):
|
|
min_python_bin = p
|
|
break
|
|
|
|
if not min_python_bin:
|
|
min_python_bin = PYTHON
|
|
|
|
# Starting from 8.0, Python 3.9 is not supported anymore.
|
|
PYTHON_VER_CHANGES = {(8, 0, 0): PYTHON}
|
|
|
|
DEFAULT_VERSIONS_TO_TEST = [
|
|
(3, 4, 3), # Minimal supported version
|
|
(5, 3, 0), # CentOS Stream 9 / AlmaLinux 9
|
|
(6, 1, 1), # Debian 12
|
|
(7, 2, 1), # openSUSE Leap 15.6
|
|
(7, 2, 6), # Ubuntu 24.04 LTS
|
|
(7, 4, 7), # Ubuntu 24.10
|
|
(7, 3, 0), # openSUSE Tumbleweed
|
|
(8, 1, 3), # Fedora 42
|
|
(8, 2, 3) # Latest version - covers rolling distros
|
|
]
|
|
|
|
# Sphinx versions to be installed and their incremental requirements
|
|
SPHINX_REQUIREMENTS = {
|
|
# Oldest versions we support for each package required by Sphinx 3.4.3
|
|
(3, 4, 3): {
|
|
"docutils": "0.16",
|
|
"alabaster": "0.7.12",
|
|
"babel": "2.8.0",
|
|
"certifi": "2020.6.20",
|
|
"docutils": "0.16",
|
|
"idna": "2.10",
|
|
"imagesize": "1.2.0",
|
|
"Jinja2": "2.11.2",
|
|
"MarkupSafe": "1.1.1",
|
|
"packaging": "20.4",
|
|
"Pygments": "2.6.1",
|
|
"PyYAML": "5.1",
|
|
"requests": "2.24.0",
|
|
"snowballstemmer": "2.0.0",
|
|
"sphinxcontrib-applehelp": "1.0.2",
|
|
"sphinxcontrib-devhelp": "1.0.2",
|
|
"sphinxcontrib-htmlhelp": "1.0.3",
|
|
"sphinxcontrib-jsmath": "1.0.1",
|
|
"sphinxcontrib-qthelp": "1.0.3",
|
|
"sphinxcontrib-serializinghtml": "1.1.4",
|
|
"urllib3": "1.25.9",
|
|
},
|
|
|
|
# Update package dependencies to a more modern base. The goal here
|
|
# is to avoid to many incremental changes for the next entries
|
|
(3, 5, 0): {
|
|
"alabaster": "0.7.13",
|
|
"babel": "2.17.0",
|
|
"certifi": "2025.6.15",
|
|
"idna": "3.10",
|
|
"imagesize": "1.4.1",
|
|
"packaging": "25.0",
|
|
"Pygments": "2.8.1",
|
|
"requests": "2.32.4",
|
|
"snowballstemmer": "3.0.1",
|
|
"sphinxcontrib-applehelp": "1.0.4",
|
|
"sphinxcontrib-htmlhelp": "2.0.1",
|
|
"sphinxcontrib-serializinghtml": "1.1.5",
|
|
"urllib3": "2.0.0",
|
|
},
|
|
|
|
# Starting from here, ensure all docutils versions are covered with
|
|
# supported Sphinx versions. Other packages are upgraded only when
|
|
# required by pip
|
|
(4, 0, 0): {
|
|
"PyYAML": "5.1",
|
|
},
|
|
(4, 1, 0): {
|
|
"docutils": "0.17",
|
|
"Pygments": "2.19.1",
|
|
"Jinja2": "3.0.3",
|
|
"MarkupSafe": "2.0",
|
|
},
|
|
(4, 3, 0): {},
|
|
(4, 4, 0): {},
|
|
(4, 5, 0): {
|
|
"docutils": "0.17.1",
|
|
},
|
|
(5, 0, 0): {},
|
|
(5, 1, 0): {},
|
|
(5, 2, 0): {
|
|
"docutils": "0.18",
|
|
"Jinja2": "3.1.2",
|
|
"MarkupSafe": "2.0",
|
|
"PyYAML": "5.3.1",
|
|
},
|
|
(5, 3, 0): {
|
|
"docutils": "0.18.1",
|
|
},
|
|
(6, 0, 0): {},
|
|
(6, 1, 0): {},
|
|
(6, 2, 0): {
|
|
"PyYAML": "5.4.1",
|
|
},
|
|
(7, 0, 0): {},
|
|
(7, 1, 0): {},
|
|
(7, 2, 0): {
|
|
"docutils": "0.19",
|
|
"PyYAML": "6.0.1",
|
|
"sphinxcontrib-serializinghtml": "1.1.9",
|
|
},
|
|
(7, 2, 6): {
|
|
"docutils": "0.20",
|
|
},
|
|
(7, 3, 0): {
|
|
"alabaster": "0.7.14",
|
|
"PyYAML": "6.0.1",
|
|
"tomli": "2.0.1",
|
|
},
|
|
(7, 4, 0): {
|
|
"docutils": "0.20.1",
|
|
"PyYAML": "6.0.1",
|
|
},
|
|
(8, 0, 0): {
|
|
"docutils": "0.21",
|
|
},
|
|
(8, 1, 0): {
|
|
"docutils": "0.21.1",
|
|
"PyYAML": "6.0.1",
|
|
"sphinxcontrib-applehelp": "1.0.7",
|
|
"sphinxcontrib-devhelp": "1.0.6",
|
|
"sphinxcontrib-htmlhelp": "2.0.6",
|
|
"sphinxcontrib-qthelp": "1.0.6",
|
|
},
|
|
(8, 2, 0): {
|
|
"docutils": "0.21.2",
|
|
"PyYAML": "6.0.1",
|
|
"sphinxcontrib-serializinghtml": "1.1.9",
|
|
},
|
|
}
|
|
|
|
|
|
class AsyncCommands:
|
|
"""Excecute command synchronously"""
|
|
|
|
def __init__(self, fp=None):
|
|
|
|
self.stdout = None
|
|
self.stderr = None
|
|
self.output = None
|
|
self.fp = fp
|
|
|
|
def log(self, out, verbose, is_info=True):
|
|
out = out.removesuffix('\n')
|
|
|
|
if verbose:
|
|
if is_info:
|
|
print(out)
|
|
else:
|
|
print(out, file=sys.stderr)
|
|
|
|
if self.fp:
|
|
self.fp.write(out + "\n")
|
|
|
|
async def _read(self, stream, verbose, is_info):
|
|
"""Ancillary routine to capture while displaying"""
|
|
|
|
while stream is not None:
|
|
line = await stream.readline()
|
|
if line:
|
|
out = line.decode("utf-8", errors="backslashreplace")
|
|
self.log(out, verbose, is_info)
|
|
if is_info:
|
|
self.stdout += out
|
|
else:
|
|
self.stderr += out
|
|
else:
|
|
break
|
|
|
|
async def run(self, cmd, capture_output=False, check=False,
|
|
env=None, verbose=True):
|
|
|
|
"""
|
|
Execute an arbitrary command, handling errors.
|
|
|
|
Please notice that this class is not thread safe
|
|
"""
|
|
|
|
self.stdout = ""
|
|
self.stderr = ""
|
|
|
|
self.log("$ " + " ".join(cmd), verbose)
|
|
|
|
proc = await asyncio.create_subprocess_exec(cmd[0],
|
|
*cmd[1:],
|
|
env=env,
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE)
|
|
|
|
# Handle input and output in realtime
|
|
await asyncio.gather(
|
|
self._read(proc.stdout, verbose, True),
|
|
self._read(proc.stderr, verbose, False),
|
|
)
|
|
|
|
await proc.wait()
|
|
|
|
if check and proc.returncode > 0:
|
|
raise subprocess.CalledProcessError(returncode=proc.returncode,
|
|
cmd=" ".join(cmd),
|
|
output=self.stdout,
|
|
stderr=self.stderr)
|
|
|
|
if capture_output:
|
|
if proc.returncode > 0:
|
|
self.log(f"Error {proc.returncode}", verbose=True, is_info=False)
|
|
return ""
|
|
|
|
return self.output
|
|
|
|
ret = subprocess.CompletedProcess(args=cmd,
|
|
returncode=proc.returncode,
|
|
stdout=self.stdout,
|
|
stderr=self.stderr)
|
|
|
|
return ret
|
|
|
|
|
|
class SphinxVenv:
|
|
"""
|
|
Installs Sphinx on one virtual env per Sphinx version with a minimal
|
|
set of dependencies, adjusting them to each specific version.
|
|
"""
|
|
|
|
def __init__(self):
|
|
"""Initialize instance variables"""
|
|
|
|
self.built_time = {}
|
|
self.first_run = True
|
|
|
|
async def _handle_version(self, args, fp,
|
|
cur_ver, cur_requirements, python_bin):
|
|
"""Handle a single Sphinx version"""
|
|
|
|
cmd = AsyncCommands(fp)
|
|
|
|
ver = ".".join(map(str, cur_ver))
|
|
|
|
if not self.first_run and args.wait_input and args.build:
|
|
ret = input("Press Enter to continue or 'a' to abort: ").strip().lower()
|
|
if ret == "a":
|
|
print("Aborted.")
|
|
sys.exit()
|
|
else:
|
|
self.first_run = False
|
|
|
|
venv_dir = f"Sphinx_{ver}"
|
|
req_file = f"requirements_{ver}.txt"
|
|
|
|
cmd.log(f"\nSphinx {ver} with {python_bin}", verbose=True)
|
|
|
|
# Create venv
|
|
await cmd.run([python_bin, "-m", "venv", venv_dir],
|
|
verbose=args.verbose, check=True)
|
|
pip = os.path.join(venv_dir, "bin/pip")
|
|
|
|
# Create install list
|
|
reqs = []
|
|
for pkg, verstr in cur_requirements.items():
|
|
reqs.append(f"{pkg}=={verstr}")
|
|
|
|
reqs.append(f"Sphinx=={ver}")
|
|
|
|
await cmd.run([pip, "install"] + reqs, check=True, verbose=args.verbose)
|
|
|
|
# Freeze environment
|
|
result = await cmd.run([pip, "freeze"], verbose=False, check=True)
|
|
|
|
# Pip install succeeded. Write requirements file
|
|
if args.req_file:
|
|
with open(req_file, "w", encoding="utf-8") as fp:
|
|
fp.write(result.stdout)
|
|
|
|
if args.build:
|
|
start_time = time.time()
|
|
|
|
# Prepare a venv environment
|
|
env = os.environ.copy()
|
|
bin_dir = os.path.join(venv_dir, "bin")
|
|
env["PATH"] = bin_dir + ":" + env["PATH"]
|
|
env["VIRTUAL_ENV"] = venv_dir
|
|
if "PYTHONHOME" in env:
|
|
del env["PYTHONHOME"]
|
|
|
|
# Test doc build
|
|
await cmd.run(["make", "cleandocs"], env=env, check=True)
|
|
make = ["make"]
|
|
|
|
if args.output:
|
|
sphinx_build = os.path.realpath(f"{bin_dir}/sphinx-build")
|
|
make += [f"O={args.output}", f"SPHINXBUILD={sphinx_build}"]
|
|
|
|
if args.make_args:
|
|
make += args.make_args
|
|
|
|
make += args.targets
|
|
|
|
if args.verbose:
|
|
cmd.log(f". {bin_dir}/activate", verbose=True)
|
|
await cmd.run(make, env=env, check=True, verbose=True)
|
|
if args.verbose:
|
|
cmd.log("deactivate", verbose=True)
|
|
|
|
end_time = time.time()
|
|
elapsed_time = end_time - start_time
|
|
hours, minutes = divmod(elapsed_time, 3600)
|
|
minutes, seconds = divmod(minutes, 60)
|
|
|
|
hours = int(hours)
|
|
minutes = int(minutes)
|
|
seconds = int(seconds)
|
|
|
|
self.built_time[ver] = f"{hours:02d}:{minutes:02d}:{seconds:02d}"
|
|
|
|
cmd.log(f"Finished doc build for Sphinx {ver}. Elapsed time: {self.built_time[ver]}", verbose=True)
|
|
|
|
async def run(self, args):
|
|
"""
|
|
Navigate though multiple Sphinx versions, handling each of them
|
|
on a loop.
|
|
"""
|
|
|
|
if args.log:
|
|
fp = open(args.log, "w", encoding="utf-8")
|
|
if not args.verbose:
|
|
args.verbose = False
|
|
else:
|
|
fp = None
|
|
if not args.verbose:
|
|
args.verbose = True
|
|
|
|
cur_requirements = {}
|
|
python_bin = min_python_bin
|
|
|
|
vers = set(SPHINX_REQUIREMENTS.keys()) | set(args.versions)
|
|
|
|
for cur_ver in sorted(vers):
|
|
if cur_ver in SPHINX_REQUIREMENTS:
|
|
new_reqs = SPHINX_REQUIREMENTS[cur_ver]
|
|
cur_requirements.update(new_reqs)
|
|
|
|
if cur_ver in PYTHON_VER_CHANGES: # pylint: disable=R1715
|
|
python_bin = PYTHON_VER_CHANGES[cur_ver]
|
|
|
|
if cur_ver not in args.versions:
|
|
continue
|
|
|
|
if args.min_version:
|
|
if cur_ver < args.min_version:
|
|
continue
|
|
|
|
if args.max_version:
|
|
if cur_ver > args.max_version:
|
|
break
|
|
|
|
await self._handle_version(args, fp, cur_ver, cur_requirements,
|
|
python_bin)
|
|
|
|
if args.build:
|
|
cmd = AsyncCommands(fp)
|
|
cmd.log("\nSummary:", verbose=True)
|
|
for ver, elapsed_time in sorted(self.built_time.items()):
|
|
cmd.log(f"\tSphinx {ver} elapsed time: {elapsed_time}",
|
|
verbose=True)
|
|
|
|
if fp:
|
|
fp.close()
|
|
|
|
def parse_version(ver_str):
|
|
"""Convert a version string into a tuple."""
|
|
|
|
return tuple(map(int, ver_str.split(".")))
|
|
|
|
|
|
DEFAULT_VERS = " - "
|
|
DEFAULT_VERS += "\n - ".join(map(lambda v: f"{v[0]}.{v[1]}.{v[2]}",
|
|
DEFAULT_VERSIONS_TO_TEST))
|
|
|
|
SCRIPT = os.path.relpath(__file__)
|
|
|
|
DESCRIPTION = f"""
|
|
This tool allows creating Python virtual environments for different
|
|
Sphinx versions that are supported by the Linux Kernel build system.
|
|
|
|
Besides creating the virtual environment, it can also test building
|
|
the documentation using "make htmldocs" (and/or other doc targets).
|
|
|
|
If called without "--versions" argument, it covers the versions shipped
|
|
on major distros, plus the lowest supported version:
|
|
|
|
{DEFAULT_VERS}
|
|
|
|
A typical usage is to run:
|
|
|
|
{SCRIPT} -m -l sphinx_builds.log
|
|
|
|
This will create one virtual env for the default version set and run
|
|
"make htmldocs" for each version, creating a log file with the
|
|
excecuted commands on it.
|
|
|
|
NOTE: The build time can be very long, specially on old versions. Also, there
|
|
is a known bug with Sphinx version 6.0.x: each subprocess uses a lot of
|
|
memory. That, together with "-jauto" may cause OOM killer to cause
|
|
failures at the doc generation. To minimize the risk, you may use the
|
|
"-a" command line parameter to constrain the built directories and/or
|
|
reduce the number of threads from "-jauto" to, for instance, "-j4":
|
|
|
|
{SCRIPT} -m -V 6.0.1 -a "SPHINXDIRS=process" "SPHINXOPTS='-j4'"
|
|
|
|
"""
|
|
|
|
MAKE_TARGETS = [
|
|
"htmldocs",
|
|
"texinfodocs",
|
|
"infodocs",
|
|
"latexdocs",
|
|
"pdfdocs",
|
|
"epubdocs",
|
|
"xmldocs",
|
|
]
|
|
|
|
async def main():
|
|
"""Main program"""
|
|
|
|
parser = argparse.ArgumentParser(description=DESCRIPTION,
|
|
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
|
|
ver_group = parser.add_argument_group("Version range options")
|
|
|
|
ver_group.add_argument('-V', '--versions', nargs="*",
|
|
default=DEFAULT_VERSIONS_TO_TEST,type=parse_version,
|
|
help='Sphinx versions to test')
|
|
ver_group.add_argument('--min-version', "--min", type=parse_version,
|
|
help='Sphinx minimal version')
|
|
ver_group.add_argument('--max-version', "--max", type=parse_version,
|
|
help='Sphinx maximum version')
|
|
ver_group.add_argument('-f', '--full', action='store_true',
|
|
help='Add all Sphinx (major,minor) supported versions to the version range')
|
|
|
|
build_group = parser.add_argument_group("Build options")
|
|
|
|
build_group.add_argument('-b', '--build', action='store_true',
|
|
help='Build documentation')
|
|
build_group.add_argument('-a', '--make-args', nargs="*",
|
|
help='extra arguments for make, like SPHINXDIRS=netlink/specs',
|
|
)
|
|
build_group.add_argument('-t', '--targets', nargs="+", choices=MAKE_TARGETS,
|
|
default=[MAKE_TARGETS[0]],
|
|
help="make build targets. Default: htmldocs.")
|
|
build_group.add_argument("-o", '--output',
|
|
help="output directory for the make O=OUTPUT")
|
|
|
|
other_group = parser.add_argument_group("Other options")
|
|
|
|
other_group.add_argument('-r', '--req-file', action='store_true',
|
|
help='write a requirements.txt file')
|
|
other_group.add_argument('-l', '--log',
|
|
help='Log command output on a file')
|
|
other_group.add_argument('-v', '--verbose', action='store_true',
|
|
help='Verbose all commands')
|
|
other_group.add_argument('-i', '--wait-input', action='store_true',
|
|
help='Wait for an enter before going to the next version')
|
|
|
|
args = parser.parse_args()
|
|
|
|
if not args.make_args:
|
|
args.make_args = []
|
|
|
|
sphinx_versions = sorted(list(SPHINX_REQUIREMENTS.keys()))
|
|
|
|
if args.full:
|
|
args.versions += list(SPHINX_REQUIREMENTS.keys())
|
|
|
|
venv = SphinxVenv()
|
|
await venv.run(args)
|
|
|
|
|
|
# Call main method
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|