docs: build the package reference pages in parallel

.docs/extensions/package_docs.py

@@ -12,44 +12,69 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Handle generation, saving, and restoration of package reference docs.
+"""Handle saving and restoration of package reference docs.
 
 Packages are not guaranteed to have compatible dependencies, so we generate their reference docs
-in separate invocations of ``sphinx-build``. If the ``package`` config option is set, we write
-an ``audodoc`` ``automodule`` directive for that package, and then save the resulting doctree and
-index information for that package. If the ``package`` config option is not set, we restore any
-saved information when doctrees are resolved.
+in separate invocations of ``sphinx-build``. If the ``package`` config option is set, we inject
+an ``audodoc`` ``automodule`` directive for that package at source-read time, and then save the
+resulting doctree and index information for that package. If the ``package`` config option is not
+set, we restore any saved information when doctrees are resolved.
+
+The placeholder rst files these builds read are written up front by the companion
+``scripts/package_docs_preprocessor.py``, and the automodule directive is only added in-memory
+during ``source-read`` for the current package. This is what makes it safe to run per-package
+sphinx-build invocations concurrently: they share the same source tree but never mutate each
+other's rst files.
 """
 
 from __future__ import annotations
 
-import json
 import pathlib
 import pickle  # noqa: S403
 import re
-import subprocess
 import typing
 
-####################
-# Sphinx extension #
-####################
-
 if typing.TYPE_CHECKING:
     import docutils.nodes
     import sphinx.application
 
 
+AUTOMODULE_TEMPLATE = """
+
+.. automodule:: {package}
+""".rstrip()
+
+
 def setup(app: sphinx.application.Sphinx) -> dict[str, str | bool]:
     """Entrypoint for Sphinx extensions, connects generation code to Sphinx event."""
-    app.connect('builder-inited', _package_docs)
+    app.connect('source-read', _append_automodule_on_source_read)
     app.connect('doctree-read', _load_on_doctree_read)
     app.connect('doctree-resolved', _save_on_doctree_resolved)
     app.add_config_value('package', default=None, rebuild='')
     return {'version': '1.0.0', 'parallel_read_safe': False, 'parallel_write_safe': False}
 
 
-def _package_docs(app: sphinx.application.Sphinx) -> None:
-    _main(docs_dir=pathlib.Path(app.confdir), package=app.config.package)
+def _append_automodule_on_source_read(
+    app: sphinx.application.Sphinx, docname: str, source: list[str]
+) -> None:
+    """Inject the automodule directive for the current per-package build.
+
+    Runs during Sphinx's ``source-read`` event, after the placeholder rst file has been loaded
+    and before it's parsed. In-memory mutation only — the on-disk file stays a placeholder, so
+    concurrent per-package builds don't step on each other.
+    """
+    package = app.config.package
+    if package is None:
+        return
+    subdir, _, p = package.rpartition('/')
+    canonical_path = ['charmlibs']
+    if subdir:
+        canonical_path.append(_normalize(subdir))
+    canonical_path.append(_normalize(p))
+    if docname != '/'.join(('reference', *canonical_path)):
+        return
+    import_name = canonical_path[-1].replace('-', '_')
+    source[0] = source[0] + AUTOMODULE_TEMPLATE.format(package=import_name)
 
 
 def _load_on_doctree_read(app: sphinx.application.Sphinx, doctree: docutils.nodes.document):
@@ -90,70 +115,9 @@ def _save_on_doctree_resolved(
     target.write_bytes(pickle.dumps((doctree, objects, modules, toc, toc_num_entries)))
 
 
-####################
-# generation logic #
-####################
-
-RST_TEMPLATE = """
-.. raw:: html
-
-   <style>
-      h1:before {{
-         content: "{import_prefix}";
-      }}
-   </style>
-
-.. _{label}:
-
-{import_name}
-{underline}
-""".strip()
-AUTOMODULE_TEMPLATE = """
-
-.. automodule:: {package}
-""".rstrip()
-
-
-def _main(docs_dir: pathlib.Path, package: str | None) -> None:
-    """Write automodule file for package and placeholders rst files for all other packages."""
-    root = docs_dir.parent
-    ref_dir = docs_dir / 'reference'
-    (ref_dir / 'charmlibs' / 'interfaces').mkdir(parents=True, exist_ok=True)
-    ls = root / '.scripts' / 'ls.py'
-    cmd = [ls, 'packages', '--exclude-examples', '--exclude-placeholders', '--exclude-testing']
-    packages = json.loads(subprocess.check_output(cmd, text=True))
-    for raw_package in packages:
-        subdir, _, p = raw_package.rpartition('/')
-        canonical_path = ['charmlibs']
-        if subdir:
-            canonical_path.append(_normalize(subdir))
-        canonical_path.append(_normalize(p))
-        *import_prefix_parts, import_name = (part.replace('-', '_') for part in canonical_path)
-        content = RST_TEMPLATE.format(
-            import_prefix='.'.join(import_prefix_parts) + '.',
-            import_name=import_name,
-            underline='=' * len(p),
-            label='-'.join(canonical_path),
-        )
-        if package is not None and package == str(pathlib.Path(subdir, p)):
-            content += AUTOMODULE_TEMPLATE.format(package=import_name)
-        path = ref_dir.joinpath(*canonical_path).with_suffix('.rst')
-        _write_if_needed(path=path, content=content)
-
-
 def _normalize(name: str) -> str:
     """Normalize distribution package name according to PyPI rules.
 
     https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization
     """
     return re.sub(r'[-_.]+', '-', name).lower()
-
-
-def _write_if_needed(path: pathlib.Path, content: str) -> None:
-    """Write to path only if contents are different.
-
-    This allows sphinx-build to skip rebuilding pages that depend on the output of this extension
-    if the output hasn't actually changed.
-    """
-    if not path.exists() or path.read_text() != content:
-        path.write_text(content)

.docs/scripts/package_docs_preprocessor.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env -S uv run --script --no-project
+
+# /// script
+# requires-python = ">=3.12"
+# ///
+
+# Copyright 2025 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Write placeholder rst files for every package's reference docs.
+
+Packages are not guaranteed to have compatible dependencies, so we generate their reference docs
+in separate invocations of ``sphinx-build``. Each of those builds needs a page to hang its
+``automodule`` output on, and this script writes those pages.
+
+This is a standalone preprocessor script rather than part of the companion ``package_docs``
+extension. The placeholders are identical no matter which package is being built, so writing
+them is a one-time preparation of the source tree rather than per-build work. Doing it once, up
+front, is also what makes it safe to run the per-package builds concurrently: they share the
+same source tree but never mutate each other's rst files. The ``package_docs`` extension appends
+the ``automodule`` directive for the current package in memory at ``source-read`` time, so the
+on-disk files stay plain placeholders.
+
+Run from ``just docs``; see ``docs.just`` for the invocation.
+"""
+
+from __future__ import annotations
+
+import json
+import pathlib
+import re
+import subprocess
+
+_DOCS_DIR = pathlib.Path(__file__).parent.parent.resolve()
+_REPO_ROOT = _DOCS_DIR.parent
+RST_TEMPLATE = """
+.. raw:: html
+
+   <style>
+      h1:before {{
+         content: "{import_prefix}";
+      }}
+   </style>
+
+.. _{label}:
+
+{import_name}
+{underline}
+""".strip()
+
+
+def _main() -> None:
+    """Write placeholder rst files for every package."""
+    ref_dir = _DOCS_DIR / 'reference'
+    (ref_dir / 'charmlibs' / 'interfaces').mkdir(parents=True, exist_ok=True)
+    ls = _REPO_ROOT / '.scripts' / 'ls.py'
+    cmd = [ls, 'packages', '--exclude-examples', '--exclude-placeholders', '--exclude-testing']
+    packages = json.loads(subprocess.check_output(cmd, text=True))
+    for raw_package in packages:
+        subdir, _, p = raw_package.rpartition('/')
+        canonical_path = ['charmlibs']
+        if subdir:
+            canonical_path.append(_normalize(subdir))
+        canonical_path.append(_normalize(p))
+        *import_prefix_parts, import_name = (part.replace('-', '_') for part in canonical_path)
+        content = RST_TEMPLATE.format(
+            import_prefix='.'.join(import_prefix_parts) + '.',
+            import_name=import_name,
+            underline='=' * len(p),
+            label='-'.join(canonical_path),
+        )
+        path = ref_dir.joinpath(*canonical_path).with_suffix('.rst')
+        _write_if_needed(path=path, content=content)
+
+
+def _normalize(name: str) -> str:
+    """Normalize distribution package name according to PyPI rules.
+
+    https://packaging.python.org/en/latest/specifications/name-normalization/#name-normalization
+    """
+    return re.sub(r'[-_.]+', '-', name).lower()
+
+
+def _write_if_needed(path: pathlib.Path, content: str) -> None:
+    """Write to path only if contents are different.
+
+    This allows sphinx-build to skip rebuilding pages that depend on this script's output if the
+    output hasn't actually changed.
+    """
+    if not path.exists() or path.read_text() != content:
+        path.write_text(content)
+
+
+if __name__ == '__main__':
+    _main()

docs.just

@@ -17,23 +17,41 @@ html *packages: _diataxis_docs (_packages packages)
 _diataxis_docs:
     ./scripts/diataxis_preprocessor.py
 
+[doc('Write the placeholder rst files that the per-package reference docs builds fill in.')]
+_package_docs:
+    ./scripts/package_docs_preprocessor.py
+
 [doc("""
 Run html builder once per package, with its deps installed, setting the package argument and suppressing reference warnings.
 
 This allows us to generate the reference docs for that package with autodoc, saving the doctrees and reference information
 to be combined into the final docs in a separate pass.
+
+Package builds run in parallel: each gets its own doctree cache and html outdir so they can't step
+on each other, and they read the placeholder rst files written once up front by the `_package_docs`
+recipe, injecting their own automodule in-memory during source-read (see
+`.docs/extensions/package_docs.py`). Since the on-disk files never carry an automodule, the parallel
+builds don't race on the source tree. Each build's captured stdout/stderr is printed atomically
+after it finishes.
 """)]
-_packages *packages:
+_packages *packages: _package_docs
     #!/usr/bin/env -S uv run --script --no-project
-    import json, pathlib, subprocess, sys
+    import concurrent.futures, json, os, pathlib, subprocess, sys
     ROOT = pathlib.Path('{{justfile_directory()}}')
+    BUILD_DIR = pathlib.Path('{{build_dir}}')
     packages = '{{packages}}'.split()
     if packages == ['-']:
         sys.exit()
     if not packages:
         cmd = [ROOT / '.scripts/ls.py', 'packages', '--exclude-examples', '--exclude-placeholders']
         packages = json.loads(subprocess.check_output(cmd, text=True))
-    for package in packages:
+
+    def build(package):
+        # Isolate per-build outputs (html, doctree cache) so parallel builds don't clobber each
+        # other. Only the .save/ pickles are shared, and they're keyed on unique per-package names.
+        pkg_slug = package.replace('/', '_')
+        outdir = BUILD_DIR / '_pkg_pass' / pkg_slug / 'html'
+        doctrees = BUILD_DIR / '_pkg_pass' / pkg_slug / 'doctrees'
         cmd = [
             'uvx',
             '--from', 'sphinx',
@@ -42,18 +60,35 @@ _packages *packages:
             'sphinx-build',
             '-T', '-W', '--keep-going',
             '-b', 'dirhtml',
-            '-d', '_dev/.doctrees',
+            '-d', str(doctrees),
             '-D', 'language=en',
             '-D', f'package={package}',
             # llms.txt is only generated in the final combined pass (the `html` recipe).
             # Leaving it enabled here triggers a nested base-mode build that regenerates the
             # interface reference docs, deleting the placeholder this package pass depends on.
             '-D', 'llms_txt_enabled=0',
             '-D', 'suppress_warnings=ref.ref,ref.doc,myst.xref_missing',
-            '.', '{{build_dir}}/html',
+            '.', str(outdir),
         ]
-        print(cmd)
-        subprocess.check_call(cmd)
+        result = subprocess.run(cmd, capture_output=True, text=True)
+        return package, cmd, result
+
+    max_workers = min(len(packages), os.cpu_count() or 1)
+    with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as pool:
+        futures = [pool.submit(build, p) for p in packages]
+        failures = 0
+        for future in concurrent.futures.as_completed(futures):
+            package, cmd, result = future.result()
+            print(f'--- {package} ---')
+            print(cmd)
+            if result.stdout:
+                sys.stdout.write(result.stdout)
+            if result.stderr:
+                sys.stderr.write(result.stderr)
+            if result.returncode != 0:
+                failures += 1
+                print(f'--- {package} FAILED (exit {result.returncode}) ---', file=sys.stderr)
+        sys.exit(failures)
 
 [doc('Show help for the docs recipes.')]
 help: