Improve schema handling in documentation

.github/workflows/pypi-publish.yml

@@ -46,3 +46,26 @@ jobs:
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
           packages-dir: packages/pytest-taskgraph/dist
+  pypi-publish-sphinx-taskgraph:
+    name: upload release to PyPI
+    if: startsWith(github.ref, 'refs/tags/sphinx-taskgraph')
+    runs-on: ubuntu-latest
+    environment: sphinx-taskgraph-release
+    permissions:
+      id-token: write
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: 'pip'
+      - name: Build sphinx-taskgraph package distributions
+        working-directory: packages/sphinx-taskgraph
+        run: |
+          pip install build
+          python -m build
+      - name: Publish package distributions to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: packages/sphinx-taskgraph/dist

docs/conf.py

@@ -32,6 +32,7 @@
     "sphinx.ext.napoleon",
     "sphinxarg.ext",
     "sphinxcontrib.mermaid",
+    "sphinx_taskgraph",
 ]
 
 # Add any paths that contain templates here, relative to this directory.

docs/reference/source/taskgraph.transforms.rst

@@ -36,6 +36,14 @@ taskgraph.transforms.cached\_tasks module
    :undoc-members:
    :show-inheritance:
 
+taskgraph.transforms.chunking module
+------------------------------------
+
+.. automodule:: taskgraph.transforms.chunking
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 taskgraph.transforms.code\_review module
 ----------------------------------------
 
@@ -60,6 +68,14 @@ taskgraph.transforms.fetch module
    :undoc-members:
    :show-inheritance:
 
+taskgraph.transforms.matrix module
+----------------------------------
+
+.. automodule:: taskgraph.transforms.matrix
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 taskgraph.transforms.notify module
 ----------------------------------
 
@@ -76,6 +92,14 @@ taskgraph.transforms.task module
    :undoc-members:
    :show-inheritance:
 
+taskgraph.transforms.task_context module
+----------------------------------------
+
+.. automodule:: taskgraph.transforms.task_context
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 Module contents
 ---------------
 

docs/reference/transforms/chunking.rst

@@ -7,6 +7,12 @@ The :mod:`taskgraph.transforms.chunking` module contains transforms that aid
 in splitting a single entry in a ``kind`` into multiple tasks. This is often
 used to parallelize expensive or slow work.
 
+Schema
+------
+
+All tasks must conform to the :py:data:`chunking schema
+<taskgraph.transforms.chunking.CHUNK_SCHEMA>`.
+
 Usage
 -----
 

docs/reference/transforms/from_deps.rst

@@ -11,6 +11,11 @@ These transforms are useful when you want to create follow-up tasks for some
 indeterminate subset of existing tasks. For example, maybe you want to run
 a signing task after each build task.
 
+Schema
+------
+
+All tasks must conform to the :py:data:`from_deps schema
+<taskgraph.transforms.from_deps.FROM_DEPS_SCHEMA>`.
 
 Usage
 -----

docs/reference/transforms/matrix.rst

@@ -9,6 +9,12 @@ task into many subtasks based on a defined matrix.
 These transforms are useful if you need to have many tasks that are very
 similar except for some small configuration differences.
 
+Schema
+------
+
+All tasks must conform to the :py:data:`matrix schema
+<taskgraph.transforms.matrix.MATRIX_SCHEMA>`.
+
 Usage
 -----
 

docs/reference/transforms/task_context.rst

@@ -9,6 +9,12 @@ until ``taskgraph`` runs.
 
 This data can be provided in a few ways, as described below.
 
+Schema
+------
+
+All tasks must conform to the :py:data:`task_context schema
+<taskgraph.transforms.task_context.SCHEMA>`.
+
 Usage
 -----
 

packages/sphinx-taskgraph/pyproject.toml

@@ -0,0 +1,17 @@
+[project]
+name = "sphinx-taskgraph"
+version = "0.1.0"
+description = "Sphinx extensions to assist with Taskgraph documentation"
+readme = "README.md"
+authors = [
+    { name = "Andrew Halberstadt", email = "ahal@mozilla.com" }
+]
+requires-python = ">=3.8"
+dependencies = [
+    "sphinx",
+    "voluptuous",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"

packages/sphinx-taskgraph/src/sphinx_taskgraph/__init__.py

@@ -0,0 +1,23 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Sphinx extension for rendering taskgraph schemas using autodoc with custom formatting.
+"""
+
+
+def setup(app):
+    """
+    Entry point for the Sphinx extension.
+    """
+    from .autoschema import SchemaDocumenter
+
+    # Register the custom autodocumenter.
+    app.add_autodocumenter(SchemaDocumenter)
+
+    # Return metadata about the extension.
+    return {
+        "version": "1.0",
+        "parallel_read_safe": True,
+        "parallel_write_safe": True,
+    }

packages/sphinx-taskgraph/src/sphinx_taskgraph/autoschema.py

@@ -0,0 +1,179 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Custom autodoc integration for rendering `taskgraph.util.schema.Schema` instances.
+"""
+
+import typing as t
+from textwrap import dedent
+from types import FunctionType, NoneType
+
+from sphinx.ext.autodoc import ClassDocumenter
+from voluptuous import All, Any, Exclusive, Length, Marker, Schema
+
+if t.TYPE_CHECKING:
+    from docutils.statemachine import StringList
+
+
+def name(item: t.Any) -> str:
+    if isinstance(item, (bool, int, str, NoneType)):
+        return str(item)
+
+    if isinstance(item, (type, FunctionType)):
+        return item.__name__
+
+    if isinstance(item, Length):
+        return repr(item)
+
+    return item.__class__.__name__
+
+
+def get_type_str(obj: t.Any) -> tuple[str, bool]:
+    # Handle simple subtypes for lists and dicts so that we display
+    # `type: dict[str, Any]` inline rather than expanded out at the
+    # bottom.
+    subtype_str = ""
+    if isinstance(obj, (list, Any)):
+        items = obj
+        if isinstance(items, Any):
+            items = items.validators
+
+        if all(not isinstance(i, (list, dict, Any)) for i in items):
+            subtype_str = f"[{' | '.join(name(i) for i in items)}]"
+
+    elif isinstance(obj, dict):
+        if all(not isinstance(k, (Marker, Any)) for k in obj.keys()) and all(
+            not isinstance(v, (list, dict, Any, All)) for v in obj.values()
+        ):
+            subtype_str = f"[{' | '.join(name(k) for k in obj.keys())}, {' | '.join({name(v) for v in obj.values()})}]"
+
+    return (
+        f"{name(obj)}{subtype_str}",
+        bool(subtype_str),
+    )
+
+
+def iter_schema_lines(obj: t.Any, indent: int = 0) -> t.Generator[str, None, None]:
+    prefix = " " * indent
+    arg_prefix = " " * (indent + 2)
+
+    if isinstance(obj, Schema):
+        # Display whether extra keys are allowed
+        extra = obj._extra_to_name[obj.extra].split("_")[0].lower()
+        yield f"{prefix}extra keys: {extra}{'d' if extra[-1] == 'e' else 'ed'}"
+        yield ""
+        yield from iter_schema_lines(obj.schema, indent)
+        return
+
+    if isinstance(obj, dict):
+        for i, (key, value) in enumerate(obj.items()):
+            subtypes_handled = False
+
+            # Handle optionally_keyed_by
+            keyed_by_str = ""
+            if isinstance(value, FunctionType):
+                keyed_by_str = ", ".join(getattr(value, "fields", ""))
+                value = getattr(value, "schema", value)
+
+            # If the key is a marker (aka Required, Optional, Exclusive),
+            # display additional information if available, like the
+            # description.
+            if isinstance(key, Marker):
+                # Add marker name and group for Exclusive.
+                marker_str = f"{name(key).lower()}"
+                if isinstance(key, Exclusive):
+                    marker_str += f"={key.group_of_exclusion}"
+
+                # Make it clear if an allowed value must be constant.
+                type_ = "type"
+                if isinstance(obj, (bool, int, str, NoneType)):
+                    type_ = "constant"
+
+                # Create the key header + type lines.
+                yield f"{prefix}{key.schema} ({marker_str})"
+                type_str, subtypes_handled = get_type_str(value)
+                yield f"{arg_prefix}{type_}: {type_str}"
+
+                # Create the keyed-by line if needed.
+                if keyed_by_str:
+                    yield ""
+                    yield f"{arg_prefix}optionally keyed by: {keyed_by_str}"
+
+                # Create the description if needed.
+                if desc := getattr(key, "description", None):
+                    yield ""
+                    yield arg_prefix + f"\n{arg_prefix}".join(
+                        dedent(l) for l in desc.splitlines()
+                    )
+
+            elif isinstance(key, Any):
+                type_str, subtypes_handled = get_type_str(key)
+                yield f"{prefix}{type_str}: {name(value)}"
+
+            else:
+                # If not a marker, simply create a `key: value` line.
+                yield f"{prefix}{name(key)}: {name(value)}"
+
+            yield ""
+
+            # Recurse into values that contain additional schema, unless the
+            # types for said containers are simple and we handled them in the
+            # type line.
+            if isinstance(value, (list, dict, All, Any)) and not subtypes_handled:
+                yield from iter_schema_lines(value, indent + 2)
+
+    elif isinstance(obj, (list, All, Any)):
+        # Recurse into list, All and Any markers.
+        op = "or" if isinstance(obj, (list, Any)) else "and"
+        if isinstance(obj, (All, Any)):
+            obj = obj.validators
+
+        for i, item in enumerate(obj):
+            if i != 0:
+                yield ""
+                yield f"{prefix}{op}"
+                yield ""
+
+            type_, subtypes_handled = get_type_str(item)
+            yield f"{arg_prefix}{type_}"
+            yield ""
+
+            if isinstance(item, (list, dict, All, Any)) and not subtypes_handled:
+                yield from iter_schema_lines(item, indent + 2)
+    else:
+        # Create line for leaf values.
+        yield prefix + name(obj)
+        yield ""
+
+
+class SchemaDocumenter(ClassDocumenter):
+    """
+    Custom Sphinx autodocumenter for instances of `Schema`.
+    """
+
+    # Document only `Schema` instances.
+    objtype = "schema"
+    directivetype = "class"
+    content_indent = "   "
+
+    # Priority over the default ClassDocumenter.
+    priority = 10
+
+    @classmethod
+    def can_document_member(
+        cls, member: object, membername: str, isattr: bool, parent: object
+    ) -> bool:
+        return isinstance(member, Schema)
+
+    def add_directive_header(self, sig: str) -> None:
+        super().add_directive_header(sig)
+
+    def add_content(
+        self, more_content: t.Union["StringList", None], no_docstring: bool = False
+    ) -> None:
+        # Format schema rules recursively.
+        for line in iter_schema_lines(self.object):
+            self.add_line(line, "")
+        self.add_line("", "")

pyproject.toml

@@ -46,9 +46,13 @@ Issues = "https://github.com/taskcluster/taskgraph/issues"
 
 [tool.uv.sources]
 pytest-taskgraph = { workspace = true }
+sphinx-taskgraph = { workspace = true }
 
 [tool.uv.workspace]
-members = ["packages/pytest-taskgraph"]
+members = [
+  "packages/pytest-taskgraph",
+  "packages/sphinx-taskgraph",
+]
 
 [tool.uv]
 dev-dependencies = [
@@ -64,6 +68,7 @@ dev-dependencies = [
   "sphinx-autobuild",
   "sphinx-argparse",
   "sphinx-book-theme >=1",
+  "sphinx-taskgraph",
   "sphinxcontrib-mermaid",
   "zstandard",
 ]

src/taskgraph/config.py

@@ -19,6 +19,8 @@
 
 logger = logging.getLogger(__name__)
 
+
+#: Schema for the graph config
 graph_config_schema = Schema(
     {
         # The trust-domain for this graph.
@@ -102,7 +104,6 @@
         Extra: object,
     }
 )
-"""Schema for GraphConfig"""
 
 
 @dataclass(frozen=True, eq=False)