diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 850a80c2b..545c0cd43 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -46,3 +46,26 @@ jobs: uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: packages/pytest-taskgraph/dist + pypi-publish-sphinx-taskgraph: + name: upload release to PyPI + if: startsWith(github.ref, 'refs/tags/sphinx-taskgraph') + runs-on: ubuntu-latest + environment: sphinx-taskgraph-release + permissions: + id-token: write + steps: + - name: Checkout sources + uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + - name: Build sphinx-taskgraph package distributions + working-directory: packages/sphinx-taskgraph + run: | + pip install build + python -m build + - name: Publish package distributions to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: packages/sphinx-taskgraph/dist diff --git a/docs/conf.py b/docs/conf.py index d4ae644eb..73edaeb3d 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -32,6 +32,7 @@ "sphinx.ext.napoleon", "sphinxarg.ext", "sphinxcontrib.mermaid", + "sphinx_taskgraph", ] # Add any paths that contain templates here, relative to this directory. diff --git a/docs/reference/source/taskgraph.transforms.rst b/docs/reference/source/taskgraph.transforms.rst index e20a5dc1a..03069d0d6 100644 --- a/docs/reference/source/taskgraph.transforms.rst +++ b/docs/reference/source/taskgraph.transforms.rst @@ -36,6 +36,14 @@ taskgraph.transforms.cached\_tasks module :undoc-members: :show-inheritance: +taskgraph.transforms.chunking module +------------------------------------ + +.. automodule:: taskgraph.transforms.chunking + :members: + :undoc-members: + :show-inheritance: + taskgraph.transforms.code\_review module ---------------------------------------- @@ -60,6 +68,14 @@ taskgraph.transforms.fetch module :undoc-members: :show-inheritance: +taskgraph.transforms.matrix module +---------------------------------- + +.. automodule:: taskgraph.transforms.matrix + :members: + :undoc-members: + :show-inheritance: + taskgraph.transforms.notify module ---------------------------------- @@ -76,6 +92,14 @@ taskgraph.transforms.task module :undoc-members: :show-inheritance: +taskgraph.transforms.task_context module +---------------------------------------- + +.. automodule:: taskgraph.transforms.task_context + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/docs/reference/transforms/chunking.rst b/docs/reference/transforms/chunking.rst index ae4759ee4..df614b32c 100644 --- a/docs/reference/transforms/chunking.rst +++ b/docs/reference/transforms/chunking.rst @@ -7,6 +7,12 @@ The :mod:`taskgraph.transforms.chunking` module contains transforms that aid in splitting a single entry in a ``kind`` into multiple tasks. This is often used to parallelize expensive or slow work. +Schema +------ + +All tasks must conform to the :py:data:`chunking schema +`. + Usage ----- diff --git a/docs/reference/transforms/from_deps.rst b/docs/reference/transforms/from_deps.rst index da16c220e..9b55f0127 100644 --- a/docs/reference/transforms/from_deps.rst +++ b/docs/reference/transforms/from_deps.rst @@ -11,6 +11,11 @@ These transforms are useful when you want to create follow-up tasks for some indeterminate subset of existing tasks. For example, maybe you want to run a signing task after each build task. +Schema +------ + +All tasks must conform to the :py:data:`from_deps schema +`. Usage ----- diff --git a/docs/reference/transforms/matrix.rst b/docs/reference/transforms/matrix.rst index 746b5472b..61c56d369 100644 --- a/docs/reference/transforms/matrix.rst +++ b/docs/reference/transforms/matrix.rst @@ -9,6 +9,12 @@ task into many subtasks based on a defined matrix. These transforms are useful if you need to have many tasks that are very similar except for some small configuration differences. +Schema +------ + +All tasks must conform to the :py:data:`matrix schema +`. + Usage ----- diff --git a/docs/reference/transforms/task_context.rst b/docs/reference/transforms/task_context.rst index 7232b9cae..491f2d723 100644 --- a/docs/reference/transforms/task_context.rst +++ b/docs/reference/transforms/task_context.rst @@ -9,6 +9,12 @@ until ``taskgraph`` runs. This data can be provided in a few ways, as described below. +Schema +------ + +All tasks must conform to the :py:data:`task_context schema +`. + Usage ----- diff --git a/packages/sphinx-taskgraph/README.md b/packages/sphinx-taskgraph/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/packages/sphinx-taskgraph/pyproject.toml b/packages/sphinx-taskgraph/pyproject.toml new file mode 100644 index 000000000..d4e40b6d3 --- /dev/null +++ b/packages/sphinx-taskgraph/pyproject.toml @@ -0,0 +1,17 @@ +[project] +name = "sphinx-taskgraph" +version = "0.1.0" +description = "Sphinx extensions to assist with Taskgraph documentation" +readme = "README.md" +authors = [ + { name = "Andrew Halberstadt", email = "ahal@mozilla.com" } +] +requires-python = ">=3.8" +dependencies = [ + "sphinx", + "voluptuous", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" diff --git a/packages/sphinx-taskgraph/src/sphinx_taskgraph/__init__.py b/packages/sphinx-taskgraph/src/sphinx_taskgraph/__init__.py new file mode 100644 index 000000000..a772f0ecf --- /dev/null +++ b/packages/sphinx-taskgraph/src/sphinx_taskgraph/__init__.py @@ -0,0 +1,23 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Sphinx extension for rendering taskgraph schemas using autodoc with custom formatting. +""" + + +def setup(app): + """ + Entry point for the Sphinx extension. + """ + from .autoschema import SchemaDocumenter + + # Register the custom autodocumenter. + app.add_autodocumenter(SchemaDocumenter) + + # Return metadata about the extension. + return { + "version": "1.0", + "parallel_read_safe": True, + "parallel_write_safe": True, + } diff --git a/packages/sphinx-taskgraph/src/sphinx_taskgraph/autoschema.py b/packages/sphinx-taskgraph/src/sphinx_taskgraph/autoschema.py new file mode 100644 index 000000000..dba9af253 --- /dev/null +++ b/packages/sphinx-taskgraph/src/sphinx_taskgraph/autoschema.py @@ -0,0 +1,179 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Custom autodoc integration for rendering `taskgraph.util.schema.Schema` instances. +""" + +import typing as t +from textwrap import dedent +from types import FunctionType, NoneType + +from sphinx.ext.autodoc import ClassDocumenter +from voluptuous import All, Any, Exclusive, Length, Marker, Schema + +if t.TYPE_CHECKING: + from docutils.statemachine import StringList + + +def name(item: t.Any) -> str: + if isinstance(item, (bool, int, str, NoneType)): + return str(item) + + if isinstance(item, (type, FunctionType)): + return item.__name__ + + if isinstance(item, Length): + return repr(item) + + return item.__class__.__name__ + + +def get_type_str(obj: t.Any) -> tuple[str, bool]: + # Handle simple subtypes for lists and dicts so that we display + # `type: dict[str, Any]` inline rather than expanded out at the + # bottom. + subtype_str = "" + if isinstance(obj, (list, Any)): + items = obj + if isinstance(items, Any): + items = items.validators + + if all(not isinstance(i, (list, dict, Any)) for i in items): + subtype_str = f"[{' | '.join(name(i) for i in items)}]" + + elif isinstance(obj, dict): + if all(not isinstance(k, (Marker, Any)) for k in obj.keys()) and all( + not isinstance(v, (list, dict, Any, All)) for v in obj.values() + ): + subtype_str = f"[{' | '.join(name(k) for k in obj.keys())}, {' | '.join({name(v) for v in obj.values()})}]" + + return ( + f"{name(obj)}{subtype_str}", + bool(subtype_str), + ) + + +def iter_schema_lines(obj: t.Any, indent: int = 0) -> t.Generator[str, None, None]: + prefix = " " * indent + arg_prefix = " " * (indent + 2) + + if isinstance(obj, Schema): + # Display whether extra keys are allowed + extra = obj._extra_to_name[obj.extra].split("_")[0].lower() + yield f"{prefix}extra keys: {extra}{'d' if extra[-1] == 'e' else 'ed'}" + yield "" + yield from iter_schema_lines(obj.schema, indent) + return + + if isinstance(obj, dict): + for i, (key, value) in enumerate(obj.items()): + subtypes_handled = False + + # Handle optionally_keyed_by + keyed_by_str = "" + if isinstance(value, FunctionType): + keyed_by_str = ", ".join(getattr(value, "fields", "")) + value = getattr(value, "schema", value) + + # If the key is a marker (aka Required, Optional, Exclusive), + # display additional information if available, like the + # description. + if isinstance(key, Marker): + # Add marker name and group for Exclusive. + marker_str = f"{name(key).lower()}" + if isinstance(key, Exclusive): + marker_str += f"={key.group_of_exclusion}" + + # Make it clear if an allowed value must be constant. + type_ = "type" + if isinstance(obj, (bool, int, str, NoneType)): + type_ = "constant" + + # Create the key header + type lines. + yield f"{prefix}{key.schema} ({marker_str})" + type_str, subtypes_handled = get_type_str(value) + yield f"{arg_prefix}{type_}: {type_str}" + + # Create the keyed-by line if needed. + if keyed_by_str: + yield "" + yield f"{arg_prefix}optionally keyed by: {keyed_by_str}" + + # Create the description if needed. + if desc := getattr(key, "description", None): + yield "" + yield arg_prefix + f"\n{arg_prefix}".join( + dedent(l) for l in desc.splitlines() + ) + + elif isinstance(key, Any): + type_str, subtypes_handled = get_type_str(key) + yield f"{prefix}{type_str}: {name(value)}" + + else: + # If not a marker, simply create a `key: value` line. + yield f"{prefix}{name(key)}: {name(value)}" + + yield "" + + # Recurse into values that contain additional schema, unless the + # types for said containers are simple and we handled them in the + # type line. + if isinstance(value, (list, dict, All, Any)) and not subtypes_handled: + yield from iter_schema_lines(value, indent + 2) + + elif isinstance(obj, (list, All, Any)): + # Recurse into list, All and Any markers. + op = "or" if isinstance(obj, (list, Any)) else "and" + if isinstance(obj, (All, Any)): + obj = obj.validators + + for i, item in enumerate(obj): + if i != 0: + yield "" + yield f"{prefix}{op}" + yield "" + + type_, subtypes_handled = get_type_str(item) + yield f"{arg_prefix}{type_}" + yield "" + + if isinstance(item, (list, dict, All, Any)) and not subtypes_handled: + yield from iter_schema_lines(item, indent + 2) + else: + # Create line for leaf values. + yield prefix + name(obj) + yield "" + + +class SchemaDocumenter(ClassDocumenter): + """ + Custom Sphinx autodocumenter for instances of `Schema`. + """ + + # Document only `Schema` instances. + objtype = "schema" + directivetype = "class" + content_indent = " " + + # Priority over the default ClassDocumenter. + priority = 10 + + @classmethod + def can_document_member( + cls, member: object, membername: str, isattr: bool, parent: object + ) -> bool: + return isinstance(member, Schema) + + def add_directive_header(self, sig: str) -> None: + super().add_directive_header(sig) + + def add_content( + self, more_content: t.Union["StringList", None], no_docstring: bool = False + ) -> None: + # Format schema rules recursively. + for line in iter_schema_lines(self.object): + self.add_line(line, "") + self.add_line("", "") diff --git a/pyproject.toml b/pyproject.toml index 5fab5cc2d..71c4726bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,9 +46,13 @@ Issues = "https://github.com/taskcluster/taskgraph/issues" [tool.uv.sources] pytest-taskgraph = { workspace = true } +sphinx-taskgraph = { workspace = true } [tool.uv.workspace] -members = ["packages/pytest-taskgraph"] +members = [ + "packages/pytest-taskgraph", + "packages/sphinx-taskgraph", +] [tool.uv] dev-dependencies = [ @@ -64,6 +68,7 @@ dev-dependencies = [ "sphinx-autobuild", "sphinx-argparse", "sphinx-book-theme >=1", + "sphinx-taskgraph", "sphinxcontrib-mermaid", "zstandard", ] diff --git a/src/taskgraph/config.py b/src/taskgraph/config.py index 1f4eca909..ef13f3d22 100644 --- a/src/taskgraph/config.py +++ b/src/taskgraph/config.py @@ -19,6 +19,8 @@ logger = logging.getLogger(__name__) + +#: Schema for the graph config graph_config_schema = Schema( { # The trust-domain for this graph. @@ -102,7 +104,6 @@ Extra: object, } ) -"""Schema for GraphConfig""" @dataclass(frozen=True, eq=False) diff --git a/src/taskgraph/decision.py b/src/taskgraph/decision.py index 69d490f0e..705e90b61 100644 --- a/src/taskgraph/decision.py +++ b/src/taskgraph/decision.py @@ -39,6 +39,7 @@ } +#: Schema for try_task_config.json version 2 try_task_config_schema_v2 = Schema( { Optional("parameters"): {str: object}, diff --git a/src/taskgraph/parameters.py b/src/taskgraph/parameters.py index 1aa169f13..1f8f640ae 100644 --- a/src/taskgraph/parameters.py +++ b/src/taskgraph/parameters.py @@ -28,7 +28,8 @@ class ParameterMismatch(Exception): """Raised when a parameters.yml has extra or missing parameters.""" -# Please keep this list sorted and in sync with docs/reference/parameters.rst +#: Schema for base parameters. +#: Please keep this list sorted and in sync with docs/reference/parameters.rst base_schema = Schema( { Required("base_repository"): str, diff --git a/src/taskgraph/transforms/chunking.py b/src/taskgraph/transforms/chunking.py index 31d7eff82..d8ad89dd2 100644 --- a/src/taskgraph/transforms/chunking.py +++ b/src/taskgraph/transforms/chunking.py @@ -10,6 +10,7 @@ from taskgraph.util.schema import Schema from taskgraph.util.templates import substitute +#: Schema for chunking transforms CHUNK_SCHEMA = Schema( { # Optional, so it can be used for a subset of tasks in a kind diff --git a/src/taskgraph/transforms/docker_image.py b/src/taskgraph/transforms/docker_image.py index 50544b6b8..33294610a 100644 --- a/src/taskgraph/transforms/docker_image.py +++ b/src/taskgraph/transforms/docker_image.py @@ -30,6 +30,7 @@ transforms = TransformSequence() +#: Schema for docker_image transforms docker_image_schema = Schema( { # Name of the docker image. diff --git a/src/taskgraph/transforms/fetch.py b/src/taskgraph/transforms/fetch.py index 38604bbfb..98f520d97 100644 --- a/src/taskgraph/transforms/fetch.py +++ b/src/taskgraph/transforms/fetch.py @@ -23,6 +23,7 @@ CACHE_TYPE = "content.v1" +#: Schema for fetch transforms FETCH_SCHEMA = Schema( { # Name of the task. @@ -53,7 +54,6 @@ } ) - # define a collection of payload builders, depending on the worker implementation fetch_builders = {} diff --git a/src/taskgraph/transforms/from_deps.py b/src/taskgraph/transforms/from_deps.py index ba46c16b3..31a967a66 100644 --- a/src/taskgraph/transforms/from_deps.py +++ b/src/taskgraph/transforms/from_deps.py @@ -23,6 +23,7 @@ from taskgraph.util.schema import Schema, validate_schema from taskgraph.util.set_name import SET_NAME_MAP +#: Schema for from_deps transforms FROM_DEPS_SCHEMA = Schema( { Required("from-deps"): { @@ -38,7 +39,7 @@ and copy attributes (if `copy-attributes` is True). """.lstrip() ), - ): list, + ): [str], Optional( "set-name", description=dedent( @@ -111,7 +112,6 @@ Extra: object, }, ) -"""Schema for from_deps transforms.""" transforms = TransformSequence() transforms.add_validate(FROM_DEPS_SCHEMA) diff --git a/src/taskgraph/transforms/matrix.py b/src/taskgraph/transforms/matrix.py index 94a55b575..11718b011 100644 --- a/src/taskgraph/transforms/matrix.py +++ b/src/taskgraph/transforms/matrix.py @@ -16,6 +16,7 @@ from taskgraph.util.schema import Schema from taskgraph.util.templates import substitute_task_fields +#: Schema for matrix transforms MATRIX_SCHEMA = Schema( { Required("name"): str, @@ -60,7 +61,6 @@ Extra: object, }, ) -"""Schema for matrix transforms.""" transforms = TransformSequence() transforms.add_validate(MATRIX_SCHEMA) diff --git a/src/taskgraph/transforms/notify.py b/src/taskgraph/transforms/notify.py index c103c8f91..9c0152dad 100644 --- a/src/taskgraph/transforms/notify.py +++ b/src/taskgraph/transforms/notify.py @@ -54,6 +54,7 @@ } """Map each type to its primary key that will be used in the route.""" +#: Schema for notify transforms NOTIFY_SCHEMA = Schema( { Exclusive("notify", "config"): { @@ -90,7 +91,6 @@ }, extra=ALLOW_EXTRA, ) -"""Notify schema.""" transforms = TransformSequence() transforms.add_validate(NOTIFY_SCHEMA) diff --git a/src/taskgraph/transforms/run/__init__.py b/src/taskgraph/transforms/run/__init__.py index 442788e18..e2706bd32 100644 --- a/src/taskgraph/transforms/run/__init__.py +++ b/src/taskgraph/transforms/run/__init__.py @@ -36,7 +36,7 @@ Optional("verify-hash"): bool, } -# Schema for a build description +#: Schema for a run transforms run_description_schema = Schema( { # The name of the task and the task's label. At least one must be specified, diff --git a/src/taskgraph/transforms/run/index_search.py b/src/taskgraph/transforms/run/index_search.py index 527ca5528..7436f010f 100644 --- a/src/taskgraph/transforms/run/index_search.py +++ b/src/taskgraph/transforms/run/index_search.py @@ -16,6 +16,8 @@ transforms = TransformSequence() + +#: Schema for run.using index-search run_task_schema = Schema( { Required("using"): "index-search", diff --git a/src/taskgraph/transforms/run/run_task.py b/src/taskgraph/transforms/run/run_task.py index 8830b9ac5..1cd46ac82 100644 --- a/src/taskgraph/transforms/run/run_task.py +++ b/src/taskgraph/transforms/run/run_task.py @@ -25,6 +25,8 @@ "powershell": ["powershell.exe", "-ExecutionPolicy", "Bypass"], } + +#: Schema for run.using run_task run_task_schema = Schema( { Required("using"): "run-task", diff --git a/src/taskgraph/transforms/run/toolchain.py b/src/taskgraph/transforms/run/toolchain.py index 02f04940c..6730ac3b2 100644 --- a/src/taskgraph/transforms/run/toolchain.py +++ b/src/taskgraph/transforms/run/toolchain.py @@ -21,6 +21,7 @@ CACHE_TYPE = "toolchains.v3" +#: Schema for run.using toolchain toolchain_run_schema = Schema( { Required("using"): "toolchain-script", diff --git a/src/taskgraph/transforms/task.py b/src/taskgraph/transforms/task.py index 320524261..38c1835c5 100644 --- a/src/taskgraph/transforms/task.py +++ b/src/taskgraph/transforms/task.py @@ -48,7 +48,7 @@ def _run_task_suffix(): return hash_path(RUN_TASK)[0:20] -# A task description is a general description of a TaskCluster task +#: Schema for the task transforms task_description_schema = Schema( { # the label for this task diff --git a/src/taskgraph/transforms/task_context.py b/src/taskgraph/transforms/task_context.py index 0b412829d..9e013e5d5 100644 --- a/src/taskgraph/transforms/task_context.py +++ b/src/taskgraph/transforms/task_context.py @@ -7,6 +7,7 @@ from taskgraph.util.templates import deep_get, substitute_task_fields from taskgraph.util.yaml import load_yaml +#: Schema for the task_context transforms SCHEMA = Schema( { Optional("name"): str, diff --git a/src/taskgraph/util/schema.py b/src/taskgraph/util/schema.py index 590273676..ba72ff079 100644 --- a/src/taskgraph/util/schema.py +++ b/src/taskgraph/util/schema.py @@ -59,6 +59,9 @@ def validator(obj): return res return Schema(schema)(obj) + # set to assist autodoc + setattr(validator, "schema", schema) + setattr(validator, "fields", fields) return validator diff --git a/uv.lock b/uv.lock index 090d7a8bc..53833e60b 100644 --- a/uv.lock +++ b/uv.lock @@ -11,6 +11,7 @@ resolution-markers = [ [manifest] members = [ "pytest-taskgraph", + "sphinx-taskgraph", "taskcluster-taskgraph", ] @@ -1820,6 +1821,25 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/9e/c41d68be04eef5b6202b468e0f90faf0c469f3a03353f2a218fd78279710/sphinx_book_theme-1.1.4-py3-none-any.whl", hash = "sha256:843b3f5c8684640f4a2d01abd298beb66452d1b2394cd9ef5be5ebd5640ea0e1", size = 433952, upload-time = "2025-02-20T16:32:31.009Z" }, ] +[[package]] +name = "sphinx-taskgraph" +version = "0.1.0" +source = { editable = "packages/sphinx-taskgraph" } +dependencies = [ + { name = "sphinx", version = "6.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "sphinx", version = "7.4.7", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.9.*'" }, + { name = "sphinx", version = "8.1.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version == '3.10.*'" }, + { name = "sphinx", version = "8.2.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, + { name = "voluptuous", version = "0.14.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, + { name = "voluptuous", version = "0.15.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, +] + +[package.metadata] +requires-dist = [ + { name = "sphinx" }, + { name = "voluptuous" }, +] + [[package]] name = "sphinxcontrib-applehelp" version = "1.0.4" @@ -2038,6 +2058,7 @@ dev = [ { name = "sphinx-autobuild", version = "2024.10.3", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, { name = "sphinx-book-theme", version = "1.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.9'" }, { name = "sphinx-book-theme", version = "1.1.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.9'" }, + { name = "sphinx-taskgraph" }, { name = "sphinxcontrib-mermaid" }, { name = "zstandard" }, ] @@ -2073,6 +2094,7 @@ dev = [ { name = "sphinx-argparse" }, { name = "sphinx-autobuild" }, { name = "sphinx-book-theme", specifier = ">=1" }, + { name = "sphinx-taskgraph", editable = "packages/sphinx-taskgraph" }, { name = "sphinxcontrib-mermaid" }, { name = "zstandard" }, ]