From 5bdd30ca0271b165b5790c759de1ef1bd9c5f60c Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Thu, 23 Oct 2025 15:27:28 -0400 Subject: [PATCH] feat: add support for outputting a mermaid flowchart of the kind graph Especially in repositories with large graphs, it can be useful to visualize the relationship between the various kinds. I looked at directly outputting svg files of the graphs here, but decided against it because rendering mermaid diagrams requires a browser or puppeteer AFAICT; and I don't want to add that dependency. I also looked at adding support for visualizing entire graphs, but those quickly get very unwieldy and I decided against it. (This might be useful in conjunction with a new `--target-task` option, but that's more than I'm willing to take on at this time.) --- src/taskgraph/generator.py | 11 ++++ src/taskgraph/main.py | 88 ++++++++++++++++++++++++++ test/test_generator.py | 41 +++++++++++++ test/test_main.py | 123 ++++++++++++++++++++++++++++++++++++- 4 files changed, 262 insertions(+), 1 deletion(-) diff --git a/src/taskgraph/generator.py b/src/taskgraph/generator.py index a9599bcdb..cad82b3b1 100644 --- a/src/taskgraph/generator.py +++ b/src/taskgraph/generator.py @@ -250,6 +250,15 @@ def graph_config(self): """ return self._run_until("graph_config") + @property + def kind_graph(self): + """ + The dependency graph of kinds. + + @type: Graph + """ + return self._run_until("kind_graph") + def _load_kinds(self, graph_config, target_kinds=None): if target_kinds: # docker-image is an implicit dependency that never appears in @@ -422,6 +431,8 @@ def _run(self): set(target_kinds) | {"docker-image"} ) + yield "kind_graph", kind_graph + logger.info("Generating full task set") # Current parallel generation relies on multiprocessing, and forking. # This causes problems on Windows and macOS due to how new processes diff --git a/src/taskgraph/main.py b/src/taskgraph/main.py index d80f4709f..f4a9ba3c8 100644 --- a/src/taskgraph/main.py +++ b/src/taskgraph/main.py @@ -64,6 +64,31 @@ def format_taskgraph_yaml(taskgraph): return yaml.safe_dump(taskgraph.to_json(), default_flow_style=False) +def format_kind_graph_mermaid(kind_graph): + """ + Convert a kind dependency graph to Mermaid flowchart format. + + @param kind_graph: Graph object containing kind nodes and dependencies + @return: String representation of the graph in Mermaid format + """ + lines = ["flowchart TD"] + + # Add nodes (kinds) + for node in sorted(kind_graph.nodes): + # Sanitize node names for Mermaid (replace hyphens with underscores for IDs) + node_id = node.replace("-", "_") + lines.append(f" {node_id}[{node}]") + + # Add edges (dependencies) + # Reverse the edge direction: if left depends on right, show right --> left + for left, right, _ in sorted(kind_graph.edges): + left_id = left.replace("-", "_") + right_id = right.replace("-", "_") + lines.append(f" {right_id} --> {left_id}") + + return "\n".join(lines) + + def get_filtered_taskgraph(taskgraph, tasksregex, exclude_keys): """ Filter all the tasks on basis of a regular expression @@ -225,6 +250,69 @@ def logfile(spec): return returncode +@command( + "kind-graph", + help="Generate a Mermaid flowchart diagram source file for the kind dependency graph. To render as a graph, run the output of this command through the Mermaid CLI or an online renderer.", +) +@argument("--root", "-r", help="root of the taskgraph definition relative to topsrcdir") +@argument("--quiet", "-q", action="store_true", help="suppress all logging output") +@argument( + "--verbose", "-v", action="store_true", help="include debug-level logging output" +) +@argument( + "--parameters", + "-p", + default=None, + help="Parameters to use for the generation. Can be a path to file (.yml or " + ".json; see `taskcluster/docs/parameters.rst`), a url, of the form " + "`project=mozilla-central` to download latest parameters file for the specified " + "project from CI, or of the form `task-id=` to download " + "parameters from the specified decision task.", +) +@argument( + "-o", + "--output-file", + default=None, + help="file path to store generated output.", +) +@argument( + "-k", + "--target-kind", + dest="target_kinds", + action="append", + default=[], + help="only return kinds and their dependencies.", +) +def show_kind_graph(options): + from taskgraph.parameters import parameters_loader # noqa: PLC0415 + + if options.pop("verbose", False): + logging.root.setLevel(logging.DEBUG) + + setup_logging() + + target_kinds = options.get("target_kinds", []) + parameters = parameters_loader( + options.get("parameters"), + strict=False, + overrides={"target-kinds": target_kinds}, + ) + + tgg = get_taskgraph_generator(options.get("root"), parameters) + kind_graph = tgg.kind_graph + + output = format_kind_graph_mermaid(kind_graph) + + if output_file := options.get("output_file"): + with open(output_file, "w") as fh: + print(output, file=fh) + print(f"Kind graph written to {output_file}", file=sys.stderr) + else: + print(output) + + return 0 + + @command( "tasks", help="Show the full task set in the task graph. The full task set includes all tasks defined by any kind, without edges (dependencies) between them.", diff --git a/test/test_generator.py b/test/test_generator.py index 58f47dbea..fe1695574 100644 --- a/test/test_generator.py +++ b/test/test_generator.py @@ -338,3 +338,44 @@ def test_kind_load_tasks(monkeypatch, graph_config, parameters, datadir, kind_co ) tasks = kind.load_tasks(parameters, {}, False) assert tasks + + +def test_kind_graph(maketgg): + "The kind_graph property has all kinds and their dependencies" + tgg = maketgg( + kinds=[ + ("_fake3", {"kind-dependencies": ["_fake2", "_fake1"]}), + ("_fake2", {"kind-dependencies": ["_fake1"]}), + ("_fake1", {"kind-dependencies": []}), + ] + ) + kind_graph = tgg.kind_graph + assert isinstance(kind_graph, graph.Graph) + assert kind_graph.nodes == {"_fake1", "_fake2", "_fake3"} + assert kind_graph.edges == { + ("_fake3", "_fake2", "kind-dependency"), + ("_fake3", "_fake1", "kind-dependency"), + ("_fake2", "_fake1", "kind-dependency"), + } + + +def test_kind_graph_with_target_kinds(maketgg): + "The kind_graph property respects target_kinds parameter" + tgg = maketgg( + kinds=[ + ("_fake3", {"kind-dependencies": ["_fake2"]}), + ("_fake2", {"kind-dependencies": ["_fake1"]}), + ("_fake1", {"kind-dependencies": []}), + ("_other", {"kind-dependencies": []}), + ("docker-image", {"kind-dependencies": []}), # Add docker-image + ], + params={"target-kinds": ["_fake2"]}, + ) + kind_graph = tgg.kind_graph + # Should only include _fake2, _fake1, and docker-image (implicit dependency) + assert "_fake2" in kind_graph.nodes + assert "_fake1" in kind_graph.nodes + assert "docker-image" in kind_graph.nodes + # _fake3 and _other should not be included + assert "_fake3" not in kind_graph.nodes + assert "_other" not in kind_graph.nodes diff --git a/test/test_main.py b/test/test_main.py index 8579da12f..aebac10e4 100644 --- a/test/test_main.py +++ b/test/test_main.py @@ -12,7 +12,7 @@ import taskgraph from taskgraph.actions import registry from taskgraph.graph import Graph -from taskgraph.main import get_filtered_taskgraph +from taskgraph.main import format_kind_graph_mermaid, get_filtered_taskgraph from taskgraph.main import main as taskgraph_main from taskgraph.task import Task from taskgraph.taskgraph import TaskGraph @@ -521,3 +521,124 @@ def test_load_task_command_with_task_id(run_load_task): user=None, custom_image=None, ) + + +def test_format_kind_graph_mermaid(): + """Test conversion of kind graph to Mermaid format""" + # Test with simple graph + kinds = frozenset(["docker-image"]) + edges = frozenset() + kind_graph = Graph(kinds, edges) + + output = format_kind_graph_mermaid(kind_graph) + assert "flowchart TD" in output + assert "docker_image[docker-image]" in output + + # Test with complex graph with dependencies + kinds = frozenset(["docker-image", "build", "test", "lint"]) + edges = frozenset( + [ + ("build", "docker-image", "kind-dependency"), + ("test", "build", "kind-dependency"), + ("lint", "docker-image", "kind-dependency"), + ] + ) + kind_graph = Graph(kinds, edges) + + output = format_kind_graph_mermaid(kind_graph) + lines = output.split("\n") + + assert lines[0] == "flowchart TD" + # Check all nodes are present + assert any("build[build]" in line for line in lines) + assert any("docker_image[docker-image]" in line for line in lines) + assert any("lint[lint]" in line for line in lines) + assert any("test[test]" in line for line in lines) + # Check edges are reversed (dependencies point to dependents) + assert any("docker_image --> build" in line for line in lines) + assert any("build --> test" in line for line in lines) + assert any("docker_image --> lint" in line for line in lines) + + +def test_show_kinds_command(run_taskgraph, capsys): + """Test the kinds command outputs Mermaid format""" + res = run_taskgraph( + ["kind-graph"], + kinds=[ + ("_fake", {"kind-dependencies": []}), + ], + ) + assert res == 0 + + out, _ = capsys.readouterr() + assert "flowchart TD" in out + assert "_fake[_fake]" in out + + +def test_show_kinds_with_dependencies(run_taskgraph, capsys): + """Test the kinds command with kind dependencies""" + res = run_taskgraph( + ["kind-graph"], + kinds=[ + ("_fake3", {"kind-dependencies": ["_fake2"]}), + ("_fake2", {"kind-dependencies": ["_fake1"]}), + ("_fake1", {"kind-dependencies": []}), + ], + ) + assert res == 0 + + out, _ = capsys.readouterr() + assert "flowchart TD" in out + # Check all kinds are present + assert "_fake1[_fake1]" in out + assert "_fake2[_fake2]" in out + assert "_fake3[_fake3]" in out + # Check edges are present and reversed + assert "_fake1 --> _fake2" in out + assert "_fake2 --> _fake3" in out + + +def test_show_kinds_output_file(run_taskgraph, tmpdir): + """Test the kinds command writes to file""" + output_file = tmpdir.join("kinds.mmd") + assert not output_file.check() + + res = run_taskgraph( + ["kind-graph", f"--output-file={output_file.strpath}"], + kinds=[ + ("_fake", {"kind-dependencies": []}), + ], + ) + assert res == 0 + assert output_file.check() + + content = output_file.read_text("utf-8") + assert "flowchart TD" in content + assert "_fake[_fake]" in content + + +def test_show_kinds_with_target_kinds(run_taskgraph, capsys): + """Test the kinds command with --target-kind filter""" + res = run_taskgraph( + ["kind-graph", "-k", "_fake2"], + kinds=[ + ("_fake3", {"kind-dependencies": ["_fake2"]}), + ("_fake2", {"kind-dependencies": ["_fake1"]}), + ("_fake1", {"kind-dependencies": []}), + ("_other", {"kind-dependencies": []}), + ("docker-image", {"kind-dependencies": []}), + ], + params={"target-kinds": ["_fake2"]}, + ) + assert res == 0 + + out, _ = capsys.readouterr() + assert "flowchart TD" in out + # Should include _fake2 and its dependencies + assert "_fake2[_fake2]" in out + assert "_fake1[_fake1]" in out + # Should include docker-image (implicit dependency for target_kinds) + assert "docker_image[docker-image]" in out + # Should not include _fake3 or _other + assert "_fake3" not in out + assert "_other" not in out