From b4ad85c32f9e906cfd68f35ec769a4be1f86bf37 Mon Sep 17 00:00:00 2001 From: Andrew Halberstadt Date: Mon, 30 Dec 2024 11:21:28 -0500 Subject: [PATCH 1/3] test(caches): refactor test for caches in the run transforms --- .../src/pytest_taskgraph/fixtures/gen.py | 6 ++ test/test_transforms_run.py | 65 ++++++++++++------- 2 files changed, 48 insertions(+), 23 deletions(-) diff --git a/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/gen.py b/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/gen.py index 4ccdfa40b..f11f459f1 100644 --- a/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/gen.py +++ b/packages/pytest-taskgraph/src/pytest_taskgraph/fixtures/gen.py @@ -85,6 +85,12 @@ def fake_load_graph_config(root_dir): "os": "linux", "worker-type": "linux", }, + "t-win": { + "provisioner": "taskgraph-t", + "implementation": "generic-worker", + "os": "windows", + "worker-type": "win", + }, } }, "task-priority": "low", diff --git a/test/test_transforms_run.py b/test/test_transforms_run.py index 820f0ed26..480d5ce51 100644 --- a/test/test_transforms_run.py +++ b/test/test_transforms_run.py @@ -58,32 +58,51 @@ def inner(task_input): return inner -@pytest.mark.parametrize( - "task", - [ - {"worker-type": "t-linux"}, - pytest.param( - {"worker-type": "releng-hardware/gecko-t-win10-64-hw"}, - marks=pytest.mark.xfail, - ), - ], - ids=["docker-worker", "generic-worker"], -) -def test_worker_caches(task, transform): - config, task, taskdesc, impl = transform(task) - add_cache(task, taskdesc, "cache1", "/cache1") - add_cache(task, taskdesc, "cache2", "/cache2", skip_untrusted=True) +@pytest.fixture +def run_caches(transform): + def inner(task): + config, task, taskdesc, impl = transform(task) + add_cache(task, taskdesc, "cache1", "/cache1") + add_cache(task, taskdesc, "cache2", "/cache2", skip_untrusted=True) - if impl not in ("docker-worker", "generic-worker"): - pytest.xfail(f"caches not implemented for '{impl}'") + if impl not in ("docker-worker", "generic-worker"): + pytest.xfail(f"caches not implemented for '{impl}'") + + # Create a new schema object with just the part relevant to caches. + key = "caches" if impl == "docker-worker" else "mounts" + partial_schema = Schema(payload_builders[impl].schema.schema[key]) + validate_schema(partial_schema, taskdesc["worker"][key], "validation error") + + result = taskdesc["worker"].get(key) + print("Dumping for copy/paste:") + pprint(result, indent=2) + return result + + return inner - key = "caches" if impl == "docker-worker" else "mounts" - assert key in taskdesc["worker"] - assert len(taskdesc["worker"][key]) == 2 - # Create a new schema object with just the part relevant to caches. - partial_schema = Schema(payload_builders[impl].schema.schema[key]) - validate_schema(partial_schema, taskdesc["worker"][key], "validation error") +def test_caches_docker_worker(run_caches): + assert run_caches({"worker-type": "t-linux"}) == [ + { + "mount-point": "/cache1", + "name": "cache1", + "skip-untrusted": False, + "type": "persistent", + }, + { + "mount-point": "/cache2", + "name": "cache2", + "skip-untrusted": True, + "type": "persistent", + }, + ] + + +def test_caches_generic_worker(run_caches): + assert run_caches({"worker-type": "t-win"}) == [ + {"cache-name": "cache1", "directory": "/cache1"}, + {"cache-name": "cache2", "directory": "/cache2"}, + ] def test_rewrite_when_to_optimization(run_transform, make_transform_config): From f7d3b47845120b1b2092d411758590b46c42c422 Mon Sep 17 00:00:00 2001 From: Andrew Halberstadt Date: Mon, 30 Dec 2024 11:21:28 -0500 Subject: [PATCH 2/3] test(caches): add a test for disabled caches --- test/test_transforms_run.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/test/test_transforms_run.py b/test/test_transforms_run.py index 480d5ce51..ee36e9ee2 100644 --- a/test/test_transforms_run.py +++ b/test/test_transforms_run.py @@ -47,8 +47,8 @@ def transform(monkeypatch, run_transform): monkeypatch.setenv("TASK_ID", "fakeid") def inner(task_input): - task = deepcopy(TASK_DEFAULTS) - task.update(task_input) + defaults = deepcopy(TASK_DEFAULTS) + task = merge(defaults, task_input) with patch("taskgraph.transforms.run.configure_taskdesc_for_run") as m: # This forces the generator to be evaluated @@ -68,14 +68,18 @@ def inner(task): if impl not in ("docker-worker", "generic-worker"): pytest.xfail(f"caches not implemented for '{impl}'") - # Create a new schema object with just the part relevant to caches. key = "caches" if impl == "docker-worker" else "mounts" - partial_schema = Schema(payload_builders[impl].schema.schema[key]) - validate_schema(partial_schema, taskdesc["worker"][key], "validation error") result = taskdesc["worker"].get(key) - print("Dumping for copy/paste:") - pprint(result, indent=2) + + if result: + print("Dumping for copy/paste:") + pprint(result, indent=2) + + # Create a new schema object with just the part relevant to caches. + partial_schema = Schema(payload_builders[impl].schema.schema[key]) + validate_schema(partial_schema, taskdesc["worker"][key], "validation error") + return result return inner @@ -105,6 +109,13 @@ def test_caches_generic_worker(run_caches): ] +@pytest.mark.parametrize("worker_type", ("t-linux", "t-win")) +def test_caches_disabled(run_caches, worker_type): + assert ( + run_caches({"run": {"use-caches": False}, "worker-type": worker_type}) is None + ) + + def test_rewrite_when_to_optimization(run_transform, make_transform_config): config = make_transform_config() From 07ab89cc614ce6e0935217eaf0fa9c59bb37fb1f Mon Sep 17 00:00:00 2001 From: Andrew Halberstadt Date: Mon, 30 Dec 2024 11:21:28 -0500 Subject: [PATCH 3/3] feat(caches): support opting into or out of specific name caches Bug: 1526028 --- src/taskgraph/transforms/run/common.py | 7 ++++++- src/taskgraph/transforms/run/run_task.py | 6 ++++-- test/test_transforms_run.py | 12 ++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/src/taskgraph/transforms/run/common.py b/src/taskgraph/transforms/run/common.py index 37095a356..450bf52fe 100644 --- a/src/taskgraph/transforms/run/common.py +++ b/src/taskgraph/transforms/run/common.py @@ -9,6 +9,7 @@ import hashlib import json +import re from taskgraph.util.taskcluster import get_artifact_prefix @@ -31,7 +32,11 @@ def add_cache(task, taskdesc, name, mount_point, skip_untrusted=False): skip_untrusted (bool): Whether cache is used in untrusted environments (default: False). Only applies to docker-worker. """ - if not task["run"].get("use-caches", True): + use_caches = task["run"].get("use-caches", True) + if isinstance(use_caches, list): + use_caches = any(re.match(pattern, name) for pattern in use_caches) + + if not use_caches: return worker = task["worker"] diff --git a/src/taskgraph/transforms/run/run_task.py b/src/taskgraph/transforms/run/run_task.py index f0b48c3f6..417c2c1cf 100644 --- a/src/taskgraph/transforms/run/run_task.py +++ b/src/taskgraph/transforms/run/run_task.py @@ -28,8 +28,10 @@ # tend to hide their caches. This cache is never added for level-1 tasks. # TODO Once bug 1526028 is fixed, this and 'use-caches' should be merged. Required("cache-dotcache"): bool, - # Whether or not to use caches. - Optional("use-caches"): bool, + # Whether or not to use caches. If a boolean, all caches will be + # enabled or disabled. If a list, entries are regexes. Only cache names + # that match one of the listed regexes are enabled. + Optional("use-caches"): Any(bool, [str]), # if true (the default), perform a checkout on the worker Required("checkout"): Any(bool, {str: dict}), Optional( diff --git a/test/test_transforms_run.py b/test/test_transforms_run.py index ee36e9ee2..3e851be6a 100644 --- a/test/test_transforms_run.py +++ b/test/test_transforms_run.py @@ -109,6 +109,18 @@ def test_caches_generic_worker(run_caches): ] +def test_caches_regex(run_caches): + task = { + "run": { + "use-caches": ["cache[^1]"], + }, + "worker-type": "t-win", + } + caches = run_caches(task) + assert len(caches) == 1 + assert caches[0]["cache-name"] == "cache2" + + @pytest.mark.parametrize("worker_type", ("t-linux", "t-win")) def test_caches_disabled(run_caches, worker_type): assert (