From 490c57a60488efbdee248b0cf53d333598f3770b Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Tue, 13 May 2025 16:13:07 +0200 Subject: [PATCH 1/3] Revert "Fix run-task/fetch-content mounts definition (#688)" This reverts commit 500a9449554912c82907776c70ed8722ca79190d. --- src/taskgraph/transforms/run/run_task.py | 4 ++-- src/taskgraph/transforms/task.py | 2 -- test/test_transforms_run_run_task.py | 21 +++------------------ 3 files changed, 5 insertions(+), 22 deletions(-) diff --git a/src/taskgraph/transforms/run/run_task.py b/src/taskgraph/transforms/run/run_task.py index 0919e5ee1..e2369cc24 100644 --- a/src/taskgraph/transforms/run/run_task.py +++ b/src/taskgraph/transforms/run/run_task.py @@ -179,7 +179,7 @@ def generic_worker_run_task(config, task, taskdesc): worker["mounts"].append( { "content": { - "task-id": {"task-reference": ""}, + "taskId": {"task-reference": ""}, "artifact": "public/run-task", "sha256": run_task_sha256, }, @@ -190,7 +190,7 @@ def generic_worker_run_task(config, task, taskdesc): worker["mounts"].append( { "content": { - "task-id": {"task-reference": ""}, + "taskId": {"task-reference": ""}, "artifact": "public/fetch-content", "sha256": fetch_content_sha256, }, diff --git a/src/taskgraph/transforms/task.py b/src/taskgraph/transforms/task.py index b92c85c53..320524261 100644 --- a/src/taskgraph/transforms/task.py +++ b/src/taskgraph/transforms/task.py @@ -622,8 +622,6 @@ def build_docker_worker_payload(config, task, task_def): # URL that supplies the content in response to an unauthenticated # GET request. Optional("url"): str, - # SHA256 checksum of the content body - Optional("sha256"): str, }, # *** Either file or directory must be specified. *** # If mounting a cache or read-only directory, the filesystem location of diff --git a/test/test_transforms_run_run_task.py b/test/test_transforms_run_run_task.py index f0841ace4..baf35a824 100644 --- a/test/test_transforms_run_run_task.py +++ b/test/test_transforms_run_run_task.py @@ -8,7 +8,7 @@ import pytest from taskgraph.transforms.run import make_task_description -from taskgraph.transforms.task import payload_builders, set_defaults +from taskgraph.transforms.task import payload_builders from taskgraph.util.caches import CACHES from taskgraph.util.schema import Schema, validate_schema from taskgraph.util.templates import merge @@ -70,7 +70,6 @@ def assert_docker_worker(task): "-cx", "echo hello world", ], - "docker-image": {"in-tree": "image"}, "env": { "CI_BASE_REPOSITORY": "http://hg.example.com", "CI_HEAD_REF": "default", @@ -88,13 +87,6 @@ def assert_docker_worker(task): }, "worker-type": "t-linux", } - taskdesc = next(set_defaults({}, [task])) - taskdesc["worker"]["max-run-time"] = 0 - validate_schema( - payload_builders[taskdesc["worker"]["implementation"]].schema, - taskdesc["worker"], - "validation error", - ) def assert_generic_worker(task): @@ -131,7 +123,7 @@ def assert_generic_worker(task): "content": { "artifact": "public/run-task", "sha256": "581ca6876fac84fa2dd8e8c2c18677d790890e9675229fd34c912c937ae19fae", - "task-id": {"task-reference": ""}, + "taskId": {"task-reference": ""}, }, "file": "./run-task", }, @@ -140,13 +132,6 @@ def assert_generic_worker(task): }, "worker-type": "b-win2012", } - taskdesc = next(set_defaults({}, [task])) - taskdesc["worker"]["max-run-time"] = 0 - validate_schema( - payload_builders[taskdesc["worker"]["implementation"]].schema, - taskdesc["worker"], - "validation error", - ) def assert_exec_with(task): @@ -192,7 +177,7 @@ def assert_run_task_command_generic_worker(task): "task", ( pytest.param( - {"worker": {"os": "linux", "docker-image": {"in-tree": "image"}}}, + {"worker": {"os": "linux"}}, id="docker_worker", ), pytest.param( From a32d08c646055d336aff39fb23c639f44aa2ee7c Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Tue, 13 May 2025 16:13:12 +0200 Subject: [PATCH 2/3] Revert "Let generic-worker verify run-task/fetch-content integrity (#666)" This causes failures because not all tasks have a direct dependency on the decision task, but generic-worker insists on that dependency existing when mounting an artifact from a task. So go back to mounting by url. This reverts commit f85ff32682d469809fd49adf703bb6ca36ecbdc6. --- src/taskgraph/transforms/run/run_task.py | 15 ++------------- test/test_transforms_run_run_task.py | 4 +--- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/src/taskgraph/transforms/run/run_task.py b/src/taskgraph/transforms/run/run_task.py index e2369cc24..8830b9ac5 100644 --- a/src/taskgraph/transforms/run/run_task.py +++ b/src/taskgraph/transforms/run/run_task.py @@ -6,9 +6,7 @@ """ import dataclasses -import hashlib import os -from pathlib import Path from voluptuous import Any, Optional, Required @@ -27,9 +25,6 @@ "powershell": ["powershell.exe", "-ExecutionPolicy", "Bypass"], } -RUN_TASK_PATH = Path(__file__).parent.parent.parent / "run-task" / "run-task" -FETCH_CONTENT_PATH = Path(__file__).parent.parent.parent / "run-task" / "fetch-content" - run_task_schema = Schema( { Required("using"): "run-task", @@ -174,14 +169,10 @@ def generic_worker_run_task(config, task, taskdesc): common_setup(config, task, taskdesc, command) worker.setdefault("mounts", []) - run_task_sha256 = hashlib.sha256(RUN_TASK_PATH.read_bytes()).hexdigest() - fetch_content_sha256 = hashlib.sha256(FETCH_CONTENT_PATH.read_bytes()).hexdigest() worker["mounts"].append( { "content": { - "taskId": {"task-reference": ""}, - "artifact": "public/run-task", - "sha256": run_task_sha256, + "url": script_url(config, "run-task"), }, "file": "./run-task", } @@ -190,9 +181,7 @@ def generic_worker_run_task(config, task, taskdesc): worker["mounts"].append( { "content": { - "taskId": {"task-reference": ""}, - "artifact": "public/fetch-content", - "sha256": fetch_content_sha256, + "url": script_url(config, "fetch-content"), }, "file": "./fetch-content", } diff --git a/test/test_transforms_run_run_task.py b/test/test_transforms_run_run_task.py index baf35a824..1bda3ae51 100644 --- a/test/test_transforms_run_run_task.py +++ b/test/test_transforms_run_run_task.py @@ -121,9 +121,7 @@ def assert_generic_worker(task): {"cache-name": "checkouts", "directory": "build"}, { "content": { - "artifact": "public/run-task", - "sha256": "581ca6876fac84fa2dd8e8c2c18677d790890e9675229fd34c912c937ae19fae", - "taskId": {"task-reference": ""}, + "url": "https://tc-tests.localhost/api/queue/v1/task//artifacts/public/run-task" }, "file": "./run-task", }, From 4f834eae0781dcb8e69613b2646a01ca620f2d9b Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Tue, 13 May 2025 16:15:55 +0200 Subject: [PATCH 3/3] chore: update 14.2.1 changelog --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d11b0eb4a..30b05407b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,8 +4,8 @@ ### Fixed -- regression in 14.2.0 causing schema validation failure for generic-worker - tasks using run-task (#688) +- Revert "generic-worker will now verify run-task/fetch-content integrity" from + 14.2.0 for being broken ## [14.2.0] - 2025-05-12