diff --git a/docs/reference/migrations.rst b/docs/reference/migrations.rst index 21d7b6a3e..e78027c3a 100644 --- a/docs/reference/migrations.rst +++ b/docs/reference/migrations.rst @@ -3,6 +3,14 @@ Migration Guide This page can help when migrating Taskgraph across major versions. +13.x -> 14.x +------------ + +* The `{task_workdir}` string in environment variables no longer gets + interpolated by `run-task`. This is backing out a feature introduced in + version 13.x, so this release introduces no new backwards incompatibilities + with version 12.x or earlier. + 12.x -> 13.x ------------ diff --git a/src/taskgraph/run-task/run-task b/src/taskgraph/run-task/run-task index eadacf084..b52e00342 100755 --- a/src/taskgraph/run-task/run-task +++ b/src/taskgraph/run-task/run-task @@ -1257,24 +1257,20 @@ def main(args): for repo in repositories: vcs_checkout_from_args(repo) - # Interpolate environment variables with defined substitution patterns. For - # example, the variable `CACHE_DIR={task_workdir}/.cache` will be - # interpolated with the task's working directory. - env_subs = { - "task_workdir": task_workdir, - } - for k, v in os.environ.items(): - for subname, subval in env_subs.items(): - # We check for an exact match rather than using a format string to - # avoid accidentally trying to interpolate environment variables - # that happen to contain brackets for some other reason. - pattern = f"{{{subname}}}" - if pattern in v: - os.environ[k] = v.replace(pattern, subval) - print_line( - b"setup", - b"%s is %s\n" % (k.encode("utf-8"), os.environ[k].encode("utf-8")), - ) + # Convert certain well known environment variables to absolute paths. + for k in [ + "CARGO_HOME", + "MOZ_FETCHES_DIR", + "PIP_CACHE_DIR", + "UPLOAD_DIR", + "UV_CACHE_DIR", + "npm_config_cache", + ] + [ + "{}_PATH".format(repository["project"].upper()) + for repository in repositories + ]: + if k in os.environ: + os.environ[k] = os.path.abspath(os.environ[k]) try: if "MOZ_FETCHES" in os.environ: diff --git a/src/taskgraph/transforms/run/__init__.py b/src/taskgraph/transforms/run/__init__.py index 685cbcf30..3728103a6 100644 --- a/src/taskgraph/transforms/run/__init__.py +++ b/src/taskgraph/transforms/run/__init__.py @@ -330,7 +330,7 @@ def cmp_artifacts(a): "task-reference": json.dumps(task_fetches, sort_keys=True) } - env.setdefault("MOZ_FETCHES_DIR", "{task_workdir}/fetches") + env.setdefault("MOZ_FETCHES_DIR", "fetches") yield task diff --git a/src/taskgraph/transforms/run/common.py b/src/taskgraph/transforms/run/common.py index 062cf275e..56f112850 100644 --- a/src/taskgraph/transforms/run/common.py +++ b/src/taskgraph/transforms/run/common.py @@ -110,8 +110,7 @@ def support_vcs_checkout(config, task, taskdesc, repo_configs, sparse=False): "REPOSITORIES": json.dumps( {repo.prefix: repo.name for repo in repo_configs.values()} ), - # If vcsdir is already absolute this will return it unmodified. - "VCS_PATH": path.join("{task_workdir}", vcsdir), + "VCS_PATH": vcsdir, } ) for repo_config in repo_configs.values(): @@ -199,5 +198,5 @@ def support_caches( # If cache_dir is already absolute, the `.join` call returns it as # is. In that case, {task_workdir} will get interpolated by # run-task. - env[cache_cfg["env"]] = path.join("{task_workdir}", cache_dir) + env[cache_cfg["env"]] = cache_dir add_cache(task, taskdesc, cache_name, cache_dir) diff --git a/test/test_scripts_run_task.py b/test/test_scripts_run_task.py index 35e2f0e7b..fc8c905ba 100644 --- a/test/test_scripts_run_task.py +++ b/test/test_scripts_run_task.py @@ -13,6 +13,7 @@ import pytest import taskgraph +from taskgraph.util.caches import CACHES @pytest.fixture(scope="module") @@ -468,21 +469,15 @@ def inner(extra_args=None, env=None): return inner -def test_main_interpolate_environment(run_main): - result, env = run_main( - env={ - "MOZ_FETCHES_DIR": "{task_workdir}/file", - "UPLOAD_DIR": "$TASK_WORKDIR/file", - "FOO": "{foo}/file", - "BAR": "file", - } - ) +def test_main_abspath_environment(run_main): + envvars = ["MOZ_FETCHES_DIR", "UPLOAD_DIR"] + envvars += [cache["env"] for cache in CACHES.values() if "env" in cache] + env = {key: "file" for key in envvars} + env["FOO"] = "file" + + result, env = run_main(env=env) assert result == 0 - assert env == { - "MOZ_FETCHES_DIR": "/builds/worker/file", - "UPLOAD_DIR": "$TASK_WORKDIR/file", - "FOO": "{foo}/file", - "BAR": "file", - "TASK_WORKDIR": "/builds/worker", - } + assert env.get("FOO") == "file" + for key in envvars: + assert env[key] == "/builds/worker/file" diff --git a/test/test_transforms_run_run_task.py b/test/test_transforms_run_run_task.py index 3f2d93237..1bda3ae51 100644 --- a/test/test_transforms_run_run_task.py +++ b/test/test_transforms_run_run_task.py @@ -114,7 +114,7 @@ def assert_generic_worker(task): "HG_STORE_PATH": "y:/hg-shared", "MOZ_SCM_LEVEL": "1", "REPOSITORIES": '{"ci": "Taskgraph"}', - "VCS_PATH": "{task_workdir}/build/src", + "VCS_PATH": "build/src", }, "implementation": "generic-worker", "mounts": [ diff --git a/uv.lock b/uv.lock index fdb06ce36..bd6de390c 100644 --- a/uv.lock +++ b/uv.lock @@ -273,7 +273,7 @@ name = "click" version = "8.1.7" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "platform_system == 'Windows'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/96/d3/f04c7bfcf5c1862a2a5b845c6b2b360488cf47af55dfa79c98f6a6bf98b5/click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de", size = 336121 } wheels = [ @@ -1071,7 +1071,7 @@ wheels = [ [[package]] name = "taskcluster-taskgraph" -version = "12.1.0" +version = "13.1.0" source = { editable = "." } dependencies = [ { name = "appdirs" },