From feea40d69de9665aaccafc14d2018d6ca11685a2 Mon Sep 17 00:00:00 2001 From: Andrew Halberstadt Date: Fri, 7 Nov 2025 10:35:09 -0500 Subject: [PATCH] refactor!: remove support for hg sparse-profiles BREAKING CHANGE: sparse profiles are no longer supported This feature isn't being used by any known Taskgraph consumers. After Gecko finishes migrating to Github, we'll be dropping support for hg.mozilla.org based repositories anyway. Cleaning this up now helps simplify the migration, and ensures we don't accidentally start depending on the feature in the meantime. --- docs/reference/migrations.rst | 7 +++++++ src/taskgraph/run-task/run-task | 10 ---------- src/taskgraph/transforms/run/common.py | 2 +- src/taskgraph/transforms/run/run_task.py | 19 ------------------- src/taskgraph/transforms/run/toolchain.py | 16 +--------------- src/taskgraph/util/caches.py | 5 ----- .../taskcluster.github.yml | 2 +- test/test_scripts_run_task.py | 2 -- test/test_transforms_run_toolchain.py | 3 --- 9 files changed, 10 insertions(+), 56 deletions(-) diff --git a/docs/reference/migrations.rst b/docs/reference/migrations.rst index 8c674cf31..311cc22da 100644 --- a/docs/reference/migrations.rst +++ b/docs/reference/migrations.rst @@ -3,6 +3,13 @@ Migration Guide This page can help when migrating Taskgraph across major versions. +17.x -> 18.x +------------ + +* Stop setting the ``run.sparse-profile`` key in all tasks which perform a + Mercurial clone. If sparse profiles are still required, the task must perform + its own clone and not rely on the ``run-task`` script to do it. + 16.x -> 17.x ------------ diff --git a/src/taskgraph/run-task/run-task b/src/taskgraph/run-task/run-task index c7a3fada0..66882632d 100755 --- a/src/taskgraph/run-task/run-task +++ b/src/taskgraph/run-task/run-task @@ -824,7 +824,6 @@ def hg_checkout( head_repo: str, base_repo: Optional[str], store_path: str, - sparse_profile: Optional[str], branch: Optional[str], revision: Optional[str], ): @@ -849,8 +848,6 @@ def hg_checkout( if base_repo: args.extend(["--upstream", base_repo]) - if sparse_profile: - args.extend(["--sparseprofile", sparse_profile]) # Specify method to checkout a revision. This defaults to revisions as # SHA-1 strings, but also supports symbolic revisions like `tip` via the @@ -914,10 +911,6 @@ def add_vcs_arguments(parser, project, name): f"--{project}-checkout", help=f"Directory where {name} checkout should be created", ) - parser.add_argument( - f"--{project}-sparse-profile", - help=f"Path to sparse profile for {name} checkout", - ) parser.add_argument( f"--{project}-shallow-clone", action="store_true", @@ -927,7 +920,6 @@ def add_vcs_arguments(parser, project, name): def collect_vcs_options(args, project, name): checkout = getattr(args, f"{project}_checkout") - sparse_profile = getattr(args, f"{project}_sparse_profile") shallow_clone = getattr(args, f"{project}_shallow_clone") env_prefix = project.upper() @@ -964,7 +956,6 @@ def collect_vcs_options(args, project, name): "name": name, "env-prefix": env_prefix, "checkout": checkout, - "sparse-profile": sparse_profile, "base-repo": base_repo, "base-rev": base_rev, "head-repo": head_repo, @@ -1030,7 +1021,6 @@ def vcs_checkout_from_args(options): options["head-repo"], options["base-repo"], options["store-path"], - options["sparse-profile"], head_ref, head_rev, ) diff --git a/src/taskgraph/transforms/run/common.py b/src/taskgraph/transforms/run/common.py index 94f5d57f3..5c33d64a8 100644 --- a/src/taskgraph/transforms/run/common.py +++ b/src/taskgraph/transforms/run/common.py @@ -82,7 +82,7 @@ def generic_worker_add_artifacts(config, task, taskdesc): add_artifacts(config, task, taskdesc, path=get_artifact_prefix(taskdesc)) -def support_vcs_checkout(config, task, taskdesc, repo_configs, sparse=False): +def support_vcs_checkout(config, task, taskdesc, repo_configs): """Update a task with parameters to enable a VCS checkout. This can only be used with ``run-task`` tasks, as the cache name is diff --git a/src/taskgraph/transforms/run/run_task.py b/src/taskgraph/transforms/run/run_task.py index 074907894..9d76380d8 100644 --- a/src/taskgraph/transforms/run/run_task.py +++ b/src/taskgraph/transforms/run/run_task.py @@ -66,15 +66,6 @@ """.lstrip() ), ): str, - Required( - "sparse-profile", - description=dedent( - """ - The sparse checkout profile to use. Value is the filename relative to the - directory where sparse profiles are defined (build/sparse-profiles/). - """.lstrip() - ), - ): Any(str, None), Required( "command", description=dedent( @@ -140,21 +131,12 @@ def common_setup(config, task, taskdesc, command): task, taskdesc, repo_configs=repo_configs, - sparse=bool(run["sparse-profile"]), ) for repo_config in repo_configs.values(): checkout_path = path.join(vcs_path, repo_config.path) command.append(f"--{repo_config.prefix}-checkout={checkout_path}") - if run["sparse-profile"]: - command.append( - "--{}-sparse-profile=build/sparse-profiles/{}".format( - repo_config.prefix, # type: ignore - run["sparse-profile"], - ) - ) - if "cwd" in run: run["cwd"] = path.normpath(run["cwd"].format(checkout=vcs_path)) elif "cwd" in run and "{checkout}" in run["cwd"]: @@ -174,7 +156,6 @@ def common_setup(config, task, taskdesc, command): worker_defaults = { "checkout": True, - "sparse-profile": None, "run-as-root": False, } diff --git a/src/taskgraph/transforms/run/toolchain.py b/src/taskgraph/transforms/run/toolchain.py index 669bcd812..58d318043 100644 --- a/src/taskgraph/transforms/run/toolchain.py +++ b/src/taskgraph/transforms/run/toolchain.py @@ -50,18 +50,6 @@ """ ), ): [str], - Required( - "sparse-profile", - description=dedent( - """ - Sparse profile to give to checkout using `run-task`. If given, - a filename in `build/sparse-profiles`. Defaults to - "toolchain-build", i.e., to - `build/sparse-profiles/toolchain-build`. If `None`, instructs - `run-task` to not use a sparse profile at all. - """ - ), - ): Any(str, None), Optional( "resources", description=dedent( @@ -205,9 +193,7 @@ def common_toolchain(config, task, taskdesc, is_docker): configure_taskdesc_for_run(config, task, taskdesc, worker["implementation"]) -toolchain_defaults = { - "sparse-profile": "toolchain-build", -} +toolchain_defaults = {} @run_task_using( diff --git a/src/taskgraph/util/caches.py b/src/taskgraph/util/caches.py index 5554cc2e4..e4bfd3455 100644 --- a/src/taskgraph/util/caches.py +++ b/src/taskgraph/util/caches.py @@ -37,11 +37,6 @@ def get_checkout_cache_name(config: "TransformConfig", task: dict[str, Any]) -> digest = hashlib.sha256(checkout_paths_str).hexdigest() cache_name += f"-repos-{digest}" - # Sparse checkouts need their own cache because they can interfere - # with clients that aren't sparse aware. - if task["run"]["sparse-profile"]: - cache_name += "-sparse" - return cache_name diff --git a/template/{{cookiecutter.project_name}}/taskcluster.github.yml b/template/{{cookiecutter.project_name}}/taskcluster.github.yml index 3076f953b..fa42ddf11 100644 --- a/template/{{cookiecutter.project_name}}/taskcluster.github.yml +++ b/template/{{cookiecutter.project_name}}/taskcluster.github.yml @@ -222,7 +222,7 @@ tasks: ACTION_CALLBACK: '${action.cb_name}' cache: - "${trustDomain}-project-${project}-level-${level}-checkouts-sparse-v2": /builds/worker/checkouts + "${trustDomain}-project-${project}-level-${level}-checkouts-v2": /builds/worker/checkouts features: taskclusterProxy: true diff --git a/test/test_scripts_run_task.py b/test/test_scripts_run_task.py index f88c7ebeb..86ad57a37 100644 --- a/test/test_scripts_run_task.py +++ b/test/test_scripts_run_task.py @@ -195,7 +195,6 @@ def test_collect_vcs_options( args.setdefault(f"{name}_checkout", checkout) args.setdefault(f"{name}_shallow_clone", False) - args.setdefault(f"{name}_sparse_profile", False) args = Namespace(**args) result = run_task_mod.collect_vcs_options(args, name, name) @@ -214,7 +213,6 @@ def test_collect_vcs_options( "repo-type": env.get("REPOSITORY_TYPE"), "shallow-clone": False, "ssh-secret-name": env.get("SSH_SECRET_NAME"), - "sparse-profile": False, "store-path": env.get("HG_STORE_PATH"), } if "PIP_REQUIREMENTS" in env: diff --git a/test/test_transforms_run_toolchain.py b/test/test_transforms_run_toolchain.py index 6aaffaaa6..3ec843dff 100644 --- a/test/test_transforms_run_toolchain.py +++ b/test/test_transforms_run_toolchain.py @@ -65,7 +65,6 @@ def assert_docker_worker(task, taskdesc): "run": { "command": ["vcs/taskcluster/scripts/toolchain/run.sh"], "cwd": "{checkout}/..", - "sparse-profile": "toolchain-build", "using": "run-task", "workdir": "/builds/worker", }, @@ -140,7 +139,6 @@ def assert_generic_worker(task, taskdesc): "run": { "command": "src/taskcluster/scripts/toolchain/run.sh --foo bar", "cwd": "{checkout}/..", - "sparse-profile": "toolchain-build", "using": "run-task", "workdir": "/builds/worker", }, @@ -192,7 +190,6 @@ def assert_powershell(task, _): "command": "src/taskcluster/scripts/toolchain/run.ps1", "cwd": "{checkout}/..", "exec-with": "powershell", - "sparse-profile": "toolchain-build", "using": "run-task", "workdir": "/builds/worker", }