refactor!: remove support for hg sparse-profiles

BREAKING CHANGE: sparse profiles are no longer supported

This feature isn't being used by any known Taskgraph consumers. After
Gecko finishes migrating to Github, we'll be dropping support for
hg.mozilla.org based repositories anyway.

Cleaning this up now helps simplify the migration, and ensures we don't
accidentally start depending on the feature in the meantime.

Author: ahal

docs/reference/migrations.rst

@@ -3,6 +3,13 @@ Migration Guide
 
 This page can help when migrating Taskgraph across major versions.
 
+17.x -> 18.x
+------------
+
+* Stop setting the ``run.sparse-profile`` key in all tasks which perform a
+  Mercurial clone. If sparse profiles are still required, the task must perform
+  its own clone and not rely on the ``run-task`` script to do it.
+
 16.x -> 17.x
 ------------
 

src/taskgraph/run-task/run-task

@@ -824,7 +824,6 @@ def hg_checkout(
     head_repo: str,
     base_repo: Optional[str],
     store_path: str,
-    sparse_profile: Optional[str],
     branch: Optional[str],
     revision: Optional[str],
 ):
@@ -849,8 +848,6 @@ def hg_checkout(
 
     if base_repo:
         args.extend(["--upstream", base_repo])
-    if sparse_profile:
-        args.extend(["--sparseprofile", sparse_profile])
 
     # Specify method to checkout a revision. This defaults to revisions as
     # SHA-1 strings, but also supports symbolic revisions like `tip` via the
@@ -914,10 +911,6 @@ def add_vcs_arguments(parser, project, name):
         f"--{project}-checkout",
         help=f"Directory where {name} checkout should be created",
     )
-    parser.add_argument(
-        f"--{project}-sparse-profile",
-        help=f"Path to sparse profile for {name} checkout",
-    )
     parser.add_argument(
         f"--{project}-shallow-clone",
         action="store_true",
@@ -927,7 +920,6 @@ def add_vcs_arguments(parser, project, name):
 
 def collect_vcs_options(args, project, name):
     checkout = getattr(args, f"{project}_checkout")
-    sparse_profile = getattr(args, f"{project}_sparse_profile")
     shallow_clone = getattr(args, f"{project}_shallow_clone")
 
     env_prefix = project.upper()
@@ -964,7 +956,6 @@ def collect_vcs_options(args, project, name):
         "name": name,
         "env-prefix": env_prefix,
         "checkout": checkout,
-        "sparse-profile": sparse_profile,
         "base-repo": base_repo,
         "base-rev": base_rev,
         "head-repo": head_repo,
@@ -1030,7 +1021,6 @@ def vcs_checkout_from_args(options):
                 options["head-repo"],
                 options["base-repo"],
                 options["store-path"],
-                options["sparse-profile"],
                 head_ref,
                 head_rev,
             )

src/taskgraph/transforms/run/common.py

@@ -82,7 +82,7 @@ def generic_worker_add_artifacts(config, task, taskdesc):
     add_artifacts(config, task, taskdesc, path=get_artifact_prefix(taskdesc))
 
 
-def support_vcs_checkout(config, task, taskdesc, repo_configs, sparse=False):
+def support_vcs_checkout(config, task, taskdesc, repo_configs):
     """Update a task with parameters to enable a VCS checkout.
 
     This can only be used with ``run-task`` tasks, as the cache name is

src/taskgraph/transforms/run/run_task.py

@@ -66,15 +66,6 @@
                 """.lstrip()
             ),
         ): str,
-        Required(
-            "sparse-profile",
-            description=dedent(
-                """
-                The sparse checkout profile to use. Value is the filename relative to the
-                directory where sparse profiles are defined (build/sparse-profiles/).
-                """.lstrip()
-            ),
-        ): Any(str, None),
         Required(
             "command",
             description=dedent(
@@ -140,21 +131,12 @@ def common_setup(config, task, taskdesc, command):
             task,
             taskdesc,
             repo_configs=repo_configs,
-            sparse=bool(run["sparse-profile"]),
         )
 
         for repo_config in repo_configs.values():
             checkout_path = path.join(vcs_path, repo_config.path)
             command.append(f"--{repo_config.prefix}-checkout={checkout_path}")
 
-        if run["sparse-profile"]:
-            command.append(
-                "--{}-sparse-profile=build/sparse-profiles/{}".format(
-                    repo_config.prefix,  # type: ignore
-                    run["sparse-profile"],
-                )
-            )
-
         if "cwd" in run:
             run["cwd"] = path.normpath(run["cwd"].format(checkout=vcs_path))
     elif "cwd" in run and "{checkout}" in run["cwd"]:
@@ -174,7 +156,6 @@ def common_setup(config, task, taskdesc, command):
 
 worker_defaults = {
     "checkout": True,
-    "sparse-profile": None,
     "run-as-root": False,
 }
 

src/taskgraph/transforms/run/toolchain.py

@@ -50,18 +50,6 @@
                 """
             ),
         ): [str],
-        Required(
-            "sparse-profile",
-            description=dedent(
-                """
-                Sparse profile to give to checkout using `run-task`. If given,
-                a filename in `build/sparse-profiles`. Defaults to
-                "toolchain-build", i.e., to
-                `build/sparse-profiles/toolchain-build`. If `None`, instructs
-                `run-task` to not use a sparse profile at all.
-                """
-            ),
-        ): Any(str, None),
         Optional(
             "resources",
             description=dedent(
@@ -205,9 +193,7 @@ def common_toolchain(config, task, taskdesc, is_docker):
     configure_taskdesc_for_run(config, task, taskdesc, worker["implementation"])
 
 
-toolchain_defaults = {
-    "sparse-profile": "toolchain-build",
-}
+toolchain_defaults = {}
 
 
 @run_task_using(

src/taskgraph/util/caches.py

@@ -37,11 +37,6 @@ def get_checkout_cache_name(config: "TransformConfig", task: dict[str, Any]) ->
         digest = hashlib.sha256(checkout_paths_str).hexdigest()
         cache_name += f"-repos-{digest}"
 
-    # Sparse checkouts need their own cache because they can interfere
-    # with clients that aren't sparse aware.
-    if task["run"]["sparse-profile"]:
-        cache_name += "-sparse"
-
     return cache_name
 
 

template/{{cookiecutter.project_name}}/taskcluster.github.yml

@@ -222,7 +222,7 @@ tasks:
                                         ACTION_CALLBACK: '${action.cb_name}'
 
                           cache:
-                              "${trustDomain}-project-${project}-level-${level}-checkouts-sparse-v2": /builds/worker/checkouts
+                              "${trustDomain}-project-${project}-level-${level}-checkouts-v2": /builds/worker/checkouts
 
                           features:
                               taskclusterProxy: true

test/test_scripts_run_task.py

@@ -195,7 +195,6 @@ def test_collect_vcs_options(
 
     args.setdefault(f"{name}_checkout", checkout)
     args.setdefault(f"{name}_shallow_clone", False)
-    args.setdefault(f"{name}_sparse_profile", False)
     args = Namespace(**args)
 
     result = run_task_mod.collect_vcs_options(args, name, name)
@@ -214,7 +213,6 @@ def test_collect_vcs_options(
         "repo-type": env.get("REPOSITORY_TYPE"),
         "shallow-clone": False,
         "ssh-secret-name": env.get("SSH_SECRET_NAME"),
-        "sparse-profile": False,
         "store-path": env.get("HG_STORE_PATH"),
     }
     if "PIP_REQUIREMENTS" in env:

test/test_transforms_run_toolchain.py

@@ -65,7 +65,6 @@ def assert_docker_worker(task, taskdesc):
         "run": {
             "command": ["vcs/taskcluster/scripts/toolchain/run.sh"],
             "cwd": "{checkout}/..",
-            "sparse-profile": "toolchain-build",
             "using": "run-task",
             "workdir": "/builds/worker",
         },
@@ -140,7 +139,6 @@ def assert_generic_worker(task, taskdesc):
         "run": {
             "command": "src/taskcluster/scripts/toolchain/run.sh --foo bar",
             "cwd": "{checkout}/..",
-            "sparse-profile": "toolchain-build",
             "using": "run-task",
             "workdir": "/builds/worker",
         },
@@ -192,7 +190,6 @@ def assert_powershell(task, _):
         "command": "src/taskcluster/scripts/toolchain/run.ps1",
         "cwd": "{checkout}/..",
         "exec-with": "powershell",
-        "sparse-profile": "toolchain-build",
         "using": "run-task",
         "workdir": "/builds/worker",
     }