diff --git a/docs/reference/migrations.rst b/docs/reference/migrations.rst index 8c674cf31..311cc22da 100644 --- a/docs/reference/migrations.rst +++ b/docs/reference/migrations.rst @@ -3,6 +3,13 @@ Migration Guide This page can help when migrating Taskgraph across major versions. +17.x -> 18.x +------------ + +* Stop setting the ``run.sparse-profile`` key in all tasks which perform a + Mercurial clone. If sparse profiles are still required, the task must perform + its own clone and not rely on the ``run-task`` script to do it. + 16.x -> 17.x ------------ diff --git a/src/taskgraph/run-task/run-task b/src/taskgraph/run-task/run-task index c7a3fada0..66882632d 100755 --- a/src/taskgraph/run-task/run-task +++ b/src/taskgraph/run-task/run-task @@ -824,7 +824,6 @@ def hg_checkout( head_repo: str, base_repo: Optional[str], store_path: str, - sparse_profile: Optional[str], branch: Optional[str], revision: Optional[str], ): @@ -849,8 +848,6 @@ def hg_checkout( if base_repo: args.extend(["--upstream", base_repo]) - if sparse_profile: - args.extend(["--sparseprofile", sparse_profile]) # Specify method to checkout a revision. This defaults to revisions as # SHA-1 strings, but also supports symbolic revisions like `tip` via the @@ -914,10 +911,6 @@ def add_vcs_arguments(parser, project, name): f"--{project}-checkout", help=f"Directory where {name} checkout should be created", ) - parser.add_argument( - f"--{project}-sparse-profile", - help=f"Path to sparse profile for {name} checkout", - ) parser.add_argument( f"--{project}-shallow-clone", action="store_true", @@ -927,7 +920,6 @@ def add_vcs_arguments(parser, project, name): def collect_vcs_options(args, project, name): checkout = getattr(args, f"{project}_checkout") - sparse_profile = getattr(args, f"{project}_sparse_profile") shallow_clone = getattr(args, f"{project}_shallow_clone") env_prefix = project.upper() @@ -964,7 +956,6 @@ def collect_vcs_options(args, project, name): "name": name, "env-prefix": env_prefix, "checkout": checkout, - "sparse-profile": sparse_profile, "base-repo": base_repo, "base-rev": base_rev, "head-repo": head_repo, @@ -1030,7 +1021,6 @@ def vcs_checkout_from_args(options): options["head-repo"], options["base-repo"], options["store-path"], - options["sparse-profile"], head_ref, head_rev, ) diff --git a/src/taskgraph/transforms/run/common.py b/src/taskgraph/transforms/run/common.py index 94f5d57f3..5c33d64a8 100644 --- a/src/taskgraph/transforms/run/common.py +++ b/src/taskgraph/transforms/run/common.py @@ -82,7 +82,7 @@ def generic_worker_add_artifacts(config, task, taskdesc): add_artifacts(config, task, taskdesc, path=get_artifact_prefix(taskdesc)) -def support_vcs_checkout(config, task, taskdesc, repo_configs, sparse=False): +def support_vcs_checkout(config, task, taskdesc, repo_configs): """Update a task with parameters to enable a VCS checkout. This can only be used with ``run-task`` tasks, as the cache name is diff --git a/src/taskgraph/transforms/run/run_task.py b/src/taskgraph/transforms/run/run_task.py index 074907894..9d76380d8 100644 --- a/src/taskgraph/transforms/run/run_task.py +++ b/src/taskgraph/transforms/run/run_task.py @@ -66,15 +66,6 @@ """.lstrip() ), ): str, - Required( - "sparse-profile", - description=dedent( - """ - The sparse checkout profile to use. Value is the filename relative to the - directory where sparse profiles are defined (build/sparse-profiles/). - """.lstrip() - ), - ): Any(str, None), Required( "command", description=dedent( @@ -140,21 +131,12 @@ def common_setup(config, task, taskdesc, command): task, taskdesc, repo_configs=repo_configs, - sparse=bool(run["sparse-profile"]), ) for repo_config in repo_configs.values(): checkout_path = path.join(vcs_path, repo_config.path) command.append(f"--{repo_config.prefix}-checkout={checkout_path}") - if run["sparse-profile"]: - command.append( - "--{}-sparse-profile=build/sparse-profiles/{}".format( - repo_config.prefix, # type: ignore - run["sparse-profile"], - ) - ) - if "cwd" in run: run["cwd"] = path.normpath(run["cwd"].format(checkout=vcs_path)) elif "cwd" in run and "{checkout}" in run["cwd"]: @@ -174,7 +156,6 @@ def common_setup(config, task, taskdesc, command): worker_defaults = { "checkout": True, - "sparse-profile": None, "run-as-root": False, } diff --git a/src/taskgraph/transforms/run/toolchain.py b/src/taskgraph/transforms/run/toolchain.py index 669bcd812..58d318043 100644 --- a/src/taskgraph/transforms/run/toolchain.py +++ b/src/taskgraph/transforms/run/toolchain.py @@ -50,18 +50,6 @@ """ ), ): [str], - Required( - "sparse-profile", - description=dedent( - """ - Sparse profile to give to checkout using `run-task`. If given, - a filename in `build/sparse-profiles`. Defaults to - "toolchain-build", i.e., to - `build/sparse-profiles/toolchain-build`. If `None`, instructs - `run-task` to not use a sparse profile at all. - """ - ), - ): Any(str, None), Optional( "resources", description=dedent( @@ -205,9 +193,7 @@ def common_toolchain(config, task, taskdesc, is_docker): configure_taskdesc_for_run(config, task, taskdesc, worker["implementation"]) -toolchain_defaults = { - "sparse-profile": "toolchain-build", -} +toolchain_defaults = {} @run_task_using( diff --git a/src/taskgraph/util/caches.py b/src/taskgraph/util/caches.py index 5554cc2e4..e4bfd3455 100644 --- a/src/taskgraph/util/caches.py +++ b/src/taskgraph/util/caches.py @@ -37,11 +37,6 @@ def get_checkout_cache_name(config: "TransformConfig", task: dict[str, Any]) -> digest = hashlib.sha256(checkout_paths_str).hexdigest() cache_name += f"-repos-{digest}" - # Sparse checkouts need their own cache because they can interfere - # with clients that aren't sparse aware. - if task["run"]["sparse-profile"]: - cache_name += "-sparse" - return cache_name diff --git a/template/{{cookiecutter.project_name}}/taskcluster.github.yml b/template/{{cookiecutter.project_name}}/taskcluster.github.yml index 3076f953b..fa42ddf11 100644 --- a/template/{{cookiecutter.project_name}}/taskcluster.github.yml +++ b/template/{{cookiecutter.project_name}}/taskcluster.github.yml @@ -222,7 +222,7 @@ tasks: ACTION_CALLBACK: '${action.cb_name}' cache: - "${trustDomain}-project-${project}-level-${level}-checkouts-sparse-v2": /builds/worker/checkouts + "${trustDomain}-project-${project}-level-${level}-checkouts-v2": /builds/worker/checkouts features: taskclusterProxy: true diff --git a/test/test_scripts_run_task.py b/test/test_scripts_run_task.py index f88c7ebeb..86ad57a37 100644 --- a/test/test_scripts_run_task.py +++ b/test/test_scripts_run_task.py @@ -195,7 +195,6 @@ def test_collect_vcs_options( args.setdefault(f"{name}_checkout", checkout) args.setdefault(f"{name}_shallow_clone", False) - args.setdefault(f"{name}_sparse_profile", False) args = Namespace(**args) result = run_task_mod.collect_vcs_options(args, name, name) @@ -214,7 +213,6 @@ def test_collect_vcs_options( "repo-type": env.get("REPOSITORY_TYPE"), "shallow-clone": False, "ssh-secret-name": env.get("SSH_SECRET_NAME"), - "sparse-profile": False, "store-path": env.get("HG_STORE_PATH"), } if "PIP_REQUIREMENTS" in env: diff --git a/test/test_transforms_run_toolchain.py b/test/test_transforms_run_toolchain.py index 6aaffaaa6..3ec843dff 100644 --- a/test/test_transforms_run_toolchain.py +++ b/test/test_transforms_run_toolchain.py @@ -65,7 +65,6 @@ def assert_docker_worker(task, taskdesc): "run": { "command": ["vcs/taskcluster/scripts/toolchain/run.sh"], "cwd": "{checkout}/..", - "sparse-profile": "toolchain-build", "using": "run-task", "workdir": "/builds/worker", }, @@ -140,7 +139,6 @@ def assert_generic_worker(task, taskdesc): "run": { "command": "src/taskcluster/scripts/toolchain/run.sh --foo bar", "cwd": "{checkout}/..", - "sparse-profile": "toolchain-build", "using": "run-task", "workdir": "/builds/worker", }, @@ -192,7 +190,6 @@ def assert_powershell(task, _): "command": "src/taskcluster/scripts/toolchain/run.ps1", "cwd": "{checkout}/..", "exec-with": "powershell", - "sparse-profile": "toolchain-build", "using": "run-task", "workdir": "/builds/worker", }