From db5e7e09a4fa4a507081b62ec3776c0ce554c0f0 Mon Sep 17 00:00:00 2001 From: AnsahMohammad Date: Wed, 5 Mar 2025 00:04:41 +0530 Subject: [PATCH 1/3] feat: support shallow git clones --- src/taskgraph/run-task/run-task | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/taskgraph/run-task/run-task b/src/taskgraph/run-task/run-task index b52e00342..c204281b5 100755 --- a/src/taskgraph/run-task/run-task +++ b/src/taskgraph/run-task/run-task @@ -600,6 +600,7 @@ def git_checkout( commit: Optional[str], ssh_key_file: Optional[Path], ssh_known_hosts_file: Optional[Path], + shallow: bool = True, ): env = { # abort if transfer speed is lower than 1kB/s for 1 minute @@ -640,6 +641,9 @@ def git_checkout( destination_path, ] + if shallow: + args.append("--depth=1") + retry_required_command(b"vcs", args, extra_env=env) if base_ref: From cea74f7ade41c1bb66414eb65a017389f7d90da6 Mon Sep 17 00:00:00 2001 From: AnsahMohammad Date: Thu, 6 Mar 2025 16:22:42 +0530 Subject: [PATCH 2/3] feat: add shallow in git_checkout --- src/taskgraph/run-task/run-task | 13 ++++++++++--- test/test_scripts_run_task.py | 20 ++++++++++++++------ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/taskgraph/run-task/run-task b/src/taskgraph/run-task/run-task index c204281b5..35ad81810 100755 --- a/src/taskgraph/run-task/run-task +++ b/src/taskgraph/run-task/run-task @@ -634,14 +634,16 @@ def git_checkout( if not os.path.exists(destination_path): # Repository doesn't already exist, needs to be cloned + repo_url = base_repo if base_repo else head_repo args = [ "git", "clone", - base_repo if base_repo else head_repo, + repo_url, destination_path, ] - - if shallow: + + # Only add --depth=1 for non-local repositories + if shallow and not os.path.exists(repo_url): args.append("--depth=1") retry_required_command(b"vcs", args, extra_env=env) @@ -934,6 +936,7 @@ def collect_vcs_options(args, project, name): "repo-type": repo_type, "ssh-secret-name": private_key_secret, "pip-requirements": pip_requirements, + "shallow": args.shallow, } @@ -982,6 +985,7 @@ def vcs_checkout_from_args(options): revision, ssh_key_file, ssh_known_hosts_file, + options.get("shallow", True), ) elif options["repo-type"] == "hg": if not revision and not ref: @@ -1065,6 +1069,9 @@ def main(args): parser.add_argument("--user", default="worker", help="user to run as") parser.add_argument("--group", default="worker", help="group to run as") parser.add_argument("--task-cwd", help="directory to run the provided command in") + parser.add_argument("--shallow", type=lambda x: x.lower() == "true", + default=True, help="perform a shallow clone (true/false, default: true)", + ) repositories = os.environ.get("REPOSITORIES") if repositories: diff --git a/test/test_scripts_run_task.py b/test/test_scripts_run_task.py index fc8c905ba..22c787895 100644 --- a/test/test_scripts_run_task.py +++ b/test/test_scripts_run_task.py @@ -178,6 +178,7 @@ def test_collect_vcs_options(monkeypatch, run_task_mod, env, extra_expected): args = Namespace() setattr(args, f"{name}_checkout", checkout) setattr(args, f"{name}_sparse_profile", False) + setattr(args, "shallow", True) result = run_task_mod.collect_vcs_options(args, name, name) @@ -197,6 +198,7 @@ def test_collect_vcs_options(monkeypatch, run_task_mod, env, extra_expected): "ssh-secret-name": env.get("SSH_SECRET_NAME"), "sparse-profile": False, "store-path": env.get("HG_STORE_PATH"), + "shallow": True, } if "PIP_REQUIREMENTS" in env: expected["pip-requirements"] = os.path.join( @@ -354,13 +356,14 @@ def _commit_file(message, filename): @pytest.mark.parametrize( - "base_ref,ref,files,hash_key", + "base_ref,ref,files,hash_key,shallow", [ - (None, None, ["mainfile"], "main"), - (None, "main", ["mainfile"], "main"), - (None, "mybranch", ["mainfile", "branchfile"], "branch"), - ("main", "main", ["mainfile"], "main"), - ("main", "mybranch", ["mainfile", "branchfile"], "branch"), + (None, None, ["mainfile"], "main", True), + (None, "main", ["mainfile"], "main", False), + (None, "mybranch", ["mainfile", "branchfile"], "branch", True), + ("main", "main", ["mainfile"], "main", False), + ("main", "mybranch", ["mainfile", "branchfile"], "branch", True), + ("main", "mybranch", ["mainfile", "branchfile"], "branch", False), ], ) def test_git_checkout( @@ -371,6 +374,7 @@ def test_git_checkout( ref, files, hash_key, + shallow, ): with tempfile.TemporaryDirectory() as workdir: destination = os.path.join(workdir, "destination") @@ -384,6 +388,7 @@ def test_git_checkout( commit=None, ssh_key_file=None, ssh_known_hosts_file=None, + shallow=shallow, ) # Check desired files exist @@ -403,10 +408,12 @@ def test_git_checkout( assert current_rev == mock_git_repo[hash_key] +@pytest.mark.parametrize("shallow", [True, False]) def test_git_checkout_with_commit( mock_stdin, run_task_mod, mock_git_repo, + shallow, ): with tempfile.TemporaryDirectory() as workdir: destination = os.path.join(workdir, "destination") @@ -420,6 +427,7 @@ def test_git_checkout_with_commit( commit=mock_git_repo["branch"], ssh_key_file=None, ssh_known_hosts_file=None, + shallow=shallow, ) From 8c6119691c22ca9c52df4e7e2d253a6f53229617 Mon Sep 17 00:00:00 2001 From: AnsahMohammad Date: Thu, 13 Mar 2025 17:38:26 +0530 Subject: [PATCH 3/3] fix: set shallow default to False Prevents potential backward compatibility issues. --- src/taskgraph/run-task/run-task | 8 ++++---- test/test_scripts_run_task.py | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/taskgraph/run-task/run-task b/src/taskgraph/run-task/run-task index eea85b4b6..405914681 100755 --- a/src/taskgraph/run-task/run-task +++ b/src/taskgraph/run-task/run-task @@ -600,7 +600,7 @@ def git_checkout( commit: Optional[str], ssh_key_file: Optional[Path], ssh_known_hosts_file: Optional[Path], - shallow: bool = True, + shallow: bool = False, ): env = { # abort if transfer speed is lower than 1kB/s for 1 minute @@ -985,7 +985,7 @@ def vcs_checkout_from_args(options): revision, ssh_key_file, ssh_known_hosts_file, - options.get("shallow", True), + options.get("shallow", False), ) elif options["repo-type"] == "hg": if not revision and not ref: @@ -1069,8 +1069,8 @@ def main(args): parser.add_argument("--user", default="worker", help="user to run as") parser.add_argument("--group", default="worker", help="group to run as") parser.add_argument("--task-cwd", help="directory to run the provided command in") - parser.add_argument("--shallow", type=lambda x: x.lower() == "true", - default=True, help="perform a shallow clone (true/false, default: true)", + parser.add_argument("--shallow", action="store_true", + default=False, help="perform a shallow clone (default: false)", ) repositories = os.environ.get("REPOSITORIES") diff --git a/test/test_scripts_run_task.py b/test/test_scripts_run_task.py index 22c787895..d73cf1b76 100644 --- a/test/test_scripts_run_task.py +++ b/test/test_scripts_run_task.py @@ -178,7 +178,7 @@ def test_collect_vcs_options(monkeypatch, run_task_mod, env, extra_expected): args = Namespace() setattr(args, f"{name}_checkout", checkout) setattr(args, f"{name}_sparse_profile", False) - setattr(args, "shallow", True) + setattr(args, "shallow", False) result = run_task_mod.collect_vcs_options(args, name, name) @@ -198,7 +198,7 @@ def test_collect_vcs_options(monkeypatch, run_task_mod, env, extra_expected): "ssh-secret-name": env.get("SSH_SECRET_NAME"), "sparse-profile": False, "store-path": env.get("HG_STORE_PATH"), - "shallow": True, + "shallow": False, } if "PIP_REQUIREMENTS" in env: expected["pip-requirements"] = os.path.join( @@ -358,12 +358,12 @@ def _commit_file(message, filename): @pytest.mark.parametrize( "base_ref,ref,files,hash_key,shallow", [ - (None, None, ["mainfile"], "main", True), + (None, None, ["mainfile"], "main", False), (None, "main", ["mainfile"], "main", False), - (None, "mybranch", ["mainfile", "branchfile"], "branch", True), + (None, "mybranch", ["mainfile", "branchfile"], "branch", False), ("main", "main", ["mainfile"], "main", False), - ("main", "mybranch", ["mainfile", "branchfile"], "branch", True), ("main", "mybranch", ["mainfile", "branchfile"], "branch", False), + ("main", "mybranch", ["mainfile", "branchfile"], "branch", True), ], ) def test_git_checkout(