feat(run-task): implement shallow git clones

Author: ahal

src/taskgraph/run-task/run-task

@@ -28,15 +28,14 @@ import os
 import platform
 import re
 import shutil
-import signal
 import socket
 import stat
 import subprocess
 import time
 import urllib.error
 import urllib.request
+from itertools import count
 from pathlib import Path
-from threading import Thread
 from typing import Dict, Optional
 
 SECRET_BASEURL_TPL = "{}/secrets/v1/secret/{{}}".format(os.environ.get("TASKCLUSTER_PROXY_URL", "http://taskcluster").rstrip('/'))
@@ -556,6 +555,7 @@ def git_fetch(
     ref: str,
     remote: str = "origin",
     tags: bool = False,
+    shallow: bool = False,
     env: Optional[Dict[str, str]] = None,
 ):
     args = ["git", "fetch"]
@@ -564,7 +564,20 @@ def git_fetch(
         args.extend(["--tags", "--force"])
 
     args.extend([remote, ref])
-    retry_required_command(b"vcs", args, cwd=destination_path, extra_env=env)
+
+    if shallow:
+        for deepen in range(10, 100, 10):
+            args[2:2] = [f"--deepen={deepen}"]
+            run_command(b"vcs", args, cwd=destination_path, extra_env=env)
+
+            ret = run_command(b"vcs", ["git", "cat-file", "-e", ref])
+            if ret == 0:
+                return
+
+        print(f"unable to fetch {ref} from {remote} in shallow clone")
+        sys.exit(1)
+    else:
+        retry_required_command(b"vcs", args, cwd=destination_path, extra_env=env)
 
 
 def _clean_git_checkout(destination_path):
@@ -615,6 +628,7 @@ def git_checkout(
     commit: Optional[str],
     ssh_key_file: Optional[Path],
     ssh_known_hosts_file: Optional[Path],
+    shallow_clone: bool = False,
 ):
     env = {
         # abort if transfer speed is lower than 1kB/s for 1 minute
@@ -651,9 +665,18 @@ def git_checkout(
         args = [
             "git",
             "clone",
+        ]
+        
+        if shallow_clone:
+            # Use shallow clone with depth 1 for minimal history
+            args.extend(["--depth=1"])
+            # Skip checkout initially
+            args.extend(["--no-checkout"])
+        
+        args.extend([
             base_repo if base_repo else head_repo,
             destination_path,
-        ]
+        ])
 
         retry_required_command(b"vcs", args, extra_env=env)
 
@@ -673,7 +696,7 @@ def git_checkout(
 
     # If a ref isn't provided, we fetch all refs from head_repo, which may be slow.
     target = ref if ref else "+refs/heads/*:refs/remotes/work/*"
-    git_fetch(destination_path, target, remote=head_repo, tags=tags, env=env)
+    git_fetch(destination_path, target, remote=head_repo, tags=tags, shallow=shallow_clone, env=env)
 
     args = [
         "git",
@@ -861,11 +884,17 @@ def add_vcs_arguments(parser, project, name):
         "--%s-sparse-profile" % project,
         help="Path to sparse profile for %s checkout" % name,
     )
+    parser.add_argument(
+        "--%s-shallow-clone" % project,
+        action="store_true",
+        help="Use shallow clone for %s" % name,
+    )
 
 
 def collect_vcs_options(args, project, name):
     checkout = getattr(args, "%s_checkout" % project)
     sparse_profile = getattr(args, "%s_sparse_profile" % project)
+    shallow_clone = getattr(args, "%s_shallow_clone" % project)
 
     env_prefix = project.upper()
 
@@ -910,6 +939,7 @@ def collect_vcs_options(args, project, name):
         "repo-type": repo_type,
         "ssh-secret-name": private_key_secret,
         "pip-requirements": pip_requirements,
+        "shallow-clone": shallow_clone,
     }
 
 
@@ -958,6 +988,7 @@ def vcs_checkout_from_args(options):
                 revision,
                 ssh_key_file,
                 ssh_known_hosts_file,
+                options.get("shallow-clone", False),
             )
         elif options["repo-type"] == "hg":
             if not revision and not ref: