feat: use big-parallel to identify large packages

Author: jfroche

nix/ext/pg_graphql/default.nix

@@ -129,7 +129,7 @@ let
     lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) supportedVersions
   );
 in
-buildEnv {
+(buildEnv {
   name = pname;
   paths = packages;
   pathsToLink = [
@@ -177,4 +177,7 @@ buildEnv {
     version =
       "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions);
   };
-}
+}).overrideAttrs
+  (_: {
+    requiredSystemFeatures = [ "big-parallel" ];
+  })

nix/ext/pg_jsonschema/default.nix

@@ -130,7 +130,7 @@ let
     lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) supportedVersions
   );
 in
-pkgs.buildEnv {
+(pkgs.buildEnv {
   name = pname;
   paths = packages;
   pathsToLink = [
@@ -170,4 +170,7 @@ pkgs.buildEnv {
     version =
       "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions);
   };
-}
+}).overrideAttrs
+  (_: {
+    requiredSystemFeatures = [ "big-parallel" ];
+  })

nix/ext/postgis.nix

@@ -179,7 +179,7 @@ let
       };
     };
 in
-buildEnv {
+(buildEnv {
   name = pname;
   paths = packages;
 
@@ -209,4 +209,7 @@ buildEnv {
     version =
       "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions);
   };
-}
+}).overrideAttrs
+  (_: {
+    requiredSystemFeatures = [ "big-parallel" ];
+  })

nix/ext/wrappers/default.nix

@@ -218,7 +218,7 @@ let
     lib.mapAttrs (name: value: build name value.hash value.rust value.pgrx) supportedVersions
   );
 in
-buildEnv {
+(buildEnv {
   name = pname;
   paths = packages;
   pathsToLink = [
@@ -303,4 +303,7 @@ buildEnv {
     version =
       "multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions);
   };
-}
+}).overrideAttrs
+  (_: {
+    requiredSystemFeatures = [ "big-parallel" ];
+  })

nix/packages/github-matrix/github_matrix.py

@@ -36,6 +36,7 @@ class NixEvalJobsOutput(TypedDict):
     neededSubstitutes: NotRequired[List[Any]]
     outputs: NotRequired[Dict[str, str]]
     error: NotRequired[str]
+    requiredSystemFeatures: NotRequired[List[str]]
 
 
 class RunsOnConfig(TypedDict):
@@ -174,11 +175,7 @@ def sort_pkgs_by_closures(jobs: List[NixEvalJobsOutput]) -> List[NixEvalJobsOutp
 
 def is_large_pkg(pkg: NixEvalJobsOutput) -> bool:
     """Determine if a package is considered large based on its attribute path."""
-    RUST_EXTENSIONS = ["exts.wrappers", "exts.pg_jsonschema", "exts.pg_graphql"]
-    LARGE_C_EXTENSION = ["exts.postgis"]
-    return any(
-        indicator in pkg["attr"] for indicator in RUST_EXTENSIONS + LARGE_C_EXTENSION
-    )
+    return "big-parallel" in pkg.get("requiredSystemFeatures", [])
 
 
 def is_kvm_pkg(pkg: NixEvalJobsOutput) -> bool:
@@ -198,7 +195,12 @@ def get_runner_for_package(pkg: NixEvalJobsOutput) -> RunsOnConfig | None:
     system = pkg["system"]
 
     if is_kvm_pkg(pkg):
-        return BUILD_RUNNER_MAP["self-hosted"].get(system)
+        runConfig = BUILD_RUNNER_MAP["self-hosted"].get(system)
+        if runConfig is None:
+            raise ValueError(
+                f"No self-hosted with kvm support available for system: {system}"
+            )
+        return runConfig
 
     if is_large_pkg(pkg) and system in ("x86_64-linux", "aarch64-linux"):
         suffix = "-arm" if system == "aarch64-linux" else ""

nix/packages/github-matrix/tests/test_github_matrix.py

@@ -56,6 +56,7 @@ def test_large_package_detection(self, attr: str, expected: bool):
             "drvPath": f"/nix/store/{attr}.drv",
             "name": attr.split(".")[-1],
             "system": "x86_64-linux",
+            "requiredSystemFeatures": ["big-parallel"] if expected else [],
         }
         assert is_large_pkg(pkg) is expected
 
@@ -96,8 +97,11 @@ def test_kvm_package_x86_64_linux(self):
             "system": "x86_64-linux",
             "requiredSystemFeatures": ["kvm"],
         }
-        result = get_runner_for_package(pkg)
-        assert result is None  # x86_64-linux not in self-hosted map
+        with pytest.raises(
+            ValueError,
+            match=r"No self-hosted with kvm support available for system: x86_64-linux",
+        ):
+            get_runner_for_package(pkg)
 
     def test_kvm_package_aarch64_linux(self):
         pkg: NixEvalJobsOutput = {
@@ -123,6 +127,7 @@ def test_large_package_x86_64_linux(self):
             "drvPath": "/nix/store/test.drv",
             "name": "postgis",
             "system": "x86_64-linux",
+            "requiredSystemFeatures": ["big-parallel"],
         }
         result = get_runner_for_package(pkg)
         assert result == {"labels": ["blacksmith-32vcpu-ubuntu-2404"]}
@@ -135,6 +140,7 @@ def test_large_package_aarch64_linux(self):
             "drvPath": "/nix/store/test.drv",
             "name": "pg_graphql",
             "system": "aarch64-linux",
+            "requiredSystemFeatures": ["big-parallel"],
         }
         result = get_runner_for_package(pkg)
         assert result == {"labels": ["blacksmith-32vcpu-ubuntu-2404-arm"]}