Lanl/mirrors

stackinator/builder.py

@@ -11,7 +11,7 @@
 import jinja2
 import yaml
 
-from . import VERSION, cache, root_logger, spack_util
+from . import VERSION, cache, root_logger, spack_util, mirror
 
 
 def install(src, dst, *, ignore=None, symlinks=False):
@@ -226,12 +226,12 @@ def generate(self, recipe):
         with (self.path / "Makefile").open("w") as f:
             f.write(
                 makefile_template.render(
-                    cache=recipe.mirror,
                     modules=recipe.with_modules,
                     post_install_hook=recipe.post_install_hook,
                     pre_install_hook=recipe.pre_install_hook,
                     spack_version=spack_version,
                     spack_meta=spack_meta,
+                    mirrors=recipe.mirrors,
                     exclude_from_cache=["nvhpc", "cuda", "perl"],
                     verbose=False,
                 )
@@ -312,11 +312,12 @@ def generate(self, recipe):
             fid.write(global_packages_yaml)
 
         # generate a mirrors.yaml file if build caches have been configured
-        if recipe.mirror:
+        key_store = self.path / ".gnupg"
+        if recipe.mirrors:
+            mirror.key_setup(recipe.mirrors, config_path, key_store)
             dst = config_path / "mirrors.yaml"
-            self._logger.debug(f"generate the build cache mirror: {dst}")
-            with dst.open("w") as fid:
-                fid.write(cache.generate_mirrors_yaml(recipe.mirror))
+            self._logger.debug(f"generate the spack mirrors.yaml: {dst}")
+            mirror.spack_yaml_setup(recipe.mirrors, dst)
 
         # Add custom spack package recipes, configured via Spack repos.
         # Step 1: copy Spack repos to store_path where they will be used to

stackinator/main.py

@@ -81,13 +81,19 @@ def log_header(args):
 def make_argparser():
     parser = argparse.ArgumentParser(description=("Generate a build configuration for a spack stack from a recipe."))
     parser.add_argument("--version", action="version", version=f"stackinator version {VERSION}")
-    parser.add_argument("-b", "--build", required=True, type=str)
+    parser.add_argument("-b", "--build", required=True, type=str,
+                        help="Where to set up the stackinator build directory. "
+                             "('/tmp' is not allowed, use '/var/tmp'")
     parser.add_argument("--no-bwrap", action="store_true", required=False)
-    parser.add_argument("-r", "--recipe", required=True, type=str)
-    parser.add_argument("-s", "--system", required=True, type=str)
+    parser.add_argument("-r", "--recipe", required=True, type=str,
+                        help="Name of (and/or path to) the Stackinator recipe.")
+    parser.add_argument("-s", "--system", required=True, type=str,
+                        help="Name of (and/or path to) the Stackinator system configuration.")
     parser.add_argument("-d", "--debug", action="store_true")
-    parser.add_argument("-m", "--mount", required=False, type=str)
-    parser.add_argument("-c", "--cache", required=False, type=str)
+    parser.add_argument("-m", "--mount", required=False, type=str,
+                        help="The mount point where the environment will be located.")
+    parser.add_argument("-c", "--cache", required=False, type=str, 
+                        help="Buildcache location or name (from system config's mirrors.yaml).")
     parser.add_argument("--develop", action="store_true", required=False)
 
     return parser

stackinator/mirror.py

@@ -0,0 +1,145 @@
+import os
+import pathlib
+import urllib.request
+from typing import Optional, List, Dict
+import magic
+
+import yaml
+
+from . import schema
+
+class MirrorConfigError(RuntimeError):
+    """Exception class for errors thrown by mirror configuration problems."""
+
+
+def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None):
+    """Configure mirrors from both the system 'mirror.yaml' file and the command line."""
+
+    path = system_config_root/"mirrors.yaml"
+    if path.exists():
+        with path.open() as fid:
+            # load the raw yaml input
+            raw = yaml.load(fid, Loader=yaml.Loader)
+
+        print(f"Configuring mirrors and buildcache from '{path}'")
+
+        # validate the yaml
+        schema.CacheValidator.validate(raw)
+
+        mirrors = [mirror for mirror in raw if mirror["enabled"]]
+    else:
+        mirrors = []
+
+    buildcache_dest_count = len([mirror for mirror in mirrors if mirror['buildcache']])
+    if buildcache_dest_count > 1:
+        raise RuntimeError("Mirror config has more than one mirror specified as the build cache destination "
+                           "in the system config's 'mirrors.yaml'.")
+    elif buildcache_dest_count == 1 and cmdline_cache:
+        raise RuntimeError("Build cache destination specified on the command line and in the system config's "
+                           "'mirrors.yaml'. It can be one or the other, but not both.")
+
+    # Add or set the cache given on the command line as the buildcache destination
+    if cmdline_cache is not None:
+        existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache][:1]
+        # If the mirror name given on the command line isn't in the config, assume it 
+        # is the URL to a build cache.
+        if not existing_mirror:
+            mirrors.append(
+                {
+                    'name': 'cmdline_cache',
+                    'url': cmdline_cache,
+                    'buildcache': True,
+                    'bootstrap': False,
+                }
+            )
+
+    for mirror in mirrors:
+        url = mirror["url"]
+        if url.beginswith("file://"):
+            # verify that the root path exists
+            path = pathlib.Path(os.path.expandvars(url))
+            if not path.is_absolute():
+                raise FileNotFoundError(f"The mirror path '{path}' is not absolute")
+            if not path.is_dir():
+                raise FileNotFoundError(f"The mirror path '{path}' is not a directory")
+
+            mirror["url"] = path
+
+        elif url.beginswith("https://"):
+            try:
+                request = urllib.request.Request(url, method='HEAD')
+                response = urllib.request.urlopen(request)
+            except urllib.error.URLError as e:
+                raise MirrorConfigError(
+                    f"Could not reach the mirror url '{url}'. " 
+                    f"Check the url listed in mirrors.yaml in system config. \n{e.reason}")
+
+        #if mirror["bootstrap"]:
+            #make bootstrap dirs
+            #bootstrap/<mirror name>/metadata.yaml
+
+        return mirrors
+
+
+def spack_yaml_setup(mirrors, config_path):
+    """Generate the mirrors.yaml for spack"""
+
+    dst = config_path / "mirrors.yaml"
+
+    yaml = {"mirrors": {}}
+
+    for m in mirrors:
+        name = m["name"]
+        url = m["url"]
+
+        yaml["mirrors"][name] = {
+            "fetch": {"url": url},
+            "push": {"url": url},
+        }
+
+    with dst.open("w") as file:
+        yaml.dump(yaml, default_flow_style=False)
+
+    # return dst
+
+
+def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: pathlib.Path):
+    """Validate mirror keys, relocate to key_store, and update mirror config with new key paths"""
+
+    for mirror in mirrors:
+        if mirror["key"]:
+            key = mirror["key"]
+
+            # key will be saved under key_store/mirror_name.gpg
+            dst = (key_store / f"'{mirror["name"]}'.gpg").resolve()
+
+            # if path, check if abs path, if not, append sys config path in front and check again
+            path = pathlib.Path(os.path.expandvars(key))
+            if path.exists():
+                if not path.is_absolute():
+                    #try prepending system config path
+                    path = system_config_path + path
+                    if not path.is_file():
+                        raise FileNotFoundError(
+                            f"The key path '{path}' is not a file. "
+                            f"Check the key listed in mirrors.yaml in system config.")
+
+                file_type = magic.from_file(path)
+
+                if not file_type.startswith("OpenPGP Public Key"):
+                    raise MirrorConfigError(
+                        f"'{path}' is not a valid GPG key. "
+                        f"Check the key listed in mirrors.yaml in system config.")
+
+                # copy key to new destination in key store
+                with open(path, 'r') as reader, open(dst, 'w') as writer:
+                    data = reader.read()
+                    writer.write(data)
+
+            else:            
+                # if PGP key, convert to binary, ???, convert back
+                with open(dst, "w") as file:
+                    file.write(key)
+
+            # update mirror with new path
+            mirror["key"] = dst

stackinator/recipe.py

@@ -4,8 +4,9 @@
 
 import jinja2
 import yaml
+from typing import Optional
 
-from . import cache, root_logger, schema, spack_util
+from . import cache, root_logger, schema, spack_util, mirror
 from .etc import envvars
 
 
@@ -169,15 +170,11 @@ def __init__(self, args):
             schema.EnvironmentsValidator.validate(raw)
             self.generate_environment_specs(raw)
 
-        # optional mirror configurtion
-        mirrors_path = self.path / "mirrors.yaml"
-        if mirrors_path.is_file():
-            self._logger.warning(
-                "mirrors.yaml have been removed from recipes, use the --cache option on stack-config instead."
-            )
-            raise RuntimeError("Unsupported mirrors.yaml file in recipe.")
-
-        self.mirror = (args.cache, self.mount)
+        # load the optional mirrors.yaml from system config, and add any additional 
+        # mirrors specified on the command line.
+        self._mirrors = None
+        self._logger.debug("Configuring mirrors.")
+        self._mirrors = mirror.configuration_from_file(self.system_config_path, args.cache)
 
         # optional post install hook
         if self.post_install_hook is not None:
@@ -236,32 +233,10 @@ def pre_install_hook(self):
             return hook_path
         return None
 
-    # Returns a dictionary with the following fields
-    #
-    # root: /path/to/cache
-    # path: /path/to/cache/user-environment
-    # key: /path/to/private-pgp-key
     @property
-    def mirror(self):
-        return self._mirror
-
-    # configuration is a tuple with two fields:
-    # - a Path of the yaml file containing the cache configuration
-    # - the mount point of the image
-    @mirror.setter
-    def mirror(self, configuration):
-        self._logger.debug(f"configuring build cache mirror with {configuration}")
-        self._mirror = None
-
-        file, mount = configuration
-
-        if file is not None:
-            mirror_config_path = pathlib.Path(file)
-            if not mirror_config_path.is_file():
-                raise FileNotFoundError(f"The cache configuration '{file}' is not a file")
-
-            self._mirror = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount))
-
+    def mirrors(self):
+        return self._mirrors
+
     @property
     def config(self):
         return self._config
@@ -541,7 +516,7 @@ def compiler_files(self):
         )
 
         makefile_template = env.get_template("Makefile.compilers")
-        push_to_cache = self.mirror is not None
+        push_to_cache = self.mirrors
         files["makefile"] = makefile_template.render(
             compilers=self.compilers,
             push_to_cache=push_to_cache,

stackinator/schema/mirror.json

@@ -0,0 +1,40 @@
+{
+    "type" : "array",
+    "items": {
+        "type": "object",
+        "required": ["name", "url"],
+        "properties": {
+            "name": {
+                "type": "string",
+                "description": "The name of this mirror. Should be follow standard variable naming syntax."
+            },
+            "url": {
+                "type": "string",
+                "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI)."
+            },
+            "enabled": {
+                "type": "boolean",
+                "default": true,
+                "description": "Whether this mirror is enabled."
+            },
+            "bootstrap": {
+                "type": "boolean",
+                "default": false,
+                "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror."
+            },
+            "buildcache": {
+                "type": "boolean",
+                "default": false,
+                "description": "Use this mirror as the buildcache push destination. Can only be enabled on a single mirror."
+            },
+            "public_key": {
+                "type": "string",
+                "description": "Public PGP key for validating binary cache packages."
+            },
+            "description": {
+                "type": "string",
+                "description": "What this mirror is for."
+            }
+        }
+    }
+}

stackinator/templates/Makefile

@@ -32,13 +32,25 @@ pre-install: spack-setup
 	$(SANDBOX) $(STORE)/pre-install-hook
 
 mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %}
-
 	{% if cache %}
+	# The old way of managing mirrors
 	$(SANDBOX) $(SPACK) buildcache keys --install --trust
 	{% if cache.key %}
 	$(SANDBOX) $(SPACK) gpg trust {{ cache.key }}
 	{% endif %}
 	{% endif %}
+	{% if mirrors %}
+	@echo "Adding mirrors and gpg keys."
+	{% for mirror_info in mirrors | reverse %}
+	$(SANDBOX) $(SPACK) mirror add --scope=site {{ mirror_info.name }} {{ mirror_info.url }}
+	$(SANDBOX) $(SPACK)	gpg trust {{ mirror_info.key_path }}
+	{% endfor %}
+	@echo "Current mirror list:"
+	$(SANDBOX) $(SPACK) mirror list
+	{% endif %}
+	{% for mirror_info in filter(lambda m: m['bootstrap'], mirrors) | filter() %}
+	$(SANDBOX) $(SPACK) bootstrap add --scope=site {{ mirror_info.name }} bootstrap/{{ mirror_info.name }}
+	{% endfor %}
 	touch mirror-setup
 
 compilers: mirror-setup