diff --git a/stackinator/builder.py b/stackinator/builder.py index 2c05e3c1..7918cb24 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -11,7 +11,7 @@ import jinja2 import yaml -from . import VERSION, cache, root_logger, spack_util +from . import VERSION, cache, root_logger, spack_util, mirror def install(src, dst, *, ignore=None, symlinks=False): @@ -226,12 +226,12 @@ def generate(self, recipe): with (self.path / "Makefile").open("w") as f: f.write( makefile_template.render( - cache=recipe.mirror, modules=recipe.with_modules, post_install_hook=recipe.post_install_hook, pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, + mirrors=recipe.mirrors, exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) @@ -312,11 +312,12 @@ def generate(self, recipe): fid.write(global_packages_yaml) # generate a mirrors.yaml file if build caches have been configured - if recipe.mirror: + key_store = self.path / ".gnupg" + if recipe.mirrors: + mirror.key_setup(recipe.mirrors, config_path, key_store) dst = config_path / "mirrors.yaml" - self._logger.debug(f"generate the build cache mirror: {dst}") - with dst.open("w") as fid: - fid.write(cache.generate_mirrors_yaml(recipe.mirror)) + self._logger.debug(f"generate the spack mirrors.yaml: {dst}") + mirror.spack_yaml_setup(recipe.mirrors, dst) # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to diff --git a/stackinator/main.py b/stackinator/main.py index 44406215..6f30ddfc 100644 --- a/stackinator/main.py +++ b/stackinator/main.py @@ -81,13 +81,19 @@ def log_header(args): def make_argparser(): parser = argparse.ArgumentParser(description=("Generate a build configuration for a spack stack from a recipe.")) parser.add_argument("--version", action="version", version=f"stackinator version {VERSION}") - parser.add_argument("-b", "--build", required=True, type=str) + parser.add_argument("-b", "--build", required=True, type=str, + help="Where to set up the stackinator build directory. " + "('/tmp' is not allowed, use '/var/tmp'") parser.add_argument("--no-bwrap", action="store_true", required=False) - parser.add_argument("-r", "--recipe", required=True, type=str) - parser.add_argument("-s", "--system", required=True, type=str) + parser.add_argument("-r", "--recipe", required=True, type=str, + help="Name of (and/or path to) the Stackinator recipe.") + parser.add_argument("-s", "--system", required=True, type=str, + help="Name of (and/or path to) the Stackinator system configuration.") parser.add_argument("-d", "--debug", action="store_true") - parser.add_argument("-m", "--mount", required=False, type=str) - parser.add_argument("-c", "--cache", required=False, type=str) + parser.add_argument("-m", "--mount", required=False, type=str, + help="The mount point where the environment will be located.") + parser.add_argument("-c", "--cache", required=False, type=str, + help="Buildcache location or name (from system config's mirrors.yaml).") parser.add_argument("--develop", action="store_true", required=False) return parser diff --git a/stackinator/mirror.py b/stackinator/mirror.py new file mode 100644 index 00000000..0e5ee5d1 --- /dev/null +++ b/stackinator/mirror.py @@ -0,0 +1,145 @@ +import os +import pathlib +import urllib.request +from typing import Optional, List, Dict +import magic + +import yaml + +from . import schema + +class MirrorConfigError(RuntimeError): + """Exception class for errors thrown by mirror configuration problems.""" + + +def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): + """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" + + path = system_config_root/"mirrors.yaml" + if path.exists(): + with path.open() as fid: + # load the raw yaml input + raw = yaml.load(fid, Loader=yaml.Loader) + + print(f"Configuring mirrors and buildcache from '{path}'") + + # validate the yaml + schema.CacheValidator.validate(raw) + + mirrors = [mirror for mirror in raw if mirror["enabled"]] + else: + mirrors = [] + + buildcache_dest_count = len([mirror for mirror in mirrors if mirror['buildcache']]) + if buildcache_dest_count > 1: + raise RuntimeError("Mirror config has more than one mirror specified as the build cache destination " + "in the system config's 'mirrors.yaml'.") + elif buildcache_dest_count == 1 and cmdline_cache: + raise RuntimeError("Build cache destination specified on the command line and in the system config's " + "'mirrors.yaml'. It can be one or the other, but not both.") + + # Add or set the cache given on the command line as the buildcache destination + if cmdline_cache is not None: + existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache][:1] + # If the mirror name given on the command line isn't in the config, assume it + # is the URL to a build cache. + if not existing_mirror: + mirrors.append( + { + 'name': 'cmdline_cache', + 'url': cmdline_cache, + 'buildcache': True, + 'bootstrap': False, + } + ) + + for mirror in mirrors: + url = mirror["url"] + if url.beginswith("file://"): + # verify that the root path exists + path = pathlib.Path(os.path.expandvars(url)) + if not path.is_absolute(): + raise FileNotFoundError(f"The mirror path '{path}' is not absolute") + if not path.is_dir(): + raise FileNotFoundError(f"The mirror path '{path}' is not a directory") + + mirror["url"] = path + + elif url.beginswith("https://"): + try: + request = urllib.request.Request(url, method='HEAD') + response = urllib.request.urlopen(request) + except urllib.error.URLError as e: + raise MirrorConfigError( + f"Could not reach the mirror url '{url}'. " + f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") + + #if mirror["bootstrap"]: + #make bootstrap dirs + #bootstrap//metadata.yaml + + return mirrors + + +def spack_yaml_setup(mirrors, config_path): + """Generate the mirrors.yaml for spack""" + + dst = config_path / "mirrors.yaml" + + yaml = {"mirrors": {}} + + for m in mirrors: + name = m["name"] + url = m["url"] + + yaml["mirrors"][name] = { + "fetch": {"url": url}, + "push": {"url": url}, + } + + with dst.open("w") as file: + yaml.dump(yaml, default_flow_style=False) + + # return dst + + +def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: pathlib.Path): + """Validate mirror keys, relocate to key_store, and update mirror config with new key paths""" + + for mirror in mirrors: + if mirror["key"]: + key = mirror["key"] + + # key will be saved under key_store/mirror_name.gpg + dst = (key_store / f"'{mirror["name"]}'.gpg").resolve() + + # if path, check if abs path, if not, append sys config path in front and check again + path = pathlib.Path(os.path.expandvars(key)) + if path.exists(): + if not path.is_absolute(): + #try prepending system config path + path = system_config_path + path + if not path.is_file(): + raise FileNotFoundError( + f"The key path '{path}' is not a file. " + f"Check the key listed in mirrors.yaml in system config.") + + file_type = magic.from_file(path) + + if not file_type.startswith("OpenPGP Public Key"): + raise MirrorConfigError( + f"'{path}' is not a valid GPG key. " + f"Check the key listed in mirrors.yaml in system config.") + + # copy key to new destination in key store + with open(path, 'r') as reader, open(dst, 'w') as writer: + data = reader.read() + writer.write(data) + + else: + # if PGP key, convert to binary, ???, convert back + with open(dst, "w") as file: + file.write(key) + + # update mirror with new path + mirror["key"] = dst \ No newline at end of file diff --git a/stackinator/recipe.py b/stackinator/recipe.py index ca8d2b3d..c59790b2 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -4,8 +4,9 @@ import jinja2 import yaml +from typing import Optional -from . import cache, root_logger, schema, spack_util +from . import cache, root_logger, schema, spack_util, mirror from .etc import envvars @@ -169,15 +170,11 @@ def __init__(self, args): schema.EnvironmentsValidator.validate(raw) self.generate_environment_specs(raw) - # optional mirror configurtion - mirrors_path = self.path / "mirrors.yaml" - if mirrors_path.is_file(): - self._logger.warning( - "mirrors.yaml have been removed from recipes, use the --cache option on stack-config instead." - ) - raise RuntimeError("Unsupported mirrors.yaml file in recipe.") - - self.mirror = (args.cache, self.mount) + # load the optional mirrors.yaml from system config, and add any additional + # mirrors specified on the command line. + self._mirrors = None + self._logger.debug("Configuring mirrors.") + self._mirrors = mirror.configuration_from_file(self.system_config_path, args.cache) # optional post install hook if self.post_install_hook is not None: @@ -236,32 +233,10 @@ def pre_install_hook(self): return hook_path return None - # Returns a dictionary with the following fields - # - # root: /path/to/cache - # path: /path/to/cache/user-environment - # key: /path/to/private-pgp-key @property - def mirror(self): - return self._mirror - - # configuration is a tuple with two fields: - # - a Path of the yaml file containing the cache configuration - # - the mount point of the image - @mirror.setter - def mirror(self, configuration): - self._logger.debug(f"configuring build cache mirror with {configuration}") - self._mirror = None - - file, mount = configuration - - if file is not None: - mirror_config_path = pathlib.Path(file) - if not mirror_config_path.is_file(): - raise FileNotFoundError(f"The cache configuration '{file}' is not a file") - - self._mirror = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount)) - + def mirrors(self): + return self._mirrors + @property def config(self): return self._config @@ -541,7 +516,7 @@ def compiler_files(self): ) makefile_template = env.get_template("Makefile.compilers") - push_to_cache = self.mirror is not None + push_to_cache = self.mirrors files["makefile"] = makefile_template.render( compilers=self.compilers, push_to_cache=push_to_cache, diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json new file mode 100644 index 00000000..a8be6ab3 --- /dev/null +++ b/stackinator/schema/mirror.json @@ -0,0 +1,40 @@ +{ + "type" : "array", + "items": { + "type": "object", + "required": ["name", "url"], + "properties": { + "name": { + "type": "string", + "description": "The name of this mirror. Should be follow standard variable naming syntax." + }, + "url": { + "type": "string", + "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI)." + }, + "enabled": { + "type": "boolean", + "default": true, + "description": "Whether this mirror is enabled." + }, + "bootstrap": { + "type": "boolean", + "default": false, + "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror." + }, + "buildcache": { + "type": "boolean", + "default": false, + "description": "Use this mirror as the buildcache push destination. Can only be enabled on a single mirror." + }, + "public_key": { + "type": "string", + "description": "Public PGP key for validating binary cache packages." + }, + "description": { + "type": "string", + "description": "What this mirror is for." + } + } + } +} \ No newline at end of file diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 10a0ea58..d1a06dd4 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -32,13 +32,25 @@ pre-install: spack-setup $(SANDBOX) $(STORE)/pre-install-hook mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} - {% if cache %} + # The old way of managing mirrors $(SANDBOX) $(SPACK) buildcache keys --install --trust {% if cache.key %} $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} {% endif %} {% endif %} + {% if mirrors %} + @echo "Adding mirrors and gpg keys." + {% for mirror_info in mirrors | reverse %} + $(SANDBOX) $(SPACK) mirror add --scope=site {{ mirror_info.name }} {{ mirror_info.url }} + $(SANDBOX) $(SPACK) gpg trust {{ mirror_info.key_path }} + {% endfor %} + @echo "Current mirror list:" + $(SANDBOX) $(SPACK) mirror list + {% endif %} + {% for mirror_info in filter(lambda m: m['bootstrap'], mirrors) | filter() %} + $(SANDBOX) $(SPACK) bootstrap add --scope=site {{ mirror_info.name }} bootstrap/{{ mirror_info.name }} + {% endfor %} touch mirror-setup compilers: mirror-setup