From f1618bfa2631cf6eda75da89e71891ba300e97ac Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Tue, 17 Feb 2026 16:46:01 -0700 Subject: [PATCH 01/16] added spack source mirrors capability, needs testing --- stackinator/builder.py | 2 ++ stackinator/schema/config.json | 7 +++++++ stackinator/templates/Makefile | 10 ++++++++++ 3 files changed, 19 insertions(+) diff --git a/stackinator/builder.py b/stackinator/builder.py index 2c05e3c1..c3de44db 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -232,6 +232,8 @@ def generate(self, recipe): pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, + # pass source_mirrors to Makefile render + source_mirrors=recipe.config.get("source_mirrors", {}), exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) diff --git a/stackinator/schema/config.json b/stackinator/schema/config.json index d6fec3a0..4b91011d 100644 --- a/stackinator/schema/config.json +++ b/stackinator/schema/config.json @@ -64,6 +64,13 @@ } } }, + "source_mirrors" : { + "type" : "object", + "additionalProperties": { + "type" : "string" + }, + "default": {} + }, "modules" : { "type": "boolean" }, diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 10a0ea58..3da2e00e 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -39,6 +39,16 @@ mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} {% endif %} {% endif %} + {% if source_mirrors %} + echo "Replacing all instances of mirror.spack.io... Just in case" + grep -rl "https://mirror.spack.io" . | xargs sed -i 's/https:\/\/mirror.spack.io/https:\/\/pe-serve.lanl.gov\/spack-mirror/g' + echo "Adding mirrors" + {% for name, url in source_mirrors.items() %} + $(SANDBOX) $(SPACK) mirror add {{ name }} {{ url }} + {% endfor %} + echo "Current mirror list:" + spack mirror list + {% endif %} touch mirror-setup compilers: mirror-setup From 954a6901a6544ef6e190bea4cdfc8ada4b6acffe Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Wed, 18 Feb 2026 14:34:05 -0700 Subject: [PATCH 02/16] removed lanl stuff --- stackinator/templates/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 3da2e00e..fcf6d68a 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -40,13 +40,13 @@ mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} {% endif %} {% endif %} {% if source_mirrors %} - echo "Replacing all instances of mirror.spack.io... Just in case" - grep -rl "https://mirror.spack.io" . | xargs sed -i 's/https:\/\/mirror.spack.io/https:\/\/pe-serve.lanl.gov\/spack-mirror/g' + echo "Removing all instances of mirror.spack.io... Just in case" + grep -rl "https://mirror.spack.io" . | xargs sed -i 's|https://mirror.spack.io||g' echo "Adding mirrors" {% for name, url in source_mirrors.items() %} $(SANDBOX) $(SPACK) mirror add {{ name }} {{ url }} {% endfor %} - echo "Current mirror list:" + echo "Spack mirrors for this recipe:" spack mirror list {% endif %} touch mirror-setup From e2c646267a6e61a41ebe64b0b538609897ac4423 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Tue, 24 Feb 2026 10:43:33 -0700 Subject: [PATCH 03/16] add source mirrors via config.yaml and retain spack default mirror --- stackinator/templates/Makefile | 30 ++++++++++++++---------------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index fcf6d68a..1d94c15a 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -34,22 +34,20 @@ pre-install: spack-setup mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} {% if cache %} - $(SANDBOX) $(SPACK) buildcache keys --install --trust - {% if cache.key %} - $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} - {% endif %} - {% endif %} - {% if source_mirrors %} - echo "Removing all instances of mirror.spack.io... Just in case" - grep -rl "https://mirror.spack.io" . | xargs sed -i 's|https://mirror.spack.io||g' - echo "Adding mirrors" - {% for name, url in source_mirrors.items() %} - $(SANDBOX) $(SPACK) mirror add {{ name }} {{ url }} - {% endfor %} - echo "Spack mirrors for this recipe:" - spack mirror list - {% endif %} - touch mirror-setup + $(SANDBOX) $(SPACK) buildcache keys --install --trust + {% if cache.key %} + $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} + {% endif %} + {% endif %} + {% if source_mirrors %} + @echo "Adding mirrors" + {% for name, url in source_mirrors.items() | reverse %} + $(SANDBOX) $(SPACK) mirror add --scope=site {{ name }} {{ url }} + {% endfor %} + @echo "Current mirror list:" + $(SANDBOX) $(SPACK) mirror list + {% endif %} + touch mirror-setup compilers: mirror-setup $(SANDBOX) $(MAKE) -C $@ From 69f9bb69df6fc700cc23d1976ef729059c1f74fd Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Tue, 24 Feb 2026 16:04:05 -0700 Subject: [PATCH 04/16] fixed spaces/tabs typo --- stackinator/templates/Makefile | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index 1d94c15a..ca8fb311 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -32,22 +32,22 @@ pre-install: spack-setup $(SANDBOX) $(STORE)/pre-install-hook mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} - + {% if cache %} - $(SANDBOX) $(SPACK) buildcache keys --install --trust - {% if cache.key %} - $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} - {% endif %} - {% endif %} - {% if source_mirrors %} - @echo "Adding mirrors" - {% for name, url in source_mirrors.items() | reverse %} - $(SANDBOX) $(SPACK) mirror add --scope=site {{ name }} {{ url }} - {% endfor %} - @echo "Current mirror list:" - $(SANDBOX) $(SPACK) mirror list - {% endif %} - touch mirror-setup + $(SANDBOX) $(SPACK) buildcache keys --install --trust + {% if cache.key %} + $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} + {% endif %} + {% endif %} + {% if source_mirrors %} + @echo "Adding mirrors" + {% for name, url in source_mirrors.items() | reverse %} + $(SANDBOX) $(SPACK) mirror add --scope=site {{ name }} {{ url }} + {% endfor %} + @echo "Current mirror list:" + $(SANDBOX) $(SPACK) mirror list + {% endif %} + touch mirror-setup compilers: mirror-setup $(SANDBOX) $(MAKE) -C $@ From 8b392c258eb75249eefbb89d5858f84a34d4416f Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 6 Mar 2026 09:16:51 -0700 Subject: [PATCH 05/16] Added mirror configuration json schema. --- stackinator/schema/config.json | 7 ------- stackinator/schema/mirror.json | 37 ++++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 7 deletions(-) create mode 100644 stackinator/schema/mirror.json diff --git a/stackinator/schema/config.json b/stackinator/schema/config.json index 4b91011d..d6fec3a0 100644 --- a/stackinator/schema/config.json +++ b/stackinator/schema/config.json @@ -64,13 +64,6 @@ } } }, - "source_mirrors" : { - "type" : "object", - "additionalProperties": { - "type" : "string" - }, - "default": {} - }, "modules" : { "type": "boolean" }, diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json new file mode 100644 index 00000000..b32a3cd1 --- /dev/null +++ b/stackinator/schema/mirror.json @@ -0,0 +1,37 @@ +# This config handles source mirrors, binary caches, and bootstrap mirrors (of both forms) +{ + # Order matters, so we need an array. + "type" : "array", + "items": { + "type": "object", + "required": ["name", "url"] + "properties": { + "name": { + "type": "string", + "description": "The name of this mirror. Should be follow standard variable naming syntax.", + }, + "url": { + "type": "string", + "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI).", + }, + "enabled": { + "type": "boolean", + "default": true, + "description": "Whether this mirror is enabled.", + }, + "bootstrap": { + "type": "boolean", + "default": false, + "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror.", + } + "public_key": { + "type": "string", + "description": "Public PGP key for validating binary cache packages.", + }, + "description": { + "type": "string", + "description": "What this mirror is for." + } + } + } +} From dba9698cd781953a4fe1d938f763ec303c588986 Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 6 Mar 2026 09:48:06 -0700 Subject: [PATCH 06/16] Incorporating Makefile changes. --- stackinator/templates/Makefile | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/stackinator/templates/Makefile b/stackinator/templates/Makefile index ca8fb311..d1a06dd4 100644 --- a/stackinator/templates/Makefile +++ b/stackinator/templates/Makefile @@ -32,21 +32,25 @@ pre-install: spack-setup $(SANDBOX) $(STORE)/pre-install-hook mirror-setup: spack-setup{% if pre_install_hook %} pre-install{% endif %} - {% if cache %} + # The old way of managing mirrors $(SANDBOX) $(SPACK) buildcache keys --install --trust {% if cache.key %} $(SANDBOX) $(SPACK) gpg trust {{ cache.key }} {% endif %} {% endif %} - {% if source_mirrors %} - @echo "Adding mirrors" - {% for name, url in source_mirrors.items() | reverse %} - $(SANDBOX) $(SPACK) mirror add --scope=site {{ name }} {{ url }} + {% if mirrors %} + @echo "Adding mirrors and gpg keys." + {% for mirror_info in mirrors | reverse %} + $(SANDBOX) $(SPACK) mirror add --scope=site {{ mirror_info.name }} {{ mirror_info.url }} + $(SANDBOX) $(SPACK) gpg trust {{ mirror_info.key_path }} {% endfor %} @echo "Current mirror list:" $(SANDBOX) $(SPACK) mirror list {% endif %} + {% for mirror_info in filter(lambda m: m['bootstrap'], mirrors) | filter() %} + $(SANDBOX) $(SPACK) bootstrap add --scope=site {{ mirror_info.name }} bootstrap/{{ mirror_info.name }} + {% endfor %} touch mirror-setup compilers: mirror-setup From b0a8071eb668af9712ad21c7b1e11b2f32306fd2 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 6 Mar 2026 10:05:24 -0700 Subject: [PATCH 07/16] mirrors --- stackinator/mirror.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 stackinator/mirror.py diff --git a/stackinator/mirror.py b/stackinator/mirror.py new file mode 100644 index 00000000..e69de29b From 6a044ae4ffaee02c903675edfb4f469d16ff371a Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 6 Mar 2026 10:26:59 -0700 Subject: [PATCH 08/16] mirrors --- stackinator/mirror.py | 58 +++++++++++++++++++++++++++++++++++++++++++ stackinator/recipe.py | 32 ++++++++++++++++++++++-- 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index e69de29b..83989b83 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -0,0 +1,58 @@ +import os +import pathlib + +import yaml + +from . import schema + + +def configuration_from_file(file, mount): + with file.open() as fid: + # load the raw yaml input + raw = yaml.load(fid, Loader=yaml.Loader) + + # validate the yaml + schema.CacheValidator.validate(raw) + + # verify that the root path exists + path = pathlib.Path(os.path.expandvars(raw["root"])) + if not path.is_absolute(): + raise FileNotFoundError(f"The build cache path '{path}' is not absolute") + if not path.is_dir(): + raise FileNotFoundError(f"The build cache path '{path}' does not exist") + + raw["root"] = path + + # Put the build cache in a sub-directory named after the mount point. + # This avoids relocation issues. + raw["path"] = pathlib.Path(path.as_posix() + mount.as_posix()) + + # verify that the key file exists if it was specified + key = raw["key"] + if key is not None: + key = pathlib.Path(os.path.expandvars(key)) + if not key.is_absolute(): + raise FileNotFoundError(f"The build cache key '{key}' is not absolute") + if not key.is_file(): + raise FileNotFoundError(f"The build cache key '{key}' does not exist") + raw["key"] = key + + return raw + + +def generate_mirrors_yaml(config): + path = config["path"].as_posix() + mirrors = { + "mirrors": { + "alpscache": { + "fetch": { + "url": f"file://{path}", + }, + "push": { + "url": f"file://{path}", + }, + } + } + } + + return yaml.dump(mirrors, default_flow_style=False) \ No newline at end of file diff --git a/stackinator/recipe.py b/stackinator/recipe.py index ca8d2b3d..d0ec018f 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -170,14 +170,20 @@ def __init__(self, args): self.generate_environment_specs(raw) # optional mirror configurtion + if mirrors_path.is_file():zx mirrors_path = self.path / "mirrors.yaml" - if mirrors_path.is_file(): self._logger.warning( "mirrors.yaml have been removed from recipes, use the --cache option on stack-config instead." ) raise RuntimeError("Unsupported mirrors.yaml file in recipe.") - self.mirror = (args.cache, self.mount) + # self.mirror = (args.cache, self.mount) + + # load the optional mirrors.yaml from system config: + mirrors_path = self.system_config_path / "mirrors.yaml" + if mirrors_path.is_file(): + self.mirrors = (mirrors_path, self.mount) + # update mirror setter and cache.configuration_from_file() # optional post install hook if self.post_install_hook is not None: @@ -262,6 +268,28 @@ def mirror(self, configuration): self._mirror = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount)) + @property + def mirrors(self): + return self._mirrors + + # old: self.mirror = (args.cache, self.mount) + # new: self.mirror = (mirrors_yaml_path, self.mount) + + @mirrors.setter + def (self, configuration): + self._logger.debug(f"configuring mirrors with {configuration}") + self._mirrors = None + + file, mount = configuration + + if file is not None: + mirror_config_path = pathlib.Path(file) + if not mirror_config_path.is_file(): + raise FileNotFoundError(f"The mirror configuration '{file}' is not a file") + + self._mirrors = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount)) + + @property def config(self): return self._config From 0ad5022265a1b95081202aea6204d6166649c7c6 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 6 Mar 2026 10:40:10 -0700 Subject: [PATCH 09/16] validate mirror config --- stackinator/mirror.py | 47 +++++++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 24 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 83989b83..be784864 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,5 +1,6 @@ import os import pathlib +import urllib.request import yaml @@ -14,30 +15,28 @@ def configuration_from_file(file, mount): # validate the yaml schema.CacheValidator.validate(raw) - # verify that the root path exists - path = pathlib.Path(os.path.expandvars(raw["root"])) - if not path.is_absolute(): - raise FileNotFoundError(f"The build cache path '{path}' is not absolute") - if not path.is_dir(): - raise FileNotFoundError(f"The build cache path '{path}' does not exist") - - raw["root"] = path - - # Put the build cache in a sub-directory named after the mount point. - # This avoids relocation issues. - raw["path"] = pathlib.Path(path.as_posix() + mount.as_posix()) - - # verify that the key file exists if it was specified - key = raw["key"] - if key is not None: - key = pathlib.Path(os.path.expandvars(key)) - if not key.is_absolute(): - raise FileNotFoundError(f"The build cache key '{key}' is not absolute") - if not key.is_file(): - raise FileNotFoundError(f"The build cache key '{key}' does not exist") - raw["key"] = key - - return raw + mirrors = [mirror for mirror in raw if mirror["enabled"]] + + for mirror in mirrors: + url = mirror["url"] + if url.beginswith("file://"): + # verify that the root path exists + path = pathlib.Path(os.path.expandvars(url)) + if not path.is_absolute(): + raise FileNotFoundError(f"The build cache path '{path}' is not absolute") + if not path.is_dir(): + raise FileNotFoundError(f"The build cache path '{path}' does not exist") + + mirror["url"] = path + + else: + try: + request = urllib.request.Request(url, method='HEAD') + response = urllib.request.urlopen(request) + except urllib.error.URLError as e: + print(f'Error: {e.reason}') + + return mirrors def generate_mirrors_yaml(config): From 3036013f07303966ac536b59b510aa7f0f7ec64c Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 6 Mar 2026 11:01:41 -0700 Subject: [PATCH 10/16] Updating recipe to handle new mirrors format. --- stackinator/mirror.py | 4 +-- stackinator/recipe.py | 72 ++++++++++--------------------------------- 2 files changed, 19 insertions(+), 57 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index be784864..08c303bb 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -7,7 +7,7 @@ from . import schema -def configuration_from_file(file, mount): +def configuration_from_file(file): with file.open() as fid: # load the raw yaml input raw = yaml.load(fid, Loader=yaml.Loader) @@ -54,4 +54,4 @@ def generate_mirrors_yaml(config): } } - return yaml.dump(mirrors, default_flow_style=False) \ No newline at end of file + return yaml.dump(mirrors, default_flow_style=False) diff --git a/stackinator/recipe.py b/stackinator/recipe.py index d0ec018f..281ab8b2 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -4,8 +4,9 @@ import jinja2 import yaml +from typing import Optional -from . import cache, root_logger, schema, spack_util +from . import cache, root_logger, schema, spack_util, mirror from .etc import envvars @@ -169,21 +170,11 @@ def __init__(self, args): schema.EnvironmentsValidator.validate(raw) self.generate_environment_specs(raw) - # optional mirror configurtion - if mirrors_path.is_file():zx - mirrors_path = self.path / "mirrors.yaml" - self._logger.warning( - "mirrors.yaml have been removed from recipes, use the --cache option on stack-config instead." - ) - raise RuntimeError("Unsupported mirrors.yaml file in recipe.") - - # self.mirror = (args.cache, self.mount) + mirrors_path = self.system_config_path/'mirrors.yaml' + self._logger.debug(f"opening {mirrors_path}") # load the optional mirrors.yaml from system config: - mirrors_path = self.system_config_path / "mirrors.yaml" - if mirrors_path.is_file(): - self.mirrors = (mirrors_path, self.mount) - # update mirror setter and cache.configuration_from_file() + self.mirrors = self.system_config_path / "mirrors.yaml" # optional post install hook if self.post_install_hook is not None: @@ -242,54 +233,25 @@ def pre_install_hook(self): return hook_path return None - # Returns a dictionary with the following fields - # - # root: /path/to/cache - # path: /path/to/cache/user-environment - # key: /path/to/private-pgp-key @property - def mirror(self): - return self._mirror + def mirrors(self): + return self._mirrors # configuration is a tuple with two fields: # - a Path of the yaml file containing the cache configuration # - the mount point of the image - @mirror.setter - def mirror(self, configuration): - self._logger.debug(f"configuring build cache mirror with {configuration}") - self._mirror = None - - file, mount = configuration - - if file is not None: - mirror_config_path = pathlib.Path(file) - if not mirror_config_path.is_file(): - raise FileNotFoundError(f"The cache configuration '{file}' is not a file") - - self._mirror = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount)) - - @property - def mirrors(self): - return self._mirrors - - # old: self.mirror = (args.cache, self.mount) - # new: self.mirror = (mirrors_yaml_path, self.mount) - @mirrors.setter - def (self, configuration): - self._logger.debug(f"configuring mirrors with {configuration}") + def mirrors(self, path: Optional[pathlib.Path]): + """Initialize the mirrors property from config.""" self._mirrors = None + if path is not None: + if not path.is_file(): + raise FileNotFoundError("The system config 'mirrors.yaml' file exists, but isn't a " + "readable file.") - file, mount = configuration - - if file is not None: - mirror_config_path = pathlib.Path(file) - if not mirror_config_path.is_file(): - raise FileNotFoundError(f"The mirror configuration '{file}' is not a file") - - self._mirrors = cache.configuration_from_file(mirror_config_path, pathlib.Path(mount)) - - + self._logger.debug(f"configuring mirrors from {path}") + self._mirrors = mirror.configuration_from_file(path) + @property def config(self): return self._config @@ -569,7 +531,7 @@ def compiler_files(self): ) makefile_template = env.get_template("Makefile.compilers") - push_to_cache = self.mirror is not None + push_to_cache = self.mirrors files["makefile"] = makefile_template.render( compilers=self.compilers, push_to_cache=push_to_cache, From a1c486d7c9d6801533f544f0b6780e474392c3aa Mon Sep 17 00:00:00 2001 From: Paul Ferrell Date: Fri, 6 Mar 2026 11:35:42 -0700 Subject: [PATCH 11/16] Updating mirror configuration more. --- stackinator/main.py | 16 ++++--- stackinator/mirror.py | 79 +++++++++++++++++++++++----------- stackinator/recipe.py | 27 +++--------- stackinator/schema/mirror.json | 7 ++- 4 files changed, 78 insertions(+), 51 deletions(-) diff --git a/stackinator/main.py b/stackinator/main.py index 44406215..6f30ddfc 100644 --- a/stackinator/main.py +++ b/stackinator/main.py @@ -81,13 +81,19 @@ def log_header(args): def make_argparser(): parser = argparse.ArgumentParser(description=("Generate a build configuration for a spack stack from a recipe.")) parser.add_argument("--version", action="version", version=f"stackinator version {VERSION}") - parser.add_argument("-b", "--build", required=True, type=str) + parser.add_argument("-b", "--build", required=True, type=str, + help="Where to set up the stackinator build directory. " + "('/tmp' is not allowed, use '/var/tmp'") parser.add_argument("--no-bwrap", action="store_true", required=False) - parser.add_argument("-r", "--recipe", required=True, type=str) - parser.add_argument("-s", "--system", required=True, type=str) + parser.add_argument("-r", "--recipe", required=True, type=str, + help="Name of (and/or path to) the Stackinator recipe.") + parser.add_argument("-s", "--system", required=True, type=str, + help="Name of (and/or path to) the Stackinator system configuration.") parser.add_argument("-d", "--debug", action="store_true") - parser.add_argument("-m", "--mount", required=False, type=str) - parser.add_argument("-c", "--cache", required=False, type=str) + parser.add_argument("-m", "--mount", required=False, type=str, + help="The mount point where the environment will be located.") + parser.add_argument("-c", "--cache", required=False, type=str, + help="Buildcache location or name (from system config's mirrors.yaml).") parser.add_argument("--develop", action="store_true", required=False) return parser diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 08c303bb..40487f48 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,42 +1,73 @@ import os import pathlib import urllib.request +from typing import Optional import yaml from . import schema -def configuration_from_file(file): - with file.open() as fid: - # load the raw yaml input - raw = yaml.load(fid, Loader=yaml.Loader) +def configuration_from_file(path: pathlib.Path, cmdline_cache: Optional[str] = None): + """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" + + if path.exists(): + with path.open() as fid: + # load the raw yaml input + raw = yaml.load(fid, Loader=yaml.Loader) + + print(f"Configuring mirrors and buildcache from '{path}'") # validate the yaml schema.CacheValidator.validate(raw) mirrors = [mirror for mirror in raw if mirror["enabled"]] + else: + mirrors = [] + + buildcache_dest_count = len([mirror for mirror in mirrors if mirror['buildcache']]) + if buildcache_dest_count > 1: + raise RuntimeError("Mirror config has more than one mirror specified as the build cache destination " + "in the system config's 'mirrors.yaml'.") + elif buildcache_dest_count == 1 and cmdline_cache: + raise RuntimeError("Build cache destination specified on the command line and in the system config's " + "'mirrors.yaml'. It can be one or the other, but not both.") + + # Add or set the cache given on the command line as the buildcache destination + if cmdline_cache is not None: + existing_mirror = [mirror for mirror in mirrors if mirror['name'] == cmdline_cache][:1] + # If the mirror name given on the command line isn't in the config, assume it + # is the URL to a build cache. + if not existing_mirror: + mirrors.append( + { + 'name': 'cmdline_cache', + 'url': cmdline_cache, + 'buildcache': True, + 'bootstrap': False, + } + ) + + for mirror in mirrors: + url = mirror["url"] + if url.beginswith("file://"): + # verify that the root path exists + path = pathlib.Path(os.path.expandvars(url)) + if not path.is_absolute(): + raise FileNotFoundError(f"The build cache path '{path}' is not absolute") + if not path.is_dir(): + raise FileNotFoundError(f"The build cache path '{path}' does not exist") + + mirror["url"] = path + + else: + try: + request = urllib.request.Request(url, method='HEAD') + response = urllib.request.urlopen(request) + except urllib.error.URLError as e: + print(f'Error: {e.reason}') - for mirror in mirrors: - url = mirror["url"] - if url.beginswith("file://"): - # verify that the root path exists - path = pathlib.Path(os.path.expandvars(url)) - if not path.is_absolute(): - raise FileNotFoundError(f"The build cache path '{path}' is not absolute") - if not path.is_dir(): - raise FileNotFoundError(f"The build cache path '{path}' does not exist") - - mirror["url"] = path - - else: - try: - request = urllib.request.Request(url, method='HEAD') - response = urllib.request.urlopen(request) - except urllib.error.URLError as e: - print(f'Error: {e.reason}') - - return mirrors + return mirrors def generate_mirrors_yaml(config): diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 281ab8b2..4b7c038c 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -170,11 +170,11 @@ def __init__(self, args): schema.EnvironmentsValidator.validate(raw) self.generate_environment_specs(raw) - mirrors_path = self.system_config_path/'mirrors.yaml' - self._logger.debug(f"opening {mirrors_path}") - - # load the optional mirrors.yaml from system config: - self.mirrors = self.system_config_path / "mirrors.yaml" + # load the optional mirrors.yaml from system config, and add any additional + # mirrors specified on the command line. + self._mirrors = None + self._logger.debug("Configuring mirrors.") + self._mirrors = mirror.configuration_from_file(self.system_config_path/"mirrors.yaml", args.cache) # optional post install hook if self.post_install_hook is not None: @@ -236,22 +236,7 @@ def pre_install_hook(self): @property def mirrors(self): return self._mirrors - - # configuration is a tuple with two fields: - # - a Path of the yaml file containing the cache configuration - # - the mount point of the image - @mirrors.setter - def mirrors(self, path: Optional[pathlib.Path]): - """Initialize the mirrors property from config.""" - self._mirrors = None - if path is not None: - if not path.is_file(): - raise FileNotFoundError("The system config 'mirrors.yaml' file exists, but isn't a " - "readable file.") - - self._logger.debug(f"configuring mirrors from {path}") - self._mirrors = mirror.configuration_from_file(path) - + @property def config(self): return self._config diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index b32a3cd1..a53cc34e 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -23,7 +23,12 @@ "type": "boolean", "default": false, "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror.", - } + }, + "buildcache": { + "type": "boolean", + "default": false, + "description": "Use this mirror as the buildcache push destination. Can only be enabled on a single mirror." + }, "public_key": { "type": "string", "description": "Public PGP key for validating binary cache packages.", From 2b606830c4daf0305b19fe7587766cd4c034353b Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 6 Mar 2026 11:25:37 -0700 Subject: [PATCH 12/16] mirror yaml generator --- stackinator/mirror.py | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 40487f48..315d2e8e 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -70,19 +70,16 @@ def configuration_from_file(path: pathlib.Path, cmdline_cache: Optional[str] = N return mirrors -def generate_mirrors_yaml(config): - path = config["path"].as_posix() - mirrors = { - "mirrors": { - "alpscache": { - "fetch": { - "url": f"file://{path}", - }, - "push": { - "url": f"file://{path}", - }, - } +def generate_mirrors_yaml(mirrors): + yaml = {"mirrors": {}} + + for m in mirrors: + name = m["name"] + url = m["url"] + + yaml["mirrors"][name] = { + "fetch": {"url": url}, + "push": {"url": url}, } - } - return yaml.dump(mirrors, default_flow_style=False) + return yaml.dump(yaml, default_flow_style=False) \ No newline at end of file From 095862c37402394424991963f83ed7dbe6bca896 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Fri, 6 Mar 2026 11:37:34 -0700 Subject: [PATCH 13/16] update mirrors --- stackinator/builder.py | 10 ++++------ stackinator/mirror.py | 15 ++++++++++++++- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index c3de44db..5b223ffe 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -226,14 +226,12 @@ def generate(self, recipe): with (self.path / "Makefile").open("w") as f: f.write( makefile_template.render( - cache=recipe.mirror, modules=recipe.with_modules, post_install_hook=recipe.post_install_hook, pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, - # pass source_mirrors to Makefile render - source_mirrors=recipe.config.get("source_mirrors", {}), + mirrors=recipe.mirrors exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) @@ -314,11 +312,11 @@ def generate(self, recipe): fid.write(global_packages_yaml) # generate a mirrors.yaml file if build caches have been configured - if recipe.mirror: + if recipe.mirrors: dst = config_path / "mirrors.yaml" - self._logger.debug(f"generate the build cache mirror: {dst}") + self._logger.debug(f"generate the spack mirrors.yaml: {dst}") with dst.open("w") as fid: - fid.write(cache.generate_mirrors_yaml(recipe.mirror)) + fid.write(cache.generate_mirrors_yaml(recipe.mirrors)) # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to diff --git a/stackinator/mirror.py b/stackinator/mirror.py index 315d2e8e..e802018a 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -67,7 +67,20 @@ def configuration_from_file(path: pathlib.Path, cmdline_cache: Optional[str] = N except urllib.error.URLError as e: print(f'Error: {e.reason}') - return mirrors + if mirror["key"]: + #if path, check if exists + path = pathlib.Path(os.path.expandvars(mirror["key"])) + if path.exists(): + if not path.is_file(): + raise FileNotFoundError(f"The key path '{path}' is not a file") + else + #if key, save to file, change to path + + if mirror["bootstrap"]: + #make bootstrap dirs + #bootstrap//metadata.yaml + + return mirrors def generate_mirrors_yaml(mirrors): From 31a16f1d6550a545c625d779afdacc138cfa1e68 Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Sat, 7 Mar 2026 00:54:59 -0700 Subject: [PATCH 14/16] validate keys in mirror config and fixed yaml generator --- stackinator/mirror.py | 74 ++++++++++++++++++++++++++++++++----------- 1 file changed, 55 insertions(+), 19 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index e802018a..afbacafb 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -2,15 +2,21 @@ import pathlib import urllib.request from typing import Optional +import magic import yaml from . import schema +class MirrorConfigError(RuntimeError): + """Exception class for errors thrown by mirror configuration problems.""" -def configuration_from_file(path: pathlib.Path, cmdline_cache: Optional[str] = None): + + +def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" + path = system_config_root/"mirrors.yaml" if path.exists(): with path.open() as fid: # load the raw yaml input @@ -54,36 +60,33 @@ def configuration_from_file(path: pathlib.Path, cmdline_cache: Optional[str] = N # verify that the root path exists path = pathlib.Path(os.path.expandvars(url)) if not path.is_absolute(): - raise FileNotFoundError(f"The build cache path '{path}' is not absolute") + raise FileNotFoundError(f"The mirror path '{path}' is not absolute") if not path.is_dir(): - raise FileNotFoundError(f"The build cache path '{path}' does not exist") + raise FileNotFoundError(f"The mirror path '{path}' does not exist") mirror["url"] = path - else: + elif url.beginswith("https://"): try: request = urllib.request.Request(url, method='HEAD') response = urllib.request.urlopen(request) except urllib.error.URLError as e: - print(f'Error: {e.reason}') - - if mirror["key"]: - #if path, check if exists - path = pathlib.Path(os.path.expandvars(mirror["key"])) - if path.exists(): - if not path.is_file(): - raise FileNotFoundError(f"The key path '{path}' is not a file") - else - #if key, save to file, change to path + raise MirrorConfigError( + f"Could not reach the mirror url '{url}'. " + f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") - if mirror["bootstrap"]: - #make bootstrap dirs - #bootstrap//metadata.yaml + if mirror["bootstrap"]: + #make bootstrap dirs + #bootstrap//metadata.yaml return mirrors -def generate_mirrors_yaml(mirrors): +def setup(mirrors, config_path): + dst = config_path / "mirrors.yaml" + self._logger.debug(f"generate the spack mirrors.yaml: {dst}") + with dst.open("w") as fid: + fid.write() yaml = {"mirrors": {}} for m in mirrors: @@ -95,4 +98,37 @@ def generate_mirrors_yaml(mirrors): "push": {"url": url}, } - return yaml.dump(yaml, default_flow_style=False) \ No newline at end of file + return yaml.dump(yaml, default_flow_style=False) + +#called from builder +def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: pathlib.Path): + for mirror in mirrors: + if mirror["key"]: + key = mirror["key"] + # if path, check if abs path, if not, append sys config path in front and check again + path = pathlib.Path(os.path.expandvars(key)) + if path.exists(): + if not path.is_absolute(): + #try prepending system config path + path = system_config_path + path + if not.path.is_file() + raise FileNotFoundError( + f"The key path '{path}' is not a file. " + f"Check the key listed in mirrors.yaml in system config.") + file_type = magic.from_file(path) + if not file_type.startswith("OpenPGP Public Key"): + raise MirrorConfigError( + f"'{key}' is not a valid GPG key. " + f"Check the key listed in mirrors.yaml in system config.") + # copy file to key store + with file open: + data = key.read + dest = mkdir(new_key_file) + dest.write(data) + # mirror["key"] = new_path + + else: + # if PGP key, convert to binary, ???, convert back + # if key, save to file, change to path + + \ No newline at end of file From 9ef113dca30631628a59da1f3e814e230579de3e Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Sat, 7 Mar 2026 00:57:53 -0700 Subject: [PATCH 15/16] validate keys in mirror config and fixed yaml generator --- stackinator/mirror.py | 44 +++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/stackinator/mirror.py b/stackinator/mirror.py index afbacafb..e521b597 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -12,7 +12,6 @@ class MirrorConfigError(RuntimeError): """Exception class for errors thrown by mirror configuration problems.""" - def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Optional[str] = None): """Configure mirrors from both the system 'mirror.yaml' file and the command line.""" @@ -62,7 +61,7 @@ def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Opt if not path.is_absolute(): raise FileNotFoundError(f"The mirror path '{path}' is not absolute") if not path.is_dir(): - raise FileNotFoundError(f"The mirror path '{path}' does not exist") + raise FileNotFoundError(f"The mirror path '{path}' is not a directory") mirror["url"] = path @@ -82,11 +81,12 @@ def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Opt return mirrors -def setup(mirrors, config_path): +def yaml_setup(mirrors, config_path): + """Generate the mirrors.yaml for spack""" + dst = config_path / "mirrors.yaml" + self._logger.debug(f"generate the spack mirrors.yaml: {dst}") - with dst.open("w") as fid: - fid.write() yaml = {"mirrors": {}} for m in mirrors: @@ -98,13 +98,22 @@ def setup(mirrors, config_path): "push": {"url": url}, } - return yaml.dump(yaml, default_flow_style=False) + with dst.open("w") as file: + yaml.dump(yaml, default_flow_style=False) + + # return dst + -#called from builder def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: pathlib.Path): + """Validate mirror keys, relocate to key_store, and update mirror config with new key paths""" + for mirror in mirrors: if mirror["key"]: key = mirror["key"] + + # key will be saved under key_store/mirror_name.gpg + dst = (key_store / f"'{mirror["name"]}'.gpg").resolve() + # if path, check if abs path, if not, append sys config path in front and check again path = pathlib.Path(os.path.expandvars(key)) if path.exists(): @@ -115,20 +124,23 @@ def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: raise FileNotFoundError( f"The key path '{path}' is not a file. " f"Check the key listed in mirrors.yaml in system config.") + file_type = magic.from_file(path) + if not file_type.startswith("OpenPGP Public Key"): raise MirrorConfigError( - f"'{key}' is not a valid GPG key. " + f"'{path}' is not a valid GPG key. " f"Check the key listed in mirrors.yaml in system config.") - # copy file to key store - with file open: - data = key.read - dest = mkdir(new_key_file) - dest.write(data) - # mirror["key"] = new_path + + # copy key to new destination in key store + with open(path, 'r') as reader, open(dst, 'w') as writer: + data = reader.read() + writer.write(data) else: # if PGP key, convert to binary, ???, convert back - # if key, save to file, change to path + with open(dst, "w") as file: + file.write(key) - \ No newline at end of file + # update mirror with new path + mirror["key"] = dst \ No newline at end of file From b9fe48e4af23f682a3360ce50300a8ac1cc8f4ea Mon Sep 17 00:00:00 2001 From: grodzki-lanl Date: Mon, 9 Mar 2026 14:33:15 -0600 Subject: [PATCH 16/16] connecting mirrors to builder.py --- stackinator/builder.py | 9 +++++---- stackinator/mirror.py | 9 ++++----- stackinator/recipe.py | 2 +- stackinator/schema/mirror.json | 16 +++++++--------- 4 files changed, 17 insertions(+), 19 deletions(-) diff --git a/stackinator/builder.py b/stackinator/builder.py index 5b223ffe..7918cb24 100644 --- a/stackinator/builder.py +++ b/stackinator/builder.py @@ -11,7 +11,7 @@ import jinja2 import yaml -from . import VERSION, cache, root_logger, spack_util +from . import VERSION, cache, root_logger, spack_util, mirror def install(src, dst, *, ignore=None, symlinks=False): @@ -231,7 +231,7 @@ def generate(self, recipe): pre_install_hook=recipe.pre_install_hook, spack_version=spack_version, spack_meta=spack_meta, - mirrors=recipe.mirrors + mirrors=recipe.mirrors, exclude_from_cache=["nvhpc", "cuda", "perl"], verbose=False, ) @@ -312,11 +312,12 @@ def generate(self, recipe): fid.write(global_packages_yaml) # generate a mirrors.yaml file if build caches have been configured + key_store = self.path / ".gnupg" if recipe.mirrors: + mirror.key_setup(recipe.mirrors, config_path, key_store) dst = config_path / "mirrors.yaml" self._logger.debug(f"generate the spack mirrors.yaml: {dst}") - with dst.open("w") as fid: - fid.write(cache.generate_mirrors_yaml(recipe.mirrors)) + mirror.spack_yaml_setup(recipe.mirrors, dst) # Add custom spack package recipes, configured via Spack repos. # Step 1: copy Spack repos to store_path where they will be used to diff --git a/stackinator/mirror.py b/stackinator/mirror.py index e521b597..0e5ee5d1 100644 --- a/stackinator/mirror.py +++ b/stackinator/mirror.py @@ -1,7 +1,7 @@ import os import pathlib import urllib.request -from typing import Optional +from typing import Optional, List, Dict import magic import yaml @@ -74,19 +74,18 @@ def configuration_from_file(system_config_root: pathlib.Path, cmdline_cache: Opt f"Could not reach the mirror url '{url}'. " f"Check the url listed in mirrors.yaml in system config. \n{e.reason}") - if mirror["bootstrap"]: + #if mirror["bootstrap"]: #make bootstrap dirs #bootstrap//metadata.yaml return mirrors -def yaml_setup(mirrors, config_path): +def spack_yaml_setup(mirrors, config_path): """Generate the mirrors.yaml for spack""" dst = config_path / "mirrors.yaml" - self._logger.debug(f"generate the spack mirrors.yaml: {dst}") yaml = {"mirrors": {}} for m in mirrors: @@ -120,7 +119,7 @@ def key_setup(mirrors: List[Dict], system_config_path: pathlib.Path, key_store: if not path.is_absolute(): #try prepending system config path path = system_config_path + path - if not.path.is_file() + if not path.is_file(): raise FileNotFoundError( f"The key path '{path}' is not a file. " f"Check the key listed in mirrors.yaml in system config.") diff --git a/stackinator/recipe.py b/stackinator/recipe.py index 4b7c038c..c59790b2 100644 --- a/stackinator/recipe.py +++ b/stackinator/recipe.py @@ -174,7 +174,7 @@ def __init__(self, args): # mirrors specified on the command line. self._mirrors = None self._logger.debug("Configuring mirrors.") - self._mirrors = mirror.configuration_from_file(self.system_config_path/"mirrors.yaml", args.cache) + self._mirrors = mirror.configuration_from_file(self.system_config_path, args.cache) # optional post install hook if self.post_install_hook is not None: diff --git a/stackinator/schema/mirror.json b/stackinator/schema/mirror.json index a53cc34e..a8be6ab3 100644 --- a/stackinator/schema/mirror.json +++ b/stackinator/schema/mirror.json @@ -1,28 +1,26 @@ -# This config handles source mirrors, binary caches, and bootstrap mirrors (of both forms) { - # Order matters, so we need an array. "type" : "array", "items": { "type": "object", - "required": ["name", "url"] + "required": ["name", "url"], "properties": { "name": { "type": "string", - "description": "The name of this mirror. Should be follow standard variable naming syntax.", + "description": "The name of this mirror. Should be follow standard variable naming syntax." }, "url": { "type": "string", - "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI).", + "description": "URL to the mirror. Can be a simple path, or any protocol Spack supports (https, OCI)." }, "enabled": { "type": "boolean", "default": true, - "description": "Whether this mirror is enabled.", + "description": "Whether this mirror is enabled." }, "bootstrap": { "type": "boolean", "default": false, - "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror.", + "description": "Whether to use as a mirror for bootstrapping. Will also use as a regular mirror." }, "buildcache": { "type": "boolean", @@ -31,7 +29,7 @@ }, "public_key": { "type": "string", - "description": "Public PGP key for validating binary cache packages.", + "description": "Public PGP key for validating binary cache packages." }, "description": { "type": "string", @@ -39,4 +37,4 @@ } } } -} +} \ No newline at end of file