Merge pull request #1452 from datajoint/fix/store-secrets-arbitrary-attr

feat(config): DJ_STORES env var + DJ_IGNORE_CONFIG_FILE flag + arbitrary .secrets/ attrs — 2.3

Author: dimitri-yatsenko

src/datajoint/settings.py

@@ -335,17 +335,32 @@ class Config(BaseSettings):
     jobs: JobsSettings = Field(default_factory=JobsSettings)
 
     # Unified stores configuration (replaces external and object_storage)
+    # ``validation_alias`` redirects pydantic-settings' env source away from the
+    # natural ``DJ_STORES`` so it doesn't auto-parse on Config() construction.
+    # ``DJ_STORES`` is handled by ``_apply_stores_env`` after the config file
+    # load so env-var precedence is honored. *New in 2.3.*
     stores: dict[str, Any] = Field(
         default_factory=dict,
+        validation_alias="_DJ_STORES_PYDANTIC_DISABLED",
         description="Unified object storage configuration. "
         "Use stores.default to designate default store. "
-        "Configure named stores as stores.<name>.protocol, stores.<name>.location, etc.",
+        "Configure named stores as stores.<name>.protocol, stores.<name>.location, etc. "
+        "Set via DJ_STORES (JSON object) or in datajoint.json. *New in 2.3* for "
+        "DJ_STORES env-var support.",
     )
 
     # Top-level settings
     loglevel: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = Field(default="INFO", validation_alias="DJ_LOG_LEVEL")
     safemode: bool = True
 
+    ignore_config_file: bool = Field(
+        default=False,
+        validation_alias="DJ_IGNORE_CONFIG_FILE",
+        description="If True, skip loading datajoint.json and the secrets directory. "
+        "Intended for env-var-only deployments (e.g. the DataJoint platform). "
+        "*New in 2.3.*",
+    )
+
     # Cache path for query results
     query_cache: Path | None = None
 
@@ -691,26 +706,50 @@ def _load_secrets(self, secrets_dir: Path) -> None:
             self.database.password = db_password
             logger.debug(f"Loaded database.password from {secrets_dir}")
 
-        # Load per-store secrets (stores.<name>.access_key, stores.<name>.secret_key)
-        # Iterate through all files in secrets directory
+        # Load per-store secrets from any stores.<name>.<attr> file.
+        # The attr name is recorded as-is on stores.<name>; this lets
+        # plugin-registered adapters define their own secret fields
+        # (e.g. a Bearer ``token`` for HTTP-based protocols) without
+        # forcing AWS-style ``access_key`` / ``secret_key`` naming.
         if secrets_dir.is_dir():
             for secret_file in secrets_dir.iterdir():
                 if not secret_file.is_file() or secret_file.name.startswith("."):
                     continue
 
                 parts = secret_file.name.split(".")
-                # Check for stores.<name>.access_key or stores.<name>.secret_key pattern
                 if len(parts) == 3 and parts[0] == "stores":
                     store_name, attr = parts[1], parts[2]
-                    if attr in ("access_key", "secret_key"):
-                        value = secret_file.read_text().strip()
-                        # Initialize store dict if needed
-                        if store_name not in self.stores:
-                            self.stores[store_name] = {}
-                        # Only set if not already present
-                        if attr not in self.stores[store_name]:
-                            self.stores[store_name][attr] = value
-                            logger.debug(f"Loaded stores.{store_name}.{attr} from {secrets_dir}")
+                    value = secret_file.read_text().strip()
+                    # Initialize store dict if needed
+                    if store_name not in self.stores:
+                        self.stores[store_name] = {}
+                    # Only set if not already present (config / env vars win)
+                    if attr not in self.stores[store_name]:
+                        self.stores[store_name][attr] = value
+                        logger.debug(f"Loaded stores.{store_name}.{attr} from {secrets_dir}")
+
+    def _apply_stores_env(self) -> None:
+        """Replace ``self.stores`` from the ``DJ_STORES`` env var if set.
+
+        ``DJ_STORES`` holds a JSON object in the same shape as the ``stores``
+        block of ``datajoint.json``. This lets env-var-only deployments
+        configure plugin-registered storage adapters with arbitrary attr
+        names (e.g. a Bearer ``token`` field) without negotiating an env-var
+        naming scheme per attr.
+
+        *New in 2.3.*
+        """
+        raw = os.environ.get("DJ_STORES")
+        if not raw:
+            return
+        try:
+            data = json.loads(raw)
+        except json.JSONDecodeError as e:
+            raise ValueError(f"DJ_STORES contains invalid JSON: {e}") from e
+        if not isinstance(data, dict):
+            raise ValueError(f"DJ_STORES must be a JSON object, got {type(data).__name__}")
+        self.stores = data
+        logger.debug("Loaded stores from DJ_STORES env var")
 
     @contextmanager
     def override(self, **kwargs: Any) -> Iterator["Config"]:
@@ -785,9 +824,13 @@ def save_template(
 
         Credentials should NOT be stored in datajoint.json. Instead, use either:
 
-        - Environment variables (``DJ_USER``, ``DJ_PASS``, ``DJ_HOST``, etc.)
+        - Environment variables (``DJ_USER``, ``DJ_PASS``, ``DJ_HOST``,
+          ``DJ_STORES`` for JSON-encoded store configs, etc.)
         - The ``.secrets/`` directory (created alongside datajoint.json)
 
+        Set ``DJ_IGNORE_CONFIG_FILE=true`` to skip both ``datajoint.json`` and
+        the secrets directory entirely (env-var-only configuration).
+
         Parameters
         ----------
         path : str or Path, optional
@@ -962,25 +1005,29 @@ def _create_config() -> Config:
     """Create and initialize the global config instance."""
     cfg = Config()
 
-    # Find config file (recursive parent search)
-    config_path = find_config_file()
+    config_path: Path | None = None
+    if not cfg.ignore_config_file:
+        config_path = find_config_file()
+        if config_path is not None:
+            try:
+                cfg.load(config_path)
+            except Exception as e:
+                warnings.warn(f"Failed to load config from {config_path}: {e}")
+        else:
+            warnings.warn(
+                f"No {CONFIG_FILENAME} found. Using defaults and environment variables. "
+                f"Run `dj.config.save_template()` to create a template configuration.",
+                stacklevel=2,
+            )
 
-    if config_path is not None:
-        try:
-            cfg.load(config_path)
-        except Exception as e:
-            warnings.warn(f"Failed to load config from {config_path}: {e}")
-    else:
-        warnings.warn(
-            f"No {CONFIG_FILENAME} found. Using defaults and environment variables. "
-            f"Run `dj.config.save_template()` to create a template configuration.",
-            stacklevel=2,
-        )
+    # DJ_STORES (if set) overrides the stores dict from the config file
+    cfg._apply_stores_env()
 
-    # Find and load secrets
-    secrets_dir = find_secrets_dir(config_path)
-    if secrets_dir is not None:
-        cfg._load_secrets(secrets_dir)
+    # Secrets fill missing attrs in whatever ended up in self.stores
+    if not cfg.ignore_config_file:
+        secrets_dir = find_secrets_dir(config_path)
+        if secrets_dir is not None:
+            cfg._load_secrets(secrets_dir)
 
     # Set initial log level
     logger.setLevel(cfg.loglevel)

src/datajoint/storage_adapter.py

@@ -86,16 +86,9 @@ def get_storage_adapter(protocol: str) -> StorageAdapter | None:
 
 def _discover_adapters() -> None:
     """Load storage adapters from datajoint.storage entry points."""
-    try:
-        from importlib.metadata import entry_points
-    except ImportError:
-        logger.debug("importlib.metadata not available, skipping adapter discovery")
-        return
-
-    try:
-        eps = entry_points(group="datajoint.storage")
-    except TypeError:
-        eps = entry_points().get("datajoint.storage", [])
+    from importlib.metadata import entry_points
+
+    eps = entry_points(group="datajoint.storage")
 
     for ep in eps:
         if ep.name in _adapter_registry:

tests/unit/test_settings.py

@@ -487,6 +487,125 @@ def test_secrets_do_not_override_existing(self, tmp_path):
         finally:
             cfg.stores = original_stores
 
+    def test_load_store_arbitrary_attr(self, tmp_path):
+        """Plugin-registered adapters can use arbitrary secret-field names."""
+        # e.g. an HTTP-based protocol that authenticates with a Bearer token
+        secrets_dir = tmp_path / SECRETS_DIRNAME
+        secrets_dir.mkdir()
+        (secrets_dir / "stores.bearer_store.token").write_text("dapibdfXXXX")
+        (secrets_dir / "stores.bearer_store.api_key").write_text("ak_yyy")
+
+        cfg = settings.Config()
+        original_stores = cfg.stores.copy()
+        try:
+            cfg._load_secrets(secrets_dir)
+
+            assert cfg.stores["bearer_store"]["token"] == "dapibdfXXXX"
+            assert cfg.stores["bearer_store"]["api_key"] == "ak_yyy"
+        finally:
+            cfg.stores = original_stores
+
+
+class TestStoreEnv:
+    """Test DJ_STORES env var and DJ_IGNORE_CONFIG_FILE flag."""
+
+    def _isolate_filesystem(self, monkeypatch, tmp_path):
+        """chdir into a tmp_path with a .git sentinel so find_config_file stops there."""
+        (tmp_path / ".git").mkdir()
+        monkeypatch.chdir(tmp_path)
+        # Defend against a /run/secrets/datajoint/ on the host
+        monkeypatch.setattr(settings, "SYSTEM_SECRETS_DIR", tmp_path / "nonexistent-system-secrets")
+
+    def test_dj_stores_sets_stores_dict(self, monkeypatch, tmp_path):
+        self._isolate_filesystem(monkeypatch, tmp_path)
+        monkeypatch.setenv(
+            "DJ_STORES",
+            '{"uc":{"protocol":"http","token":"dapibd","workspace_url":"https://x"}}',
+        )
+
+        with pytest.warns(UserWarning):  # "No datajoint.json found"
+            cfg = settings._create_config()
+
+        assert cfg.stores["uc"]["protocol"] == "http"
+        assert cfg.stores["uc"]["token"] == "dapibd"
+        assert cfg.stores["uc"]["workspace_url"] == "https://x"
+
+    def test_dj_stores_overrides_config_file(self, monkeypatch, tmp_path):
+        self._isolate_filesystem(monkeypatch, tmp_path)
+        (tmp_path / CONFIG_FILENAME).write_text('{"stores": {"main": {"protocol": "s3", "location": "from-file"}}}')
+        monkeypatch.setenv(
+            "DJ_STORES",
+            '{"main": {"protocol": "http", "location": "from-env"}}',
+        )
+
+        cfg = settings._create_config()
+
+        assert cfg.stores["main"]["protocol"] == "http"
+        assert cfg.stores["main"]["location"] == "from-env"
+
+    def test_dj_stores_invalid_json_raises(self, monkeypatch, tmp_path):
+        self._isolate_filesystem(monkeypatch, tmp_path)
+        monkeypatch.setenv("DJ_STORES", "{not json")
+        with pytest.raises(ValueError, match="DJ_STORES.*invalid JSON"):
+            settings._create_config()
+
+    def test_dj_stores_non_object_raises(self, monkeypatch, tmp_path):
+        self._isolate_filesystem(monkeypatch, tmp_path)
+        monkeypatch.setenv("DJ_STORES", '["a", "b"]')
+        with pytest.raises(ValueError, match="DJ_STORES must be a JSON object"):
+            settings._create_config()
+
+    def test_dj_stores_plus_secrets_dir(self, monkeypatch, tmp_path):
+        """Secrets dir fills attrs that DJ_STORES omits."""
+        self._isolate_filesystem(monkeypatch, tmp_path)
+        # config file lets find_secrets_dir locate .secrets/ next to it
+        (tmp_path / CONFIG_FILENAME).write_text("{}")
+        secrets_dir = tmp_path / SECRETS_DIRNAME
+        secrets_dir.mkdir()
+        (secrets_dir / "stores.uc.token").write_text("from-secrets")
+        monkeypatch.setenv("DJ_STORES", '{"uc": {"protocol": "http"}}')
+
+        cfg = settings._create_config()
+
+        assert cfg.stores["uc"]["protocol"] == "http"
+        assert cfg.stores["uc"]["token"] == "from-secrets"
+
+    def test_ignore_config_file_skips_json(self, monkeypatch, tmp_path):
+        self._isolate_filesystem(monkeypatch, tmp_path)
+        (tmp_path / CONFIG_FILENAME).write_text('{"database": {"host": "should-not-load"}}')
+        monkeypatch.setenv("DJ_IGNORE_CONFIG_FILE", "true")
+
+        cfg = settings._create_config()
+
+        assert cfg.database.host == "localhost"
+
+    def test_ignore_config_file_skips_secrets(self, monkeypatch, tmp_path):
+        self._isolate_filesystem(monkeypatch, tmp_path)
+        # Place secrets where find_secrets_dir would find them if not ignored
+        monkeypatch.setattr(settings, "SYSTEM_SECRETS_DIR", tmp_path / SECRETS_DIRNAME)
+        secrets_dir = tmp_path / SECRETS_DIRNAME
+        secrets_dir.mkdir()
+        (secrets_dir / "database.password").write_text("should-not-load")
+        monkeypatch.setenv("DJ_IGNORE_CONFIG_FILE", "true")
+
+        cfg = settings._create_config()
+
+        assert cfg.database.password is None
+
+    def test_ignore_config_file_default_loads_both(self, monkeypatch, tmp_path):
+        """Default (env unset) preserves today's behavior."""
+        self._isolate_filesystem(monkeypatch, tmp_path)
+        (tmp_path / CONFIG_FILENAME).write_text('{"database": {"host": "from-file"}}')
+        secrets_dir = tmp_path / SECRETS_DIRNAME
+        secrets_dir.mkdir()
+        (secrets_dir / "database.user").write_text("dbuser")
+        monkeypatch.delenv("DJ_IGNORE_CONFIG_FILE", raising=False)
+
+        cfg = settings._create_config()
+
+        assert cfg.database.host == "from-file"
+        assert cfg.database.user == "dbuser"
+
 
 class TestDisplaySettings:
     """Test display-related settings."""