Fix config precedence: environment variables now override config files

Following the 12-Factor App methodology, environment variables now take
precedence over config file values. This is the standard DevOps practice
for deployments where secrets and environment-specific settings should
be injected via environment variables (Docker, Kubernetes, CI/CD).

Priority order (highest to lowest):
1. Environment variables (DJ_*)
2. Secrets files (.secrets/)
3. Config file (datajoint.json)
4. Defaults

Added ENV_VAR_MAPPING to track which settings have env var overrides.
The _update_from_flat_dict() method now skips file values when the
corresponding env var is set.

Added test_env_var_overrides_config_file to verify the new behavior.

Bump version to 2.0.0a3.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: dimitri-yatsenko

src/datajoint/settings.py

@@ -28,6 +28,7 @@
 
 import json
 import logging
+import os
 import warnings
 from contextlib import contextmanager
 from copy import deepcopy
@@ -45,6 +46,18 @@
 SYSTEM_SECRETS_DIR = Path("/run/secrets/datajoint")
 DEFAULT_SUBFOLDING = (2, 2)
 
+# Mapping of config keys to environment variables
+# Environment variables take precedence over config file values
+ENV_VAR_MAPPING = {
+    "database.host": "DJ_HOST",
+    "database.user": "DJ_USER",
+    "database.password": "DJ_PASS",
+    "database.port": "DJ_PORT",
+    "external.aws_access_key_id": "DJ_AWS_ACCESS_KEY_ID",
+    "external.aws_secret_access_key": "DJ_AWS_SECRET_ACCESS_KEY",
+    "loglevel": "DJ_LOG_LEVEL",
+}
+
 Role = Enum("Role", "manual lookup imported computed job")
 role_to_prefix = {
     Role.manual: "",
@@ -541,26 +554,47 @@ def load(self, filename: str | Path) -> None:
         self._config_path = filepath
 
     def _update_from_flat_dict(self, data: dict[str, Any]) -> None:
-        """Update settings from a dict (flat dot-notation or nested)."""
+        """
+        Update settings from a dict (flat dot-notation or nested).
+
+        Environment variables take precedence over config file values.
+        If an env var is set for a setting, the file value is skipped.
+        """
         for key, value in data.items():
             # Handle nested dicts by recursively updating
             if isinstance(value, dict) and hasattr(self, key):
                 group_obj = getattr(self, key)
                 for nested_key, nested_value in value.items():
                     if hasattr(group_obj, nested_key):
+                        # Check if env var is set for this nested key
+                        full_key = f"{key}.{nested_key}"
+                        env_var = ENV_VAR_MAPPING.get(full_key)
+                        if env_var and os.environ.get(env_var):
+                            logger.debug(f"Skipping {full_key} from file (env var {env_var} takes precedence)")
+                            continue
                         setattr(group_obj, nested_key, nested_value)
                 continue
 
             # Handle flat dot-notation keys
             parts = key.split(".")
             if len(parts) == 1:
                 if hasattr(self, key) and not key.startswith("_"):
+                    # Check if env var is set for this key
+                    env_var = ENV_VAR_MAPPING.get(key)
+                    if env_var and os.environ.get(env_var):
+                        logger.debug(f"Skipping {key} from file (env var {env_var} takes precedence)")
+                        continue
                     setattr(self, key, value)
             elif len(parts) == 2:
                 group, attr = parts
                 if hasattr(self, group):
                     group_obj = getattr(self, group)
                     if hasattr(group_obj, attr):
+                        # Check if env var is set for this key
+                        env_var = ENV_VAR_MAPPING.get(key)
+                        if env_var and os.environ.get(env_var):
+                            logger.debug(f"Skipping {key} from file (env var {env_var} takes precedence)")
+                            continue
                         setattr(group_obj, attr, value)
             elif len(parts) == 4:
                 # Handle object_storage.stores.<name>.<attr> pattern

src/datajoint/version.py

@@ -1,4 +1,4 @@
 # version bump auto managed by Github Actions:
 # label_prs.yaml(prep), release.yaml(bump), post_release.yaml(edit)
 # manually set this version will be eventually overwritten by the above actions
-__version__ = "2.0.0a2"
+__version__ = "2.0.0a3"

tests/test_settings.py

@@ -249,18 +249,48 @@ def test_override_restores_on_exception(self):
 class TestLoad:
     """Test loading configuration."""
 
-    def test_load_config_file(self, tmp_path):
-        """Test loading configuration from file."""
+    def test_load_config_file(self, tmp_path, monkeypatch):
+        """Test loading configuration from file.
+
+        Note: Environment variables take precedence over config file values.
+        We need to clear DJ_HOST to test file loading.
+        """
         filename = tmp_path / "test_config.json"
         filename.write_text('{"database": {"host": "loaded_host"}}')
         original_host = dj.config.database.host
 
+        # Clear env var so file value takes effect
+        monkeypatch.delenv("DJ_HOST", raising=False)
+
         try:
             dj.config.load(filename)
             assert dj.config.database.host == "loaded_host"
         finally:
             dj.config.database.host = original_host
 
+    def test_env_var_overrides_config_file(self, tmp_path, monkeypatch):
+        """Test that environment variables take precedence over config file.
+
+        When DJ_HOST is set, loading a config file should NOT override the value.
+        The env var value should be preserved.
+        """
+        filename = tmp_path / "test_config.json"
+        filename.write_text('{"database": {"host": "file_host"}}')
+        original_host = dj.config.database.host
+
+        # Set env var - it should take precedence over file
+        monkeypatch.setenv("DJ_HOST", "env_host")
+        # Reset config to pick up new env var
+        dj.config.database.host = "env_host"
+
+        try:
+            dj.config.load(filename)
+            # File value should be skipped because DJ_HOST is set
+            # The env var value should be preserved
+            assert dj.config.database.host == "env_host"
+        finally:
+            dj.config.database.host = original_host
+
     def test_load_nonexistent_file(self):
         """Test loading nonexistent file raises FileNotFoundError."""
         with pytest.raises(FileNotFoundError):