Merge pull request #801 from simvue-io/789-invalid-python-environment-metadata-keys

🐛 Handle Conda file based dependencies in metadata

Author: kzscisoft

CHANGELOG.md

@@ -1,6 +1,6 @@
 [project]
 name = "simvue"
-version = "2.1.2"
+version = "2.1.3"
 description = "Simulation tracking and monitoring"
 authors = [
   {name = "Simvue Development Team", email = "info@simvue.io"}
@@ -55,6 +55,7 @@ dependencies = [
   "deepmerge (>=2.0,<3.0)",
   "geocoder (>=1.38.1,<2.0.0)",
   "pydantic-extra-types (>=2.10.5,<3.0.0)",
+  "pyyaml (>=6.0.2,<7.0.0)",
 ]
 
 [project.urls]

poetry.lock

@@ -10,6 +10,7 @@
 import typing
 import json
 import toml
+import yaml
 import logging
 import pathlib
 
@@ -76,9 +77,71 @@ def git_info(repository: str) -> dict[str, typing.Any]:
         return {}
 
 
+def _conda_dependency_parse(dependency: str) -> tuple[str, str] | None:
+    """Parse a dependency definition into module-version."""
+    if dependency.startswith("::"):
+        logger.warning(
+            f"Skipping Conda specific channel definition '{dependency}' in Python environment metadata."
+        )
+        return None
+    elif ">=" in dependency:
+        module, version = dependency.split(">=")
+        logger.warning(
+            f"Ignoring '>=' constraint in Python package version, naively storing '{module}=={version}', "
+            "for a more accurate record use 'conda env export > environment.yml'"
+        )
+    elif "~=" in dependency:
+        module, version = dependency.split("~=")
+        logger.warning(
+            f"Ignoring '~=' constraint in Python package version, naively storing '{module}=={version}', "
+            "for a more accurate record use 'conda env export > environment.yml'"
+        )
+    elif dependency.startswith("-e"):
+        _, version = dependency.split("-e")
+        version = version.strip()
+        module = pathlib.Path(version).name
+    elif dependency.startswith("file://"):
+        _, version = dependency.split("file://")
+        module = pathlib.Path(version).stem
+    elif dependency.startswith("git+"):
+        _, version = dependency.split("git+")
+        if "#egg=" in version:
+            repo, module = version.split("#egg=")
+            module = repo.split("/")[-1].replace(".git", "")
+        else:
+            module = version.split("/")[-1].replace(".git", "")
+    elif "==" not in dependency:
+        logger.warning(
+            f"Ignoring '{dependency}' in Python environment record as no version constraint specified."
+        )
+        return None
+    else:
+        module, version = dependency.split("==")
+
+    return module, version
+
+
+def _conda_env(environment_file: pathlib.Path) -> dict[str, str]:
+    """Parse/interpret a Conda environment file."""
+    content = yaml.load(environment_file.open(), Loader=yaml.SafeLoader)
+    python_environment: dict[str, str] = {}
+    pip_dependencies: list[str] = []
+    for dependency in content.get("dependencies", []):
+        if isinstance(dependency, dict) and dependency.get("pip"):
+            pip_dependencies = dependency["pip"]
+            break
+
+    for dependency in pip_dependencies:
+        if not (parsed := _conda_dependency_parse(dependency)):
+            continue
+        module, version = parsed
+        python_environment[module.strip().replace("-", "_")] = version.strip()
+    return python_environment
+
+
 def _python_env(repository: pathlib.Path) -> dict[str, typing.Any]:
     """Retrieve a dictionary of Python dependencies if lock file is available"""
-    python_meta: dict[str, str] = {}
+    python_meta: dict[str, dict] = {}
 
     if (pyproject_file := pathlib.Path(repository).joinpath("pyproject.toml")).exists():
         content = toml.load(pyproject_file)
@@ -103,22 +166,37 @@ def _python_env(repository: pathlib.Path) -> dict[str, typing.Any]:
         python_meta["environment"] = {
             package["name"]: package["version"] for package in content
         }
+    # Handle Conda case, albeit naively given the user may or may not have used 'conda env'
+    # to dump their exact dependency versions
+    elif (
+        environment_file := pathlib.Path(repository).joinpath("environment.yml")
+    ).exists():
+        python_meta["environment"] = _conda_env(environment_file)
     else:
         with contextlib.suppress((KeyError, ImportError)):
             from pip._internal.operations.freeze import freeze
 
-            python_meta["environment"] = {
-                entry[0]: entry[-1]
-                for line in freeze(local_only=True)
-                if (entry := line.split("=="))
-            }
+            # Conda supports having file names with @ as entries
+            # in the requirements.txt file as opposed to ==
+            python_meta["environment"] = {}
+
+            for line in freeze(local_only=True):
+                if line.startswith("-e"):
+                    python_meta["environment"]["local_install"] = line.split(" ")[-1]
+                    continue
+                if "@" in line:
+                    entry = line.split("@")
+                    python_meta["environment"][entry[0].strip()] = entry[-1].strip()
+                elif "==" in line:
+                    entry = line.split("==")
+                    python_meta["environment"][entry[0].strip()] = entry[-1].strip()
 
     return python_meta
 
 
 def _rust_env(repository: pathlib.Path) -> dict[str, typing.Any]:
     """Retrieve a dictionary of Rust dependencies if lock file available"""
-    rust_meta: dict[str, str] = {}
+    rust_meta: dict[str, dict] = {}
 
     if (cargo_file := pathlib.Path(repository).joinpath("Cargo.toml")).exists():
         content = toml.load(cargo_file).get("package", {})
@@ -134,15 +212,15 @@ def _rust_env(repository: pathlib.Path) -> dict[str, typing.Any]:
     cargo_dat = toml.load(cargo_lock)
     rust_meta["environment"] = {
         dependency["name"]: dependency["version"]
-        for dependency in cargo_dat.get("package")
+        for dependency in cargo_dat.get("package", [])
     }
 
     return rust_meta
 
 
 def _julia_env(repository: pathlib.Path) -> dict[str, typing.Any]:
     """Retrieve a dictionary of Julia dependencies if a project file is available"""
-    julia_meta: dict[str, str] = {}
+    julia_meta: dict[str, dict] = {}
     if (project_file := pathlib.Path(repository).joinpath("Project.toml")).exists():
         content = toml.load(project_file)
         julia_meta["project"] = {
@@ -155,7 +233,7 @@ def _julia_env(repository: pathlib.Path) -> dict[str, typing.Any]:
 
 
 def _node_js_env(repository: pathlib.Path) -> dict[str, typing.Any]:
-    js_meta: dict[str, str] = {}
+    js_meta: dict[str, dict] = {}
     if (
         project_file := pathlib.Path(repository).joinpath("package-lock.json")
     ).exists():

pyproject.toml

@@ -0,0 +1,40 @@
+name: advanced_env
+channels:
+  - conda-forge
+  - anaconda
+  - defaults
+dependencies:
+  # Basic Conda packages with different version specifiers
+  - python=3.10.12
+  - numpy>=1.23.5
+  - pandas
+  - scikit-learn<1.2
+  - openjdk>=11,<12
+
+  # Platform-specific dependencies
+  - libsass  # Standard dependency
+  - vsix-installer  # Standard dependency
+  - openblas  # A package that may have platform-specific builds
+
+  # Using a sub-channel (also called a label)
+  - ::my-package-from-subchannel
+  
+  # A 'pip' section for installing packages from PyPI and other sources
+  - pip
+  - pip:
+    # Public PyPI packages with different version specifiers
+    - requests==2.31.0
+    - black
+    - jupyterlab~=4.0.0
+    - numpy==2.32.2
+    
+    # A local package from a path
+    - -e ./path/to/my-local-package
+    - file:///path/to/my-local-wheel.whl
+    
+    # A package from a Git repository
+    - git+https://github.com/myuser/myrepo.git#egg=myproject
+
+variables:
+  # Define environment variables
+  MY_ENV_VAR: "some_value"

simvue/metadata.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import toml
 import os
 import pytest
 import requests
@@ -1315,3 +1316,28 @@ def test_reconnect_with_process() -> None:
             remove_runs=True,
             recursive=True
         )
+
+@pytest.mark.parametrize(
+    "environment", ("python_conda", "python_poetry", "python_uv", "julia", "rust", "nodejs")
+)
+def test_run_environment_metadata(environment: str, mocker: pytest_mock.MockerFixture) -> None:
+    """Tests that the environment information is compatible with the server."""
+    from simvue.config.user import SimvueConfiguration
+    from simvue.metadata import environment as env_func
+    _data_dir = pathlib.Path(__file__).parents[1].joinpath("example_data")
+    _target_dir = _data_dir
+    if "python" in environment:
+        _target_dir = _data_dir.joinpath(environment)
+    _config = SimvueConfiguration.fetch()
+
+    with sv_run.Run(server_token=_config.server.token, server_url=_config.server.url) as run:
+        _uuid = f"{uuid.uuid4()}".split("-")[0]
+        run.init(
+            name=f"test_run_environment_metadata_{environment}",
+            folder=f"/simvue_unit_testing/{_uuid}",
+            retention_period=os.environ.get("SIMVUE_TESTING_RETENTION_PERIOD", "2 mins"),
+            running=False,
+            visibility="tenant" if os.environ.get("CI") else None,
+        )
+        run.update_metadata(env_func(_target_dir))
+

tests/example_data/python_conda/environment.yml

@@ -14,7 +14,7 @@ def test_cargo_env() -> None:
 @pytest.mark.metadata
 @pytest.mark.local
 @pytest.mark.parametrize(
-    "backend", ("poetry", "uv", None)
+    "backend", ("poetry", "uv", "conda", None)
 )
 def test_python_env(backend: str | None) -> None:
     if backend == "poetry":
@@ -23,6 +23,9 @@ def test_python_env(backend: str | None) -> None:
     elif backend == "uv":
         metadata = sv_meta._python_env(pathlib.Path(__file__).parents[1].joinpath("example_data", "python_uv"))
         assert metadata["project"]["name"] == "example-repo"
+    elif backend == "conda":
+        metadata = sv_meta._python_env(pathlib.Path(__file__).parents[1].joinpath("example_data", "python_conda"))
+        assert metadata["environment"]["requests"]
     else:
         metadata = sv_meta._python_env(pathlib.Path(__file__).parents[1].joinpath("example_data"))
 
@@ -51,4 +54,4 @@ def test_environment() -> None:
     assert metadata["python"]["project"]["name"] == "example-repo"
     assert metadata["rust"]["project"]["name"] == "example_project"
     assert metadata["julia"]["project"]["name"] == "Julia Demo Project"
-    assert metadata["javascript"]["project"]["name"] == "my-awesome-project"
+    assert metadata["javascript"]["project"]["name"] == "my-awesome-project"