Dynamische Dependencies / Umfassende Dependency-Resolution

pyproject.toml

@@ -7,7 +7,7 @@ name = "questionpy-server"
 description = "QuestionPy application server"
 license = { file = "LICENSE.md" }
 urls = { homepage = "https://questionpy.org" }
-version = "0.8.0"
+version = "0.9.0"
 authors = [
     { name = "TU Berlin innoCampus" },
     { email = "info@isis.tu-berlin.de" }
@@ -21,7 +21,8 @@ dependencies = [
     "semver >=3.0.4, <4.0.0",
     "psutil >=7.0.0, <8.0.0",
     "jinja2 >=3.1.6, <4.0.0",
-    "pyyaml >=6.0.2, <7.0.0"
+    "pyyaml >=6.0.2, <7.0.0",
+    "resolvelib >=1.2.1, <2.0.0"
 ]
 
 [tool.poetry.group.dev.dependencies]

questionpy_common/__init__.py

@@ -2,7 +2,9 @@
 #  QuestionPy is free software released under terms of the MIT license. See LICENSE.md.
 #  (c) Technische Universität Berlin, innoCampus <info@isis.tu-berlin.de>
 from abc import ABC, abstractmethod
+from typing import Any, NamedTuple, Self
 
+from pydantic import GetCoreSchemaHandler
 from pydantic_core import CoreSchema, core_schema
 
 
@@ -20,3 +22,32 @@ def __get_pydantic_core_schema__(cls, *_: object) -> CoreSchema:
 
 
 TranslatableString.register(str)
+
+
+class PackageNamespaceAndShortName(NamedTuple):
+    """Tuple of namespace and short name, identifying any version of a specific package."""
+
+    namespace: str
+    short_name: str
+
+    def __str__(self) -> str:
+        return f"@{self.namespace}/{self.short_name}"
+
+    @classmethod
+    def from_string(cls, value: str) -> Self:
+        """Parse an NSSN in the same format as produced by `__str__`."""
+        value = value.strip()
+        if not value.startswith("@") or value.count("/") != 1:
+            msg = f"Invalid package identifier (NSSN): '{value}'"
+            raise ValueError(msg)
+
+        ns, sn = value.removeprefix("@").split("/", maxsplit=1)
+        return cls(ns, sn)
+
+    @classmethod
+    def __get_pydantic_core_schema__(cls, source_type: Any, handle: GetCoreSchemaHandler) -> CoreSchema:
+        return core_schema.no_info_before_validator_function(
+            lambda obj: cls.from_string(obj) if isinstance(obj, str) else obj,
+            handle(source_type),
+            serialization=core_schema.to_string_ser_schema(),
+        )

questionpy_common/constants.py

@@ -27,4 +27,19 @@
     r"^([a-zA-Z_][a-zA-Z0-9_]*|\.\.)(\[([a-zA-Z_][a-zA-Z0-9_]*|\.\.)?])*$"
 )
 
+# Regular expressions.
+
 ENVIRONMENT_VARIABLE_REGEX: Final[str] = r"[a-zA-Z_][a-zA-Z0-9_]*"
+
+RE_SEMVER = (
+    r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)"
+    r"(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
+)
+
+RE_API = r"^(0|[1-9]\d*)\.(0|[1-9]\d*)$"
+
+# The SemVer and Api version patterns are used on pydantic fields, which uses Rust regexes, so re.compiling them makes
+# no sense. We match RE_VALID_CHARS_NAME in Python though, so here it does.
+RE_VALID_CHARS_NAME = re.compile(r"^[a-z\d_]+$")
+
+NAME_MAX_LENGTH = 127

questionpy_common/dependencies.py

@@ -0,0 +1,58 @@
+from dataclasses import dataclass, field
+from typing import Annotated
+
+from pydantic import Field
+
+from questionpy_common import PackageNamespaceAndShortName
+from questionpy_common.constants import RE_SEMVER
+from questionpy_common.manifest import DistDependencies
+from questionpy_common.package_location import PackageLocation
+
+
+@dataclass(frozen=True)
+class StaticDependencySolution:
+    """Indicates that a package in the tree provides a static dependency that is to be used.
+
+    Usually, this is a solution for the static dependency itself, but if there is also a dynamic dependency in the tree
+    for that NSSN _and_ that dynamic dependency allows the static version, a `StaticDependencySolution` might be used to
+    also solve a dynamic dependency.
+
+    If multiple packages provide the same version of a static dependency, any of them may be used as the solution.
+    Static dependencies on different versions of the same NSSN will always lead to a `DependencyConflictError`.
+    """
+
+    nssn: PackageNamespaceAndShortName
+
+    owner: PackageNamespaceAndShortName
+    """The package that includes this static dependency."""
+
+    hash: str
+    # semver.Version is avoided to allow solutions to be passed to the package.
+    version: Annotated[str, Field(pattern=RE_SEMVER)]
+
+    dependencies: DistDependencies = field(compare=False)
+    """Transitive dependencies of this dependency."""
+
+    def __str__(self) -> str:
+        return f"{self.hash} ({self.version}, statically packaged in '{self.owner}')"
+
+
+@dataclass(frozen=True)
+class DynamicDependencySolution:
+    """Indicates that the given version should be used to supply all usages of the NSSN."""
+
+    nssn: PackageNamespaceAndShortName
+
+    hash: str
+    # semver.Version is avoided to allow solutions to be passed to the package.
+    version: Annotated[str, Field(pattern=RE_SEMVER)]
+    dependencies: DistDependencies = field(compare=False)
+    """Transitive dependencies of this dependency."""
+
+    def __str__(self) -> str:
+        return f"{self.hash} ({self.version}, dynamic)"
+
+
+type DependencySolution = StaticDependencySolution | DynamicDependencySolution
+
+type SolutionAndLocation = tuple[DynamicDependencySolution, PackageLocation] | tuple[StaticDependencySolution, None]

questionpy_common/environment.py

@@ -7,10 +7,11 @@
 from enum import Enum
 from functools import total_ordering
 from importlib.resources.abc import Traversable
-from typing import NamedTuple, Protocol
+from typing import Protocol
 
 from pydantic import BaseModel, JsonValue
 
+from questionpy_common import PackageNamespaceAndShortName
 from questionpy_common.api.package import QPyPackageInterface
 from questionpy_common.manifest import Bcp47LanguageTag, Manifest
 
@@ -21,7 +22,6 @@
     "OnRequestCallback",
     "Package",
     "PackageInitFunction",
-    "PackageNamespaceAndShortName",
     "PackageNotInitializedError",
     "PackageNotLoadedError",
     "PackagePermissions",
@@ -79,16 +79,6 @@ def __lt__(self, other: object) -> bool:
         return NotImplemented
 
 
-class PackageNamespaceAndShortName(NamedTuple):
-    """Tuple of namespace and short name, identifying any version of a specific package."""
-
-    namespace: str
-    short_name: str
-
-    def __str__(self) -> str:
-        return f"@{self.namespace}/{self.short_name}"
-
-
 class Package(Protocol):
     @property
     def manifest(self) -> Manifest: ...

questionpy_common/manifest.py

@@ -2,15 +2,31 @@
 #  QuestionPy is free software released under terms of the MIT license. See LICENSE.md.
 #  (c) Technische Universität Berlin, innoCampus <info@isis.tu-berlin.de>
 
-import re
+from abc import ABC
 from enum import StrEnum
 from keyword import iskeyword, issoftkeyword
-from typing import Annotated, NewType
-
-from pydantic import BaseModel, ByteSize, PositiveInt, StringConstraints, conset, field_validator
+from typing import Annotated, Literal, NewType
+
+from pydantic import (
+    AfterValidator,
+    BaseModel,
+    ByteSize,
+    PositiveInt,
+    StringConstraints,
+    conset,
+    field_validator,
+)
 from pydantic.fields import Field
 
-from questionpy_common.constants import ENVIRONMENT_VARIABLE_REGEX
+from questionpy_common import PackageNamespaceAndShortName
+from questionpy_common.constants import (
+    ENVIRONMENT_VARIABLE_REGEX,
+    NAME_MAX_LENGTH,
+    RE_API,
+    RE_SEMVER,
+    RE_VALID_CHARS_NAME,
+)
+from questionpy_common.version_specifiers import QPyDependencyVersionSpecifier
 
 
 class PackageType(StrEnum):
@@ -19,22 +35,9 @@ class PackageType(StrEnum):
     QUESTION = "QUESTION"
 
 
-# Defaults.
 DEFAULT_NAMESPACE = "local"
 DEFAULT_PACKAGETYPE = PackageType.QUESTIONTYPE
 
-# Regular expressions.
-RE_SEMVER = (
-    r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)"
-    r"(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
-)
-RE_API = r"^(0|[1-9]\d*)\.(0|[1-9]\d*)$"
-# The SemVer and Api version patterns are used on pydantic fields, which uses Rust regexes, so re.compiling them makes
-# no sense. We match RE_VALID_CHARS_NAME in Python though, so here it does.
-RE_VALID_CHARS_NAME = re.compile(r"^[a-z\d_]+$")
-
-NAME_MAX_LENGTH = 127
-
 
 # Validators.
 def ensure_is_valid_name(name: str) -> str:
@@ -139,6 +142,10 @@ def ensure_contains_english_translation(
     def identifier(self) -> str:
         return f"@{self.namespace}/{self.short_name}"
 
+    @property
+    def nssn(self) -> PackageNamespaceAndShortName:
+        return PackageNamespaceAndShortName(self.namespace, self.short_name)
+
 
 class PackageFile(BaseModel):
     """Represents a static file included in a built package."""
@@ -148,13 +155,46 @@ class PackageFile(BaseModel):
 
 
 class DistStaticQPyDependency(BaseModel):
-    dir_name: str
-    """Name (without `dist/dependencies/qpy/`) of the directory the dependency package contents reside in."""
+    namespace: Annotated[str, AfterValidator(ensure_is_valid_name)]
+    short_name: Annotated[str, AfterValidator(ensure_is_valid_name)]
+    version: Annotated[str, Field(pattern=RE_SEMVER)]
+
+    dependencies: "DistDependencies"
+    """Transitive dependencies of this dependency."""
+
     hash: str
-    """Hash of the ZIP package whose contents lie in `dir_name`."""
+    """Hash of the ZIP package whose contents are included in this package."""
+
+    @property
+    def nssn(self) -> PackageNamespaceAndShortName:
+        return PackageNamespaceAndShortName(self.namespace, self.short_name)
+
+
+type DependencyLockStrategy = Literal["required", "preferred-no-downgrade", "preferred-allow-downgrade"]
+
+
+class LockedDependencyInfo(BaseModel):
+    strategy: DependencyLockStrategy
+    locked_version: Annotated[str, Field(pattern=RE_SEMVER)]
+    locked_hash: str
+
+
+class AbstractDynamicQPyDependency(BaseModel, ABC):
+    namespace: Annotated[str, AfterValidator(ensure_is_valid_name)]
+    short_name: Annotated[str, AfterValidator(ensure_is_valid_name)]
+    version: QPyDependencyVersionSpecifier | None = None
+    include_prereleases: bool = False
+
+    @property
+    def nssn(self) -> PackageNamespaceAndShortName:
+        return PackageNamespaceAndShortName(self.namespace, self.short_name)
+
+
+class DistDynamicQPyDependency(AbstractDynamicQPyDependency):
+    locked: LockedDependencyInfo | None = None
 
 
-type DistQPyDependency = DistStaticQPyDependency
+type DistQPyDependency = DistStaticQPyDependency | DistDynamicQPyDependency
 
 
 class DistDependencies(BaseModel):