Add hugging-face command support

CHANGES/+hugging-face.feature

@@ -0,0 +1 @@
+Added the `hugging-face` command group to manage Hugging Face remotes, repositories, and distributions for pull-through caching of Hugging Face Hub content.

CHANGES/pulp-glue/+hugging-face.feature

@@ -0,0 +1 @@
+Added entity contexts for the `hugging_face` plugin (remote, repository, repository version, and distribution).

pulp-glue/src/pulp_glue/hugging_face/context.py

@@ -0,0 +1,56 @@
+from pulp_glue.common.context import (
+    EntityDefinition,
+    PluginRequirement,
+    PulpDistributionContext,
+    PulpRemoteContext,
+    PulpRepositoryContext,
+    PulpRepositoryVersionContext,
+)
+from pulp_glue.common.i18n import get_translation
+
+translation = get_translation(__package__)
+_ = translation.gettext
+
+
+class PulpHuggingFaceRemoteContext(PulpRemoteContext):
+    PLUGIN = "hugging_face"
+    RESOURCE_TYPE = "hugging-face"
+    ENTITY = _("hugging face remote")
+    ENTITIES = _("hugging face remotes")
+    HREF = "hugging_face_hugging_face_remote_href"
+    ID_PREFIX = "remotes_hugging_face_hugging_face"
+    NEEDS_PLUGINS = [PluginRequirement("hugging_face", specifier=">=0.1.0")]
+
+
+class PulpHuggingFaceDistributionContext(PulpDistributionContext):
+    PLUGIN = "hugging_face"
+    RESOURCE_TYPE = "hugging-face"
+    ENTITY = _("hugging face distribution")
+    ENTITIES = _("hugging face distributions")
+    HREF = "hugging_face_hugging_face_distribution_href"
+    ID_PREFIX = "distributions_hugging_face_hugging_face"
+    NEEDS_PLUGINS = [PluginRequirement("hugging_face", specifier=">=0.1.0")]
+
+    def preprocess_entity(self, body: EntityDefinition, partial: bool = False) -> EntityDefinition:
+        body = super().preprocess_entity(body, partial=partial)
+        if not partial and self.pulp_ctx.has_plugin(PluginRequirement("core", specifier=">=3.16.0")):
+            body.setdefault("repository", None)
+            body.setdefault("remote", None)
+        return body
+
+
+class PulpHuggingFaceRepositoryVersionContext(PulpRepositoryVersionContext):
+    HREF = "hugging_face_hugging_face_repository_version_href"
+    ID_PREFIX = "repositories_hugging_face_hugging_face_versions"
+    NEEDS_PLUGINS = [PluginRequirement("hugging_face", specifier=">=0.1.0")]
+
+
+class PulpHuggingFaceRepositoryContext(PulpRepositoryContext):
+    PLUGIN = "hugging_face"
+    RESOURCE_TYPE = "hugging-face"
+    ENTITY = _("hugging face repository")
+    ENTITIES = _("hugging face repositories")
+    HREF = "hugging_face_hugging_face_repository_href"
+    ID_PREFIX = "repositories_hugging_face_hugging_face"
+    VERSION_CONTEXT = PulpHuggingFaceRepositoryVersionContext
+    NEEDS_PLUGINS = [PluginRequirement("hugging_face", specifier=">=0.1.0")]

pyproject.toml

@@ -49,6 +49,7 @@ common = "pulpcore.cli.common"
 container = "pulpcore.cli.container"
 core = "pulpcore.cli.core"
 file = "pulpcore.cli.file"
+hugging-face = "pulpcore.cli.hugging_face"
 python = "pulpcore.cli.python"
 rpm = "pulpcore.cli.rpm"
 

src/pulpcore/cli/hugging_face/__init__.py

@@ -0,0 +1,27 @@
+import typing as t
+
+import click
+
+from pulp_glue.common.i18n import get_translation
+
+from pulp_cli.generic import pulp_group
+from pulpcore.cli.hugging_face.distribution import distribution
+from pulpcore.cli.hugging_face.remote import remote
+from pulpcore.cli.hugging_face.repository import repository
+
+translation = get_translation(__package__)
+_ = translation.gettext
+
+__version__ = "0.1.0"
+
+
+@pulp_group(name="hugging-face")
+def hugging_face() -> None:
+    """Manage Hugging Face Hub content via Pulp."""
+
+
+def mount(main: click.Group, **kwargs: t.Any) -> None:
+    hugging_face.add_command(remote)
+    hugging_face.add_command(distribution)
+    hugging_face.add_command(repository)
+    main.add_command(hugging_face)

src/pulpcore/cli/hugging_face/distribution.py

@@ -0,0 +1,81 @@
+import click
+
+from pulp_glue.common.context import PulpRemoteContext, PulpRepositoryContext
+from pulp_glue.common.i18n import get_translation
+from pulp_glue.hugging_face.context import (
+    PulpHuggingFaceDistributionContext,
+    PulpHuggingFaceRemoteContext,
+    PulpHuggingFaceRepositoryContext,
+)
+
+from pulp_cli.generic import (
+    common_distribution_create_options,
+    create_command,
+    destroy_command,
+    distribution_filter_options,
+    distribution_lookup_option,
+    href_option,
+    label_command,
+    list_command,
+    name_option,
+    pulp_group,
+    pulp_labels_option,
+    resource_option,
+    show_command,
+    type_option,
+    update_command,
+)
+
+translation = get_translation(__package__)
+_ = translation.gettext
+
+
+remote_option = resource_option(
+    "--remote",
+    default_plugin="hugging_face",
+    default_type="hugging-face",
+    context_table={"hugging_face:hugging-face": PulpHuggingFaceRemoteContext},
+    href_pattern=PulpRemoteContext.HREF_PATTERN,
+    help=_(
+        "Hugging Face remote to use for pull-through caching."
+        " Specified as '[[<plugin>:]<type>:]<name>' or as href."
+        " Pass an empty string to detach the remote."
+    ),
+)
+repository_option = resource_option(
+    "--repository",
+    default_plugin="hugging_face",
+    default_type="hugging-face",
+    context_table={"hugging_face:hugging-face": PulpHuggingFaceRepositoryContext},
+    href_pattern=PulpRepositoryContext.HREF_PATTERN,
+    help=_(
+        "Hugging Face repository whose latest version is served."
+        " Specified as '[[<plugin>:]<type>:]<name>' or as href."
+        " Pass an empty string to detach the repository."
+    ),
+)
+
+
+@pulp_group()
+@type_option(choices={"hugging-face": PulpHuggingFaceDistributionContext})
+def distribution() -> None:
+    pass
+
+
+lookup_options = [href_option, name_option, distribution_lookup_option]
+nested_lookup_options = [distribution_lookup_option]
+update_options = [
+    remote_option,
+    repository_option,
+    pulp_labels_option,
+]
+create_options = common_distribution_create_options + update_options
+
+distribution.add_command(list_command(decorators=distribution_filter_options))
+distribution.add_command(show_command(decorators=lookup_options))
+distribution.add_command(create_command(decorators=create_options))
+distribution.add_command(
+    update_command(decorators=lookup_options + update_options + [click.option("--base-path")])
+)
+distribution.add_command(destroy_command(decorators=lookup_options))
+distribution.add_command(label_command(decorators=nested_lookup_options))

src/pulpcore/cli/hugging_face/remote.py

@@ -0,0 +1,62 @@
+import click
+
+from pulp_glue.common.i18n import get_translation
+from pulp_glue.hugging_face.context import PulpHuggingFaceRemoteContext
+
+from pulp_cli.generic import (
+    common_remote_create_options,
+    common_remote_update_options,
+    create_command,
+    destroy_command,
+    href_option,
+    label_command,
+    list_command,
+    name_option,
+    pulp_group,
+    remote_filter_options,
+    remote_lookup_option,
+    show_command,
+    type_option,
+    update_command,
+)
+
+translation = get_translation(__package__)
+_ = translation.gettext
+
+
+@pulp_group()
+@type_option(choices={"hugging-face": PulpHuggingFaceRemoteContext})
+def remote() -> None:
+    pass
+
+
+lookup_options = [href_option, name_option, remote_lookup_option]
+nested_lookup_options = [remote_lookup_option]
+hugging_face_remote_options = [
+    click.option(
+        "--hf-token",
+        "hf_token",
+        help=_(
+            "Hugging Face API token for accessing private repositories."
+            " This value is write-only and will not appear in API responses."
+        ),
+    ),
+    click.option(
+        "--policy",
+        type=click.Choice(["immediate", "on_demand", "streamed"], case_sensitive=False),
+        help=_("Policy for downloading content (use 'on_demand' for pull-through caching)."),
+    ),
+]
+
+remote.add_command(list_command(decorators=remote_filter_options))
+remote.add_command(show_command(decorators=lookup_options))
+remote.add_command(
+    create_command(decorators=common_remote_create_options + hugging_face_remote_options)
+)
+remote.add_command(
+    update_command(
+        decorators=lookup_options + common_remote_update_options + hugging_face_remote_options
+    )
+)
+remote.add_command(destroy_command(decorators=lookup_options))
+remote.add_command(label_command(decorators=nested_lookup_options))

src/pulpcore/cli/hugging_face/repository.py

@@ -0,0 +1,57 @@
+import click
+
+from pulp_glue.common.i18n import get_translation
+from pulp_glue.hugging_face.context import PulpHuggingFaceRepositoryContext
+
+from pulp_cli.generic import (
+    create_command,
+    destroy_command,
+    href_option,
+    label_command,
+    label_select_option,
+    list_command,
+    name_option,
+    pulp_group,
+    pulp_labels_option,
+    repository_href_option,
+    repository_lookup_option,
+    retained_versions_option,
+    show_command,
+    type_option,
+    update_command,
+    version_command,
+)
+
+translation = get_translation(__package__)
+_ = translation.gettext
+
+
+@pulp_group()
+@type_option(choices={"hugging-face": PulpHuggingFaceRepositoryContext})
+def repository() -> None:
+    pass
+
+
+lookup_options = [href_option, name_option, repository_lookup_option]
+nested_lookup_options = [repository_href_option, repository_lookup_option]
+update_options = [
+    click.option("--description"),
+    retained_versions_option,
+    pulp_labels_option,
+]
+create_options = update_options + [click.option("--name", required=True)]
+
+repository.add_command(
+    list_command(
+        decorators=[
+            label_select_option,
+            click.option("--name-contains", "name__contains"),
+        ]
+    )
+)
+repository.add_command(show_command(decorators=lookup_options))
+repository.add_command(create_command(decorators=create_options))
+repository.add_command(update_command(decorators=lookup_options + update_options))
+repository.add_command(destroy_command(decorators=lookup_options))
+repository.add_command(version_command(decorators=nested_lookup_options))
+repository.add_command(label_command(decorators=nested_lookup_options))

tests/scripts/pulp_hugging_face/test_distribution.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+set -eu
+# shellcheck source=tests/scripts/config.source
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
+
+pulp debug has-plugin --name "hugging_face" || exit 23
+
+cleanup() {
+  pulp hugging-face distribution destroy --name "cli_test_hugging_face_distro" || true
+  pulp hugging-face repository destroy --name "cli_test_hugging_face_distro_repo" || true
+  pulp hugging-face remote destroy --name "cli_test_hugging_face_distro_remote" || true
+}
+trap cleanup EXIT
+
+REMOTE_HREF="$(pulp hugging-face remote create --name "cli_test_hugging_face_distro_remote" --url "https://huggingface.co/" --policy "on_demand" | jq -r '.pulp_href')"
+REPO_HREF="$(pulp hugging-face repository create --name "cli_test_hugging_face_distro_repo" | jq -r '.pulp_href')"
+
+expect_succ pulp hugging-face distribution create --name "cli_test_hugging_face_distro" --base-path "cli_test_hugging_face_distro" --remote "cli_test_hugging_face_distro_remote"
+expect_succ pulp hugging-face distribution show --distribution "cli_test_hugging_face_distro"
+test "$(echo "$OUTPUT" | jq -r '.remote')" = "$REMOTE_HREF"
+
+expect_succ pulp hugging-face distribution update --distribution "cli_test_hugging_face_distro" --remote "" --repository "cli_test_hugging_face_distro_repo"
+expect_succ pulp hugging-face distribution show --distribution "cli_test_hugging_face_distro"
+test "$(echo "$OUTPUT" | jq -r '.remote')" = "null"
+test "$(echo "$OUTPUT" | jq -r '.repository')" = "$REPO_HREF"
+
+expect_succ pulp hugging-face distribution update --distribution "cli_test_hugging_face_distro" --repository ""
+expect_succ pulp hugging-face distribution show --distribution "cli_test_hugging_face_distro"
+test "$(echo "$OUTPUT" | jq -r '.repository')" = "null"
+
+expect_succ pulp hugging-face distribution list --base-path "cli_test_hugging_face_distro"
+test "$(echo "$OUTPUT" | jq -r length)" -eq 1
+
+expect_succ pulp hugging-face distribution destroy --distribution "cli_test_hugging_face_distro"

tests/scripts/pulp_hugging_face/test_remote.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -eu
+# shellcheck source=tests/scripts/config.source
+. "$(dirname "$(dirname "$(realpath "$0")")")"/config.source
+
+pulp debug has-plugin --name "hugging_face" || exit 23
+
+cleanup() {
+  pulp hugging-face remote destroy --name "cli_test_hugging_face_remote" || true
+}
+trap cleanup EXIT
+
+expect_succ pulp hugging-face remote list
+
+expect_succ pulp hugging-face remote create --name "cli_test_hugging_face_remote" --url "https://huggingface.co/" --policy "on_demand" --hf-token "s3cr3t"
+expect_succ pulp hugging-face remote show --remote "cli_test_hugging_face_remote"
+HREF="$(echo "$OUTPUT" | jq -r '.pulp_href')"
+test "$(echo "$OUTPUT" | jq -r '.policy')" = "on_demand"
+# hf_token is write-only and must never be returned in API responses.
+test "$(echo "$OUTPUT" | jq -r '.hf_token')" = "null"
+
+expect_succ pulp hugging-face remote update --remote "$HREF" --policy "immediate"
+expect_succ pulp hugging-face remote show --remote "cli_test_hugging_face_remote"
+test "$(echo "$OUTPUT" | jq -r '.policy')" = "immediate"
+
+expect_succ pulp hugging-face remote list --name-contains "li_test_hugging_face_remot"
+test "$(echo "$OUTPUT" | jq -r '.|length')" = "1"
+
+expect_succ pulp hugging-face remote destroy --remote "cli_test_hugging_face_remote"