From 64e02dac13a7bf5b16fb72d711de91a156e5104a Mon Sep 17 00:00:00 2001 From: Gerrod Ubben Date: Mon, 16 Feb 2026 15:54:53 -0500 Subject: [PATCH] Migrate to Django 5 STORAGES and get_domain() Assisted by: cursor composer-1.5 Co-authored-by: Cursor --- .ci/assets/ci_constraints.txt | 3 +- .ci/scripts/check_release.py | 11 ++- .github/workflows/ci.yml | 8 +- .github/workflows/scripts/install.sh | 4 +- pulp_container/app/registry.py | 7 +- pulp_container/app/registry_api.py | 16 ++-- pulp_container/app/tasks/sign.py | 3 +- pulp_container/app/utils.py | 3 +- .../functional/api/test_content_cache.py | 17 +--- template_config.yml | 87 +++++++++++++------ 10 files changed, 98 insertions(+), 61 deletions(-) diff --git a/.ci/assets/ci_constraints.txt b/.ci/assets/ci_constraints.txt index c8070aff5..c9198f19a 100644 --- a/.ci/assets/ci_constraints.txt +++ b/.ci/assets/ci_constraints.txt @@ -1,5 +1,6 @@ # Pulpcore versions without the openapi command do no longer work in the CI -pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.* +# Pulpcore versions without the django 5 storage compatibility will fail, >3.63,<3.70 +pulpcore>=3.21.30,!=3.23.*,!=3.24.*,!=3.25.*,!=3.26.*,!=3.27.*,!=3.29.*,!=3.30.*,!=3.31.*,!=3.32.*,!=3.33.*,!=3.34.*,!=3.35.*,!=3.36.*,!=3.37.*,!=3.38.*,!=3.40.*,!=3.41.*,!=3.42.*,!=3.43.*,!=3.44.*,!=3.45.*,!=3.46.*,!=3.47.*,!=3.48.*,!=3.50.*,!=3.51.*,!=3.52.*,!=3.53.*,!=3.54.*,!=3.64.*,!=3.65.*,!=3.66.*,!=3.67.*,!=3.68.*,!=3.69.* tablib!=3.6.0 diff --git a/.ci/scripts/check_release.py b/.ci/scripts/check_release.py index 81bae1fd1..86e250e4c 100755 --- a/.ci/scripts/check_release.py +++ b/.ci/scripts/check_release.py @@ -1,11 +1,20 @@ #!/usr/bin/env python +# /// script +# requires-python = ">=3.13" +# dependencies = [ +# "gitpython>=3.1.46,<3.2.0", +# "packaging>=26.0,<26.1", +# "pyyaml>=6.0.3,<6.1.0", +# ] +# /// import argparse import re import os import tomllib -import yaml from pathlib import Path + +import yaml from packaging.version import Version from git import Repo diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0eb61fc05..664689748 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,13 +94,13 @@ jobs: run_docs: ${{ needs.check-changes.outputs.run_docs }} lint: - needs: - - "check-changes" - if: needs.check-changes.outputs.run_tests == '1' uses: "./.github/workflows/lint.yml" build: - needs: "lint" + needs: + - "check-changes" + - "lint" + if: needs.check-changes.outputs.run_tests == '1' uses: "./.github/workflows/build.yml" test: diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index b0c202752..36c82ab2a 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -95,7 +95,7 @@ if [ "$TEST" = "s3" ]; then sed -i -e '$a s3_test: true\ minio_access_key: "'$MINIO_ACCESS_KEY'"\ minio_secret_key: "'$MINIO_SECRET_KEY'"\ -pulp_scenario_settings: {"flatpak_index": false, "token_auth_disabled": true}\ +pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "flatpak_index": false, "token_auth_disabled": true}\ pulp_scenario_env: {}\ ' vars/main.yaml export PULP_API_ROOT="/rerouted/djnd/" @@ -109,7 +109,7 @@ if [ "$TEST" = "azure" ]; then - ./azurite:/etc/pulp\ command: "azurite-blob --skipApiVersionCheck --blobHost 0.0.0.0"' vars/main.yaml sed -i -e '$a azure_test: true\ -pulp_scenario_settings: {"flatpak_index": true}\ +pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.azure_storage.AzureStorage", "OPTIONS": {"account_key": "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", "account_name": "devstoreaccount1", "azure_container": "pulp-test", "connection_string": "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;", "expiration_secs": 120, "location": "pulp3", "overwrite_files": true}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "flatpak_index": true}\ pulp_scenario_env: {}\ ' vars/main.yaml fi diff --git a/pulp_container/app/registry.py b/pulp_container/app/registry.py index be6b1cced..44f0796fd 100644 --- a/pulp_container/app/registry.py +++ b/pulp_container/app/registry.py @@ -12,12 +12,12 @@ from aiohttp.web_exceptions import HTTPTooManyRequests from django_guid import set_guid from django_guid.utils import generate_guid -from django.conf import settings from django.core.exceptions import ObjectDoesNotExist from django.db import IntegrityError from multidict import MultiDict from pulpcore.plugin.content import Handler, PathNotResolved +from pulpcore.plugin.util import get_domain from pulpcore.plugin.models import RemoteArtifact, Content, ContentArtifact from pulpcore.plugin.content import ArtifactResponse from pulpcore.plugin.tasking import dispatch @@ -83,9 +83,10 @@ async def _dispatch(artifact, headers): full_headers["Docker-Content-Digest"] = headers["Docker-Content-Digest"] full_headers["Docker-Distribution-API-Version"] = "registry/2.0" - if settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem": + domain = get_domain() + if domain.storage_class == "pulpcore.app.models.storage.FileSystem": file = artifact.file - path = os.path.join(settings.MEDIA_ROOT, file.name) + path = os.path.join(domain.get_storage().location, file.name) if not os.path.exists(path): raise Exception("Expected path '{}' is not found".format(path)) return web.FileResponse(path, headers=full_headers) diff --git a/pulp_container/app/registry_api.py b/pulp_container/app/registry_api.py index 86ac20a0f..a8114c162 100644 --- a/pulp_container/app/registry_api.py +++ b/pulp_container/app/registry_api.py @@ -18,7 +18,6 @@ from tempfile import NamedTemporaryFile from django.core.exceptions import ObjectDoesNotExist -from django.core.files.storage import default_storage as storage from django.core.files.base import ContentFile, File from django.db import IntegrityError, transaction from django.db.models import F, Value @@ -30,7 +29,7 @@ from pulpcore.plugin.models import Artifact, ContentArtifact, UploadChunk from pulpcore.plugin.files import PulpTemporaryUploadedFile from pulpcore.plugin.tasking import add_and_remove, dispatch -from pulpcore.plugin.util import get_objects_for_user, get_url +from pulpcore.plugin.util import get_objects_for_user, get_url, get_domain from pulpcore.plugin.exceptions import TimeoutException from rest_framework.exceptions import ( @@ -596,7 +595,7 @@ def get_manifest_config(self, manifest): except Artifact.DoesNotExist: log.warning(f"Manifest {manifest.pk}'s config blob was not found.") else: - with storage.open(config_artifact.file.name) as file: + with get_domain().get_storage().open(config_artifact.file.name) as file: raw_data = file.read() config_data = json.loads(raw_data) config["labels"] = config_data.get("config", {}).get("Labels") @@ -989,14 +988,15 @@ def __init__(self, *args, **kwargs): """ super().__init__(*args, **kwargs) + domain = get_domain() if ( - settings.DEFAULT_FILE_STORAGE == "pulpcore.app.models.storage.FileSystem" - or not settings.REDIRECT_TO_OBJECT_STORAGE + domain.storage_class == "pulpcore.app.models.storage.FileSystem" + or not domain.redirect_to_object_storage ): self.redirects_class = FileStorageRedirects - elif settings.DEFAULT_FILE_STORAGE == "storages.backends.s3boto3.S3Boto3Storage": + elif domain.storage_class == "storages.backends.s3boto3.S3Boto3Storage": self.redirects_class = S3StorageRedirects - elif settings.DEFAULT_FILE_STORAGE == "storages.backends.azure_storage.AzureStorage": + elif domain.storage_class == "storages.backends.azure_storage.AzureStorage": self.redirects_class = AzureStorageRedirects else: raise NotImplementedError() @@ -1195,6 +1195,8 @@ def put(self, request, path, pk=None): """ Responds with the actual manifest """ + domain = get_domain() + storage = domain.get_storage() # iterate over all the layers and create chunk = request.META["wsgi.input"] artifact = self.receive_artifact(chunk) diff --git a/pulp_container/app/tasks/sign.py b/pulp_container/app/tasks/sign.py index 958c80b8b..94d54bca4 100644 --- a/pulp_container/app/tasks/sign.py +++ b/pulp_container/app/tasks/sign.py @@ -8,6 +8,7 @@ from django.db.models import Q from pulpcore.plugin.models import Repository +from pulpcore.plugin.util import get_domain from pulp_container.app.models import ( ManifestSignature, @@ -106,7 +107,7 @@ async def create_signature(manifest, reference, signing_service): if not manifest.data: # TODO: BACKWARD COMPATIBILITY - remove after fully migrating to artifactless manifest artifact = await manifest._artifacts.aget() - if settings.DEFAULT_FILE_STORAGE != "pulpcore.app.models.storage.FileSystem": + if get_domain().storage_class != "pulpcore.app.models.storage.FileSystem": async with tempfile.NamedTemporaryFile(dir=".", mode="wb", delete=False) as tf: await tf.write(await sync_to_async(artifact.file.read)()) await tf.flush() diff --git a/pulp_container/app/utils.py b/pulp_container/app/utils.py index 746649d19..75853c7a0 100644 --- a/pulp_container/app/utils.py +++ b/pulp_container/app/utils.py @@ -10,7 +10,7 @@ from asgiref.sync import sync_to_async from jsonschema import Draft7Validator, validate, ValidationError -from django.core.files.storage import default_storage as storage +from pulpcore.plugin.util import get_domain from django.db import IntegrityError from functools import partial from rest_framework.exceptions import Throttled @@ -310,6 +310,7 @@ async def save_artifact(artifact_attributes): def get_content_data(saved_artifact): + storage = get_domain().get_storage() with storage.open(saved_artifact.file.name, mode="rb") as file: raw_data = file.read() content_data = json.loads(raw_data) diff --git a/pulp_container/tests/functional/api/test_content_cache.py b/pulp_container/tests/functional/api/test_content_cache.py index d3bf14204..0c013f958 100644 --- a/pulp_container/tests/functional/api/test_content_cache.py +++ b/pulp_container/tests/functional/api/test_content_cache.py @@ -30,13 +30,7 @@ from pulp_container.constants import MEDIA_TYPE -STANDARD_FILE_STORAGE_FRAMEWORKS = [ - "django.core.files.storage.FileSystemStorage", - "pulpcore.app.models.storage.FileSystem", -] - cli_client = cli.Client(config.get_config()) -DEFAULT_FILE_STORAGE = utils.get_pulp_setting(cli_client, "DEFAULT_FILE_STORAGE") CACHE_ENABLED = utils.get_pulp_setting(cli_client, "CACHE_ENABLED") PULP_CONTENT_HOST_BASE_URL = config.get_config().get_base_url() @@ -190,13 +184,10 @@ def check_cache(self, url, expected_metadata, headers): def fetch_response_metadata(self, response): """Retrieve metadata from the passed response and normalize status code for redirects.""" - if DEFAULT_FILE_STORAGE in STANDARD_FILE_STORAGE_FRAMEWORKS: - return response.status_code, response.headers.get("X-PULP-CACHE") - else: - if response.history: - response = response.history[0] - response.status_code = 200 - return response.status_code, response.headers.get("X-PULP-CACHE") + if response.history: + response = response.history[0] + response.status_code = 200 + return response.status_code, response.headers.get("X-PULP-CACHE") def cache_status_first_func(i): diff --git a/template_config.yml b/template_config.yml index b561a6cad..b18705d7b 100644 --- a/template_config.yml +++ b/template_config.yml @@ -2,18 +2,21 @@ # were not present before running plugin-template have been added with their default values. # generated with plugin_template +# +# After editing this file please always reapply the plugin template before committing any changes. -api_root: /pulp/ +--- +api_root: "/pulp/" black: true check_commit_message: true check_gettext: true check_manifest: true check_stray_pulpcore_imports: true -ci_base_image: ghcr.io/pulp/pulp-ci-centos9 +ci_base_image: "ghcr.io/pulp/pulp-ci-centos9" ci_env: {} -ci_trigger: '{pull_request: {branches: [''*'']}}' -cli_package: pulp-cli -cli_repo: https://github.com/pulp/pulp-cli.git +ci_trigger: "{pull_request: {branches: ['*']}}" +cli_package: "pulp-cli" +cli_repo: "https://github.com/pulp/pulp-cli.git" core_import_allowed: [] deploy_client_to_pypi: true deploy_client_to_rubygems: true @@ -23,54 +26,81 @@ docker_fixtures: false extra_files: [] flake8: true flake8_ignore: [] -github_org: pulp -latest_release_branch: '2.21' +github_org: "pulp" +latest_release_branch: "2.21" lint_requirements: true os_required_packages: [] parallel_test_workers: 8 -plugin_app_label: container -plugin_default_branch: main -plugin_name: pulp_container +plugin_app_label: "container" +plugin_default_branch: "main" +plugin_name: "pulp_container" plugins: -- app_label: container - name: pulp_container -post_job_template: null -pre_job_template: null + - app_label: "container" + name: "pulp_container" pulp_env: {} pulp_env_azure: {} pulp_env_gcp: {} pulp_env_s3: {} -pulp_scheme: https +pulp_scheme: "https" pulp_settings: allowed_content_checksums: - - sha1 - - sha224 - - sha256 - - sha384 - - sha512 + - "sha1" + - "sha224" + - "sha256" + - "sha384" + - "sha512" allowed_export_paths: - - /tmp + - "/tmp" allowed_import_paths: - - /tmp + - "/tmp" flatpak_index: true pulp_settings_azure: + MEDIA_ROOT: "" + STORAGES: + default: + BACKEND: "storages.backends.azure_storage.AzureStorage" + OPTIONS: + account_key: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + account_name: "devstoreaccount1" + azure_container: "pulp-test" + connection_string: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://ci-azurite:10000/devstoreaccount1;" + expiration_secs: 120 + location: "pulp3" + overwrite_files: true + staticfiles: + BACKEND: "django.contrib.staticfiles.storage.StaticFilesStorage" flatpak_index: true pulp_settings_gcp: null pulp_settings_s3: + MEDIA_ROOT: "" + STORAGES: + default: + BACKEND: "storages.backends.s3boto3.S3Boto3Storage" + OPTIONS: + access_key: "AKIAIT2Z5TDYPX3ARJBA" + addressing_style: "path" + bucket_name: "pulp3" + default_acl: "@none" + endpoint_url: "http://minio:9000" + region_name: "eu-central-1" + secret_key: "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS" + signature_version: "s3v4" + staticfiles: + BACKEND: "django.contrib.staticfiles.storage.StaticFilesStorage" flatpak_index: false token_auth_disabled: true pydocstyle: true -release_email: pulp-infra@redhat.com -release_user: pulpbot +release_email: "pulp-infra@redhat.com" +release_user: "pulpbot" stalebot: true stalebot_days_until_close: 30 stalebot_days_until_stale: 90 stalebot_limit_to_pulls: true supported_release_branches: -- '2.14' -- '2.15' -- '2.16' -- '2.20' + - "2.14" + - "2.15" + - "2.16" + - "2.20" sync_ci: true test_azure: true test_cli: true @@ -81,4 +111,5 @@ test_performance: false test_reroute: true test_s3: true use_issue_template: true +...