Update CI files

Author: pulpbot

.ci/ansible/Containerfile.j2

@@ -44,4 +44,8 @@ RUN PULP_STATIC_ROOT=/var/lib/operator/static/ PULP_CONTENT_ORIGIN=localhost \
 RUN mkdir /var/lib/pulp/.config
 USER root:root
 
+# Lots of plugins try to use this path, and throw warnings if they cannot access it.
+RUN mkdir /.pytest_cache
+RUN chown pulp:pulp /.pytest_cache
+
 ENTRYPOINT ["/init"]

.ci/ansible/build_container.yaml

@@ -1,6 +1,7 @@
 # Ansible playbook to create the pulp service containers image
 ---
-- hosts: "localhost"
+- name: "Build CI Container Image"
+  hosts: "localhost"
   gather_facts: false
   vars_files:
     - "vars/main.yaml"
@@ -9,6 +10,7 @@
       ansible.builtin.template:
         src: "Containerfile.j2"
         dest: "Containerfile"
+
     - name: "Build pulp image"
       # We build from the ../.. (parent dir of pulpcore git repo) Docker build
       # "context" so that repos like pulp-smash are accessible to Docker
@@ -18,9 +20,20 @@
       # 1-off-builds and CI purposes (which has no cache across CI runs.)
       # Run build.yaml with -e cache=false if your builds are using outdated
       # layers.
-      command: "docker build --network host --no-cache={{ not cache | default(true) | bool }} -t {{ image.name }}:{{ image.tag }} -f {{ playbook_dir }}/Containerfile ../../.."
+      ansible.builtin.command:
+        argv:
+          - "docker"
+          - "build"
+          - "--network"
+          - "host"
+          - "--no-cache={{ not cache | default(true) | bool }}"
+          - "-t"
+          - "{{ image.name }}:{{ image.tag }}"
+          - "-f"
+          - "{{ playbook_dir }}/Containerfile"
+          - "../../.."
 
     - name: "Clean image cache"
-      docker_prune:
-        images : true
+      community.docker.docker_prune:
+        images: true
 ...

.ci/ansible/settings.py.j2

@@ -10,10 +10,6 @@ REDIS_HOST = "localhost"
 REDIS_PORT = 6379
 ANALYTICS = False
 
-{% if api_root is defined %}
-API_ROOT = {{ api_root | repr }}
-{% endif %}
-
 {% if pulp_settings %}
 {% for key, value in pulp_settings.items() %}
 {{ key | upper }} = {{ value | repr }}

.ci/ansible/start_container.yaml

@@ -1,6 +1,7 @@
 # Ansible playbook to start the pulp service container and its supporting services
 ---
-- hosts: "localhost"
+- name: "Start CI Containers"
+  hosts: "localhost"
   gather_facts: false
   vars_files:
     - "vars/main.yaml"
@@ -14,16 +15,16 @@
         - "settings"
 
     - name: "Generate Pulp Settings"
-      template:
+      ansible.builtin.template:
         src: "settings.py.j2"
         dest: "settings/settings.py"
 
     - name: "Setup docker networking"
-      docker_network:
+      community.docker.docker_network:
         name: "pulp_ci_bridge"
 
     - name: "Start Service Containers"
-      docker_container:
+      community.docker.docker_container:
         name: "{{ item.name }}"
         image: "{{ item.image }}"
         auto_remove: true
@@ -39,12 +40,12 @@
       loop: "{{ services | default([]) }}"
 
     - name: "Retrieve Docker Network Info"
-      docker_network_info:
+      community.docker.docker_network_info:
         name: "pulp_ci_bridge"
       register: "pulp_ci_bridge_info"
 
     - name: "Update /etc/hosts"
-      lineinfile:
+      ansible.builtin.lineinfile:
         path: "/etc/hosts"
         regexp: "\\s{{ item.value.Name }}\\s*$"
         line: "{{ item.value.IPv4Address | ansible.utils.ipaddr('address') }}\t{{ item.value.Name }}"
@@ -55,16 +56,27 @@
       amazon.aws.s3_bucket:
         aws_access_key: "{{ minio_access_key }}"
         aws_secret_key: "{{ minio_secret_key }}"
-        s3_url: "http://minio:9000"
+        endpoint_url: "http://minio:9000"
         region: "eu-central-1"
         name: "pulp3"
         state: "present"
       when: "s3_test | default(false)"
 
-    - block:
+    - name: "Wait on Services"
+      block:
+        - name: "Wait for azurite"
+          ansible.builtin.uri:
+            url: "http://ci-azurite:10000/"
+            status_code:
+              - 200
+              - 400
+          when: "azure_test | default(false)"
+          retries: 2
+          delay: 5
+
         - name: "Wait for Pulp"
-          uri:
-            url: "http://pulp{{ lookup('env', 'PULP_API_ROOT') | default('\/pulp\/', True) }}api/v3/status/"
+          ansible.builtin.uri:
+            url: "http://pulp{{ pulp_scenario_settings.api_root | default(pulp_settings.api_root | default('\/pulp\/', True), True) }}api/v3/status/"
             follow_redirects: "all"
             validate_certs: "no"
           register: "result"
@@ -73,11 +85,12 @@
           delay: 5
       rescue:
         - name: "Output pulp container log"
-          command: "docker logs pulp"
+          ansible.builtin.command:
+            cmd: "docker logs pulp"
           failed_when: true
 
     - name: "Check version of component being tested"
-      assert:
+      ansible.builtin.assert:
         that:
           - "(result.json.versions | items2dict(key_name='component', value_name='version'))[item.app_label] | canonical_semver == (component_version | canonical_semver)"
         fail_msg: |
@@ -86,14 +99,15 @@
       loop: "{{ 'plugins' | ansible.builtin.extract(lookup('ansible.builtin.file', '../../template_config.yml') | from_yaml) }}"
 
     - name: "Set pulp password in .netrc"
-      copy:
+      ansible.builtin.copy:
         dest: "~/.netrc"
         content: |
           machine pulp
           login admin
           password password
 
-- hosts: "pulp"
+- name: "Prepare Pulp Application Container"
+  hosts: "pulp"
   gather_facts: false
   tasks:
     - name: "Create directory for pulp-smash config"
@@ -108,6 +122,6 @@
         dest: "/var/lib/pulp/.config/pulp_smash/settings.json"
 
     - name: "Set pulp admin password"
-      command:
+      ansible.builtin.command:
         cmd: "pulpcore-manager reset-admin-password --password password"
 ...

.github/workflows/scripts/before_install.sh

@@ -29,7 +29,7 @@ if [ "$TEST" = "s3" ]; then
   COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[s3] git+https://github.com/gerrod3/botocore.git@fix-100-continue"
 fi
 if [ "$TEST" = "azure" ]; then
-  COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[azure]"
+  COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[azure,uvloop]"
 fi
 
 if [[ "$TEST" = "pulp" ]]; then
@@ -39,10 +39,6 @@ if [[ "$TEST" = "lowerbounds" ]]; then
   python3 .ci/scripts/calc_constraints.py requirements.txt > lowerbounds_constraints.txt
 fi
 
-export PULP_API_ROOT=$(test "${TEST}" = "s3" && echo "/rerouted/djnd/" || echo "/pulp/")
-
-echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV"
-
 # Compose the scenario definition.
 mkdir -p .ci/ansible/vars
 
@@ -54,9 +50,8 @@ legacy_component_name: "pulp_container"
 component_name: "container"
 component_version: "${COMPONENT_VERSION}"
 pulp_env: {}
-pulp_settings: {"allowed_content_checksums": ["sha1", "sha224", "sha256", "sha384", "sha512"], "allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"], "flatpak_index": true}
+pulp_settings: {"allowed_content_checksums": ["sha1", "sha224", "sha256", "sha384", "sha512"], "allowed_export_paths": ["/tmp"], "allowed_import_paths": ["/tmp"], "api_root": "/pulp/", "flatpak_index": true}
 pulp_scheme: "https"
-api_root: "${PULP_API_ROOT}"
 image:
   name: "pulp"
   tag: "ci_build"
@@ -93,7 +88,7 @@ if [ "$TEST" = "s3" ]; then
 s3_test: true
 minio_access_key: "${MINIO_ACCESS_KEY}"
 minio_secret_key: "${MINIO_SECRET_KEY}"
-pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "flatpak_index": false, "token_auth_disabled": true}
+pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "api_root": "/rerouted/djnd/", "flatpak_index": false, "token_auth_disabled": true}
 pulp_scenario_env: {}
 VARSYAML
 fi

.github/workflows/scripts/install.sh

@@ -15,15 +15,15 @@ REPO_ROOT="$PWD"
 
 source .github/workflows/scripts/utils.sh
 
-PIP_REQUIREMENTS=("pulp-cli")
+PIP_REQUIREMENTS=("pulp-cli" "yq")
 
 # This must be the **only** call to "pip install" on the test runner.
 pip install ${PIP_REQUIREMENTS[*]}
 
 if [[ "$TEST" = "s3" ]]; then
 for i in {1..3}
 do
-  ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3
+  ansible-galaxy collection install "amazon.aws:11.1.0" && s=0 && break || s=$? && sleep 3
 done
 if [[ $s -gt 0 ]]
 then
@@ -36,18 +36,15 @@ fi
 PULP_CLI_VERSION="$(pip freeze | sed -n -e 's/pulp-cli==//p')"
 git clone --depth 1 --branch "$PULP_CLI_VERSION" https://github.com/pulp/pulp-cli.git ../pulp-cli
 
-cd .ci/ansible/
+PULP_API_ROOT="$(yq -r '.pulp_scenario_settings.api_root // .pulp_settings.api_root // "/pulp/"' < .ci/ansible/vars/main.yaml)"
 
 pulp config create --base-url https://pulp --api-root "${PULP_API_ROOT}" --username "admin" --password "password"
-cp ~/.config/pulp/cli.toml "${REPO_ROOT}/../pulp-cli/tests/cli.toml"
+cp ~/.config/pulp/cli.toml "../pulp-cli/tests/cli.toml"
+
+cd .ci/ansible/
 
 ansible-playbook build_container.yaml
 ansible-playbook start_container.yaml
-
-# Plugins often write to ~/.config/pulp/cli.toml from the host
-chmod 777 ~/.config/pulp
-chmod 666 ~/.config/pulp/cli.toml
-sudo chown -R 700:700 ~/.config
 echo ::group::SSL
 # Copy pulp CA
 sudo docker cp pulp:/etc/pulp/certs/pulp_webserver.crt /usr/local/share/ca-certificates/pulp_webserver.crt
@@ -72,7 +69,3 @@ fi
 # Needed for some functional tests
 cmd_prefix bash -c "echo '%wheel        ALL=(ALL)       NOPASSWD: ALL' > /etc/sudoers.d/nopasswd"
 cmd_prefix bash -c "usermod -a -G wheel pulp"
-
-# Lots of plugins try to use this path, and throw warnings if they cannot access it.
-cmd_prefix mkdir /.pytest_cache
-cmd_prefix chown pulp:pulp /.pytest_cache

.github/workflows/scripts/release.sh

@@ -24,4 +24,5 @@ towncrier build --yes --version "${NEW_VERSION}"
 bump-my-version bump release --commit --message "Release {new_version}" --tag --tag-name "{new_version}" --tag-message "Release {new_version}" --allow-dirty
 bump-my-version bump patch --commit
 
-git push origin "${BRANCH}" "${NEW_VERSION}"
+# Git push is not atomic by default!
+git push --atomic origin "${BRANCH}" "${NEW_VERSION}"

.github/workflows/test.yml

@@ -134,7 +134,7 @@ jobs:
         if: always()
         run: |
           echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate"
-          http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true
+          pulp status || true
           docker images || true
           docker ps -a || true
           docker logs pulp || true

.gitleaks.toml

@@ -1,10 +1,7 @@
 [allowlist]
-  description = "Our test install exports a test only MINIO ACCESS KEY"
+  description = "Allow specific test-keys."
   paths = [
     ".github/workflows/scripts/install.sh",
     ".travis/install.sh",
   ]
-  regexes = [
-    "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IkdNMkQ6SU9CVDpHQVpEOk1aUlE6RzQyVzpDWkJaOkdWUlQ6R00zRzpNRTJUOlFNSlk6R1JURDpNTUpRIn0.eyJhY2Nlc3MiOlt7InR5cGUiOiIiLCJuYW1lIjoiIiwiYWN0aW9ucyI6W119XSwiYXVkIjoicHVscDMtc291cmNlLWZlZG9yYTMxLmxvY2FsaG9zdC5leGFtcGxlLmNvbSIsImV4cCI6MTU5NDYzNDU0NSwiaWF0IjoxNTk0NjM0MjQ1LCJpc3MiOiJodHRwOi8vbG9jYWxob3N0OjI0ODE3L3Rva2VuLyIsImp0aSI6ImU4ZTUyYzVhLWYxMzAtNGJlMi1iNjFhLTUwNzVhMjhkMTA0YSIsIm5iZiI6MTU5NDYzNDI0NSwic3ViIjoiIn0.ySDUHooaURbsyKLkHoXqA1JJPwlcDtpz_u6GgcqA8fmFGmSWJFlAGYtA2GLXDzPioH-bh1JkMJdBDs61c5JnFw",
-    "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IkhBM1Q6SVlSUjpHUTNUOklPTEM6TVE0RzpFT0xDOkdGUVQ6QVpURTpHQlNXOkNaUlY6TUlZVzpLTkpWIn0.eyJhY2Nlc3MiOlt7InR5cGUiOiIiLCJuYW1lIjoiIiwiYWN0aW9ucyI6W119XSwiYXVkIjoibG9jYWxob3N0OjI0ODE2IiwiZXhwIjoxNTcxMDcxOTUzLCJpYXQiOjE1NzEwNzE2NTMsImlzcyI6ImxvY2FsaG9zdDoyNDgxNi90b2tlbiIsImp0aSI6IjRmYTliYTYwLTY0ZTUtNDA3MC1hMzMyLWZmZTRlMTk2YzVjNyIsIm5iZiI6MTU3MTA3MTY1Mywic3ViIjoiIn0.pirj8yhbjYnldxmZ-jIZ72VJrzxkAnwLXLu1ND9QAL-kl3gZrvPbp98w2xdhEoQ_7WEka4veb6uU5ZzmD87X1Q",
-  ]
+  regexes = ['''AKIAIT2Z5TDYPX3ARJBA''', '''Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw''', "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IkdNMkQ6SU9CVDpHQVpEOk1aUlE6RzQyVzpDWkJaOkdWUlQ6R00zRzpNRTJUOlFNSlk6R1JURDpNTUpRIn0.eyJhY2Nlc3MiOlt7InR5cGUiOiIiLCJuYW1lIjoiIiwiYWN0aW9ucyI6W119XSwiYXVkIjoicHVscDMtc291cmNlLWZlZG9yYTMxLmxvY2FsaG9zdC5leGFtcGxlLmNvbSIsImV4cCI6MTU5NDYzNDU0NSwiaWF0IjoxNTk0NjM0MjQ1LCJpc3MiOiJodHRwOi8vbG9jYWxob3N0OjI0ODE3L3Rva2VuLyIsImp0aSI6ImU4ZTUyYzVhLWYxMzAtNGJlMi1iNjFhLTUwNzVhMjhkMTA0YSIsIm5iZiI6MTU5NDYzNDI0NSwic3ViIjoiIn0.ySDUHooaURbsyKLkHoXqA1JJPwlcDtpz_u6GgcqA8fmFGmSWJFlAGYtA2GLXDzPioH-bh1JkMJdBDs61c5JnFw", "eyJ0eXAiOiJKV1QiLCJhbGciOiJFUzI1NiIsImtpZCI6IkhBM1Q6SVlSUjpHUTNUOklPTEM6TVE0RzpFT0xDOkdGUVQ6QVpURTpHQlNXOkNaUlY6TUlZVzpLTkpWIn0.eyJhY2Nlc3MiOlt7InR5cGUiOiIiLCJuYW1lIjoiIiwiYWN0aW9ucyI6W119XSwiYXVkIjoibG9jYWxob3N0OjI0ODE2IiwiZXhwIjoxNTcxMDcxOTUzLCJpYXQiOjE1NzEwNzE2NTMsImlzcyI6ImxvY2FsaG9zdDoyNDgxNi90b2tlbiIsImp0aSI6IjRmYTliYTYwLTY0ZTUtNDA3MC1hMzMyLWZmZTRlMTk2YzVjNyIsIm5iZiI6MTU3MTA3MTY1Mywic3ViIjoiIn0.pirj8yhbjYnldxmZ-jIZ72VJrzxkAnwLXLu1ND9QAL-kl3gZrvPbp98w2xdhEoQ_7WEka4veb6uU5ZzmD87X1Q", '''fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS''', '''qR\+vjWPU50fCqQuUWbj9Fain/j2pV\+ZtBCiDiieS''']

template_config.yml

@@ -6,7 +6,6 @@
 # After editing this file please always reapply the plugin template before committing any changes.
 
 ---
-api_root: "/pulp/"
 black: true
 check_commit_message: true
 check_gettext: true
@@ -53,6 +52,7 @@ pulp_settings:
     - "/tmp"
   allowed_import_paths:
     - "/tmp"
+  api_root: "/pulp/"
   flatpak_index: true
 pulp_settings_azure:
   MEDIA_ROOT: ""
@@ -87,6 +87,7 @@ pulp_settings_s3:
         signature_version: "s3v4"
     staticfiles:
       BACKEND: "django.contrib.staticfiles.storage.StaticFilesStorage"
+  api_root: "/rerouted/djnd/"
   flatpak_index: false
   token_auth_disabled: true
 pydocstyle: true