From f8da9f04f3b38b78bf7a77027becc62df7a9c64d Mon Sep 17 00:00:00 2001 From: Maxim Sava Date: Mon, 21 Jul 2025 09:08:54 +0300 Subject: [PATCH] [multiple] Discover and configure Glance with Ceph RGW secrets This patch enhances the cifmw_ceph_client/cifmw_cephadm roles to: - Automatically discover Ceph RGW (RADOS Gateway) endpoint and credentials - Create Glance secrets using the discovered RGW settings - Configure glance with Ceph RGW secrets - Creating a dedicated 'glance' S3 user in RGW (cifmw_cephadm role) This integration configures Glance with Ceph RGW access credentials, enabling it to use RGW as an Glance backend when object storage is enabled Changes: - Add RGW discovery playbook - Add logic to create Glance secrets with RGW config Signed-off-by: Maxim Sava --- hooks/playbooks/ceph.yml | 5 + .../tasks/glance-rgw-discovery.yaml | 60 +++++++++ roles/cifmw_ceph_client/tasks/main.yml | 6 + .../templates/k8s_ceph_rgw_glance_secret.j2 | 13 ++ roles/cifmw_cephadm/README.md | 3 + roles/cifmw_cephadm/defaults/main.yml | 1 + roles/cifmw_cephadm/tasks/glance_s3_info.yaml | 124 ++++++++++++++++++ .../templates/ceph_client.yaml.j2 | 4 + 8 files changed, 216 insertions(+) create mode 100644 roles/cifmw_ceph_client/tasks/glance-rgw-discovery.yaml create mode 100644 roles/cifmw_ceph_client/templates/k8s_ceph_rgw_glance_secret.j2 create mode 100644 roles/cifmw_cephadm/tasks/glance_s3_info.yaml diff --git a/hooks/playbooks/ceph.yml b/hooks/playbooks/ceph.yml index be0db4ed18..63de7c9ffc 100644 --- a/hooks/playbooks/ceph.yml +++ b/hooks/playbooks/ceph.yml @@ -473,6 +473,11 @@ name: cifmw_cephadm tasks_from: rbd_mirror + - name: Create RGW S3 user and fetch S3 user info + ansible.builtin.import_role: + name: cifmw_cephadm + tasks_from: glance_s3_info + - name: Create Cephx Keys for OpenStack ansible.builtin.import_role: name: cifmw_cephadm diff --git a/roles/cifmw_ceph_client/tasks/glance-rgw-discovery.yaml b/roles/cifmw_ceph_client/tasks/glance-rgw-discovery.yaml new file mode 100644 index 0000000000..f5a57782f5 --- /dev/null +++ b/roles/cifmw_ceph_client/tasks/glance-rgw-discovery.yaml @@ -0,0 +1,60 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Set fact for glance S3 backend secrets + ansible.builtin.set_fact: + ceph_s3_access_key: "{{ cifmw_ceph_client_s3_glance_user['keys'][0]['access_key'] }}" + ceph_s3_secret_key: "{{ cifmw_ceph_client_s3_glance_user['keys'][0]['secret_key'] }}" + ceph_s3_bucket: "glance-s3-bucket" + ceph_s3_endpoint: "{{ cifmw_ceph_client_s3_internal_swift_url }}" + when: + - cifmw_ceph_client_s3_glance_user is defined + - cifmw_ceph_client_s3_internal_swift_url is defined + no_log: true + +- name: Generate Glance S3 backend secrets using Ceph RGW credentials + ansible.builtin.template: + src: templates/k8s_ceph_rgw_glance_secret.j2 + dest: k8s_ceph_rgw_glance_secret.yaml + mode: "0600" + force: true + when: + - ceph_s3_access_key is defined + - ceph_s3_secret_key is defined + no_log: true + register: reg_k8s_ceph_rgw_glance_secret + +- name: Create glance S3 backend secrets + kubernetes.core.k8s: + kubeconfig: "{{ cifmw_openshift_kubeconfig }}" + api_key: "{{ cifmw_openshift_token | default(omit)}}" + context: "{{ cifmw_openshift_context | default(omit) }}" + namespace: openstack + state: present + apply: true + force: true + src: k8s_ceph_rgw_glance_secret.yaml + register: k8s_glance_s3_secret + delegate_to: localhost + no_log: true + when: reg_k8s_ceph_rgw_glance_secret is not skipped + +- name: Delete the rendered secret file + ansible.builtin.file: + path: k8s_ceph_rgw_glance_secret.yaml + state: absent + delegate_to: localhost + when: k8s_glance_s3_secret | default(true) diff --git a/roles/cifmw_ceph_client/tasks/main.yml b/roles/cifmw_ceph_client/tasks/main.yml index bae1384d4e..b2c5952112 100644 --- a/roles/cifmw_ceph_client/tasks/main.yml +++ b/roles/cifmw_ceph_client/tasks/main.yml @@ -78,6 +78,12 @@ mode: "0600" force: true +- name: Import glance Ceph RGW discovery playbook + when: + - cifmw_ceph_client_s3_glance_user is defined + - cifmw_ceph_client_s3_internal_swift_url is defined + ansible.builtin.include_tasks: glance-rgw-discovery.yaml + - name: Create edpm-values-post-ceph ConfigMap if sample path provided ansible.builtin.include_tasks: edpm_values_post_ceph.yml when: diff --git a/roles/cifmw_ceph_client/templates/k8s_ceph_rgw_glance_secret.j2 b/roles/cifmw_ceph_client/templates/k8s_ceph_rgw_glance_secret.j2 new file mode 100644 index 0000000000..73ac51088c --- /dev/null +++ b/roles/cifmw_ceph_client/templates/k8s_ceph_rgw_glance_secret.j2 @@ -0,0 +1,13 @@ +# Glance secrets used to configure glance with ceph S3 backend +apiVersion: v1 +kind: Secret +metadata: + name: s3glance +stringData: + s3glance.conf: | + [default_backend] + s3_store_host = {{ ceph_s3_endpoint }} + s3_store_access_key = {{ ceph_s3_access_key }} + s3_store_secret_key = {{ ceph_s3_secret_key }} + s3_store_bucket = {{ ceph_s3_bucket }} + s3_store_cacert = "/etc/pki/tls/certs/ca-bundle.crt" diff --git a/roles/cifmw_cephadm/README.md b/roles/cifmw_cephadm/README.md index 368af7ca40..3892939922 100644 --- a/roles/cifmw_cephadm/README.md +++ b/roles/cifmw_cephadm/README.md @@ -108,6 +108,9 @@ that they do not need to be changed for a typical EDPM deployment. Squid). Set it to false if the target Ceph release is equal to or greater than Tentacle. +* `cifmw_cephadm_rgw_s3_glance`: (Bool) If this is value is `true`, then + cephadm will create glance secrets using the discovered RGW settings + Use the `cifmw_cephadm_pools` list of dictionaries to define pools for Nova (vms), Cinder (volumes), Cinder-backups (backups), and Glance (images). ``` diff --git a/roles/cifmw_cephadm/defaults/main.yml b/roles/cifmw_cephadm/defaults/main.yml index c407da07a3..47084ade15 100644 --- a/roles/cifmw_cephadm/defaults/main.yml +++ b/roles/cifmw_cephadm/defaults/main.yml @@ -156,3 +156,4 @@ cifmw_cephadm_rgw_ingress_service_id: "rgw.default" # set ssl_backward compatibily to False if ceph version is equal or greater # than Tentacle cifmw_rgw_ssl_backward_compatibility: true +cifmw_cephadm_rgw_s3_glance: false diff --git a/roles/cifmw_cephadm/tasks/glance_s3_info.yaml b/roles/cifmw_cephadm/tasks/glance_s3_info.yaml new file mode 100644 index 0000000000..e77eb8c208 --- /dev/null +++ b/roles/cifmw_cephadm/tasks/glance_s3_info.yaml @@ -0,0 +1,124 @@ +--- + # Copyright 2025 Red Hat, Inc. + # All Rights Reserved. + # + # Licensed under the Apache License, Version 2.0 (the "License"); you may + # not use this file except in compliance with the License. You may obtain + # a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + # License for the specific language governing permissions and limitations + # under the License. + +- name: Create glance S3 RGW user and fetch object-store endpoint + when: cifmw_cephadm_rgw_s3_glance | default(false) | bool + block: + - name: Refresh ceph_cli + ansible.builtin.include_tasks: ceph_cli.yml + vars: + ceph_command: "radosgw-admin" + + - name: Wait for RGW daemons to be running + ansible.builtin.command: + cmd: "{{ cifmw_cephadm_ceph_cli }} orch ps --daemon-type rgw --format json" + register: rgw_daemon_status + become: true + changed_when: false + failed_when: false + until: > + rgw_daemon_status.rc == 0 and + (rgw_daemon_status.stdout | default('[]') | from_json | default([], true)) | + selectattr('status_desc', 'equalto', 'running') | + list | length > 0 + retries: 30 + delay: 10 + + - name: Check if Ceph S3 glance user exists + ansible.builtin.command: + cmd: >- + {{ cifmw_cephadm_ceph_cli }} user info --uid glance + no_log: true + become: true + ignore_errors: true + changed_when: false + register: cifmw_ceph_s3_glance_user_check + when: cifmw_cephadm_ceph_cli is defined + + - name: Create Ceph S3 glance user + ansible.builtin.command: + cmd: >- + {{ cifmw_cephadm_ceph_cli }} user create + --uid="glance" + --display-name="Glance S3 User" + become: true + when: + - cifmw_ceph_s3_glance_user_check is not skipped + - cifmw_ceph_s3_glance_user_check.rc != 0 + register: glance_rgw_user_create + retries: 3 + delay: 5 + until: glance_rgw_user_create.rc == 0 + + - name: Get RGW glance user info + ansible.builtin.command: + cmd: >- + {{ cifmw_cephadm_ceph_cli }} user info --uid="glance" + become: true + no_log: true + changed_when: false + register: ceph_rgw_glance_user_info + + - name: Show RGW daemon status on failure + when: ceph_rgw_glance_user_info.rc != 0 + block: + - name: Get RGW daemon status for debugging + ansible.builtin.command: + cmd: "{{ cifmw_cephadm_ceph_cli }} orch ps --daemon-type rgw" + become: true + register: rgw_debug_status + changed_when: false + + - name: Display RGW status + ansible.builtin.debug: + msg: "RGW daemon status: {{ rgw_debug_status.stdout }}" + + - name: Fail with context + ansible.builtin.fail: + msg: | + Failed to fetch glance user info after creation. + User creation result: {{ glance_rgw_user_create | default('not attempted') }} + RGW daemon status shown above. + + - name: Set facts RGW glance user + ansible.builtin.set_fact: + cifmw_ceph_s3_glance_user: "{{ ceph_rgw_glance_user_info.stdout | from_json }}" + when: + - ceph_rgw_glance_user_info is defined + - ceph_rgw_glance_user_info.rc == 0 + no_log: true + + - name: Get OpenStack internal object-store endpoints + cifmw.general.ci_script: + extra_args: + KUBECONFIG: "{{ cifmw_openshift_kubeconfig }}" + output_dir: "{{ cifmw_cephadm_basedir }}/artifacts" + script: |- + oc -n {{ cifmw_cephadm_ns }} rsh openstackclient openstack endpoint list --service object-store --interface internal -f value -c URL + delegate_to: localhost + register: object_store_endpoint_raw + when: + - cifmw_openshift_kubeconfig is defined + - cifmw_cephadm_basedir is defined + - cifmw_cephadm_rgw_s3_glance is defined + + - name: Extract base URL without path + ansible.builtin.set_fact: + rgw_internal_url: >- + {{ + object_store_endpoint_raw.stdout + | regex_replace('(/swift.*)$', '') + }} diff --git a/roles/cifmw_cephadm/templates/ceph_client.yaml.j2 b/roles/cifmw_cephadm/templates/ceph_client.yaml.j2 index 09f2b55dbb..c9df09dbe5 100644 --- a/roles/cifmw_cephadm/templates/ceph_client.yaml.j2 +++ b/roles/cifmw_cephadm/templates/ceph_client.yaml.j2 @@ -1,4 +1,8 @@ --- +{% if cifmw_ceph_s3_glance_user is defined and rgw_internal_url is defined %} +cifmw_ceph_client_s3_glance_user: {{ cifmw_ceph_s3_glance_user }} +cifmw_ceph_client_s3_internal_swift_url: {{ rgw_internal_url }} +{% endif %} cifmw_ceph_client_fsid: {{ cifmw_cephadm_fsid }} cifmw_ceph_client_cluster: {{ cifmw_cephadm_cluster }} cifmw_ceph_client_external_cluster_mon_ips: "{{ cifmw_ceph_client_external_cluster_mon_ips }}"