diff --git a/docs/dictionary/en-custom.txt b/docs/dictionary/en-custom.txt index 67aae5cf2..b6ff9720c 100644 --- a/docs/dictionary/en-custom.txt +++ b/docs/dictionary/en-custom.txt @@ -16,6 +16,7 @@ IdP Idempotency LDAP LLM +LVs MachineConfig Marjanovic Nemanja diff --git a/hooks/playbooks/ceph.yml b/hooks/playbooks/ceph.yml index 13f5ce610..8cac4c953 100644 --- a/hooks/playbooks/ceph.yml +++ b/hooks/playbooks/ceph.yml @@ -159,6 +159,9 @@ hostvars[_target]['ansible_ssh_private_key_file'] | default(lookup('env', 'ANSIBLE_SSH_PRIVATE_KEY')) }} + cifmw_block_device_thin_pool: "{{ cifmw_ceph_thin_pool | default('') }}" + cifmw_block_device_thin_lv_size: "{{ cifmw_ceph_thin_lv_size | default('50G') }}" + cifmw_block_device_thin_lv_name: "ceph_osd_{{ i }}" cifmw_block_device_image_file: /var/lib/ceph-osd-{{ i }}.img cifmw_block_device_loop: /dev/loop{{ i + 3 }} cifmw_block_lv_name: ceph_lv{{ i }} @@ -170,6 +173,17 @@ loop_var: i loop: "{{ range(0, cifmw_num_osds_perhost|int) }}" + - name: Build data_devices for ceph spec from cifmw_block_device outputs + ansible.builtin.set_fact: + cifmw_ceph_spec_data_devices: | + data_devices: + paths: + {% for p in cifmw_block_device_paths %} + - {{ p }} + {% endfor %} + delegate_to: localhost + delegate_facts: true + - name: Build Ceph spec and conf from gathered IPs of the target inventory group tags: spec hosts: localhost diff --git a/roles/cifmw_block_device/README.md b/roles/cifmw_block_device/README.md index 54f7eaafe..80ca68908 100644 --- a/roles/cifmw_block_device/README.md +++ b/roles/cifmw_block_device/README.md @@ -1,25 +1,31 @@ # cifmw_block_device -Creates a virtual block device with logical volumes. Useful for -deploying Ceph on a virtual machine which does not have any -block devices except for root. Creates a systemd unit so the -virtual block device comes back online during reboot. +Creates block devices with logical volumes for Ceph OSD testing. +Supports two modes: -The target system must have 7 GB of available disk space at minimum. +- **Loop mode** (default): creates a loop-backed file with LVM on top + and a systemd unit to restore it across reboots. Useful for VMs + that have no spare block devices. +- **Thin-pool mode**: creates thin-provisioned LVs from an existing + VG thin pool. Useful for bare-metal hosts that already have a + thin pool with available space. -This role will recreate the block device on each run. Thus, if there -is data on the block device from the previous run it will delete it. -The assumption is that the block device exists for testing and that -rebuilding the environment quickly is more important preserving any -test data. +The mode is selected by `cifmw_block_device_thin_pool`: when non-empty +the role uses thin-pool mode, otherwise loop mode. ## Privilege escalation -Requires root on the remote system to create loop back device and -systemd unit. +Requires root on the remote system to create devices and LVM objects. ## Parameters +### Common + +* `cifmw_block_device_thin_pool`: VG/pool path for thin-pool mode, + e.g. `vg/lv_thinpool`. When empty (default), loop mode is used. + +### Loop mode + * `cifmw_block_device_image_file`: Name of the `dd'd` image file (default `/var/lib/ceph-osd.img`) * `cifmw_block_device_size`: Size of the virtual block device (default @@ -34,8 +40,25 @@ systemd unit. restores the device on system startup (default `/etc/systemd/system/ceph-osd-losetup.service`) +### Thin-pool mode + +* `cifmw_block_device_thin_lv_size`: Size of each thin LV (default + `50G`) +* `cifmw_block_device_thin_lv_name`: Name of the thin LV to create + (default `ceph_osd`) + +## Output + +Both modes append to the `cifmw_block_device_paths` list fact. +Each role invocation adds one entry, so when called in a loop +the list accumulates all created device paths (e.g. +`["/dev/ceph_vg0/ceph_lv0", "/dev/ceph_vg1/ceph_lv1"]` or +`["/dev/vg/ceph_osd_0", "/dev/vg/ceph_osd_1"]`). + ## Examples +### Loop mode (default) + The following will create a 7 GB block device on the target system using the defaults above. ``` @@ -60,11 +83,30 @@ data_devices: paths: - /dev/ceph_vg/ceph_lv_data ``` -The following will stop and disable the systemd unit file which starts -the virtual block device, remove the logical volume, volume group, and -physical volume, and delete the loopback device and its backing file. + +### Thin-pool mode + +```yaml +- include_role: + name: cifmw_block_device + vars: + cifmw_block_device_thin_pool: "vg/lv_thinpool" + cifmw_block_device_thin_lv_size: "50G" + cifmw_block_device_thin_lv_name: "ceph_osd_0" +``` +Ceph spec entry: +```yaml +data_devices: + paths: + - /dev/vg/ceph_osd_0 ``` + +### Cleanup + +```yaml - import_role: name: cifmw_block_device tasks_from: cleanup ``` +For thin-pool mode, pass `cifmw_block_device_thin_pool` and +`cifmw_block_device_thin_lv_name` so the correct cleanup path runs. diff --git a/roles/cifmw_block_device/defaults/main.yml b/roles/cifmw_block_device/defaults/main.yml index aa2eb9764..d3bca17a2 100644 --- a/roles/cifmw_block_device/defaults/main.yml +++ b/roles/cifmw_block_device/defaults/main.yml @@ -18,9 +18,17 @@ # All variables intended for modification should be placed in this file. # All variables within this role should have a prefix of "cifmw_block_device" +# --- Loop-device mode (default) --- cifmw_block_device_image_file: /var/lib/ceph-osd.img cifmw_block_device_size: 7G cifmw_block_device_loop: /dev/loop3 cifmw_block_vg_name: ceph_vg cifmw_block_lv_name: ceph_lv_data cifmw_block_systemd_unit_file: /etc/systemd/system/ceph-osd-losetup.service + +# --- Thin-pool mode --- +# When non-empty, create thin-provisioned LVs from this existing thin pool +# instead of loop-backed files. Value is "VG/pool", e.g. "vg/lv_thinpool". +cifmw_block_device_thin_pool: "" +cifmw_block_device_thin_lv_size: "50G" +cifmw_block_device_thin_lv_name: "ceph_osd" diff --git a/roles/cifmw_block_device/tasks/cleanup.yml b/roles/cifmw_block_device/tasks/cleanup.yml index 721a74577..d9606d6d6 100644 --- a/roles/cifmw_block_device/tasks/cleanup.yml +++ b/roles/cifmw_block_device/tasks/cleanup.yml @@ -14,31 +14,47 @@ # License for the specific language governing permissions and limitations # under the License. -- name: Ensure ceph-osd-losetup is not running and disabled - tags: systemd +- name: Clean up thin-provisioned LV + when: cifmw_block_device_thin_pool | length > 0 become: true - ansible.builtin.systemd: - state: stopped - enabled: false - name: "{{ cifmw_block_systemd_unit_file | basename }}" + ansible.builtin.command: + cmd: >- + lvremove --force + /dev/{{ cifmw_block_device_thin_pool.split('/')[0] }}/{{ cifmw_block_device_thin_lv_name }} + register: _thin_cleanup + failed_when: + - _thin_cleanup.rc != 0 + - "'not found' not in (_thin_cleanup.stderr | default(''))" -- name: Remove unit file - become: true - ansible.builtin.file: - path: "{{ cifmw_block_systemd_unit_file }}" - state: absent +- name: Clean up loop-backed block device + when: cifmw_block_device_thin_pool | default('') | length == 0 + block: + - name: Ensure ceph-osd-losetup is not running and disabled + tags: systemd + become: true + ansible.builtin.systemd: + state: stopped + enabled: false + name: "{{ cifmw_block_systemd_unit_file | basename }}" -- name: Use {pv,vg,lv}remove to remove logical volume on loop device - become: true - ansible.builtin.shell: - cmd: |- - lvremove --force /dev/{{ cifmw_block_vg_name }}/{{ cifmw_block_lv_name }} - vgremove --force {{ cifmw_block_vg_name }} - pvremove --force {{ cifmw_block_device_loop }} - lvs + - name: Remove unit file + become: true + ansible.builtin.file: + path: "{{ cifmw_block_systemd_unit_file }}" + state: absent + + - name: Use {pv,vg,lv}remove to remove logical volume on loop device + become: true + ansible.builtin.shell: + cmd: |- + lvremove --force /dev/{{ cifmw_block_vg_name }}/{{ cifmw_block_lv_name }} + vgremove --force {{ cifmw_block_vg_name }} + pvremove --force {{ cifmw_block_device_loop }} + lvs -- name: Use losetup and rm to cremove the loop device and backing image file +- name: Use losetup and rm to remove the loop device and backing image file become: true + when: cifmw_block_device_thin_pool | length > 0 ansible.builtin.shell: cmd: |- losetup -d {{ cifmw_block_device_loop }} diff --git a/roles/cifmw_block_device/tasks/loop.yml b/roles/cifmw_block_device/tasks/loop.yml new file mode 100644 index 000000000..bc54afac5 --- /dev/null +++ b/roles/cifmw_block_device/tasks/loop.yml @@ -0,0 +1,74 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Install packages + become: true + ansible.builtin.dnf: + name: + - util-linux + - lvm2 + - jq + - podman + state: present + +- name: Stat loop device see if it already exists + ansible.builtin.stat: + path: "{{ cifmw_block_device_loop }}" + register: cifmw_block_device_loop_res + +- name: Log to syslog if loop device exists + community.general.syslogger: + msg: "Warning {{ cifmw_block_device_loop }} already exists" + when: cifmw_block_device_loop_res.stat.exists + +- name: Use dd and losetup to create the loop device + become: true + ansible.builtin.shell: + cmd: |- + dd if=/dev/zero of={{ cifmw_block_device_image_file }} bs=1 count=0 seek={{ cifmw_block_device_size }} + losetup {{ cifmw_block_device_loop }} {{ cifmw_block_device_image_file }} + lsblk + +- name: Use {pv,vg,lv}create to create logical volume on loop device + become: true + ansible.builtin.shell: + cmd: |- + pvcreate {{ cifmw_block_device_loop }} + vgcreate {{ cifmw_block_vg_name }} {{ cifmw_block_device_loop }} + lvcreate -n {{ cifmw_block_lv_name }} -l +100%FREE {{ cifmw_block_vg_name }} + lvs + +- name: Create a systemd service that restores the device on startup + become: true + ansible.builtin.template: + src: templates/ceph-osd-losetup.service.j2 + dest: "{{ cifmw_block_systemd_unit_file }}" + mode: "0644" + force: true + +- name: Ensure ceph-osd-losetup is running and enabled + become: true + tags: systemd + ansible.builtin.systemd: + state: started + enabled: true + name: "{{ cifmw_block_systemd_unit_file | regex_replace('/etc/systemd/system/', '') }}" + +- name: Collect created device path + ansible.builtin.set_fact: + cifmw_block_device_paths: >- + {{ (cifmw_block_device_paths | default([])) + + ['/dev/' ~ cifmw_block_vg_name ~ '/' ~ cifmw_block_lv_name] }} diff --git a/roles/cifmw_block_device/tasks/main.yml b/roles/cifmw_block_device/tasks/main.yml index 9d2e4ff97..9706a0c7b 100644 --- a/roles/cifmw_block_device/tasks/main.yml +++ b/roles/cifmw_block_device/tasks/main.yml @@ -14,55 +14,10 @@ # License for the specific language governing permissions and limitations # under the License. -- name: Install packages - become: true - ansible.builtin.dnf: - name: - - util-linux - - lvm2 - - jq - - podman - state: present +- name: Create block device using thin-pool LV + when: cifmw_block_device_thin_pool | length > 0 + ansible.builtin.include_tasks: thin.yml -- name: Stat loop device see if it already exists - ansible.builtin.stat: - path: "{{ cifmw_block_device_loop }}" - register: cifmw_block_device_loop_res - -- name: Log to syslog if loop device exists - community.general.syslogger: - msg: "Warning {{ cifmw_block_device_loop }} already exists" - when: cifmw_block_device_loop_res.stat.exists - -- name: Use dd and losetup to create the loop device - become: true - ansible.builtin.shell: - cmd: |- - dd if=/dev/zero of={{ cifmw_block_device_image_file }} bs=1 count=0 seek={{ cifmw_block_device_size }} - losetup {{ cifmw_block_device_loop }} {{ cifmw_block_device_image_file }} - lsblk - -- name: Use {pv,vg,lv}create to create logical volume on loop device - become: true - ansible.builtin.shell: - cmd: |- - pvcreate {{ cifmw_block_device_loop }} - vgcreate {{ cifmw_block_vg_name }} {{ cifmw_block_device_loop }} - lvcreate -n {{ cifmw_block_lv_name }} -l +100%FREE {{ cifmw_block_vg_name }} - lvs - -- name: Create a systemd service that restores the device on startup - become: true - ansible.builtin.template: - src: templates/ceph-osd-losetup.service.j2 - dest: "{{ cifmw_block_systemd_unit_file }}" - mode: "0644" - force: true - -- name: Ensure ceph-osd-losetup is running and enabled - become: true - tags: systemd - ansible.builtin.systemd: - state: started - enabled: true - name: "{{ cifmw_block_systemd_unit_file | regex_replace('/etc/systemd/system/', '') }}" +- name: Create block device using loop-backed file + when: cifmw_block_device_thin_pool | length == 0 + ansible.builtin.include_tasks: loop.yml diff --git a/roles/cifmw_block_device/tasks/thin.yml b/roles/cifmw_block_device/tasks/thin.yml new file mode 100644 index 000000000..16304fc40 --- /dev/null +++ b/roles/cifmw_block_device/tasks/thin.yml @@ -0,0 +1,42 @@ +--- +# Copyright Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +- name: Install lvm2 for thin-pool operations + become: true + ansible.builtin.dnf: + name: + - lvm2 + - jq + state: present + +- name: Create thin LV from existing thin pool + become: true + ansible.builtin.command: + cmd: >- + lvcreate -V {{ cifmw_block_device_thin_lv_size }} + --thin -n {{ cifmw_block_device_thin_lv_name }} + {{ cifmw_block_device_thin_pool }} + creates: "/dev/{{ cifmw_block_device_thin_pool.split('/')[0] }}/{{ cifmw_block_device_thin_lv_name }}" + register: _thin_lvcreate_result + failed_when: + - _thin_lvcreate_result.rc != 0 + - "'already exists' not in (_thin_lvcreate_result.stderr | default(''))" + +- name: Collect created device path + ansible.builtin.set_fact: + cifmw_block_device_paths: >- + {{ (cifmw_block_device_paths | default([])) + + ['/dev/' ~ cifmw_block_device_thin_pool.split('/')[0] ~ '/' ~ cifmw_block_device_thin_lv_name] }}