Add role to deploy MongoDB as quadlet on lovelace

ansible/group_vars/all/nftables.yml

@@ -86,8 +86,12 @@ nftables_configuration: |
   {% if "databases" in group_names %}
       # PostgreSQL connections
       iifname {{ ansible_default_ipv4.interface }} ip saddr @possible_lke_ipv4_addrs tcp dport postgresql ct state new accept
+      # MongoDB connections
+      iifname {{ ansible_default_ipv4.interface }} ip saddr @possible_lke_ipv4_addrs tcp dport 27017 ct state new accept
   {% if ansible_default_ipv6 is defined %}
+      # And now via IPv6
       iifname {{ ansible_default_ipv6.interface }} ip6 saddr @possible_lke_ipv6_addrs tcp dport postgresql ct state new accept
+      iifname {{ ansible_default_ipv6.interface }} ip6 saddr @possible_lke_ipv6_addrs tcp dport 27017 ct state new accept
   {% endif %}
   {% endif %}
 
@@ -116,7 +120,8 @@ nftables_configuration: |
 
     chain forward {
       type filter hook forward priority 0
-      policy drop
+      # podman
+      policy accept
       ct state invalid drop
       ct state established,related accept
 

ansible/playbook.yml

@@ -62,11 +62,12 @@
   roles:
     - git-mirrors
 
-- name: Deploy our PostgreSQL database hosts
+- name: Deploy our database hosts
   hosts: databases
   roles:
     - postgres
     - prometheus-postgres-exporter
+    - mongodb
 
 - name: Deploy our LDAP server environment to the LDAP host
   hosts: ldap

ansible/roles/mongodb/README.md

@@ -0,0 +1,29 @@
+# mongodb
+
+This role deploys MongoDB using podman quadlets.
+
+The container starts as root, but drops privileges after startup.
+
+
+## Troubleshooting
+
+- **MongoDB refuses connections**
+
+  Verify that:
+
+  1. Your outbound IP is within the whitelisted LKE ranges in `/etc/nftables` -
+     the `nftables` Ansible role may be re-run to fetch them freshly
+
+  2. Your outbound IP is not banned by `fail2ban`, see `fail2ban-client banned`
+
+  3. You are using a sane piece of software that does not randomly appear to not
+     bind on ports properly:
+
+     > I encountered the same problem for an half an hour or so but it was not
+     > an issue after that. I changed nothing in my code or my firewall
+     > settings. Just waited a bit and the program worked as usual. So I think
+     > this may be something that is wrong with the Mongodb servers or it may be
+     > due to a slow network connection. Maybe check the uri and just waiting
+     > for a while. It worked for me.
+
+     When in doubt, `sudo systemctl restart mongodb`

ansible/roles/mongodb/handlers/main.yml

@@ -0,0 +1,7 @@
+---
+- name: Restart mongodb
+  ansible.builtin.service:
+    name: mongodb
+    state: restarted
+  tags:
+    - role::mongodb

ansible/roles/mongodb/meta/main.yml

@@ -0,0 +1,5 @@
+---
+dependencies:
+  - certbot
+  - podman
+  - systemd

ansible/roles/mongodb/tasks/main.yml

@@ -0,0 +1,90 @@
+---
+- name: Template quadlet services
+  ansible.builtin.template:
+    src: mongodb.{{ item }}.j2
+    dest: /etc/containers/systemd/mongodb.{{ item }}
+    owner: root
+    group: root
+    mode: "0444"
+  register: mongodb_units
+  tags:
+    - role::mongodb
+  notify:
+    - Reload the systemd daemon
+    - Restart mongodb
+  loop:
+    - container
+    - image
+    - volume
+
+- name: Create secrets directory
+  ansible.builtin.file:
+    path: /etc/opt/mongodb
+    state: directory
+    owner: "{{ mongodb_uid }}"
+    group: "{{ mongodb_gid }}"
+    mode: "0700"
+  tags:
+    - role::mongodb
+
+- name: Create TLS key file
+  ansible.builtin.shell: >
+    umask 0077
+
+    cat
+    /etc/letsencrypt/live/{{ ansible_fqdn }}/fullchain.pem
+    /etc/letsencrypt/live/{{ ansible_fqdn }}/privkey.pem
+    > /etc/opt/mongodb/fullchain-plus-key.pem
+
+    chown "{{ mongodb_uid }}:{{ mongodb_gid }}" /etc/opt/mongodb/fullchain-plus-key.pem
+  args:
+    creates: /etc/opt/mongodb/fullchain-plus-key.pem
+  tags:
+    - role::mongodb
+
+- name: Copy TLS fullchain
+  ansible.builtin.copy:
+    remote_src: true
+    src: /etc/letsencrypt/live/{{ ansible_fqdn }}/fullchain.pem
+    dest: /etc/opt/mongodb/fullchain.pem
+    owner: "{{ mongodb_uid }}"
+    group: "{{ mongodb_gid }}"
+    mode: "0400"
+  tags:
+    - role::mongodb
+
+- name: Create letsencrypt post-renewal hook
+  ansible.builtin.copy:
+    content: |
+      #!/bin/sh
+
+      set -ex
+
+      umask 0077
+
+      cat \
+        /etc/letsencrypt/live/{{ ansible_fqdn }}/fullchain.pem \
+        /etc/letsencrypt/live/{{ ansible_fqdn }}/privkey.pem \
+        > /etc/opt/mongodb/fullchain-plus-key.pem
+
+      chown "{{ mongodb_uid }}:{{ mongodb_gid }}" /etc/opt/mongodb/fullchain-plus-key.pem
+
+      cp /etc/letsencrypt/live/{{ ansible_fqdn }}/fullchain.pem /etc/opt/mongodb/fullchain.pem
+
+      # XXX: kill -HUP $PID OK?
+      systemctl restart mongodb
+    dest: /etc/letsencrypt/renewal-hooks/deploy/update-mongodb-cert
+    owner: root
+    group: root
+    mode: "0744"
+  tags:
+    - role::mongodb
+
+- name: Start and enable the quadlet
+  ansible.builtin.service:
+    name: mongodb.service
+    daemon_reload: true # "{{ mongodb_units is changed }}"
+    state: started
+    enabled: true
+  tags:
+    - role::mongodb

ansible/roles/mongodb/templates/mongodb.container.j2

@@ -0,0 +1,44 @@
+# {{ ansible_managed }}
+
+[Unit]
+Description = Mongo NoSQL (NonSensical Query Language) Server
+
+[Container]
+Image = mongodb.image
+Pull = missing
+Volume = mongodb.volume:/data/db
+# XXX: role should to have a dependency for the lovelace cert
+Volume = /etc/opt/mongodb:/certs:ro
+NoNewPrivileges = true
+PublishPort = 27017:27017
+Exec = \
+  --auth \
+  --tlsMode requireTLS \
+  --tlsCertificateKeyFile /certs/fullchain-plus-key.pem \
+  --tlsCAFile /certs/fullchain.pem \
+  --tlsAllowConnectionsWithoutCertificates
+
+[Service]
+# Resource control
+CPUQuota = 20%
+MemoryMax = 900M
+TasksMax = 200
+
+# Sandboxing
+NoNewPrivileges = true
+PrivateDevices = true
+ProtectHome = true
+ProtectKernelLogs = true
+LockPersonality = true
+MemoryDenyWriteExecute = true
+ProtectKernelModules = true
+ProtectSystem = true
+PrivateMounts = true
+RestrictRealtime = true
+RestrictSUIDSGID = true
+UMask = 0077
+
+[Install]
+WantedBy = network-online.target
+
+# vim: ft=dosini.jinja2:

ansible/roles/mongodb/templates/mongodb.image.j2

@@ -0,0 +1,10 @@
+# {{ ansible_managed }}
+
+[Unit]
+Description = Mongo NoSQL (NonSensical Query Language) Server image
+After = network-online.target
+
+[Image]
+Image = docker.io/library/mongo:4.4
+
+# vim: ft=dosini.jinja2:

ansible/roles/mongodb/templates/mongodb.volume.j2

@@ -0,0 +1,8 @@
+# {{ ansible_managed }}
+
+[Unit]
+Description = Mongo NoSQL (NonSensical Query Language) volume
+
+[Volume]
+
+# vim: ft=dosini.jinja2:

ansible/roles/mongodb/vars/main.yml

@@ -0,0 +1,4 @@
+---
+# MongoDB UID in the container
+mongodb_uid: 999
+mongodb_gid: 999

kubernetes/namespaces/databases/blackbox/blackbox-configmap.yaml

@@ -8,7 +8,7 @@ data:
     databases:
       mongodb:
         main_mongodb:
-          connection_string: mongodb://{{ MONGO_INITDB_ROOT_USERNAME }}:{{ MONGO_INITDB_ROOT_PASSWORD }}@mongodb.databases.svc.cluster.local:27017
+          connection_string: mongodb://{{ MONGO_INITDB_ROOT_USERNAME }}:{{ MONGO_INITDB_ROOT_PASSWORD }}@lovelace.box.pydis.wtf:27017
 
       postgres:
         lovelace_postgres: