Use encrypt nginx/mariadb/node exporter traffic

vmcj · vmcj · commit 263afe782651 · 2024-04-10T15:34:12.000+02:00
This is not possible (yet) for php_fpm but that one also exposes the
least interesting metrics for people snooping. If we would want that we
would need to do something with nginx for this.

This is relevant for the online judge but also makes sense as teams can
always query the node endpoint on domservers.
diff --git a/provision-contest/ansible/Makefile b/provision-contest/ansible/Makefile
@@ -15,6 +15,8 @@ LIBVENDORTGZ=roles/domjudge_checkout/files/lib-vendor.tgz
 SSHKEY=roles/ssh/files/id_ed25519
 SSL_DOMSERVER=roles/ssl/files/domserver
 SSL_DOMSERVER_FILES=$(addprefix $(SSL_DOMSERVER),.key .crt)
+SSL_NODEEXPORT=roles/prometheus_target_all/files/node_exporter
+SSL_NODEEXPORT_FILES=$(addprefix $(SSL_NODEEXPORT),.key .crt)
 SSL_LOCALHOST=roles/ssl/files/localhost
 SSL_LOCALHOST_FILES=$(addprefix $(SSL_LOCALHOST),.key .crt)
 SSL_CDS=roles/ssl/files/cds
@@ -57,9 +59,12 @@ ansible-master:
 	done
 
 admin: $(SSL_LOCALHOST_FILES)
-grafana: $(SSL_GRAFANA_FILES)
-domserver: $(SSL_DOMSERVER_FILES)
-cds: $(SSL_CDS_FILES)
+grafana: $(SSL_GRAFANA_FILES) $(SSL_NODEEXPORT)
+domserver: $(SSL_DOMSERVER_FILES) $(SSL_NODEEXPORT)
+judgehost: $(SSL_NODEEXPORT)
+cds: $(SSL_CDS_FILES) $(SSL_NODEEXPORT)
+scoreboard: $(SSL_NODEEXPORT)
+mgmt: $(SSL_NODEEXPORT)
 
 $(SSHKEY) $(SSHKEY).pub:
 	ssh-keygen -t ed25519 -f $(SSHKEY) -P ''
@@ -68,6 +73,9 @@ $(SSL_DOMSERVER_FILES):
 	openssl req -x509 -nodes -newkey rsa:4096 -subj "/O=DOMjudge/CN=domjudge" \
         -addext "subjectAltName = DNS:wf46-domjudge,DNS:wf47-domjudge,DNS:analyst" \
 		-sha256 -days 365 -keyout $(SSL_DOMSERVER).key -out $(SSL_DOMSERVER).crt
+$(SSL_NODEEXPORT_FILES):
+	openssl req -x509 -nodes -newkey rsa:4096 -subj "/O=DOMjudge/CN=metricexporter" \
+		-sha256 -days 365 -keyout $(SSL_NODEEXPORT).key -out $(SSL_NODEEXPORT).crt
 $(SSL_LOCALHOST_FILES):
 	openssl req -x509 -nodes -newkey rsa:4096 -subj "/O=DOMjudge/CN=localhost" \
 		-sha256 -days 365 -keyout $(SSL_LOCALHOST).key -out $(SSL_LOCALHOST).crt
@@ -84,6 +92,7 @@ clean:
 distclean: clean
 	rm -f $(SSHKEY) $(SSHKEY).pub
 	rm -f $(SSL_DOMSERVER_FILES)
+	rm -f $(SSL_NODEEXPORT)
 	rm -f $(SSL_LOCALHOST_FILES)
 	rm -f $(SSL_CDS_FILES)
 	rm -f $(SSL_GRAFANA_FILES)
diff --git a/provision-contest/ansible/roles/grafana/templates/prometheus.yml.j2 b/provision-contest/ansible/roles/grafana/templates/prometheus.yml.j2
@@ -16,36 +16,72 @@ scrape_configs:
         - {{ hostvars[host].ansible_host }}:9104
 {% endfor %}
   - job_name: node_domserver
+    basic_auth:
+      username: "prometheus"
+      password: "{{ PROMETHEUS_PASS }}"
+    tls_config:
+      insecure_skip_verify: true
+    scheme: https
     static_configs:
       - targets:
 {% for host in groups["domserver"] %}
         - {{ hostvars[host].ansible_host }}:9100
 {% endfor %}
   - job_name: node_judgehost
+    basic_auth:
+      username: "prometheus"
+      password: "{{ PROMETHEUS_PASS }}"
+    tls_config:
+      insecure_skip_verify: true
+    scheme: https
     static_configs:
       - targets:
 {% for host in groups["judgehost"] %}
         - {{ hostvars[host].ansible_host }}:9100
 {% endfor %}
   - job_name: node_grafana
+    basic_auth:
+      username: "prometheus"
+      password: "{{ PROMETHEUS_PASS }}"
+    tls_config:
+      insecure_skip_verify: true
+    scheme: https
     static_configs:
       - targets:
 {% for host in groups["grafana"] %}
         - {{ hostvars[host].ansible_host }}:9100
 {% endfor %}
   - job_name: node_mgmt
+    basic_auth:
+      username: "prometheus"
+      password: "{{ PROMETHEUS_PASS }}"
+    tls_config:
+      insecure_skip_verify: true
+    scheme: https
     static_configs:
       - targets:
 {% for host in groups["mgmt"] %}
         - {{ hostvars[host].ansible_host }}:9100
 {% endfor %}
   - job_name: node_scoreboard
+    basic_auth:
+      username: "prometheus"
+      password: "{{ PROMETHEUS_PASS }}"
+    tls_config:
+      insecure_skip_verify: true
+    scheme: https
     static_configs:
       - targets:
 {% for host in groups["scoreboard"] %}
         - {{ hostvars[host].ansible_host }}:9100
 {% endfor %}
   - job_name: node_cds
+    basic_auth:
+      username: "prometheus"
+      password: "{{ PROMETHEUS_PASS }}"
+    tls_config:
+      insecure_skip_verify: true
+    scheme: https
     static_configs:
       - targets:
 {% for host in groups["cds"] %}
@@ -65,18 +101,36 @@ scrape_configs:
         - {{ hostvars[host].ansible_host }}
 {% endfor %}
   - job_name: 'web_nginx_cds'
+    basic_auth:
+      username: "prometheus"
+      password: "{{ PROMETHEUS_PASS }}"
+    tls_config:
+      insecure_skip_verify: true
+    scheme: https
     static_configs:
       - targets:
 {% for host in groups["cds"] %}
         - {{ hostvars[host].ansible_host }}:9113
 {% endfor %}
   - job_name: 'web_nginx_scoreboard'
+    basic_auth:
+      username: "prometheus"
+      password: "{{ PROMETHEUS_PASS }}"
+    tls_config:
+      insecure_skip_verify: true
+    scheme: https
     static_configs:
       - targets:
 {% for host in groups["scoreboard"] %}
         - {{ hostvars[host].ansible_host }}:9113
 {% endfor %}
   - job_name: 'web_nginx_domserver'
+    basic_auth:
+      username: "prometheus"
+      password: "{{ PROMETHEUS_PASS }}"
+    tls_config:
+      insecure_skip_verify: true
+    scheme: https
     static_configs:
       - targets:
 {% for host in groups["domserver"] %}
diff --git a/provision-contest/ansible/roles/prometheus_target_all/handlers/main.yml b/provision-contest/ansible/roles/prometheus_target_all/handlers/main.yml
@@ -5,3 +5,9 @@
     enabled: true
     state: restarted
     daemon_reload: true
+
+- name: Restart node-exporter
+  service:
+    name: prometheus-node-exporter
+    enabled: true
+    state: restarted
diff --git a/provision-contest/ansible/roles/prometheus_target_all/tasks/main.yml b/provision-contest/ansible/roles/prometheus_target_all/tasks/main.yml
@@ -6,3 +6,43 @@
     state: present
     pkg:
       - prometheus-node-exporter
+
+- name: Collect prometheus settings
+  file:
+    path: /etc/prometheus
+    owner: root
+    group: root
+    mode: 0755
+    state: directory
+
+- name: Install SSL server certificates
+  copy:
+    src: "node_exporter.{{ item }}"
+    dest: "/etc/prometheus/node_exporter.{{ item }}"
+    owner: root
+    group: root
+    mode: 0644
+  loop:
+    - crt
+    - key
+
+- name: Get HTPassword
+  delegate_to: localhost
+  shell: "echo {{ PROMETHEUS_PASS }} | htpasswd -inBC 10 \"\" | tr -d ':\n'"
+  register: htpassd_shell
+
+- name: Set certificate to encrypt node_exporter traffic
+  template:
+    owner: prometheus
+    group: prometheus
+    mode: 0644
+    src: web.yml.j2
+    dest: /etc/prometheus/prometheus-authentication.yml
+
+- name: Scrape with TLS encryption
+  lineinfile:
+    dest: /etc/default/prometheus-node-exporter
+    state: present
+    regexp: '^ARGS=""'
+    line: 'ARGS="--web.config /etc/prometheus/prometheus-authentication.yml"'
+  notify: Restart node-exporter
diff --git a/provision-contest/ansible/roles/prometheus_target_web/meta/main.yml b/provision-contest/ansible/roles/prometheus_target_web/meta/main.yml
@@ -0,0 +1,5 @@
+# Role dependencies
+---
+dependencies:
+  - role: prometheus_target_all
+    tags: prometheus_target_all
diff --git a/provision-contest/ansible/roles/prometheus_target_web/tasks/main.yml b/provision-contest/ansible/roles/prometheus_target_web/tasks/main.yml
@@ -16,6 +16,14 @@
       - prometheus-mysqld-exporter
   notify: Restart mysqld-exporter
 
+- name: Scrape mysql exporter with TLS encryption
+  lineinfile:
+     dest: /etc/default/prometheus-mysqld-exporter
+    state: present
+    regexp: '^ARGS=""'
+    line: 'ARGS="--web.config /etc/prometheus/prometheus-authentication.yml"'
+   notify: Restart mysqld-exporter
+
 # Gather PHP-FPM statistics
 # The exporter from this is currently not in deb sources
 # so we need to download this from GitHub see the README in files
@@ -68,7 +76,7 @@
     dest: /etc/default/prometheus-nginx-exporter
     state: present
     regexp: '^ARGS=""'
-    line: 'ARGS="-nginx.scrape-uri=http://localhost:8787/basic_status"'
+    line: 'ARGS="--nginx.scrape-uri=http://localhost:8787/basic_status --web.config /etc/prometheus/prometheus-authentication.yml"'
   notify: Restart nginx-exporter
 
 - name: Create storage dir for exporter settings
