WIP - Observability stack

docker-compose.yml

@@ -20,6 +20,7 @@ services:
       - .:/boulder:cached
       - ./.gocache:/root/.cache/go-build:cached
       - ./test/certs/.softhsm-tokens/:/var/lib/softhsm/tokens/:cached
+      - logs:/var/log/
     networks:
       bouldernet:
         ipv4_address: 10.77.77.77
@@ -56,7 +57,7 @@ services:
       - bredis_1
       - bredis_2
       - bconsul
-      - bjaeger
+      - otel-collector
       - bpkimetal
     entrypoint: test/entrypoint.sh
     working_dir: &boulder_working_dir /boulder
@@ -135,8 +136,58 @@ services:
         ipv4_address: 10.77.77.10
     command: "consul agent -dev -config-format=hcl -config-file=/test/consul/config.hcl"
 
-  bjaeger:
-    image: jaegertracing/all-in-one:1.50
+  clickhouse:
+    image: clickhouse/clickhouse-server:latest
+    container_name: clickhouse
+    ports:
+      - "8123:8123"  # HTTP interface
+      - "9000:9000"  # Native interface
+    environment:
+      CLICKHOUSE_DB: otel
+    volumes:
+      - clickhouse_data:/var/lib/clickhouse
+      - ./test/clickhouse/users.xml:/etc/clickhouse-server/users.d/users.xml:ro
+    ulimits:
+      nofile:
+        soft: 262144
+        hard: 262144
+    restart: unless-stopped
+    networks:
+      - bouldernet
+
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:latest
+    container_name: otel-collector
+    command: ["--config=/etc/otel-collector-config.yaml", "--feature-gates=clickhouse.json"]
+    volumes:
+      - logs:/var/log/boulder/
+      - ./test/otelcol/config.yaml:/etc/otel-collector-config.yaml:ro
+    ports:
+      - "4317:4317"   # OTLP gRPC receiver
+      - "4318:4318"   # OTLP HTTP receiver
+      - "8888:8888"   # Prometheus metrics
+      - "8889:8889"   # Prometheus exporter metrics
+      - "5514:5514/udp" # Syslog receiver
+    depends_on:
+      - clickhouse
+    restart: unless-stopped
+    networks:
+      - bouldernet
+
+  grafana:
+    image: grafana/grafana:latest
+    container_name: grafana
+    ports:
+      - "3000:3000"
+    environment:
+      - GF_SECURITY_ADMIN_PASSWORD=admin_very_bad_password
+      - GF_INSTALL_PLUGINS=grafana-clickhouse-datasource
+    volumes:
+      - grafana_data:/var/lib/grafana
+      - ./test/grafana/provisioning:/etc/grafana/provisioning:ro
+    depends_on:
+      - clickhouse
+    restart: unless-stopped
     networks:
       - bouldernet
 
@@ -162,6 +213,11 @@ services:
         aliases:
           - boulder-vitess
 
+volumes:
+  clickhouse_data:
+  grafana_data:
+  logs:
+
 networks:
   # This network represents the data-center internal network. It is used for
   # boulder services and their infrastructure, such as consul, mariadb, and

test/clickhouse/users.xml

@@ -0,0 +1,29 @@
+<?xml version="1.0"?>
+<clickhouse>
+    <users>
+        <default>
+            <password>default_user_very_bad_password</password>
+            <networks>
+                <ip>::/0</ip>
+            </networks>
+            <profile>default</profile>
+            <quota>default</quota>
+        </default>
+        <otel_writer>
+            <password>otel_writer_very_bad_password</password>
+            <networks>
+                <ip>::/0</ip>
+            </networks>
+            <profile>default</profile>
+            <quota>default</quota>
+        </otel_writer>
+        <grafana_reader>
+            <password>grafana_reader_very_bad_password</password>
+            <networks>
+                <ip>::/0</ip>
+            </networks>
+            <profile>default</profile>
+            <quota>default</quota>
+        </grafana_reader>
+    </users>
+</clickhouse>

test/config-next/admin.json

@@ -36,7 +36,7 @@
 		"sysloglevel": -1
 	},
 	"openTelemetry": {
-		"endpoint": "bjaeger:4317",
+		"endpoint": "otel-collector:4317",
 		"sampleratio": 1
 	}
 }

test/config-next/bad-key-revoker.json

@@ -30,7 +30,7 @@
 		"sysloglevel": -1
 	},
 	"openTelemetry": {
-		"endpoint": "bjaeger:4317",
+		"endpoint": "otel-collector:4317",
 		"sampleratio": 1
 	}
 }

test/config-next/ca.json

@@ -202,7 +202,7 @@
 		"sysloglevel": -1
 	},
 	"openTelemetry": {
-		"endpoint": "bjaeger:4317",
+		"endpoint": "otel-collector:4317",
 		"sampleratio": 1
 	}
 }

test/config-next/crl-storer.json

@@ -38,7 +38,7 @@
 		"sysloglevel": -1
 	},
 	"openTelemetry": {
-		"endpoint": "bjaeger:4317",
+		"endpoint": "otel-collector:4317",
 		"sampleratio": 1
 	}
 }

test/config-next/crl-updater.json

@@ -59,7 +59,7 @@
 		"sysloglevel": -1
 	},
 	"openTelemetry": {
-		"endpoint": "bjaeger:4317",
+		"endpoint": "otel-collector:4317",
 		"sampleratio": 1
 	}
 }

test/config-next/log-validator.json

@@ -3,7 +3,7 @@
 		"stdoutLevel": 7
 	},
 	"openTelemetry": {
-		"endpoint": "bjaeger:4317",
+		"endpoint": "otel-collector:4317",
 		"sampleratio": 1
 	},
 	"files": [

test/config-next/nonce-a.json

@@ -9,7 +9,7 @@
 			"syslogLevel": -1
 		},
 		"openTelemetry": {
-			"endpoint": "bjaeger:4317",
+			"endpoint": "otel-collector:4317",
 			"sampleratio": 1
 		},
 		"grpc": {

test/config-next/nonce-b.json

@@ -9,7 +9,7 @@
 			"syslogLevel": -1
 		},
 		"openTelemetry": {
-			"endpoint": "bjaeger:4317",
+			"endpoint": "otel-collector:4317",
 			"sampleratio": 1
 		},
 		"grpc": {

test/config-next/publisher.json

@@ -47,7 +47,7 @@
 		"sysloglevel": -1
 	},
 	"openTelemetry": {
-		"endpoint": "bjaeger:4317",
+		"endpoint": "otel-collector:4317",
 		"sampleratio": 1
 	}
 }

test/config-next/ra.json

@@ -183,7 +183,7 @@
 		"sysloglevel": 6
 	},
 	"openTelemetry": {
-		"endpoint": "bjaeger:4317",
+		"endpoint": "otel-collector:4317",
 		"sampleratio": 1
 	}
 }

test/config-next/remoteva-a.json

@@ -49,7 +49,7 @@
 		"sysloglevel": -1
 	},
 	"openTelemetry": {
-		"endpoint": "bjaeger:4317",
+		"endpoint": "otel-collector:4317",
 		"sampleratio": 1
 	}
 }

test/config-next/remoteva-b.json

@@ -49,7 +49,7 @@
 		"sysloglevel": -1
 	},
 	"openTelemetry": {
-		"endpoint": "bjaeger:4317",
+		"endpoint": "otel-collector:4317",
 		"sampleratio": 1
 	}
 }

test/config-next/remoteva-c.json

@@ -49,7 +49,7 @@
 		"sysloglevel": -1
 	},
 	"openTelemetry": {
-		"endpoint": "bjaeger:4317",
+		"endpoint": "otel-collector:4317",
 		"sampleratio": 1
 	}
 }

test/config-next/sa.json

@@ -57,7 +57,7 @@
 		"sysloglevel": -1
 	},
 	"openTelemetry": {
-		"endpoint": "bjaeger:4317",
+		"endpoint": "otel-collector:4317",
 		"sampleratio": 1
 	}
 }

test/config-next/sfe.json

@@ -92,7 +92,7 @@
 		"sysloglevel": -1
 	},
 	"openTelemetry": {
-		"endpoint": "bjaeger:4317",
+		"endpoint": "otel-collector:4317",
 		"sampleratio": 1
 	},
 	"openTelemetryHttpConfig": {

test/config-next/va.json

@@ -73,7 +73,7 @@
 		"sysloglevel": 6
 	},
 	"openTelemetry": {
-		"endpoint": "bjaeger:4317",
+		"endpoint": "otel-collector:4317",
 		"sampleratio": 1
 	}
 }

test/config-next/wfe2.json

@@ -156,7 +156,7 @@
 		"sysloglevel": -1
 	},
 	"openTelemetry": {
-		"endpoint": "bjaeger:4317",
+		"endpoint": "otel-collector:4317",
 		"sampleratio": 1
 	},
 	"openTelemetryHttpConfig": {

test/config/log-validator.json

@@ -3,7 +3,7 @@
 		"stdoutLevel": 7
 	},
 	"openTelemetry": {
-		"endpoint": "bjaeger:4317",
+		"endpoint": "otel-collector:4317",
 		"sampleratio": 1
 	},
 	"files": [

test/config/sfe.json

@@ -39,7 +39,7 @@
 		"sysloglevel": -1
 	},
 	"openTelemetry": {
-		"endpoint": "bjaeger:4317",
+		"endpoint": "otel-collector:4317",
 		"sampleratio": 1
 	},
 	"openTelemetryHttpConfig": {

test/grafana/provisioning/datasources/clickhouse.yaml

@@ -0,0 +1,27 @@
+apiVersion: 1
+
+datasources:
+  - name: ClickHouse
+    type: grafana-clickhouse-datasource
+    jsonData:
+      defaultDatabase: otel
+      port: 9000
+      server: clickhouse
+      username: grafana_reader
+      logs:
+        defaultDatabase: otel
+        defaultTable: otel_logs
+        otelEnabled: true
+        otelVersion: latest
+        selectContextColumns: true
+        contextColumns:
+          - ServiceName
+      traces:
+        defaultDatabase: otel
+        defaultTable: otel_traces
+        otelEnabled: true
+        otelVersion: latest
+    secureJsonData:
+      password: grafana_reader_very_bad_password
+    isDefault: true
+    editable: true