diff --git a/.github/workflows/validate-charts.yml b/.github/workflows/validate-charts.yml index 9a64848..0ae7d00 100644 --- a/.github/workflows/validate-charts.yml +++ b/.github/workflows/validate-charts.yml @@ -104,10 +104,19 @@ jobs: ;; countly-mongodb) helm template test-release "${chart}" \ + --set users.admin.password=test \ --set users.app.password=test \ --set users.metrics.password=test \ > /dev/null || exit_code=1 ;; + countly-cluster-secret-store) + helm template test-release "${chart}" \ + --set secretStore.secretManagerProjectID=test-project \ + --set secretStore.clusterProjectID=test-cluster-project \ + --set secretStore.clusterName=test-cluster \ + --set secretStore.clusterLocation=test-location \ + > /dev/null || exit_code=1 + ;; countly-migration) helm template test-release "${chart}" \ --set backingServices.mongodb.password=test \ diff --git a/.gitignore b/.gitignore index 8809778..5c88993 100644 --- a/.gitignore +++ b/.gitignore @@ -8,9 +8,6 @@ overlay-secrets.yaml *-secrets.yaml secrets-*.yaml -# Exception: reference environment templates (contain no real secrets) -!environments/reference/secrets-*.yaml - # Helmfile state helmfile.lock .helmfile/ diff --git a/README.md b/README.md index bda7fae..126936e 100644 --- a/README.md +++ b/README.md @@ -157,8 +157,11 @@ Install required operators before deploying Countly. See [docs/PREREQUISITES.md] - Choose `global.observability`: `disabled`, `full`, `external-grafana`, or `external` - Choose `global.kafkaConnect`: `throughput`, `balanced`, or `low-latency` - Choose `global.security`: `open` or `hardened` + - Keep `global.imageSource.mode: direct` for the current direct-pull flow, or switch to `gcpArtifactRegistry` and set `global.imageSource.gcpArtifactRegistry.repositoryPrefix` + - Set `global.imagePullSecrets` when pulling from a private registry such as GAR -3. **Fill in required secrets** in the chart-specific files. See `environments/reference/secrets.example.yaml` for a complete reference. +3. **Fill in required credentials** in the chart-specific files. See `environments/reference/secrets.example.yaml` for a complete reference. + Keep `secrets.mode: values` for direct YAML values, switch to `secrets.mode: externalSecret` to have the charts create `ExternalSecret` resources backed by your Secret Manager store. 4. **Register your environment** in `helmfile.yaml.gotmpl`: ```yaml @@ -173,6 +176,81 @@ Install required operators before deploying Countly. See [docs/PREREQUISITES.md] helmfile -e my-deployment apply ``` +For a GAR-backed production example, see [environments/example-production/global.yaml](/Users/admin/cly/helm/environments/example-production/global.yaml) and replace `countly-gar` with your Kubernetes docker-registry secret name. +For GitOps-managed pull secrets, start from [environments/reference/image-pull-secrets.example.yaml](/Users/admin/cly/helm/environments/reference/image-pull-secrets.example.yaml) and encrypt or template it before committing. +For Secret Manager + External Secrets Operator, set `global.imagePullSecretExternalSecret` in your environment `global.yaml` so Countly can create its namespaced `dockerconfigjson` pull secret. +Application secrets can use the same pattern in `credentials-countly.yaml`, `credentials-kafka.yaml`, `credentials-clickhouse.yaml`, and `credentials-mongodb.yaml` by switching `secrets.mode` to `externalSecret` and filling `secrets.externalSecret.remoteRefs`. +Countly ingress TLS can also use the same pattern: set customer `tls: provided`, then enable `ingress.tls.externalSecret` in `countly.yaml` to materialize a `kubernetes.io/tls` secret from Secret Manager. The default scaffold already points all customers at the shared keys `countly-prod-tls-crt` and `countly-prod-tls-key`; override them only when a customer needs a dedicated certificate. + +Recommended Secret Manager naming convention: +- `-gar-dockerconfig` +- `-countly-encryption-reports-key` +- `-countly-web-session-secret` +- `-countly-password-secret` +- `-countly-clickhouse-password` +- `-kafka-connect-clickhouse-password` +- `-clickhouse-default-user-password` +- `-mongodb-admin-password` +- `-mongodb-app-password` +- `-mongodb-metrics-password` + +### GitOps Customer Onboarding + +For Argo CD managed deployments, scaffold a new customer/cluster with: + +```bash +./scripts/new-argocd-customer.sh [--secret-mode values|gcp-secrets] +``` + +This creates: +- `environments//` +- `argocd/customers/.yaml` + +For Secret Manager from day one, prefer: + +```bash +./scripts/new-argocd-customer.sh --secret-mode gcp-secrets +``` + +Then: +1. fill in `environments//credentials-*.yaml` +2. commit +3. sync `countly-bootstrap` + +## Image Sources + +This table shows which images are used by the platform, where they are pulled from, and whether they are Countly-provided or official upstream/vendor images. + +| Component | Image / Pattern | Source Registry | Ownership | Private/GAR Ready | +|-------|-------|-------|-------|-------| +| Countly app pods (`api`, `frontend`, `ingestor`, `aggregator`, `jobserver`) | `gcr.io/countly-dev-313620/countly-unified:26.01` or `/countly-unified` | `gcr.io` or `us-docker.pkg.dev` | Countly-provided | Yes | +| Kafka Connect ClickHouse | `countly/strimzi-kafka-connect-clickhouse:kafka4.2.0-ch1.3.5-strimzi0.51-otel2.12.0` | Docker Hub | Countly-provided custom image | Public by default | +| ClickHouse server | `clickhouse/clickhouse-server:26.3` | Docker Hub style namespace | Official provider image | No, not via current GAR toggle | +| ClickHouse keeper | `clickhouse/clickhouse-keeper:26.3` | Docker Hub style namespace | Official provider image | No, not via current GAR toggle | +| MongoDB database | chosen by MongoDB Kubernetes Operator from `version: 8.2.5` | operator-resolved upstream image | Official provider image | No, not via current chart values | +| MongoDB exporter | `percona/mongodb_exporter:0.47.2` | Docker Hub style namespace | Official provider/vendor image | No | +| Migration service | `countly/migration:` | configurable, default public-style repo | Countly-provided | Not wired to GAR automatically | +| Prometheus | `prom/prometheus:v3.8.1` | Docker Hub style namespace | Official provider image | Only via `global.imageRegistry` mirror | +| Loki | `grafana/loki:3.6.3` | Docker Hub style namespace | Official provider image | Only via `global.imageRegistry` mirror | +| Tempo | `grafana/tempo:2.8.1` | Docker Hub style namespace | Official provider image | Only via `global.imageRegistry` mirror | +| Pyroscope | `grafana/pyroscope:1.16.0` | Docker Hub style namespace | Official provider image | Only via `global.imageRegistry` mirror | +| Grafana | `grafana/grafana:12.3.5` | Docker Hub style namespace | Official provider image | Only via `global.imageRegistry` mirror | +| Alloy / Alloy OTLP / Alloy Metrics | `grafana/alloy:v1.14.0` | Docker Hub style namespace | Official provider image | Only via `global.imageRegistry` mirror | +| kube-state-metrics | `registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.17.0` | `registry.k8s.io` | Official provider image | Only via `global.imageRegistry` mirror | +| node-exporter | `prom/node-exporter:v1.10.2` | Docker Hub style namespace | Official provider image | Only via `global.imageRegistry` mirror | +| busybox init/test containers | `busybox:1.37.0` | Docker Hub | Official provider image | No explicit mirror logic | + +Operator and platform apps are pinned by Helm chart version in `argocd/operators/`, so this repo controls the chart source and version, but not every underlying container image directly: + +| Operator/App | Source | Version | Ownership | +|-------|-------|-------|-------| +| cert-manager | Jetstack chart | `v1.17.2` | Official provider | +| External Secrets Operator | external-secrets chart | `1.3.1` | Official provider | +| Strimzi Kafka Operator | Strimzi chart | `0.51.0` | Official provider | +| ClickHouse Operator | GHCR OCI chart | `0.0.2` | Official provider | +| MongoDB Kubernetes Operator | MongoDB chart | `1.7.0` | Official provider | +| F5 NGINX Ingress | NGINX chart | `2.1.0` | Official provider | + ### Manual Installation (without Helmfile) Substitute your profile choices from `global.yaml` into the commands below. @@ -193,7 +271,7 @@ helm install countly-mongodb ./charts/countly-mongodb -n mongodb --create-namesp -f profiles/sizing/$SIZING/mongodb.yaml \ -f profiles/security/$SECURITY/mongodb.yaml \ -f environments/$ENV/mongodb.yaml \ - -f environments/$ENV/secrets-mongodb.yaml + -f environments/$ENV/credentials-mongodb.yaml helm install countly-clickhouse ./charts/countly-clickhouse -n clickhouse --create-namespace \ --wait --timeout 10m \ @@ -201,7 +279,7 @@ helm install countly-clickhouse ./charts/countly-clickhouse -n clickhouse --crea -f profiles/sizing/$SIZING/clickhouse.yaml \ -f profiles/security/$SECURITY/clickhouse.yaml \ -f environments/$ENV/clickhouse.yaml \ - -f environments/$ENV/secrets-clickhouse.yaml + -f environments/$ENV/credentials-clickhouse.yaml helm install countly-kafka ./charts/countly-kafka -n kafka --create-namespace \ --wait --timeout 10m \ @@ -211,7 +289,7 @@ helm install countly-kafka ./charts/countly-kafka -n kafka --create-namespace \ -f profiles/observability/$OBS/kafka.yaml \ -f profiles/security/$SECURITY/kafka.yaml \ -f environments/$ENV/kafka.yaml \ - -f environments/$ENV/secrets-kafka.yaml + -f environments/$ENV/credentials-kafka.yaml helm install countly ./charts/countly -n countly --create-namespace \ --wait --timeout 10m \ @@ -221,7 +299,7 @@ helm install countly ./charts/countly -n countly --create-namespace \ -f profiles/observability/$OBS/countly.yaml \ -f profiles/security/$SECURITY/countly.yaml \ -f environments/$ENV/countly.yaml \ - -f environments/$ENV/secrets-countly.yaml + -f environments/$ENV/credentials-countly.yaml helm install countly-observability ./charts/countly-observability -n observability --create-namespace \ --wait --timeout 10m \ @@ -233,11 +311,12 @@ helm install countly-observability ./charts/countly-observability -n observabili -f environments/$ENV/secrets-observability.yaml # Optional: MongoDB to ClickHouse batch migration (includes bundled Redis) +helm dependency build ./charts/countly-migration helm install countly-migration ./charts/countly-migration -n countly-migration --create-namespace \ --wait --timeout 5m \ -f environments/$ENV/global.yaml \ -f environments/$ENV/migration.yaml \ - -f environments/$ENV/secrets-migration.yaml + -f environments/$ENV/credentials-migration.yaml ``` ## Configuration Model @@ -263,7 +342,7 @@ Composable profile dimensions — select one value per dimension in `global.yaml Environments contain deployment-specific choices: - `global.yaml` — Profile selectors, hostname, backing service modes - `.yaml` — Per-chart overrides (tuning, network policy, OTEL) -- `secrets-.yaml` — Per-chart secrets (gitignored) +- `credentials-.yaml` — Per-chart credentials overrides ### Deployment Modes diff --git a/argocd/ONBOARDING.md b/argocd/ONBOARDING.md new file mode 100644 index 0000000..33e0fa3 --- /dev/null +++ b/argocd/ONBOARDING.md @@ -0,0 +1,976 @@ +# Customer Onboarding Guide + +This guide is written as a slow, step-by-step walkthrough for adding one new customer cluster. + +Use this when you want to: +- create a new customer deployment +- choose between direct secrets and Secret Manager +- connect Argo CD to the new cluster +- troubleshoot the common issues we already hit once + +## What You Are Building + +For each customer, you will end up with: +- one Argo CD customer metadata file +- one environment folder with customer overrides +- one target Kubernetes cluster +- one set of Argo CD applications created automatically by `countly-bootstrap` +- one secret strategy: + - direct values in Git, or + - Google Secret Manager + External Secrets Operator + +### Visual Map + +```mermaid +flowchart TD + meta["argocd/customers/.yaml"] + env["environments//"] + bootstrap["Argo CD app: countly-bootstrap"] + appsets["ApplicationSets"] + cluster["Customer cluster"] + + meta --> bootstrap + env --> bootstrap + bootstrap --> appsets + + appsets --> eso["-external-secrets"] + appsets --> store["-cluster-secret-store"] + appsets --> mongo["-mongodb"] + appsets --> ch["-clickhouse"] + appsets --> kafka["-kafka"] + appsets --> countly["-countly"] + appsets --> obs["-observability (optional)"] + appsets --> mig["-migration (optional)"] + + eso --> cluster + store --> cluster + mongo --> cluster + ch --> cluster + kafka --> cluster + countly --> cluster + obs --> cluster + mig --> cluster +``` + +## The Secret Naming Rule + +Use this naming convention everywhere: + +```text +-- +``` + +Examples for customer `northstar`: + +```text +northstar-gar-dockerconfig +northstar-countly-encryption-reports-key +northstar-countly-web-session-secret +northstar-countly-password-secret +northstar-countly-clickhouse-password +northstar-kafka-connect-clickhouse-password +northstar-clickhouse-default-user-password +northstar-mongodb-admin-password +northstar-mongodb-app-password +northstar-mongodb-metrics-password +``` + +Keep the `` slug exactly the same in: +- `argocd/customers/.yaml` +- `environments//` +- Secret Manager secret names + +## Before You Start + +Make sure these are already true: + +1. You can access the repo. +2. You can access the Argo CD instance. +3. The target cluster exists. +4. The target cluster is registered in Argo CD. +5. DNS is ready for the customer hostname. + +Useful checks: + +```bash +argocd app list +argocd cluster list +kubectl config current-context +``` + +## Step 1: Create The Customer Scaffold + +Run: + +```bash +./scripts/new-argocd-customer.sh [--secret-mode values|gcp-secrets] +``` + +Example: + +```bash +./scripts/new-argocd-customer.sh northstar https://1.2.3.4 analytics.northstar.example.com +``` + +If you plan to use Google Secret Manager from the start, use: + +```bash +./scripts/new-argocd-customer.sh --secret-mode gcp-secrets northstar https://1.2.3.4 analytics.northstar.example.com +``` + +This creates: +- `argocd/customers/northstar.yaml` +- `environments/northstar/` + +The difference is: +- `values` writes the credential files in direct-password mode +- `gcp-secrets` writes the credential files already wired for External Secrets and the standard Google Secret Manager key names + +## How To Read Argo CD For One Customer + +Yes, in the current setup all customer apps appear in the same Argo CD dashboard view. +That is normal. + +The important trick is: every generated app starts with the customer slug. + +For customer `northstar`, you should expect app names like: + +```text +northstar-cluster-secret-store +northstar-external-secrets +northstar-mongodb +northstar-clickhouse +northstar-kafka +northstar-countly +northstar-observability +northstar-migration +``` + +So when the UI feels crowded, think: +- first filter by the customer slug +- then read the apps from platform first, then data stores, then Countly + +### Fast CLI Filters + +List only one customer's apps: + +```bash +kubectl get applications -n argocd | grep '^northstar-' +``` + +Get only one app: + +```bash +argocd app get northstar-countly +argocd app get northstar-kafka +``` + +Refresh one app: + +```bash +argocd app get northstar-countly --hard-refresh +``` + +Sync one app: + +```bash +argocd app sync northstar-countly +``` + +Terminate a stuck sync: + +```bash +argocd app terminate-op northstar-countly +``` + +### Best Order To Read A Broken Customer + +When one customer is failing, read the apps in this order: + +1. `northstar-cluster-secret-store` +2. `northstar-external-secrets` +3. `northstar-mongodb` +4. `northstar-clickhouse` +5. `northstar-kafka` +6. `northstar-countly` + +Why this order: +- if Secret Manager auth is broken, app secrets fail later +- if MongoDB or ClickHouse is broken, Countly can still show as unhealthy later +- if Kafka is broken, Countly ingestion can fail later + +### Healthy First-Rollout Shape + +This is the rough order you want to see in Argo CD: + +```mermaid +flowchart LR + A["-cluster-secret-store"] --> B["-external-secrets"] + B --> C["-mongodb"] + B --> D["-clickhouse"] + D --> E["-kafka"] + C --> F["-countly"] + D --> F + E --> F +``` + +If `countly` is unhealthy, do not start there immediately. +Walk backward to Kafka, ClickHouse, MongoDB, and secret-store health first. + +## Step 2: Fill In Customer Metadata + +Open: + +- `argocd/customers/.yaml` + +Set these carefully: +- `server` +- `gcpServiceAccountEmail` +- `secretManagerProjectID` +- `clusterProjectID` +- `clusterName` +- `clusterLocation` +- `hostname` +- `sizing` +- `security` +- `tls` +- `observability` +- `kafkaConnect` +- `migration` + +Important for `server`: +- use the actual cluster API server URL Argo CD knows +- do not guess or paste a random external IP +- for GKE, the safest source is: + +```bash +gcloud container clusters describe \ + --zone \ + --project \ + --format="value(endpoint)" +``` + +Then use: + +```text +https:// +``` + +Example: + +```yaml +customer: northstar +environment: northstar +project: customer-platform +server: https://1.2.3.4 +gcpServiceAccountEmail: northstar-eso@example-secrets-project.iam.gserviceaccount.com +secretManagerProjectID: example-secrets-project +clusterProjectID: example-gke-project +clusterName: northstar-prod +clusterLocation: us-central1-a +hostname: analytics.northstar.example.com +sizing: tier1 +security: hardened +tls: letsencrypt +observability: disabled +kafkaConnect: balanced +migration: disabled +``` + +## Step 3: Choose Your Secret Mode + +You have two valid ways to run a customer. + +### Option A: Direct Values + +Use this when: +- you are testing quickly +- you are on an internal sandbox +- you are not ready to set up Secret Manager yet + +What to do: +- keep `secrets.mode: values` +- fill the passwords directly in: + - `environments//credentials-countly.yaml` + - `environments//credentials-kafka.yaml` + - `environments//credentials-clickhouse.yaml` + - `environments//credentials-mongodb.yaml` + +### Option B: Secret Manager + +Use this when: +- you do not want app passwords in Git +- you want customer isolation +- you want the production path + +What to do: +- set `secrets.mode: externalSecret` in the files that should read from GSM +- create the matching secrets in Google Secret Manager +- let External Secrets Operator create the Kubernetes `Secret`s for you + +## Step 4: If Using GAR, Decide Image Pull Mode + +There are two image pull patterns: + +### Direct / Public Pulls + +Use: + +```yaml +global: + imageSource: + mode: direct +``` + +### GAR + Secret Manager Pull Secret + +Use: + +```yaml +global: + imageSource: + mode: gcpArtifactRegistry + gcpArtifactRegistry: + repositoryPrefix: us-docker.pkg.dev// + imagePullSecrets: + - name: countly-registry + imagePullSecretExternalSecret: + enabled: true + refreshInterval: "1h" + secretStoreRef: + name: gcp-secrets + kind: ClusterSecretStore + remoteRef: + key: -gar-dockerconfig +``` + +This GAR pull-secret path is for Countly application images. Kafka Connect uses the public `countly/strimzi-kafka-connect-clickhouse` image by default. + +### Provided TLS + Secret Manager + +If you want to use your own certificate instead of Let's Encrypt: + +1. set customer `tls: provided` +2. keep or enable the generated `countly.yaml` TLS External Secret block +3. create these Secret Manager keys once if you want to reuse the same cert for many customers: + - `countly-prod-tls-crt` + - `countly-prod-tls-key` + +This is the default path now. New customers generated in `gcp-secrets` mode already point at these shared TLS keys, so you do not need to rename them per customer unless a customer needs its own certificate. + +Example: + +```yaml +ingress: + tls: + externalSecret: + enabled: true + refreshInterval: "1h" + secretStoreRef: + name: gcp-secrets + kind: ClusterSecretStore + remoteRefs: + tlsCrt: countly-prod-tls-crt + tlsKey: countly-prod-tls-key +``` + +This creates the Countly ingress TLS secret automatically in the `countly` namespace, so you do not need a separate manual TLS manifest per customer. + +## Step 5: If Using Secret Manager, Prepare The Cluster + +This is the production path. + +### 5.1 Install ESO Through Argo + +The repo already has the Argo pieces for: +- External Secrets Operator +- per-customer `ClusterSecretStore` + +After customer metadata is committed, `countly-bootstrap` will create them. + +### 5.2 Enable Workload Identity On The Cluster + +This is the part that usually feels confusing the first time. + +Simple version: +- Kubernetes pods inside the customer cluster need a safe way to prove who they are +- Google Secret Manager only gives secrets to identities it trusts +- Workload Identity is the bridge between those two things + +If this is not configured, External Secrets will not be able to read passwords from Google Secret Manager. + +#### 5.2.1 Check Whether Workload Identity Is Already Enabled + +Who runs this: +- the person onboarding the customer cluster + +What this does: +- asks GKE whether the cluster already supports Workload Identity + +Why this matters: +- without this, the `external-secrets` pod cannot authenticate to Google Secret Manager + +Command: + +```bash +gcloud container clusters describe \ + --zone \ + --project \ + --format="value(workloadIdentityConfig.workloadPool)" +``` + +What good looks like: + +```text +.svc.id.goog +``` + +If the output is empty: +- Workload Identity is not enabled yet +- you must enable it before moving on + +#### 5.2.2 Turn Workload Identity On If It Is Missing + +Who runs this: +- the cluster administrator + +What this does: +- tells the cluster to trust Kubernetes service accounts as Google identities + +Why this matters: +- this is what allows the `external-secrets` pod to read from Google Secret Manager without using a static key file + +Command: + +```bash +gcloud container clusters update \ + --zone \ + --project \ + --workload-pool=.svc.id.goog +``` + +What good looks like: +- the command succeeds +- running the check again shows `.svc.id.goog` + +#### 5.2.3 Check The Node Pool Metadata Mode + +Who runs this: +- the cluster administrator + +What this does: +- checks whether the node pool is exposing the GKE metadata server in the correct way + +Why this matters: +- even if Workload Identity is enabled on the cluster, pods still need the node pool configured correctly to use it + +First list the node pools: + +```bash +gcloud container node-pools list \ + --cluster \ + --zone \ + --project +``` + +Then check each node pool: + +```bash +gcloud container node-pools describe \ + --cluster \ + --zone \ + --project \ + --format="value(config.workloadMetadataConfig.mode)" +``` + +What good looks like: + +```text +GKE_METADATA +``` + +If it is not `GKE_METADATA`, update it: + +```bash +gcloud container node-pools update \ + --cluster \ + --zone \ + --project \ + --workload-metadata=GKE_METADATA +``` + +#### 5.2.4 Quick Mental Model + +If you want a very simple way to remember this: + +- cluster Workload Identity: + - lets the cluster speak Google IAM +- node pool metadata mode: + - lets the pod actually use that identity on the node + +You need both. + +### 5.3 Bind The Kubernetes Service Account To The GCP Service Account + +This is the second half of the setup. + +Simple version: +- the `external-secrets` pod runs as a Kubernetes service account +- that Kubernetes service account must be linked to a Google service account +- that Google service account is the one allowed to read secrets + +#### 5.3.1 Understand The Two Identities + +There are two different identities here: + +1. Kubernetes service account: + - usually `external-secrets` in namespace `external-secrets` + - this is the identity used by the pod inside the cluster + +2. Google service account: + - something like `northstar-eso@example-secrets-project.iam.gserviceaccount.com` + - this is the identity Google Secret Manager trusts + +Workload Identity links those two together. + +#### 5.3.2 Allow The Kubernetes Service Account To Act As The Google Service Account + +Who runs this: +- someone with IAM permission on the Google service account project + +What this does: +- tells Google IAM that the `external-secrets` Kubernetes service account is allowed to act as the chosen Google service account + +Why this matters: +- without this, the pod exists, but Google still does not trust it + +Command: + +```bash +gcloud iam service-accounts add-iam-policy-binding \ + \ + --project= \ + --role=roles/iam.workloadIdentityUser \ + --member="serviceAccount:.svc.id.goog[external-secrets/external-secrets]" +``` + +What good looks like: +- the command succeeds +- the binding appears in the Google service account IAM policy + +#### 5.3.3 Allow The Google Service Account To Read Secrets + +Who runs this: +- someone with IAM permission on the Secret Manager project + +What this does: +- gives the Google service account permission to read secrets from the chosen Secret Manager project + +Why this matters: +- the identity link can be correct, but secret reads still fail if this permission is missing + +Command: + +```bash +gcloud projects add-iam-policy-binding \ + --member="serviceAccount:" \ + --role=roles/secretmanager.secretAccessor +``` + +What good looks like: +- the command succeeds +- the Google service account can read the expected secrets + +#### 5.3.4 Verify The Kubernetes Service Account Annotation + +Who runs this: +- the platform operator after Argo has installed the External Secrets Operator + +What this does: +- checks that the in-cluster Kubernetes service account is annotated with the Google service account email + +Why this matters: +- this annotation is how GKE knows which Google service account the pod should use + +Command: + +```bash +kubectl get sa -n external-secrets external-secrets -o yaml +``` + +What you should see: + +```yaml +iam.gke.io/gcp-service-account: +``` + +### 5.4 Verify The ClusterSecretStore + +Check: + +```bash +kubectl get clustersecretstores.external-secrets.io +kubectl describe clustersecretstores.external-secrets.io gcp-secrets +``` + +Healthy looks like: +- `STATUS=Valid` +- `READY=True` + +If you see `InvalidProviderConfig`, first check Workload Identity. + +## Step 6: Create Secrets In Google Secret Manager + +Use names like: + +```text +northstar-countly-encryption-reports-key +northstar-countly-web-session-secret +northstar-countly-password-secret +northstar-countly-clickhouse-password +northstar-mongodb-app-password +northstar-kafka-connect-clickhouse-password +northstar-clickhouse-default-user-password +northstar-mongodb-admin-password +northstar-mongodb-app-password +northstar-mongodb-metrics-password +northstar-gar-dockerconfig +``` + +If your org policy blocks global replication, create secrets with user-managed regional replication: + +```bash +gcloud secrets create northstar-countly-clickhouse-password \ + --replication-policy=user-managed \ + --locations=europe-west1 +printf '%s' 'StrongPasswordHere' | gcloud secrets versions add northstar-countly-clickhouse-password --data-file=- +``` + +Create one secret at a time if you are debugging. It is easier to spot mistakes. + +## Step 7: Map The Secrets In Environment Files + +### Countly + +File: +- `environments/reference/credentials-countly.yaml` + +Secret Manager mode example: + +```yaml +secrets: + mode: externalSecret + externalSecret: + secretStoreRef: + name: gcp-secrets + kind: ClusterSecretStore + remoteRefs: + common: + encryptionReportsKey: northstar-countly-encryption-reports-key + webSessionSecret: northstar-countly-web-session-secret + passwordSecret: northstar-countly-password-secret + clickhouse: + password: northstar-countly-clickhouse-password + mongodb: + password: northstar-mongodb-app-password + clickhouse: + username: default + database: countly_drill + kafka: + securityProtocol: PLAINTEXT +``` + +### Kafka + +File: +- `environments/reference/credentials-kafka.yaml` + +```yaml +secrets: + mode: externalSecret + externalSecret: + secretStoreRef: + name: gcp-secrets + kind: ClusterSecretStore + remoteRefs: + clickhouse: + password: northstar-kafka-connect-clickhouse-password +``` + +### ClickHouse + +File: +- `environments/reference/credentials-clickhouse.yaml` + +```yaml +secrets: + mode: externalSecret + externalSecret: + secretStoreRef: + name: gcp-secrets + kind: ClusterSecretStore + remoteRefs: + defaultUserPassword: northstar-clickhouse-default-user-password +``` + +### MongoDB + +File: +- `environments/reference/credentials-mongodb.yaml` + +```yaml +secrets: + mode: externalSecret + externalSecret: + secretStoreRef: + name: gcp-secrets + kind: ClusterSecretStore + remoteRefs: + admin: + password: northstar-mongodb-admin-password + app: + password: northstar-mongodb-app-password + metrics: + password: northstar-mongodb-metrics-password +``` + +Important: +- Countly and MongoDB `app` must use the same password +- for new customers, reuse the same Secret Manager key in both files: + - `credentials-countly.yaml` + - `credentials-mongodb.yaml` +- use `-mongodb-app-password` for both + +## Step 8: Commit And Sync + +Commit: + +```bash +git add argocd/customers/.yaml environments/ +git commit -m "Add customer" +git push origin +``` + +Sync bootstrap: + +```bash +argocd app sync countly-bootstrap +``` + +Then inspect the generated customer apps: + +```bash +kubectl get applications -n argocd | grep +``` + +If needed, sync apps one by one: + +```bash +argocd app sync -cluster-secret-store +argocd app sync -external-secrets +argocd app sync -mongodb +argocd app sync -clickhouse +argocd app sync -kafka +argocd app sync -countly +``` + +## Step 9: Verify That Secrets Landed + +Check ExternalSecrets: + +```bash +kubectl get externalsecrets.external-secrets.io -n countly +kubectl get externalsecrets.external-secrets.io -n kafka +kubectl get externalsecrets.external-secrets.io -n clickhouse +kubectl get externalsecrets.external-secrets.io -n mongodb +``` + +Check the created Kubernetes secrets: + +```bash +kubectl get secret -n countly +kubectl get secret -n kafka +kubectl get secret -n clickhouse +kubectl get secret -n mongodb +``` + +If you want to inspect only one customer's secret-related resources, these are useful: + +```bash +kubectl get externalsecrets.external-secrets.io -n countly +kubectl get externalsecrets.external-secrets.io -n kafka +kubectl get externalsecrets.external-secrets.io -n clickhouse +kubectl get externalsecrets.external-secrets.io -n mongodb + +kubectl describe clustersecretstores.external-secrets.io gcp-secrets +kubectl describe externalsecret -n countly countly-common +kubectl describe externalsecret -n countly countly-clickhouse +kubectl describe externalsecret -n countly countly-mongodb +kubectl describe externalsecret -n kafka clickhouse-auth +``` + +## Step 10: Verify The Workloads + +Check pods: + +```bash +kubectl get pods -n countly +kubectl get pods -n kafka +kubectl get pods -n clickhouse +kubectl get pods -n mongodb +``` + +Check ingress: + +```bash +kubectl get ingress -n countly +curl -Ik https:// +``` + +## Switching Between Direct Values And Secret Manager + +This is meant to be easy. + +### To Move From Direct Values To Secret Manager + +1. Create the secrets in Google Secret Manager. +2. Change `secrets.mode: values` to `secrets.mode: externalSecret`. +3. Add the matching `remoteRefs`. +4. Commit and sync. + +### To Move From Secret Manager Back To Direct Values + +1. Put the values back into the `credentials-*.yaml` files. +2. Change `secrets.mode: externalSecret` to `secrets.mode: values`. +3. Remove the `remoteRefs`. +4. Commit and sync. + +The charts are designed so this is a values change, not a template rewrite. + +## Troubleshooting + +### `ClusterSecretStore` says `InvalidProviderConfig` + +Usually means: +- Workload Identity is not enabled +- node pool metadata mode is wrong +- GCP service account binding is wrong + +Check: + +```bash +kubectl describe clustersecretstores.external-secrets.io gcp-secrets +kubectl get sa -n external-secrets external-secrets -o yaml +kubectl logs -n external-secrets deploy/external-secrets +``` + +### `ExternalSecret` says secret does not exist + +Usually means: +- the secret name in `remoteRefs` is wrong +- the secret exists in the wrong GCP project +- the GCP service account cannot read it + +Check: + +```bash +gcloud secrets list --project= +kubectl describe externalsecret -n +``` + +### Argo CD UI shows too many apps and it is hard to focus on one customer + +Use the customer slug as your filter. + +Examples: + +```bash +kubectl get applications -n argocd | grep '^northstar-' +argocd app get northstar-cluster-secret-store +argocd app get northstar-external-secrets +argocd app get northstar-kafka +argocd app get northstar-countly +``` + +If you are debugging one customer, do not scan the whole dashboard. +Filter down to that one slug first. + +### One customer is broken but others are healthy + +That usually means the shared templates are fine and the customer-specific inputs are wrong. + +Check these first: +- `argocd/customers/.yaml` +- `environments//global.yaml` +- `environments//credentials-*.yaml` +- the Secret Manager secret names for that customer +- the Argo destination server for that customer + +This is a good quick drill: + +```bash +argocd app get -cluster-secret-store +argocd app get -external-secrets +argocd app get -mongodb +argocd app get -clickhouse +argocd app get -kafka +argocd app get -countly +``` + +If all other customers are fine and only one is broken, assume: +- wrong customer metadata +- wrong GSM secret names +- wrong IAM / Workload Identity for that cluster +- wrong customer-specific overrides + +before assuming the shared chart templates are broken. + +### Secret creation fails with location policy errors + +Use: + +```bash +gcloud secrets create \ + --replication-policy=user-managed \ + --locations=europe-west1 +``` + +### Argo app is on the right revision but old ExternalSecret specs still exist + +Delete the stale `ExternalSecret` objects and resync the app. + +### Kafka is degraded with `Pod is unschedulable or is not starting` + +This is usually capacity or topology, not Argo. + +Check: + +```bash +kubectl get pods -n kafka -o wide +kubectl get events -n kafka --sort-by=.lastTimestamp | tail -50 +kubectl get nodes +``` + +## Multi-Customer Rule Of Thumb + +For every new customer, keep this structure: +- one customer metadata file +- one environment folder +- one cluster +- one GCP service account +- one set of GSM secrets + +Do not share customer passwords across customers. +Do not reuse one customer secret name for another customer. + +## What To Do Next + +Once this guide is in place, the next normal step is: + +1. scaffold the customer +2. choose secret mode +3. fill metadata +4. create secrets if using GSM +5. commit +6. sync `countly-bootstrap` +7. verify the generated apps diff --git a/argocd/README.md b/argocd/README.md new file mode 100644 index 0000000..af3b173 --- /dev/null +++ b/argocd/README.md @@ -0,0 +1,314 @@ +# Argo CD Customer Deployment Guide + +This folder contains the GitOps setup used to deploy Countly to many customer clusters with Argo CD. + +The short version: + +1. Register the customer cluster in Argo CD. +2. Create a customer scaffold with the helper script. +3. Fill in the customer secrets and profile choices. +4. Commit the customer files. +5. Sync `countly-bootstrap`. +6. Argo CD creates the per-customer apps automatically. + +For a slower, step-by-step walkthrough, see [ONBOARDING.md](/Users/admin/cly/helm/argocd/ONBOARDING.md). + +## Folder Overview + +- `root-application.yaml` + - The parent Argo CD application. + - Sync this when you want Argo CD to pick up Git changes in `argocd/`. +- `projects/customers.yaml` + - Shared Argo CD project for customer apps. +- `operators/` + - Per-customer platform apps such as cert-manager, ingress, MongoDB operator, ClickHouse operator, and Strimzi. +- `applicationsets/` + - Generates one Argo CD `Application` per component per customer. +- `customers/` + - One small metadata file per customer. +- `../environments//` + - Helm values and secrets for that customer. + +## What Gets Created For Each Customer + +Core apps: +- MongoDB +- ClickHouse +- Kafka +- Countly + +Optional apps: +- Observability +- Migration + +Platform apps: +- cert-manager +- MongoDB CRDs/operator +- ClickHouse operator +- Strimzi Kafka operator +- NGINX ingress +- Let’s Encrypt issuer +- ClusterSecretStore for Google Secret Manager + +## Before You Start + +Make sure these are already true: + +1. Argo CD is installed in the tools cluster. +2. `countly-bootstrap` exists and is healthy. +3. The target customer cluster is registered in Argo CD. +4. DNS for the customer hostname points to the ingress load balancer you expect to use. + +Helpful checks: + +```bash +argocd app list +argocd cluster list +``` + +## Add A New Customer + +### 1. Create the customer scaffold + +Run: + +```bash +./scripts/new-argocd-customer.sh [--secret-mode values|gcp-secrets] +``` + +Example: + +```bash +./scripts/new-argocd-customer.sh acme https://1.2.3.4 acme.count.ly +``` + +If you know the customer will use Google Secret Manager, start with: + +```bash +./scripts/new-argocd-customer.sh --secret-mode gcp-secrets acme https://1.2.3.4 acme.count.ly +``` + +This creates: + +- `argocd/customers/.yaml` +- `environments//` + +### 2. Edit the customer metadata + +File: + +- `argocd/customers/.yaml` + +This file is the source of truth for: + +- `server` +- `gcpServiceAccountEmail` +- `secretManagerProjectID` +- `clusterProjectID` +- `clusterName` +- `clusterLocation` +- `hostname` +- `sizing` +- `security` +- `tls` +- `observability` +- `kafkaConnect` +- `migration` + +Typical example: + +```yaml +customer: acme +environment: acme +project: countly-customers +server: https://1.2.3.4 +gcpServiceAccountEmail: eso-acme@my-project.iam.gserviceaccount.com +secretManagerProjectID: countly-tools +clusterProjectID: countly-dev-313620 +clusterName: acme-prod +clusterLocation: us-central1 +hostname: acme.count.ly +sizing: tier1 +security: open +tls: letsencrypt +observability: disabled +kafkaConnect: balanced +migration: disabled +``` + +### 3. Fill in the customer secrets + +Files to review: + +- `environments//credentials-countly.yaml` +- `environments//credentials-clickhouse.yaml` +- `environments//credentials-kafka.yaml` +- `environments//credentials-mongodb.yaml` +- `environments//credentials-observability.yaml` +- `environments//credentials-migration.yaml` + +For direct-value deployments: + +- set `secrets.mode: values` where used +- fill in the real passwords and secrets +- keep matching passwords consistent across Countly, ClickHouse, Kafka, and MongoDB + +For external secret deployments: + +- use your external secret setup instead of committing direct values +- set `gcpServiceAccountEmail` in the customer metadata so the per-customer External Secrets operator can use Workload Identity +- for GAR image pulls, store Docker config JSON in Google Secret Manager and point `global.imagePullSecretExternalSecret.remoteRef.key` to that secret +- use the flat secret naming convention `--` + +Recommended secret names: + +- `-gar-dockerconfig` +- `-countly-encryption-reports-key` +- `-countly-web-session-secret` +- `-countly-password-secret` +- `-countly-clickhouse-password` +- `-kafka-connect-clickhouse-password` +- `-clickhouse-default-user-password` +- `-mongodb-admin-password` +- `-mongodb-app-password` +- `-mongodb-metrics-password` + +Use the same Secret Manager key for: +- Countly MongoDB password +- MongoDB `app` user password + +That means new customers should point both charts at: +- `-mongodb-app-password` + +Note: +- existing customer environments may still use older secret names +- use the new convention for all new customers +- migrate older customers only as a planned change + +## Important Rules + +### Customer metadata wins + +The customer file in `argocd/customers/` is the source of truth for: + +- cluster destination +- domain +- sizing +- TLS mode +- observability mode +- migration mode + +### Do not set these in `environments//countly.yaml` + +Do not manually set: + +- `ingress.hostname` +- `ingress.tls.mode` + +These are passed from customer metadata by the Countly `ApplicationSet`. + +### Kafka when migration is disabled + +If `migration: disabled`, make sure the drill ClickHouse sink connector is not enabled in: + +- `environments//kafka.yaml` + +This avoids creating a Kafka connector that depends on migration-owned tables. + +## Commit And Deploy + +After the customer files are ready: + +```bash +git add argocd/customers/.yaml environments/ +git commit -m "Add customer" +git push origin +``` + +Then tell Argo CD to pick it up: + +```bash +argocd app get countly-bootstrap --refresh +argocd app sync countly-bootstrap +kubectl get applications -n argocd | grep +``` + +## Expected App Order + +The apps are designed to settle roughly in this order: + +1. Platform operators and ingress +2. MongoDB and ClickHouse +3. Kafka +4. Countly +5. Observability +6. Migration + +It is normal for some apps to show `Progressing` for a while during first rollout. + +## Quick Verification + +After sync, useful checks are: + +```bash +kubectl get applications -n argocd | grep +kubectl get pods -A +kubectl get ingress -n countly +kubectl get certificate -n countly +curl -Ik https:// +``` + +## Removing A Customer + +1. Delete: + - `argocd/customers/.yaml` + - `environments//` +2. Commit and push. +3. Sync `countly-bootstrap`. +4. Confirm the customer apps disappear from Argo CD. + +## Common Problems + +### Countly still renders `countly.example.com` + +Cause: +- stale customer env overrides, or the `countly-app` `ApplicationSet` has not refreshed yet + +Fix: +- sync `countly-bootstrap` +- make sure the generated Countly app includes `ingress.hostname` and `ingress.tls.mode` + +### Kafka fails because of the drill sink connector + +Cause: +- migration is disabled, but the connector is still enabled + +Fix: +- disable `ch-sink-drill-events` in `environments//kafka.yaml` + +### Bootstrap changes are not reaching generated apps + +Cause: +- `countly-bootstrap` was not refreshed or synced + +Fix: + +```bash +argocd app get countly-bootstrap --refresh +argocd app sync countly-bootstrap +``` + +## Recommended Workflow For Engineers + +For each new customer: + +1. Register the cluster in Argo CD. +2. Run the scaffold script. +3. Edit `argocd/customers/.yaml`. +4. Fill in `environments//credentials-*.yaml`. +5. Review `environments//kafka.yaml` if migration is disabled. +6. Commit and push. +7. Sync `countly-bootstrap`. +8. Verify the generated apps, ingress, and certificate. + +If you follow that flow, you should not need to manually create Argo CD apps one by one. diff --git a/argocd/applicationsets/00-mongodb.yaml b/argocd/applicationsets/00-mongodb.yaml new file mode 100644 index 0000000..4cbd4e3 --- /dev/null +++ b/argocd/applicationsets/00-mongodb.yaml @@ -0,0 +1,69 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: countly-mongodb + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/helm.git + revision: main + files: + - path: argocd/customers/*.yaml + template: + metadata: + name: "{{ .customer }}-mongodb" + annotations: + argocd.argoproj.io/sync-wave: "0" + spec: + project: "{{ .project }}" + source: + repoURL: https://github.com/Countly/helm.git + targetRevision: main + path: charts/countly-mongodb + helm: + releaseName: countly-mongodb + valueFiles: + - "../../environments/{{ .environment }}/global.yaml" + - "../../profiles/sizing/{{ .sizing }}/mongodb.yaml" + - "../../profiles/security/{{ .security }}/mongodb.yaml" + - "../../environments/{{ .environment }}/mongodb.yaml" + - "../../environments/{{ .environment }}/credentials-mongodb.yaml" + parameters: + - name: argocd.enabled + value: "true" + - name: global.sizing + value: "{{ .sizing }}" + - name: global.security + value: "{{ .security }}" + destination: + server: "{{ .server }}" + namespace: mongodb + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m + ignoreDifferences: + - group: mongodbcommunity.mongodb.com + kind: MongoDBCommunity + jsonPointers: + - /status + - group: external-secrets.io + kind: ExternalSecret + jqPathExpressions: + - .spec.data[]?.remoteRef.conversionStrategy + - .spec.data[]?.remoteRef.decodingStrategy + - .spec.data[]?.remoteRef.metadataPolicy diff --git a/argocd/applicationsets/01-clickhouse.yaml b/argocd/applicationsets/01-clickhouse.yaml new file mode 100644 index 0000000..817fe76 --- /dev/null +++ b/argocd/applicationsets/01-clickhouse.yaml @@ -0,0 +1,75 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: countly-clickhouse + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/helm.git + revision: main + files: + - path: argocd/customers/*.yaml + template: + metadata: + name: "{{ .customer }}-clickhouse" + annotations: + argocd.argoproj.io/sync-wave: "0" + spec: + project: "{{ .project }}" + source: + repoURL: https://github.com/Countly/helm.git + targetRevision: main + path: charts/countly-clickhouse + helm: + releaseName: countly-clickhouse + valueFiles: + - "../../environments/{{ .environment }}/global.yaml" + - "../../profiles/sizing/{{ .sizing }}/clickhouse.yaml" + - "../../profiles/security/{{ .security }}/clickhouse.yaml" + - "../../environments/{{ .environment }}/clickhouse.yaml" + - "../../environments/{{ .environment }}/credentials-clickhouse.yaml" + parameters: + - name: argocd.enabled + value: "true" + - name: global.sizing + value: "{{ .sizing }}" + - name: global.security + value: "{{ .security }}" + destination: + server: "{{ .server }}" + namespace: clickhouse + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m + ignoreDifferences: + - group: clickhouse.com + kind: ClickHouseCluster + jsonPointers: + - /status + - group: clickhouse.com + kind: KeeperCluster + jsonPointers: + - /status + - /spec/containerTemplate/resources/requests/memory + - /spec/containerTemplate/resources/limits/memory + - group: external-secrets.io + kind: ExternalSecret + jqPathExpressions: + - .spec.data[]?.remoteRef.conversionStrategy + - .spec.data[]?.remoteRef.decodingStrategy + - .spec.data[]?.remoteRef.metadataPolicy diff --git a/argocd/applicationsets/02-kafka.yaml b/argocd/applicationsets/02-kafka.yaml new file mode 100644 index 0000000..db700d9 --- /dev/null +++ b/argocd/applicationsets/02-kafka.yaml @@ -0,0 +1,87 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: countly-kafka + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/helm.git + revision: main + files: + - path: argocd/customers/*.yaml + template: + metadata: + name: "{{ .customer }}-kafka" + annotations: + argocd.argoproj.io/sync-wave: "5" + spec: + project: "{{ .project }}" + source: + repoURL: https://github.com/Countly/helm.git + targetRevision: main + path: charts/countly-kafka + helm: + releaseName: countly-kafka + valueFiles: + - "../../environments/{{ .environment }}/global.yaml" + - "../../profiles/sizing/{{ .sizing }}/kafka.yaml" + - "../../profiles/kafka-connect/{{ .kafkaConnect }}/kafka.yaml" + - "../../profiles/observability/{{ .observability }}/kafka.yaml" + - "../../profiles/security/{{ .security }}/kafka.yaml" + - "../../environments/{{ .environment }}/kafka.yaml" + - "../../environments/{{ .environment }}/credentials-kafka.yaml" + parameters: + - name: argocd.enabled + value: "true" + - name: global.sizing + value: "{{ .sizing }}" + - name: global.security + value: "{{ .security }}" + - name: global.observability + value: "{{ .observability }}" + - name: global.kafkaConnect + value: "{{ .kafkaConnect }}" + destination: + server: "{{ .server }}" + namespace: kafka + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m + ignoreDifferences: + - group: kafka.strimzi.io + kind: Kafka + jsonPointers: + - /status + - group: kafka.strimzi.io + kind: KafkaConnect + jsonPointers: + - /status + - group: kafka.strimzi.io + kind: KafkaConnector + jsonPointers: + - /status + - group: kafka.strimzi.io + kind: KafkaNodePool + jsonPointers: + - /status + - group: external-secrets.io + kind: ExternalSecret + jqPathExpressions: + - .spec.data[]?.remoteRef.conversionStrategy + - .spec.data[]?.remoteRef.decodingStrategy + - .spec.data[]?.remoteRef.metadataPolicy diff --git a/argocd/applicationsets/03-countly.yaml b/argocd/applicationsets/03-countly.yaml new file mode 100644 index 0000000..3e80361 --- /dev/null +++ b/argocd/applicationsets/03-countly.yaml @@ -0,0 +1,81 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: countly-app + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/helm.git + revision: main + files: + - path: argocd/customers/*.yaml + template: + metadata: + name: "{{ .customer }}-countly" + annotations: + argocd.argoproj.io/sync-wave: "10" + spec: + project: "{{ .project }}" + source: + repoURL: https://github.com/Countly/helm.git + targetRevision: main + path: charts/countly + helm: + releaseName: countly + valueFiles: + - "../../environments/{{ .environment }}/global.yaml" + - "../../profiles/sizing/{{ .sizing }}/countly.yaml" + - "../../profiles/tls/{{ .tls }}/countly.yaml" + - "../../profiles/observability/{{ .observability }}/countly.yaml" + - "../../profiles/security/{{ .security }}/countly.yaml" + - "../../environments/{{ .environment }}/countly.yaml" + - "../../environments/{{ .environment }}/credentials-countly.yaml" + parameters: + - name: argocd.enabled + value: "true" + - name: ingress.hostname + value: "{{ .hostname }}" + - name: ingress.tls.mode + value: '{{ if eq .tls "none" }}http{{ else if eq .tls "provided" }}existingSecret{{ else }}{{ .tls }}{{ end }}' + - name: global.sizing + value: "{{ .sizing }}" + - name: global.security + value: "{{ .security }}" + - name: global.observability + value: "{{ .observability }}" + - name: global.tls + value: "{{ .tls }}" + - name: global.kafkaConnect + value: "{{ .kafkaConnect }}" + destination: + server: "{{ .server }}" + namespace: countly + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m + ignoreDifferences: + - group: networking.k8s.io + kind: Ingress + jsonPointers: + - /status + - group: external-secrets.io + kind: ExternalSecret + jqPathExpressions: + - .spec.data[]?.remoteRef.conversionStrategy + - .spec.data[]?.remoteRef.decodingStrategy + - .spec.data[]?.remoteRef.metadataPolicy diff --git a/argocd/applicationsets/04-observability.yaml b/argocd/applicationsets/04-observability.yaml new file mode 100644 index 0000000..1d69582 --- /dev/null +++ b/argocd/applicationsets/04-observability.yaml @@ -0,0 +1,61 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: countly-observability + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/helm.git + revision: main + files: + - path: argocd/customers/*.yaml + template: + metadata: + name: "{{ .customer }}-observability" + annotations: + argocd.argoproj.io/sync-wave: "15" + spec: + project: "{{ .project }}" + source: + repoURL: https://github.com/Countly/helm.git + targetRevision: main + path: '{{ if eq .observability "disabled" }}charts/noop{{ else }}charts/countly-observability{{ end }}' + helm: + releaseName: countly-observability + valueFiles: + - "../../environments/{{ .environment }}/global.yaml" + - "../../profiles/sizing/{{ .sizing }}/observability.yaml" + - "../../profiles/observability/{{ .observability }}/observability.yaml" + - "../../profiles/security/{{ .security }}/observability.yaml" + - "../../environments/{{ .environment }}/observability.yaml" + - "../../environments/{{ .environment }}/credentials-observability.yaml" + parameters: + - name: argocd.enabled + value: "true" + - name: global.sizing + value: "{{ .sizing }}" + - name: global.security + value: "{{ .security }}" + - name: global.observability + value: "{{ .observability }}" + destination: + server: "{{ .server }}" + namespace: observability + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m diff --git a/argocd/applicationsets/05-migration.yaml b/argocd/applicationsets/05-migration.yaml new file mode 100644 index 0000000..5bf7143 --- /dev/null +++ b/argocd/applicationsets/05-migration.yaml @@ -0,0 +1,52 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: countly-migration + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/helm.git + revision: main + files: + - path: argocd/customers/*.yaml + template: + metadata: + name: "{{ .customer }}-migration" + annotations: + argocd.argoproj.io/sync-wave: "10" + spec: + project: "{{ .project }}" + source: + repoURL: https://github.com/Countly/helm.git + targetRevision: main + path: '{{ if eq .migration "enabled" }}charts/countly-migration{{ else }}charts/noop{{ end }}' + helm: + releaseName: countly-migration + valueFiles: + - "../../environments/{{ .environment }}/global.yaml" + - "../../environments/{{ .environment }}/migration.yaml" + - "../../environments/{{ .environment }}/credentials-migration.yaml" + parameters: + - name: argocd.enabled + value: "true" + destination: + server: "{{ .server }}" + namespace: countly-migration + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + retry: + limit: 5 + backoff: + duration: 5s + factor: 2 + maxDuration: 3m diff --git a/argocd/operator-manifests/letsencrypt-prod-issuer/clusterissuer.yaml b/argocd/operator-manifests/letsencrypt-prod-issuer/clusterissuer.yaml new file mode 100644 index 0000000..4eca398 --- /dev/null +++ b/argocd/operator-manifests/letsencrypt-prod-issuer/clusterissuer.yaml @@ -0,0 +1,15 @@ +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-prod +spec: + acme: + email: devops@count.ly + server: https://acme-v02.api.letsencrypt.org/directory + privateKeySecretRef: + name: letsencrypt-prod + solvers: + - http01: + ingress: + class: nginx + diff --git a/argocd/operators/00-cert-manager.yaml b/argocd/operators/00-cert-manager.yaml new file mode 100644 index 0000000..4bd96cd --- /dev/null +++ b/argocd/operators/00-cert-manager.yaml @@ -0,0 +1,41 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: customer-cert-manager + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/helm.git + revision: main + files: + - path: argocd/customers/*.yaml + template: + metadata: + name: "{{ .customer }}-cert-manager" + annotations: + argocd.argoproj.io/sync-wave: "-30" + spec: + project: default + source: + repoURL: https://charts.jetstack.io + chart: cert-manager + targetRevision: v1.17.2 + helm: + releaseName: cert-manager + parameters: + - name: installCRDs + value: "true" + destination: + server: "{{ .server }}" + namespace: cert-manager + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/argocd/operators/01-mongodb-crds.yaml b/argocd/operators/01-mongodb-crds.yaml new file mode 100644 index 0000000..ab09f9b --- /dev/null +++ b/argocd/operators/01-mongodb-crds.yaml @@ -0,0 +1,38 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: customer-mongodb-crds + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/helm.git + revision: main + files: + - path: argocd/customers/*.yaml + template: + metadata: + name: "{{ .customer }}-mongodb-crds" + annotations: + argocd.argoproj.io/sync-wave: "-29" + spec: + project: default + source: + repoURL: https://github.com/mongodb/mongodb-kubernetes.git + targetRevision: "1.7.0" + path: public + directory: + include: crds.yaml + destination: + server: "{{ .server }}" + namespace: mongodb + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/argocd/operators/02-mongodb-operator.yaml b/argocd/operators/02-mongodb-operator.yaml new file mode 100644 index 0000000..59a68df --- /dev/null +++ b/argocd/operators/02-mongodb-operator.yaml @@ -0,0 +1,42 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: customer-mongodb-kubernetes-operator + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/helm.git + revision: main + files: + - path: argocd/customers/*.yaml + template: + metadata: + name: "{{ .customer }}-mongodb-kubernetes-operator" + annotations: + argocd.argoproj.io/sync-wave: "-28" + spec: + project: default + source: + repoURL: https://mongodb.github.io/helm-charts + chart: mongodb-kubernetes + targetRevision: 1.7.0 + helm: + releaseName: mongodb-kubernetes-operator + valuesObject: + operator: + watchedResources: + - mongodbcommunity + destination: + server: "{{ .server }}" + namespace: mongodb + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/argocd/operators/03-clickhouse-operator.yaml b/argocd/operators/03-clickhouse-operator.yaml new file mode 100644 index 0000000..569db87 --- /dev/null +++ b/argocd/operators/03-clickhouse-operator.yaml @@ -0,0 +1,41 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: customer-clickhouse-operator + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/helm.git + revision: main + files: + - path: argocd/customers/*.yaml + template: + metadata: + name: "{{ .customer }}-clickhouse-operator" + annotations: + argocd.argoproj.io/sync-wave: "-27" + spec: + project: default + source: + repoURL: ghcr.io/clickhouse + chart: clickhouse-operator-helm + targetRevision: 0.0.2 + helm: + releaseName: clickhouse-operator + valuesObject: + certManager: + install: false + destination: + server: "{{ .server }}" + namespace: clickhouse-operator-system + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/argocd/operators/04-strimzi-operator.yaml b/argocd/operators/04-strimzi-operator.yaml new file mode 100644 index 0000000..3d83405 --- /dev/null +++ b/argocd/operators/04-strimzi-operator.yaml @@ -0,0 +1,38 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: customer-strimzi-kafka-operator + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/helm.git + revision: main + files: + - path: argocd/customers/*.yaml + template: + metadata: + name: "{{ .customer }}-strimzi-kafka-operator" + annotations: + argocd.argoproj.io/sync-wave: "-26" + spec: + project: default + source: + repoURL: https://strimzi.io/charts/ + chart: strimzi-kafka-operator + targetRevision: 0.51.0 + helm: + releaseName: strimzi-kafka-operator + destination: + server: "{{ .server }}" + namespace: kafka + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/argocd/operators/05-nginx-ingress.yaml b/argocd/operators/05-nginx-ingress.yaml new file mode 100644 index 0000000..2883136 --- /dev/null +++ b/argocd/operators/05-nginx-ingress.yaml @@ -0,0 +1,79 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: customer-nginx-ingress + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/helm.git + revision: main + files: + - path: argocd/customers/*.yaml + template: + metadata: + name: "{{ .customer }}-nginx-ingress" + annotations: + argocd.argoproj.io/sync-wave: "-25" + spec: + project: default + sources: + - repoURL: https://helm.nginx.com/stable + chart: nginx-ingress + targetRevision: 2.1.0 + helm: + releaseName: nginx-ingress + valueFiles: + - $values/nginx-ingress-values.yaml + - repoURL: https://github.com/Countly/helm.git + targetRevision: main + ref: values + destination: + server: "{{ .server }}" + namespace: ingress-nginx + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + ignoreDifferences: + - group: apiextensions.k8s.io + kind: CustomResourceDefinition + name: apdoslogconfs.appprotectdos.f5.com + jsonPointers: + - /spec/preserveUnknownFields + - group: apiextensions.k8s.io + kind: CustomResourceDefinition + name: apdospolicies.appprotectdos.f5.com + jsonPointers: + - /spec/preserveUnknownFields + - group: apiextensions.k8s.io + kind: CustomResourceDefinition + name: aplogconfs.appprotect.f5.com + jsonPointers: + - /spec/preserveUnknownFields + - group: apiextensions.k8s.io + kind: CustomResourceDefinition + name: appolicies.appprotect.f5.com + jsonPointers: + - /spec/preserveUnknownFields + - group: apiextensions.k8s.io + kind: CustomResourceDefinition + name: apusersigs.appprotect.f5.com + jsonPointers: + - /spec/preserveUnknownFields + - group: "" + kind: Service + name: nginx-ingress-controller + namespace: ingress-nginx + jsonPointers: + - /metadata/annotations/cloud.google.com~1neg + - /spec/healthCheckNodePort + - /spec/ports/0/nodePort + - /spec/ports/1/nodePort diff --git a/argocd/operators/06-letsencrypt-prod-issuer-app.yaml b/argocd/operators/06-letsencrypt-prod-issuer-app.yaml new file mode 100644 index 0000000..6046cf4 --- /dev/null +++ b/argocd/operators/06-letsencrypt-prod-issuer-app.yaml @@ -0,0 +1,37 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: customer-letsencrypt-prod-issuer + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/helm.git + revision: main + files: + - path: argocd/customers/*.yaml + template: + metadata: + name: "{{ .customer }}-letsencrypt-prod-issuer" + annotations: + argocd.argoproj.io/sync-wave: "-24" + spec: + project: default + source: + repoURL: https://github.com/Countly/helm.git + targetRevision: main + path: '{{ if eq .tls "letsencrypt" }}argocd/operator-manifests/letsencrypt-prod-issuer{{ else }}charts/noop{{ end }}' + directory: + recurse: true + destination: + server: "{{ .server }}" + namespace: cert-manager + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - ServerSideApply=true diff --git a/argocd/operators/07-external-secrets-operator.yaml b/argocd/operators/07-external-secrets-operator.yaml new file mode 100644 index 0000000..f20e732 --- /dev/null +++ b/argocd/operators/07-external-secrets-operator.yaml @@ -0,0 +1,44 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: customer-external-secrets + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/helm.git + revision: main + files: + - path: argocd/customers/*.yaml + template: + metadata: + name: "{{ .customer }}-external-secrets" + annotations: + argocd.argoproj.io/sync-wave: "-23" + spec: + project: default + source: + repoURL: https://charts.external-secrets.io + chart: external-secrets + targetRevision: 1.3.1 + helm: + releaseName: external-secrets + values: | + installCRDs: true + serviceAccount: + create: true + annotations: + iam.gke.io/gcp-service-account: "{{ .gcpServiceAccountEmail }}" + destination: + server: "{{ .server }}" + namespace: external-secrets + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - CreateNamespace=true + - ServerSideApply=true diff --git a/argocd/operators/08-cluster-secret-store.yaml b/argocd/operators/08-cluster-secret-store.yaml new file mode 100644 index 0000000..7a39007 --- /dev/null +++ b/argocd/operators/08-cluster-secret-store.yaml @@ -0,0 +1,48 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: customer-cluster-secret-store + namespace: argocd +spec: + goTemplate: true + goTemplateOptions: + - missingkey=error + generators: + - git: + repoURL: https://github.com/Countly/helm.git + revision: main + files: + - path: argocd/customers/*.yaml + template: + metadata: + name: "{{ .customer }}-cluster-secret-store" + annotations: + argocd.argoproj.io/sync-wave: "-22" + spec: + project: default + source: + repoURL: https://github.com/Countly/helm.git + targetRevision: main + path: charts/countly-cluster-secret-store + helm: + releaseName: countly-cluster-secret-store + parameters: + - name: secretStore.name + value: "gcp-secrets" + - name: secretStore.secretManagerProjectID + value: "{{ .secretManagerProjectID }}" + - name: secretStore.clusterProjectID + value: "{{ .clusterProjectID }}" + - name: secretStore.clusterName + value: "{{ .clusterName }}" + - name: secretStore.clusterLocation + value: "{{ .clusterLocation }}" + destination: + server: "{{ .server }}" + namespace: external-secrets + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - ServerSideApply=true diff --git a/argocd/projects/customers.yaml b/argocd/projects/customers.yaml new file mode 100644 index 0000000..d48d4f7 --- /dev/null +++ b/argocd/projects/customers.yaml @@ -0,0 +1,36 @@ +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: countly-customers + namespace: argocd +spec: + description: "Shared AppProject for GitOps-managed Countly customer environments" + sourceRepos: + - '*' + destinations: + - server: '*' + namespace: mongodb + - server: '*' + namespace: clickhouse + - server: '*' + namespace: kafka + - server: '*' + namespace: countly + - server: '*' + namespace: observability + - server: '*' + namespace: countly-migration + clusterResourceWhitelist: + - group: "" + kind: Namespace + - group: storage.k8s.io + kind: StorageClass + - group: rbac.authorization.k8s.io + kind: ClusterRole + - group: rbac.authorization.k8s.io + kind: ClusterRoleBinding + - group: cert-manager.io + kind: ClusterIssuer + namespaceResourceWhitelist: + - group: '*' + kind: '*' diff --git a/argocd/root-application.yaml b/argocd/root-application.yaml new file mode 100644 index 0000000..a84dae0 --- /dev/null +++ b/argocd/root-application.yaml @@ -0,0 +1,23 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: countly-bootstrap + namespace: argocd +spec: + project: default + source: + repoURL: https://github.com/Countly/helm.git + targetRevision: main + path: argocd + directory: + recurse: true + exclude: "{operator-manifests/**,customers/**}" + destination: + server: https://kubernetes.default.svc + namespace: argocd + syncPolicy: + automated: + prune: true + selfHeal: true + syncOptions: + - ServerSideApply=true diff --git a/charts/countly-argocd/templates/app-clickhouse.yaml b/charts/countly-argocd/templates/app-clickhouse.yaml index b164050..fde892f 100644 --- a/charts/countly-argocd/templates/app-clickhouse.yaml +++ b/charts/countly-argocd/templates/app-clickhouse.yaml @@ -23,7 +23,7 @@ spec: - ../../profiles/sizing/{{ .Values.global.sizing }}/clickhouse.yaml - ../../profiles/security/{{ .Values.global.security }}/clickhouse.yaml - ../../environments/{{ .Values.environment }}/clickhouse.yaml - - ../../environments/{{ .Values.environment }}/secrets-clickhouse.yaml + - ../../environments/{{ .Values.environment }}/credentials-clickhouse.yaml parameters: - name: argocd.enabled value: "true" diff --git a/charts/countly-argocd/templates/app-countly.yaml b/charts/countly-argocd/templates/app-countly.yaml index 54c8592..e71346d 100644 --- a/charts/countly-argocd/templates/app-countly.yaml +++ b/charts/countly-argocd/templates/app-countly.yaml @@ -25,7 +25,7 @@ spec: - ../../profiles/observability/{{ .Values.global.observability }}/countly.yaml - ../../profiles/security/{{ .Values.global.security }}/countly.yaml - ../../environments/{{ .Values.environment }}/countly.yaml - - ../../environments/{{ .Values.environment }}/secrets-countly.yaml + - ../../environments/{{ .Values.environment }}/credentials-countly.yaml parameters: - name: argocd.enabled value: "true" diff --git a/charts/countly-argocd/templates/app-kafka.yaml b/charts/countly-argocd/templates/app-kafka.yaml index b373087..9ad325f 100644 --- a/charts/countly-argocd/templates/app-kafka.yaml +++ b/charts/countly-argocd/templates/app-kafka.yaml @@ -25,7 +25,7 @@ spec: - ../../profiles/observability/{{ .Values.global.observability }}/kafka.yaml - ../../profiles/security/{{ .Values.global.security }}/kafka.yaml - ../../environments/{{ .Values.environment }}/kafka.yaml - - ../../environments/{{ .Values.environment }}/secrets-kafka.yaml + - ../../environments/{{ .Values.environment }}/credentials-kafka.yaml parameters: - name: argocd.enabled value: "true" diff --git a/charts/countly-argocd/templates/app-migration.yaml b/charts/countly-argocd/templates/app-migration.yaml index ab30a27..8fbb152 100644 --- a/charts/countly-argocd/templates/app-migration.yaml +++ b/charts/countly-argocd/templates/app-migration.yaml @@ -21,7 +21,7 @@ spec: valueFiles: - ../../environments/{{ .Values.environment }}/global.yaml - ../../environments/{{ .Values.environment }}/migration.yaml - - ../../environments/{{ .Values.environment }}/secrets-migration.yaml + - ../../environments/{{ .Values.environment }}/credentials-migration.yaml parameters: - name: argocd.enabled value: "true" diff --git a/charts/countly-argocd/templates/app-mongodb.yaml b/charts/countly-argocd/templates/app-mongodb.yaml index ce470a3..86460b9 100644 --- a/charts/countly-argocd/templates/app-mongodb.yaml +++ b/charts/countly-argocd/templates/app-mongodb.yaml @@ -23,7 +23,7 @@ spec: - ../../profiles/sizing/{{ .Values.global.sizing }}/mongodb.yaml - ../../profiles/security/{{ .Values.global.security }}/mongodb.yaml - ../../environments/{{ .Values.environment }}/mongodb.yaml - - ../../environments/{{ .Values.environment }}/secrets-mongodb.yaml + - ../../environments/{{ .Values.environment }}/credentials-mongodb.yaml parameters: - name: argocd.enabled value: "true" diff --git a/charts/countly-argocd/templates/app-observability.yaml b/charts/countly-argocd/templates/app-observability.yaml index 27276d7..9876d82 100644 --- a/charts/countly-argocd/templates/app-observability.yaml +++ b/charts/countly-argocd/templates/app-observability.yaml @@ -24,7 +24,7 @@ spec: - ../../profiles/observability/{{ .Values.global.observability }}/observability.yaml - ../../profiles/security/{{ .Values.global.security }}/observability.yaml - ../../environments/{{ .Values.environment }}/observability.yaml - - ../../environments/{{ .Values.environment }}/secrets-observability.yaml + - ../../environments/{{ .Values.environment }}/credentials-observability.yaml parameters: - name: argocd.enabled value: "true" diff --git a/charts/countly-clickhouse/Chart.yaml b/charts/countly-clickhouse/Chart.yaml index 221a694..f995595 100644 --- a/charts/countly-clickhouse/Chart.yaml +++ b/charts/countly-clickhouse/Chart.yaml @@ -3,7 +3,7 @@ name: countly-clickhouse description: ClickHouse for Countly analytics via ClickHouse Operator type: application version: 0.1.0 -appVersion: "26.2" +appVersion: "26.3" home: https://countly.com icon: https://count.ly/images/logos/countly-logo.svg sources: diff --git a/charts/countly-clickhouse/templates/external-secret-default-password.yaml b/charts/countly-clickhouse/templates/external-secret-default-password.yaml new file mode 100644 index 0000000..f234c51 --- /dev/null +++ b/charts/countly-clickhouse/templates/external-secret-default-password.yaml @@ -0,0 +1,22 @@ +{{- if and (eq (.Values.secrets.mode | default "values") "externalSecret") (not .Values.auth.defaultUserPassword.existingSecret) }} +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: {{ .Values.auth.defaultUserPassword.secretName }} + labels: + {{- include "countly-clickhouse.labels" . | nindent 4 }} + annotations: + {{- include "countly-clickhouse.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} +spec: + refreshInterval: {{ .Values.secrets.externalSecret.refreshInterval | default "1h" }} + secretStoreRef: + name: {{ required "secrets.externalSecret.secretStoreRef.name is required when secrets.mode=externalSecret" .Values.secrets.externalSecret.secretStoreRef.name }} + kind: {{ .Values.secrets.externalSecret.secretStoreRef.kind | default "ClusterSecretStore" }} + target: + name: {{ .Values.auth.defaultUserPassword.secretName }} + creationPolicy: Owner + data: + - secretKey: {{ .Values.auth.defaultUserPassword.key }} + remoteRef: + key: {{ required "secrets.externalSecret.remoteRefs.defaultUserPassword is required when secrets.mode=externalSecret" .Values.secrets.externalSecret.remoteRefs.defaultUserPassword }} +{{- end }} diff --git a/charts/countly-clickhouse/templates/secret-default-password.yaml b/charts/countly-clickhouse/templates/secret-default-password.yaml index cbeaa37..87cf770 100644 --- a/charts/countly-clickhouse/templates/secret-default-password.yaml +++ b/charts/countly-clickhouse/templates/secret-default-password.yaml @@ -1,4 +1,4 @@ -{{- if not .Values.auth.defaultUserPassword.existingSecret }} +{{- if and (ne (.Values.secrets.mode | default "values") "externalSecret") (not .Values.auth.defaultUserPassword.existingSecret) }} apiVersion: v1 kind: Secret metadata: diff --git a/charts/countly-clickhouse/values.schema.json b/charts/countly-clickhouse/values.schema.json index 3bb5f65..404976d 100644 --- a/charts/countly-clickhouse/values.schema.json +++ b/charts/countly-clickhouse/values.schema.json @@ -143,7 +143,30 @@ "secrets": { "type": "object", "properties": { - "keep": { "type": "boolean" } + "keep": { "type": "boolean" }, + "mode": { + "type": "string", + "enum": ["values", "existingSecret", "externalSecret"] + }, + "externalSecret": { + "type": "object", + "properties": { + "refreshInterval": { "type": "string" }, + "secretStoreRef": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "kind": { "type": "string" } + } + }, + "remoteRefs": { + "type": "object", + "properties": { + "defaultUserPassword": { "type": "string" } + } + } + } + } } } } diff --git a/charts/countly-clickhouse/values.yaml b/charts/countly-clickhouse/values.yaml index 9051065..665f32a 100644 --- a/charts/countly-clickhouse/values.yaml +++ b/charts/countly-clickhouse/values.yaml @@ -29,8 +29,7 @@ argocd: clickhouseOperator: apiVersion: clickhouse.com/v1alpha1 -# -- ClickHouse server version -version: "26.2" +version: "26.3" # -- Number of shards in the cluster shards: 1 # -- Number of replicas per shard @@ -213,3 +212,11 @@ serviceMonitor: secrets: # -- Preserve secrets on helm uninstall/upgrade keep: true + mode: values # values | existingSecret | externalSecret + externalSecret: + refreshInterval: "1h" + secretStoreRef: + name: "" + kind: ClusterSecretStore + remoteRefs: + defaultUserPassword: "" diff --git a/charts/countly-cluster-secret-store/Chart.yaml b/charts/countly-cluster-secret-store/Chart.yaml new file mode 100644 index 0000000..219c646 --- /dev/null +++ b/charts/countly-cluster-secret-store/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: countly-cluster-secret-store +description: ClusterSecretStore for External Secrets Operator with GCP Secret Manager +type: application +version: 0.1.0 +appVersion: "1.0" diff --git a/charts/countly-cluster-secret-store/templates/clustersecretstore.yaml b/charts/countly-cluster-secret-store/templates/clustersecretstore.yaml new file mode 100644 index 0000000..991843c --- /dev/null +++ b/charts/countly-cluster-secret-store/templates/clustersecretstore.yaml @@ -0,0 +1,16 @@ +apiVersion: external-secrets.io/v1 +kind: ClusterSecretStore +metadata: + name: {{ .Values.secretStore.name }} +spec: + provider: + gcpsm: + projectID: {{ required "secretStore.secretManagerProjectID is required" .Values.secretStore.secretManagerProjectID | quote }} + auth: + workloadIdentity: + clusterProjectID: {{ required "secretStore.clusterProjectID is required" .Values.secretStore.clusterProjectID | quote }} + clusterName: {{ required "secretStore.clusterName is required" .Values.secretStore.clusterName | quote }} + clusterLocation: {{ required "secretStore.clusterLocation is required" .Values.secretStore.clusterLocation | quote }} + serviceAccountRef: + name: {{ .Values.secretStore.serviceAccountRef.name | quote }} + namespace: {{ .Values.secretStore.serviceAccountRef.namespace | quote }} diff --git a/charts/countly-cluster-secret-store/values.yaml b/charts/countly-cluster-secret-store/values.yaml new file mode 100644 index 0000000..7b72520 --- /dev/null +++ b/charts/countly-cluster-secret-store/values.yaml @@ -0,0 +1,9 @@ +secretStore: + name: gcp-secrets + secretManagerProjectID: "" + clusterProjectID: "" + clusterName: "" + clusterLocation: "" + serviceAccountRef: + name: external-secrets + namespace: external-secrets diff --git a/charts/countly-kafka/templates/_helpers.tpl b/charts/countly-kafka/templates/_helpers.tpl index a4f35c3..d67f4d2 100644 --- a/charts/countly-kafka/templates/_helpers.tpl +++ b/charts/countly-kafka/templates/_helpers.tpl @@ -97,3 +97,14 @@ ClickHouse Connect secret name {{ .Values.kafkaConnect.clickhouse.secretName }} {{- end -}} {{- end -}} + +{{/* +Resolve the Kafka Connect image. + +Kafka Connect now uses the public Countly image by default. We intentionally +do not rewrite it through global.imageSource.mode because Countly app images +and Kafka Connect images can follow different distribution paths. +*/}} +{{- define "countly-kafka.connectImage" -}} +{{- .Values.kafkaConnect.image -}} +{{- end -}} diff --git a/charts/countly-kafka/templates/external-secret-clickhouse-connect.yaml b/charts/countly-kafka/templates/external-secret-clickhouse-connect.yaml new file mode 100644 index 0000000..ff8af55 --- /dev/null +++ b/charts/countly-kafka/templates/external-secret-clickhouse-connect.yaml @@ -0,0 +1,39 @@ +{{- if and .Values.kafkaConnect.enabled (eq (.Values.secrets.mode | default "values") "externalSecret") (not .Values.kafkaConnect.clickhouse.existingSecret) }} +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: {{ .Values.kafkaConnect.clickhouse.secretName }} + labels: + {{- include "countly-kafka.labels" . | nindent 4 }} + annotations: + {{- include "countly-kafka.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} +spec: + refreshInterval: {{ .Values.secrets.externalSecret.refreshInterval | default "1h" }} + secretStoreRef: + name: {{ required "secrets.externalSecret.secretStoreRef.name is required when secrets.mode=externalSecret" .Values.secrets.externalSecret.secretStoreRef.name }} + kind: {{ .Values.secrets.externalSecret.secretStoreRef.kind | default "ClusterSecretStore" }} + target: + name: {{ .Values.kafkaConnect.clickhouse.secretName }} + creationPolicy: Owner + template: + engineVersion: v2 + mergePolicy: Merge + data: + {{- if not .Values.secrets.externalSecret.remoteRefs.clickhouse.username }} + username: {{ .Values.kafkaConnect.clickhouse.username | quote }} + {{- end }} + {{- if not .Values.secrets.externalSecret.remoteRefs.clickhouse.password }} + password: {{ .Values.kafkaConnect.clickhouse.password | quote }} + {{- end }} + data: + {{- if .Values.secrets.externalSecret.remoteRefs.clickhouse.username }} + - secretKey: username + remoteRef: + key: {{ required "secrets.externalSecret.remoteRefs.clickhouse.username is required when secrets.mode=externalSecret" .Values.secrets.externalSecret.remoteRefs.clickhouse.username }} + {{- end }} + {{- if .Values.secrets.externalSecret.remoteRefs.clickhouse.password }} + - secretKey: password + remoteRef: + key: {{ required "secrets.externalSecret.remoteRefs.clickhouse.password is required when secrets.mode=externalSecret" .Values.secrets.externalSecret.remoteRefs.clickhouse.password }} + {{- end }} +{{- end }} diff --git a/charts/countly-kafka/templates/kafkaconnect.yaml b/charts/countly-kafka/templates/kafkaconnect.yaml index fc6994c..401df80 100644 --- a/charts/countly-kafka/templates/kafkaconnect.yaml +++ b/charts/countly-kafka/templates/kafkaconnect.yaml @@ -20,7 +20,7 @@ spec: name: {{ include "countly-kafka.fullname" . }}-metrics key: connect-metrics-config.yml {{- end }} - image: {{ .Values.kafkaConnect.image }} + image: {{ include "countly-kafka.connectImage" . | quote }} groupId: {{ .Values.kafkaConnect.workerConfig | dig "group.id" "connect-cluster" }} offsetStorageTopic: {{ .Values.kafkaConnect.workerConfig | dig "offset.storage.topic" "connect-offsets" }} configStorageTopic: {{ .Values.kafkaConnect.workerConfig | dig "config.storage.topic" "connect-configs" }} diff --git a/charts/countly-kafka/templates/secret-clickhouse-connect.yaml b/charts/countly-kafka/templates/secret-clickhouse-connect.yaml index 6039fca..54f2090 100644 --- a/charts/countly-kafka/templates/secret-clickhouse-connect.yaml +++ b/charts/countly-kafka/templates/secret-clickhouse-connect.yaml @@ -1,4 +1,4 @@ -{{- if and .Values.kafkaConnect.enabled (not .Values.kafkaConnect.clickhouse.existingSecret) }} +{{- if and .Values.kafkaConnect.enabled (ne (.Values.secrets.mode | default "values") "externalSecret") (not .Values.kafkaConnect.clickhouse.existingSecret) }} apiVersion: v1 kind: Secret metadata: diff --git a/charts/countly-kafka/values.schema.json b/charts/countly-kafka/values.schema.json index d282159..7f77465 100644 --- a/charts/countly-kafka/values.schema.json +++ b/charts/countly-kafka/values.schema.json @@ -3,7 +3,16 @@ "type": "object", "required": ["version"], "properties": { - "global": { "type": "object" }, + "global": { + "type": "object", + "properties": { + "imageRegistry": { "type": "string" }, + "imagePullSecrets": { "type": "array" }, + "storageClass": { "type": "string" }, + "sizing": { "type": "string" }, + "scheduling": { "type": "object" } + } + }, "createNamespace": { "type": "boolean" }, "strimzi": { "type": "object", @@ -130,7 +139,36 @@ "secrets": { "type": "object", "properties": { - "keep": { "type": "boolean" } + "keep": { "type": "boolean" }, + "mode": { + "type": "string", + "enum": ["values", "existingSecret", "externalSecret"] + }, + "externalSecret": { + "type": "object", + "properties": { + "refreshInterval": { "type": "string" }, + "secretStoreRef": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "kind": { "type": "string" } + } + }, + "remoteRefs": { + "type": "object", + "properties": { + "clickhouse": { + "type": "object", + "properties": { + "username": { "type": "string" }, + "password": { "type": "string" } + } + } + } + } + } + } } } } diff --git a/charts/countly-kafka/values.yaml b/charts/countly-kafka/values.yaml index 94ccff9..cb6938d 100644 --- a/charts/countly-kafka/values.yaml +++ b/charts/countly-kafka/values.yaml @@ -136,8 +136,7 @@ kafkaConnect: enabled: true # -- Connect cluster name (used in Strimzi resource name) name: connect-ch - # -- Container image with the ClickHouse sink connector baked in - image: "gcr.io/countly-dev-313620/strimzi/kafka-connect-clickhouse:4.2.0-1.3.5-strimzi-amd64" + image: "countly/strimzi-kafka-connect-clickhouse:kafka4.2.0-ch1.3.5-strimzi0.51-otel2.12.0" # -- Number of Connect worker replicas replicas: 2 # -- Override bootstrap servers (empty = auto-resolve from this Kafka cluster) @@ -339,3 +338,13 @@ networkPolicy: secrets: # -- Preserve secrets on helm uninstall/upgrade keep: true + mode: values # values | existingSecret | externalSecret + externalSecret: + refreshInterval: "1h" + secretStoreRef: + name: "" + kind: ClusterSecretStore + remoteRefs: + clickhouse: + username: "" + password: "" diff --git a/charts/countly-migration/templates/external-secret.yaml b/charts/countly-migration/templates/external-secret.yaml index 1daeb50..14b81da 100644 --- a/charts/countly-migration/templates/external-secret.yaml +++ b/charts/countly-migration/templates/external-secret.yaml @@ -1,5 +1,5 @@ {{- if eq (.Values.secrets.mode | default "values") "externalSecret" }} -apiVersion: external-secrets.io/v1beta1 +apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: name: {{ include "countly-migration.fullname" . }} diff --git a/charts/countly-mongodb/templates/mongodbcommunity.yaml b/charts/countly-mongodb/templates/mongodbcommunity.yaml index e3ba8d5..242aaab 100644 --- a/charts/countly-mongodb/templates/mongodbcommunity.yaml +++ b/charts/countly-mongodb/templates/mongodbcommunity.yaml @@ -19,6 +19,17 @@ spec: enabled: true {{- end }} users: + {{- if .Values.users.admin.enabled }} + - name: {{ .Values.users.admin.name }} + db: {{ .Values.users.admin.database }} + passwordSecretRef: + name: {{ .Values.users.admin.passwordSecretName }} + key: {{ .Values.users.admin.passwordSecretKey }} + roles: + {{- toYaml .Values.users.admin.roles | nindent 8 }} + scramCredentialsSecretName: {{ .Values.users.admin.name }}-scram + connectionStringSecretName: {{ include "countly-mongodb.fullname" . }}-{{ .Values.users.admin.name }}-mongodb-conn + {{- end }} - name: {{ .Values.users.app.name }} db: {{ .Values.users.app.database }} passwordSecretRef: diff --git a/charts/countly-mongodb/templates/secret-passwords.yaml b/charts/countly-mongodb/templates/secret-passwords.yaml index 1200229..06cfc2b 100644 --- a/charts/countly-mongodb/templates/secret-passwords.yaml +++ b/charts/countly-mongodb/templates/secret-passwords.yaml @@ -1,3 +1,29 @@ +{{- if eq (.Values.secrets.mode | default "values") "values" }} +{{- if .Values.users.admin.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.users.admin.passwordSecretName }} + labels: + {{- include "countly-mongodb.labels" . | nindent 4 }} + annotations: + {{- if .Values.secrets.keep }} + helm.sh/resource-policy: keep + {{- end }} + {{- include "countly-mongodb.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} +type: Opaque +data: + {{ .Values.users.admin.passwordSecretKey }}: |- + {{- $existing := lookup "v1" "Secret" .Release.Namespace .Values.users.admin.passwordSecretName -}} + {{- if and $existing (not .Values.users.admin.password) }} + {{ index $existing.data .Values.users.admin.passwordSecretKey }} + {{- else if .Values.users.admin.password }} + {{ .Values.users.admin.password | b64enc }} + {{- else }} + {{- fail "MongoDB admin user password is required on first install when users.admin.enabled=true. Set users.admin.password." }} + {{- end }} +--- +{{- end }} apiVersion: v1 kind: Secret metadata: @@ -41,7 +67,77 @@ data: {{ index $existing.data .Values.users.metrics.passwordSecretKey }} {{- else if .Values.users.metrics.password }} {{ .Values.users.metrics.password | b64enc }} - {{- else }} - {{- fail "MongoDB metrics user password is required on first install. Set users.metrics.password." }} - {{- end }} +{{- else }} +{{- fail "MongoDB metrics user password is required on first install. Set users.metrics.password." }} +{{- end }} +{{- end }} +{{- end }} +{{- if eq (.Values.secrets.mode | default "values") "externalSecret" }} +{{- if .Values.users.admin.enabled }} +--- +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: {{ .Values.users.admin.passwordSecretName }} + labels: + {{- include "countly-mongodb.labels" . | nindent 4 }} + annotations: + {{- include "countly-mongodb.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} +spec: + refreshInterval: {{ .Values.secrets.externalSecret.refreshInterval | default "1h" }} + secretStoreRef: + name: {{ required "secrets.externalSecret.secretStoreRef.name is required when secrets.mode=externalSecret" .Values.secrets.externalSecret.secretStoreRef.name }} + kind: {{ .Values.secrets.externalSecret.secretStoreRef.kind | default "ClusterSecretStore" }} + target: + name: {{ .Values.users.admin.passwordSecretName }} + creationPolicy: Owner + data: + - secretKey: {{ .Values.users.admin.passwordSecretKey }} + remoteRef: + key: {{ required "secrets.externalSecret.remoteRefs.admin.password is required when users.admin.enabled=true and secrets.mode=externalSecret" .Values.secrets.externalSecret.remoteRefs.admin.password }} +{{- end }} +--- +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: {{ .Values.users.app.passwordSecretName }} + labels: + {{- include "countly-mongodb.labels" . | nindent 4 }} + annotations: + {{- include "countly-mongodb.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} +spec: + refreshInterval: {{ .Values.secrets.externalSecret.refreshInterval | default "1h" }} + secretStoreRef: + name: {{ required "secrets.externalSecret.secretStoreRef.name is required when secrets.mode=externalSecret" .Values.secrets.externalSecret.secretStoreRef.name }} + kind: {{ .Values.secrets.externalSecret.secretStoreRef.kind | default "ClusterSecretStore" }} + target: + name: {{ .Values.users.app.passwordSecretName }} + creationPolicy: Owner + data: + - secretKey: {{ .Values.users.app.passwordSecretKey }} + remoteRef: + key: {{ required "secrets.externalSecret.remoteRefs.app.password is required when secrets.mode=externalSecret" .Values.secrets.externalSecret.remoteRefs.app.password }} +{{- if .Values.users.metrics.enabled }} +--- +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: {{ .Values.users.metrics.passwordSecretName }} + labels: + {{- include "countly-mongodb.labels" . | nindent 4 }} + annotations: + {{- include "countly-mongodb.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} +spec: + refreshInterval: {{ .Values.secrets.externalSecret.refreshInterval | default "1h" }} + secretStoreRef: + name: {{ required "secrets.externalSecret.secretStoreRef.name is required when secrets.mode=externalSecret" .Values.secrets.externalSecret.secretStoreRef.name }} + kind: {{ .Values.secrets.externalSecret.secretStoreRef.kind | default "ClusterSecretStore" }} + target: + name: {{ .Values.users.metrics.passwordSecretName }} + creationPolicy: Owner + data: + - secretKey: {{ .Values.users.metrics.passwordSecretKey }} + remoteRef: + key: {{ required "secrets.externalSecret.remoteRefs.metrics.password is required when secrets.mode=externalSecret" .Values.secrets.externalSecret.remoteRefs.metrics.password }} +{{- end }} {{- end }} diff --git a/charts/countly-mongodb/values.schema.json b/charts/countly-mongodb/values.schema.json index 4276610..d3c27e7 100644 --- a/charts/countly-mongodb/values.schema.json +++ b/charts/countly-mongodb/values.schema.json @@ -31,6 +31,18 @@ "users": { "type": "object", "properties": { + "admin": { + "type": "object", + "properties": { + "enabled": { "type": "boolean" }, + "name": { "type": "string" }, + "database": { "type": "string" }, + "roles": { "type": "array" }, + "passwordSecretName": { "type": "string" }, + "passwordSecretKey": { "type": "string" }, + "password": { "type": "string" } + } + }, "app": { "type": "object", "required": ["name", "database", "passwordSecretName", "passwordSecretKey"], @@ -69,7 +81,47 @@ "secrets": { "type": "object", "properties": { - "keep": { "type": "boolean" } + "keep": { "type": "boolean" }, + "mode": { + "type": "string", + "enum": ["values", "existingSecret", "externalSecret"] + }, + "externalSecret": { + "type": "object", + "properties": { + "refreshInterval": { "type": "string" }, + "secretStoreRef": { + "type": "object", + "properties": { + "name": { "type": "string" }, + "kind": { "type": "string" } + } + }, + "remoteRefs": { + "type": "object", + "properties": { + "app": { + "type": "object", + "properties": { + "password": { "type": "string" } + } + }, + "admin": { + "type": "object", + "properties": { + "password": { "type": "string" } + } + }, + "metrics": { + "type": "object", + "properties": { + "password": { "type": "string" } + } + } + } + } + } + } } } } diff --git a/charts/countly-mongodb/values.yaml b/charts/countly-mongodb/values.yaml index d031e69..63de999 100644 --- a/charts/countly-mongodb/values.yaml +++ b/charts/countly-mongodb/values.yaml @@ -66,7 +66,20 @@ mongodb: # -- MongoDB users created by the operator users: - # -- Application user (used by Countly services) + admin: + enabled: true + name: admin + database: admin + roles: + - name: root + db: admin + - name: userAdminAnyDatabase + db: admin + - name: dbAdminAnyDatabase + db: admin + passwordSecretName: admin-user-password + passwordSecretKey: password + password: "" app: # -- Username name: app @@ -105,8 +118,7 @@ users: exporter: # -- Deploy the exporter sidecar enabled: true - # -- Exporter container image - image: percona/mongodb_exporter:0.40.0 + image: percona/mongodb_exporter:0.47.2 # -- Metrics port exposed by the exporter port: 9216 # -- Exporter resource requests and limits @@ -154,3 +166,16 @@ networkPolicy: secrets: # -- Preserve secrets on helm uninstall/upgrade keep: true + mode: values # values | existingSecret | externalSecret + externalSecret: + refreshInterval: "1h" + secretStoreRef: + name: "" + kind: ClusterSecretStore + remoteRefs: + admin: + password: "" + app: + password: "" + metrics: + password: "" diff --git a/charts/countly-observability/README.md b/charts/countly-observability/README.md index e34001e..823e4a2 100644 --- a/charts/countly-observability/README.md +++ b/charts/countly-observability/README.md @@ -278,13 +278,13 @@ grafana: | Component | Image | Version | |---|---|---| -| Prometheus | `prom/prometheus` | v3.10.0 | -| Grafana | `grafana/grafana` | 12.4.0 | -| Loki | `grafana/loki` | 3.6.7 | -| Tempo | `grafana/tempo` | 2.10.1 | -| Pyroscope | `grafana/pyroscope` | 1.18.1 | -| Alloy | `grafana/alloy` | v1.13.2 | -| kube-state-metrics | `registry.k8s.io/kube-state-metrics/kube-state-metrics` | v2.18.0 | +| Prometheus | `prom/prometheus` | v3.8.1 | +| Grafana | `grafana/grafana` | 12.3.5 | +| Loki | `grafana/loki` | 3.6.3 | +| Tempo | `grafana/tempo` | 2.8.1 | +| Pyroscope | `grafana/pyroscope` | 1.16.0 | +| Alloy | `grafana/alloy` | v1.14.0 | +| kube-state-metrics | `registry.k8s.io/kube-state-metrics/kube-state-metrics` | v2.17.0 | | node-exporter | `prom/node-exporter` | v1.10.2 | --- diff --git a/charts/countly-observability/templates/loki/statefulset.yaml b/charts/countly-observability/templates/loki/statefulset.yaml index c295619..dc0ef64 100644 --- a/charts/countly-observability/templates/loki/statefulset.yaml +++ b/charts/countly-observability/templates/loki/statefulset.yaml @@ -25,7 +25,7 @@ spec: spec: initContainers: - name: init-data-dir - image: busybox:1.35 + image: busybox:1.37.0 command: ['sh', '-c', 'mkdir -p /loki/chunks /loki/rules /loki/tsdb-shipper-cache /loki/wal && chown -R 10001:10001 /loki && chmod -R 755 /loki'] volumeMounts: - name: data diff --git a/charts/countly-observability/templates/prometheus/statefulset.yaml b/charts/countly-observability/templates/prometheus/statefulset.yaml index 42320a0..282e7a4 100644 --- a/charts/countly-observability/templates/prometheus/statefulset.yaml +++ b/charts/countly-observability/templates/prometheus/statefulset.yaml @@ -26,7 +26,7 @@ spec: serviceAccountName: {{ include "obs.fullname" . }}-prometheus initContainers: - name: init-data-dir - image: busybox:1.35 + image: busybox:1.37.0 command: ['sh', '-c', 'mkdir -p /prometheus && chown -R 65534:65534 /prometheus && chmod -R 755 /prometheus'] volumeMounts: - name: data diff --git a/charts/countly-observability/templates/pyroscope/statefulset.yaml b/charts/countly-observability/templates/pyroscope/statefulset.yaml index b5db317..65866e6 100644 --- a/charts/countly-observability/templates/pyroscope/statefulset.yaml +++ b/charts/countly-observability/templates/pyroscope/statefulset.yaml @@ -25,7 +25,7 @@ spec: spec: initContainers: - name: init-data-dir - image: busybox:1.35 + image: busybox:1.37.0 command: ['sh', '-c', 'mkdir -p /data && chown -R 10001:10001 /data && chmod -R 755 /data'] volumeMounts: - name: data diff --git a/charts/countly-observability/templates/tempo/statefulset.yaml b/charts/countly-observability/templates/tempo/statefulset.yaml index 18fff84..d8b94c0 100644 --- a/charts/countly-observability/templates/tempo/statefulset.yaml +++ b/charts/countly-observability/templates/tempo/statefulset.yaml @@ -25,7 +25,7 @@ spec: spec: initContainers: - name: init-data-dir - image: busybox:1.35 + image: busybox:1.37.0 command: ['sh', '-c', 'mkdir -p /var/tempo/traces /var/tempo/wal /var/tempo/generator/wal /var/tempo/generator/traces && chown -R 10001:10001 /var/tempo && chmod -R 755 /var/tempo'] volumeMounts: - name: data diff --git a/charts/countly-observability/templates/tests/test-backends.yaml b/charts/countly-observability/templates/tests/test-backends.yaml index 7941108..610893a 100644 --- a/charts/countly-observability/templates/tests/test-backends.yaml +++ b/charts/countly-observability/templates/tests/test-backends.yaml @@ -13,7 +13,7 @@ spec: restartPolicy: Never containers: - name: test - image: busybox:1.35 + image: busybox:1.37.0 command: ['sh', '-c', 'wget -qO- --timeout=10 http://{{ include "obs.fullname" . }}-prometheus:9090/-/ready'] --- {{- end }} @@ -32,7 +32,7 @@ spec: restartPolicy: Never containers: - name: test - image: busybox:1.35 + image: busybox:1.37.0 command: ['sh', '-c', 'wget -qO- --timeout=10 http://{{ include "obs.fullname" . }}-loki:3100/ready'] --- {{- end }} @@ -51,7 +51,7 @@ spec: restartPolicy: Never containers: - name: test - image: busybox:1.35 + image: busybox:1.37.0 command: ['sh', '-c', 'wget -qO- --timeout=10 http://{{ include "obs.fullname" . }}-tempo:3200/ready'] --- {{- end }} @@ -70,7 +70,7 @@ spec: restartPolicy: Never containers: - name: test - image: busybox:1.35 + image: busybox:1.37.0 command: ['sh', '-c', 'wget -qO- --timeout=10 http://{{ include "obs.fullname" . }}-grafana:3000/api/health'] --- {{- end }} @@ -89,6 +89,6 @@ spec: restartPolicy: Never containers: - name: test - image: busybox:1.35 + image: busybox:1.37.0 command: ['sh', '-c', 'wget -qO- --timeout=10 http://{{ include "obs.fullname" . }}-alloy-otlp:12345/-/ready'] {{- end }} diff --git a/charts/countly-observability/values.yaml b/charts/countly-observability/values.yaml index 5929a36..a114393 100644 --- a/charts/countly-observability/values.yaml +++ b/charts/countly-observability/values.yaml @@ -83,7 +83,7 @@ profiling: prometheus: image: repository: prom/prometheus - tag: "v3.10.0" + tag: "v3.8.1" retention: time: "30d" size: "50GB" @@ -112,7 +112,7 @@ prometheus: loki: image: repository: grafana/loki - tag: "3.6.7" + tag: "3.6.3" retention: "30d" storage: backend: "filesystem" # filesystem | s3 | gcs | azure @@ -158,7 +158,7 @@ loki: tempo: image: repository: grafana/tempo - tag: "2.10.1" + tag: "2.8.1" retention: "12h" storage: backend: "local" # local | s3 | gcs | azure @@ -203,7 +203,7 @@ tempo: pyroscope: image: repository: grafana/pyroscope - tag: "1.18.1" + tag: "1.16.0" retention: "72h" storage: backend: "filesystem" # filesystem | s3 | gcs | azure | swift @@ -242,7 +242,7 @@ grafana: enabled: true image: repository: grafana/grafana - tag: "12.4.0" + tag: "12.3.5" admin: # -- Use an existing Secret for admin credentials existingSecret: "" @@ -287,7 +287,7 @@ grafana: alloy: image: repository: grafana/alloy - tag: "v1.13.2" + tag: "v1.14.0" resources: requests: cpu: "500m" @@ -306,7 +306,7 @@ alloy: alloyOtlp: image: repository: grafana/alloy - tag: "v1.13.2" + tag: "v1.14.0" replicas: 1 resources: requests: @@ -329,7 +329,7 @@ alloyOtlp: alloyMetrics: image: repository: grafana/alloy - tag: "v1.13.2" + tag: "v1.14.0" replicas: 1 resources: requests: @@ -352,7 +352,7 @@ kubeStateMetrics: enabled: true image: repository: registry.k8s.io/kube-state-metrics/kube-state-metrics - tag: "v2.18.0" + tag: "v2.17.0" resources: requests: cpu: "10m" diff --git a/charts/countly/templates/_countly-component.tpl b/charts/countly/templates/_countly-component.tpl index 7468e76..8a91f57 100644 --- a/charts/countly/templates/_countly-component.tpl +++ b/charts/countly/templates/_countly-component.tpl @@ -60,7 +60,7 @@ spec: type: RuntimeDefault containers: - name: {{ $component }} - image: "{{ if $root.Values.global.imageRegistry }}{{ $root.Values.global.imageRegistry }}/{{ end }}{{ $root.Values.image.repository }}{{ if $root.Values.image.digest }}@{{ $root.Values.image.digest }}{{ else }}:{{ $root.Values.image.tag | default $root.Chart.AppVersion }}{{ end }}" + image: "{{ include "countly.image" $root }}{{ if $root.Values.image.digest }}@{{ $root.Values.image.digest }}{{ else }}:{{ $root.Values.image.tag | default $root.Chart.AppVersion }}{{ end }}" imagePullPolicy: {{ $root.Values.image.pullPolicy }} securityContext: runAsNonRoot: true @@ -173,6 +173,23 @@ spec: {{- end }} {{- end -}} +{{/* +Resolve the Countly image repository based on the selected source mode. +*/}} +{{- define "countly.image" -}} +{{- $mode := .Values.global.imageSource.mode | default "direct" -}} +{{- if eq $mode "gcpArtifactRegistry" -}} +{{- $prefix := required "global.imageSource.gcpArtifactRegistry.repositoryPrefix is required when global.imageSource.mode is gcpArtifactRegistry" .Values.global.imageSource.gcpArtifactRegistry.repositoryPrefix -}} +{{- printf "%s/%s" ($prefix | trimSuffix "/") .Values.image.artifactRepository -}} +{{- else -}} +{{- if .Values.global.imageRegistry -}} +{{- printf "%s/%s" (.Values.global.imageRegistry | trimSuffix "/") .Values.image.repository -}} +{{- else -}} +{{- .Values.image.repository -}} +{{- end -}} +{{- end -}} +{{- end -}} + {{/* Service for a Countly component */}} @@ -218,6 +235,10 @@ metadata: labels: {{- include "countly.labels" $root | nindent 4 }} app.kubernetes.io/component: {{ $component }} + {{- if $root.Values.argocd.enabled }} + annotations: + {{- include "countly.syncWave" (dict "wave" "6" "root" $root) | nindent 4 }} + {{- end }} spec: scaleTargetRef: apiVersion: apps/v1 diff --git a/charts/countly/templates/_helpers.tpl b/charts/countly/templates/_helpers.tpl index 5902dfa..2045971 100644 --- a/charts/countly/templates/_helpers.tpl +++ b/charts/countly/templates/_helpers.tpl @@ -119,6 +119,16 @@ Effective TLS secret name. {{- ((.Values.ingress).tls).secretName | default (printf "%s-tls" (include "countly.fullname" .)) -}} {{- end -}} +{{/* +Escape MongoDB URI user-info values safely. +urlquery handles reserved characters but encodes spaces as "+", which is +query-style encoding. Replace "+" with "%20" so the result is safe in URI +user-info segments too. +*/}} +{{- define "countly.mongodb.escapeUserInfo" -}} +{{- . | urlquery | replace "+" "%20" -}} +{{- end -}} + {{/* MongoDB connection string computation. Reads from backingServices.mongodb; constructs from service DNS if not provided. @@ -138,7 +148,31 @@ Reads from backingServices.mongodb; constructs from service DNS if not provided. {{- $user := $bs.username | default "app" -}} {{- $db := $bs.database | default "admin" -}} {{- $rs := $bs.replicaSet | default (printf "%s-mongodb" .Release.Name) -}} -mongodb://{{ $user }}:{{ $pass }}@{{ $host }}:{{ $port }}/{{ $db }}?replicaSet={{ $rs }}&ssl=false +mongodb://{{ include "countly.mongodb.escapeUserInfo" $user }}:{{ include "countly.mongodb.escapeUserInfo" $pass }}@{{ $host }}:{{ $port }}/{{ $db }}?replicaSet={{ $rs }}&ssl=false +{{- end -}} +{{- end -}} + +{{/* +MongoDB connection string computation using an explicit password value. +Used by ExternalSecret templates where the password may come from the secret backend. +*/}} +{{- define "countly.mongodb.connectionStringWithPassword" -}} +{{- $root := .root -}} +{{- $pass := .password -}} +{{- $bs := ($root.Values.backingServices).mongodb | default dict -}} +{{- $connStr := $bs.connectionString -}} +{{- if $connStr -}} +{{- $connStr -}} +{{- else -}} +{{- if not $pass -}} +{{- fail "MongoDB password is required. Set backingServices.mongodb.password, secrets.mongodb.password, or secrets.externalSecret.remoteRefs.mongodb.password." -}} +{{- end -}} +{{- $host := $bs.host | default (printf "%s-mongodb-svc.%s.svc.cluster.local" $root.Release.Name ($root.Values.mongodbNamespace | default "mongodb")) -}} +{{- $port := $bs.port | default "27017" -}} +{{- $user := $bs.username | default "app" -}} +{{- $db := $bs.database | default "admin" -}} +{{- $rs := $bs.replicaSet | default (printf "%s-mongodb" $root.Release.Name) -}} +mongodb://{{ include "countly.mongodb.escapeUserInfo" $user }}:{{ $pass }}@{{ $host }}:{{ $port }}/{{ $db }}?replicaSet={{ $rs }}&ssl=false {{- end -}} {{- end -}} @@ -193,3 +227,13 @@ ArgoCD sync-wave annotation (only when argocd.enabled). argocd.argoproj.io/sync-wave: {{ .wave | quote }} {{- end }} {{- end -}} + +{{/* +Resolve the first configured imagePullSecret name. +*/}} +{{- define "countly.imagePullSecretName" -}} +{{- $pullSecrets := .Values.global.imagePullSecrets | default list -}} +{{- if gt (len $pullSecrets) 0 -}} +{{- (index $pullSecrets 0).name -}} +{{- end -}} +{{- end -}} diff --git a/charts/countly/templates/external-secret-clickhouse.yaml b/charts/countly/templates/external-secret-clickhouse.yaml index ae7673c..601dea9 100644 --- a/charts/countly/templates/external-secret-clickhouse.yaml +++ b/charts/countly/templates/external-secret-clickhouse.yaml @@ -1,6 +1,6 @@ {{- if eq (.Values.secrets.mode | default "values") "externalSecret" }} {{- if not .Values.secrets.clickhouse.existingSecret }} -apiVersion: external-secrets.io/v1beta1 +apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: name: {{ include "countly.fullname" . }}-clickhouse @@ -18,18 +18,42 @@ spec: target: name: {{ include "countly.fullname" . }}-clickhouse creationPolicy: Owner + template: + engineVersion: v2 + mergePolicy: Merge + data: + {{- if not .Values.secrets.externalSecret.remoteRefs.clickhouse.url }} + COUNTLY_CONFIG__CLICKHOUSE_URL: {{ include "countly.clickhouse.url" . | quote }} + {{- end }} + {{- if not .Values.secrets.externalSecret.remoteRefs.clickhouse.username }} + COUNTLY_CONFIG__CLICKHOUSE_USERNAME: {{ (.Values.secrets.clickhouse.username | default ((.Values.backingServices).clickhouse).username | default "default") | quote }} + {{- end }} + {{- if not .Values.secrets.externalSecret.remoteRefs.clickhouse.password }} + COUNTLY_CONFIG__CLICKHOUSE_PASSWORD: {{ (.Values.secrets.clickhouse.password | default ((.Values.backingServices).clickhouse).password) | quote }} + {{- end }} + {{- if not .Values.secrets.externalSecret.remoteRefs.clickhouse.database }} + COUNTLY_CONFIG__CLICKHOUSE_DATABASE: {{ (.Values.secrets.clickhouse.database | default ((.Values.backingServices).clickhouse).database | default "countly_drill") | quote }} + {{- end }} data: + {{- if .Values.secrets.externalSecret.remoteRefs.clickhouse.url }} - secretKey: COUNTLY_CONFIG__CLICKHOUSE_URL remoteRef: key: {{ required "secrets.externalSecret.remoteRefs.clickhouse.url is required" .Values.secrets.externalSecret.remoteRefs.clickhouse.url }} + {{- end }} + {{- if .Values.secrets.externalSecret.remoteRefs.clickhouse.username }} - secretKey: COUNTLY_CONFIG__CLICKHOUSE_USERNAME remoteRef: key: {{ required "secrets.externalSecret.remoteRefs.clickhouse.username is required" .Values.secrets.externalSecret.remoteRefs.clickhouse.username }} + {{- end }} + {{- if .Values.secrets.externalSecret.remoteRefs.clickhouse.password }} - secretKey: COUNTLY_CONFIG__CLICKHOUSE_PASSWORD remoteRef: key: {{ required "secrets.externalSecret.remoteRefs.clickhouse.password is required" .Values.secrets.externalSecret.remoteRefs.clickhouse.password }} + {{- end }} + {{- if .Values.secrets.externalSecret.remoteRefs.clickhouse.database }} - secretKey: COUNTLY_CONFIG__CLICKHOUSE_DATABASE remoteRef: key: {{ required "secrets.externalSecret.remoteRefs.clickhouse.database is required" .Values.secrets.externalSecret.remoteRefs.clickhouse.database }} + {{- end }} {{- end }} {{- end }} diff --git a/charts/countly/templates/external-secret-common.yaml b/charts/countly/templates/external-secret-common.yaml index 76fdad1..c1570b1 100644 --- a/charts/countly/templates/external-secret-common.yaml +++ b/charts/countly/templates/external-secret-common.yaml @@ -1,6 +1,9 @@ {{- if eq (.Values.secrets.mode | default "values") "externalSecret" }} {{- if not .Values.secrets.common.existingSecret }} -apiVersion: external-secrets.io/v1beta1 +{{- $commonRemote := .Values.secrets.externalSecret.remoteRefs.common | default dict -}} +{{- $commonUsesExternal := or $commonRemote.encryptionReportsKey $commonRemote.webSessionSecret $commonRemote.passwordSecret -}} +{{- if $commonUsesExternal }} +apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: name: {{ include "countly.fullname" . }}-common @@ -18,15 +21,38 @@ spec: target: name: {{ include "countly.fullname" . }}-common creationPolicy: Owner + {{- $hasCommonTemplateData := not (and $commonRemote.encryptionReportsKey $commonRemote.webSessionSecret $commonRemote.passwordSecret) }} + {{- if $hasCommonTemplateData }} + template: + engineVersion: v2 + mergePolicy: Merge + data: + {{- if not .Values.secrets.externalSecret.remoteRefs.common.encryptionReportsKey }} + COUNTLY_CONFIG__ENCRYPTION_REPORTS_KEY: {{ .Values.secrets.common.encryptionReportsKey | quote }} + {{- end }} + {{- if not .Values.secrets.externalSecret.remoteRefs.common.webSessionSecret }} + COUNTLY_CONFIG__WEB_SESSION_SECRET: {{ .Values.secrets.common.webSessionSecret | quote }} + {{- end }} + {{- if not .Values.secrets.externalSecret.remoteRefs.common.passwordSecret }} + COUNTLY_CONFIG__PASSWORDSECRET: {{ .Values.secrets.common.passwordSecret | quote }} + {{- end }} + {{- end }} data: + {{- if .Values.secrets.externalSecret.remoteRefs.common.encryptionReportsKey }} - secretKey: COUNTLY_CONFIG__ENCRYPTION_REPORTS_KEY remoteRef: key: {{ required "secrets.externalSecret.remoteRefs.common.encryptionReportsKey is required" .Values.secrets.externalSecret.remoteRefs.common.encryptionReportsKey }} + {{- end }} + {{- if .Values.secrets.externalSecret.remoteRefs.common.webSessionSecret }} - secretKey: COUNTLY_CONFIG__WEB_SESSION_SECRET remoteRef: key: {{ required "secrets.externalSecret.remoteRefs.common.webSessionSecret is required" .Values.secrets.externalSecret.remoteRefs.common.webSessionSecret }} + {{- end }} + {{- if .Values.secrets.externalSecret.remoteRefs.common.passwordSecret }} - secretKey: COUNTLY_CONFIG__PASSWORDSECRET remoteRef: key: {{ required "secrets.externalSecret.remoteRefs.common.passwordSecret is required" .Values.secrets.externalSecret.remoteRefs.common.passwordSecret }} + {{- end }} +{{- end }} {{- end }} {{- end }} diff --git a/charts/countly/templates/external-secret-image-pull.yaml b/charts/countly/templates/external-secret-image-pull.yaml new file mode 100644 index 0000000..032f9c1 --- /dev/null +++ b/charts/countly/templates/external-secret-image-pull.yaml @@ -0,0 +1,26 @@ +{{- if .Values.global.imagePullSecretExternalSecret.enabled }} +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: {{ required "global.imagePullSecrets[0].name is required when global.imagePullSecretExternalSecret.enabled is true" (include "countly.imagePullSecretName" .) }} + labels: + {{- include "countly.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly.syncWave" (dict "wave" "0" "root" .) | nindent 4 }} + {{- end }} +spec: + refreshInterval: {{ .Values.global.imagePullSecretExternalSecret.refreshInterval | default "1h" }} + secretStoreRef: + name: {{ required "global.imagePullSecretExternalSecret.secretStoreRef.name is required when global.imagePullSecretExternalSecret.enabled is true" .Values.global.imagePullSecretExternalSecret.secretStoreRef.name }} + kind: {{ .Values.global.imagePullSecretExternalSecret.secretStoreRef.kind | default "ClusterSecretStore" }} + target: + name: {{ required "global.imagePullSecrets[0].name is required when global.imagePullSecretExternalSecret.enabled is true" (include "countly.imagePullSecretName" .) }} + creationPolicy: Owner + template: + type: kubernetes.io/dockerconfigjson + data: + - secretKey: .dockerconfigjson + remoteRef: + key: {{ required "global.imagePullSecretExternalSecret.remoteRef.key is required when global.imagePullSecretExternalSecret.enabled is true" .Values.global.imagePullSecretExternalSecret.remoteRef.key }} +{{- end }} diff --git a/charts/countly/templates/external-secret-ingress-tls.yaml b/charts/countly/templates/external-secret-ingress-tls.yaml new file mode 100644 index 0000000..3fb8d9f --- /dev/null +++ b/charts/countly/templates/external-secret-ingress-tls.yaml @@ -0,0 +1,30 @@ +{{- $tlsMode := include "countly.tls.mode" . -}} +{{- if and (eq $tlsMode "existingSecret") .Values.ingress.tls.externalSecret.enabled }} +apiVersion: external-secrets.io/v1 +kind: ExternalSecret +metadata: + name: {{ include "countly.tls.secretName" . }} + labels: + {{- include "countly.labels" . | nindent 4 }} + {{- if .Values.argocd.enabled }} + annotations: + {{- include "countly.syncWave" (dict "wave" "1" "root" .) | nindent 4 }} + {{- end }} +spec: + refreshInterval: {{ .Values.ingress.tls.externalSecret.refreshInterval | default "1h" }} + secretStoreRef: + name: {{ required "ingress.tls.externalSecret.secretStoreRef.name is required when ingress.tls.externalSecret.enabled is true" .Values.ingress.tls.externalSecret.secretStoreRef.name }} + kind: {{ .Values.ingress.tls.externalSecret.secretStoreRef.kind | default "ClusterSecretStore" }} + target: + name: {{ include "countly.tls.secretName" . }} + creationPolicy: Owner + template: + type: kubernetes.io/tls + data: + - secretKey: tls.crt + remoteRef: + key: {{ required "ingress.tls.externalSecret.remoteRefs.tlsCrt is required when ingress.tls.externalSecret.enabled is true" .Values.ingress.tls.externalSecret.remoteRefs.tlsCrt }} + - secretKey: tls.key + remoteRef: + key: {{ required "ingress.tls.externalSecret.remoteRefs.tlsKey is required when ingress.tls.externalSecret.enabled is true" .Values.ingress.tls.externalSecret.remoteRefs.tlsKey }} +{{- end }} diff --git a/charts/countly/templates/external-secret-kafka.yaml b/charts/countly/templates/external-secret-kafka.yaml index cc9b590..cda228a 100644 --- a/charts/countly/templates/external-secret-kafka.yaml +++ b/charts/countly/templates/external-secret-kafka.yaml @@ -1,6 +1,9 @@ {{- if eq (.Values.secrets.mode | default "values") "externalSecret" }} {{- if not .Values.secrets.kafka.existingSecret }} -apiVersion: external-secrets.io/v1beta1 +{{- $kafkaRemote := .Values.secrets.externalSecret.remoteRefs.kafka | default dict -}} +{{- $kafkaUsesExternal := or $kafkaRemote.brokers $kafkaRemote.securityProtocol $kafkaRemote.saslMechanism $kafkaRemote.saslUsername $kafkaRemote.saslPassword -}} +{{- if $kafkaUsesExternal }} +apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: name: {{ include "countly.fullname" . }}-kafka @@ -18,13 +21,37 @@ spec: target: name: {{ include "countly.fullname" . }}-kafka creationPolicy: Owner + template: + engineVersion: v2 + mergePolicy: Merge + data: + {{- if not .Values.secrets.externalSecret.remoteRefs.kafka.brokers }} + COUNTLY_CONFIG__KAFKA_RDKAFKA_BROKERS: {{ include "countly.kafka.brokers" . | quote }} + {{- end }} + {{- if not .Values.secrets.externalSecret.remoteRefs.kafka.securityProtocol }} + COUNTLY_CONFIG__KAFKA_RDKAFKA_SECURITYPROTOCOL: {{ (.Values.secrets.kafka.securityProtocol | default ((.Values.backingServices).kafka).securityProtocol | default "PLAINTEXT") | quote }} + {{- end }} + {{- $saslMechanism := .Values.secrets.kafka.saslMechanism | default ((.Values.backingServices).kafka).saslMechanism }} + {{- if and $saslMechanism (not .Values.secrets.externalSecret.remoteRefs.kafka.saslMechanism) }} + COUNTLY_CONFIG__KAFKA_RDKAFKA_SASLMECHANISM: {{ $saslMechanism | quote }} + {{- end }} + {{- if and $saslMechanism (not .Values.secrets.externalSecret.remoteRefs.kafka.saslUsername) }} + COUNTLY_CONFIG__KAFKA_RDKAFKA_SASLUSERNAME: {{ (.Values.secrets.kafka.saslUsername | default ((.Values.backingServices).kafka).saslUsername) | quote }} + {{- end }} + {{- if and $saslMechanism (not .Values.secrets.externalSecret.remoteRefs.kafka.saslPassword) }} + COUNTLY_CONFIG__KAFKA_RDKAFKA_SASLPASSWORD: {{ (.Values.secrets.kafka.saslPassword | default ((.Values.backingServices).kafka).saslPassword) | quote }} + {{- end }} data: + {{- if .Values.secrets.externalSecret.remoteRefs.kafka.brokers }} - secretKey: COUNTLY_CONFIG__KAFKA_RDKAFKA_BROKERS remoteRef: key: {{ required "secrets.externalSecret.remoteRefs.kafka.brokers is required" .Values.secrets.externalSecret.remoteRefs.kafka.brokers }} + {{- end }} + {{- if .Values.secrets.externalSecret.remoteRefs.kafka.securityProtocol }} - secretKey: COUNTLY_CONFIG__KAFKA_RDKAFKA_SECURITYPROTOCOL remoteRef: key: {{ required "secrets.externalSecret.remoteRefs.kafka.securityProtocol is required" .Values.secrets.externalSecret.remoteRefs.kafka.securityProtocol }} + {{- end }} {{- if .Values.secrets.externalSecret.remoteRefs.kafka.saslMechanism }} - secretKey: COUNTLY_CONFIG__KAFKA_RDKAFKA_SASLMECHANISM remoteRef: @@ -42,3 +69,4 @@ spec: {{- end }} {{- end }} {{- end }} +{{- end }} diff --git a/charts/countly/templates/external-secret-mongodb.yaml b/charts/countly/templates/external-secret-mongodb.yaml index 748c196..f6d35cb 100644 --- a/charts/countly/templates/external-secret-mongodb.yaml +++ b/charts/countly/templates/external-secret-mongodb.yaml @@ -1,6 +1,6 @@ {{- if eq (.Values.secrets.mode | default "values") "externalSecret" }} {{- if not .Values.secrets.mongodb.existingSecret }} -apiVersion: external-secrets.io/v1beta1 +apiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: name: {{ include "countly.fullname" . }}-mongodb @@ -18,9 +18,24 @@ spec: target: name: {{ include "countly.fullname" . }}-mongodb creationPolicy: Owner + template: + engineVersion: v2 + data: + {{- if .Values.secrets.externalSecret.remoteRefs.mongodb.connectionString }} + {{- else if .Values.secrets.externalSecret.remoteRefs.mongodb.password }} + {{ .Values.secrets.mongodb.key | default "connectionString.standard" }}: {{ include "countly.mongodb.connectionStringWithPassword" (dict "root" . "password" "{{ .mongodbPassword | urlquery | replace \"+\" \"%20\" }}" ) | quote }} + {{- else }} + {{ .Values.secrets.mongodb.key | default "connectionString.standard" }}: {{ include "countly.mongodb.connectionString" . | quote }} + {{- end }} data: + {{- if .Values.secrets.externalSecret.remoteRefs.mongodb.connectionString }} - secretKey: {{ .Values.secrets.mongodb.key | default "connectionString.standard" }} remoteRef: key: {{ required "secrets.externalSecret.remoteRefs.mongodb.connectionString is required" .Values.secrets.externalSecret.remoteRefs.mongodb.connectionString }} + {{- else if .Values.secrets.externalSecret.remoteRefs.mongodb.password }} + - secretKey: mongodbPassword + remoteRef: + key: {{ required "secrets.externalSecret.remoteRefs.mongodb.password is required" .Values.secrets.externalSecret.remoteRefs.mongodb.password }} + {{- end }} {{- end }} {{- end }} diff --git a/charts/countly/templates/ingress.yaml b/charts/countly/templates/ingress.yaml index 413d051..e2c5f6d 100644 --- a/charts/countly/templates/ingress.yaml +++ b/charts/countly/templates/ingress.yaml @@ -14,6 +14,8 @@ metadata: {{- end }} {{- if eq $tlsMode "letsencrypt" }} cert-manager.io/cluster-issuer: {{ .Values.ingress.tls.clusterIssuer | default "letsencrypt-prod" | quote }} + acme.cert-manager.io/http01-edit-in-place: "true" + cert-manager.io/issue-temporary-certificate: "true" {{- end }} {{- if eq $tlsMode "selfSigned" }} cert-manager.io/cluster-issuer: {{ (.Values.ingress.tls.selfSigned).issuerName | default (printf "%s-ca-issuer" (include "countly.fullname" .)) | quote }} diff --git a/charts/countly/templates/secret-common.yaml b/charts/countly/templates/secret-common.yaml index 2308a71..8d17ecb 100644 --- a/charts/countly/templates/secret-common.yaml +++ b/charts/countly/templates/secret-common.yaml @@ -1,4 +1,6 @@ -{{- if and (ne (.Values.secrets.mode | default "values") "externalSecret") (not .Values.secrets.common.existingSecret) }} +{{- $commonRemote := .Values.secrets.externalSecret.remoteRefs.common | default dict -}} +{{- $commonUsesExternal := or $commonRemote.encryptionReportsKey $commonRemote.webSessionSecret $commonRemote.passwordSecret -}} +{{- if and (or (ne (.Values.secrets.mode | default "values") "externalSecret") (not $commonUsesExternal)) (not .Values.secrets.common.existingSecret) }} apiVersion: v1 kind: Secret metadata: diff --git a/charts/countly/templates/secret-kafka.yaml b/charts/countly/templates/secret-kafka.yaml index 18ed231..f71473a 100644 --- a/charts/countly/templates/secret-kafka.yaml +++ b/charts/countly/templates/secret-kafka.yaml @@ -1,4 +1,6 @@ -{{- if and (ne (.Values.secrets.mode | default "values") "externalSecret") (not .Values.secrets.kafka.existingSecret) }} +{{- $kafkaRemote := .Values.secrets.externalSecret.remoteRefs.kafka | default dict -}} +{{- $kafkaUsesExternal := or $kafkaRemote.brokers $kafkaRemote.securityProtocol $kafkaRemote.saslMechanism $kafkaRemote.saslUsername $kafkaRemote.saslPassword -}} +{{- if and (or (ne (.Values.secrets.mode | default "values") "externalSecret") (not $kafkaUsesExternal)) (not .Values.secrets.kafka.existingSecret) }} apiVersion: v1 kind: Secret metadata: diff --git a/charts/countly/values.schema.json b/charts/countly/values.schema.json index 56d4381..35af06f 100644 --- a/charts/countly/values.schema.json +++ b/charts/countly/values.schema.json @@ -11,6 +11,56 @@ "imageRegistry": { "type": "string" }, + "imageSource": { + "type": "object", + "properties": { + "mode": { + "type": "string", + "enum": [ + "direct", + "gcpArtifactRegistry" + ] + }, + "gcpArtifactRegistry": { + "type": "object", + "properties": { + "repositoryPrefix": { + "type": "string" + } + } + } + } + }, + "imagePullSecretExternalSecret": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "refreshInterval": { + "type": "string" + }, + "secretStoreRef": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "kind": { + "type": "string" + } + } + }, + "remoteRef": { + "type": "object", + "properties": { + "key": { + "type": "string" + } + } + } + } + }, "imagePullSecrets": { "type": "array" }, @@ -41,6 +91,11 @@ "type": "string", "minLength": 1 }, + "artifactRepository": { + "type": "string", + "minLength": 1, + "description": "Repository path appended to global.imageSource.gcpArtifactRegistry.repositoryPrefix when using GCP Artifact Registry" + }, "digest": { "type": [ "string", @@ -315,6 +370,39 @@ "secretName": { "type": "string" }, + "externalSecret": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "refreshInterval": { + "type": "string" + }, + "secretStoreRef": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "kind": { + "type": "string" + } + } + }, + "remoteRefs": { + "type": "object", + "properties": { + "tlsCrt": { + "type": "string" + }, + "tlsKey": { + "type": "string" + } + } + } + } + }, "selfSigned": { "type": "object", "properties": { diff --git a/charts/countly/values.yaml b/charts/countly/values.yaml index df6c196..d9efdaa 100644 --- a/charts/countly/values.yaml +++ b/charts/countly/values.yaml @@ -2,7 +2,18 @@ global: # -- Override container image registry for all images imageRegistry: "" - # -- Global image pull secrets + imageSource: + mode: direct + gcpArtifactRegistry: + repositoryPrefix: "" + imagePullSecretExternalSecret: + enabled: false + refreshInterval: "1h" + secretStoreRef: + name: "" + kind: ClusterSecretStore + remoteRef: + key: "" imagePullSecrets: [] # -- Default StorageClass for PVCs (empty = cluster default) storageClass: "" @@ -38,6 +49,7 @@ serviceAccount: image: # -- Image repository repository: gcr.io/countly-dev-313620/countly-unified + artifactRepository: countly-unified # -- Image digest (takes precedence over tag when set) digest: "sha256:f81b39d4488c596f76a5c385d088a8998b7c1b20933366ad994f5315597ec48b" # -- Image tag (used when digest is empty; defaults to appVersion) @@ -521,6 +533,7 @@ secrets: saslPassword: "" mongodb: connectionString: "" + password: "" # --- Network Policy --- @@ -586,9 +599,18 @@ ingress: mode: http # -- cert-manager ClusterIssuer name (for letsencrypt mode) clusterIssuer: letsencrypt-prod - # -- TLS Secret name (auto-derived if empty: -tls) - secretName: "" - # -- Self-signed CA configuration (for selfSigned mode) + secretName: "" # Auto-derived if empty: -tls + externalSecret: + enabled: false + refreshInterval: "1h" + secretStoreRef: + name: "" + kind: ClusterSecretStore + remoteRefs: + # Shared TLS certificate defaults for all customers. + # Override per customer only when a customer needs its own certificate. + tlsCrt: "countly-prod-tls-crt" + tlsKey: "countly-prod-tls-key" selfSigned: # -- Issuer name (auto-derived if empty: -ca-issuer) issuerName: "" diff --git a/charts/noop/Chart.yaml b/charts/noop/Chart.yaml new file mode 100644 index 0000000..0969edd --- /dev/null +++ b/charts/noop/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: noop +description: No-op chart used when a GitOps-managed component is intentionally disabled. +type: application +version: 0.1.0 +appVersion: "0.1.0" diff --git a/docs/ARGOCD.md b/docs/ARGOCD.md index a45e161..c96382a 100644 --- a/docs/ARGOCD.md +++ b/docs/ARGOCD.md @@ -18,7 +18,7 @@ Deploy the Countly analytics platform using ArgoCD's GitOps model. The `countly- cp -r environments/reference environments/my-env ``` Edit `environments/my-env/global.yaml` — set `ingress.hostname`, sizing, TLS, security profiles. - Fill in secrets files (`secrets-mongodb.yaml`, `secrets-clickhouse.yaml`, etc.). + Fill in credential files (`credentials-mongodb.yaml`, `credentials-clickhouse.yaml`, etc.). 2. **Install the ArgoCD chart:** ```bash diff --git a/docs/DEPLOYING.md b/docs/DEPLOYING.md index ee6a1d0..9bd328b 100644 --- a/docs/DEPLOYING.md +++ b/docs/DEPLOYING.md @@ -38,18 +38,20 @@ See [DEPLOYMENT-MODES.md](DEPLOYMENT-MODES.md) for all mode options. ## Step 3: Configure Secrets -Fill in the required passwords in per-chart secret files (`secrets-.yaml`), which are gitignored to prevent accidental credential commits. Every chart needs credentials on first install: +Your copied environment already includes the chart-specific secret files from `environments/reference/`. + +Fill in the required passwords in the per-chart credential files (`credentials-.yaml`). Every chart needs credentials on first install: | Secret File | Required Secrets | |-------------|-----------------| -| `secrets-countly.yaml` | `secrets.common.*` (3 keys), `secrets.clickhouse.password`, `secrets.mongodb.password` | -| `secrets-mongodb.yaml` | `users.app.password`, `users.metrics.password` | -| `secrets-clickhouse.yaml` | `auth.defaultUserPassword.password` | -| `secrets-kafka.yaml` | `kafkaConnect.clickhouse.password` | +| `credentials-countly.yaml` | `secrets.common.*` (3 keys), `secrets.clickhouse.password`, `secrets.mongodb.password` | +| `credentials-mongodb.yaml` | `users.app.password`, `users.metrics.password` | +| `credentials-clickhouse.yaml` | `auth.defaultUserPassword.password` | +| `credentials-kafka.yaml` | `kafkaConnect.clickhouse.password` | -See `environments/reference/secrets.example.yaml` for a complete template you can copy. +See `environments/reference/secrets.example.yaml` for a complete reference. -**Important:** The ClickHouse password must match across `secrets-countly.yaml`, `secrets-clickhouse.yaml`, and `secrets-kafka.yaml`. The MongoDB password must match across `secrets-countly.yaml` and `secrets-mongodb.yaml`. +**Important:** The ClickHouse password must match across `credentials-countly.yaml`, `credentials-clickhouse.yaml`, and `credentials-kafka.yaml`. The MongoDB password must match across `credentials-countly.yaml` and `credentials-mongodb.yaml`. For production secret management options, see [SECRET-MANAGEMENT.md](SECRET-MANAGEMENT.md). @@ -71,6 +73,8 @@ helmfile -e my-deployment apply This installs all charts in dependency order with a 10-minute timeout per chart. +If you prefer plain Helm instead of Helmfile, use the same values layering shown in [README.md](/Users/admin/cly/helm/README.md) under manual installation. + ## Step 5: Verify ```bash @@ -102,6 +106,27 @@ Note: PVCs are not deleted by default. Clean up manually if needed. ## Troubleshooting +### Helmfile is not installed locally + +If `helmfile` is not available on your machine, either: + +- install Helmfile and keep using this document, or +- run plain `helm install` / `helm upgrade` commands using the same values files + +The charts themselves do not require Argo CD. + +### Migration chart fails with missing `redis` dependency + +The `countly-migration` chart includes a Redis dependency. + +Before rendering or installing it directly with Helm, run: + +```bash +helm dependency build ./charts/countly-migration +``` + +If you are not deploying migration, you can ignore this. + ### Kafka startup: UNKNOWN_TOPIC_OR_PARTITION errors On a fresh deployment, Countly pods (aggregator, ingestor) may log `UNKNOWN_TOPIC_OR_PARTITION` errors for the first 2-5 minutes. This is expected behavior: diff --git a/docs/DEPLOYMENT-MODES.md b/docs/DEPLOYMENT-MODES.md index d6365b4..0c75b78 100644 --- a/docs/DEPLOYMENT-MODES.md +++ b/docs/DEPLOYMENT-MODES.md @@ -8,7 +8,7 @@ Set in `global.yaml` -> `ingress.tls.mode`: |------|-------------|-------------| | `http` | No TLS (default) | None | | `letsencrypt` | Auto-provisioned via cert-manager | cert-manager + ClusterIssuer, DNS pointing to ingress | -| `existingSecret` | Pre-created TLS secret | Kubernetes TLS secret in countly namespace | +| `existingSecret` | Pre-created TLS secret or ExternalSecret-created TLS secret | Kubernetes TLS secret in countly namespace | | `selfSigned` | Self-signed CA via cert-manager | cert-manager (dev/local only) | ### Let's Encrypt Example @@ -29,6 +29,25 @@ ingress: secretName: my-tls-cert # Must exist in countly namespace ``` +### Existing Certificate From Secret Manager Example +```yaml +ingress: + hostname: analytics.example.com + tls: + mode: existingSecret + secretName: my-tls-cert + externalSecret: + enabled: true + secretStoreRef: + name: gcp-secrets + kind: ClusterSecretStore + remoteRefs: + # Shared TLS keys for all customers by default. + # Override only for customer-specific certificates. + tlsCrt: countly-prod-tls-crt + tlsKey: countly-prod-tls-key +``` + ## Backing Service Modes Set in `global.yaml` -> `backingServices..mode`: diff --git a/docs/QUICKSTART.md b/docs/QUICKSTART.md index 4a7faa0..2a8dedf 100644 --- a/docs/QUICKSTART.md +++ b/docs/QUICKSTART.md @@ -57,9 +57,25 @@ environments/local/global.yaml # Global settings (profile sel profiles/sizing/local/.yaml # Sizing (resources, replicas, HA) profiles///.yaml # Optional dimension profiles environments/local/.yaml # Environment choices (ingress, OTEL, etc.) -environments/local/secrets-.yaml # Credentials (gitignored) +environments/local/credentials-.yaml # Credentials overrides ``` +Important: +- the repo does not ship real local secret files +- create them before installing by copying from `environments/reference/` + +Recommended setup: + +```bash +cp environments/reference/credentials-countly.yaml environments/local/credentials-countly.yaml +cp environments/reference/credentials-mongodb.yaml environments/local/credentials-mongodb.yaml +cp environments/reference/credentials-clickhouse.yaml environments/local/credentials-clickhouse.yaml +cp environments/reference/credentials-kafka.yaml environments/local/credentials-kafka.yaml +cp environments/reference/credentials-observability.yaml environments/local/credentials-observability.yaml +``` + +Then fill in the required passwords. + ## Install Charts Run from the `helm/` directory. Order matters — each chart must complete before the next starts. @@ -72,8 +88,9 @@ helm install countly-mongodb ./charts/countly-mongodb \ --wait --timeout 10m \ -f environments/local/global.yaml \ -f profiles/sizing/local/mongodb.yaml \ + -f profiles/security/open/mongodb.yaml \ -f environments/local/mongodb.yaml \ - -f environments/local/secrets-mongodb.yaml + -f environments/local/credentials-mongodb.yaml ``` ### 2. ClickHouse @@ -84,8 +101,9 @@ helm install countly-clickhouse ./charts/countly-clickhouse \ --wait --timeout 10m \ -f environments/local/global.yaml \ -f profiles/sizing/local/clickhouse.yaml \ + -f profiles/security/open/clickhouse.yaml \ -f environments/local/clickhouse.yaml \ - -f environments/local/secrets-clickhouse.yaml + -f environments/local/credentials-clickhouse.yaml ``` ### 3. Kafka @@ -96,8 +114,11 @@ helm install countly-kafka ./charts/countly-kafka \ --wait --timeout 10m \ -f environments/local/global.yaml \ -f profiles/sizing/local/kafka.yaml \ + -f profiles/kafka-connect/balanced/kafka.yaml \ + -f profiles/observability/full/kafka.yaml \ + -f profiles/security/open/kafka.yaml \ -f environments/local/kafka.yaml \ - -f environments/local/secrets-kafka.yaml + -f environments/local/credentials-kafka.yaml ``` ### 4. Countly @@ -108,8 +129,11 @@ helm install countly ./charts/countly \ --wait --timeout 10m \ -f environments/local/global.yaml \ -f profiles/sizing/local/countly.yaml \ + -f profiles/tls/selfSigned/countly.yaml \ + -f profiles/observability/full/countly.yaml \ + -f profiles/security/open/countly.yaml \ -f environments/local/countly.yaml \ - -f environments/local/secrets-countly.yaml + -f environments/local/credentials-countly.yaml ``` ### 5. Observability @@ -120,8 +144,10 @@ helm install countly-observability ./charts/countly-observability \ --wait --timeout 10m \ -f environments/local/global.yaml \ -f profiles/sizing/local/observability.yaml \ + -f profiles/observability/full/observability.yaml \ + -f profiles/security/open/observability.yaml \ -f environments/local/observability.yaml \ - -f environments/local/secrets-observability.yaml + -f environments/local/credentials-observability.yaml ``` ## Verify @@ -141,6 +167,12 @@ Grafana: https://grafana.local Replace `helm install` with `helm upgrade` (same flags, omit `--create-namespace`). +If you install the optional migration chart directly, first run: + +```bash +helm dependency build ./charts/countly-migration +``` + ## Uninstall Reverse order: @@ -165,11 +197,11 @@ environments/local/ clickhouse.yaml # ServiceMonitor disabled (no Prometheus Operator CRD) kafka.yaml # JMX metrics disabled (KafkaNodePool CRD limitation) observability.yaml # mode: full, Grafana ingress (grafana.local, selfSigned TLS) - secrets-countly.yaml # App secrets (encryption key, session, password) - secrets-mongodb.yaml # MongoDB user passwords - secrets-clickhouse.yaml # ClickHouse default password - secrets-kafka.yaml # Kafka Connect ClickHouse password - secrets-observability.yaml # Empty stub (no secrets needed) + credentials-countly.yaml # Create from environments/reference/ + credentials-mongodb.yaml # Create from environments/reference/ + credentials-clickhouse.yaml # Create from environments/reference/ + credentials-kafka.yaml # Create from environments/reference/ + credentials-observability.yaml # Create from environments/reference/ ``` ## Known Issues (Local) diff --git a/docs/SECRET-MANAGEMENT.md b/docs/SECRET-MANAGEMENT.md index 879fd27..aaae835 100644 --- a/docs/SECRET-MANAGEMENT.md +++ b/docs/SECRET-MANAGEMENT.md @@ -54,21 +54,27 @@ secrets: kind: ClusterSecretStore remoteRefs: common: - encryptionReportsKey: "countly/encryption-reports-key" - webSessionSecret: "countly/web-session-secret" - passwordSecret: "countly/password-secret" + encryptionReportsKey: "acme-countly-encryption-reports-key" + webSessionSecret: "acme-countly-web-session-secret" + passwordSecret: "acme-countly-password-secret" clickhouse: - url: "countly/clickhouse-url" - username: "countly/clickhouse-username" - password: "countly/clickhouse-password" - database: "countly/clickhouse-database" - kafka: - brokers: "countly/kafka-brokers" - securityProtocol: "countly/kafka-security-protocol" + password: "acme-countly-clickhouse-password" mongodb: - connectionString: "countly/mongodb-connection-string" + password: "acme-mongodb-app-password" ``` +Recommended naming convention: +- `-gar-dockerconfig` +- `-countly-encryption-reports-key` +- `-countly-web-session-secret` +- `-countly-password-secret` +- `-countly-clickhouse-password` +- `-kafka-connect-clickhouse-password` +- `-clickhouse-default-user-password` +- `-mongodb-admin-password` +- `-mongodb-app-password` +- `-mongodb-metrics-password` + ## Required Secrets All secrets are required on first install. On upgrades, existing values are preserved automatically. @@ -79,7 +85,7 @@ All secrets are required on first install. On upgrades, existing values are pres | countly | common | webSessionSecret | Session cookie signing (min 8 chars) | | countly | common | passwordSecret | Password hashing (min 8 chars) | | countly | clickhouse | password | ClickHouse default user auth | -| countly | mongodb | password | MongoDB app user auth | +| countly | mongodb | password | MongoDB app user auth, reuse the same GSM key as `countly-mongodb.users.app.password` | | countly-mongodb | users.app | password | Must match countly secrets.mongodb.password | | countly-mongodb | users.metrics | password | Prometheus exporter auth | | countly-clickhouse | auth.defaultUserPassword | password | Must match countly secrets.clickhouse.password | diff --git a/docs/migration-guide.md b/docs/migration-guide.md index 9f526b3..7e27ee5 100644 --- a/docs/migration-guide.md +++ b/docs/migration-guide.md @@ -134,7 +134,7 @@ Create environment files for repeatable deploys: {} ``` -**`environments/my-env/secrets-migration.yaml`:** +**`environments/my-env/credentials-migration.yaml`:** ```yaml backingServices: mongodb: @@ -150,7 +150,7 @@ helm install countly-migration ./charts/countly-migration \ --wait --timeout 5m \ -f environments/my-env/global.yaml \ -f environments/my-env/migration.yaml \ - -f environments/my-env/secrets-migration.yaml + -f environments/my-env/credentials-migration.yaml ``` ### External MongoDB/ClickHouse diff --git a/environments/example-production/global.yaml b/environments/example-production/global.yaml index ea3c15b..9ed5760 100644 --- a/environments/example-production/global.yaml +++ b/environments/example-production/global.yaml @@ -6,6 +6,20 @@ global: tls: letsencrypt observability: full kafkaConnect: balanced + imageSource: + mode: gcpArtifactRegistry + gcpArtifactRegistry: + repositoryPrefix: us-docker.pkg.dev/countly-01/countly-unified + imagePullSecretExternalSecret: + enabled: true + refreshInterval: "1h" + secretStoreRef: + name: gcp-secrets + kind: ClusterSecretStore + remoteRef: + key: customer-a-gar-dockerconfig + imagePullSecrets: + - name: countly-registry storageClass: gp3 ingress: diff --git a/environments/local/clickhouse.yaml b/environments/local/clickhouse.yaml index 05ac971..3197512 100644 --- a/environments/local/clickhouse.yaml +++ b/environments/local/clickhouse.yaml @@ -1,6 +1,6 @@ # Local environment — ClickHouse chart overrides (non-sizing) # Profile defaults come from profiles/sizing/local/clickhouse.yaml -# Secrets come from secrets-clickhouse.yaml +# Credentials come from credentials-clickhouse.yaml # Enable server-side OpenTelemetry span logging # Queries with W3C traceparent headers will be logged to system.opentelemetry_span_log diff --git a/environments/local/kafka.yaml b/environments/local/kafka.yaml index 5eb3b42..ed80d66 100644 --- a/environments/local/kafka.yaml +++ b/environments/local/kafka.yaml @@ -1,10 +1,10 @@ # Local environment — Kafka chart overrides (non-sizing) # Profile defaults come from profiles/sizing/local/kafka.yaml -# Secrets come from secrets-kafka.yaml +# Credentials come from credentials-kafka.yaml # Use OTel-enabled image (includes /opt/otel/opentelemetry-javaagent.jar) kafkaConnect: - image: "gcr.io/countly-dev-313620/strimzi/kafka-connect-clickhouse:4.2.0-1.3.5-otel-strimzi-amd64" + image: "countly/strimzi-kafka-connect-clickhouse:kafka4.2.0-ch1.3.5-strimzi0.51-otel2.12.0" otel: enabled: true resourceAttributes: "service.namespace=countly,deployment.environment=local" diff --git a/environments/local/mongodb.yaml b/environments/local/mongodb.yaml index d7a4f40..4378233 100644 --- a/environments/local/mongodb.yaml +++ b/environments/local/mongodb.yaml @@ -1,4 +1,4 @@ # Local environment — MongoDB chart overrides (non-sizing) # Profile defaults come from profiles/sizing/local/mongodb.yaml -# Secrets come from secrets-mongodb.yaml +# Credentials come from credentials-mongodb.yaml {} diff --git a/environments/reference/README.md b/environments/reference/README.md index 12b374b..c83d349 100644 --- a/environments/reference/README.md +++ b/environments/reference/README.md @@ -10,19 +10,22 @@ This directory is a complete starting point for a new Countly deployment. ``` 2. Edit `global.yaml`: - - Set `ingress.hostname` to your domain - - Choose `global.sizing`: `local`, `small`, or `production` - - Choose `global.tls`: `none`, `letsencrypt`, `provided`, or `selfSigned` - - Choose `global.observability`: `disabled`, `full`, `external-grafana`, or `external` - - Choose `global.kafkaConnect`: `throughput`, `balanced`, or `low-latency` - - Choose `global.security`: `open` or `hardened` - - Choose backing service modes (bundled or external) + - Set `ingress.hostname` to your domain + - Choose `global.sizing`: `local`, `small`, or `production` + - Choose `global.tls`: `none`, `letsencrypt`, `provided`, or `selfSigned` + - Choose `global.observability`: `disabled`, `full`, `external-grafana`, or `external` + - Choose `global.kafkaConnect`: `throughput`, `balanced`, or `low-latency` + - Choose `global.security`: `open` or `hardened` + - Choose backing service modes (bundled or external) + - For GAR, set `global.imageSource`, `global.imagePullSecrets`, and optionally `global.imagePullSecretExternalSecret` + - For `global.tls: provided`, either point Countly at a pre-created TLS secret or enable `ingress.tls.externalSecret` in `countly.yaml` 3. Fill in required secrets in the chart-specific files: - - `countly.yaml` → `secrets.common.*` and `secrets.clickhouse.password`, `secrets.mongodb.password` - - `mongodb.yaml` → `users.app.password`, `users.metrics.password` - - `clickhouse.yaml` → `auth.defaultUserPassword.password` - - `kafka.yaml` → `kafkaConnect.clickhouse.password` + - `credentials-countly.yaml` → `secrets.common.*` and `secrets.clickhouse.password`, `secrets.mongodb.password` + - `credentials-mongodb.yaml` → `users.app.password`, `users.metrics.password` + - `credentials-clickhouse.yaml` → `auth.defaultUserPassword.password` + - `credentials-kafka.yaml` → `kafkaConnect.clickhouse.password` + - `image-pull-secrets.example.yaml` → private registry pull secret manifests for `countly` and `kafka` Or use `secrets.example.yaml` as a complete reference. @@ -44,11 +47,16 @@ This directory is a complete starting point for a new Countly deployment. See `secrets.example.yaml` for a complete list of all required secrets. For production, choose one of: -- **Direct values**: Fill secrets in chart-specific YAML files (split into `secrets-countly.yaml`, `secrets-mongodb.yaml`, etc.) +- **Direct values**: Fill credentials in chart-specific YAML files (split into `credentials-countly.yaml`, `credentials-mongodb.yaml`, etc.) - **existingSecret**: Pre-create Kubernetes secrets and reference them -- **externalSecret**: Use External Secrets Operator (see `external-secrets.example.yaml`) +- **externalSecret**: Use External Secrets Operator and Secret Manager-backed remote refs in the same `credentials-*.yaml` files - **SOPS**: Encrypt secret files with SOPS (see `secrets.sops.example.yaml`) +For private registries such as GAR, also create namespaced image pull secrets. +Use `image-pull-secrets.example.yaml` as a starting point, then encrypt it with SOPS or manage it through your GitOps secret workflow. +If you use External Secrets Operator with Google Secret Manager, point `global.imagePullSecretExternalSecret.remoteRef.key` at a secret whose value is the Docker config JSON content for `us-docker.pkg.dev`. +You can use the same External Secrets pattern for Countly ingress TLS when `global.tls` is `provided`; see `countly.yaml` and `external-secrets.example.yaml`. + ## Files | File | Purpose | @@ -59,11 +67,14 @@ For production, choose one of: | `clickhouse.yaml` | ClickHouse chart values (topology, auth, keeper) | | `kafka.yaml` | Kafka chart values (brokers, controllers, connect, connectors) | | `observability.yaml` | Observability chart values (signals, backends, Grafana, Alloy) | -| `secrets-countly.yaml` | Countly secrets (encryption keys, DB passwords) | -| `secrets-mongodb.yaml` | MongoDB user passwords | -| `secrets-clickhouse.yaml` | ClickHouse auth password | -| `secrets-kafka.yaml` | Kafka Connect ClickHouse password | -| `secrets-observability.yaml` | Observability secrets (external backend creds if needed) | +| `credentials-countly.yaml` | Countly secrets (encryption keys, DB passwords) | +| `credentials-mongodb.yaml` | MongoDB user passwords | +| `credentials-clickhouse.yaml` | ClickHouse auth password | +| `credentials-kafka.yaml` | Kafka Connect ClickHouse password | +| `credentials-observability.yaml` | Observability secrets (external backend creds if needed) | +| `countly-tls.env` | Manual TLS secret helper for bring-your-own certificate workflows | | `secrets.example.yaml` | Combined secrets reference (all charts in one file) | | `secrets.sops.example.yaml` | SOPS encryption guide | | `external-secrets.example.yaml` | External Secrets Operator guide | +| `image-pull-secrets.example.yaml` | Example GAR/private registry image pull secrets for `countly` and `kafka` | +| `cluster-secret-store.gcp.example.yaml` | Example `ClusterSecretStore` for Google Secret Manager with Workload Identity | diff --git a/environments/reference/clickhouse.yaml b/environments/reference/clickhouse.yaml index d7899d3..22b2187 100644 --- a/environments/reference/clickhouse.yaml +++ b/environments/reference/clickhouse.yaml @@ -27,7 +27,7 @@ clickhouseOperator: # ============================================================================= # Cluster Topology # ============================================================================= -version: "26.2" +version: "26.3" shards: 1 replicas: 2 diff --git a/environments/reference/cluster-secret-store.gcp.example.yaml b/environments/reference/cluster-secret-store.gcp.example.yaml new file mode 100644 index 0000000..7bb563f --- /dev/null +++ b/environments/reference/cluster-secret-store.gcp.example.yaml @@ -0,0 +1,31 @@ +# ============================================================================= +# External Secrets Operator + Google Secret Manager +# ClusterSecretStore Example +# ============================================================================= +# Apply this once per cluster after External Secrets Operator is installed. +# +# Prerequisites: +# - The external-secrets controller service account is annotated for Workload +# Identity with a GCP service account that can read Secret Manager secrets. +# - The GCP service account has at least: +# roles/secretmanager.secretAccessor +# +# This file is a reference only. Adapt project IDs and names to your cluster. +# ============================================================================= + +apiVersion: external-secrets.io/v1 +kind: ClusterSecretStore +metadata: + name: gcp-secrets +spec: + provider: + gcpsm: + projectID: countly-dev-313620 + auth: + workloadIdentity: + clusterLocation: us-central1 + clusterName: change-me + clusterProjectID: countly-dev-313620 + serviceAccountRef: + name: external-secrets + namespace: external-secrets diff --git a/environments/reference/countly-tls.env b/environments/reference/countly-tls.env index dd467a5..568b026 100644 --- a/environments/reference/countly-tls.env +++ b/environments/reference/countly-tls.env @@ -1,7 +1,9 @@ # Countly TLS Certificate Configuration - Template -# Copy this file to countly-tls.env and update with real values +# Use this only for manual bring-your-own TLS secret workflows. +# If you use Secret Manager + External Secrets, prefer ingress.tls.externalSecret +# in countly.yaml instead of managing this file. # Base64 encoded TLS certificate (full chain) TLS_CRT= # Base64 encoded TLS private key -TLS_KEY= \ No newline at end of file +TLS_KEY= diff --git a/environments/reference/countly.yaml b/environments/reference/countly.yaml index 4ba46fa..5ae7cfe 100644 --- a/environments/reference/countly.yaml +++ b/environments/reference/countly.yaml @@ -29,7 +29,8 @@ serviceAccount: # --- Image --- image: repository: gcr.io/countly-dev-313620/countly-unified - digest: "sha256:f81b39d4488c596f76a5c385d088a8998b7c1b20933366ad994f5315597ec48b" + artifactRepository: countly-unified + digest: "sha256:b42efb9713ee11d173fe409924fb9e2a208b5c0beafed9e42f349b996b6650a4" tag: "26.01" # Fallback when digest is empty pullPolicy: IfNotPresent @@ -555,16 +556,27 @@ ingress: proxy_next_upstream_tries 3; proxy_temp_file_write_size 1m; client_body_timeout 120s; - hostname: countly.example.com + hostname: "" # Set via argocd/customers/.yaml tls: # TLS mode: letsencrypt | existingSecret | selfSigned | http # http: No TLS # letsencrypt: cert-manager + Let's Encrypt (recommended for production) # existingSecret: Bring your own TLS secret # selfSigned: cert-manager self-signed CA (for development) - mode: http + mode: "" # Set via argocd/customers/.yaml clusterIssuer: letsencrypt-prod # Used with mode=letsencrypt secretName: "" # Auto-derived if empty: -tls + externalSecret: + enabled: false # Create the TLS Secret from External Secrets when mode=existingSecret + refreshInterval: "1h" + secretStoreRef: + name: "" # e.g. gcp-secrets + kind: ClusterSecretStore + remoteRefs: + # Shared TLS certificate defaults for all customers. + # Override only if a specific customer needs its own certificate. + tlsCrt: "countly-prod-tls-crt" + tlsKey: "countly-prod-tls-key" selfSigned: issuerName: "" # Auto-derived if empty: -ca-issuer caSecretName: "" # Auto-derived if empty: -ca-keypair diff --git a/environments/reference/credentials-clickhouse.yaml b/environments/reference/credentials-clickhouse.yaml new file mode 100644 index 0000000..c7a7799 --- /dev/null +++ b/environments/reference/credentials-clickhouse.yaml @@ -0,0 +1,18 @@ +# ClickHouse secrets — FILL IN before first deploy +secrets: + mode: values + +auth: + defaultUserPassword: + password: "" # REQUIRED: must match credentials-countly.yaml secrets.clickhouse.password + +# For Secret Manager / External Secrets instead of direct values: +# secrets: +# mode: externalSecret +# externalSecret: +# refreshInterval: "1h" +# secretStoreRef: +# name: gcp-secrets +# kind: ClusterSecretStore +# remoteRefs: +# defaultUserPassword: "acme-clickhouse-default-user-password" diff --git a/environments/reference/credentials-countly.yaml b/environments/reference/credentials-countly.yaml new file mode 100644 index 0000000..23c147e --- /dev/null +++ b/environments/reference/credentials-countly.yaml @@ -0,0 +1,34 @@ +# Countly secrets — FILL IN before first deploy +# Passwords must match across charts (see secrets.example.yaml) +secrets: + mode: values + common: + encryptionReportsKey: "" # REQUIRED: min 8 chars + webSessionSecret: "" # REQUIRED: min 8 chars + passwordSecret: "" # REQUIRED: min 8 chars + clickhouse: + username: "default" + password: "" # REQUIRED: must match credentials-clickhouse.yaml + database: "countly_drill" + kafka: + securityProtocol: "PLAINTEXT" + mongodb: + password: "" # REQUIRED: must match credentials-mongodb.yaml users.app.password + +# For Secret Manager / External Secrets instead of direct values: +# secrets: +# mode: externalSecret +# externalSecret: +# refreshInterval: "1h" +# secretStoreRef: +# name: gcp-secrets +# kind: ClusterSecretStore +# remoteRefs: +# common: +# encryptionReportsKey: "acme-countly-encryption-reports-key" +# webSessionSecret: "acme-countly-web-session-secret" +# passwordSecret: "acme-countly-password-secret" +# clickhouse: +# password: "acme-countly-clickhouse-password" +# mongodb: +# password: "acme-mongodb-app-password" diff --git a/environments/reference/credentials-kafka.yaml b/environments/reference/credentials-kafka.yaml new file mode 100644 index 0000000..44d1609 --- /dev/null +++ b/environments/reference/credentials-kafka.yaml @@ -0,0 +1,19 @@ +# Kafka secrets — FILL IN before first deploy +secrets: + mode: values + +kafkaConnect: + clickhouse: + password: "" # REQUIRED: must match ClickHouse default user password + +# For Secret Manager / External Secrets instead of direct values: +# secrets: +# mode: externalSecret +# externalSecret: +# refreshInterval: "1h" +# secretStoreRef: +# name: gcp-secrets +# kind: ClusterSecretStore +# remoteRefs: +# clickhouse: +# password: "acme-kafka-connect-clickhouse-password" diff --git a/environments/reference/credentials-migration.yaml b/environments/reference/credentials-migration.yaml new file mode 100644 index 0000000..6fe5890 --- /dev/null +++ b/environments/reference/credentials-migration.yaml @@ -0,0 +1,2 @@ +# Migration secrets placeholder. +# Fill when `migration: enabled` is used for a customer. diff --git a/environments/reference/credentials-mongodb.yaml b/environments/reference/credentials-mongodb.yaml new file mode 100644 index 0000000..0aef2b3 --- /dev/null +++ b/environments/reference/credentials-mongodb.yaml @@ -0,0 +1,29 @@ +# MongoDB secrets — FILL IN before first deploy +secrets: + mode: values + +users: + admin: + enabled: true + password: "" # REQUIRED: MongoDB super admin/root-style user + app: + password: "" # REQUIRED: must match credentials-countly.yaml secrets.mongodb.password + metrics: + enabled: true + password: "" # REQUIRED: metrics exporter password + +# For Secret Manager / External Secrets instead of direct values: +# secrets: +# mode: externalSecret +# externalSecret: +# refreshInterval: "1h" +# secretStoreRef: +# name: gcp-secrets +# kind: ClusterSecretStore +# remoteRefs: +# admin: +# password: "acme-mongodb-admin-password" +# app: +# password: "acme-mongodb-app-password" # Reuse this same GSM secret in credentials-countly.yaml +# metrics: +# password: "acme-mongodb-metrics-password" diff --git a/environments/reference/secrets-observability.yaml b/environments/reference/credentials-observability.yaml similarity index 100% rename from environments/reference/secrets-observability.yaml rename to environments/reference/credentials-observability.yaml diff --git a/environments/reference/external-secrets.example.yaml b/environments/reference/external-secrets.example.yaml index 7bf93ef..092f526 100644 --- a/environments/reference/external-secrets.example.yaml +++ b/environments/reference/external-secrets.example.yaml @@ -2,30 +2,94 @@ # External Secrets Operator (ESO) Configuration Example # ============================================================================= # When using secrets.mode=externalSecret, configure the ESO remoteRefs -# in environments//countly.yaml: +# in the chart-specific secrets files under environments//: # -# secrets: -# mode: externalSecret -# externalSecret: +# environments//credentials-countly.yaml +# secrets: +# mode: externalSecret +# externalSecret: +# refreshInterval: "1h" +# secretStoreRef: +# name: gcp-secrets +# kind: ClusterSecretStore +# remoteRefs: +# common: +# encryptionReportsKey: "acme-countly-encryption-reports-key" +# webSessionSecret: "acme-countly-web-session-secret" +# passwordSecret: "acme-countly-password-secret" +# clickhouse: +# password: "acme-countly-clickhouse-password" +# mongodb: +# password: "acme-mongodb-app-password" +# +# environments//credentials-clickhouse.yaml +# secrets: +# mode: externalSecret +# externalSecret: +# refreshInterval: "1h" +# secretStoreRef: +# name: gcp-secrets +# kind: ClusterSecretStore +# remoteRefs: +# defaultUserPassword: "acme-clickhouse-default-user-password" +# +# environments//credentials-kafka.yaml +# secrets: +# mode: externalSecret +# externalSecret: +# refreshInterval: "1h" +# secretStoreRef: +# name: gcp-secrets +# kind: ClusterSecretStore +# remoteRefs: +# clickhouse: +# password: "acme-kafka-connect-clickhouse-password" +# +# environments//credentials-mongodb.yaml +# secrets: +# mode: externalSecret +# externalSecret: +# refreshInterval: "1h" +# secretStoreRef: +# name: gcp-secrets +# kind: ClusterSecretStore +# remoteRefs: +# admin: +# password: "acme-mongodb-admin-password" +# app: +# password: "acme-mongodb-app-password" +# metrics: +# password: "acme-mongodb-metrics-password" +# +# For GAR image pulls, configure this in environments//global.yaml: +# +# global: +# imagePullSecrets: +# - name: countly-registry +# imagePullSecretExternalSecret: +# enabled: true # refreshInterval: "1h" # secretStoreRef: -# name: my-secret-store +# name: gcp-secrets # kind: ClusterSecretStore -# remoteRefs: -# common: -# encryptionReportsKey: "countly/encryption-reports-key" -# webSessionSecret: "countly/web-session-secret" -# passwordSecret: "countly/password-secret" -# clickhouse: -# url: "countly/clickhouse-url" -# username: "countly/clickhouse-username" -# password: "countly/clickhouse-password" -# database: "countly/clickhouse-database" -# kafka: -# brokers: "countly/kafka-brokers" -# securityProtocol: "countly/kafka-security-protocol" -# mongodb: -# connectionString: "countly/mongodb-connection-string" +# remoteRef: +# key: "acme-gar-dockerconfig" +# +# For bring-your-own Countly TLS from Secret Manager, configure this in +# environments//countly.yaml and set global.tls=provided: +# +# ingress: +# tls: +# secretName: "countly-tls" +# externalSecret: +# enabled: true +# refreshInterval: "1h" +# secretStoreRef: +# name: gcp-secrets +# kind: ClusterSecretStore +# remoteRefs: +# tlsCrt: "countly-prod-tls-crt" +# tlsKey: "countly-prod-tls-key" # # Prerequisites: # 1. Install External Secrets Operator: https://external-secrets.io/ diff --git a/environments/reference/global.yaml b/environments/reference/global.yaml index 541701c..a487420 100644 --- a/environments/reference/global.yaml +++ b/environments/reference/global.yaml @@ -18,6 +18,18 @@ global: # --- Registry & Storage --- imageRegistry: "" # Private registry prefix (leave empty for public images) + imageSource: + mode: direct # direct | gcpArtifactRegistry + gcpArtifactRegistry: + repositoryPrefix: "" # e.g. us-central1-docker.pkg.dev/my-project/my-repo + imagePullSecretExternalSecret: + enabled: false + refreshInterval: "1h" + secretStoreRef: + name: "" # e.g. gcp-secrets + kind: ClusterSecretStore + remoteRef: + key: "" # Secret Manager key containing the Docker config JSON storageClass: "" # Default storage class for all PVCs (leave empty for cluster default) imagePullSecrets: [] # Docker config secrets for private registries diff --git a/environments/reference/image-pull-secrets.example.yaml b/environments/reference/image-pull-secrets.example.yaml new file mode 100644 index 0000000..b1af340 --- /dev/null +++ b/environments/reference/image-pull-secrets.example.yaml @@ -0,0 +1,41 @@ +# ============================================================================= +# Image Pull Secrets Example +# ============================================================================= +# DO NOT COMMIT THIS FILE WITH REAL VALUES UNENCRYPTED. +# +# Use this when Countly and Kafka Connect pull from a private registry +# such as GCP Artifact Registry (GAR). +# +# Replace: +# - metadata.name with your actual secret name if not using "countly-registry" +# - namespaces if your releases run elsewhere +# - .dockerconfigjson with the base64-encoded contents of your Docker config +# +# You need one secret per namespace because imagePullSecrets are namespaced. +# For the default layout in this repo, create the same secret in: +# - countly +# - kafka +# +# Example source for the Docker config: +# cat ~/.docker/config.json | base64 | tr -d '\n' +# +# Kubernetes secret type must be: kubernetes.io/dockerconfigjson +# ============================================================================= + +apiVersion: v1 +kind: Secret +metadata: + name: countly-registry + namespace: countly +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: CHANGEME_BASE64_DOCKER_CONFIG_JSON +--- +apiVersion: v1 +kind: Secret +metadata: + name: countly-registry + namespace: kafka +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: CHANGEME_BASE64_DOCKER_CONFIG_JSON diff --git a/environments/reference/kafka.yaml b/environments/reference/kafka.yaml index 544557d..6100f36 100644 --- a/environments/reference/kafka.yaml +++ b/environments/reference/kafka.yaml @@ -8,6 +8,10 @@ # --- Global Settings (inherited from helmfile globals) --- global: imageRegistry: "" + imageSource: + mode: direct + gcpArtifactRegistry: + repositoryPrefix: "" imagePullSecrets: [] storageClass: "" sizing: small # local | small | production @@ -134,7 +138,7 @@ cruiseControl: kafkaConnect: enabled: true name: connect-ch - image: "gcr.io/countly-dev-313620/strimzi/kafka-connect-clickhouse:4.2.0-1.3.5-strimzi-amd64" + image: "countly/strimzi-kafka-connect-clickhouse:kafka4.2.0-ch1.3.5-strimzi0.51-otel2.12.0" replicas: 2 bootstrapServers: "" # Auto-derived from cluster if empty diff --git a/environments/reference/migration.yaml b/environments/reference/migration.yaml new file mode 100644 index 0000000..6fa760c --- /dev/null +++ b/environments/reference/migration.yaml @@ -0,0 +1,3 @@ +# Migration overrides for optional countly-migration app. +# Enable per customer by setting `migration: enabled` in argocd/customers/.yaml +# and then filling this file with environment-specific overrides as needed. diff --git a/environments/reference/mongodb.yaml b/environments/reference/mongodb.yaml index 31f230e..5631bd2 100644 --- a/environments/reference/mongodb.yaml +++ b/environments/reference/mongodb.yaml @@ -90,7 +90,7 @@ users: # ============================================================================= exporter: enabled: true - image: percona/mongodb_exporter:0.40.0 + image: percona/mongodb_exporter:0.47.2 port: 9216 resources: requests: diff --git a/environments/reference/observability.yaml b/environments/reference/observability.yaml index 79a4b39..43b25e9 100644 --- a/environments/reference/observability.yaml +++ b/environments/reference/observability.yaml @@ -84,7 +84,7 @@ profiling: prometheus: image: repository: prom/prometheus - tag: "v3.10.0" + tag: "v3.8.1" retention: time: "30d" size: "50GB" @@ -112,7 +112,7 @@ prometheus: loki: image: repository: grafana/loki - tag: "3.6.7" + tag: "3.6.3" retention: "30d" storage: backend: "filesystem" # filesystem | s3 | gcs | azure @@ -188,7 +188,7 @@ loki: tempo: image: repository: grafana/tempo - tag: "2.10.1" + tag: "2.8.1" retention: "12h" storage: backend: "local" # local | s3 | gcs | azure @@ -231,7 +231,7 @@ tempo: pyroscope: image: repository: grafana/pyroscope - tag: "1.18.1" + tag: "1.16.0" retention: "72h" storage: backend: "filesystem" # filesystem | s3 | gcs | azure | swift @@ -268,7 +268,7 @@ grafana: enabled: true # Only deployed when mode == "full" image: repository: grafana/grafana - tag: "12.4.0" + tag: "12.3.5" admin: existingSecret: "" # Use an existing Secret for admin credentials userKey: "admin-user" @@ -310,7 +310,7 @@ grafana: alloy: image: repository: grafana/alloy - tag: "v1.13.2" + tag: "v1.14.0" resources: requests: cpu: "500m" @@ -329,7 +329,7 @@ alloy: alloyOtlp: image: repository: grafana/alloy - tag: "v1.13.2" + tag: "v1.14.0" replicas: 1 resources: requests: @@ -351,7 +351,7 @@ alloyOtlp: alloyMetrics: image: repository: grafana/alloy - tag: "v1.13.2" + tag: "v1.14.0" replicas: 1 resources: requests: @@ -374,7 +374,7 @@ kubeStateMetrics: enabled: true image: repository: registry.k8s.io/kube-state-metrics/kube-state-metrics - tag: "v2.18.0" + tag: "v2.17.0" resources: requests: cpu: "10m" diff --git a/environments/reference/secrets-clickhouse.yaml b/environments/reference/secrets-clickhouse.yaml deleted file mode 100644 index 17c22f1..0000000 --- a/environments/reference/secrets-clickhouse.yaml +++ /dev/null @@ -1,4 +0,0 @@ -# ClickHouse secrets — FILL IN before first deploy -auth: - defaultUserPassword: - password: "" # REQUIRED: must match secrets-countly.yaml secrets.clickhouse.password diff --git a/environments/reference/secrets-countly.yaml b/environments/reference/secrets-countly.yaml deleted file mode 100644 index 1d60be2..0000000 --- a/environments/reference/secrets-countly.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Countly secrets — FILL IN before first deploy -# Passwords must match across charts (see secrets.example.yaml) -secrets: - mode: values - common: - encryptionReportsKey: "" # REQUIRED: min 8 chars - webSessionSecret: "" # REQUIRED: min 8 chars - passwordSecret: "" # REQUIRED: min 8 chars - clickhouse: - username: "default" - password: "" # REQUIRED: must match secrets-clickhouse.yaml - database: "countly_drill" - kafka: - securityProtocol: "PLAINTEXT" - mongodb: - password: "" # REQUIRED: must match secrets-mongodb.yaml diff --git a/environments/reference/secrets-kafka.yaml b/environments/reference/secrets-kafka.yaml deleted file mode 100644 index 9b587b7..0000000 --- a/environments/reference/secrets-kafka.yaml +++ /dev/null @@ -1,4 +0,0 @@ -# Kafka secrets — FILL IN before first deploy -kafkaConnect: - clickhouse: - password: "" # REQUIRED: must match ClickHouse default user password diff --git a/environments/reference/secrets-mongodb.yaml b/environments/reference/secrets-mongodb.yaml deleted file mode 100644 index ae59611..0000000 --- a/environments/reference/secrets-mongodb.yaml +++ /dev/null @@ -1,7 +0,0 @@ -# MongoDB secrets — FILL IN before first deploy -users: - app: - password: "" # REQUIRED: must match secrets-countly.yaml secrets.mongodb.password - metrics: - enabled: true - password: "" # REQUIRED: metrics exporter password diff --git a/environments/reference/secrets.example.yaml b/environments/reference/secrets.example.yaml index 282eb0d..181cb54 100644 --- a/environments/reference/secrets.example.yaml +++ b/environments/reference/secrets.example.yaml @@ -13,8 +13,9 @@ # - SOPS encryption (see secrets.sops.example.yaml) # ============================================================================= -# --- countly chart (environments//secrets-countly.yaml) --- +# --- countly chart (environments//credentials-countly.yaml) --- secrets: + mode: values common: encryptionReportsKey: "CHANGEME-min-8-chars" webSessionSecret: "CHANGEME-min-8-chars" @@ -24,19 +25,38 @@ secrets: mongodb: password: "CHANGEME-match-mongodb-chart" -# --- countly-mongodb chart (environments//secrets-mongodb.yaml) --- +# --- countly-mongodb chart (environments//credentials-mongodb.yaml) --- +secrets: + mode: values users: + admin: + enabled: true + password: "CHANGEME-super-admin" app: password: "CHANGEME-match-secrets.mongodb.password" metrics: password: "CHANGEME-metrics-exporter" -# --- countly-clickhouse chart (environments//secrets-clickhouse.yaml) --- +# --- countly-clickhouse chart (environments//credentials-clickhouse.yaml) --- +secrets: + mode: values auth: defaultUserPassword: password: "CHANGEME-match-secrets.clickhouse.password" -# --- countly-kafka chart (environments//secrets-kafka.yaml) --- +# --- countly-kafka chart (environments//credentials-kafka.yaml) --- +secrets: + mode: values kafkaConnect: clickhouse: password: "CHANGEME-match-clickhouse-password" + +# For External Secrets Operator, switch the per-chart file to: +# +# secrets: +# mode: externalSecret +# externalSecret: +# refreshInterval: "1h" +# secretStoreRef: +# name: gcp-secrets +# kind: ClusterSecretStore diff --git a/environments/reference/secrets.sops.example.yaml b/environments/reference/secrets.sops.example.yaml index 9b652d1..7335b8d 100644 --- a/environments/reference/secrets.sops.example.yaml +++ b/environments/reference/secrets.sops.example.yaml @@ -2,11 +2,11 @@ # SOPS Encrypted Secrets Example # ============================================================================= # Encrypt this file with SOPS before committing: -# sops --encrypt --in-place environments//secrets-countly.yaml +# sops --encrypt --in-place environments//credentials-countly.yaml # # Configure helmfile to decrypt with the helm-secrets plugin: # values: -# - secrets://environments//secrets-countly.yaml +# - secrets://environments//credentials-countly.yaml # # See: https://github.com/jkroepke/helm-secrets # ============================================================================= diff --git a/helmfile.yaml.gotmpl b/helmfile.yaml.gotmpl index 3759daf..90b78fe 100644 --- a/helmfile.yaml.gotmpl +++ b/helmfile.yaml.gotmpl @@ -36,7 +36,7 @@ releases: - profiles/sizing/{{ .Values | get "global.sizing" "small" }}/mongodb.yaml - profiles/security/{{ .Values | get "global.security" "open" }}/mongodb.yaml - environments/{{ .Environment.Name }}/mongodb.yaml - - environments/{{ .Environment.Name }}/secrets-mongodb.yaml + - environments/{{ .Environment.Name }}/credentials-mongodb.yaml - name: countly-clickhouse installed: {{ ne (.Values | get "backingServices.clickhouse.mode" "bundled") "external" }} @@ -47,7 +47,7 @@ releases: - profiles/sizing/{{ .Values | get "global.sizing" "small" }}/clickhouse.yaml - profiles/security/{{ .Values | get "global.security" "open" }}/clickhouse.yaml - environments/{{ .Environment.Name }}/clickhouse.yaml - - environments/{{ .Environment.Name }}/secrets-clickhouse.yaml + - environments/{{ .Environment.Name }}/credentials-clickhouse.yaml - name: countly-kafka installed: {{ ne (.Values | get "backingServices.kafka.mode" "bundled") "external" }} @@ -60,7 +60,7 @@ releases: - profiles/observability/{{ .Values | get "global.observability" "full" }}/kafka.yaml - profiles/security/{{ .Values | get "global.security" "open" }}/kafka.yaml - environments/{{ .Environment.Name }}/kafka.yaml - - environments/{{ .Environment.Name }}/secrets-kafka.yaml + - environments/{{ .Environment.Name }}/credentials-kafka.yaml needs: - mongodb/countly-mongodb - clickhouse/countly-clickhouse @@ -75,7 +75,7 @@ releases: - profiles/observability/{{ .Values | get "global.observability" "full" }}/countly.yaml - profiles/security/{{ .Values | get "global.security" "open" }}/countly.yaml - environments/{{ .Environment.Name }}/countly.yaml - - environments/{{ .Environment.Name }}/secrets-countly.yaml + - environments/{{ .Environment.Name }}/credentials-countly.yaml needs: - mongodb/countly-mongodb - clickhouse/countly-clickhouse @@ -92,7 +92,7 @@ releases: - profiles/observability/{{ .Values | get "global.observability" "full" }}/observability.yaml - profiles/security/{{ .Values | get "global.security" "open" }}/observability.yaml - environments/{{ .Environment.Name }}/observability.yaml - - environments/{{ .Environment.Name }}/secrets-observability.yaml + - environments/{{ .Environment.Name }}/credentials-observability.yaml needs: - countly/countly @@ -105,7 +105,7 @@ releases: values: - environments/{{ .Environment.Name }}/global.yaml - environments/{{ .Environment.Name }}/migration.yaml - - environments/{{ .Environment.Name }}/secrets-migration.yaml + - environments/{{ .Environment.Name }}/credentials-migration.yaml needs: - mongodb/countly-mongodb - clickhouse/countly-clickhouse diff --git a/nginx-ingress-values.yaml b/nginx-ingress-values.yaml index b95f8d3..7d5bcea 100644 --- a/nginx-ingress-values.yaml +++ b/nginx-ingress-values.yaml @@ -36,10 +36,10 @@ controller: resources: requests: cpu: "1" - memory: "1Gi" + memory: "2Gi" limits: - cpu: "1" - memory: "1Gi" + cpu: "2" + memory: "2Gi" # --- Service --- service: diff --git a/scripts/new-argocd-customer.sh b/scripts/new-argocd-customer.sh new file mode 100755 index 0000000..2061c04 --- /dev/null +++ b/scripts/new-argocd-customer.sh @@ -0,0 +1,373 @@ +#!/usr/bin/env bash + +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: + scripts/new-argocd-customer.sh [--secret-mode values|gcp-secrets] [project] + +Example: + scripts/new-argocd-customer.sh acme https://1.2.3.4 acme.count.ly + scripts/new-argocd-customer.sh --secret-mode gcp-secrets acme https://1.2.3.4 acme.count.ly + +This command: + 1. copies environments/reference to environments/ + 2. updates environments//global.yaml with the hostname and default profiles + 3. writes credentials files for either direct values or GCP Secret Manager + 4. creates argocd/customers/.yaml for the ApplicationSets + +Defaults: + project countly-customers + secretMode values + sizing production + security open + tls letsencrypt + observability full + kafkaConnect balanced + migration disabled + gcpSA set after scaffold for External Secrets Workload Identity +EOF +} + +secret_mode="values" +positionals=() + +while [[ $# -gt 0 ]]; do + case "$1" in + --secret-mode) + if [[ $# -lt 2 ]]; then + echo "Missing value for --secret-mode" >&2 + exit 1 + fi + secret_mode="$2" + shift 2 + ;; + -h|--help) + usage + exit 0 + ;; + --) + shift + while [[ $# -gt 0 ]]; do + positionals+=("$1") + shift + done + ;; + -*) + echo "Unknown option: $1" >&2 + usage + exit 1 + ;; + *) + positionals+=("$1") + shift + ;; + esac +done + +case "${secret_mode}" in + values|direct) + secret_mode="values" + ;; + gcp-secrets) + ;; + *) + echo "Unsupported --secret-mode: ${secret_mode}" >&2 + echo "Supported values: values, gcp-secrets" >&2 + exit 1 + ;; +esac + +if [[ ${#positionals[@]} -lt 3 || ${#positionals[@]} -gt 4 ]]; then + usage + exit 1 +fi + +customer="${positionals[0]}" +server="${positionals[1]}" +hostname="${positionals[2]}" +project="${positionals[3]:-countly-customers}" + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +env_dir="${repo_root}/environments/${customer}" +customer_file="${repo_root}/argocd/customers/${customer}.yaml" + +if [[ -e "${env_dir}" ]]; then + echo "Environment already exists: ${env_dir}" >&2 + exit 1 +fi + +if [[ -e "${customer_file}" ]]; then + echo "Customer metadata already exists: ${customer_file}" >&2 + exit 1 +fi + +mkdir -p "$(dirname "${customer_file}")" + +cp -R "${repo_root}/environments/reference" "${env_dir}" + +cat > "${env_dir}/global.yaml" < "${env_dir}/kafka.yaml" <<'EOF' +# Customer-specific Kafka overrides only. +# Leave this file minimal so sizing / kafka-connect / observability / security profiles apply cleanly. +EOF + +cat > "${env_dir}/clickhouse.yaml" <<'EOF' +# Customer-specific ClickHouse overrides only. +# Leave this file minimal so sizing / security profiles apply cleanly. +EOF + +cat > "${env_dir}/mongodb.yaml" <<'EOF' +# Customer-specific MongoDB overrides only. +# Leave this file minimal so sizing / security profiles apply cleanly. +EOF + +cat > "${env_dir}/observability.yaml" <<'EOF' +# Customer-specific observability overrides only. +EOF + +cat > "${env_dir}/migration.yaml" <<'EOF' +# Customer-specific migration overrides only. +EOF + +if [[ "${secret_mode}" == "gcp-secrets" ]]; then + cat > "${env_dir}/countly.yaml" <<'EOF' +# Customer-specific Countly overrides only. +# TLS Secret Manager support is prewired below and becomes active only when: +# - argocd/customers/.yaml sets tls: provided +# - the shared Secret Manager keys exist +# By default this reuses one shared certificate for every customer: +# - countly-prod-tls-crt +# - countly-prod-tls-key +# Override these remoteRefs only if a specific customer needs its own cert. +ingress: + tls: + externalSecret: + enabled: true + refreshInterval: "1h" + secretStoreRef: + name: gcp-secrets + kind: ClusterSecretStore + remoteRefs: + tlsCrt: countly-prod-tls-crt + tlsKey: countly-prod-tls-key +EOF + + cat > "${env_dir}/credentials-countly.yaml" < "${env_dir}/credentials-kafka.yaml" < "${env_dir}/credentials-clickhouse.yaml" < "${env_dir}/credentials-mongodb.yaml" < "${env_dir}/countly.yaml" <<'EOF' +# Customer-specific Countly overrides only. +# Leave this file minimal so sizing / TLS / observability / security profiles apply cleanly. +EOF + + cat > "${env_dir}/credentials-countly.yaml" <<'EOF' +# Countly secrets — FILL IN before first deploy +# Passwords must match across charts (see secrets.example.yaml) +secrets: + mode: values + common: + encryptionReportsKey: "" # REQUIRED: min 8 chars + webSessionSecret: "" # REQUIRED: min 8 chars + passwordSecret: "" # REQUIRED: min 8 chars + clickhouse: + username: "default" + password: "" # REQUIRED: must match credentials-clickhouse.yaml + database: "countly_drill" + kafka: + securityProtocol: "PLAINTEXT" + mongodb: + password: "" # REQUIRED: must match credentials-mongodb.yaml users.app.password +EOF + + cat > "${env_dir}/credentials-kafka.yaml" <<'EOF' +# Kafka secrets — FILL IN before first deploy +secrets: + mode: values + +kafkaConnect: + clickhouse: + password: "" # REQUIRED: must match ClickHouse default user password +EOF + + cat > "${env_dir}/credentials-clickhouse.yaml" <<'EOF' +# ClickHouse secrets — FILL IN before first deploy +secrets: + mode: values + +auth: + defaultUserPassword: + password: "" # REQUIRED: must match credentials-countly.yaml secrets.clickhouse.password +EOF + + cat > "${env_dir}/credentials-mongodb.yaml" <<'EOF' +# MongoDB secrets — FILL IN before first deploy +secrets: + mode: values + +users: + admin: + enabled: true + password: "" # REQUIRED: MongoDB super admin/root-style user + app: + password: "" # REQUIRED: must match credentials-countly.yaml secrets.mongodb.password + metrics: + enabled: true + password: "" # REQUIRED: metrics exporter password +EOF +fi + +cat > "${customer_file}" <- convention + 5. Commit and sync countly-bootstrap +EOF