Fixes and updates for Concourse deployment

.tool-versions

@@ -1,10 +1,10 @@
 concourse 7.13.1
 gcloud 561.0.0
 helm 3.20.1
+opentofu 1.11.5
 kapp 0.65.1
 kubectl 1.35.3
 python 3.14.3
-terraform 1.13.3
 terraform-lsp 0.0.12
 terragrunt 0.99.4
 vendir 0.45.2

README.md

@@ -18,7 +18,7 @@ Please see [ARC Readme](<./docs/actions-runner-controller/README.md>)
 The required tools to work with this repository are:
   * glcoud
   * helm
-  * terraform
+  * opentofu
   * terragrunt
   * kapp
   * ytt

docs/concourse/concourse_minor_version_upgrade.md

@@ -18,7 +18,7 @@ Please note the process should be also useful for upgrading major versions.
 
 3. Confirm there are no pending changes for the Concourse stack
    ```
-   terragrunt run-all plan --terragrunt-source-update
+   terragrunt run --all plan --source-update
    ```
    Note: The Cloud SQL database has "automatic storage increases" enabled. So the disk could have grown larger than the initial value from the `config.yaml` file. In that case, Terraform would have to shrink the disk which is not possible. Instead, it tries to destroy the database and recreate it, losing all data including backups:
    ```
@@ -52,7 +52,7 @@ Make sure the following flags are enabled:
 
 7. Apply roll-out for new Concourse version
    ```
-   terragrunt run-all apply --terragrunt-source-update
+   terragrunt run --all apply --source-update
    ```
 
 At this point depending on your use case:
@@ -67,7 +67,7 @@ At this point depending on your use case:
 3. For `wg-ci-test` cluster: execute end-to-end test
    ```
    cd e2e_test
-   terragrunt apply --terragrunt-source-update
+   terragrunt apply --source-update
    cd ..
    ```
 
@@ -85,8 +85,8 @@ To guarantee SQL databases consistency delete Concourse deployment and restore S
 
 3. Run Concourse stack deployment
    ```
-   terragrunt run-all plan
-   terragrunt run-all apply
+   terragrunt run --all plan
+   terragrunt run --all apply
    ```
 
 4. Set back to the previous version of the `fly` binary

docs/concourse/developer_notes.md

@@ -3,7 +3,7 @@
 # Plan/apply terragrunt for changes to modules
 Update your terragrunt cache folders when terraform source modules code would change
 ```sh
-terragrunt run-all plan --terragrunt-source-update
+terragrunt run --all plan --source-update
 ```
 
 ## Upgrade components managed by kapp and vendir (when needed)

terraform-modules/concourse/backend/databases.tf

@@ -1,22 +1,15 @@
-data "google_sql_database_instance" "concourse" {
-  name    = var.sql_instance_name
-  project = var.project
-
-}
-
 resource "google_sql_database" "concourse" {
-
   for_each = toset([
     "concourse",
     "credhub",
     "uaa"
   ])
-  charset    = "UTF8"
-  collation  = "en_US.UTF8"
-  instance   = data.google_sql_database_instance.concourse.name
+  charset   = "UTF8"
+  collation = "en_US.UTF8"
+  # The SQL instance is created by the infra stack; use its configured name directly.
+  instance   = var.sql_instance_name
   name       = each.key
   project    = var.project
-  depends_on = [data.google_sql_database_instance.concourse, carvel_kapp.sqlproxy, carvel_kapp.carvel_secretgen]
-
+  depends_on = [carvel_kapp.sqlproxy, carvel_kapp.carvel_secretgen]
 }
 

terraform-modules/concourse/backend/files/config/cloud_sql_proxy/sql_accounts.yml

@@ -27,6 +27,12 @@ kind: Password
 metadata:
   name: #@ name + "-postgresql-password"
   namespace: concourse
+spec:
+  length: 32
+  digits: 8
+  uppercaseLetters: 8
+  lowercaseLetters: 16
+  symbols: 0
 #@ end
 
 --- #@ template.replace(add_database_user("concourse"))

terraform-modules/concourse/backend/iam_sql-proxy.tf

@@ -1,8 +1,8 @@
 resource "google_service_account" "sql_proxy" {
-  account_id  = "${var.gke_name}-sql-proxy"
+  account_id   = "${var.gke_name}-sql-proxy"
   display_name = "Used by Cloud SQL Auth proxy [${var.gke_name}]"
-  disabled    = "false"
-  project     = var.project
+  disabled     = "false"
+  project      = var.project
 }
 
 resource "google_service_account_iam_member" "sql_proxy" {
@@ -14,7 +14,7 @@ resource "google_service_account_iam_member" "sql_proxy" {
 resource "google_project_iam_member" "sql_proxy" {
   project = var.project
   member  = "serviceAccount:${google_service_account.sql_proxy.email}"
-  role = "roles/cloudsql.client"
+  role    = "roles/cloudsql.client"
 
 }
 
@@ -29,5 +29,5 @@ resource "kubectl_manifest" "sql_proxy_service_account" {
        iam.gke.io/gcp-service-account: ${google_service_account.sql_proxy.email}
   YAML
 
-  depends_on = [data.google_container_cluster.wg_ci, google_service_account.sql_proxy, kubectl_manifest.config_connector, kubernetes_namespace.concourse ]
+  depends_on = [google_service_account.sql_proxy, kubectl_manifest.config_connector, kubernetes_namespace.concourse]
 }

terraform-modules/concourse/backend/kapp_sqlproxy.tf

@@ -4,10 +4,10 @@ data "carvel_ytt" "sqlproxy" {
     "files/config/cloud_sql_proxy",
   ]
   values = {
-    "google.project_id"       = var.project
-    "google.region"           = var.region
-    "database.instance"       = var.sql_instance_name
-    "sql_proxy_account.name"  = "${var.gke_name}-sql-proxy"
+    "google.project_id"      = var.project
+    "google.region"          = var.region
+    "database.instance"      = var.sql_instance_name
+    "sql_proxy_account.name" = "${var.gke_name}-sql-proxy"
   }
 }
 

terraform-modules/concourse/backend/providers.tf

@@ -30,8 +30,8 @@ data "google_container_cluster" "wg_ci" {
 provider "carvel" {
   kapp {
     kubeconfig {
-      server = "https://${data.google_container_cluster.wg_ci.endpoint}"
-      token  = data.google_client_config.provider.access_token
+      server  = "https://${data.google_container_cluster.wg_ci.endpoint}"
+      token   = data.google_client_config.provider.access_token
       ca_cert = base64decode(data.google_container_cluster.wg_ci.master_auth[0].cluster_ca_certificate)
     }
   }
@@ -49,7 +49,7 @@ provider "kubectl" {
 
 
 provider "kubernetes" {
-  host  = "https://${data.google_container_cluster.wg_ci.endpoint}"
-  token = data.google_client_config.provider.access_token
+  host                   = "https://${data.google_container_cluster.wg_ci.endpoint}"
+  token                  = data.google_client_config.provider.access_token
   cluster_ca_certificate = base64decode(data.google_container_cluster.wg_ci.master_auth[0].cluster_ca_certificate)
 }

terragrunt/concourse-wg-ci-test/app/terragrunt.hcl

@@ -54,4 +54,5 @@ inputs = {
   concourse_container_placement_strategy = local.config.concourse_container_placement_strategy
   concourse_max_days_to_retain_build_logs = local.config.concourse_max_days_to_retain_build_logs
   concourse_max_build_logs_to_retain = local.config.concourse_max_build_logs_to_retain
+  concourse_enable_redact_secrets = try(local.config.concourse_enable_redact_secrets, null)
 }

terragrunt/concourse-wg-ci-test/backend/terragrunt.hcl

@@ -13,10 +13,10 @@ remote_state {
     if_exists = "overwrite"
   }
   config = {
-    bucket         = "${local.config.gcs_bucket}"
-    prefix         = "${local.config.gcs_prefix}/concourse-backend"
-    project        = "${local.config.project}"
-    location       = "${local.config.region}"
+    bucket   = local.config.gcs_bucket
+    prefix   = "${local.config.gcs_prefix}/concourse-backend"
+    project  = local.config.project
+    location = local.config.region
     # use for uniform bucket-level access
     # (https://cloud.google.com/storage/docs/uniform-bucket-level-access)
     enable_bucket_policy_only = false

terragrunt/concourse-wg-ci-test/config.yaml

@@ -21,15 +21,17 @@ concourse_github_mainTeam: "cloudfoundry:wg-app-runtime-interfaces-autoscaler-ap
 concourse_github_mainTeamUser: ""
 
 # Concourse worker placement strategy: https://concourse-ci.org/container-placement.html
-# Use default strategy for test system
-concourse_container_placement_strategy: "volume-locality"
+# The cloud controller unit tests cause a high system load on workers, so place them on workers with few containers
+concourse_container_placement_strategy: "fewest-build-containers"
 # Optional: Set the maximum number of days to retain Concourse build logs.
 # If not set, the default Concourse is configured to feel very snappy!.
 concourse_max_days_to_retain_build_logs: 30
 # Optional: Set the maximum number of build logs to retain.
 # If not set or set to 0, the default Concourse is configured to feel very snappy!.
 concourse_max_build_logs_to_retain: 200
 
+concourse_enable_redact_secrets: true
+
 # Concourse helm chart
 concourse_helm_version: "18.1.1"
 
@@ -74,7 +76,7 @@ sql_instance_disk_size: 38
 
 # Other GKE vars
 gke_deletion_protection : true
-gke_controlplane_version: "1.31"
+gke_controlplane_version: "1.34"
 gke_cluster_ipv4_cidr: 10.104.0.0/14
 gke_services_ipv4_cidr_block: 10.108.0.0/20
 gke_master_ipv4_cidr_block: 172.16.0.32/28

terragrunt/concourse-wg-ci/backend/terragrunt.hcl

@@ -13,10 +13,10 @@ remote_state {
     if_exists = "overwrite"
   }
   config = {
-    bucket         = "${local.config.gcs_bucket}"
-    prefix         = "${local.config.gcs_prefix}/concourse-backend"
-    project        = "${local.config.project}"
-    location       = "${local.config.region}"
+    bucket   = local.config.gcs_bucket
+    prefix   = "${local.config.gcs_prefix}/concourse-backend"
+    project  = local.config.project
+    location = local.config.region
     # use for uniform bucket-level access
     # (https://cloud.google.com/storage/docs/uniform-bucket-level-access)
     enable_bucket_policy_only = false