From 19509091783bb4ea837522cdea1c7d8601c49fc0 Mon Sep 17 00:00:00 2001 From: jpaodev Date: Tue, 3 Feb 2026 23:31:41 +0100 Subject: [PATCH] feat: dont use kube-rbac-proxy and reinject properly --- config/default/kustomization.yaml | 3 +- config/default/manager_auth_proxy_patch.yaml | 32 +++------- config/manager/kustomization.yaml | 14 ++--- config/rbac/kustomization.yaml | 6 +- docs/api.md | 6 +- go.mod | 30 +++++++++- go.sum | 61 ++++++++++++++++++++ internal/controller/pod_controller_test.go | 58 ++++++++++++++++--- internal/workload/podspec_updates.go | 41 +++++++++++-- main.go | 31 ++++++++-- 10 files changed, 220 insertions(+), 62 deletions(-) diff --git a/config/default/kustomization.yaml b/config/default/kustomization.yaml index 782a32ce..7c42e8ee 100644 --- a/config/default/kustomization.yaml +++ b/config/default/kustomization.yaml @@ -34,7 +34,8 @@ bases: # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. #- ../prometheus patchesStrategicMerge: - # Protect the /metrics endpoint by putting it behind auth. + # Configure the controller-manager to serve metrics securely using + # controller-runtime's built-in authentication and authorization. # If you want your controller-manager to expose the /metrics # endpoint w/o any authn/z, please comment the following line. - manager_auth_proxy_patch.yaml diff --git a/config/default/manager_auth_proxy_patch.yaml b/config/default/manager_auth_proxy_patch.yaml index 2366eb6b..761a8936 100644 --- a/config/default/manager_auth_proxy_patch.yaml +++ b/config/default/manager_auth_proxy_patch.yaml @@ -11,8 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# This patch inject a sidecar container which is a HTTP proxy for the -# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. +# This patch configures the controller manager to serve metrics securely +# using controller-runtime's built-in authentication and authorization. apiVersion: apps/v1 kind: Deployment metadata: @@ -22,31 +22,13 @@ spec: template: spec: containers: - - name: kube-rbac-proxy - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - "ALL" - image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.0 + - name: manager args: - - "--secure-listen-address=0.0.0.0:8443" - - "--upstream=http://127.0.0.1:8080/" - - "--logtostderr=true" - - "--v=0" + - "--health-probe-bind-address=:8081" + - "--metrics-bind-address=:8443" + - "--metrics-secure=true" + - "--leader-elect" ports: - containerPort: 8443 protocol: TCP name: https - resources: - limits: - cpu: 500m - memory: 128Mi - requests: - cpu: 5m - memory: 64Mi - - name: manager - args: - - "--health-probe-bind-address=:8081" - - "--metrics-bind-address=127.0.0.1:8080" - - "--leader-elect" diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index bcf04cb6..48fd1106 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -12,16 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. resources: - - manager.yaml +- manager.yaml generatorOptions: disableNameSuffixHash: true configMapGenerator: - - files: - - controller_manager_config.yaml - name: manager-config +- files: + - controller_manager_config.yaml + name: manager-config apiVersion: kustomize.config.k8s.io/v1beta1 kind: Kustomization images: - - name: controller - newName: cloud-sql-proxy-operator - newTag: latest +- name: controller + newName: cloud-sql-proxy-operator + newTag: latest diff --git a/config/rbac/kustomization.yaml b/config/rbac/kustomization.yaml index fdeeb9d3..fed2d66b 100644 --- a/config/rbac/kustomization.yaml +++ b/config/rbac/kustomization.yaml @@ -22,9 +22,9 @@ resources: - role_binding.yaml - leader_election_role.yaml - leader_election_role_binding.yaml - # Comment the following 4 lines if you want to disable - # the Auth Proxy (https://github.com/brancz/kube-rbac-proxy) - # which protects your /metrics endpoint. + # The following resources are required for the controller-runtime's + # built-in metrics authentication and authorization. + # Comment these lines if you want to disable secure metrics. - auth_proxy_service.yaml - auth_proxy_role.yaml - auth_proxy_role_binding.yaml diff --git a/docs/api.md b/docs/api.md index 5f1d7262..d5061245 100644 --- a/docs/api.md +++ b/docs/api.md @@ -58,7 +58,7 @@ _Appears in:_ | `maxSigtermDelay` _integer_ | MaxSigtermDelay is the maximum number of seconds to wait for connections to
close after receiving a TERM signal. This sets the proxy container's
CLI argument `--max-sigterm-delay` and
configures `terminationGracePeriodSeconds` on the workload's PodSpec. | | Minimum: 0
Optional: \{\}
| | `minSigtermDelay` _integer_ | MinSigtermDelay is the minimum number of seconds to wait for connections to
close after receiving a TERM signal. This sets the proxy container's
CLI argument `--min-sigterm-delay` | | Minimum: 0
Optional: \{\}
| | `sqlAdminAPIEndpoint` _string_ | SQLAdminAPIEndpoint is a debugging parameter that when specified will
change the Google Cloud api endpoint used by the proxy. | | Optional: \{\}
| -| `image` _string_ | Image is the URL to the proxy image. Optional, by default the operator
will use the latest Cloud SQL Auth Proxy version as of the release of the
operator.

The operator ensures that all workloads configured with the default proxy
image are upgraded automatically to use to the latest released proxy image.

When the customer upgrades the operator, the operator upgrades all
workloads using the default proxy image to the latest proxy image. The
change to the proxy container image is applied in accordance with
the RolloutStrategy. | | Optional: \{\}
| +| `image` _string_ | Image is the URL to the proxy image. Optional, by default the operator
will use the latest Cloud SQL Auth Proxy version as of the release of the
operator.
The operator ensures that all workloads configured with the default proxy
image are upgraded automatically to use to the latest released proxy image.
When the customer upgrades the operator, the operator upgrades all
workloads using the default proxy image to the latest proxy image. The
change to the proxy container image is applied in accordance with
the RolloutStrategy. | | Optional: \{\}
| | `rolloutStrategy` _string_ | RolloutStrategy indicates the strategy to use when rolling out changes to
the workloads affected by the results. When this is set to
`Workload`, changes to this resource will be automatically applied
to a running Deployment, StatefulSet, DaemonSet, or ReplicaSet in
accordance with the Strategy set on that workload. When this is set to
`None`, the operator will take no action to roll out changes to affected
workloads. `Workload` will be used by default if no value is set.
See: https://kubernetes.io/docs/concepts/workloads/controllers/deployment/#strategy | Workload | Enum: [Workload None]
Optional: \{\}
| | `refreshStrategy` _string_ | RefreshStrategy indicates which refresh strategy the proxy should use.
When this is set to `lazy`, the proxy will use a lazy refresh strategy,
and will be configured to run with the --lazy-refresh flag. When this
omitted or set to `background`, the proxy will use the default background
refresh strategy.
See: https://github.com/GoogleCloudPlatform/cloud-sql-proxy/?tab=readme-ov-file#configuring-a-lazy-refresh | background | Enum: [lazy background]
Optional: \{\}
| | `quiet` _boolean_ | Quiet configures the proxy's --quiet flag to limit the amount of
logging generated by the proxy container. | | | @@ -130,25 +130,21 @@ _Appears in:_ InstanceSpec describes the configuration for how the proxy should expose a Cloud SQL database instance to a workload. - In the minimum recommended configuration, the operator will choose a non-conflicting TCP port and set environment variables MY_DB_SERVER_PORT MY_DB_SERVER_HOST with the value of the TCP port and hostname. The application can read these values to connect to the database through the proxy. For example: - `{ "connectionString":"my-project:us-central1:my-db-server", "portEnvName":"MY_DB_SERVER_PORT" "hostEnvName":"MY_DB_SERVER_HOST" }` - If you want to assign a specific port number for a database, set the `port` field. For example: - `{ "connectionString":"my-project:us-central1:my-db-server", "port":5000 }` diff --git a/go.mod b/go.mod index cf7d0d92..c594cf5e 100644 --- a/go.mod +++ b/go.mod @@ -14,13 +14,20 @@ require ( ) require ( + cel.dev/expr v0.24.0 // indirect + github.com/antlr4-go/antlr/v4 v4.13.0 // indirect + github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch/v5 v5.9.11 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-logr/zapr v1.3.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect @@ -28,10 +35,13 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.3 // indirect + github.com/google/cel-go v0.22.0 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/mailru/easyjson v0.7.7 // indirect @@ -43,12 +53,22 @@ require ( github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect - github.com/rogpeppe/go-internal v1.13.1 // indirect + github.com/spf13/cobra v1.8.1 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/stretchr/testify v1.10.0 // indirect + github.com/stoewer/go-strcase v1.3.0 // indirect github.com/x448/float16 v0.8.4 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 // indirect + go.opentelemetry.io/otel v1.37.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 // indirect + go.opentelemetry.io/otel/metric v1.37.0 // indirect + go.opentelemetry.io/otel/sdk v1.37.0 // indirect + go.opentelemetry.io/otel/trace v1.37.0 // indirect + go.opentelemetry.io/proto/otlp v1.3.1 // indirect go.uber.org/multierr v1.11.0 // indirect go.yaml.in/yaml/v2 v2.4.2 // indirect + golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/net v0.43.0 // indirect golang.org/x/oauth2 v0.30.0 // indirect golang.org/x/sync v0.16.0 // indirect @@ -57,14 +77,20 @@ require ( golang.org/x/text v0.28.0 // indirect golang.org/x/time v0.12.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250811230008-5f3141c8851a // indirect + google.golang.org/grpc v1.74.2 // indirect google.golang.org/protobuf v1.36.7 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.32.1 // indirect + k8s.io/apiserver v0.32.1 // indirect + k8s.io/component-base v0.32.1 // indirect k8s.io/klog/v2 v2.130.1 // indirect k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f // indirect k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect + sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 // indirect sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.2 // indirect ) diff --git a/go.sum b/go.sum index 14034a1a..480ef547 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,20 @@ +cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= +cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= cloud.google.com/go/cloudsqlconn v1.18.0 h1:mP6TY/7I+nrnIh6vmbWCRJPxpFBZSL6AZhW6HaYC/OI= cloud.google.com/go/cloudsqlconn v1.18.0/go.mod h1:58bxZZ17Mz5D83ddMT8x6w56yKpcmVXyaOwGWkzGcMw= +github.com/antlr4-go/antlr/v4 v4.13.0 h1:lxCg3LAv+EUK6t1i0y1V6/SLeUi0eKEKdhQAlS8TVTI= +github.com/antlr4-go/antlr/v4 v4.13.0/go.mod h1:pfChB/xh/Unjila75QW7+VU4TSnWnnk9UTnmpPaOR2g= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4B6AGu/h5Sxe66HYVdqdGu2l9Iebqhi/AEoA= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -15,12 +26,17 @@ github.com/evanphx/json-patch v0.5.2 h1:xVCHIVMUu1wtM/VkR9jVZ45N3FhZfYMMYGorLCR8 github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch/v5 v5.9.11 h1:/8HVnzMq13/3x9TPvjG08wUGqBTmZBsCWzjTM0wiaDU= github.com/evanphx/json-patch/v5 v5.9.11/go.mod h1:3j+LviiESTElxA4p3EMKAB9HXj3/XEtnUf6OZxqIQTM= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= @@ -39,6 +55,8 @@ github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= github.com/google/btree v1.1.3/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= +github.com/google/cel-go v0.22.0 h1:b3FJZxpiv1vTMo2/5RDUqAHPxkT8mmMfJIrq1llbf7g= +github.com/google/cel-go v0.22.0/go.mod h1:BuznPXXfQDpXKWQ9sPW3TzlAJN5zzFe+i9tIs0yC4s8= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -51,6 +69,10 @@ github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgY github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0 h1:bkypFPDjIYGfCYD5mRBvpqxfYX1YCS1PXdKYWi8FsN0= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.20.0/go.mod h1:P+Lt/0by1T8bfcF3z737NnSbmxQAppXMRziHUxPOC8k= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -92,8 +114,13 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.3.0 h1:g0eASXYtp+yvN9fK8sH94oCIk0fau9uV1/ZdJ0AVEzs= +github.com/stoewer/go-strcase v1.3.0/go.mod h1:fAH5hQ5pehh+j3nZfvwdk2RgEgQjAoM8wodgtPmh1xo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= @@ -107,6 +134,26 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0 h1:Hf9xI/XLML9ElpiHVDNwvqI0hIFlzV8dgIr35kV1kRU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.62.0/go.mod h1:NfchwuyNoMcZ5MLHwPrODwUF1HWCXWrL31s8gSAdIKY= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0 h1:3Q/xZUyC1BBkualc9ROb4G8qkH90LXEIICcs5zv1OYY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.28.0/go.mod h1:s75jGIWA9OfCMzF0xr+ZgfrB5FEbbV7UuYo32ahUiFI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0 h1:qFffATk0X+HD+f1Z8lswGiOQYKHRlzfmdJm0wEaVrFA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.27.0/go.mod h1:MOiCmryaYtc+V0Ei+Tx9o5S1ZjA7kzLucuVuyzBZloQ= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0= +go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -120,6 +167,8 @@ go.yaml.in/yaml/v3 v3.0.3/go.mod h1:tBHosrYAkRZjRAOREWbDnBXUf08JOwYq++0QNwQiWzI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= +golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -160,6 +209,12 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a h1:SGktgSolFCo75dnHJF2yMvnns6jCmHFJ0vE4Vn2JKvQ= +google.golang.org/genproto/googleapis/api v0.0.0-20250528174236-200df99c418a/go.mod h1:a77HrdMjoeKbnd2jmgcWdaS++ZLZAEq3orIOAEIKiVw= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250811230008-5f3141c8851a h1:tPE/Kp+x9dMSwUm/uM0JKK0IfdiJkwAbSMSeZBXXJXc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250811230008-5f3141c8851a/go.mod h1:gw1tLEfykwDz2ET4a12jcXt4couGAm7IwsVaTy0Sflo= +google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4= +google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM= google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -178,14 +233,20 @@ k8s.io/apiextensions-apiserver v0.32.1 h1:hjkALhRUeCariC8DiVmb5jj0VjIc1N0DREP32+ k8s.io/apiextensions-apiserver v0.32.1/go.mod h1:sxWIGuGiYov7Io1fAS2X06NjMIk5CbRHc2StSmbaQto= k8s.io/apimachinery v0.32.8 h1:95I+2jX71Tev+C+UlhNbmKfv+A/TQII42HLskiHZpBg= k8s.io/apimachinery v0.32.8/go.mod h1:GpHVgxoKlTxClKcteaeuF1Ul/lDVb74KpZcxcmLDElE= +k8s.io/apiserver v0.32.1 h1:oo0OozRos66WFq87Zc5tclUX2r0mymoVHRq8JmR7Aak= +k8s.io/apiserver v0.32.1/go.mod h1:UcB9tWjBY7aryeI5zAgzVJB/6k7E97bkr1RgqDz0jPw= k8s.io/client-go v0.32.8 h1:BkSFWUtRz/BbE3DJF98KPg7ix6lwMnIQ9DnHw3iWiSw= k8s.io/client-go v0.32.8/go.mod h1:vGkCzRxZ7BuRX2zdW7+kOwCdcgOkq9omDWb26wk/sE0= +k8s.io/component-base v0.32.1 h1:/5IfJ0dHIKBWysGV0yKTFfacZ5yNV1sulPh3ilJjRZk= +k8s.io/component-base v0.32.1/go.mod h1:j1iMMHi/sqAHeG5z+O9BFNCF698a1u0186zkjMZQ28w= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f h1:GA7//TjRY9yWGy1poLzYYJJ4JRdzg3+O6e8I+e+8T5Y= k8s.io/kube-openapi v0.0.0-20241105132330-32ad38e42d3f/go.mod h1:R/HEjbvWI0qdfb8viZUeVZm0X6IZnxAydC7YU42CMw4= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 h1:M3sRQVHv7vB20Xc2ybTt7ODCeFj6JSWYFzOFnYeS6Ro= k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0 h1:CPT0ExVicCzcpeN4baWEV2ko2Z/AsiZgEdwgcfwLgMo= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.0/go.mod h1:Ve9uj1L+deCXFrPOk1LpFXqTg7LCFzFso6PA48q/XZw= sigs.k8s.io/controller-runtime v0.20.4 h1:X3c+Odnxz+iPTRobG4tp092+CvBU9UK0t/bRf+n0DGU= sigs.k8s.io/controller-runtime v0.20.4/go.mod h1:xg2XB0K5ShQzAgsoujxuKN4LNXR2LfwwHsPj7Iaw+XY= sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 h1:/Rv+M11QRah1itp8VhT6HoVx1Ray9eB4DBr+K+/sCJ8= diff --git a/internal/controller/pod_controller_test.go b/internal/controller/pod_controller_test.go index ae31d7d3..b6520e41 100644 --- a/internal/controller/pod_controller_test.go +++ b/internal/controller/pod_controller_test.go @@ -185,13 +185,37 @@ func TestPodDeleteController(t *testing.T) { d *appsv1.Deployment wantNotFound bool setPodError bool + podErrorReason string + setInitContainerErr bool setSidecarContainers bool }{ { - name: "matching pod with error gets deleted", - d: dMatch, - setPodError: true, - wantNotFound: true, + name: "matching pod with CrashLoopBackOff gets deleted", + d: dMatch, + setPodError: true, + podErrorReason: "CrashLoopBackOff", + wantNotFound: true, + }, + { + name: "matching pod with ImagePullBackOff gets deleted", + d: dMatch, + setPodError: true, + podErrorReason: "ImagePullBackOff", + wantNotFound: true, + }, + { + name: "matching pod with ErrImagePull gets deleted", + d: dMatch, + setPodError: true, + podErrorReason: "ErrImagePull", + wantNotFound: true, + }, + { + name: "matching pod with init container error gets deleted", + d: dMatch, + setInitContainerErr: true, + podErrorReason: "CrashLoopBackOff", + wantNotFound: true, }, { name: "matching pod with no error", @@ -206,15 +230,17 @@ func TestPodDeleteController(t *testing.T) { wantNotFound: false, }, { - name: "no matching workload, pod error", - d: dNoMatch, - setPodError: true, - wantNotFound: false, + name: "no matching workload, pod error", + d: dNoMatch, + setPodError: true, + podErrorReason: "CrashLoopBackOff", + wantNotFound: false, }, { name: "matching workload, pod error, has containers", d: dNoMatch, setPodError: true, + podErrorReason: "CrashLoopBackOff", setSidecarContainers: true, wantNotFound: false, }, @@ -245,9 +271,23 @@ func TestPodDeleteController(t *testing.T) { t.Fatal(err) } if tc.setPodError { + reason := tc.podErrorReason + if reason == "" { + reason = "CrashLoopBackOff" + } pods[0].Status.ContainerStatuses = []corev1.ContainerStatus{{ Name: pods[0].Spec.Containers[0].Name, - State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: "CrashLoopBackOff"}}, + State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: reason}}, + }} + } + if tc.setInitContainerErr { + reason := tc.podErrorReason + if reason == "" { + reason = "CrashLoopBackOff" + } + pods[0].Status.InitContainerStatuses = []corev1.ContainerStatus{{ + Name: "init-container", + State: corev1.ContainerState{Waiting: &corev1.ContainerStateWaiting{Reason: reason}}, }} } diff --git a/internal/workload/podspec_updates.go b/internal/workload/podspec_updates.go index 046c898b..1c8fd9b9 100644 --- a/internal/workload/podspec_updates.go +++ b/internal/workload/podspec_updates.go @@ -217,7 +217,7 @@ func (u *Updater) filterMatchingInstances(pl *cloudsqlapi.AuthProxyWorkloadList, // CheckWorkloadContainers determines if a pod is configured incorrectly and // therefore needs to be deleted. Pods must be (1) missing one or more proxy -// sidecar containers and (2) have a terminated container. +// sidecar containers and (2) have a container in an error or problematic waiting state. func (u *Updater) CheckWorkloadContainers(wl *PodWorkload, matches []*cloudsqlapi.AuthProxyWorkload) error { // Find the names of all AuthProxyWorkload resources that should have a @@ -246,13 +246,44 @@ func (u *Updater) CheckWorkloadContainers(wl *PodWorkload, matches []*cloudsqlap missingSidecars := strings.Join(missing, ", ") - // Some proxy containers are missing. Are the remaining pod containers failing? - for _, cs := range wl.Pod.Status.ContainerStatuses { + // problematicWaitingReasons are waiting states that indicate the pod is stuck + // and unlikely to recover without intervention. + problematicWaitingReasons := map[string]bool{ + "CrashLoopBackOff": true, + "ImagePullBackOff": true, + "ErrImagePull": true, + "CreateContainerConfigError": true, + "InvalidImageName": true, + "CreateContainerError": true, + "RunContainerError": true, + } + + // checkContainerStatus checks if a container status indicates a problematic state + checkContainerStatus := func(cs corev1.ContainerStatus) error { if cs.State.Terminated != nil && cs.State.Terminated.Reason == "Error" { return fmt.Errorf("pod is in an error state and missing sidecar containers %v", missingSidecars) } - if cs.State.Waiting != nil && cs.State.Waiting.Reason == "CrashLoopBackOff" { - return fmt.Errorf("pod is in a CrashLoopBackOff state and missing sidecar containers %v", missingSidecars) + if cs.State.Waiting != nil { + if problematicWaitingReasons[cs.State.Waiting.Reason] { + return fmt.Errorf("pod is in a %s state and missing sidecar containers %v", + cs.State.Waiting.Reason, missingSidecars) + } + } + return nil + } + + // Check regular container statuses + for _, cs := range wl.Pod.Status.ContainerStatuses { + if err := checkContainerStatus(cs); err != nil { + return err + } + } + + // Check init container statuses - these are critical because if the proxy + // sidecar wasn't injected as an init container, other init containers may fail + for _, cs := range wl.Pod.Status.InitContainerStatuses { + if err := checkContainerStatus(cs); err != nil { + return err } } diff --git a/main.go b/main.go index 57f340c8..103a7078 100644 --- a/main.go +++ b/main.go @@ -14,6 +14,7 @@ package main import ( + "crypto/tls" "flag" "fmt" "os" @@ -21,6 +22,7 @@ import ( "github.com/GoogleCloudPlatform/cloud-sql-proxy-operator/internal/controller" "github.com/GoogleCloudPlatform/cloud-sql-proxy-operator/internal/workload" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" "sigs.k8s.io/controller-runtime/pkg/webhook" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) @@ -51,11 +53,14 @@ func main() { var metricsAddr string var enableLeaderElection bool var probeAddr string - flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") + var secureMetrics bool + flag.StringVar(&metricsAddr, "metrics-bind-address", ":8443", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") + flag.BoolVar(&secureMetrics, "metrics-secure", true, + "If set, the metrics endpoint is served securely via HTTPS. Use --metrics-secure=false to use HTTP instead.") opts := zap.Options{ Development: true, } @@ -67,11 +72,27 @@ func main() { ctrl.Log.Info(fmt.Sprintf("Version: %v Build: %v", version, buildID)) ctrl.Log.Info(fmt.Sprintf("Runtime: %v %v/%v", runtime.Version(), runtime.GOOS, runtime.GOARCH)) + // Configure metrics server options + metricsServerOptions := metricsserver.Options{ + BindAddress: metricsAddr, + } + + if secureMetrics { + // Use controller-runtime's built-in authentication and authorization + // for the metrics endpoint, replacing the deprecated kube-rbac-proxy. + metricsServerOptions.SecureServing = true + metricsServerOptions.FilterProvider = filters.WithAuthenticationAndAuthorization + // TLS config with reasonable defaults + metricsServerOptions.TLSOpts = []func(*tls.Config){ + func(c *tls.Config) { + c.MinVersion = tls.VersionTLS12 + }, + } + } + mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - Metrics: metricsserver.Options{ - BindAddress: metricsAddr, - }, + Scheme: scheme, + Metrics: metricsServerOptions, WebhookServer: &webhook.DefaultServer{ Options: webhook.Options{Port: 9443}, },