diff --git a/Makefile b/Makefile index 1227787..ad93916 100644 --- a/Makefile +++ b/Makefile @@ -61,10 +61,6 @@ vet: ## Run go vet against code. test: manifests generate fmt vet setup-envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test -v $$(go list ./... | grep -v /e2e) -coverprofile cover.out -# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'. -# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. -# CertManager is installed by default; skip with: -# - CERT_MANAGER_INSTALL_SKIP=true .PHONY: test-e2e test-e2e: manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind. @command -v kind >/dev/null 2>&1 || { \ @@ -75,7 +71,20 @@ test-e2e: manifests generate fmt vet ## Run the e2e tests. Expected an isolated echo "No Kind cluster is running. Please start a Kind cluster before running the e2e tests."; \ exit 1; \ } - go test ./test/e2e/ -v -ginkgo.v + CERT_MANAGER_INSTALL_SKIP=true go test ./test/e2e/ -v -ginkgo.v + +.PHONY: e2e-deploy +e2e-deploy: manifests helm + $(HELM) template -n managed-postgres-operator-system managed-postgres-operator-controller-manager deploy/charts/managed-postgres-operator \ + --set podLabels.control-plane=controller-manager \ + --set image.repository=$(firstword $(subst :, ,$(IMG))) \ + --set image.tag=$(lastword $(subst :, ,$(IMG))) \ + | $(KUBECTL) apply -n managed-postgres-operator-system -f - + +.PHONY: e2e-undeploy +e2e-undeploy: manifests helm + $(HELM) template -n managed-postgres-operator-system managed-postgres-operator \ + | $(KUBECTL) delete -n managed-postgres-operator-system -f - .PHONY: lint lint: golangci-lint ## Run golangci-lint linter @@ -104,11 +113,11 @@ run: manifests generate fmt vet ## Run a controller from your host. # More info: https://docs.docker.com/develop/develop-images/build_enhancements/ .PHONY: docker-build docker-build: ## Build docker image with the manager. - $(CONTAINER_TOOL) build -t ${IMG} . + $(CONTAINER_TOOL) build -t $(IMG) . .PHONY: docker-push docker-push: ## Push docker image with the manager. - $(CONTAINER_TOOL) push ${IMG} + $(CONTAINER_TOOL) push $(IMG) # PLATFORMS defines the target platforms for the manager image be built to provide support to multiple # architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to: @@ -123,16 +132,10 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross - $(CONTAINER_TOOL) buildx create --name managed-postgres-operator-builder $(CONTAINER_TOOL) buildx use managed-postgres-operator-builder - - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMG} -f Dockerfile.cross . + - $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag $(IMG) -f Dockerfile.cross . - $(CONTAINER_TOOL) buildx rm managed-postgres-operator-builder rm Dockerfile.cross -.PHONY: build-installer -build-installer: manifests generate kustomize ## Generate a consolidated YAML with CRDs and deployment. - mkdir -p dist - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default > dist/install.yaml - ##@ Deployment ifndef ignore-not-found @@ -140,21 +143,22 @@ ifndef ignore-not-found endif .PHONY: install -install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config. - $(KUSTOMIZE) build config/crd | $(KUBECTL) apply -f - +install: manifests ## Install CRDs into the K8s cluster specified in ~/.kube/config. + $(KUBECTL) apply -f deploy/crds/ .PHONY: uninstall -uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - +uninstall: manifests ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f deploy/crds/ .PHONY: deploy -deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config. - cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG} - $(KUSTOMIZE) build config/default | $(KUBECTL) apply -f - +deploy: manifests helm ## Deploy controller to the K8s cluster specified in ~/.kube/config. + $(HELM) install managed-postgres-operator-controller-manager deploy/charts/managed-postgres-operator \ + | $(KUBECTL) apply -n managed-postgres-operator-system -f - .PHONY: undeploy -undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. - $(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f - +undeploy: helm ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion. + $(HELM) template managed-postgres-operator deploy/charts/managed-postgres-operator \ + | $(KUBECTL) delete -n managed-postgres-operator-system -f - ##@ Dependencies @@ -165,13 +169,13 @@ $(LOCALBIN): ## Tool Binaries KUBECTL ?= kubectl -KUSTOMIZE ?= $(LOCALBIN)/kustomize +HELM ?= $(LOCALBIN)/helm CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ENVTEST ?= $(LOCALBIN)/setup-envtest GOLANGCI_LINT = $(LOCALBIN)/golangci-lint ## Tool Versions -KUSTOMIZE_VERSION ?= v5.5.0 +HELM_VERSION ?= v4.0.0 CONTROLLER_TOOLS_VERSION ?= v0.17.2 #ENVTEST_VERSION is the version of controller-runtime release branch to fetch the envtest setup script (i.e. release-0.20) ENVTEST_VERSION ?= $(shell go list -m -f "{{ .Version }}" sigs.k8s.io/controller-runtime | awk -F'[v.]' '{printf "release-%d.%d", $$2, $$3}') @@ -179,10 +183,10 @@ ENVTEST_VERSION ?= $(shell go list -m -f "{{ .Version }}" sigs.k8s.io/controller ENVTEST_K8S_VERSION ?= $(shell go list -m -f "{{ .Version }}" k8s.io/api | awk -F'[v.]' '{printf "1.%d", $$3}') GOLANGCI_LINT_VERSION ?= v1.63.4 -.PHONY: kustomize -kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. -$(KUSTOMIZE): $(LOCALBIN) - $(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION)) +.PHONY: helm +helm: $(HELM) ## Download helm locally if necessary. +$(HELM): $(LOCALBIN) + $(call go-install-tool,$(HELM),helm.sh/helm/v4/cmd/helm,$(HELM_VERSION)) .PHONY: controller-gen controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. diff --git a/deploy/charts/managed-postgres-operator/values.yaml b/deploy/charts/managed-postgres-operator/values.yaml index eeec297..e61e7d6 100644 --- a/deploy/charts/managed-postgres-operator/values.yaml +++ b/deploy/charts/managed-postgres-operator/values.yaml @@ -22,9 +22,16 @@ serviceAccount: podAnnotations: {} podLabels: {} -podSecurityContext: {} - -securityContext: {} +podSecurityContext: + runAsNonRoot: true + runAsUser: 1000 + seccompProfile: + type: RuntimeDefault + +securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] resources: {} diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 6301c2a..a643894 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -17,11 +17,8 @@ limitations under the License. package e2e import ( - "encoding/json" "fmt" - "os" "os/exec" - "path/filepath" "time" . "github.com/onsi/ginkgo/v2" @@ -33,15 +30,6 @@ import ( // namespace where the project is deployed in const namespace = "managed-postgres-operator-system" -// serviceAccountName created for the project -const serviceAccountName = "managed-postgres-operator-controller-manager" - -// metricsServiceName is the name of the metrics service of the project -const metricsServiceName = "managed-postgres-operator-controller-manager-metrics-service" - -// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data -const metricsRoleBindingName = "managed-postgres-operator-metrics-binding" - var _ = Describe("Manager", Ordered, func() { var controllerPodName string @@ -66,7 +54,7 @@ var _ = Describe("Manager", Ordered, func() { Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs") By("deploying the controller-manager") - cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage)) + cmd = exec.Command("make", "e2e-deploy", fmt.Sprintf("IMG=%s", projectImage)) _, err = utils.Run(cmd) Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager") }) @@ -79,7 +67,7 @@ var _ = Describe("Manager", Ordered, func() { _, _ = utils.Run(cmd) By("undeploying the controller-manager") - cmd = exec.Command("make", "undeploy") + cmd = exec.Command("make", "e2e-undeploy") _, _ = utils.Run(cmd) By("uninstalling CRDs") @@ -169,161 +157,5 @@ var _ = Describe("Manager", Ordered, func() { } Eventually(verifyControllerUp).Should(Succeed()) }) - - It("should ensure the metrics endpoint is serving metrics", func() { - By("creating a ClusterRoleBinding for the service account to allow access to metrics") - cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName, - "--clusterrole=managed-postgres-operator-metrics-reader", - fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName), - ) - _, err := utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding") - - By("validating that the metrics service is available") - cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace) - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Metrics service should exist") - - By("getting the service account token") - token, err := serviceAccountToken() - Expect(err).NotTo(HaveOccurred()) - Expect(token).NotTo(BeEmpty()) - - By("waiting for the metrics endpoint to be ready") - verifyMetricsEndpointReady := func(g Gomega) { - cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace) - output, err := utils.Run(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready") - } - Eventually(verifyMetricsEndpointReady).Should(Succeed()) - - By("verifying that the controller manager is serving the metrics server") - verifyMetricsServerStarted := func(g Gomega) { - cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) - output, err := utils.Run(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(output).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"), - "Metrics server not yet started") - } - Eventually(verifyMetricsServerStarted).Should(Succeed()) - - By("creating the curl-metrics pod to access the metrics endpoint") - cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never", - "--namespace", namespace, - "--image=curlimages/curl:latest", - "--overrides", - fmt.Sprintf(`{ - "spec": { - "containers": [{ - "name": "curl", - "image": "curlimages/curl:latest", - "command": ["/bin/sh", "-c"], - "args": ["curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics"], - "securityContext": { - "allowPrivilegeEscalation": false, - "capabilities": { - "drop": ["ALL"] - }, - "runAsNonRoot": true, - "runAsUser": 1000, - "seccompProfile": { - "type": "RuntimeDefault" - } - } - }], - "serviceAccount": "%s" - } - }`, token, metricsServiceName, namespace, serviceAccountName)) - _, err = utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod") - - By("waiting for the curl-metrics pod to complete.") - verifyCurlUp := func(g Gomega) { - cmd := exec.Command("kubectl", "get", "pods", "curl-metrics", - "-o", "jsonpath={.status.phase}", - "-n", namespace) - output, err := utils.Run(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(output).To(Equal("Succeeded"), "curl pod in wrong status") - } - Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed()) - - By("getting the metrics by checking curl-metrics logs") - metricsOutput := getMetricsOutput() - Expect(metricsOutput).To(ContainSubstring( - "controller_runtime_reconcile_total", - )) - }) - - // +kubebuilder:scaffold:e2e-webhooks-checks - - // TODO: Customize the e2e test suite with scenarios specific to your project. - // Consider applying sample/CR(s) and check their status and/or verifying - // the reconciliation by using the metrics, i.e.: - // metricsOutput := getMetricsOutput() - // Expect(metricsOutput).To(ContainSubstring( - // fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`, - // strings.ToLower(), - // )) }) }) - -// serviceAccountToken returns a token for the specified service account in the given namespace. -// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request -// and parsing the resulting token from the API response. -func serviceAccountToken() (string, error) { - const tokenRequestRawString = `{ - "apiVersion": "authentication.k8s.io/v1", - "kind": "TokenRequest" - }` - - // Temporary file to store the token request - secretName := fmt.Sprintf("%s-token-request", serviceAccountName) - tokenRequestFile := filepath.Join("/tmp", secretName) - err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644)) - if err != nil { - return "", err - } - - var out string - verifyTokenCreation := func(g Gomega) { - // Execute kubectl command to create the token - cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf( - "/api/v1/namespaces/%s/serviceaccounts/%s/token", - namespace, - serviceAccountName, - ), "-f", tokenRequestFile) - - output, err := cmd.CombinedOutput() - g.Expect(err).NotTo(HaveOccurred()) - - // Parse the JSON output to extract the token - var token tokenRequest - err = json.Unmarshal(output, &token) - g.Expect(err).NotTo(HaveOccurred()) - - out = token.Status.Token - } - Eventually(verifyTokenCreation).Should(Succeed()) - - return out, err -} - -// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint. -func getMetricsOutput() string { - By("getting the curl-metrics logs") - cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace) - metricsOutput, err := utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod") - Expect(metricsOutput).To(ContainSubstring("< HTTP/1.1 200 OK")) - return metricsOutput -} - -// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response, -// containing only the token field that we need to extract. -type tokenRequest struct { - Status struct { - Token string `json:"token"` - } `json:"status"` -}