diff --git a/assets/optional/kube-state-metrics/00-namespace.yaml b/assets/optional/kube-state-metrics/00-namespace.yaml new file mode 100644 index 0000000000..17f727565a --- /dev/null +++ b/assets/optional/kube-state-metrics/00-namespace.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: openshift-monitoring + labels: + name: openshift-monitoring + pod-security.kubernetes.io/enforce: privileged + pod-security.kubernetes.io/audit: privileged + pod-security.kubernetes.io/warn: privileged diff --git a/assets/optional/kube-state-metrics/01-cluster-role-binding.yaml b/assets/optional/kube-state-metrics/01-cluster-role-binding.yaml new file mode 100644 index 0000000000..c8e3419960 --- /dev/null +++ b/assets/optional/kube-state-metrics/01-cluster-role-binding.yaml @@ -0,0 +1,18 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/managed-by: cluster-monitoring-operator + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/part-of: openshift-monitoring + app.kubernetes.io/version: 2.19.0 + name: kube-state-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kube-state-metrics +subjects: +- kind: ServiceAccount + name: kube-state-metrics + namespace: openshift-monitoring diff --git a/assets/optional/kube-state-metrics/01-cluster-role.yaml b/assets/optional/kube-state-metrics/01-cluster-role.yaml new file mode 100644 index 0000000000..ab123ee6cd --- /dev/null +++ b/assets/optional/kube-state-metrics/01-cluster-role.yaml @@ -0,0 +1,153 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/managed-by: cluster-monitoring-operator + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/part-of: openshift-monitoring + app.kubernetes.io/version: 2.19.0 + name: kube-state-metrics +rules: +- apiGroups: + - "" + resources: + - configmaps + - secrets + - nodes + - pods + - services + - serviceaccounts + - resourcequotas + - replicationcontrollers + - limitranges + - persistentvolumeclaims + - persistentvolumes + - namespaces + - endpoints + verbs: + - list + - watch +- apiGroups: + - apps + resources: + - statefulsets + - daemonsets + - deployments + - replicasets + verbs: + - list + - watch +- apiGroups: + - batch + resources: + - cronjobs + - jobs + verbs: + - list + - watch +- apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - list + - watch +- apiGroups: + - authentication.k8s.io + resources: + - tokenreviews + verbs: + - create +- apiGroups: + - authorization.k8s.io + resources: + - subjectaccessreviews + verbs: + - create +- apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - list + - watch +- apiGroups: + - certificates.k8s.io + resources: + - certificatesigningrequests + verbs: + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - volumeattachments + verbs: + - list + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - mutatingwebhookconfigurations + - validatingwebhookconfigurations + verbs: + - list + - watch +- apiGroups: + - networking.k8s.io + resources: + - networkpolicies + - ingressclasses + - ingresses + verbs: + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - list + - watch +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterrolebindings + - clusterroles + - rolebindings + - roles + verbs: + - list + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch +- apiGroups: + - autoscaling.k8s.io + resources: + - verticalpodautoscalers + verbs: + - list + - watch +- apiGroups: + - gateway.networking.k8s.io + resources: + - gatewayclasses + - gateways + verbs: + - list + - watch diff --git a/assets/optional/kube-state-metrics/01-service-account.yaml b/assets/optional/kube-state-metrics/01-service-account.yaml new file mode 100644 index 0000000000..7f3fe4b1ce --- /dev/null +++ b/assets/optional/kube-state-metrics/01-service-account.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +automountServiceAccountToken: false +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/managed-by: cluster-monitoring-operator + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/part-of: openshift-monitoring + app.kubernetes.io/version: 2.19.0 + name: kube-state-metrics + namespace: openshift-monitoring diff --git a/assets/optional/kube-state-metrics/02-custom-resource-state-configmap.yaml b/assets/optional/kube-state-metrics/02-custom-resource-state-configmap.yaml new file mode 100644 index 0000000000..63adb89f96 --- /dev/null +++ b/assets/optional/kube-state-metrics/02-custom-resource-state-configmap.yaml @@ -0,0 +1,544 @@ +apiVersion: v1 +data: + custom-resource-state-configmap.yaml: |- + "kind": "CustomResourceStateMetrics" + "spec": + "resources": + - "groupVersionKind": + "group": "autoscaling.k8s.io" + "kind": "VerticalPodAutoscaler" + "version": "v1" + "metrics": + - "commonLabels": null + "each": + "stateSet": + "labelName": "updatemode" + "list": + - "Off" + - "Initial" + - "Recreate" + - "Auto" + "path": + - "spec" + - "updatePolicy" + - "updateMode" + "type": "StateSet" + "help": "Update mode of the VerticalPodAutoscaler." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_spec_updatepolicy_updatemode" + - "commonLabels": + "resource": "cpu" + "unit": "cores" + "each": + "gauge": + "labelsFromPath": + "container": + - "containerName" + "path": + - "status" + - "recommendation" + - "containerRecommendations" + "valueFrom": + - "lowerBound" + - "cpu" + "type": "Gauge" + "help": "Minimum cpu resources the container can use before the VerticalPodAutoscaler updater evicts it." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound_cpu" + - "commonLabels": + "resource": "memory" + "unit": "bytes" + "each": + "gauge": + "labelsFromPath": + "container": + - "containerName" + "path": + - "status" + - "recommendation" + - "containerRecommendations" + "valueFrom": + - "lowerBound" + - "memory" + "type": "Gauge" + "help": "Minimum memory resources the container can use before the VerticalPodAutoscaler updater evicts it." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound_memory" + - "commonLabels": + "resource": "cpu" + "unit": "cores" + "each": + "gauge": + "labelsFromPath": + "container": + - "containerName" + "path": + - "status" + - "recommendation" + - "containerRecommendations" + "valueFrom": + - "upperBound" + - "cpu" + "type": "Gauge" + "help": "Maximum cpu resources the container can use before the VerticalPodAutoscaler updater evicts it." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound_cpu" + - "commonLabels": + "resource": "memory" + "unit": "bytes" + "each": + "gauge": + "labelsFromPath": + "container": + - "containerName" + "path": + - "status" + - "recommendation" + - "containerRecommendations" + "valueFrom": + - "upperBound" + - "memory" + "type": "Gauge" + "help": "Maximum memory resources the container can use before the VerticalPodAutoscaler updater evicts it." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound_memory" + - "commonLabels": + "resource": "cpu" + "unit": "cores" + "each": + "gauge": + "labelsFromPath": + "container": + - "containerName" + "path": + - "status" + - "recommendation" + - "containerRecommendations" + "valueFrom": + - "target" + - "cpu" + "type": "Gauge" + "help": "Target cpu resources the VerticalPodAutoscaler recommends for the container." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_status_recommendation_containerrecommendations_target_cpu" + - "commonLabels": + "resource": "memory" + "unit": "bytes" + "each": + "gauge": + "labelsFromPath": + "container": + - "containerName" + "path": + - "status" + - "recommendation" + - "containerRecommendations" + "valueFrom": + - "target" + - "memory" + "type": "Gauge" + "help": "Target memory resources the VerticalPodAutoscaler recommends for the container." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_status_recommendation_containerrecommendations_target_memory" + - "commonLabels": + "resource": "cpu" + "unit": "cores" + "each": + "gauge": + "labelsFromPath": + "container": + - "containerName" + "path": + - "status" + - "recommendation" + - "containerRecommendations" + "valueFrom": + - "uncappedTarget" + - "cpu" + "type": "Gauge" + "help": "Target cpu resources the VerticalPodAutoscaler recommends for the container ignoring bounds." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget_cpu" + - "commonLabels": + "resource": "memory" + "unit": "bytes" + "each": + "gauge": + "labelsFromPath": + "container": + - "containerName" + "path": + - "status" + - "recommendation" + - "containerRecommendations" + "valueFrom": + - "uncappedTarget" + - "memory" + "type": "Gauge" + "help": "Target memory resources the VerticalPodAutoscaler recommends for the container ignoring bounds." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget_memory" + - "commonLabels": + "resource": "cpu" + "unit": "cores" + "each": + "gauge": + "labelsFromPath": + "container": + - "containerName" + "path": + - "spec" + - "resourcePolicy" + - "containerPolicies" + "valueFrom": + - "minAllowed" + - "cpu" + "type": "Gauge" + "help": "Minimum cpu resources the VerticalPodAutoscaler can set for containers matching the name." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed_cpu" + - "commonLabels": + "resource": "memory" + "unit": "bytes" + "each": + "gauge": + "labelsFromPath": + "container": + - "containerName" + "path": + - "spec" + - "resourcePolicy" + - "containerPolicies" + "valueFrom": + - "minAllowed" + - "memory" + "type": "Gauge" + "help": "Minimum memory resources the VerticalPodAutoscaler can set for containers matching the name." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed_memory" + - "commonLabels": + "resource": "cpu" + "unit": "cores" + "each": + "gauge": + "labelsFromPath": + "container": + - "containerName" + "path": + - "spec" + - "resourcePolicy" + - "containerPolicies" + "valueFrom": + - "maxAllowed" + - "cpu" + "type": "Gauge" + "help": "Maximum cpu resources the VerticalPodAutoscaler can set for containers matching the name." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed_cpu" + - "commonLabels": + "resource": "memory" + "unit": "bytes" + "each": + "gauge": + "labelsFromPath": + "container": + - "containerName" + "path": + - "spec" + - "resourcePolicy" + - "containerPolicies" + "valueFrom": + - "maxAllowed" + - "memory" + "type": "Gauge" + "help": "Maximum memory resources the VerticalPodAutoscaler can set for containers matching the name." + "labelsFromPath": + "namespace": + - "metadata" + - "namespace" + "target_api_version": + - "spec" + - "targetRef" + - "apiVersion" + "target_kind": + - "spec" + - "targetRef" + - "kind" + "target_name": + - "spec" + - "targetRef" + - "name" + "verticalpodautoscaler": + - "metadata" + - "name" + "name": "verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed_memory" + - "groupVersionKind": + "group": "gateway.networking.k8s.io" + "kind": "GatewayClass" + "version": "v1" + "metrics": + - "each": + "info": + "labelsFromPath": + "accepted": + - "status" + - "conditions" + - "[type=Accepted]" + - "status" + "controller": + - "spec" + - "controllerName" + "gateway_class": + - "metadata" + - "name" + "type": "Info" + "help": "Information about GatewayClasses" + "name": "gateway_class_info" + - "groupVersionKind": + "group": "gateway.networking.k8s.io" + "kind": "Gateway" + "version": "v1" + "metrics": + - "each": + "info": + "labelsFromPath": + "gateway": + - "metadata" + - "name" + "gateway_class": + - "spec" + - "gatewayClassName" + "namespace": + - "metadata" + - "namespace" + "programmed": + - "status" + - "conditions" + - "[type=Programmed]" + - "status" + "type": "Info" + "help": "Information about Gateways" + "name": "gateway_info" +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/managed-by: cluster-monitoring-operator + app.kubernetes.io/part-of: openshift-monitoring + name: kube-state-metrics-custom-resource-state-configmap + namespace: openshift-monitoring diff --git a/assets/optional/kube-state-metrics/02-kube-rbac-proxy-secret.yaml b/assets/optional/kube-state-metrics/02-kube-rbac-proxy-secret.yaml new file mode 100644 index 0000000000..1cae041683 --- /dev/null +++ b/assets/optional/kube-state-metrics/02-kube-rbac-proxy-secret.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +data: {} +kind: Secret +metadata: + labels: + app.kubernetes.io/managed-by: cluster-monitoring-operator + app.kubernetes.io/part-of: openshift-monitoring + name: kube-state-metrics-kube-rbac-proxy-config + namespace: openshift-monitoring +stringData: + config.yaml: |- + "authorization": + "static": + - "path": "/metrics" + "resourceRequest": false + "user": + "name": "system:serviceaccount:openshift-monitoring:prometheus-k8s" + "verb": "get" +type: Opaque diff --git a/assets/optional/kube-state-metrics/03-deployment.yaml b/assets/optional/kube-state-metrics/03-deployment.yaml new file mode 100644 index 0000000000..ad8fc8f8cf --- /dev/null +++ b/assets/optional/kube-state-metrics/03-deployment.yaml @@ -0,0 +1,158 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/managed-by: cluster-monitoring-operator + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/part-of: openshift-monitoring + app.kubernetes.io/version: 2.19.0 + name: kube-state-metrics + namespace: openshift-monitoring +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: exporter + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/part-of: openshift-monitoring + strategy: + type: Recreate + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: kube-state-metrics + openshift.io/required-scc: restricted-v2 + target.workload.openshift.io/management: '{"effect": "PreferredDuringScheduling"}' + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/managed-by: cluster-monitoring-operator + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/part-of: openshift-monitoring + app.kubernetes.io/version: 2.19.0 + spec: + automountServiceAccountToken: true + containers: + - args: + - --host=127.0.0.1 + - --port=8081 + - --telemetry-host=127.0.0.1 + - --telemetry-port=8082 + - --custom-resource-state-config-file=/etc/kube-state-metrics/custom-resource-state-configmap.yaml + - --metric-denylist=^kube_secret_labels$,^kube_.+_annotations$,^kube_customresource_.+_annotations_info$,^kube_customresource_.+_labels_info$,^kube_.+_created$,^kube_.+_metadata_resource_version$,^kube_replicaset_metadata_generation$,^kube_replicaset_status_observed_generation$,^kube_pod_restart_policy$,^kube_pod_init_container_status_terminated$,^kube_pod_init_container_status_running$,^kube_pod_container_status_terminated$,^kube_pod_container_status_running$,^kube_pod_completion_time$,^kube_pod_status_scheduled$ + - --metric-labels-allowlist=pods=[*],nodes=[*],namespaces=[*],persistentvolumes=[*],persistentvolumeclaims=[*],poddisruptionbudgets=[*] + image: quay.io/openshift/kube-state-metrics + name: kube-state-metrics + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + readinessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 + resources: + limits: + cpu: 100m + memory: 200Mi + requests: + cpu: 2m + memory: 80Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /tmp + name: volume-directive-shadow + readOnly: false + - mountPath: /etc/kube-state-metrics + name: kube-state-metrics-custom-resource-state-configmap + readOnly: true + - args: + - --secure-listen-address=:8443 + - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - --upstream=http://127.0.0.1:8081/ + - --tls-cert-file=/etc/tls/private/tls.crt + - --tls-private-key-file=/etc/tls/private/tls.key + - --config-file=/etc/kube-rbac-policy/config.yaml + image: quay.io/openshift/kube-rbac-proxy + name: kube-rbac-proxy-main + ports: + - containerPort: 8443 + name: https-main + resources: + limits: + cpu: 20m + memory: 40Mi + requests: + cpu: 1m + memory: 15Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /etc/tls/private + name: kube-state-metrics-tls + readOnly: true + - mountPath: /etc/kube-rbac-policy + name: kube-state-metrics-kube-rbac-proxy-config + readOnly: true + - args: + - --secure-listen-address=:9443 + - --tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305 + - --upstream=http://127.0.0.1:8082/ + - --tls-cert-file=/etc/tls/private/tls.crt + - --tls-private-key-file=/etc/tls/private/tls.key + - --config-file=/etc/kube-rbac-policy/config.yaml + image: quay.io/openshift/kube-rbac-proxy + name: kube-rbac-proxy-self + ports: + - containerPort: 9443 + name: https-self + resources: + limits: + cpu: 20m + memory: 40Mi + requests: + cpu: 1m + memory: 15Mi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + runAsNonRoot: true + terminationMessagePolicy: FallbackToLogsOnError + volumeMounts: + - mountPath: /etc/tls/private + name: kube-state-metrics-tls + readOnly: true + - mountPath: /etc/kube-rbac-policy + name: kube-state-metrics-kube-rbac-proxy-config + readOnly: true + nodeSelector: + kubernetes.io/os: linux + priorityClassName: system-cluster-critical + securityContext: + runAsNonRoot: true + serviceAccountName: kube-state-metrics + volumes: + - emptyDir: {} + name: volume-directive-shadow + - name: kube-state-metrics-tls + secret: + secretName: kube-state-metrics-tls + - name: kube-state-metrics-kube-rbac-proxy-config + secret: + secretName: kube-state-metrics-kube-rbac-proxy-config + - configMap: + name: kube-state-metrics-custom-resource-state-configmap + name: kube-state-metrics-custom-resource-state-configmap diff --git a/assets/optional/kube-state-metrics/04-service.yaml b/assets/optional/kube-state-metrics/04-service.yaml new file mode 100644 index 0000000000..94b982309d --- /dev/null +++ b/assets/optional/kube-state-metrics/04-service.yaml @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: Service +metadata: + annotations: + openshift.io/description: |- + Expose kube-state-metrics `/metrics` endpoints within the cluster on the following ports: + * Port 8443 provides access to the Kubernetes resource metrics. This port is for internal use, and no other usage is guaranteed. + * Port 9443 provides access to the internal kube-state-metrics metrics. This port is for internal use, and no other usage is guaranteed. + service.beta.openshift.io/serving-cert-secret-name: kube-state-metrics-tls + labels: + app.kubernetes.io/component: exporter + app.kubernetes.io/managed-by: cluster-monitoring-operator + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/part-of: openshift-monitoring + app.kubernetes.io/version: 2.19.0 + name: kube-state-metrics + namespace: openshift-monitoring +spec: + clusterIP: None + ports: + - name: https-main + port: 8443 + targetPort: https-main + - name: https-self + port: 9443 + targetPort: https-self + selector: + app.kubernetes.io/component: exporter + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/part-of: openshift-monitoring diff --git a/assets/optional/kube-state-metrics/kustomization.aarch64.yaml b/assets/optional/kube-state-metrics/kustomization.aarch64.yaml new file mode 100644 index 0000000000..f5b48a4fbe --- /dev/null +++ b/assets/optional/kube-state-metrics/kustomization.aarch64.yaml @@ -0,0 +1,7 @@ +images: + - name: quay.io/openshift/kube-state-metrics + newName: quay.io/openshift-release-dev/ocp-v5.0-art-dev + digest: sha256:ad7ae7a3c499ed390a36ae17acd5251aa2a5a3833cd4144d1976f4d2b968b654 + - name: quay.io/openshift/kube-rbac-proxy + newName: quay.io/openshift-release-dev/ocp-v5.0-art-dev + digest: sha256:0d6a1c6ebba722e09ff2850010cb8114a8d097ccee1198c1f59680c8c7581d48 diff --git a/assets/optional/kube-state-metrics/kustomization.x86_64.yaml b/assets/optional/kube-state-metrics/kustomization.x86_64.yaml new file mode 100644 index 0000000000..77878bb0e8 --- /dev/null +++ b/assets/optional/kube-state-metrics/kustomization.x86_64.yaml @@ -0,0 +1,7 @@ +images: + - name: quay.io/openshift/kube-state-metrics + newName: quay.io/openshift-release-dev/ocp-v5.0-art-dev + digest: sha256:f2c7f8cb3995b165ed4acf4c2f546b9993986862b427c4b2ef224521e05d1594 + - name: quay.io/openshift/kube-rbac-proxy + newName: quay.io/openshift-release-dev/ocp-v5.0-art-dev + digest: sha256:3b2676dd92a952c620e067cc158c2a0942c602471645d8e367104293cb964147 diff --git a/assets/optional/kube-state-metrics/kustomization.yaml b/assets/optional/kube-state-metrics/kustomization.yaml new file mode 100644 index 0000000000..17942badc5 --- /dev/null +++ b/assets/optional/kube-state-metrics/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - 00-namespace.yaml + - 01-service-account.yaml + - 01-cluster-role.yaml + - 01-cluster-role-binding.yaml + - 02-kube-rbac-proxy-secret.yaml + - 02-custom-resource-state-configmap.yaml + - 03-deployment.yaml + - 04-service.yaml diff --git a/assets/optional/kube-state-metrics/release-kube-state-metrics-aarch64.json b/assets/optional/kube-state-metrics/release-kube-state-metrics-aarch64.json new file mode 100644 index 0000000000..ea791b9165 --- /dev/null +++ b/assets/optional/kube-state-metrics/release-kube-state-metrics-aarch64.json @@ -0,0 +1,8 @@ +{ + "release": { + "base": "placeholder" + }, + "images": { + "kube_state_metrics": "quay.io/openshift-release-dev/ocp-v5.0-art-dev@sha256:ad7ae7a3c499ed390a36ae17acd5251aa2a5a3833cd4144d1976f4d2b968b654" + } +} diff --git a/assets/optional/kube-state-metrics/release-kube-state-metrics-x86_64.json b/assets/optional/kube-state-metrics/release-kube-state-metrics-x86_64.json new file mode 100644 index 0000000000..842b4c01a3 --- /dev/null +++ b/assets/optional/kube-state-metrics/release-kube-state-metrics-x86_64.json @@ -0,0 +1,8 @@ +{ + "release": { + "base": "placeholder" + }, + "images": { + "kube_state_metrics": "quay.io/openshift-release-dev/ocp-v5.0-art-dev@sha256:f2c7f8cb3995b165ed4acf4c2f546b9993986862b427c4b2ef224521e05d1594" + } +} diff --git a/packaging/observability/otelcol.d/microshift-metrics-kube-state-metrics.yaml b/packaging/observability/otelcol.d/microshift-metrics-kube-state-metrics.yaml new file mode 100644 index 0000000000..b8bb4c76dc --- /dev/null +++ b/packaging/observability/otelcol.d/microshift-metrics-kube-state-metrics.yaml @@ -0,0 +1,26 @@ +receivers: + prometheus/kube_state_metrics: + config: + scrape_configs: + - job_name: kube-state-metrics + scrape_interval: 30s + scheme: https + tls_config: + ca_file: /var/lib/microshift/certs/service-ca/ca.crt + server_name: kube-state-metrics.openshift-monitoring.svc + kubernetes_sd_configs: + - kubeconfig_file: /var/lib/microshift/resources/observability-client/kubeconfig + role: endpoints + namespaces: + names: [openshift-monitoring] + relabel_configs: + - source_labels: [__meta_kubernetes_service_name, __meta_kubernetes_endpoint_port_name] + action: keep + regex: kube-state-metrics;https-main + +service: + pipelines: + metrics/kube_state_metrics: + receivers: [prometheus/kube_state_metrics] + processors: [batch] + exporters: [otlp] diff --git a/packaging/rpm/microshift.spec b/packaging/rpm/microshift.spec index 6362e4f552..605613f350 100644 --- a/packaging/rpm/microshift.spec +++ b/packaging/rpm/microshift.spec @@ -236,6 +236,7 @@ and can be used to embed those images into osbuilder blueprints or bootc contain Summary: OpenTelemetry-Collector configured for MicroShift BuildArch: noarch Requires: microshift = %{version} +Requires: microshift-metrics-kube-state-metrics = %{version} Requires: opentelemetry-collector %description observability @@ -261,6 +262,25 @@ The microshift-cert-manager-release-info package provides release information fi release. These files contain the list of container image references used by Cert Manager and can be used to embed those images into osbuilder blueprints or bootc containerfiles. +%package metrics-kube-state-metrics +Summary: Kubernetes kube-state-metrics for MicroShift +ExclusiveArch: x86_64 aarch64 +Requires: microshift = %{version} + +%description metrics-kube-state-metrics +The microshift-metrics-kube-state-metrics package provides kube-state-metrics for MicroShift. +Install this package to expose Kubernetes object state metrics via a secure endpoint. + +%package metrics-kube-state-metrics-release-info +Summary: Release information for kube-state-metrics for MicroShift +BuildArch: noarch +Requires: microshift-release-info = %{version} + +%description metrics-kube-state-metrics-release-info +The microshift-metrics-kube-state-metrics-release-info package provides release information files for this +release. These files contain the list of container image references used by kube-state-metrics +and can be used to embed those images into osbuilder blueprints or bootc containerfiles. + %package sriov Summary: SR-IOV Network Operator for MicroShift ExclusiveArch: x86_64 aarch64 @@ -562,7 +582,9 @@ install -p -m644 assets/optional/ai-model-serving/release-ai-model-serving-x86_6 # observability install -d -m755 %{buildroot}/%{_sysconfdir}/microshift/observability +install -d -m755 %{buildroot}/%{_sysconfdir}/microshift/observability/otelcol.d install -p -m644 packaging/observability/*.yaml -D %{buildroot}%{_sysconfdir}/microshift/observability/ +install -p -m644 packaging/observability/otelcol.d/microshift-metrics-kube-state-metrics.yaml %{buildroot}%{_sysconfdir}/microshift/observability/otelcol.d/ # Explicit copy of large config as default. Not using symlink to avoid accidental package upgrade overwriting user config if the user edits the config without copying (i.e. edits the target of symlink). install -p -m644 packaging/observability/opentelemetry-collector-large.yaml -D %{buildroot}%{_sysconfdir}/microshift/observability/opentelemetry-collector.yaml install -p -m644 packaging/observability/microshift-observability.service %{buildroot}%{_unitdir}/ @@ -599,6 +621,28 @@ cat assets/optional/cert-manager/manager/images-x86_64.yaml >> %{buildroot}/%{_p mkdir -p -m755 %{buildroot}%{_datadir}/microshift/release install -p -m644 assets/optional/cert-manager/release-cert-manager-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/ +# kube-state-metrics +install -d -m755 %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics +install -p -m644 assets/optional/kube-state-metrics/00-namespace.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics +install -p -m644 assets/optional/kube-state-metrics/01-service-account.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics +install -p -m644 assets/optional/kube-state-metrics/01-cluster-role.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics +install -p -m644 assets/optional/kube-state-metrics/01-cluster-role-binding.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics +install -p -m644 assets/optional/kube-state-metrics/02-kube-rbac-proxy-secret.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics +install -p -m644 assets/optional/kube-state-metrics/02-custom-resource-state-configmap.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics +install -p -m644 assets/optional/kube-state-metrics/03-deployment.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics +install -p -m644 assets/optional/kube-state-metrics/04-service.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics +install -p -m644 assets/optional/kube-state-metrics/kustomization.yaml %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics + +%ifarch %{arm} aarch64 +cat assets/optional/kube-state-metrics/kustomization.aarch64.yaml >> %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics/kustomization.yaml +%endif +%ifarch x86_64 +cat assets/optional/kube-state-metrics/kustomization.x86_64.yaml >> %{buildroot}/%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics/kustomization.yaml +%endif + +# kube-state-metrics-release-info +install -p -m644 assets/optional/kube-state-metrics/release-kube-state-metrics-{x86_64,aarch64}.json %{buildroot}%{_datadir}/microshift/release/ + # sriov install -d -m755 %{buildroot}/%{_prefix}/lib/microshift/manifests.d/070-microshift-sriov install -d -m755 %{buildroot}/%{_prefix}/lib/microshift/manifests.d/070-microshift-sriov/crd @@ -790,10 +834,12 @@ fi %files observability %dir %{_prefix}/lib/microshift/manifests.d/003-microshift-observability %dir %{_sysconfdir}/microshift/observability/ +%dir %{_sysconfdir}/microshift/observability/otelcol.d %{_unitdir}/microshift-observability.service %config(noreplace) %{_sysconfdir}/microshift/observability/opentelemetry-collector.yaml %{_sysconfdir}/microshift/observability/opentelemetry-collector-*.yaml %{_prefix}/lib/microshift/manifests.d/003-microshift-observability/* +%config(noreplace) %{_sysconfdir}/microshift/observability/otelcol.d/microshift-metrics-kube-state-metrics.yaml %files cert-manager %dir %{_prefix}/lib/microshift/manifests.d/060-microshift-cert-manager @@ -802,6 +848,13 @@ fi %files cert-manager-release-info %{_datadir}/microshift/release/release-cert-manager-{x86_64,aarch64}.json +%files metrics-kube-state-metrics +%dir %{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics +%{_prefix}/lib/microshift/manifests.d/081-microshift-kube-state-metrics/* + +%files metrics-kube-state-metrics-release-info +%{_datadir}/microshift/release/release-kube-state-metrics-{x86_64,aarch64}.json + %files sriov %dir %{_prefix}/lib/microshift/manifests.d/070-microshift-sriov %dir %{_prefix}/lib/microshift/manifests.d/070-microshift-sriov/crd diff --git a/pkg/healthcheck/microshift_optional_workloads.go b/pkg/healthcheck/microshift_optional_workloads.go index 80e2d9a3b0..cdf928666c 100644 --- a/pkg/healthcheck/microshift_optional_workloads.go +++ b/pkg/healthcheck/microshift_optional_workloads.go @@ -38,6 +38,21 @@ var optionalWorkloadPaths = map[string]optionalWorkloads{ Namespace: "sriov-network-operator", Workloads: NamespaceWorkloads{Deployments: []string{"sriov-network-operator"}}, }, + + "/usr/lib/microshift/manifests.d/081-microshift-kube-state-metrics": { + Namespace: "openshift-monitoring", + Workloads: NamespaceWorkloads{Deployments: []string{"kube-state-metrics"}}, + }, +} + +// mergeWorkloads merges two NamespaceWorkloads, returning a new NamespaceWorkloads. This is helpful for cases +// where components from multiple sources are deployed to the same namespace. +func mergeWorkloads(existing, incoming NamespaceWorkloads) NamespaceWorkloads { + return NamespaceWorkloads{ + Deployments: append(existing.Deployments, incoming.Deployments...), + DaemonSets: append(existing.DaemonSets, incoming.DaemonSets...), + StatefulSets: append(existing.StatefulSets, incoming.StatefulSets...), + } } // fillOptionalMicroShiftWorkloads assembles list of optional MicroShift workloads @@ -73,7 +88,7 @@ func fillOptionalMicroShiftWorkloads(workloadsToCheck map[string]NamespaceWorklo } klog.Infof("Optional component path exists and is configured: %s - expecting %v in namespace %q", path, ow.Workloads.String(), ow.Namespace) - workloadsToCheck[ow.Namespace] = ow.Workloads + workloadsToCheck[ow.Namespace] = mergeWorkloads(workloadsToCheck[ow.Namespace], ow.Workloads) } return nil } diff --git a/scripts/auto-rebase/assets.yaml b/scripts/auto-rebase/assets.yaml index b4f34d3f6c..e2d70cb981 100644 --- a/scripts/auto-rebase/assets.yaml +++ b/scripts/auto-rebase/assets.yaml @@ -301,6 +301,27 @@ assets: - file: service.yaml - file: serviceaccount.yaml + - dir: optional/kube-state-metrics/ + ignore: "MicroShift-specific kube-state-metrics manifests sourced from CMO" + files: + - file: 00-namespace.yaml + - file: 01-cluster-role-binding.yaml + - file: 01-cluster-role.yaml + - file: 01-service-account.yaml + - file: 02-custom-resource-state-configmap.yaml + - file: 02-kube-rbac-proxy-secret.yaml + - file: 03-deployment.yaml + - file: 04-service.yaml + - file: kustomization.yaml + - file: kustomization.x86_64.yaml + ignore: "gets generated during image rebase" + - file: kustomization.aarch64.yaml + ignore: "gets generated during image rebase" + - file: release-kube-state-metrics-x86_64.json + ignore: "gets generated during image rebase" + - file: release-kube-state-metrics-aarch64.json + ignore: "gets generated during image rebase" + - dir: optional/observability/ ignore: "they don't exist in upstream repository - only in microshift" files: diff --git a/test/bin/common.sh b/test/bin/common.sh index ef682a676f..9e65e45877 100644 --- a/test/bin/common.sh +++ b/test/bin/common.sh @@ -388,6 +388,7 @@ MICROSHIFT_Y2_OPTIONAL_RPMS_LIST=( microshift-cert-manager-release-info microshift-sriov microshift-sriov-release-info + microshift-metrics-kube-state-metrics ) MICROSHIFT_Y1_OPTIONAL_RPMS_LIST=( "${MICROSHIFT_Y2_OPTIONAL_RPMS_LIST[@]}"