From 1622b78d9bef3e2172486d8c33f74ff06f03eab0 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Mon, 5 May 2025 15:20:10 +0200 Subject: [PATCH 1/9] Adding docs how to authenticate with prometheus to nifi 2.x.x metrics --- .../nifi/pages/usage_guide/monitoring.adoc | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/docs/modules/nifi/pages/usage_guide/monitoring.adoc b/docs/modules/nifi/pages/usage_guide/monitoring.adoc index 98d92b10..5c9b99e2 100644 --- a/docs/modules/nifi/pages/usage_guide/monitoring.adoc +++ b/docs/modules/nifi/pages/usage_guide/monitoring.adoc @@ -44,3 +44,29 @@ https://simple-nifi-node-default-0.simple-nifi-node-default..svc.clus ``` IMPORTANT: If NiFi is configured to do any user authentication, requests to the metric endpoint must be authenticated and authorized. + +=== Authentication with NiFi `2.x.x` +To authenticate, you can use a bearer token created by your NiFi instance e.g. + +[source,bash] +---- +curl -X POST https://:/nifi-api/access/token -d 'username=&password=' -k +---- + +where `-k` equals `verify=false`. The reply is your bearer token. + +You then can use the bearer token to authenticate with Prometheus replacing `basic_auth` with + +[source,yaml] +---- +# basic_auth: + # username: + # password: +authorization: + type: Bearer + credentials: "" +tls_config: + insecure_skip_verify: true +---- + +according to https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_config[Prometheus documentation]. From bec0c7b9fe518f8f4efcf787ec7ce26791dcda9e Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Mon, 5 May 2025 15:23:28 +0200 Subject: [PATCH 2/9] using address rather then placeholders --- docs/modules/nifi/pages/usage_guide/monitoring.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/nifi/pages/usage_guide/monitoring.adoc b/docs/modules/nifi/pages/usage_guide/monitoring.adoc index 5c9b99e2..91c1d382 100644 --- a/docs/modules/nifi/pages/usage_guide/monitoring.adoc +++ b/docs/modules/nifi/pages/usage_guide/monitoring.adoc @@ -50,7 +50,7 @@ To authenticate, you can use a bearer token created by your NiFi instance e.g. [source,bash] ---- -curl -X POST https://:/nifi-api/access/token -d 'username=&password=' -k +curl -X POST https://simple-nifi-node-default-0.simple-nifi-node-default..svc.cluster.local:8443/nifi-api/access/token -d 'username=&password=' -k ---- where `-k` equals `verify=false`. The reply is your bearer token. From c7fb1101f786971dc25fcf7c8912ac57615f9dda Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 11 Jun 2025 14:48:03 +0200 Subject: [PATCH 3/9] Updating docs to newest findings --- .../nifi/pages/usage_guide/monitoring.adoc | 69 ++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/docs/modules/nifi/pages/usage_guide/monitoring.adoc b/docs/modules/nifi/pages/usage_guide/monitoring.adoc index 91c1d382..e97033e5 100644 --- a/docs/modules/nifi/pages/usage_guide/monitoring.adoc +++ b/docs/modules/nifi/pages/usage_guide/monitoring.adoc @@ -59,6 +59,7 @@ You then can use the bearer token to authenticate with Prometheus replacing `bas [source,yaml] ---- +--- # basic_auth: # username: # password: @@ -67,6 +68,72 @@ authorization: credentials: "" tls_config: insecure_skip_verify: true +static_configs: + - targets: + - '..svc.cluster.local:8443' <1> ---- +<1> Static targets only scrapes one pod. More about this in Known Limitations. -according to https://prometheus.io/docs/prometheus/latest/configuration/configuration/#http_config[Prometheus documentation]. +or use it in a nifi secret which should look like +[source,yaml] +---- +--- +apiVersion: v1 +kind: Secret +metadata: + name: nifi-authorization-secret +type: Opaque +stringData: + nifi_token: "" +---- + +If you want to use a `ServiceMonitor` you'd need to configure it as follows: +[source,yaml] +---- +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: scrape-nifi2 + labels: + stackable.tech/vendor: Stackable + release: prometheus +spec: + endpoints: + - port: https + path: 'nifi-api/flow/metrics/prometheus' + scheme: https + interval: 5s + tlsConfig: + insecureSkipVerify: true + authorization: + credentials: <1> + key: "nifi_token" + name: "nifi-authorization-secret" + optional: false + type: "Bearer" + relabelings: <2> + - sourceLabels: + - __meta_kubernetes_pod_name + - __meta_kubernetes_service_name + - __meta_kubernetes_namespace + - __meta_kubernetes_pod_container_port_number + targetLabel: __address__ + replacement: ${1}.${2}.${3}.svc.cluster.local:${4} + regex: (.+);(.+?)(?:-metrics)?;(.+);(.+) + selector: + matchLabels: + prometheus.io/scrape: "true" + namespaceSelector: + any: true + jobLabel: app.kubernetes.io/instance +---- +<1> Authorization via Bearer Token stored in a secret +<2> Relabel __address__ to be a FQDN rather then the IP-Address of target pod + +NOTE: As of xref:listener-operator:listener.adoc[Listener] integration, SDP exposes a Service with `-metrics` thus we need to regex this suffix. + +==== Known Limitations + +NiFi only allows authentication with JWT on pod level. Therefore you will need one endpoint per NiFi pod and a valid bearer token for each. This is a consequence of NiFi +moving their metrics endpoint behind a strong authentication mechanism. From 97d6c1870907d22766eacfde9c2b99dbd202342f Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 11 Jun 2025 14:49:12 +0200 Subject: [PATCH 4/9] Fixing typo and wording --- docs/modules/nifi/pages/usage_guide/monitoring.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/modules/nifi/pages/usage_guide/monitoring.adoc b/docs/modules/nifi/pages/usage_guide/monitoring.adoc index e97033e5..ada00213 100644 --- a/docs/modules/nifi/pages/usage_guide/monitoring.adoc +++ b/docs/modules/nifi/pages/usage_guide/monitoring.adoc @@ -72,9 +72,9 @@ static_configs: - targets: - '..svc.cluster.local:8443' <1> ---- -<1> Static targets only scrapes one pod. More about this in Known Limitations. +<1> Static targets only scrapes one pod. -or use it in a nifi secret which should look like +or use it in a NiFi secret which should look like [source,yaml] ---- --- From 675f3dcc50a6c15dfbace355ab88d300d3085583 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 11 Jun 2025 14:50:19 +0200 Subject: [PATCH 5/9] Fixing escaping of __ --- docs/modules/nifi/pages/usage_guide/monitoring.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/nifi/pages/usage_guide/monitoring.adoc b/docs/modules/nifi/pages/usage_guide/monitoring.adoc index ada00213..8972b30f 100644 --- a/docs/modules/nifi/pages/usage_guide/monitoring.adoc +++ b/docs/modules/nifi/pages/usage_guide/monitoring.adoc @@ -129,7 +129,7 @@ spec: jobLabel: app.kubernetes.io/instance ---- <1> Authorization via Bearer Token stored in a secret -<2> Relabel __address__ to be a FQDN rather then the IP-Address of target pod +<2> Relabel \\__address__ to be a FQDN rather then the IP-Address of target pod NOTE: As of xref:listener-operator:listener.adoc[Listener] integration, SDP exposes a Service with `-metrics` thus we need to regex this suffix. From 2d68691961885b1dd4e5225ec59e3cdbc6af1700 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 11 Jun 2025 14:57:36 +0200 Subject: [PATCH 6/9] Making precommit happy --- docs/modules/nifi/pages/usage_guide/monitoring.adoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/modules/nifi/pages/usage_guide/monitoring.adoc b/docs/modules/nifi/pages/usage_guide/monitoring.adoc index 8972b30f..ce284da1 100644 --- a/docs/modules/nifi/pages/usage_guide/monitoring.adoc +++ b/docs/modules/nifi/pages/usage_guide/monitoring.adoc @@ -100,7 +100,7 @@ metadata: release: prometheus spec: endpoints: - - port: https + - port: https path: 'nifi-api/flow/metrics/prometheus' scheme: https interval: 5s @@ -109,7 +109,7 @@ spec: authorization: credentials: <1> key: "nifi_token" - name: "nifi-authorization-secret" + name: "nifi-authorization-secret" optional: false type: "Bearer" relabelings: <2> @@ -118,7 +118,7 @@ spec: - __meta_kubernetes_service_name - __meta_kubernetes_namespace - __meta_kubernetes_pod_container_port_number - targetLabel: __address__ + targetLabel: __address__ replacement: ${1}.${2}.${3}.svc.cluster.local:${4} regex: (.+);(.+?)(?:-metrics)?;(.+);(.+) selector: From c3053ae2573457ac7664865cad6f9ca84e6fe4d6 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 11 Jun 2025 14:59:28 +0200 Subject: [PATCH 7/9] precommit no2 --- docs/modules/nifi/pages/usage_guide/monitoring.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/nifi/pages/usage_guide/monitoring.adoc b/docs/modules/nifi/pages/usage_guide/monitoring.adoc index ce284da1..74fe1882 100644 --- a/docs/modules/nifi/pages/usage_guide/monitoring.adoc +++ b/docs/modules/nifi/pages/usage_guide/monitoring.adoc @@ -83,7 +83,7 @@ kind: Secret metadata: name: nifi-authorization-secret type: Opaque -stringData: +stringData: nifi_token: "" ---- From 82e7fe2dc9d134ebecf8501ba01e370bae540a70 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Wed, 11 Jun 2025 15:05:14 +0200 Subject: [PATCH 8/9] Adding a little more information --- docs/modules/nifi/pages/usage_guide/monitoring.adoc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/modules/nifi/pages/usage_guide/monitoring.adoc b/docs/modules/nifi/pages/usage_guide/monitoring.adoc index 74fe1882..24f33982 100644 --- a/docs/modules/nifi/pages/usage_guide/monitoring.adoc +++ b/docs/modules/nifi/pages/usage_guide/monitoring.adoc @@ -71,6 +71,8 @@ tls_config: static_configs: - targets: - '..svc.cluster.local:8443' <1> +metrics_path: '/nifi-api/flow/metrics/prometheus' +scheme: https ---- <1> Static targets only scrapes one pod. From 1de48db6ac40cce53e1e3532006e8198c515c262 Mon Sep 17 00:00:00 2001 From: Maxi Wittich Date: Tue, 24 Jun 2025 13:30:31 +0200 Subject: [PATCH 9/9] Updating docs to adress review --- .../nifi/pages/usage_guide/monitoring.adoc | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/docs/modules/nifi/pages/usage_guide/monitoring.adoc b/docs/modules/nifi/pages/usage_guide/monitoring.adoc index 24f33982..355f96b6 100644 --- a/docs/modules/nifi/pages/usage_guide/monitoring.adoc +++ b/docs/modules/nifi/pages/usage_guide/monitoring.adoc @@ -46,6 +46,12 @@ https://simple-nifi-node-default-0.simple-nifi-node-default..svc.clus IMPORTANT: If NiFi is configured to do any user authentication, requests to the metric endpoint must be authenticated and authorized. === Authentication with NiFi `2.x.x` + +[IMPORTANT] +=== +The NiFi metrics endpoints are behind a strong authentication mechanism which require credentials for each individual pod. +=== + To authenticate, you can use a bearer token created by your NiFi instance e.g. [source,bash] @@ -53,28 +59,27 @@ To authenticate, you can use a bearer token created by your NiFi instance e.g. curl -X POST https://simple-nifi-node-default-0.simple-nifi-node-default..svc.cluster.local:8443/nifi-api/access/token -d 'username=&password=' -k ---- -where `-k` equals `verify=false`. The reply is your bearer token. +where `-k` equals `verify=false` to allow self-signed certificates. The reply is your bearer token. -You then can use the bearer token to authenticate with Prometheus replacing `basic_auth` with +The following example shows how to configure the Prometheus scraper to use the bearer token to authenticate against a NiFi pod. [source,yaml] ---- --- -# basic_auth: - # username: - # password: -authorization: +authorization: <1> type: Bearer - credentials: "" + credentials: "" <2> tls_config: insecure_skip_verify: true static_configs: - targets: - - '..svc.cluster.local:8443' <1> + - '..svc.cluster.local:8443' <3> metrics_path: '/nifi-api/flow/metrics/prometheus' scheme: https ---- -<1> Static targets only scrapes one pod. +<1> Use the `authorization` property instead if the `basic_auth`. +<2> Add the previously obtained token here. +<3> Static targets only scrapes one pod. or use it in a NiFi secret which should look like [source,yaml] @@ -134,8 +139,3 @@ spec: <2> Relabel \\__address__ to be a FQDN rather then the IP-Address of target pod NOTE: As of xref:listener-operator:listener.adoc[Listener] integration, SDP exposes a Service with `-metrics` thus we need to regex this suffix. - -==== Known Limitations - -NiFi only allows authentication with JWT on pod level. Therefore you will need one endpoint per NiFi pod and a valid bearer token for each. This is a consequence of NiFi -moving their metrics endpoint behind a strong authentication mechanism.