From 0900f8d875e475b52832cfaeb6f6ecc5de9384db Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 20 Feb 2025 10:24:38 +0100 Subject: [PATCH 01/13] chore(nifi): Bump to 2.2.0, upgrade postgres jdbc driver --- .github/workflows/dev_nifi.yaml | 2 +- demos/signal-processing/Dockerfile-nifi | 13 +++++++++++-- stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml | 2 +- stacks/nifi-kafka-druid-superset-s3/nifi.yaml | 2 +- stacks/signal-processing/nifi.yaml | 4 ++-- 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dev_nifi.yaml b/.github/workflows/dev_nifi.yaml index 5e456f55..017210c4 100644 --- a/.github/workflows/dev_nifi.yaml +++ b/.github/workflows/dev_nifi.yaml @@ -5,7 +5,7 @@ env: IMAGE_NAME: nifi # TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that # the demo is reproducable for the release and it will be automatically replaced for the release branch. - IMAGE_VERSION: 1.27.0-postgresql + IMAGE_VERSION: 2.2.0-postgresql REGISTRY_PATH: stackable DOCKERFILE_PATH: "demos/signal-processing/Dockerfile-nifi" diff --git a/demos/signal-processing/Dockerfile-nifi b/demos/signal-processing/Dockerfile-nifi index b53549da..5cd45332 100644 --- a/demos/signal-processing/Dockerfile-nifi +++ b/demos/signal-processing/Dockerfile-nifi @@ -1,3 +1,12 @@ -FROM oci.stackable.tech/sdp/nifi:1.27.0-stackable0.0.0-dev +FROM oci.stackable.tech/sdp/nifi:2.2.0-stackable0.0.0-dev -RUN curl --fail -o /stackable/nifi/postgresql-42.6.0.jar "https://repo.stackable.tech/repository/misc/postgresql-timescaledb/postgresql-42.6.0.jar" +# This is the postgresql JDBC driver from https://jdbc.postgresql.org/download/ +# There appear to be no signatures to validate against 😬 +# +# VERSION="42.7.5" +# curl -O "https://jdbc.postgresql.org/download/postgresql-$VERSION.jar" +# curl --fail -u "your_username" --upload-file "postgresql-$VERSION.jar" 'https://repo.stackable.tech/repository/misc/jdbc/' +# rm "postgresql-$VERSION.jar" + +ARG DRIVER_VERSION="42.7.5" +RUN curl --fail -o "/stackable/nifi/postgresql-$DRIVER_VERSION.jar" "https://repo.stackable.tech/repository/misc/jdbc/postgresql-$DRIVER_VERSION.jar" diff --git a/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml b/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml index de3f4d99..bae55688 100644 --- a/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml +++ b/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml @@ -5,7 +5,7 @@ metadata: name: nifi spec: image: - productVersion: 1.27.0 + productVersion: 2.2.0 clusterConfig: authentication: - authenticationClass: nifi-admin-credentials diff --git a/stacks/nifi-kafka-druid-superset-s3/nifi.yaml b/stacks/nifi-kafka-druid-superset-s3/nifi.yaml index 8105be96..225fce94 100644 --- a/stacks/nifi-kafka-druid-superset-s3/nifi.yaml +++ b/stacks/nifi-kafka-druid-superset-s3/nifi.yaml @@ -5,7 +5,7 @@ metadata: name: nifi spec: image: - productVersion: 1.27.0 + productVersion: 2.2.0 clusterConfig: authentication: - authenticationClass: nifi-admin-credentials diff --git a/stacks/signal-processing/nifi.yaml b/stacks/signal-processing/nifi.yaml index 7f3d3964..b0a48f0e 100644 --- a/stacks/signal-processing/nifi.yaml +++ b/stacks/signal-processing/nifi.yaml @@ -5,10 +5,10 @@ metadata: name: nifi spec: image: - productVersion: 1.27.0 + productVersion: 2.2.0 # TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that # the demo is reproducable for the release and it will be automatically replaced for the release branch. - custom: oci.stackable.tech/stackable/nifi:1.27.0-postgresql + custom: oci.stackable.tech/stackable/nifi:2.2.0-postgresql clusterConfig: listenerClass: external-unstable zookeeperConfigMapName: nifi-znode From fc5daaca4782d920bdcc8cbdd87cf68e6b13075b Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 27 Feb 2025 09:23:44 +0100 Subject: [PATCH 02/13] chore(demo/signal-processing): Update postgresql version NiFi template --- demos/signal-processing/Dockerfile-nifi | 3 ++- demos/signal-processing/DownloadAndWriteToDB.xml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/demos/signal-processing/Dockerfile-nifi b/demos/signal-processing/Dockerfile-nifi index 5cd45332..faba9af2 100644 --- a/demos/signal-processing/Dockerfile-nifi +++ b/demos/signal-processing/Dockerfile-nifi @@ -1,4 +1,4 @@ -FROM oci.stackable.tech/sdp/nifi:2.2.0-stackable0.0.0-dev +FROM oci.stackable.tech/sdp/nifi:1.28.1-stackable0.0.0-dev # This is the postgresql JDBC driver from https://jdbc.postgresql.org/download/ # There appear to be no signatures to validate against 😬 @@ -8,5 +8,6 @@ FROM oci.stackable.tech/sdp/nifi:2.2.0-stackable0.0.0-dev # curl --fail -u "your_username" --upload-file "postgresql-$VERSION.jar" 'https://repo.stackable.tech/repository/misc/jdbc/' # rm "postgresql-$VERSION.jar" +# IMPORTANT (@NickLarsenNZ): Changing this version requires a change in the NiFi template (DownloadAndWriteToDB.xml) ARG DRIVER_VERSION="42.7.5" RUN curl --fail -o "/stackable/nifi/postgresql-$DRIVER_VERSION.jar" "https://repo.stackable.tech/repository/misc/jdbc/postgresql-$DRIVER_VERSION.jar" diff --git a/demos/signal-processing/DownloadAndWriteToDB.xml b/demos/signal-processing/DownloadAndWriteToDB.xml index 28fcb414..f4ddccd6 100644 --- a/demos/signal-processing/DownloadAndWriteToDB.xml +++ b/demos/signal-processing/DownloadAndWriteToDB.xml @@ -189,7 +189,7 @@ database-driver-locations - /stackable/nifi/postgresql-42.6.0.jar + /stackable/nifi/postgresql-42.7.5.jar kerberos-user-service From d9b61030c034236e8c7f22a62bc2704a3f0086b8 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 27 Feb 2025 09:57:37 +0100 Subject: [PATCH 03/13] ci(container/nifi): Drop back to NiFi 1.28.1 due to issues with NiFi 2.x --- .github/workflows/dev_nifi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/dev_nifi.yaml b/.github/workflows/dev_nifi.yaml index 017210c4..7384a735 100644 --- a/.github/workflows/dev_nifi.yaml +++ b/.github/workflows/dev_nifi.yaml @@ -5,7 +5,7 @@ env: IMAGE_NAME: nifi # TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that # the demo is reproducable for the release and it will be automatically replaced for the release branch. - IMAGE_VERSION: 2.2.0-postgresql + IMAGE_VERSION: 1.28.1-postgresql REGISTRY_PATH: stackable DOCKERFILE_PATH: "demos/signal-processing/Dockerfile-nifi" From c9e54d77cc77a7366063a42b51ed5546c03479c1 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 27 Feb 2025 09:58:31 +0100 Subject: [PATCH 04/13] chore(demo/signal-processing): Improve readability of job scripts --- .../create-nifi-ingestion-job.yaml | 29 ++++++++++++++----- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/demos/signal-processing/create-nifi-ingestion-job.yaml b/demos/signal-processing/create-nifi-ingestion-job.yaml index 4b44a343..63187d78 100644 --- a/demos/signal-processing/create-nifi-ingestion-job.yaml +++ b/demos/signal-processing/create-nifi-ingestion-job.yaml @@ -10,16 +10,28 @@ spec: initContainers: - name: wait-for-timescale-job image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev - command: ["bash", "-c", "echo 'Waiting for timescaleDB tables to be ready' - && kubectl wait --for=condition=complete job/create-timescale-tables-job" - ] + command: + - bash + - -c + - -euo + - pipefail + - | + echo 'Waiting for timescaleDB tables and NiFi to be ready' + kubectl wait --for=condition=complete job/create-timescale-tables-job --timeout=300s + kubectl wait --for=condition=Ready pod/nifi-node-default-0 --timeout=300s containers: - name: create-nifi-ingestion-job image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev - command: ["bash", "-c", "export PGPASSWORD=$(cat /timescale-admin-credentials/password) && \ - curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/signal-processing/DownloadAndWriteToDB.xml && \ - sed -i \"s/PLACEHOLDERPGPASSWORD/$PGPASSWORD/g\" DownloadAndWriteToDB.xml && \ - python -u /tmp/script/script.py"] + command: + - bash + - -c + - -euo + - pipefail + - | + export PGPASSWORD=$(cat /timescale-admin-credentials/password) + curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/signal-processing/DownloadAndWriteToDB.xml + sed -i "s/PLACEHOLDERPGPASSWORD/$PGPASSWORD/g" DownloadAndWriteToDB.xml + python -u /tmp/script/script.py volumeMounts: - name: script mountPath: /tmp/script @@ -70,11 +82,12 @@ data: nipyapi.config.nifi_config.host = f"{ENDPOINT}/nifi-api" nipyapi.config.nifi_config.verify_ssl = False - print("Logging in") + print(f"Logging in as {USERNAME}") service_login(username=USERNAME, password=PASSWORD) print("Logged in") pg_id = get_root_pg_id() + print(f"pgid={pg_id}") upload_template(pg_id, TEMPLATE_FILE) From bc22c583f866918301d783fabd19944e58c2c460 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 27 Feb 2025 09:59:18 +0100 Subject: [PATCH 05/13] docs(demo/signal-processing): Improve demo docs --- docs/modules/demos/pages/signal-processing.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/modules/demos/pages/signal-processing.adoc b/docs/modules/demos/pages/signal-processing.adoc index 4e1ac075..256b76f6 100644 --- a/docs/modules/demos/pages/signal-processing.adoc +++ b/docs/modules/demos/pages/signal-processing.adoc @@ -138,14 +138,14 @@ There are two located in the "Stackable Data Platform" folder. === Measurements -This is the original data. The first graph plots two measurments (`r1`, `r2`), together with the model scores (`r1_score`, `r2_score`, `r1_score_lttb`).# +The _Gas measurements_ dashboard shows the original data. The first graph plots two measurments (`r1`, `r2`), together with the model scores (`r1_score`, `r2_score`, `r1_score_lttb`).# These are superimposed on each other for ease of comparison. image::signal-processing/measurements.png[] === Predictions -In this second dashboard the predictions for all r-values are plotted: the top graph takes an average across all measurements, with a threshold marked as a red line across the top. +The _Spectral Residuals_ dashboard shows the predictions for all r-values are plotted: the top graph takes an average across all measurements, with a threshold marked as a red line across the top. This can be used for triggering email alerts. Underneath the individual r-values are plotted, firstly as raw data and then the same using downsampling. Downsampling uses a built-in Timescale extension to significantly reduce the number of data plotted while retaining the same overall shape. From b9fa60ec1019f252c5c28078325bb195d3cbd3dc Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 27 Feb 2025 10:00:57 +0100 Subject: [PATCH 06/13] chore(demo/signal-processing): Drop back to NiFi 1.28.1 due to NiFi 2.x problems --- stacks/signal-processing/nifi.yaml | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/stacks/signal-processing/nifi.yaml b/stacks/signal-processing/nifi.yaml index b0a48f0e..2fa82c9a 100644 --- a/stacks/signal-processing/nifi.yaml +++ b/stacks/signal-processing/nifi.yaml @@ -5,10 +5,12 @@ metadata: name: nifi spec: image: - productVersion: 2.2.0 + productVersion: 1.28.1 # TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that # the demo is reproducable for the release and it will be automatically replaced for the release branch. - custom: oci.stackable.tech/stackable/nifi:2.2.0-postgresql + # custom: oci.stackable.tech/stackable/nifi:2.2.0-postgresql + custom: oci.stackable.tech/stackable/nifi:1.28.1-postgresql + # pullPolicy: IfNotPresent clusterConfig: listenerClass: external-unstable zookeeperConfigMapName: nifi-znode @@ -18,6 +20,13 @@ spec: keySecret: nifi-sensitive-property-key autoGenerate: true nodes: + podOverrides: + metadata: + annotations: + sidecar.opentelemetry.io/inject: "default/otel-collector-grpc" + instrumentation.opentelemetry.io/inject-java: "true" + # Todo: We probably want to add the following annotation by default. + instrumentation.opentelemetry.io/java-container-names: "nifi" config: resources: cpu: From 472811a939144902498e6fa59da09291e9ecd649 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 27 Feb 2025 10:07:13 +0100 Subject: [PATCH 07/13] chore(demo/nifi-kafka-druid-superset-s3): Drop back to NiFi 1.28.1 due to NiFi 2.x problems --- stacks/nifi-kafka-druid-superset-s3/nifi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/nifi-kafka-druid-superset-s3/nifi.yaml b/stacks/nifi-kafka-druid-superset-s3/nifi.yaml index 225fce94..3736dd8e 100644 --- a/stacks/nifi-kafka-druid-superset-s3/nifi.yaml +++ b/stacks/nifi-kafka-druid-superset-s3/nifi.yaml @@ -5,7 +5,7 @@ metadata: name: nifi spec: image: - productVersion: 2.2.0 + productVersion: 1.28.1 clusterConfig: authentication: - authenticationClass: nifi-admin-credentials From c7e17e904df90eed9d436f232a699e1d4d8fec2f Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 27 Feb 2025 10:07:32 +0100 Subject: [PATCH 08/13] chore(demo/data-lakehouse-iceberg-trino-spark): Drop back to NiFi 1.28.1 due to NiFi 2.x problems --- stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml b/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml index bae55688..1ea8a69d 100644 --- a/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml +++ b/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml @@ -5,7 +5,7 @@ metadata: name: nifi spec: image: - productVersion: 2.2.0 + productVersion: 1.28.1 clusterConfig: authentication: - authenticationClass: nifi-admin-credentials From bf2d291b75976441c422ad62383f08bcb5de6738 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 27 Feb 2025 13:03:17 +0100 Subject: [PATCH 09/13] chore(demo/signal-processing): Improve readability of setup jobs --- .../create-nifi-ingestion-job.yaml | 17 ++++----- .../create-timescale-tables.yaml | 36 ++++++++++++++----- 2 files changed, 37 insertions(+), 16 deletions(-) diff --git a/demos/signal-processing/create-nifi-ingestion-job.yaml b/demos/signal-processing/create-nifi-ingestion-job.yaml index 63187d78..ea3aede3 100644 --- a/demos/signal-processing/create-nifi-ingestion-job.yaml +++ b/demos/signal-processing/create-nifi-ingestion-job.yaml @@ -8,7 +8,7 @@ spec: spec: serviceAccountName: demo-serviceaccount initContainers: - - name: wait-for-timescale-job + - name: wait-for-timescale-job-and-nifi image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev command: - bash @@ -16,9 +16,10 @@ spec: - -euo - pipefail - | - echo 'Waiting for timescaleDB tables and NiFi to be ready' - kubectl wait --for=condition=complete job/create-timescale-tables-job --timeout=300s - kubectl wait --for=condition=Ready pod/nifi-node-default-0 --timeout=300s + echo 'Waiting for timescaleDB tables to be ready' + kubectl wait --for=condition=complete job/create-timescale-tables-job --timeout=30m + echo 'Waiting for NiFi to be ready' + kubectl wait --for=condition=Ready pod/nifi-node-default-0 --timeout=30m containers: - name: create-nifi-ingestion-job image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev @@ -28,10 +29,10 @@ spec: - -euo - pipefail - | - export PGPASSWORD=$(cat /timescale-admin-credentials/password) - curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/signal-processing/DownloadAndWriteToDB.xml - sed -i "s/PLACEHOLDERPGPASSWORD/$PGPASSWORD/g" DownloadAndWriteToDB.xml - python -u /tmp/script/script.py + export PGPASSWORD=$(cat /timescale-admin-credentials/password) + curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/signal-processing/DownloadAndWriteToDB.xml + sed -i "s/PLACEHOLDERPGPASSWORD/$PGPASSWORD/g" DownloadAndWriteToDB.xml + python -u /tmp/script/script.py volumeMounts: - name: script mountPath: /tmp/script diff --git a/demos/signal-processing/create-timescale-tables.yaml b/demos/signal-processing/create-timescale-tables.yaml index 853ea310..3937b847 100644 --- a/demos/signal-processing/create-timescale-tables.yaml +++ b/demos/signal-processing/create-timescale-tables.yaml @@ -10,17 +10,37 @@ spec: initContainers: - name: wait-for-timescale image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev - command: ["bash", "-c", "echo 'Waiting for timescaleDB to be ready' - && kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/name=postgresql-timescaledb" - ] + command: + - bash + - -c + - -euo + - pipefail + - | + echo 'Waiting for timescaleDB to be ready' + kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=postgresql-timescaledb --timeout=30m containers: - name: create-timescale-tables-job image: postgres - command: ["bash", "-c", "export PGPASSWORD=$(cat /timescale-admin-credentials/password) && \ - echo 'Submitting DDL...' && \ - psql -U admin -h postgresql-timescaledb.default.svc.cluster.local postgres -c '\\x' -c 'CREATE DATABASE tsdb' -c '\\c tsdb' -f /tmp/sql/timescaledb.sql -c 'select count(*) from conditions' -c '\\q' && \ - echo 'Creating extension as superuser...' && \ - psql -U postgres -h postgresql-timescaledb.default.svc.cluster.local postgres -c '\\x' -c '\\c tsdb' -c 'CREATE EXTENSION timescaledb_toolkit' -c '\\q'"] + command: + - bash + - -c + - -euo + - pipefail + - | + export PGPASSWORD=$(cat /timescale-admin-credentials/password) + echo 'Submitting DDL...' + psql -U admin -h postgresql-timescaledb.default.svc.cluster.local postgres \ + -c '\x' -c 'CREATE DATABASE tsdb' \ + -c '\c tsdb' \ + -f /tmp/sql/timescaledb.sql \ + -c 'select count(*) from conditions' \ + -c '\q' + echo 'Creating extension as superuser...' + psql -U postgres -h postgresql-timescaledb.default.svc.cluster.local postgres \ + -c '\x' \ + -c '\c tsdb' \ + -c 'CREATE EXTENSION timescaledb_toolkit' \ + -c '\q' volumeMounts: - name: script mountPath: /tmp/sql From 3647bb27bd5e9a21a2da0700e28c4d5b1ae50c07 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 27 Feb 2025 20:51:10 +0100 Subject: [PATCH 10/13] chore(demo/signal-processing): Improve setup jobs --- .../signal-processing/create-nifi-ingestion-job.yaml | 12 +++++++++--- demos/signal-processing/create-timescale-tables.yaml | 9 ++++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/demos/signal-processing/create-nifi-ingestion-job.yaml b/demos/signal-processing/create-nifi-ingestion-job.yaml index ea3aede3..8f9790fd 100644 --- a/demos/signal-processing/create-nifi-ingestion-job.yaml +++ b/demos/signal-processing/create-nifi-ingestion-job.yaml @@ -12,12 +12,18 @@ spec: image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev command: - bash - - -c - -euo - pipefail + - -c - | - echo 'Waiting for timescaleDB tables to be ready' + echo 'Waiting for timescaleDB job to be created' + until kubectl get job create-timescale-tables-job 2>/dev/null; do sleep 5; done + kubectl wait --for=create job/create-timescale-tables-job --timeout=30m + echo 'Waiting for timescaleDB job to be completed' kubectl wait --for=condition=complete job/create-timescale-tables-job --timeout=30m + + echo 'Waiting for NiFi to be created' + kubectl wait --for=create pod/nifi-node-default-0 --timeout=30m echo 'Waiting for NiFi to be ready' kubectl wait --for=condition=Ready pod/nifi-node-default-0 --timeout=30m containers: @@ -25,9 +31,9 @@ spec: image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev command: - bash - - -c - -euo - pipefail + - -c - | export PGPASSWORD=$(cat /timescale-admin-credentials/password) curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/signal-processing/DownloadAndWriteToDB.xml diff --git a/demos/signal-processing/create-timescale-tables.yaml b/demos/signal-processing/create-timescale-tables.yaml index 3937b847..efa8c7d6 100644 --- a/demos/signal-processing/create-timescale-tables.yaml +++ b/demos/signal-processing/create-timescale-tables.yaml @@ -12,20 +12,23 @@ spec: image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev command: - bash - - -c - -euo - pipefail + - -c - | + echo 'Waiting for timescaleDB to be created' + kubectl wait --for=create pod/postgresql-timescaledb-0 --timeout=30m + echo 'Waiting for timescaleDB to be ready' - kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=postgresql-timescaledb --timeout=30m + kubectl wait --for=condition=ready pod/postgresql-timescaledb-0 --timeout=30m containers: - name: create-timescale-tables-job image: postgres command: - bash - - -c - -euo - pipefail + - -c - | export PGPASSWORD=$(cat /timescale-admin-credentials/password) echo 'Submitting DDL...' From de4ae6a47852dc0c0930684eaa30322faae93952 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 27 Feb 2025 20:52:04 +0100 Subject: [PATCH 11/13] chore(demo/nifi-kafka-druid-water-level-data): Improve initialization jobs --- demos/demos-v2.yaml | 1 + .../create-druid-ingestion-job.yaml | 24 +++++++++- .../create-nifi-ingestion-job.yaml | 23 ++++++++- .../serviceaccount.yaml | 47 +++++++++++++++++++ .../setup-superset.yaml | 23 ++++++++- 5 files changed, 115 insertions(+), 3 deletions(-) create mode 100644 demos/nifi-kafka-druid-water-level-data/serviceaccount.yaml diff --git a/demos/demos-v2.yaml b/demos/demos-v2.yaml index a31ff1ea..ff275071 100644 --- a/demos/demos-v2.yaml +++ b/demos/demos-v2.yaml @@ -85,6 +85,7 @@ demos: - s3 - water-levels manifests: + - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/serviceaccount.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/setup-superset.yaml diff --git a/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml b/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml index 01570f29..3ce637dc 100644 --- a/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml +++ b/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml @@ -6,10 +6,32 @@ metadata: spec: template: spec: + serviceAccountName: demo-serviceaccount + initContainers: + - name: wait-for-druid-coordinator + image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev + command: + - bash + - -euo + - pipefail + - -c + - | + echo 'Waiting for Druid Coordinator to be created' + kubectl wait --for=create pod/druid-coordinator-default-0 + echo 'Waiting for Druid Coordinator to be ready' + kubectl wait --for=condition=Ready pod/druid-coordinator-default-0 --timeout=30m containers: - name: create-druid-ingestion-job image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev - command: ["bash", "-c", "curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/stations-ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor && curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/measurements-ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor && curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/measurements-compaction-job-spec.json https://druid-coordinator:8281/druid/coordinator/v1/config/compaction"] + command: + - bash + - -euo + - pipefail + - -c + - | + curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/stations-ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor + curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/measurements-ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor + curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/measurements-compaction-job-spec.json https://druid-coordinator:8281/druid/coordinator/v1/config/compaction volumeMounts: - name: ingestion-job-spec mountPath: /tmp/ingestion-job-spec diff --git a/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml b/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml index 7dcf70ed..edd8fac2 100644 --- a/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml +++ b/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml @@ -6,10 +6,31 @@ metadata: spec: template: spec: + serviceAccountName: demo-serviceaccount + initContainers: + - name: wait-for-nifi + image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev + command: + - bash + - -euo + - pipefail + - -c + - | + echo 'Waiting for NiFi to be created' + kubectl wait --for=create pod/nifi-node-default-0 --timeout=30m + echo 'Waiting for NiFi to be ready' + kubectl wait --for=condition=Ready pod/nifi-node-default-0 --timeout=30m containers: - name: create-nifi-ingestion-job image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev - command: ["bash", "-c", "curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/IngestWaterLevelsToKafka.xml && python -u /tmp/script/script.py"] + command: + - bash + - -euo + - pipefail + - -c + - | + curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/IngestWaterLevelsToKafka.xml + python -u /tmp/script/script.py volumeMounts: - name: script mountPath: /tmp/script diff --git a/demos/nifi-kafka-druid-water-level-data/serviceaccount.yaml b/demos/nifi-kafka-druid-water-level-data/serviceaccount.yaml new file mode 100644 index 00000000..8a2d0b19 --- /dev/null +++ b/demos/nifi-kafka-druid-water-level-data/serviceaccount.yaml @@ -0,0 +1,47 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: demo-serviceaccount + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: demo-clusterrolebinding +subjects: + - kind: ServiceAccount + name: demo-serviceaccount + namespace: default +roleRef: + kind: ClusterRole + name: demo-clusterrole + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: demo-clusterrole +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + - apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - pods/exec + verbs: + - create diff --git a/demos/nifi-kafka-druid-water-level-data/setup-superset.yaml b/demos/nifi-kafka-druid-water-level-data/setup-superset.yaml index fad556f2..0d85b0b0 100644 --- a/demos/nifi-kafka-druid-water-level-data/setup-superset.yaml +++ b/demos/nifi-kafka-druid-water-level-data/setup-superset.yaml @@ -6,10 +6,31 @@ metadata: spec: template: spec: + serviceAccountName: demo-serviceaccount + initContainers: + - name: wait-for-superset + image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev + command: + - bash + - -euo + - pipefail + - -c + - | + echo 'Waiting for Superset to be created' + kubectl wait --for=create pod/superset-node-default-0 --timeout=30m + echo 'Waiting for Superset to be ready' + kubectl wait --for=condition=Ready pod/superset-node-default-0 --timeout=30m containers: - name: setup-superset image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev - command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/superset-assets.zip && python -u /tmp/script/script.py"] + command: + - bash + - -euo + - pipefail + - -c + - | + curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/superset-assets.zip + python -u /tmp/script/script.py volumeMounts: - name: script mountPath: /tmp/script From 117fd6630b4c09a4a09bd2fa4488789f6c09d2f2 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 27 Feb 2025 20:52:23 +0100 Subject: [PATCH 12/13] chore(demo/nifi-kafka-druid-earthquake-data): Improve initialization jobs --- demos/demos-v2.yaml | 1 + .../create-druid-ingestion-job.yaml | 22 ++++++++- .../create-nifi-ingestion-job.yaml | 23 ++++++++- .../serviceaccount.yaml | 47 +++++++++++++++++++ .../setup-superset.yaml | 23 ++++++++- 5 files changed, 113 insertions(+), 3 deletions(-) create mode 100644 demos/nifi-kafka-druid-earthquake-data/serviceaccount.yaml diff --git a/demos/demos-v2.yaml b/demos/demos-v2.yaml index ff275071..f8c92aed 100644 --- a/demos/demos-v2.yaml +++ b/demos/demos-v2.yaml @@ -64,6 +64,7 @@ demos: - s3 - earthquakes manifests: + - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/serviceaccount.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml diff --git a/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml b/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml index 421ba348..2c25c32c 100644 --- a/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml +++ b/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml @@ -6,10 +6,30 @@ metadata: spec: template: spec: + serviceAccountName: demo-serviceaccount + initContainers: + - name: wait-for-druid-coordinator + image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev + command: + - bash + - -euo + - pipefail + - -c + - | + echo 'Waiting for Druid Coordinator to be created' + kubectl wait --for=create pod/druid-coordinator-default-0 + echo 'Waiting for Druid Coordinator to be ready' + kubectl wait --for=condition=Ready pod/druid-coordinator-default-0 --timeout=30m containers: - name: create-druid-ingestion-job image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev - command: ["bash", "-c", "curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor"] + command: + - bash + - -euo + - pipefail + - -c + - | + curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor volumeMounts: - name: ingestion-job-spec mountPath: /tmp/ingestion-job-spec diff --git a/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml b/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml index 19bb7675..efddab0c 100644 --- a/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml +++ b/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml @@ -6,10 +6,31 @@ metadata: spec: template: spec: + serviceAccountName: demo-serviceaccount + initContainers: + - name: wait-for-nifi + image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev + command: + - bash + - -euo + - pipefail + - -c + - | + echo 'Waiting for NiFi to be created' + kubectl wait --for=create pod/nifi-node-default-0 --timeout=30m + echo 'Waiting for NiFi to be ready' + kubectl wait --for=condition=Ready pod/nifi-node-default-0 --timeout=30m containers: - name: create-nifi-ingestion-job image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev - command: ["bash", "-c", "curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/IngestEarthquakesToKafka.xml && python -u /tmp/script/script.py"] + command: + - bash + - -euo + - pipefail + - -c + - | + curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/IngestEarthquakesToKafka.xml + python -u /tmp/script/script.py volumeMounts: - name: script mountPath: /tmp/script diff --git a/demos/nifi-kafka-druid-earthquake-data/serviceaccount.yaml b/demos/nifi-kafka-druid-earthquake-data/serviceaccount.yaml new file mode 100644 index 00000000..8a2d0b19 --- /dev/null +++ b/demos/nifi-kafka-druid-earthquake-data/serviceaccount.yaml @@ -0,0 +1,47 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: demo-serviceaccount + namespace: default +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: demo-clusterrolebinding +subjects: + - kind: ServiceAccount + name: demo-serviceaccount + namespace: default +roleRef: + kind: ClusterRole + name: demo-clusterrole + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: demo-clusterrole +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - watch + - apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - pods/exec + verbs: + - create diff --git a/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml b/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml index 15ee14c9..9be84027 100644 --- a/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml +++ b/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml @@ -6,10 +6,31 @@ metadata: spec: template: spec: + serviceAccountName: demo-serviceaccount + initContainers: + - name: wait-for-superset + image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev + command: + - bash + - -euo + - pipefail + - -c + - | + echo 'Waiting for Superset to be created' + kubectl wait --for=create pod/superset-node-default-0 --timeout=30m + echo 'Waiting for Superset to be ready' + kubectl wait --for=condition=Ready pod/superset-node-default-0 --timeout=30m containers: - name: setup-superset image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev - command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/superset-assets.zip && python -u /tmp/script/script.py"] + command: + - bash + - -euo + - pipefail + - -c + - | + curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/superset-assets.zip + python -u /tmp/script/script.py volumeMounts: - name: script mountPath: /tmp/script From da1b1fe2762c997807e99ae94eac35f12ae8e5f5 Mon Sep 17 00:00:00 2001 From: Nick Larsen Date: Thu, 27 Feb 2025 21:06:07 +0100 Subject: [PATCH 13/13] chore(demo/signal-processing): Remove accidentally committed changes --- stacks/signal-processing/nifi.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/stacks/signal-processing/nifi.yaml b/stacks/signal-processing/nifi.yaml index 2fa82c9a..3af838e7 100644 --- a/stacks/signal-processing/nifi.yaml +++ b/stacks/signal-processing/nifi.yaml @@ -20,13 +20,6 @@ spec: keySecret: nifi-sensitive-property-key autoGenerate: true nodes: - podOverrides: - metadata: - annotations: - sidecar.opentelemetry.io/inject: "default/otel-collector-grpc" - instrumentation.opentelemetry.io/inject-java: "true" - # Todo: We probably want to add the following annotation by default. - instrumentation.opentelemetry.io/java-container-names: "nifi" config: resources: cpu: