chore(nifi): Bump to 1.28.1, upgrade postgres jdbc driver (#154)

NickLarsenNZ · web-flow · commit d7eeebaa07ea · 2025-03-11T10:42:06.000Z
* chore(nifi): Bump to 2.2.0, upgrade postgres jdbc driver

* chore(demo/signal-processing): Update postgresql version NiFi template

* ci(container/nifi): Drop back to NiFi 1.28.1 due to issues with NiFi 2.x

* chore(demo/signal-processing): Improve readability of job scripts

* docs(demo/signal-processing): Improve demo docs

* chore(demo/signal-processing): Drop back to NiFi 1.28.1 due to NiFi 2.x problems

* chore(demo/nifi-kafka-druid-superset-s3): Drop back to NiFi 1.28.1 due to NiFi 2.x problems

* chore(demo/data-lakehouse-iceberg-trino-spark): Drop back to NiFi 1.28.1 due to NiFi 2.x problems

* chore(demo/signal-processing): Improve readability of setup jobs

* chore(demo/signal-processing): Improve setup jobs

* chore(demo/nifi-kafka-druid-water-level-data): Improve initialization jobs

* chore(demo/nifi-kafka-druid-earthquake-data): Improve initialization jobs

* chore(demo/signal-processing): Remove accidentally committed changes
diff --git a/.github/workflows/dev_nifi.yaml b/.github/workflows/dev_nifi.yaml
@@ -5,7 +5,7 @@ env:
   IMAGE_NAME: nifi
   # TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that
   # the demo is reproducable for the release and it will be automatically replaced for the release branch.
-  IMAGE_VERSION: 1.27.0-postgresql
+  IMAGE_VERSION: 1.28.1-postgresql
   REGISTRY_PATH: stackable
   DOCKERFILE_PATH: "demos/signal-processing/Dockerfile-nifi"
 
diff --git a/demos/demos-v2.yaml b/demos/demos-v2.yaml
@@ -64,6 +64,7 @@ demos:
       - s3
       - earthquakes
     manifests:
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/serviceaccount.yaml
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml
@@ -85,6 +86,7 @@ demos:
       - s3
       - water-levels
     manifests:
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/serviceaccount.yaml
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/setup-superset.yaml
diff --git a/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml b/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml
@@ -6,10 +6,30 @@ metadata:
 spec:
   template:
     spec:
+      serviceAccountName: demo-serviceaccount
+      initContainers:
+        - name: wait-for-druid-coordinator
+          image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              echo 'Waiting for Druid Coordinator to be created'
+              kubectl wait --for=create pod/druid-coordinator-default-0
+              echo 'Waiting for Druid Coordinator to be ready'
+              kubectl wait --for=condition=Ready pod/druid-coordinator-default-0 --timeout=30m
       containers:
         - name: create-druid-ingestion-job
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
-          command: ["bash", "-c", "curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor"]
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor
           volumeMounts:
             - name: ingestion-job-spec
               mountPath: /tmp/ingestion-job-spec
diff --git a/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml b/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml
@@ -6,10 +6,31 @@ metadata:
 spec:
   template:
     spec:
+      serviceAccountName: demo-serviceaccount
+      initContainers:
+        - name: wait-for-nifi
+          image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              echo 'Waiting for NiFi to be created'
+              kubectl wait --for=create pod/nifi-node-default-0 --timeout=30m
+              echo 'Waiting for NiFi to be ready'
+              kubectl wait --for=condition=Ready pod/nifi-node-default-0 --timeout=30m
       containers:
         - name: create-nifi-ingestion-job
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
-          command: ["bash", "-c", "curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/IngestEarthquakesToKafka.xml && python -u /tmp/script/script.py"]
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/IngestEarthquakesToKafka.xml
+              python -u /tmp/script/script.py
           volumeMounts:
             - name: script
               mountPath: /tmp/script
diff --git a/demos/nifi-kafka-druid-earthquake-data/serviceaccount.yaml b/demos/nifi-kafka-druid-earthquake-data/serviceaccount.yaml
@@ -0,0 +1,47 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: demo-serviceaccount
+  namespace: default
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: demo-clusterrolebinding
+subjects:
+  - kind: ServiceAccount
+    name: demo-serviceaccount
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: demo-clusterrole
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: demo-clusterrole
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - batch
+    resources:
+      - jobs
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - pods/exec
+    verbs:
+      - create
diff --git a/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml b/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml
@@ -6,10 +6,31 @@ metadata:
 spec:
   template:
     spec:
+      serviceAccountName: demo-serviceaccount
+      initContainers:
+        - name: wait-for-superset
+          image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              echo 'Waiting for Superset to be created'
+              kubectl wait --for=create pod/superset-node-default-0 --timeout=30m
+              echo 'Waiting for Superset to be ready'
+              kubectl wait --for=condition=Ready pod/superset-node-default-0 --timeout=30m
       containers:
         - name: setup-superset
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
-          command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/superset-assets.zip && python -u /tmp/script/script.py"]
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/superset-assets.zip
+              python -u /tmp/script/script.py
           volumeMounts:
             - name: script
               mountPath: /tmp/script
diff --git a/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml b/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml
@@ -6,10 +6,32 @@ metadata:
 spec:
   template:
     spec:
+      serviceAccountName: demo-serviceaccount
+      initContainers:
+        - name: wait-for-druid-coordinator
+          image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              echo 'Waiting for Druid Coordinator to be created'
+              kubectl wait --for=create pod/druid-coordinator-default-0
+              echo 'Waiting for Druid Coordinator to be ready'
+              kubectl wait --for=condition=Ready pod/druid-coordinator-default-0 --timeout=30m
       containers:
         - name: create-druid-ingestion-job
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
-          command: ["bash", "-c", "curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/stations-ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor && curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/measurements-ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor && curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/measurements-compaction-job-spec.json https://druid-coordinator:8281/druid/coordinator/v1/config/compaction"]
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/stations-ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor
+              curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/measurements-ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor
+              curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/measurements-compaction-job-spec.json https://druid-coordinator:8281/druid/coordinator/v1/config/compaction
           volumeMounts:
             - name: ingestion-job-spec
               mountPath: /tmp/ingestion-job-spec
diff --git a/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml b/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml
@@ -6,10 +6,31 @@ metadata:
 spec:
   template:
     spec:
+      serviceAccountName: demo-serviceaccount
+      initContainers:
+        - name: wait-for-nifi
+          image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              echo 'Waiting for NiFi to be created'
+              kubectl wait --for=create pod/nifi-node-default-0 --timeout=30m
+              echo 'Waiting for NiFi to be ready'
+              kubectl wait --for=condition=Ready pod/nifi-node-default-0 --timeout=30m
       containers:
         - name: create-nifi-ingestion-job
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
-          command: ["bash", "-c", "curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/IngestWaterLevelsToKafka.xml && python -u /tmp/script/script.py"]
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/IngestWaterLevelsToKafka.xml
+              python -u /tmp/script/script.py
           volumeMounts:
             - name: script
               mountPath: /tmp/script
diff --git a/demos/nifi-kafka-druid-water-level-data/serviceaccount.yaml b/demos/nifi-kafka-druid-water-level-data/serviceaccount.yaml
@@ -0,0 +1,47 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: demo-serviceaccount
+  namespace: default
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: demo-clusterrolebinding
+subjects:
+  - kind: ServiceAccount
+    name: demo-serviceaccount
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: demo-clusterrole
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: demo-clusterrole
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - batch
+    resources:
+      - jobs
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - pods/exec
+    verbs:
+      - create
diff --git a/demos/nifi-kafka-druid-water-level-data/setup-superset.yaml b/demos/nifi-kafka-druid-water-level-data/setup-superset.yaml
@@ -6,10 +6,31 @@ metadata:
 spec:
   template:
     spec:
+      serviceAccountName: demo-serviceaccount
+      initContainers:
+        - name: wait-for-superset
+          image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              echo 'Waiting for Superset to be created'
+              kubectl wait --for=create pod/superset-node-default-0 --timeout=30m
+              echo 'Waiting for Superset to be ready'
+              kubectl wait --for=condition=Ready pod/superset-node-default-0 --timeout=30m
       containers:
         - name: setup-superset
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
-          command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/superset-assets.zip && python -u /tmp/script/script.py"]
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/superset-assets.zip
+              python -u /tmp/script/script.py
           volumeMounts:
             - name: script
               mountPath: /tmp/script
diff --git a/demos/signal-processing/Dockerfile-nifi b/demos/signal-processing/Dockerfile-nifi
@@ -1,3 +1,13 @@
-FROM oci.stackable.tech/sdp/nifi:1.27.0-stackable0.0.0-dev
+FROM oci.stackable.tech/sdp/nifi:1.28.1-stackable0.0.0-dev
 
-RUN curl --fail -o /stackable/nifi/postgresql-42.6.0.jar "https://repo.stackable.tech/repository/misc/postgresql-timescaledb/postgresql-42.6.0.jar"
+# This is the postgresql JDBC driver from https://jdbc.postgresql.org/download/
+# There appear to be no signatures to validate against 😬
+#
+# VERSION="42.7.5"
+# curl -O "https://jdbc.postgresql.org/download/postgresql-$VERSION.jar"
+# curl --fail -u "your_username" --upload-file "postgresql-$VERSION.jar" 'https://repo.stackable.tech/repository/misc/jdbc/'
+# rm "postgresql-$VERSION.jar"
+
+# IMPORTANT (@NickLarsenNZ): Changing this version requires a change in the NiFi template (DownloadAndWriteToDB.xml)
+ARG DRIVER_VERSION="42.7.5"
+RUN curl --fail -o "/stackable/nifi/postgresql-$DRIVER_VERSION.jar" "https://repo.stackable.tech/repository/misc/jdbc/postgresql-$DRIVER_VERSION.jar"
diff --git a/demos/signal-processing/DownloadAndWriteToDB.xml b/demos/signal-processing/DownloadAndWriteToDB.xml
@@ -189,7 +189,7 @@
                 </entry>
                 <entry>
                     <key>database-driver-locations</key>
-                    <value>/stackable/nifi/postgresql-42.6.0.jar</value>
+                    <value>/stackable/nifi/postgresql-42.7.5.jar</value>
                 </entry>
                 <entry>
                     <key>kerberos-user-service</key>
diff --git a/demos/signal-processing/create-nifi-ingestion-job.yaml b/demos/signal-processing/create-nifi-ingestion-job.yaml
diff --git a/demos/signal-processing/create-timescale-tables.yaml b/demos/signal-processing/create-timescale-tables.yaml
diff --git a/docs/modules/demos/pages/signal-processing.adoc b/docs/modules/demos/pages/signal-processing.adoc
diff --git a/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml b/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml
diff --git a/stacks/nifi-kafka-druid-superset-s3/nifi.yaml b/stacks/nifi-kafka-druid-superset-s3/nifi.yaml
diff --git a/stacks/signal-processing/nifi.yaml b/stacks/signal-processing/nifi.yaml
