From a386fa9d17d74dbc72a8186463df3a199e11f617 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Tue, 18 Mar 2025 12:06:50 +0200 Subject: [PATCH 1/4] wip --- demos/demos-v2.yaml | 3 ++- .../spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml | 2 +- stacks/stacks-v2.yaml | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/demos/demos-v2.yaml b/demos/demos-v2.yaml index 117e4a0b..6fcb31dc 100644 --- a/demos/demos-v2.yaml +++ b/demos/demos-v2.yaml @@ -107,7 +107,8 @@ demos: - ny-taxi-data manifests: - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/serviceaccount.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml + # - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml + - plainYaml: demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml supportedNamespaces: [] diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml b/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml index 299d473a..a937d3df 100644 --- a/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml +++ b/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml @@ -10,7 +10,7 @@ spec: - name: load-ny-taxi-data image: "bitnami/minio:2022-debian-10" # yamllint disable-line rule:line-length - command: ["bash", "-c", "cd /tmp && for month in 2020-09 2020-10 2020-11 2020-12; do curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/fhvhv_tripdata_$month.parquet && mc --insecure alias set minio http://minio:9000/ $(cat /minio-s3-credentials/accessKey) $(cat /minio-s3-credentials/secretKey) && mc cp fhvhv_tripdata_$month.parquet minio/demo/ny-taxi-data/raw/ && mc mb --ignore-existing minio/prediction; done"] + command: ["bash", "-c", "cd /tmp && for month in 2020-09 2020-10 2020-11 2020-12; do curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/fhvhv_tripdata_$month.parquet && mc --insecure alias set minio https://minio:9000/ $(cat /minio-s3-credentials/accessKey) $(cat /minio-s3-credentials/secretKey) && mc cp fhvhv_tripdata_$month.parquet minio/demo/ny-taxi-data/raw/ && mc mb --ignore-existing minio/prediction; done"] volumeMounts: - name: minio-s3-credentials mountPath: /minio-s3-credentials diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 3296ce73..b4bf0069 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -254,7 +254,7 @@ stacks: - minio - s3 manifests: - - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/minio.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/minio-tls/rendered-chart.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-hive.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-hive-iceberg.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-superset.yaml From be9081fb4371a657175c1420d66e13b958795f65 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 18 Mar 2025 15:48:39 +0100 Subject: [PATCH 2/4] use cert for load test data, remove unsecure call --- .../load-test-data.yaml | 29 ++++++++++++++++++- 1 file changed, 28 insertions(+), 1 deletion(-) diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml b/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml index a937d3df..bc10d79b 100644 --- a/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml +++ b/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml @@ -10,13 +10,40 @@ spec: - name: load-ny-taxi-data image: "bitnami/minio:2022-debian-10" # yamllint disable-line rule:line-length - command: ["bash", "-c", "cd /tmp && for month in 2020-09 2020-10 2020-11 2020-12; do curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/fhvhv_tripdata_$month.parquet && mc --insecure alias set minio https://minio:9000/ $(cat /minio-s3-credentials/accessKey) $(cat /minio-s3-credentials/secretKey) && mc cp fhvhv_tripdata_$month.parquet minio/demo/ny-taxi-data/raw/ && mc mb --ignore-existing minio/prediction; done"] + command: + - "bash" + - "-ce" + - | + cd /tmp + for month in 2020-09 2020-10 2020-11 2020-12; do + curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/fhvhv_tripdata_$month.parquet + mc alias set minio https://minio.default.svc.cluster.local:9000/ $(cat /minio-s3-credentials/accessKey) $(cat /minio-s3-credentials/secretKey) + mc cp fhvhv_tripdata_$month.parquet minio/demo/ny-taxi-data/raw/ + mc mb --ignore-existing minio/prediction; + done volumeMounts: - name: minio-s3-credentials mountPath: /minio-s3-credentials + - name: tls + mountPath: /.mc/certs/CAs volumes: - name: minio-s3-credentials secret: secretName: minio-s3-credentials + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: |- + service=minio + spec: + storageClassName: secrets.stackable.tech + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" restartPolicy: OnFailure backoffLimit: 50 From 01fe9b6c10c4d30a18333323d3c76712a9b36b80 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 18 Mar 2025 15:51:10 +0100 Subject: [PATCH 3/4] remove local reference --- demos/demos-v2.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/demos/demos-v2.yaml b/demos/demos-v2.yaml index 6fcb31dc..117e4a0b 100644 --- a/demos/demos-v2.yaml +++ b/demos/demos-v2.yaml @@ -107,8 +107,7 @@ demos: - ny-taxi-data manifests: - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/serviceaccount.yaml - # - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml - - plainYaml: demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml supportedNamespaces: [] From dffc817f872702174fa5e6145db7292bd6e797c3 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 18 Mar 2025 15:56:47 +0100 Subject: [PATCH 4/4] fix linter --- demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml b/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml index bc10d79b..0276431c 100644 --- a/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml +++ b/demos/spark-k8s-anomaly-detection-taxi-data/load-test-data.yaml @@ -44,6 +44,6 @@ spec: - ReadWriteOnce resources: requests: - storage: "1" + storage: "1" restartPolicy: OnFailure backoffLimit: 50