From a3c53543e8a27f8a94b75fd0368ca342b8b2a7d3 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 17 Dec 2025 13:56:00 +0100 Subject: [PATCH 1/5] enable decision logs --- .../hive-metastore-regorules.yaml | 66 +++++++++++++++++++ .../end-to-end-security/hive-metastore.yaml | 4 ++ stacks/end-to-end-security/opa.yaml | 4 ++ 3 files changed, 74 insertions(+) create mode 100644 stacks/end-to-end-security/hive-metastore-regorules.yaml diff --git a/stacks/end-to-end-security/hive-metastore-regorules.yaml b/stacks/end-to-end-security/hive-metastore-regorules.yaml new file mode 100644 index 00000000..6c7c77f4 --- /dev/null +++ b/stacks/end-to-end-security/hive-metastore-regorules.yaml @@ -0,0 +1,66 @@ +--- +# {% raw %} +apiVersion: v1 +kind: ConfigMap +metadata: + name: hive-metastore-regorules + labels: + opa.stackable.tech/bundle: "true" +data: + actual_permissions.rego: | + package hms + + trino_user := "trino" + spark_user := "spark" + customer_analytics_db := "customer_analytics" + compliance_analytics_db := "compliance_analytics" + + default database_allow = false + default table_allow = false + default column_allow = false + default partition_allow = false + default user_allow = false + + database_allow if { + input.identity.username == spark_user + input.resources.database.name == customer_analytics_db + } + + # Allow 'SELECT * FROM lakehouse.customer_analytics.customer' + table_allow if { + input.identity.username == spark_user + input.resources.table.dbName == customer_analytics_db + input.resources.table.tableName == "customer" + input.privileges.readRequiredPriv[0].priv == "SELECT" + } + + # Allow: 'CREATE TABLE IF NOT EXISTS lakehouse.customer_analytics.spark_report AS SELECT c_birth_country, count(*) FROM ..' + table_allow if { + input.identity.username == spark_user + input.resources.table.dbName == customer_analytics_db + input.resources.table.tableName == "spark_report" + input.privileges.writeRequiredPriv[0].priv == "CREATE" + } + + # Trino + database_allow if { + input.identity.username == trino_user + } + + table_allow if { + input.identity.username == trino_user + } + + column_allow if { + input.identity.username == trino_user + } + + partition_allow if { + input.identity.username == trino_user + } + + user_allow if { + input.identity.username == trino_user + } + +# {% endraw %} diff --git a/stacks/end-to-end-security/hive-metastore.yaml b/stacks/end-to-end-security/hive-metastore.yaml index e1c08899..e140296a 100644 --- a/stacks/end-to-end-security/hive-metastore.yaml +++ b/stacks/end-to-end-security/hive-metastore.yaml @@ -7,6 +7,10 @@ spec: image: productVersion: 4.0.0 clusterConfig: + authorization: + opa: + configMapName: opa + package: hms database: connString: jdbc:postgresql://postgresql-hive-iceberg:5432/hive credentialsSecret: postgres-credentials diff --git a/stacks/end-to-end-security/opa.yaml b/stacks/end-to-end-security/opa.yaml index cb8bd725..ec46fd74 100644 --- a/stacks/end-to-end-security/opa.yaml +++ b/stacks/end-to-end-security/opa.yaml @@ -25,6 +25,10 @@ spec: logging: containers: opa: + console: + level: INFO + file: + level: INFO loggers: decision: level: INFO From 91e620eed3ca94c17f4a934b348b21582a94deed Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 17 Dec 2025 14:18:30 +0100 Subject: [PATCH 2/5] rename opa package to hive-iceberg --- stacks/end-to-end-security/hive-metastore-regorules.yaml | 2 +- stacks/end-to-end-security/hive-metastore.yaml | 2 +- stacks/stacks-v2.yaml | 5 +++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/stacks/end-to-end-security/hive-metastore-regorules.yaml b/stacks/end-to-end-security/hive-metastore-regorules.yaml index 6c7c77f4..ce7fba1a 100644 --- a/stacks/end-to-end-security/hive-metastore-regorules.yaml +++ b/stacks/end-to-end-security/hive-metastore-regorules.yaml @@ -8,7 +8,7 @@ metadata: opa.stackable.tech/bundle: "true" data: actual_permissions.rego: | - package hms + package hive-iceberg trino_user := "trino" spark_user := "spark" diff --git a/stacks/end-to-end-security/hive-metastore.yaml b/stacks/end-to-end-security/hive-metastore.yaml index e140296a..e9290c44 100644 --- a/stacks/end-to-end-security/hive-metastore.yaml +++ b/stacks/end-to-end-security/hive-metastore.yaml @@ -10,7 +10,7 @@ spec: authorization: opa: configMapName: opa - package: hms + package: hive-iceberg database: connString: jdbc:postgresql://postgresql-hive-iceberg:5432/hive credentialsSecret: postgres-credentials diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 508753a1..98c07716 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -584,11 +584,12 @@ stacks: - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-superset.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/krb5.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/kerberos-secretclass.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/opa.yaml + - plainYaml: stacks/end-to-end-security/opa.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/zookeeper.yaml # TODO: Add authentication - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/hdfs.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/hdfs-regorules.yaml - - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/hive-metastore.yaml + - plainYaml: stacks/end-to-end-security/hive-metastore.yaml + - plainYaml: stacks/end-to-end-security/hive-metastore-regorules.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/trino.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/trino-regorules.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/trino-policies.yaml From 8fbd29165173ee918224df5582244551731906dc Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 17 Dec 2025 14:23:09 +0100 Subject: [PATCH 3/5] improve comments --- .../hive-metastore-regorules.yaml | 12 +++++++++--- stacks/stacks-v2.yaml | 6 +++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/stacks/end-to-end-security/hive-metastore-regorules.yaml b/stacks/end-to-end-security/hive-metastore-regorules.yaml index ce7fba1a..2194d015 100644 --- a/stacks/end-to-end-security/hive-metastore-regorules.yaml +++ b/stacks/end-to-end-security/hive-metastore-regorules.yaml @@ -21,12 +21,16 @@ data: default partition_allow = false default user_allow = false + ### SPARK ### + # These rules are tailored for the create-spark-report job. + + # Allow the spark user access to the 'customer_analytics_db' database_allow if { input.identity.username == spark_user input.resources.database.name == customer_analytics_db } - # Allow 'SELECT * FROM lakehouse.customer_analytics.customer' + # Allow the 'SELECT * FROM lakehouse.customer_analytics.customer' query in create-spark-report table_allow if { input.identity.username == spark_user input.resources.table.dbName == customer_analytics_db @@ -34,7 +38,8 @@ data: input.privileges.readRequiredPriv[0].priv == "SELECT" } - # Allow: 'CREATE TABLE IF NOT EXISTS lakehouse.customer_analytics.spark_report AS SELECT c_birth_country, count(*) FROM ..' + # Allow the 'CREATE TABLE IF NOT EXISTS lakehouse.customer_analytics.spark_report AS SELECT c_birth_country, count(*) FROM ..' + # query in create-spark-report table_allow if { input.identity.username == spark_user input.resources.table.dbName == customer_analytics_db @@ -42,7 +47,8 @@ data: input.privileges.writeRequiredPriv[0].priv == "CREATE" } - # Trino + ### TRINO ### + # We allow everything here for the technical trino user in order to still do data exploration database_allow if { input.identity.username == trino_user } diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 98c07716..3702de14 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -584,12 +584,12 @@ stacks: - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-superset.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/krb5.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/kerberos-secretclass.yaml - - plainYaml: stacks/end-to-end-security/opa.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/opa.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/zookeeper.yaml # TODO: Add authentication - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/hdfs.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/hdfs-regorules.yaml - - plainYaml: stacks/end-to-end-security/hive-metastore.yaml - - plainYaml: stacks/end-to-end-security/hive-metastore-regorules.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/hive-metastore.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/hive-metastore-regorules.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/trino.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/trino-regorules.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/end-to-end-security/trino-policies.yaml From 4bba00d261aea2f4add538a3ee31a52b7e440b81 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 17 Dec 2025 14:31:25 +0100 Subject: [PATCH 4/5] fix package name --- stacks/end-to-end-security/hive-metastore-regorules.yaml | 2 +- stacks/end-to-end-security/hive-metastore.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stacks/end-to-end-security/hive-metastore-regorules.yaml b/stacks/end-to-end-security/hive-metastore-regorules.yaml index 2194d015..1df9cd51 100644 --- a/stacks/end-to-end-security/hive-metastore-regorules.yaml +++ b/stacks/end-to-end-security/hive-metastore-regorules.yaml @@ -8,7 +8,7 @@ metadata: opa.stackable.tech/bundle: "true" data: actual_permissions.rego: | - package hive-iceberg + package hive_iceberg trino_user := "trino" spark_user := "spark" diff --git a/stacks/end-to-end-security/hive-metastore.yaml b/stacks/end-to-end-security/hive-metastore.yaml index e9290c44..a15b9962 100644 --- a/stacks/end-to-end-security/hive-metastore.yaml +++ b/stacks/end-to-end-security/hive-metastore.yaml @@ -10,7 +10,7 @@ spec: authorization: opa: configMapName: opa - package: hive-iceberg + package: hive_iceberg database: connString: jdbc:postgresql://postgresql-hive-iceberg:5432/hive credentialsSecret: postgres-credentials From 66c134428751c17a77d1d46b62ca0b47dfb04583 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 17 Dec 2025 15:11:54 +0100 Subject: [PATCH 5/5] fix hive rego filename --- stacks/end-to-end-security/hive-metastore-regorules.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/end-to-end-security/hive-metastore-regorules.yaml b/stacks/end-to-end-security/hive-metastore-regorules.yaml index 1df9cd51..86c73e4b 100644 --- a/stacks/end-to-end-security/hive-metastore-regorules.yaml +++ b/stacks/end-to-end-security/hive-metastore-regorules.yaml @@ -7,7 +7,7 @@ metadata: labels: opa.stackable.tech/bundle: "true" data: - actual_permissions.rego: | + hive-metastore.rego: | package hive_iceberg trino_user := "trino"