fix(spark-anomaly-detection): Adjust Trino HTTPS connection (#251)

* wip

* fix assert zip content

* fix zip

* fix zip 2

* use main branch zip

* set version back to 4.1.2

* Apply suggestions from code review

Co-authored-by: Nick <[email protected]>

* Update demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml

Co-authored-by: Nick <[email protected]>

* use listener service instead of headless

---------

Co-authored-by: Nick <[email protected]>

Author: maltesander

demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml

@@ -9,11 +9,25 @@ spec:
       initContainers:
         - name: wait-for-testdata
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
-          command: ["bash", "-c", "echo 'Waiting for job load-ny-taxi-data to finish' && kubectl wait --for=condition=complete --timeout=30m job/load-ny-taxi-data"]
+          command:
+          - bash
+          - -euo
+          - pipefail
+          - -c
+          - |
+              echo 'Waiting for job load-ny-taxi-data to finish'
+              kubectl wait --for=condition=complete --timeout=30m job/load-ny-taxi-data
       containers:
         - name: create-spark-anomaly-detection-job
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
-          command: ["bash", "-c", "echo 'Submitting Spark job' && kubectl apply -f /tmp/manifest/spark-ad-job.yaml"]
+          command:
+          - bash
+          - -euo
+          - pipefail
+          - -c
+          - |
+              echo 'Submitting Spark job'
+              kubectl apply -f /tmp/manifest/spark-ad-job.yaml
           volumeMounts:
             - name: manifest
               mountPath: /tmp/manifest
@@ -89,7 +103,7 @@ data:
         spark.sql.extensions: org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions
         spark.sql.catalog.prediction: org.apache.iceberg.spark.SparkCatalog
         spark.sql.catalog.prediction.type: hive
-        spark.sql.catalog.prediction.uri: thrift://hive-iceberg:9083
+        spark.sql.catalog.prediction.uri: thrift://hive-iceberg-metastore:9083
         # AFAIK This can be removed in 24.3
         #spark.driver.userClassPathFirst: "false"
         #spark.executor.userClassPathFirst: "false"

demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml

@@ -9,7 +9,14 @@ spec:
       containers:
         - name: setup-superset
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
-          command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip && python -u /tmp/script/script.py"]
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              curl -L -o superset-assets.zip sets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip
+              python -u /tmp/script/script.py
           volumeMounts:
             - name: script
               mountPath: /tmp/script
@@ -39,7 +46,7 @@ data:
     import logging
     import requests
 
-    base_url = "http://superset-node-default:8088" # For local testing / developing replace it, afterwards change back to http://superset-node-default:8088
+    base_url = "http://superset-node:8088" # For local testing / developing replace it, afterwards change back to http://superset-node:8088
     superset_username = open("/superset-credentials/adminUser.username").read()
     superset_password =  open("/superset-credentials/adminUser.password").read()
     trino_username = "admin"

demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip

@@ -26,8 +26,8 @@ stringData:
   adminUser.firstname: Superset
   adminUser.lastname: Admin
   adminUser.email: [email protected]
-  adminUser.password: {{ supersetAdminPassword }}
-  connections.secretKey: {{ supersetSecretKey }}
+  adminUser.password: "{{ supersetAdminPassword }}"
+  connections.secretKey: "{{ supersetSecretKey }}"
   connections.sqlalchemyDatabaseUri: postgresql://superset:superset@postgresql-superset/superset
 ---
 apiVersion: v1