diff --git a/demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml b/demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml index 929ff124..ed7edd5b 100644 --- a/demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml +++ b/demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml @@ -13,14 +13,18 @@ spec: # restarted. Additionally, the db-init job takes a few minutes to complete before the cluster is deployed. The wait/watch steps # below are not "water-tight" but add a layer of stability by at least ensuring that the db is initialized and ready and that # all pods are reachable (albeit independent of each other). - command: ["bash", "-c", " - kubectl rollout status --watch statefulset/airflow-webserver-default - && kubectl rollout status --watch statefulset/airflow-scheduler-default - && export AIRFLOW_ADMIN_PASSWORD=$(cat /airflow-credentials/adminUser.password) - && export ACCESS_TOKEN=$(curl -XPOST http://airflow-webserver-default:8080/auth/token -H 'Content-Type: application/json' -d '{\"username\": \"admin\", \"password\": \"'$AIRFLOW_ADMIN_PASSWORD'\"}' | jq '.access_token' | tr -d '\"') - && curl -H \"Authorization: Bearer $ACCESS_TOKEN\" -H 'Content-Type: application/json' -XPATCH http://airflow-webserver-default:8080/api/v2/dags/sparkapp_dag -d '{\"is_paused\": false}' | jq - && curl -H \"Authorization: Bearer $ACCESS_TOKEN\" -H 'Content-Type: application/json' -XPOST http://airflow-webserver-default:8080/api/v2/dags/sparkapp_dag/dagRuns -d '{\"logical_date\": null}' | jq - "] + command: + - bash + - -euo + - pipefail + - -c + - | + kubectl rollout status --watch statefulset/airflow-webserver-default + kubectl rollout status --watch statefulset/airflow-scheduler-default + AIRFLOW_ADMIN_PASSWORD=$(cat /airflow-credentials/adminUser.password) + ACCESS_TOKEN=$(curl -XPOST http://airflow-webserver-default-headless:8080/auth/token -H 'Content-Type: application/json' -d '{"username": "admin", "password": "'$AIRFLOW_ADMIN_PASSWORD'"}' | jq -r .access_token) + curl -H "Authorization: Bearer $ACCESS_TOKEN" -H 'Content-Type: application/json' -XPATCH http://airflow-webserver-default-headless:8080/api/v2/dags/sparkapp_dag -d '{"is_paused": false}' | jq + curl -H "Authorization: Bearer $ACCESS_TOKEN" -H 'Content-Type: application/json' -XPOST http://airflow-webserver-default-headless:8080/api/v2/dags/sparkapp_dag/dagRuns -d '{"logical_date": null}' | jq volumeMounts: - name: airflow-credentials mountPath: /airflow-credentials diff --git a/demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml b/demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml index 087d7c9c..08eaccf3 100644 --- a/demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml +++ b/demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml @@ -13,14 +13,18 @@ spec: # restarted. Additionally, the db-init job takes a few minutes to complete before the cluster is deployed. The wait/watch steps # below are not "water-tight" but add a layer of stability by at least ensuring that the db is initialized and ready and that # all pods are reachable (albeit independent of each other). - command: ["bash", "-c", " - kubectl rollout status --watch statefulset/airflow-webserver-default - && kubectl rollout status --watch statefulset/airflow-scheduler-default - && export AIRFLOW_ADMIN_PASSWORD=$(cat /airflow-credentials/adminUser.password) - && export ACCESS_TOKEN=$(curl -XPOST http://airflow-webserver-default:8080/auth/token -H 'Content-Type: application/json' -d '{\"username\": \"admin\", \"password\": \"'$AIRFLOW_ADMIN_PASSWORD'\"}' | jq '.access_token' | tr -d '\"') - && curl -H \"Authorization: Bearer $ACCESS_TOKEN\" -H 'Content-Type: application/json' -XPATCH http://airflow-webserver-default:8080/api/v2/dags/date_demo -d '{\"is_paused\": false}' | jq - && curl -H \"Authorization: Bearer $ACCESS_TOKEN\" -H 'Content-Type: application/json' -XPOST http://airflow-webserver-default:8080/api/v2/dags/date_demo/dagRuns -d '{\"logical_date\": null}' | jq - "] + command: + - bash + - -euo + - pipefail + - -c + - | + kubectl rollout status --watch statefulset/airflow-webserver-default + kubectl rollout status --watch statefulset/airflow-scheduler-default + AIRFLOW_ADMIN_PASSWORD=$(cat /airflow-credentials/adminUser.password) + ACCESS_TOKEN=$(curl -XPOST http://airflow-webserver-default-headless:8080/auth/token -H 'Content-Type: application/json' -d '{"username": "admin", "password": "'$AIRFLOW_ADMIN_PASSWORD'"}' | jq -r .access_token) + curl -H "Authorization: Bearer $ACCESS_TOKEN" -H 'Content-Type: application/json' -XPATCH http://airflow-webserver-default-headless:8080/api/v2/dags/date_demo -d '{"is_paused": false}' | jq + curl -H "Authorization: Bearer $ACCESS_TOKEN" -H 'Content-Type: application/json' -XPOST http://airflow-webserver-default-headless:8080/api/v2/dags/date_demo/dagRuns -d '{"logical_date": null}' | jq volumeMounts: - name: airflow-credentials mountPath: /airflow-credentials diff --git a/stacks/airflow/airflow.yaml b/stacks/airflow/airflow.yaml index 5284b46d..1a02b1ef 100644 --- a/stacks/airflow/airflow.yaml +++ b/stacks/airflow/airflow.yaml @@ -26,6 +26,8 @@ spec: mountPath: /dags/pyspark_pi.yaml subPath: pyspark_pi.yaml webservers: + roleConfig: + listenerClass: external-unstable config: resources: cpu: @@ -34,7 +36,6 @@ spec: memory: limit: 2Gi gracefulShutdownTimeout: 30s - listenerClass: external-unstable roleGroups: default: envOverrides: