From b6f5ae58abe32355ab36be2cedad04b86eba220d Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 11 Apr 2025 18:40:55 +0200 Subject: [PATCH 01/64] wip --- stacks/_templates/argo-cd.yaml | 18 ++++++++ .../applications/commons-operator.yaml | 42 ++++++++++++++++++ .../argo-cd/applications/secret-operator.yaml | 43 +++++++++++++++++++ .../argo-cd/projects/stackable-operators.yaml | 19 ++++++++ stacks/stacks-v2.yaml | 30 +++++++++++++ 5 files changed, 152 insertions(+) create mode 100644 stacks/_templates/argo-cd.yaml create mode 100644 stacks/argo-cd/applications/commons-operator.yaml create mode 100644 stacks/argo-cd/applications/secret-operator.yaml create mode 100644 stacks/argo-cd/projects/stackable-operators.yaml diff --git a/stacks/_templates/argo-cd.yaml b/stacks/_templates/argo-cd.yaml new file mode 100644 index 00000000..1c72ab9c --- /dev/null +++ b/stacks/_templates/argo-cd.yaml @@ -0,0 +1,18 @@ +--- +releaseName: argocd +name: argo-cd +repo: + name: argo-cd + url: https://argoproj.github.io/argo-helm +version: v7.8.23 +options: + configs: + secret: + argocdServerAdminPassword: "{{ argocdAdminPassword }}" + # We have to set the time otherwise error message: + # invalid session: Account password has changed since token issued + argocdServerAdminPasswordMtime: "2025-01-01T00:00:00Z" + dex: + enabled: false + notifications: + enabled: false diff --git a/stacks/argo-cd/applications/commons-operator.yaml b/stacks/argo-cd/applications/commons-operator.yaml new file mode 100644 index 00000000..97fafa90 --- /dev/null +++ b/stacks/argo-cd/applications/commons-operator.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: commons-operator +spec: + # this interferes with stackablectl's templating and + # cannot really be used here properly but must be provided. + generators: + - list: + elements: + - cluster: development + template: + metadata: + name: commons-operator + spec: + project: stackable-operators + ignoreDifferences: + # mitigating: https://github.com/stackabletech/hdfs-operator/issues/626 + - group: "apiextensions.k8s.io" + kind: "CustomResourceDefinition" + jqPathExpressions: + - .spec.names.categories | select(. == []) + - .spec.names.shortNames | select(. == []) + - .spec.versions[].additionalPrinterColumns | select(. == []) + source: + repoURL: https://repo.stackable.tech/repository/helm-stable/ + targetRevision: "{{ stackableVersion }}" + chart: commons-operator + helm: + releaseName: commons-operator + destination: + server: https://kubernetes.default.svc + namespace: "{{ stackableOperatorNamespace }}" + syncPolicy: + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + automated: + selfHeal: true + prune: true diff --git a/stacks/argo-cd/applications/secret-operator.yaml b/stacks/argo-cd/applications/secret-operator.yaml new file mode 100644 index 00000000..e8e46e42 --- /dev/null +++ b/stacks/argo-cd/applications/secret-operator.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: secret-operator +spec: + generators: + - list: + elements: + - cluster: development + url: https://kubernetes.default.svc + helmChartRevision: 25.3.0 + targetRevision: HEAD + template: + metadata: + name: secret-operator + spec: + project: stackable-operators + ignoreDifferences: + # mitigating: https://github.com/stackabletech/hdfs-operator/issues/626 + - group: "apiextensions.k8s.io" + kind: "CustomResourceDefinition" + jqPathExpressions: + - .spec.names.categories | select(. == []) + - .spec.names.shortNames | select(. == []) + - .spec.versions[].additionalPrinterColumns | select(. == []) + source: + repoURL: https://repo.stackable.tech/repository/helm-stable/ + targetRevision: 25.3.0 + chart: secret-operator + helm: + releaseName: secret-operator + destination: + server: "{{url}}" + namespace: stackable-operators + syncPolicy: + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + automated: + selfHeal: true + prune: true diff --git a/stacks/argo-cd/projects/stackable-operators.yaml b/stacks/argo-cd/projects/stackable-operators.yaml new file mode 100644 index 00000000..3e3e82a0 --- /dev/null +++ b/stacks/argo-cd/projects/stackable-operators.yaml @@ -0,0 +1,19 @@ +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: stackable-operators +spec: + description: Stackable operators ArgoCD Project + sourceRepos: + - "*" + destinations: + - namespace: argo-cd + server: https://kubernetes.default.svc + - namespace: stackable-operators + server: https://kubernetes.default.svc + - namespace: stackable-products + server: https://kubernetes.default.svc + # required to deploy clusterwide resources like clusteroles etc. + clusterResourceWhitelist: + - group: "*" + kind: "*" diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index d013c95f..d48df9dc 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -1,5 +1,35 @@ --- stacks: + argocd: + description: TODO + stackableRelease: dev + # we dont not want to install operators... + stackableOperators: + - listener + labels: + - argocd + manifests: + #- helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/argocd.yaml + - helmChart: stacks/_templates/argo-cd.yaml + - plainYaml: stacks/argo-cd/projects/stackable-operators.yaml + - plainYaml: stacks/argo-cd/applications/commons-operator.yaml + supportedNamespaces: + - argo-cd + resourceRequests: + memory: 2000Mi + cpu: 2000m + pvc: 20Gi + parameters: + - name: stackableVersion + description: Stackable release to be installed via Argo + default: 25.3.0 + - name: stackableOperatorNamespace + description: Stackable namespace for the operators + default: stackable-operators + - name: argocdAdminPassword + description: Password of the ArgoCD admin user + # generated via: `htpasswd -nbBC 10 "" adminadmin | tr -d ':\n'` + default: $2y$10$HhJC3pGHTlk8RyBoS39N/.wC72mdWxV2X8QS1wROUwCFxl.2tGfky monitoring: description: Stack containing Prometheus and Grafana stackableRelease: dev From 3e983a028465c5a3bb8ec8b4194dcc4c7d0fd0ca Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 11 Apr 2025 20:02:36 +0200 Subject: [PATCH 02/64] argocd / airflow stack working --- .../applications/airflow-postgres.yaml | 31 ++++++++++++++ stacks/argo-cd/applications/airflow.yaml | 20 +++++++++ .../applicationsets/airflow-operator.yaml | 42 +++++++++++++++++++ .../commons-operator.yaml | 2 +- .../applicationsets/listener-operator.yaml | 42 +++++++++++++++++++ .../secret-operator.yaml | 11 +++-- stacks/argo-cd/projects/airflow.yaml | 14 +++++++ .../argo-cd/projects/stackable-operators.yaml | 6 +-- stacks/stacks-v2.yaml | 23 +++++++++- 9 files changed, 177 insertions(+), 14 deletions(-) create mode 100644 stacks/argo-cd/applications/airflow-postgres.yaml create mode 100644 stacks/argo-cd/applications/airflow.yaml create mode 100644 stacks/argo-cd/applicationsets/airflow-operator.yaml rename stacks/argo-cd/{applications => applicationsets}/commons-operator.yaml (95%) create mode 100644 stacks/argo-cd/applicationsets/listener-operator.yaml rename stacks/argo-cd/{applications => applicationsets}/secret-operator.yaml (79%) create mode 100644 stacks/argo-cd/projects/airflow.yaml diff --git a/stacks/argo-cd/applications/airflow-postgres.yaml b/stacks/argo-cd/applications/airflow-postgres.yaml new file mode 100644 index 00000000..d59f9961 --- /dev/null +++ b/stacks/argo-cd/applications/airflow-postgres.yaml @@ -0,0 +1,31 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: airflow-postgres +spec: + project: airflow + destination: + server: https://kubernetes.default.svc + namespace: stackable-airflow + source: + repoURL: "registry-1.docker.io/bitnamicharts" + path: postgresql + # helm inspect chart oci://registry-1.docker.io/bitnamicharts/postgresql + targetRevision: 16.6.3 # 17.4.0 + chart: postgresql + helm: + # TODO this breaks naming as long as we use the airflow stack yaml which needs this svc name + releaseName: postgresql-airflow + valuesObject: + commonLabels: + stackable.tech/vendor: Stackable + auth: + username: airflow + password: airflow + database: airflow + syncPolicy: + syncOptions: + - CreateNamespace=true + automated: + selfHeal: true + prune: true diff --git a/stacks/argo-cd/applications/airflow.yaml b/stacks/argo-cd/applications/airflow.yaml new file mode 100644 index 00000000..4789efe9 --- /dev/null +++ b/stacks/argo-cd/applications/airflow.yaml @@ -0,0 +1,20 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: airflow +spec: + project: airflow + destination: + namespace: stackable-airflow + server: https://kubernetes.default.svc + source: + repoURL: https://github.com/stackabletech/demos.git + targetRevision: "{{ demoTargetRevision }}" + # TODO: change to other directory + path: stacks/airflow/ + syncPolicy: + syncOptions: + - CreateNamespace=true + automated: + selfHeal: true + prune: true diff --git a/stacks/argo-cd/applicationsets/airflow-operator.yaml b/stacks/argo-cd/applicationsets/airflow-operator.yaml new file mode 100644 index 00000000..109ecc94 --- /dev/null +++ b/stacks/argo-cd/applicationsets/airflow-operator.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: airflow-operator +spec: + # this interferes with stackablectl's templating and + # cannot really be used here properly but must be provided. + generators: + - list: + elements: + - cluster: development + template: + metadata: + name: airflow-operator + spec: + project: stackable-operators + ignoreDifferences: + # mitigating: https://github.com/stackabletech/hdfs-operator/issues/626 + - group: "apiextensions.k8s.io" + kind: "CustomResourceDefinition" + jqPathExpressions: + - .spec.names.categories | select(. == []) + - .spec.names.shortNames | select(. == []) + - .spec.versions[].additionalPrinterColumns | select(. == []) + source: + repoURL: https://repo.stackable.tech/repository/helm-stable/ + targetRevision: "{{ stackableReleaseVersion }}" + chart: airflow-operator + helm: + releaseName: airflow-operator + destination: + server: https://kubernetes.default.svc + namespace: "{{ stackableOperatorNamespace }}" + syncPolicy: + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + automated: + selfHeal: true + prune: true diff --git a/stacks/argo-cd/applications/commons-operator.yaml b/stacks/argo-cd/applicationsets/commons-operator.yaml similarity index 95% rename from stacks/argo-cd/applications/commons-operator.yaml rename to stacks/argo-cd/applicationsets/commons-operator.yaml index 97fafa90..19a2bbc6 100644 --- a/stacks/argo-cd/applications/commons-operator.yaml +++ b/stacks/argo-cd/applicationsets/commons-operator.yaml @@ -25,7 +25,7 @@ spec: - .spec.versions[].additionalPrinterColumns | select(. == []) source: repoURL: https://repo.stackable.tech/repository/helm-stable/ - targetRevision: "{{ stackableVersion }}" + targetRevision: "{{ stackableReleaseVersion }}" chart: commons-operator helm: releaseName: commons-operator diff --git a/stacks/argo-cd/applicationsets/listener-operator.yaml b/stacks/argo-cd/applicationsets/listener-operator.yaml new file mode 100644 index 00000000..9053f49e --- /dev/null +++ b/stacks/argo-cd/applicationsets/listener-operator.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: listener-operator +spec: + # this interferes with stackablectl's templating and + # cannot really be used here properly but must be provided. + generators: + - list: + elements: + - cluster: development + template: + metadata: + name: listener-operator + spec: + project: stackable-operators + ignoreDifferences: + # mitigating: https://github.com/stackabletech/hdfs-operator/issues/626 + - group: "apiextensions.k8s.io" + kind: "CustomResourceDefinition" + jqPathExpressions: + - .spec.names.categories | select(. == []) + - .spec.names.shortNames | select(. == []) + - .spec.versions[].additionalPrinterColumns | select(. == []) + source: + repoURL: https://repo.stackable.tech/repository/helm-stable/ + targetRevision: "{{ stackableReleaseVersion }}" + chart: listener-operator + helm: + releaseName: listener-operator + destination: + server: https://kubernetes.default.svc + namespace: "{{ stackableOperatorNamespace }}" + syncPolicy: + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + automated: + selfHeal: true + prune: true diff --git a/stacks/argo-cd/applications/secret-operator.yaml b/stacks/argo-cd/applicationsets/secret-operator.yaml similarity index 79% rename from stacks/argo-cd/applications/secret-operator.yaml rename to stacks/argo-cd/applicationsets/secret-operator.yaml index e8e46e42..7312c0bd 100644 --- a/stacks/argo-cd/applications/secret-operator.yaml +++ b/stacks/argo-cd/applicationsets/secret-operator.yaml @@ -4,13 +4,12 @@ kind: ApplicationSet metadata: name: secret-operator spec: + # this interferes with stackablectl's templating and + # cannot really be used here properly but must be provided. generators: - list: elements: - cluster: development - url: https://kubernetes.default.svc - helmChartRevision: 25.3.0 - targetRevision: HEAD template: metadata: name: secret-operator @@ -26,13 +25,13 @@ spec: - .spec.versions[].additionalPrinterColumns | select(. == []) source: repoURL: https://repo.stackable.tech/repository/helm-stable/ - targetRevision: 25.3.0 + targetRevision: "{{ stackableReleaseVersion }}" chart: secret-operator helm: releaseName: secret-operator destination: - server: "{{url}}" - namespace: stackable-operators + server: https://kubernetes.default.svc + namespace: "{{ stackableOperatorNamespace }}" syncPolicy: syncOptions: - CreateNamespace=true diff --git a/stacks/argo-cd/projects/airflow.yaml b/stacks/argo-cd/projects/airflow.yaml new file mode 100644 index 00000000..3661434f --- /dev/null +++ b/stacks/argo-cd/projects/airflow.yaml @@ -0,0 +1,14 @@ +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: airflow +spec: + description: Project to manage Stackable Airflow via Git + sourceRepos: + - "*" + destinations: + - namespace: stackable-airflow + server: https://kubernetes.default.svc + clusterResourceWhitelist: + - group: "*" + kind: "*" diff --git a/stacks/argo-cd/projects/stackable-operators.yaml b/stacks/argo-cd/projects/stackable-operators.yaml index 3e3e82a0..3ee37fc2 100644 --- a/stacks/argo-cd/projects/stackable-operators.yaml +++ b/stacks/argo-cd/projects/stackable-operators.yaml @@ -3,16 +3,12 @@ kind: AppProject metadata: name: stackable-operators spec: - description: Stackable operators ArgoCD Project + description: Project to manage Stackable Operators via Helm sourceRepos: - "*" destinations: - - namespace: argo-cd - server: https://kubernetes.default.svc - namespace: stackable-operators server: https://kubernetes.default.svc - - namespace: stackable-products - server: https://kubernetes.default.svc # required to deploy clusterwide resources like clusteroles etc. clusterResourceWhitelist: - group: "*" diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index d48df9dc..f9bf053f 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -11,8 +11,18 @@ stacks: manifests: #- helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/argocd.yaml - helmChart: stacks/_templates/argo-cd.yaml + # projects - plainYaml: stacks/argo-cd/projects/stackable-operators.yaml - - plainYaml: stacks/argo-cd/applications/commons-operator.yaml + - plainYaml: stacks/argo-cd/projects/airflow.yaml + # operators + - plainYaml: stacks/argo-cd/applicationsets/commons-operator.yaml + # listener is currently deployed via stackablectl since it complains if no operators are deployed... + #- plainYaml: stacks/argo-cd/applicationsets/listener-operator.yaml + - plainYaml: stacks/argo-cd/applicationsets/secret-operator.yaml + - plainYaml: stacks/argo-cd/applicationsets/airflow-operator.yaml + # products + - plainYaml: stacks/argo-cd/applications/airflow-postgres.yaml + - plainYaml: stacks/argo-cd/applications/airflow.yaml supportedNamespaces: - argo-cd resourceRequests: @@ -20,9 +30,12 @@ stacks: cpu: 2000m pvc: 20Gi parameters: - - name: stackableVersion + - name: stackableReleaseVersion description: Stackable release to be installed via Argo default: 25.3.0 + - name: demoTargetRevision + description: The target revision, HEAD or e.g. release-25.3 + default: release-25.3 - name: stackableOperatorNamespace description: Stackable namespace for the operators default: stackable-operators @@ -30,6 +43,12 @@ stacks: description: Password of the ArgoCD admin user # generated via: `htpasswd -nbBC 10 "" adminadmin | tr -d ':\n'` default: $2y$10$HhJC3pGHTlk8RyBoS39N/.wC72mdWxV2X8QS1wROUwCFxl.2tGfky + - name: airflowAdminPassword + description: Password of the Airflow admin user + default: adminadmin + - name: airflowSecretKey + description: Airflow's secret key used to generate e.g. user session tokens + default: airflowSecretKey monitoring: description: Stack containing Prometheus and Grafana stackableRelease: dev From d64a4170f5e01668b74d4e19c1945dd12452457e Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 11 Apr 2025 20:23:55 +0200 Subject: [PATCH 03/64] added spark op --- .../applicationsets/spark-k8s-operator.yaml | 42 +++++++++++++++++++ stacks/stacks-v2.yaml | 18 ++++---- 2 files changed, 53 insertions(+), 7 deletions(-) create mode 100644 stacks/argo-cd/applicationsets/spark-k8s-operator.yaml diff --git a/stacks/argo-cd/applicationsets/spark-k8s-operator.yaml b/stacks/argo-cd/applicationsets/spark-k8s-operator.yaml new file mode 100644 index 00000000..b5686b60 --- /dev/null +++ b/stacks/argo-cd/applicationsets/spark-k8s-operator.yaml @@ -0,0 +1,42 @@ +--- +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: spark-k8s-operator +spec: + # this interferes with stackablectl's templating and + # cannot really be used here properly but must be provided. + generators: + - list: + elements: + - cluster: development + template: + metadata: + name: spark-k8s-operator + spec: + project: stackable-operators + ignoreDifferences: + # mitigating: https://github.com/stackabletech/hdfs-operator/issues/626 + - group: "apiextensions.k8s.io" + kind: "CustomResourceDefinition" + jqPathExpressions: + - .spec.names.categories | select(. == []) + - .spec.names.shortNames | select(. == []) + - .spec.versions[].additionalPrinterColumns | select(. == []) + source: + repoURL: https://repo.stackable.tech/repository/helm-stable/ + targetRevision: "{{ stackableReleaseVersion }}" + chart: spark-k8s-operator + helm: + releaseName: spark-k8s-operator + destination: + server: https://kubernetes.default.svc + namespace: "{{ stackableOperatorNamespace }}" + syncPolicy: + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + automated: + selfHeal: true + prune: true diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index f9bf053f..89befdf7 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -3,31 +3,35 @@ stacks: argocd: description: TODO stackableRelease: dev - # we dont not want to install operators... + # TODO: We actually want to deploy all operators via ArgoCD, but we currently *have to* install + # operators with stackablectl. Therefore we install the internal operators via stackablectl. + # stackableOperators: [] stackableOperators: - listener + - commons + - secret labels: - argocd manifests: - #- helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/argocd.yaml - helmChart: stacks/_templates/argo-cd.yaml # projects - plainYaml: stacks/argo-cd/projects/stackable-operators.yaml - plainYaml: stacks/argo-cd/projects/airflow.yaml # operators - - plainYaml: stacks/argo-cd/applicationsets/commons-operator.yaml - # listener is currently deployed via stackablectl since it complains if no operators are deployed... + # currently deployed via stackablectl since it complains if no operators are deployed... + #- plainYaml: stacks/argo-cd/applicationsets/commons-operator.yaml #- plainYaml: stacks/argo-cd/applicationsets/listener-operator.yaml - - plainYaml: stacks/argo-cd/applicationsets/secret-operator.yaml + #- plainYaml: stacks/argo-cd/applicationsets/secret-operator.yaml - plainYaml: stacks/argo-cd/applicationsets/airflow-operator.yaml + - plainYaml: stacks/argo-cd/applicationsets/spark-k8s-operator.yaml # products - plainYaml: stacks/argo-cd/applications/airflow-postgres.yaml - plainYaml: stacks/argo-cd/applications/airflow.yaml supportedNamespaces: - argo-cd resourceRequests: - memory: 2000Mi - cpu: 2000m + memory: 10000Mi + cpu: 8000m pvc: 20Gi parameters: - name: stackableReleaseVersion From 64daa28a701a667ac240fddfdfe37c40c3f61479 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 11 Apr 2025 22:57:10 +0200 Subject: [PATCH 04/64] wip --- stacks/argo-cd/applications/airflow.yaml | 3 +- .../argo-cd/applications/sealed-secrets.yaml | 28 ++ stacks/argo-cd/manifests/airflow/airflow.yaml | 308 ++++++++++++++++++ .../airflow/sealed-airflow-credentials.yaml | 24 ++ .../argo-cd/secrets/sealed-secrets-key.yaml | 90 +++++ stacks/stacks-v2.yaml | 22 +- 6 files changed, 465 insertions(+), 10 deletions(-) create mode 100644 stacks/argo-cd/applications/sealed-secrets.yaml create mode 100644 stacks/argo-cd/manifests/airflow/airflow.yaml create mode 100644 stacks/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml create mode 100644 stacks/argo-cd/secrets/sealed-secrets-key.yaml diff --git a/stacks/argo-cd/applications/airflow.yaml b/stacks/argo-cd/applications/airflow.yaml index 4789efe9..9db9f6a7 100644 --- a/stacks/argo-cd/applications/airflow.yaml +++ b/stacks/argo-cd/applications/airflow.yaml @@ -10,8 +10,7 @@ spec: source: repoURL: https://github.com/stackabletech/demos.git targetRevision: "{{ demoTargetRevision }}" - # TODO: change to other directory - path: stacks/airflow/ + path: stacks/argo-cd/manifests/airflow/ syncPolicy: syncOptions: - CreateNamespace=true diff --git a/stacks/argo-cd/applications/sealed-secrets.yaml b/stacks/argo-cd/applications/sealed-secrets.yaml new file mode 100644 index 00000000..7faa76b4 --- /dev/null +++ b/stacks/argo-cd/applications/sealed-secrets.yaml @@ -0,0 +1,28 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: sealed-secrets +spec: + project: default + sources: + - repoURL: "registry-1.docker.io/bitnamicharts" + path: sealed-secrets + targetRevision: 2.5.9 # 0.29.0 + chart: sealed-secrets + helm: + releaseName: sealed-secrets-controller + valuesObject: + secretName: sealed-secrets-key + - repoURL: https://github.com/stackabletech/demos.git + # TODO: adapt to release + targetRevision: HEAD + path: stacks/argo-cd/secrets/ + destination: + server: https://kubernetes.default.svc + namespace: kube-system + syncPolicy: + syncOptions: + - CreateNamespace=true + automated: + selfHeal: true + prune: true diff --git a/stacks/argo-cd/manifests/airflow/airflow.yaml b/stacks/argo-cd/manifests/airflow/airflow.yaml new file mode 100644 index 00000000..703f409c --- /dev/null +++ b/stacks/argo-cd/manifests/airflow/airflow.yaml @@ -0,0 +1,308 @@ +--- +# {% raw %} +apiVersion: airflow.stackable.tech/v1alpha1 +kind: AirflowCluster +metadata: + name: airflow + namespace: stackable-airflow +spec: + image: + productVersion: 2.10.4 + clusterConfig: + listenerClass: external-unstable + loadExamples: false + exposeConfig: false + credentialsSecret: airflow-credentials + volumes: + - name: airflow-dags + configMap: + name: airflow-dags + volumeMounts: + - name: airflow-dags + mountPath: /dags/date_demo.py + subPath: date_demo.py + - name: airflow-dags + mountPath: /dags/pyspark_pi.py + subPath: pyspark_pi.py + - name: airflow-dags + mountPath: /dags/pyspark_pi.yaml + subPath: pyspark_pi.yaml + webservers: + config: + resources: + cpu: + min: 400m + max: "1" + memory: + limit: 2Gi + gracefulShutdownTimeout: 30s + roleGroups: + default: + envOverrides: + AIRFLOW__CORE__DAGS_FOLDER: "/dags" + AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" + replicas: 1 + kubernetesExecutors: + envOverrides: + AIRFLOW__CORE__DAGS_FOLDER: "/dags" + AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" + schedulers: + config: + gracefulShutdownTimeout: 30s + resources: + cpu: + min: 400m + max: "1" + memory: + limit: 1Gi + roleGroups: + default: + envOverrides: + AIRFLOW__CORE__DAGS_FOLDER: "/dags" + AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" + replicas: 1 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: airflow-dags + namespace: stackable-airflow +data: + date_demo.py: | + """Example DAG returning the current date""" + from datetime import datetime, timedelta + + from airflow import DAG + from airflow.operators.bash import BashOperator + + with DAG( + dag_id='date_demo', + schedule_interval='0-59 * * * *', + start_date=datetime(2021, 1, 1), + catchup=False, + dagrun_timeout=timedelta(minutes=5), + tags=['example'], + params={}, + ) as dag: + + run_this = BashOperator( + task_id='run_every_minute', + bash_command='date', + ) + pyspark_pi.py: | + """Example DAG demonstrating how to apply a Kubernetes Resource from Airflow running in-cluster""" + from datetime import datetime, timedelta + from airflow import DAG + from typing import TYPE_CHECKING, Optional, Sequence, Dict + from kubernetes import client + from airflow.exceptions import AirflowException + from airflow.sensors.base import BaseSensorOperator + from airflow.models import BaseOperator + from airflow.providers.cncf.kubernetes.hooks.kubernetes import KubernetesHook + import yaml + from airflow.utils import yaml + import os + + if TYPE_CHECKING: + from airflow.utils.context import Context + + class SparkKubernetesOperator(BaseOperator): + template_fields: Sequence[str] = ('application_file', 'namespace') + template_ext: Sequence[str] = ('.yaml', '.yml', '.json') + ui_color = '#f4a460' + + def __init__( + self, + *, + application_file: str, + namespace: Optional[str] = None, + kubernetes_conn_id: str = 'kubernetes_in_cluster', + api_group: str = 'spark.stackable.tech', + api_version: str = 'v1alpha1', + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.application_file = application_file + self.namespace = namespace + self.kubernetes_conn_id = kubernetes_conn_id + self.api_group = api_group + self.api_version = api_version + self.plural = "sparkapplications" + + def execute(self, context: 'Context'): + hook = KubernetesHook(conn_id=self.kubernetes_conn_id) + self.log.info("Creating SparkApplication...") + response = hook.create_custom_object( + group=self.api_group, + version=self.api_version, + plural=self.plural, + body=self.application_file, + namespace=self.namespace, + ) + return response + + + class SparkKubernetesSensor(BaseSensorOperator): + template_fields = ("application_name", "namespace") + # See https://github.com/stackabletech/spark-k8s-operator/pull/460/files#diff-d737837121132af6b60f50279a78464b05dcfd06c05d1d090f4198a5e962b5f6R371 + # Unknown is set immediately so it must be excluded from the failed states. + FAILURE_STATES = ("Failed") + SUCCESS_STATES = ("Succeeded") + + def __init__( + self, + *, + application_name: str, + attach_log: bool = False, + namespace: Optional[str] = None, + kubernetes_conn_id: str = 'kubernetes_in_cluster', + api_group: str = 'spark.stackable.tech', + api_version: str = 'v1alpha1', + poke_interval: float = 60, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.application_name = application_name + self.attach_log = attach_log + self.namespace = namespace + self.kubernetes_conn_id = kubernetes_conn_id + self.hook = KubernetesHook(conn_id=self.kubernetes_conn_id) + self.api_group = api_group + self.api_version = api_version + self.poke_interval = poke_interval + + def _log_driver(self, application_state: str, response: dict) -> None: + if not self.attach_log: + return + status_info = response["status"] + if "driverInfo" not in status_info: + return + driver_info = status_info["driverInfo"] + if "podName" not in driver_info: + return + driver_pod_name = driver_info["podName"] + namespace = response["metadata"]["namespace"] + log_method = self.log.error if application_state in self.FAILURE_STATES else self.log.info + try: + log = "" + for line in self.hook.get_pod_logs(driver_pod_name, namespace=namespace): + log += line.decode() + log_method(log) + except client.rest.ApiException as e: + self.log.warning( + "Could not read logs for pod %s. It may have been disposed.\n" + "Make sure timeToLiveSeconds is set on your SparkApplication spec.\n" + "underlying exception: %s", + driver_pod_name, + e, + ) + + def poke(self, context: Dict) -> bool: + self.log.info("Poking: %s", self.application_name) + response = self.hook.get_custom_object( + group=self.api_group, + version=self.api_version, + plural="sparkapplications", + name=self.application_name, + namespace=self.namespace, + ) + try: + application_state = response["status"]["phase"] + except KeyError: + self.log.debug(f"SparkApplication status could not be established: {response}") + return False + if self.attach_log and application_state in self.FAILURE_STATES + self.SUCCESS_STATES: + self._log_driver(application_state, response) + if application_state in self.FAILURE_STATES: + raise AirflowException(f"SparkApplication failed with state: {application_state}") + elif application_state in self.SUCCESS_STATES: + self.log.info("SparkApplication ended successfully") + return True + else: + self.log.info("SparkApplication is still in state: %s", application_state) + return False + + with DAG( + dag_id='sparkapp_dag', + schedule_interval=None, + start_date=datetime(2022, 1, 1), + catchup=False, + dagrun_timeout=timedelta(minutes=60), + tags=['example'], + params={"example_key": "example_value"}, + ) as dag: + + def load_body_to_dict(body): + try: + body_dict = yaml.safe_load(body) + except yaml.YAMLError as e: + raise AirflowException(f"Exception when loading resource definition: {e}\n") + return body_dict + + yaml_path = os.path.join(os.environ.get('AIRFLOW__CORE__DAGS_FOLDER'), 'pyspark_pi.yaml') + + with open(yaml_path, 'r') as file: + crd = file.read() + with open('/run/secrets/kubernetes.io/serviceaccount/namespace', 'r') as file: + ns = file.read() + + document=load_body_to_dict(crd) + application_name='pyspark-pi-'+datetime.utcnow().strftime('%Y%m%d%H%M%S') + document.update({'metadata': {'name': application_name, 'namespace': ns}}) + + t1 = SparkKubernetesOperator( + task_id='spark_pi_submit', + namespace=ns, + application_file=document, + do_xcom_push=True, + dag=dag, + ) + + t2 = SparkKubernetesSensor( + task_id='spark_pi_monitor', + namespace=ns, + application_name="{{ task_instance.xcom_pull(task_ids='spark_pi_submit')['metadata']['name'] }}", + poke_interval=5, + dag=dag, + ) + + t1 >> t2 + pyspark_pi.yaml: | + --- + apiVersion: spark.stackable.tech/v1alpha1 + kind: SparkApplication + metadata: + name: pyspark-pi + spec: + version: "1.0" + sparkImage: + productVersion: 3.5.2 + mode: cluster + mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py + job: + config: + resources: + cpu: + min: 500m + max: 500m + memory: + limit: 512Mi + driver: + config: + resources: + cpu: + min: 1000m + max: 1200m + memory: + limit: 1024Mi + executor: + config: + resources: + cpu: + min: 500m + max: 1000m + memory: + limit: 1024Mi + replicas: 3 + +# {% endraw %} diff --git a/stacks/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml b/stacks/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml new file mode 100644 index 00000000..bc631bc7 --- /dev/null +++ b/stacks/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: airflow-credentials + namespace: stackable-airflow +spec: + encryptedData: + adminUser.email: AgCmhs/OLHZ0j4CnTdZvfj76m+y7e62IuAeJ/LksxaTOWbHlvTJ0p8dBsp+WX3RfiuRK8M7kJDundrSKYzbkw8MISjoHF4/v4pSu6qWLBBuDPaMGh3i1bpAJ9gKUti4SGLhp7dr/Lx1oFWnY5T69WMARQqTbhHnpH70LPkFWKRvlIkCN26PbuYsE7stKXaVweEP204sfUChK5qzafE4Lcm5l+1yGFfILBBKAbBGxLX2I3O4n4LuP/iv0Ne14wGnxss80Qapt+ue8Ce16EZiJPtY+ZCXa05fYgtitVr393pYZks/fO9as76ABjWFc2lnE5GaGV58gK9V/5VRd1TBfkK3Y8UZPxqHNULlkwVh4Vy2EHHBMzhtec0m6w72aWuWFecyUNkrwIuivZ8e9tiH6aulpYqgLshnbDZCQi8yQVDNx/xLv4GH0/VvDUEIB7O13CHRL4M+OH+GYgLpJYqhTTkJNvXdrmjyK6kWeiErw8yG77akVplDGSG6LschN0DGhJVoZLLwbjvvM9em6tHu3gRSXAYhfG80bB9PMwmU9Lei+cdq6Fp7RceqxR3F6lnigP2e5gCaEpc6YYzBKEnzgcdlR1iA57D+5lBK9SwvWkppH5/dbPRkzt5+kqWaExRq8VmEnwooiBJ/4Jx4i4FKWFnx3cnDF5OYXovIKPyYPZW1HVvifrjHHZAOe1jORxYpMOwPYOLraDIO83comkVEBzCVUOh8v + adminUser.firstname: AgB3NcEnKhwMsyFPOfSTfxKFkhIg1TyeixtdqW8RSq5I6zrMX9we431QLuOSfxQlaizsm3o670wjO+4yDNpOGB4/vXaSSpA+835tiMDymJ/URR7vb1vvLV/xx6azdn5JmmHWd+yEja47TxQ1ip5KDai1Oxc0BKbgzu2Ogalq1U/56riSKSe8MvRjJ9KY8IwBpoY1rKAjTFJWavUVVt5Ms0fdSB7sL69NoKVzUQEpucTemy6k8RQFabK+KwirQ2KSDcQ5J+df/ODx1/HZKG8j81Wla7dBJ0nwIeie0StAZdZ+j9QlsWQF4zCRaEPxn9p1vken7Na/EqQdYwIV62o3mo66p06t+ObdjxKRHEO5BiY2lYqY3XcgaETBpaDfl2PiMbneLngGpBfOQSKnVZC60svqB/BpvBlMCiwcr+0ZJczFM1f+ptmJGHLJirf1kTYKvm6ezstGussB848Ca/ayhzzt9cestW9UFS/SlAo7d+N+YUKv3O+8QPIb5AdN2nPN9kcQxdh2T+ogtg3yIKsb6makP8LC6Z0Hl8C1DuzSh3Inj5VxcmkRuB9LRYymfNu+6GjkVlAWk58FUdQM5+5syMDgiHdVIMmnoTjRNm3mCm3l6g6rG33ggaAI4X1ypyjnmMlxbpRJi/+KgsQNNByYYQG1Foo3MZ7JFQojABBFMVSCPumMyjR7i8kvYCTr7acrHS3ZSjvR+Msz + adminUser.lastname: AgCYaEJ973OSMSL0vBrQcHbFtmqQls6Ids1xcPVz3/uVpxZw89Phn67Xx6dPSNHqZIYHWbZ/gV24xOJd0XbtlFINUknGgjwqjwGJHm4Q+0F1nmCVmhrRqB+Jc8I7qqb5BT6XpsgQOa0vazDc+NVKlgm2DmbIKz8VvHrhUNRb1rd33RYd3lm0Kj8lSl4QZm8tIX+BZi3J9gSLDIMzGQftg5BxS8Wt8cbYsMW8lFeZ6Dg7Fmx9r0GCeAa7kZWy8UME0saeKj13tg/oYj79MNFCu/Q510Q+YbLcQAJ22ifPgIaPdN4YK03DYL9qiIeo4im8rlgfKqHaNkfNzmXXh758T4s74Nau9KLdG+qE2+j175B9nrYjBYlT2EDoJIF6m1iiOCAc93tk4FIc3fw5RNLkOsAYdEWoECWLOdDloQsDHa+DGwHnF6d9ZpzlATnKLMOJ6F3+RAW10NzD38bWMlUUJoGZTuhVOz9tQCyFs2/SqMC8/LauMrHx2TFbKEq2kpc4MRTYwUQe9L+zQFI2Oc5HXGulXDdteShL6ayT8sXGRIsNI0lCpNhHsUauGJ7sSkW4bklGMEPzRh48+Lua2/QQKOcLWxgPx53b492/7SvWExx7rs+POSZouLdwUlngXC2D1H5yvmnPXEnnTLVGwBNDRBa+BBefocVOmq6yZGX5trkhudTr/Nu/rZQkDGUtSfiMDzd6Uo73LQ== + adminUser.password: AgA7BxW4/FggUvHMVvnYxdP6Pmju3uSskl29Lc1ywr6AbtCSqiysp29Ravyr0muT+PlEb5zmUYrct+HaffstMOudDCYw5wSm50hhelRPFgqppJgoUvXwfeUMkx73rrkddfN2RnCepUZxmkJ8PcXA38+KPwTZM4zZmOHeq7lgKDlsjgin/O0MrAeochZGJ1JmsJ0B3QUxFit4aURq4Pko67ytrWMJBO2KQRLHUm7Swf3umqjCBWuqfqhbRtKrWEOhoAEHDzKOq4fUCaV5rf2uxpvNCC0KJY6zmhPcPgZAO+Wuc+W9iNmM4ROoTFCvDS/stXcneDR7oRObgtXN5gn/0xno3xhdypg0lqfm2bEoEiokwu2GVwrzPr5iNI2HlUVhXeo9bNwjJ1alWjt0faUvwtZRSbKI1AihBF0gelbXhKT9Iyya336TTuz0riW3sFeDixHDTRoVC3ZaHzvCTXLTLtw3xgmhxM1mCebU+pxS8DV4AAIy9MfkK4ZHkjpcCdVbV5D4Mt2A9Ltm1O7qHlKsdWF0+vMmllLn7dZzAmtAiCiI9srOUh0xj+bIOqqN2Ky8a2K73JPh3kXxO5njW9N0/lFFQ4D98VpgI/DDqoZwX97yzLcthLb54xr6OUtduvX6rDakgrRZT4OCSvlGUIWzvtTcoDwuBubSuOnLh6AFNd7xz/Ik+eepInSOCMpd3HIdp5KBB3TPm9eLFcLW + adminUser.username: AgAQWpaTaFTvYsZx0p9Am3iTwoUEZ/pWiETvhVSF0NANgKe17V5b1lQj+DamMVlFUGfGgV8Qcxdyw4Lc71yzGgwEB0fQERoy54Z2lwVpYLJm2qLcXuolkzFSyZEMBBulddkTGhF/zhDZWfpOnayhfAbQyyp/T3KUuZira220DN9HiXDb5BiId1+Gzu6xTW6WD5/c+7yFiD6SWFReGpIuqHofh4y02p8A5RED2jtcr/p7ltCZ/tqjIpnZcTP5K7wauEjDiJmDbLTcdAmzdSEOI4Q952+oK3xjPAIrhhivC3DAtQ0lgV2ye9859tx+CDEEkLkjewascVNwmX1yeYPxnh8ayJfabkEpNM8KOfmrDE/AZB82FeKChmvAcpQAu3fUuxBMyUit6RkNW73fvDf8hIyOmNSZdr0f21FPhJWFYnsF4Qfm3zOo/fXuG1LJz2WB2Qy7RuTmRGW1AE4VQOvzvVDCEOq5BsVfPdlGrSfo5oeI6ctCqTL4S+2CXS3VRUkIX61QdN54CU7ZeT4cAzZGpkUP6elfsrLv0KnQc4KvvzfBNu71j3xgjjYN41aGDNi8dqWaCVebYbuQtDqbcxLxUQNX+VqBdCflMcnCOC4kAEI+B2n/rUrGLWjKUIMhZvZIKJQd437j1fvJo2AHO8TbqtORspg3/H8OuwZpwcDsWl1dVQ1B0EczZweo3I5NkytTVxNEEXjsQA== + connections.celeryBrokerUrl: AgBEl833Xp1UogzJpkpJ1zhjlw6KKAqcfWzSqQ8LIPQ96GaIUfo6G8wx7jozw7yEF0vnAuTTdzdEZ7x8/PVwWQyqNJKiS9o6mIEdkhv9mQ+EG/wzpB8dnbQR4Xl758f+dPRqbIA0RYetmcD9NsiF08obXIIzZA+G95kR6x24WiixwJL+txMNdno0Qu92fRqasv6jV22CUju/kUTRFEov5IrzGOj7gLKQ7d2yrCGZd/W9SnP9sgX1ltz78C2yiEXX7aI0n066OjJUxE7DFTJCUWNrknDcJuSjGFrWTzf8GNydrPmkVp6g5JA3fGgUTm76osischHr/0/+cwzQzEyQKyb5H/fxcAd/5yqtXhD/xibPDoVHs484PQAmgy8M3PMG8fNQ9XyrXNB/ZuxcCnt+rqPkeP0jqEsdxog2Q3U1lXfH/FU2eJr5tLHb1zNISV7vNX28VRzLoefyYFtHkp1F2yVYThmuSCPj1cqkfOpt02rYsIDIYqwryxxOvUpTAH0uNbdoIWSvXTzX2ZARUVVm0bboDgfe4RxVQofxKjk1mNPHhEZ9jYd6GX1G1SCYuSVdrRN4fLSOxlyP+vPkyzOJtQ7WmugBZFTmZ/iUN38ce4HbDIajVKtMvDqD35kkN/q4NZ3d+/YjmZA9YvJ03Hd0XNznImr8R4wqOt5oZsK2+XaMxHTzPfxV2MpLT9Xgb+CAHiGKvCaheKdRNWODhFQQXAw8l6t4z8dvYNkZPAZAD6ehFUtlvpTbHVVqtEdiTA== + connections.celeryResultBackend: AgBywQulzF0AZOPR8ypHDnsQIicOWUri9rV2sLwR8kiD+J98eT6CWvblzUv8x3/I4gJWwHbwjmACv5Ke+MD1GxqVsJYl2zc/A78SkVvNT6V+kaUFqxXMEQ3EJN7q+BHUPFphxQDS2Ur5+ssON1hth56L+HAxZLmt4XC9Fjz20cO7Dcx+vpgXG7KVtdRmLvchs0PCaMnMZDANg/rgGC9lMcgi3BROhobGapbp3+iLjiIDtNJ4SwwQpQaA5KWKXqC8Dmfvtm18lyNAK5oWSrMQI6+eWe+ML56C8bMlz3jtlZsFO6upe7eOueD1WoK1+YI+NEfvw3HDChgMaQ559x3y/pIG94F7QNtltZbfTfXptStQru+H2rxmCUBYb/BaHv3oqZsOgQe95LubQNfrpH9tIIygGZcQDZGjZsnfjmu1ahYzqiKb6UEoPxZJskIMy5BAakWQf0FiVJQa7stt7/k/NosBrUfJbyPJNWA6xVwilV7K4Auz4MuBsMk1fRofgpw3N85SeipVmU3673GIs+L1IFy8UEYAtYpv14aFOvqJrE4YBpkufr+K/IvVpYfvIBL2zAF3CKm9SEzYAQnADwb3UgOCIm8w233vogjHo5oqL8wacUeZgzrZ+T1FIR9e1tXrYxk2VLmyv5d0VOWYe0AleUk+O3rVdLpi5yUJQjvKnExVXzMMLsApTJ3dy5OHwou99Po1mlg2D6xh439hDUlAcUQz4NfGYvJtYSgAb5cjMs3bEMUcAuadGIGidlIy74HQArfXosundworwKC8 + connections.secretKey: AgBpwR5AZJUjGxJwi7rS6dLH7WJKYza4wKjPxLF+RVskwtwtkGtYSpbRZ+vXxE6pn47eJQC0OpJzreiNqYZtSR1CHv6HyHZsnqCSnLSzlUBCPp9H51nfVQ57V+S967/jEVJ3V7446gqQqdJb3IS4uwbb+G8CJsjsZiEcwx/z/OK5vu+k7TiG5jiyksxXhC7EFzYKfc/CD8Lsr6LBimUEa2i1W/PhOoDcqKa7qvDMIM/iAEilvtHXjA+EU8kxplDYy2an1cadockyyMSoHvDmSOg6BU62bK8Xqx9o0X6uaSrTufeueymtBYb/wVHB0bTxm1UXb8QdGYfDePemnP0oRNwvETrupuD0HRgEutetyxLYEEezC98ZpRM377kirX9Zqb0/fZ9YVwECBL7hznl/sp8w1hE2O76orf3iAGNMftrFtOjEyzro+41zd+bz+sy1ivGluIWwovQbgondk25AoYRC0LhGDl7kQDVX3ND/XeH2NYErAuCEyjQ1MGHLJxQGGDxoCPaDXtLMuHvXlg3H71jibzT7rV+YikAExjfUVim0kuDtKqdBwAhvRqX+Si/kCRMrIUhRowWSqx0g54U9mwC6wfbSHV90RnYtvBPbFc144WbgV7irc5bfnekidrCj8MiIPKegUByqlB26ziyexEmiRN74BH2yCJ/M2yf1IWSad0uqb5d/48BcgEP8E2BOgeeaCJ6WOMeR80qvas07hN5e + connections.sqlalchemyDatabaseUri: AgAFm5FacxXw3q9rxKT+9eTpY3FPf7NSp3wdPUUzkBpQPsEa0JvafDeIZ5/+MC9WaFBWuSt0awrIh/FyHlNW4/KRhTWzFL4h2MrDAfCEdCaOgB4wrgYJF7tGz3968eNhBn0ALX7MU6dgiGawPGvX91FgPo4dwz/5Ob5+f5MhA/paH8uBXR8W96cTTeJlcJaqLV/xMzGCInurUpOwyrLWu8zjOHNdn/11P3n5gKVlwbi6w57AKAYtqux869oh9al4kIbgIEbRYRI42Q2Xa3v+RtmFCX3CelCs6uoWtzNgt+wUbQbfi48MuHrakiJf49yrqFHdc2vfj03O8jlKLGSGcSsy5Epbb1VBuTh04jVXAO03xb6Top3gqZvxY8/gWOWPs0SejeilvubNIwj40bhzRxPj+r7TnE2dmS+KnTN1fYR1fk39C6SNjSKVncMie8PH/xfq8WOVZSGhzlIwYNMs7/WpXYLVs58Wz+pVVRcbYf/CQjRNKy4ikEf+eHwMLvmsvHnslxr9S9Bwt2An7mFkkO0VvGubCYfOK/W1SoN9NBhFmJO6Es3vfTKuS1gqgcJAzd1oz0+hz6Kno8XRpZUzdUa+P8LPo4Y+mhif+y9OH0sXFH5rYGWvLbUxEgFfdUMWQRKpGRqJiGNKFBLo0kpRt/DqG3v9wcWHbYLl+/cRLCsC46FUsPW5L73T4AEtV3TLh6Q5hZiHCWTEyxW4WyeFD6Bd7FLPNbIA8gMD0oCP3ztd+odbSDecURUzhQifICH/SCJl4VH8jvr40Rd+TjrqEj5t + template: + metadata: + creationTimestamp: null + name: airflow-credentials + namespace: stackable-airflow + type: Opaque diff --git a/stacks/argo-cd/secrets/sealed-secrets-key.yaml b/stacks/argo-cd/secrets/sealed-secrets-key.yaml new file mode 100644 index 00000000..60e7e522 --- /dev/null +++ b/stacks/argo-cd/secrets/sealed-secrets-key.yaml @@ -0,0 +1,90 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: sealed-secrets-key +stringData: + tls.crt: | + -----BEGIN CERTIFICATE----- + MIIFEzCCAvugAwIBAgIUTIeBhvXRk1w2QJRP2RpyQWJCB48wDQYJKoZIhvcNAQEL + BQAwGTEXMBUGA1UEAwwOc2VhbGVkLXNlY3JldHMwHhcNMjUwNDExMTk1NzUzWhcN + MjYwNDExMTk1NzUzWjAZMRcwFQYDVQQDDA5zZWFsZWQtc2VjcmV0czCCAiIwDQYJ + KoZIhvcNAQEBBQADggIPADCCAgoCggIBAMobRqdxhAE8Nr45tCRRI6kz2WopwnKb + 8bWGM0LbM/p04xfcLLAVzK4nC5HHBmfPycz6XyGjv7ViUSaHVk1j3OLxFEX4sd+2 + JoTQRenRc/oUHdwaGskBr5frXFAO3Tiou+pJ6Tu8ewCeCvGUzB9HtiRq5pCGPu6M + wzXbmR6GKaCJTu80LZ8fwFdSWI0gDoqeWffCTfcTgJfTa+DgMfTM3zzUNwJVfLo3 + r596wGmlxKfMcUtzw2J5iLr0dXJuJnlmA0gU31kwVcf8Wo1DAdh8hhxmUAkmGss8 + AktmB8OvvNygVVzzwmX/L2MH1zPXzgM9sYenkJwLSZhMx/uyfgX5x54/QaC7lwfk + wPnp52pK0JA2VmfFBsvlnPsVOWgwWU0jUtMGBtKlu/OonELAGaffbkhjD3eDe0B3 + X0gwtju+T4WCzQmgTFyXFdp18/S2rz19TBrq2XponRRpcXBo4qe8P59Eq++NNjAg + rYymFcTfrDZqMIt3jBv3jSnLlAnwg/H1neRfgx6kxXqxApF+vMDD7VVDaLYnLbFV + lwdkWxPrHStk05eBsLu/CkGv2ykkUAa0qqg/Dm/bkOl5lZlauVFC9wuiLh2cHgZr + tBpGuYlMFMRvx7A0+IBUTjTQswBF1+mBStUyBThOw1aJYbE2FCWj+lGrMX8qr+ad + sKsS+5DjOwCVAgMBAAGjUzBRMB0GA1UdDgQWBBRFs1lkbmy+f7TN10DGuHLa75sj + BDAfBgNVHSMEGDAWgBRFs1lkbmy+f7TN10DGuHLa75sjBDAPBgNVHRMBAf8EBTAD + AQH/MA0GCSqGSIb3DQEBCwUAA4ICAQA+6nknVdXMATXxbyk0E5fDlM/D2aceDw6D + 3kydc6wBT6k9SkhxFtrCI0IBO0f067ppFPj4ApJI0mrh3dsNiAzjqGetlGCqXjKq + pwdVxWYrmn/ELTKgvxToQG4J4PMUeo7tlhviU5HJo90DroDpoZHALubl+XsjzspA + MHH94CiF24z2nxoVolVa1qqq7WDlMWhZuit3WDo+jIgrpBgcPegNrZqrZqt/xlzM + Q91iOwwd3ykfo+xLdNuG1i8Qm1UFOEmXsustHBPDCtcI/ddLda4NbZtrNeZVmDbE + Px4Zzsjrbd6IKhVbdD5E4d9K/XMNsBIl4sidqdaUMhkNTYXkI/a5dTFVlFprLFnM + yB7A6OplQPp+LTITXW2IvD/eZetdl5073X0OQtuJrsm4mAJZ74sqXgcyVs7bGPIw + aXt1ttNH0qA5lvbp0cOIdXmpSWQfDRvlAExqX4p86549J3GMhIsVCMw5KsqduN5e + rSi0HCi24S8CeXiYV1phkSuxf0sROPuRb8SvY/6qcb73DMEMQx3zPWLV/+pG7DcG + uD327xvb3uJG0kFLi4MDnoCHLHYZeoPtwIEwG+1LjwHj+oKn12wlKJvYTxWALi4k + tKVW3QYL8beLaI3XfF+Vd3kCVpU6N9aRt3l+PpzkE1VCkayz67Sf5dJy88YIB7o4 + A08j/AH+GA== + -----END CERTIFICATE----- + tls.key: | + -----BEGIN PRIVATE KEY----- + MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDKG0ancYQBPDa+ + ObQkUSOpM9lqKcJym/G1hjNC2zP6dOMX3CywFcyuJwuRxwZnz8nM+l8ho7+1YlEm + h1ZNY9zi8RRF+LHftiaE0EXp0XP6FB3cGhrJAa+X61xQDt04qLvqSek7vHsAngrx + lMwfR7YkauaQhj7ujMM125kehimgiU7vNC2fH8BXUliNIA6Knln3wk33E4CX02vg + 4DH0zN881DcCVXy6N6+fesBppcSnzHFLc8NieYi69HVybiZ5ZgNIFN9ZMFXH/FqN + QwHYfIYcZlAJJhrLPAJLZgfDr7zcoFVc88Jl/y9jB9cz184DPbGHp5CcC0mYTMf7 + sn4F+ceeP0Ggu5cH5MD56edqStCQNlZnxQbL5Zz7FTloMFlNI1LTBgbSpbvzqJxC + wBmn325IYw93g3tAd19IMLY7vk+Fgs0JoExclxXadfP0tq89fUwa6tl6aJ0UaXFw + aOKnvD+fRKvvjTYwIK2MphXE36w2ajCLd4wb940py5QJ8IPx9Z3kX4MepMV6sQKR + frzAw+1VQ2i2Jy2xVZcHZFsT6x0rZNOXgbC7vwpBr9spJFAGtKqoPw5v25DpeZWZ + WrlRQvcLoi4dnB4Ga7QaRrmJTBTEb8ewNPiAVE400LMARdfpgUrVMgU4TsNWiWGx + NhQlo/pRqzF/Kq/mnbCrEvuQ4zsAlQIDAQABAoICAB2p4TXkWF6iCtrit0j5S8Wi + 4Y9Ob8bIkrJ07wMqDzf8ffRbkYeKu474Fh+gWZVVDyZxvRK/6PxjYMvJ42XaeWop + vjMC7ICMSJ0HwKsv3djfqP5MKpBEGhSvK85oUBnn0iTyEjR9VoRhrfOFRmx05wvz + UMH3ojb7HA/EmnW7dVKltJsxSlznmSB36p1UDi5UTZoqv6y7BQ3DrzrDTTHD/CP+ + ddB24aanU7SRnLok5XdHb32GkS/b7LCb/rz749O/oE2tGpcJnB2NMXE4W4X9yOVU + Mv0Uxav8s82DPTKAkbTEJ6NL98VmrfrYBMMMMqWRIYRqmLP+iB+9bUDJEuwnnB+g + HQ+W9MliiKfpVx3LpcwaT68SNLSoSHt+hHcd0b8iccyvpxag7LExe36NEmg5Xvn7 + NfTcsCeRrbFW7NWmJnS0yZ5vNpf1K6Y9MVPLlNLOYL61QJAVf5EKQxxT66xvFjE6 + L068mFAn1w0vreXIf/Z7eS+A5nU3EBHigdNJO+EjrJ9Myr5fCJewE6tCptTUdkep + ahC2VvWj7d5JOmGVgdvO7/lb2N1Zv8Z7M3aj2tj1xLIeGh5/T/GaGBn56CZpoRtl + i76E08ehrXoKVZM+ej4awmipqkW3x3qDXFTP0Khdrr0FvFptT4/kJ7rNFnrHbNhC + 9fQsHMzxohtlq285hehFAoIBAQDldt9FiaWJr1YTcaaCopJO3VR39M+Ap+Nc3LCm + G320DMzBV310q8zjnbpQ72wCm8zOV8ZRoGLfSEI0cJBsw/w0fkPK3FqPdHosUQte + 4TvYqhz7qNV4fMqgqeFgwPXLzfKjeJwd1GSyBt6+LtL6vIVXsbiFa04wTi3Zal0S + aiKMt2l2qLC1RjIIQC/go3+Be5n4r/DsmPuIKXydsBJbmYxwIp8zGxE5SqEle66i + kg8CXm3TzMnydaSXW69s4/E4TsNdsdoUMEVFX9TR9QaYBGbYa4uBgblxnwapmoCG + 6/Qx72Wwn6jclnIJ6O4oCxRwt4+PFQoB1iggqYXvUVka7pdrAoIBAQDhen7s+u5/ + Gmkna0Z2gxwpUleEAeoqAPEab5HEZYvUfQFpmsCvavSLVH7I72HmXwxox41a634U + t19JGiPajRVy3d7NWJwT775Rr3JvIPOxJz7z0Erthvovt+usmybjM0GUAygcL3KX + NI7NIE/20+FrE1t5KydSDpnwGDjUOP3qke2a2LH3gkxye98zUiSP1beS9ZtND0t0 + tb15H6kE0RzKzoDr3g0H96uIYN3f/u/vd0QcoeUbeZ4zAfFT13FLGkTzM9hryi+c + u2GlNx8jlxjEdP1WQ3R16F2wiNUDXEO0rIJ39yJxq4Q+HitHs1II0ku8Zsl4j2ji + 4NjDWnVlGsf/AoIBACq3PkkLnq1K3pMwDPc0nFfiC99oy1f/6OtYieniJDxoRZWY + W9pkQPf+XLjEWS8DIdUnsdtwJ/e409o/OiOFOGXtO2GAM7uQy02EqME43JMw18Rh + YZvUX0QZW8NPKHKQHCMiSK2k5Dnf3Yc54Dyc65Wvl9H+b8e+Rfq7Oig7acMYQaG9 + NWgJcTbn/AGtbgcEo10QI6FnuStXzcLtC/3RT7twNoKk/0fTBLydkZzh47llKrzK + 2q0wCTvD7Zwvsq/wuZ2vhhFMCxknBc8v9dJGv356RrYNGCB2oh7gjByRwBurnRqM + HjsyJniBzBPYPC3fffH80KgW3dF2vWAXnKbO05ECggEBAN2KBfvB8WNhB8B7XphC + 6gCPrv09BF6Q3lQjGw3k2CbHTAP+0SOJ6wya0JeM3JwE96RbrT+P8ilYon6o+GFc + DZ728FrCnVfZTx3jTIRz0/xTmz7jgTswS9Fm3GCTcPn6+ov7LgXDeMuoEEbrYKYc + OHeEZXQAOnodbpLAudWKybEYAGUMVI/jrkkt/HfoVZQdYGk8eIKWbIUXrDq+KMta + yvdZPsKBQjx51EgDJP1Y91ZJ0NpxFzKPpChp2DuQivhHr0dlwVbnKTDNy3sRVb8a + TnN4nxGK5XqT/LnfN6w9kyiJ9wb4Axe8dgqyicPBJPEV7uQF+h5CicOhOFl5H69f + SPECggEAFQUHIFV+GqiBwoJv7PESfHWiCwmCNWsA5q1lRLhIiWHJ8veIXAizzYGj + B+AWnMIccF3KLPvX++7o7qY4kkhAjob6AoulRg49C/g3YtuuxPe3CI7MV12nOHKc + ltzRO/CZbcg7LMNoPDrsmVoxYsN91pAg7hH83MNLjEiHhsJMKW5MXcnV+ZEiSukK + gYEyN7T2sI7sGQYEYERfFybOyqNw9RjojHKFEAlfrVa+rDbpU837AATcd4+E+aav + iwVBz6BCvn+ozhvmu2ZkOminlve3Bfnr7eJ5p8NHBwB9ELscKBZK2SNcKlu3wtg4 + w6IEdTFRspEvNwyrTH9geMzYmqbIsA== + -----END PRIVATE KEY----- diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 89befdf7..6fa318c9 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -1,7 +1,7 @@ --- stacks: argocd: - description: TODO + description: Deploying Stackable Demos with ArgoCD stackableRelease: dev # TODO: We actually want to deploy all operators via ArgoCD, but we currently *have to* install # operators with stackablectl. Therefore we install the internal operators via stackablectl. @@ -12,20 +12,32 @@ stacks: - secret labels: - argocd + - sealed secrets manifests: - helmChart: stacks/_templates/argo-cd.yaml + ################################ # projects + ################################ - plainYaml: stacks/argo-cd/projects/stackable-operators.yaml - plainYaml: stacks/argo-cd/projects/airflow.yaml + ################################ # operators + ################################ # currently deployed via stackablectl since it complains if no operators are deployed... #- plainYaml: stacks/argo-cd/applicationsets/commons-operator.yaml #- plainYaml: stacks/argo-cd/applicationsets/listener-operator.yaml #- plainYaml: stacks/argo-cd/applicationsets/secret-operator.yaml - plainYaml: stacks/argo-cd/applicationsets/airflow-operator.yaml - plainYaml: stacks/argo-cd/applicationsets/spark-k8s-operator.yaml - # products + ################################ + # prerequisites + ################################ + - plainYaml: stacks/argo-cd/applications/sealed-secrets.yaml - plainYaml: stacks/argo-cd/applications/airflow-postgres.yaml + ################################ + # products + ################################ + # via argo - plainYaml: stacks/argo-cd/applications/airflow.yaml supportedNamespaces: - argo-cd @@ -47,12 +59,6 @@ stacks: description: Password of the ArgoCD admin user # generated via: `htpasswd -nbBC 10 "" adminadmin | tr -d ':\n'` default: $2y$10$HhJC3pGHTlk8RyBoS39N/.wC72mdWxV2X8QS1wROUwCFxl.2tGfky - - name: airflowAdminPassword - description: Password of the Airflow admin user - default: adminadmin - - name: airflowSecretKey - description: Airflow's secret key used to generate e.g. user session tokens - default: airflowSecretKey monitoring: description: Stack containing Prometheus and Grafana stackableRelease: dev From 15612f7eb363274e932fbf168bbc31bfb6faa92c Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 11 Apr 2025 22:59:22 +0200 Subject: [PATCH 05/64] fix sealed secret location --- stacks/argo-cd/applications/sealed-secrets.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/argo-cd/applications/sealed-secrets.yaml b/stacks/argo-cd/applications/sealed-secrets.yaml index 7faa76b4..27310ea5 100644 --- a/stacks/argo-cd/applications/sealed-secrets.yaml +++ b/stacks/argo-cd/applications/sealed-secrets.yaml @@ -15,7 +15,7 @@ spec: secretName: sealed-secrets-key - repoURL: https://github.com/stackabletech/demos.git # TODO: adapt to release - targetRevision: HEAD + targetRevision: spike/argocd-demo path: stacks/argo-cd/secrets/ destination: server: https://kubernetes.default.svc From e81624d7042b0f1addbe19fe5281b39de37ec07c Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 11 Apr 2025 23:03:13 +0200 Subject: [PATCH 06/64] fix demo branches --- stacks/argo-cd/applications/sealed-secrets.yaml | 2 +- stacks/stacks-v2.yaml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/stacks/argo-cd/applications/sealed-secrets.yaml b/stacks/argo-cd/applications/sealed-secrets.yaml index 27310ea5..aef6042a 100644 --- a/stacks/argo-cd/applications/sealed-secrets.yaml +++ b/stacks/argo-cd/applications/sealed-secrets.yaml @@ -15,7 +15,7 @@ spec: secretName: sealed-secrets-key - repoURL: https://github.com/stackabletech/demos.git # TODO: adapt to release - targetRevision: spike/argocd-demo + targetRevision: "{{ demoTargetRevision }}" path: stacks/argo-cd/secrets/ destination: server: https://kubernetes.default.svc diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 6fa318c9..47080633 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -51,7 +51,8 @@ stacks: default: 25.3.0 - name: demoTargetRevision description: The target revision, HEAD or e.g. release-25.3 - default: release-25.3 + #default: release-25.3 + default: spike/argocd-demo - name: stackableOperatorNamespace description: Stackable namespace for the operators default: stackable-operators From 822dee502614ec04d8e040665f5101da40fb7416 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 11 Apr 2025 23:05:02 +0200 Subject: [PATCH 07/64] fixes --- stacks/argo-cd/applications/airflow.yaml | 2 +- stacks/argo-cd/applications/sealed-secrets.yaml | 3 +-- stacks/stacks-v2.yaml | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/stacks/argo-cd/applications/airflow.yaml b/stacks/argo-cd/applications/airflow.yaml index 9db9f6a7..dcd20380 100644 --- a/stacks/argo-cd/applications/airflow.yaml +++ b/stacks/argo-cd/applications/airflow.yaml @@ -9,7 +9,7 @@ spec: server: https://kubernetes.default.svc source: repoURL: https://github.com/stackabletech/demos.git - targetRevision: "{{ demoTargetRevision }}" + targetRevision: "{{ demoReleaseVersion }}" path: stacks/argo-cd/manifests/airflow/ syncPolicy: syncOptions: diff --git a/stacks/argo-cd/applications/sealed-secrets.yaml b/stacks/argo-cd/applications/sealed-secrets.yaml index aef6042a..145df9de 100644 --- a/stacks/argo-cd/applications/sealed-secrets.yaml +++ b/stacks/argo-cd/applications/sealed-secrets.yaml @@ -14,8 +14,7 @@ spec: valuesObject: secretName: sealed-secrets-key - repoURL: https://github.com/stackabletech/demos.git - # TODO: adapt to release - targetRevision: "{{ demoTargetRevision }}" + targetRevision: "{{ demoReleaseVersion }}" path: stacks/argo-cd/secrets/ destination: server: https://kubernetes.default.svc diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 47080633..f8a7d0bc 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -49,7 +49,7 @@ stacks: - name: stackableReleaseVersion description: Stackable release to be installed via Argo default: 25.3.0 - - name: demoTargetRevision + - name: demoReleaseVersion description: The target revision, HEAD or e.g. release-25.3 #default: release-25.3 default: spike/argocd-demo From 4bf262686fed388a6aecfbed1d9336109b2e0bdb Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 11 Apr 2025 23:17:37 +0200 Subject: [PATCH 08/64] add role and binding for airflow / spark --- .../airflow/airflow-spark-clusterrole.yaml | 36 +++++++++++++++++++ .../airflow-spark-clusterrolebinding.yaml | 13 +++++++ 2 files changed, 49 insertions(+) create mode 100644 stacks/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml create mode 100644 stacks/argo-cd/manifests/airflow/airflow-spark-clusterrolebinding.yaml diff --git a/stacks/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml b/stacks/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml new file mode 100644 index 00000000..66abed2f --- /dev/null +++ b/stacks/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: airflow-spark-clusterrole +rules: + - apiGroups: + - spark.stackable.tech + resources: + - sparkapplications + verbs: + - create + - get + - list + - apiGroups: + - airflow.stackable.tech + resources: + - airflowdbs + verbs: + - create + - get + - list + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - watch + - list + - apiGroups: + - "" + resources: + - persistentvolumeclaims + verbs: + - list diff --git a/stacks/argo-cd/manifests/airflow/airflow-spark-clusterrolebinding.yaml b/stacks/argo-cd/manifests/airflow/airflow-spark-clusterrolebinding.yaml new file mode 100644 index 00000000..1f9e1b5d --- /dev/null +++ b/stacks/argo-cd/manifests/airflow/airflow-spark-clusterrolebinding.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: airflow-spark-clusterrole-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: airflow-spark-clusterrole +subjects: + - apiGroup: rbac.authorization.k8s.io + kind: Group + name: system:serviceaccounts From b1e33bc706814e876c3be6e305971ae57d11b580 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Fri, 11 Apr 2025 23:19:02 +0200 Subject: [PATCH 09/64] remove ns --- stacks/argo-cd/manifests/airflow/airflow.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/stacks/argo-cd/manifests/airflow/airflow.yaml b/stacks/argo-cd/manifests/airflow/airflow.yaml index 703f409c..a3399671 100644 --- a/stacks/argo-cd/manifests/airflow/airflow.yaml +++ b/stacks/argo-cd/manifests/airflow/airflow.yaml @@ -4,7 +4,6 @@ apiVersion: airflow.stackable.tech/v1alpha1 kind: AirflowCluster metadata: name: airflow - namespace: stackable-airflow spec: image: productVersion: 2.10.4 @@ -66,7 +65,6 @@ apiVersion: v1 kind: ConfigMap metadata: name: airflow-dags - namespace: stackable-airflow data: date_demo.py: | """Example DAG returning the current date""" From f8ab04351e989238abe1354955d8d1f5810ca2c5 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sat, 12 Apr 2025 01:19:00 +0200 Subject: [PATCH 10/64] test minio --- stacks/argo-cd/applications/minio.yaml | 44 +++++++++++++++++++ stacks/argo-cd/manifests/airflow/airflow.yaml | 14 +++--- stacks/argo-cd/projects/minio.yaml | 14 ++++++ stacks/stacks-v2.yaml | 4 +- 4 files changed, 68 insertions(+), 8 deletions(-) create mode 100644 stacks/argo-cd/applications/minio.yaml create mode 100644 stacks/argo-cd/projects/minio.yaml diff --git a/stacks/argo-cd/applications/minio.yaml b/stacks/argo-cd/applications/minio.yaml new file mode 100644 index 00000000..76ec47d3 --- /dev/null +++ b/stacks/argo-cd/applications/minio.yaml @@ -0,0 +1,44 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: minio +spec: + project: minio + destination: + server: https://kubernetes.default.svc + namespace: minio + source: + repoURL: https://charts.min.io/ + targetRevision: 5.4.0 # RELEASE.2024-12-18T13-15-44Z + chart: minio + helm: + releaseName: minio + valuesObject: + additionalLabels: + stackable.tech/vendor: Stackable + podLabels: + stackable.tech/vendor: Stackable + rootUser: admin + rootPassword: adminadmin + mode: standalone + persistence: + size: 10Gi + buckets: + - name: demo + policy: public + resources: + requests: + cpu: 1 + memory: 2Gi + service: + type: NodePort + nodePort: null + consoleService: + type: NodePort + nodePort: null + syncPolicy: + syncOptions: + - CreateNamespace=true + automated: + selfHeal: true + prune: true diff --git a/stacks/argo-cd/manifests/airflow/airflow.yaml b/stacks/argo-cd/manifests/airflow/airflow.yaml index a3399671..ce4975ad 100644 --- a/stacks/argo-cd/manifests/airflow/airflow.yaml +++ b/stacks/argo-cd/manifests/airflow/airflow.yaml @@ -37,14 +37,16 @@ spec: gracefulShutdownTimeout: 30s roleGroups: default: - envOverrides: + envOverrides: &envOverrides AIRFLOW__CORE__DAGS_FOLDER: "/dags" AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" + AIRFLOW_CONN_MINIO_DEFAULT: "s3://admin:adminadmin@minio.minio.svc.cluster.local:9000?extra=%7B%22aws_access_key_id%22%3A%22admin%22%2C%22aws_secret_access_key%22%3A%22adminadmin%22%2C%22endpoint_url%22%3A%22https%3A%2F%2Fminio.minio.svc.cluster.local%3A9000%22%7D" + AIRFLOW__LOGGING__REMOTE_LOGGING: "True" + AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER: s3://demo/airflow-task-logs/ + AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID: minio_default replicas: 1 kubernetesExecutors: - envOverrides: - AIRFLOW__CORE__DAGS_FOLDER: "/dags" - AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" + envOverrides: *envOverrides schedulers: config: gracefulShutdownTimeout: 30s @@ -56,9 +58,7 @@ spec: limit: 1Gi roleGroups: default: - envOverrides: - AIRFLOW__CORE__DAGS_FOLDER: "/dags" - AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" + envOverrides: *envOverrides replicas: 1 --- apiVersion: v1 diff --git a/stacks/argo-cd/projects/minio.yaml b/stacks/argo-cd/projects/minio.yaml new file mode 100644 index 00000000..aaa033ba --- /dev/null +++ b/stacks/argo-cd/projects/minio.yaml @@ -0,0 +1,14 @@ +apiVersion: argoproj.io/v1alpha1 +kind: AppProject +metadata: + name: minio +spec: + description: Project to manage Minio via Helm + sourceRepos: + - "*" + destinations: + - namespace: minio + server: https://kubernetes.default.svc + clusterResourceWhitelist: + - group: "*" + kind: "*" diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index f8a7d0bc..35e45149 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -13,6 +13,7 @@ stacks: labels: - argocd - sealed secrets + - airflow manifests: - helmChart: stacks/_templates/argo-cd.yaml ################################ @@ -20,6 +21,7 @@ stacks: ################################ - plainYaml: stacks/argo-cd/projects/stackable-operators.yaml - plainYaml: stacks/argo-cd/projects/airflow.yaml + - plainYaml: stacks/argo-cd/projects/minio.yaml ################################ # operators ################################ @@ -34,10 +36,10 @@ stacks: ################################ - plainYaml: stacks/argo-cd/applications/sealed-secrets.yaml - plainYaml: stacks/argo-cd/applications/airflow-postgres.yaml + - plainYaml: stacks/argo-cd/applications/minio.yaml ################################ # products ################################ - # via argo - plainYaml: stacks/argo-cd/applications/airflow.yaml supportedNamespaces: - argo-cd From 3d8e66429512c9e4a87e195080cc13acd78ef427 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sat, 12 Apr 2025 01:22:46 +0200 Subject: [PATCH 11/64] fix sync policy --- stacks/argo-cd/applications/minio.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/stacks/argo-cd/applications/minio.yaml b/stacks/argo-cd/applications/minio.yaml index 76ec47d3..d78139a0 100644 --- a/stacks/argo-cd/applications/minio.yaml +++ b/stacks/argo-cd/applications/minio.yaml @@ -36,9 +36,9 @@ spec: consoleService: type: NodePort nodePort: null - syncPolicy: - syncOptions: - - CreateNamespace=true - automated: - selfHeal: true - prune: true + syncPolicy: + syncOptions: + - CreateNamespace=true + automated: + selfHeal: true + prune: true From 150fd766a3437d1c135493296689e4f0d028afad Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sat, 12 Apr 2025 01:58:24 +0200 Subject: [PATCH 12/64] testing --- stacks/argo-cd/applications/minio.yaml | 31 +- stacks/argo-cd/manifests/airflow/airflow.yaml | 2 +- stacks/argo-cd/manifests/minio/minio.yaml | 712 ++++++++++++++++++ 3 files changed, 716 insertions(+), 29 deletions(-) create mode 100644 stacks/argo-cd/manifests/minio/minio.yaml diff --git a/stacks/argo-cd/applications/minio.yaml b/stacks/argo-cd/applications/minio.yaml index d78139a0..05d27e88 100644 --- a/stacks/argo-cd/applications/minio.yaml +++ b/stacks/argo-cd/applications/minio.yaml @@ -8,34 +8,9 @@ spec: server: https://kubernetes.default.svc namespace: minio source: - repoURL: https://charts.min.io/ - targetRevision: 5.4.0 # RELEASE.2024-12-18T13-15-44Z - chart: minio - helm: - releaseName: minio - valuesObject: - additionalLabels: - stackable.tech/vendor: Stackable - podLabels: - stackable.tech/vendor: Stackable - rootUser: admin - rootPassword: adminadmin - mode: standalone - persistence: - size: 10Gi - buckets: - - name: demo - policy: public - resources: - requests: - cpu: 1 - memory: 2Gi - service: - type: NodePort - nodePort: null - consoleService: - type: NodePort - nodePort: null + repoURL: https://github.com/stackabletech/demos.git + targetRevision: "{{ demoReleaseVersion }}" + path: stacks/argo-cd/manifests/minio/ syncPolicy: syncOptions: - CreateNamespace=true diff --git a/stacks/argo-cd/manifests/airflow/airflow.yaml b/stacks/argo-cd/manifests/airflow/airflow.yaml index ce4975ad..dc155030 100644 --- a/stacks/argo-cd/manifests/airflow/airflow.yaml +++ b/stacks/argo-cd/manifests/airflow/airflow.yaml @@ -40,7 +40,7 @@ spec: envOverrides: &envOverrides AIRFLOW__CORE__DAGS_FOLDER: "/dags" AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" - AIRFLOW_CONN_MINIO_DEFAULT: "s3://admin:adminadmin@minio.minio.svc.cluster.local:9000?extra=%7B%22aws_access_key_id%22%3A%22admin%22%2C%22aws_secret_access_key%22%3A%22adminadmin%22%2C%22endpoint_url%22%3A%22https%3A%2F%2Fminio.minio.svc.cluster.local%3A9000%22%7D" + AIRFLOW_CONN_MINIO_DEFAULT: "s3://admin:adminadmin@minio.minio.svc.cluster.local:9000?extra=%7B%22endpoint_url%22%3A%22https%3A%2F%2Fminio.minio.svc.cluster.local%3A9000%22%7D" AIRFLOW__LOGGING__REMOTE_LOGGING: "True" AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER: s3://demo/airflow-task-logs/ AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID: minio_default diff --git a/stacks/argo-cd/manifests/minio/minio.yaml b/stacks/argo-cd/manifests/minio/minio.yaml new file mode 100644 index 00000000..8681086f --- /dev/null +++ b/stacks/argo-cd/manifests/minio/minio.yaml @@ -0,0 +1,712 @@ +--- +# Source: minio/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "minio-sa" +--- +# Source: minio/templates/secrets.yaml +apiVersion: v1 +kind: Secret +metadata: + name: minio + labels: + app: minio + chart: minio-5.4.0 + release: minio + heritage: Helm +type: Opaque +data: + rootUser: "YWRtaW4=" + rootPassword: "YWRtaW5hZG1pbg==" +--- +# Source: minio/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: minio + labels: + app: minio + chart: minio-5.4.0 + release: minio + heritage: Helm +data: + initialize: |- + #!/bin/sh + set -e # Have script exit in the event of a failed command. + MC_CONFIG_DIR="/etc/minio/mc/" + MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" + + # connectToMinio + # Use a check-sleep-check loop to wait for MinIO service to be available + connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 + LIMIT=29 # Allow 30 attempts + set -e # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) + SECRET=$(cat /config/rootPassword) + set +e # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" + $MC_COMMAND + STATUS=$? + until [ $STATUS = 0 ]; do + ATTEMPTS=$(expr $ATTEMPTS + 1) + echo \"Failed attempts: $ATTEMPTS\" + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 + fi + sleep 2 # 1 second intervals between attempts + $MC_COMMAND + STATUS=$? + done + set -e # reset `e` as active + return 0 + } + + # checkBucketExists ($bucket) + # Check if the bucket exists, by using the exit code of `mc ls` + checkBucketExists() { + BUCKET=$1 + CMD=$(${MC} stat myminio/$BUCKET >/dev/null 2>&1) + return $? + } + + # createBucket ($bucket, $policy, $purge) + # Ensure bucket exists, purging if asked to + createBucket() { + BUCKET=$1 + POLICY=$2 + PURGE=$3 + VERSIONING=$4 + OBJECTLOCKING=$5 + + # Purge the bucket, if set & exists + # Since PURGE is user input, check explicitly for `true` + if [ $PURGE = true ]; then + if checkBucketExists $BUCKET; then + echo "Purging bucket '$BUCKET'." + set +e # don't exit if this fails + ${MC} rm -r --force myminio/$BUCKET + set -e # reset `e` as active + else + echo "Bucket '$BUCKET' does not exist, skipping purge." + fi + fi + + # Create the bucket if it does not exist and set objectlocking if enabled (NOTE: versioning will be not changed if OBJECTLOCKING is set because it enables versioning to the Buckets created) + if ! checkBucketExists $BUCKET; then + if [ ! -z $OBJECTLOCKING ]; then + if [ $OBJECTLOCKING = true ]; then + echo "Creating bucket with OBJECTLOCKING '$BUCKET'" + ${MC} mb --with-lock myminio/$BUCKET + elif [ $OBJECTLOCKING = false ]; then + echo "Creating bucket '$BUCKET'" + ${MC} mb myminio/$BUCKET + fi + elif [ -z $OBJECTLOCKING ]; then + echo "Creating bucket '$BUCKET'" + ${MC} mb myminio/$BUCKET + else + echo "Bucket '$BUCKET' already exists." + fi + fi + + # set versioning for bucket if objectlocking is disabled or not set + if [ $OBJECTLOCKING = false ]; then + if [ ! -z $VERSIONING ]; then + if [ $VERSIONING = true ]; then + echo "Enabling versioning for '$BUCKET'" + ${MC} version enable myminio/$BUCKET + elif [ $VERSIONING = false ]; then + echo "Suspending versioning for '$BUCKET'" + ${MC} version suspend myminio/$BUCKET + fi + fi + else + echo "Bucket '$BUCKET' versioning unchanged." + fi + + # At this point, the bucket should exist, skip checking for existence + # Set policy on the bucket + echo "Setting policy of bucket '$BUCKET' to '$POLICY'." + ${MC} anonymous set $POLICY myminio/$BUCKET + } + + # Try connecting to MinIO instance + scheme=https + connectToMinio $scheme + + + + # Create the buckets + createBucket demo "public" false false false + + add-user: |- + #!/bin/sh + set -e ; # Have script exit in the event of a failed command. + MC_CONFIG_DIR="/etc/minio/mc/" + MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" + + # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. + # Special characters for example : ',",<,>,{,} + MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_tmp" + + # connectToMinio + # Use a check-sleep-check loop to wait for MinIO service to be available + connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 1 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 + } + + # checkUserExists () + # Check if the user exists, by using the exit code of `mc admin user info` + checkUserExists() { + CMD=$(${MC} admin user info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) + return $? + } + + # createUser ($policy) + createUser() { + POLICY=$1 + #check accessKey_and_secretKey_tmp file + if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then + echo "credentials file does not exist" + return 1 + fi + if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then + echo "credentials file is invalid" + rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP + return 1 + fi + USER=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) + # Create the user if it does not exist + if ! checkUserExists ; then + echo "Creating user '$USER'" + cat $MINIO_ACCESSKEY_SECRETKEY_TMP | ${MC} admin user add myminio + else + echo "User '$USER' already exists." + fi + #clean up credentials files. + rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP + + # set policy for user + if [ ! -z $POLICY -a $POLICY != " " ] ; then + echo "Adding policy '$POLICY' for '$USER'" + set +e ; # policy already attach errors out, allow it. + ${MC} admin policy attach myminio $POLICY --user=$USER + set -e + else + echo "User '$USER' has no policy attached." + fi + } + + # Try connecting to MinIO instance + scheme=https + connectToMinio $scheme + + + + # Create the users + echo console > $MINIO_ACCESSKEY_SECRETKEY_TMP + echo console123 >> $MINIO_ACCESSKEY_SECRETKEY_TMP + createUser consoleAdmin + + add-policy: |- + #!/bin/sh + set -e ; # Have script exit in the event of a failed command. + MC_CONFIG_DIR="/etc/minio/mc/" + MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" + + # connectToMinio + # Use a check-sleep-check loop to wait for MinIO service to be available + connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 1 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 + } + + # checkPolicyExists ($policy) + # Check if the policy exists, by using the exit code of `mc admin policy info` + checkPolicyExists() { + POLICY=$1 + CMD=$(${MC} admin policy info myminio $POLICY > /dev/null 2>&1) + return $? + } + + # createPolicy($name, $filename) + createPolicy () { + NAME=$1 + FILENAME=$2 + + # Create the name if it does not exist + echo "Checking policy: $NAME (in /config/$FILENAME.json)" + if ! checkPolicyExists $NAME ; then + echo "Creating policy '$NAME'" + else + echo "Policy '$NAME' already exists." + fi + ${MC} admin policy create myminio $NAME /config/$FILENAME.json + + } + + # Try connecting to MinIO instance + scheme=https + connectToMinio $scheme + + add-svcacct: |- + #!/bin/sh + set -e ; # Have script exit in the event of a failed command. + MC_CONFIG_DIR="/etc/minio/mc/" + MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" + + # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. + # Special characters for example : ',",<,>,{,} + MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_svcacct_tmp" + + # connectToMinio + # Use a check-sleep-check loop to wait for MinIO service to be available + connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 2 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 + } + + # checkSvcacctExists () + # Check if the svcacct exists, by using the exit code of `mc admin user svcacct info` + checkSvcacctExists() { + CMD=$(${MC} admin user svcacct info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) + return $? + } + + # createSvcacct ($user) + createSvcacct () { + USER=$1 + FILENAME=$2 + #check accessKey_and_secretKey_tmp file + if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then + echo "credentials file does not exist" + return 1 + fi + if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then + echo "credentials file is invalid" + rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP + return 1 + fi + SVCACCT=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) + # Create the svcacct if it does not exist + if ! checkSvcacctExists ; then + echo "Creating svcacct '$SVCACCT'" + # Check if policy file is define + if [ -z $FILENAME ]; then + ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) myminio $USER + else + ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --policy /config/$FILENAME.json myminio $USER + fi + else + echo "Svcacct '$SVCACCT' already exists." + fi + #clean up credentials files. + rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP + } + + # Try connecting to MinIO instance + scheme=https + connectToMinio $scheme + + custom-command: |- + #!/bin/sh + set -e ; # Have script exit in the event of a failed command. + MC_CONFIG_DIR="/etc/minio/mc/" + MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" + + # connectToMinio + # Use a check-sleep-check loop to wait for MinIO service to be available + connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 1 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 + } + + # runCommand ($@) + # Run custom mc command + runCommand() { + ${MC} "$@" + return $? + } + + # Try connecting to MinIO instance + scheme=https + connectToMinio $scheme +--- +# Source: minio/templates/pvc.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: minio + labels: + app: minio + chart: minio-5.4.0 + release: minio + heritage: Helm +spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: "10Gi" +--- +# Source: minio/templates/console-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: minio-console + labels: + app: minio + chart: minio-5.4.0 + release: minio + heritage: Helm +spec: + type: NodePort + externalTrafficPolicy: "Cluster" + ports: + - name: https + port: 9001 + protocol: TCP + targetPort: 9001 + selector: + app: minio + release: minio +--- +# Source: minio/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: minio + labels: + app: minio + chart: minio-5.4.0 + release: minio + heritage: Helm + monitoring: "true" +spec: + type: NodePort + externalTrafficPolicy: "Cluster" + ports: + - name: https + port: 9000 + protocol: TCP + targetPort: 9000 + selector: + app: minio + release: minio +--- +# Source: minio/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio + labels: + app: minio + chart: minio-5.4.0 + release: minio + heritage: Helm + stackable.tech/vendor: Stackable +spec: + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 100% + maxUnavailable: 0 + replicas: 1 + selector: + matchLabels: + app: minio + release: minio + template: + metadata: + name: minio + labels: + app: minio + release: minio + stackable.tech/vendor: Stackable + annotations: + checksum/secrets: fa63e34a92c817c84057e2d452fa683e66462a57b0529388fb96a57e05f38e57 + checksum/config: ebea49cc4c1bfbd1b156a58bf770a776ff87fe199f642d31c2816b5515112e72 + spec: + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + runAsGroup: 1000 + runAsUser: 1000 + + serviceAccountName: minio-sa + containers: + - name: minio + image: "quay.io/minio/minio:RELEASE.2024-12-18T13-15-44Z" + imagePullPolicy: IfNotPresent + command: + - "/bin/sh" + - "-ce" + - | + # minio requires the TLS key pair to be specially named + # mkdir -p /etc/minio/certs + cp -v /etc/minio/original_certs/tls.crt /etc/minio/certs/public.crt + cp -v /etc/minio/original_certs/tls.key /etc/minio/certs/private.key + + /usr/bin/docker-entrypoint.sh minio server /export -S /etc/minio/certs/ --address :9000 --console-address :9001 + volumeMounts: + - name: minio-user + mountPath: "/tmp/credentials" + readOnly: true + - name: export + mountPath: /export + - mountPath: /etc/minio/original_certs + name: tls + - mountPath: /etc/minio/certs + name: certs + ports: + - name: https + containerPort: 9000 + - name: https-console + containerPort: 9001 + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: minio + key: rootUser + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: minio + key: rootPassword + - name: MINIO_PROMETHEUS_AUTH_TYPE + value: "public" + resources: + requests: + cpu: 1 + memory: 2Gi + securityContext: + readOnlyRootFilesystem: false + volumes: + - name: export + persistentVolumeClaim: + claimName: minio + - name: minio-user + secret: + secretName: minio + + - ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: service=minio + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1 + storageClassName: secrets.stackable.tech + name: tls + - emptyDir: + medium: Memory + sizeLimit: 5Mi + name: certs +--- +# Source: minio/templates/post-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: minio-post-job + labels: + app: minio-post-job + chart: minio-5.4.0 + release: minio + heritage: Helm + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation +spec: + template: + metadata: + labels: + app: minio-job + release: minio + stackable.tech/vendor: Stackable + spec: + restartPolicy: OnFailure + volumes: + - name: etc-path + emptyDir: {} + - name: tmp + emptyDir: {} + - name: minio-configuration + projected: + sources: + - configMap: + name: minio + - secret: + name: minio + - ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: service=minio + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1 + storageClassName: secrets.stackable.tech + name: tls + - emptyDir: + medium: Memory + sizeLimit: 5Mi + name: certs + serviceAccountName: minio-sa + containers: + - name: minio-make-bucket + image: "quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z" + imagePullPolicy: IfNotPresent + command: + - "/bin/sh" + - "-ce" + - | + # Copy the CA cert from the "tls" SecretClass + # mkdir -p /etc/minio/mc/certs/CAs + cp -v /etc/minio/mc/original_certs/ca.crt /etc/minio/mc/certs/CAs/public.crt + + . /config/initialize + env: + - name: MINIO_ENDPOINT + value: minio + - name: MINIO_PORT + value: "9000" + volumeMounts: + - name: etc-path + mountPath: /etc/minio/mc + - name: tmp + mountPath: /tmp + - name: minio-configuration + mountPath: /config + - name: tls + mountPath: /etc/minio/mc/original_certs + - name: certs + mountPath: /etc/minio/mc/certs/CAs + resources: + requests: + memory: 128Mi + - name: minio-make-user + image: "quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z" + imagePullPolicy: IfNotPresent + command: + - "/bin/sh" + - "-ce" + - | + # Copy the CA cert from the "tls" SecretClass + # mkdir -p /etc/minio/mc/certs/CAs + cp -v /etc/minio/mc/original_certs/ca.crt /etc/minio/mc/certs/CAs/public.crt + + . /config/add-user + env: + - name: MINIO_ENDPOINT + value: minio + - name: MINIO_PORT + value: "9000" + volumeMounts: + - name: etc-path + mountPath: /etc/minio/mc + - name: tmp + mountPath: /tmp + - name: minio-configuration + mountPath: /config + - name: tls + mountPath: /etc/minio/mc/original_certs + - name: certs + mountPath: /etc/minio/mc/certs/CAs + resources: + requests: + memory: 128Mi From f05edcd8efb390fe90998312f06110dd3337d11c Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sat, 12 Apr 2025 04:09:58 +0200 Subject: [PATCH 13/64] add airflow logs minio --- stacks/argo-cd/manifests/airflow/airflow.yaml | 21 +++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/stacks/argo-cd/manifests/airflow/airflow.yaml b/stacks/argo-cd/manifests/airflow/airflow.yaml index dc155030..66f8088c 100644 --- a/stacks/argo-cd/manifests/airflow/airflow.yaml +++ b/stacks/argo-cd/manifests/airflow/airflow.yaml @@ -16,6 +16,20 @@ spec: - name: airflow-dags configMap: name: airflow-dags + - name: minio-tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: pod,node + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech volumeMounts: - name: airflow-dags mountPath: /dags/date_demo.py @@ -26,6 +40,8 @@ spec: - name: airflow-dags mountPath: /dags/pyspark_pi.yaml subPath: pyspark_pi.yaml + - name: minio-tls + mountPath: /stackable/minio-tls webservers: config: resources: @@ -40,10 +56,11 @@ spec: envOverrides: &envOverrides AIRFLOW__CORE__DAGS_FOLDER: "/dags" AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" - AIRFLOW_CONN_MINIO_DEFAULT: "s3://admin:adminadmin@minio.minio.svc.cluster.local:9000?extra=%7B%22endpoint_url%22%3A%22https%3A%2F%2Fminio.minio.svc.cluster.local%3A9000%22%7D" + AIRFLOW_CONN_MINIO: "aws://admin:adminadmin@/?endpoint_url=https%3A%2F%2Fminio.minio.svc.cluster.local%3A9000" + AWS_CA_BUNDLE: "/stackable/minio-tls/ca.crt" AIRFLOW__LOGGING__REMOTE_LOGGING: "True" AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER: s3://demo/airflow-task-logs/ - AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID: minio_default + AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID: minio replicas: 1 kubernetesExecutors: envOverrides: *envOverrides From 3cedcfaa5b4fbc2fafeaf5a0f9887ac2e0063687 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sat, 12 Apr 2025 04:15:04 +0200 Subject: [PATCH 14/64] fixes --- stacks/argo-cd/applications/airflow.yaml | 2 +- stacks/argo-cd/projects/minio.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stacks/argo-cd/applications/airflow.yaml b/stacks/argo-cd/applications/airflow.yaml index dcd20380..9d938bc9 100644 --- a/stacks/argo-cd/applications/airflow.yaml +++ b/stacks/argo-cd/applications/airflow.yaml @@ -5,8 +5,8 @@ metadata: spec: project: airflow destination: - namespace: stackable-airflow server: https://kubernetes.default.svc + namespace: stackable-airflow source: repoURL: https://github.com/stackabletech/demos.git targetRevision: "{{ demoReleaseVersion }}" diff --git a/stacks/argo-cd/projects/minio.yaml b/stacks/argo-cd/projects/minio.yaml index aaa033ba..ca58ee8a 100644 --- a/stacks/argo-cd/projects/minio.yaml +++ b/stacks/argo-cd/projects/minio.yaml @@ -3,7 +3,7 @@ kind: AppProject metadata: name: minio spec: - description: Project to manage Minio via Helm + description: Project to manage Minio sourceRepos: - "*" destinations: From 871d45912d3359dddb05740898431811cf739e57 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 09:48:17 +0200 Subject: [PATCH 15/64] extend cert expiry to 10 years --- stacks/argo-cd/manifests/airflow/airflow.yaml | 2 +- .../argo-cd/secrets/sealed-secrets-key.yaml | 159 +++++++++--------- 2 files changed, 82 insertions(+), 79 deletions(-) diff --git a/stacks/argo-cd/manifests/airflow/airflow.yaml b/stacks/argo-cd/manifests/airflow/airflow.yaml index 66f8088c..de8613d1 100644 --- a/stacks/argo-cd/manifests/airflow/airflow.yaml +++ b/stacks/argo-cd/manifests/airflow/airflow.yaml @@ -65,6 +65,7 @@ spec: kubernetesExecutors: envOverrides: *envOverrides schedulers: + envOverrides: *envOverrides config: gracefulShutdownTimeout: 30s resources: @@ -75,7 +76,6 @@ spec: limit: 1Gi roleGroups: default: - envOverrides: *envOverrides replicas: 1 --- apiVersion: v1 diff --git a/stacks/argo-cd/secrets/sealed-secrets-key.yaml b/stacks/argo-cd/secrets/sealed-secrets-key.yaml index 60e7e522..7196e082 100644 --- a/stacks/argo-cd/secrets/sealed-secrets-key.yaml +++ b/stacks/argo-cd/secrets/sealed-secrets-key.yaml @@ -4,87 +4,90 @@ kind: Secret metadata: name: sealed-secrets-key stringData: + # Generated on 2025/04/12 + # openssl req -x509 -nodes -newkey rsa:4096 -keyout tls.key -out tls.crt -subj "/CN=sealed-secrets" -days 3650 + # required for sealed secrets decryption tls.crt: | -----BEGIN CERTIFICATE----- - MIIFEzCCAvugAwIBAgIUTIeBhvXRk1w2QJRP2RpyQWJCB48wDQYJKoZIhvcNAQEL - BQAwGTEXMBUGA1UEAwwOc2VhbGVkLXNlY3JldHMwHhcNMjUwNDExMTk1NzUzWhcN - MjYwNDExMTk1NzUzWjAZMRcwFQYDVQQDDA5zZWFsZWQtc2VjcmV0czCCAiIwDQYJ - KoZIhvcNAQEBBQADggIPADCCAgoCggIBAMobRqdxhAE8Nr45tCRRI6kz2WopwnKb - 8bWGM0LbM/p04xfcLLAVzK4nC5HHBmfPycz6XyGjv7ViUSaHVk1j3OLxFEX4sd+2 - JoTQRenRc/oUHdwaGskBr5frXFAO3Tiou+pJ6Tu8ewCeCvGUzB9HtiRq5pCGPu6M - wzXbmR6GKaCJTu80LZ8fwFdSWI0gDoqeWffCTfcTgJfTa+DgMfTM3zzUNwJVfLo3 - r596wGmlxKfMcUtzw2J5iLr0dXJuJnlmA0gU31kwVcf8Wo1DAdh8hhxmUAkmGss8 - AktmB8OvvNygVVzzwmX/L2MH1zPXzgM9sYenkJwLSZhMx/uyfgX5x54/QaC7lwfk - wPnp52pK0JA2VmfFBsvlnPsVOWgwWU0jUtMGBtKlu/OonELAGaffbkhjD3eDe0B3 - X0gwtju+T4WCzQmgTFyXFdp18/S2rz19TBrq2XponRRpcXBo4qe8P59Eq++NNjAg - rYymFcTfrDZqMIt3jBv3jSnLlAnwg/H1neRfgx6kxXqxApF+vMDD7VVDaLYnLbFV - lwdkWxPrHStk05eBsLu/CkGv2ykkUAa0qqg/Dm/bkOl5lZlauVFC9wuiLh2cHgZr - tBpGuYlMFMRvx7A0+IBUTjTQswBF1+mBStUyBThOw1aJYbE2FCWj+lGrMX8qr+ad - sKsS+5DjOwCVAgMBAAGjUzBRMB0GA1UdDgQWBBRFs1lkbmy+f7TN10DGuHLa75sj - BDAfBgNVHSMEGDAWgBRFs1lkbmy+f7TN10DGuHLa75sjBDAPBgNVHRMBAf8EBTAD - AQH/MA0GCSqGSIb3DQEBCwUAA4ICAQA+6nknVdXMATXxbyk0E5fDlM/D2aceDw6D - 3kydc6wBT6k9SkhxFtrCI0IBO0f067ppFPj4ApJI0mrh3dsNiAzjqGetlGCqXjKq - pwdVxWYrmn/ELTKgvxToQG4J4PMUeo7tlhviU5HJo90DroDpoZHALubl+XsjzspA - MHH94CiF24z2nxoVolVa1qqq7WDlMWhZuit3WDo+jIgrpBgcPegNrZqrZqt/xlzM - Q91iOwwd3ykfo+xLdNuG1i8Qm1UFOEmXsustHBPDCtcI/ddLda4NbZtrNeZVmDbE - Px4Zzsjrbd6IKhVbdD5E4d9K/XMNsBIl4sidqdaUMhkNTYXkI/a5dTFVlFprLFnM - yB7A6OplQPp+LTITXW2IvD/eZetdl5073X0OQtuJrsm4mAJZ74sqXgcyVs7bGPIw - aXt1ttNH0qA5lvbp0cOIdXmpSWQfDRvlAExqX4p86549J3GMhIsVCMw5KsqduN5e - rSi0HCi24S8CeXiYV1phkSuxf0sROPuRb8SvY/6qcb73DMEMQx3zPWLV/+pG7DcG - uD327xvb3uJG0kFLi4MDnoCHLHYZeoPtwIEwG+1LjwHj+oKn12wlKJvYTxWALi4k - tKVW3QYL8beLaI3XfF+Vd3kCVpU6N9aRt3l+PpzkE1VCkayz67Sf5dJy88YIB7o4 - A08j/AH+GA== + MIIFEzCCAvugAwIBAgIUFWEYhsQ0cTg6XATOSu70TgA/BGQwDQYJKoZIhvcNAQEL + BQAwGTEXMBUGA1UEAwwOc2VhbGVkLXNlY3JldHMwHhcNMjUwNDEyMDIxODEyWhcN + MzUwNDEwMDIxODEyWjAZMRcwFQYDVQQDDA5zZWFsZWQtc2VjcmV0czCCAiIwDQYJ + KoZIhvcNAQEBBQADggIPADCCAgoCggIBAMbJgXSu4ELhULYyraB4l/fnGHf8NXNM + z/PG452XrKBGdElOScxYhkentBtGUlfsYeoanOwZikKLr3ngoMzvYIuoLCqkQbA2 + u5jlP8VUtTsMc0amYPAJeEvW/KLbH/6/BFJC9Qrt/kglvVRyDWOb2VN1Dx8RbY0Q + uWqdF460OGacStyHnRgpp43a7vb4YRKxNkb7Na1s+KYUIa6hJPgIIfLGxksiQKAr + zuRKFATImFDVzXiXLHP8c7qcQa4+bg9C194CUoWt+hC2SbrGGFEJ9M6c4TzU7/5y + aE9f4TTZfadH77XpukraI26fDnLKuspK5r3VFmEWE7d8qft6J8i9Dwl9OI7DJfUx + /dHOFAb/6oiN8TmIRtR6fnreuBv5qWq3B8wnfxmVNf22C0rS8KQaj019IQdZk1Sm + hOXfTtoZ4vZk9pw+gFdkoNF1rMqzmQnoR+IcKmWRydLx0pftQ3IpGYBqncfNnaK0 + tCag+dZbmPVtL8M5ovWX1WUoAfKSczBaxMnYO3I57R5jEEOR4BfWIdPEVVMgni2u + tO560gYMHpQHGzkaH8Doa0ZyV64rctEQhCIU4bV6JzwvDnYE8O96ej3n75NIYwKF + NmSNslExd5DP2fLlIK9mnh5q8FwFqGjqefTVUJ+0pCCMRCvID8FjcqCPEF4Xvpf7 + 6LRNSF5gerelAgMBAAGjUzBRMB0GA1UdDgQWBBTuOIW0EZNXykIIvU0L8xhOfU5z + mzAfBgNVHSMEGDAWgBTuOIW0EZNXykIIvU0L8xhOfU5zmzAPBgNVHRMBAf8EBTAD + AQH/MA0GCSqGSIb3DQEBCwUAA4ICAQBQlrQ0pfAjUSu44ZWwXpkSxNERwKI1eMUO + XroKPRSAmdB/tB7JgH3wdT/Yz7pdzvt0LKu1SlcIQXRJ5VsG9k7clhJXQ1Ogse7w + g7TvQWp49uw8OlVY/mJXlia7dgKiyREGSGjpSsUC+pHcFRmZ4N2los7agY62hZS0 + 68esldNIHLPR9PfM84jGfkk1CkFefayROL7kITVemBKHBPw5Jo8JSLnahSmfiJ1e + K7SY7u/j5PY3MobnjB0MtxaDovmJjjvZ9eQZsTSRnDsHOWZEcHZ1Eoga6NEjaGG2 + kYX8RFp8TM1xhq77/+uTWxhXyekrlxmhBY/ft/C34cm4jM0qw8db8fMXsYtPo/G1 + Cq3ry8+XpqM28TtqHy8hj9nK/WQNvJ9z892FVk7nbBcFzctomdXO9vh+p9C5PW6u + dkofuMW3EC0HuinXZoUiFMwVSGF6lm7lxgkNnn9JJa7v1OCJVqQxFGoJk2gDh+Ub + zHCuy/s4nu5MsBX3xJgp2h7/DPAr1DO04tmAUZ9OwpEyeYzWogM2cIm6yeMRNl5+ + xqhWZGh7/5s5iik1g0wB5o08IwKlTMI/b1cNbJBymgjwqGlJRRAKP2IwvFGae3sX + no3+9FB1FJjEX7sKIHFqN03w1GyrPW4qqc/9is6UaLmxTlu98QeMikel+Wjpd8bP + DAIkrU/MPw== -----END CERTIFICATE----- tls.key: | -----BEGIN PRIVATE KEY----- - MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDKG0ancYQBPDa+ - ObQkUSOpM9lqKcJym/G1hjNC2zP6dOMX3CywFcyuJwuRxwZnz8nM+l8ho7+1YlEm - h1ZNY9zi8RRF+LHftiaE0EXp0XP6FB3cGhrJAa+X61xQDt04qLvqSek7vHsAngrx - lMwfR7YkauaQhj7ujMM125kehimgiU7vNC2fH8BXUliNIA6Knln3wk33E4CX02vg - 4DH0zN881DcCVXy6N6+fesBppcSnzHFLc8NieYi69HVybiZ5ZgNIFN9ZMFXH/FqN - QwHYfIYcZlAJJhrLPAJLZgfDr7zcoFVc88Jl/y9jB9cz184DPbGHp5CcC0mYTMf7 - sn4F+ceeP0Ggu5cH5MD56edqStCQNlZnxQbL5Zz7FTloMFlNI1LTBgbSpbvzqJxC - wBmn325IYw93g3tAd19IMLY7vk+Fgs0JoExclxXadfP0tq89fUwa6tl6aJ0UaXFw - aOKnvD+fRKvvjTYwIK2MphXE36w2ajCLd4wb940py5QJ8IPx9Z3kX4MepMV6sQKR - frzAw+1VQ2i2Jy2xVZcHZFsT6x0rZNOXgbC7vwpBr9spJFAGtKqoPw5v25DpeZWZ - WrlRQvcLoi4dnB4Ga7QaRrmJTBTEb8ewNPiAVE400LMARdfpgUrVMgU4TsNWiWGx - NhQlo/pRqzF/Kq/mnbCrEvuQ4zsAlQIDAQABAoICAB2p4TXkWF6iCtrit0j5S8Wi - 4Y9Ob8bIkrJ07wMqDzf8ffRbkYeKu474Fh+gWZVVDyZxvRK/6PxjYMvJ42XaeWop - vjMC7ICMSJ0HwKsv3djfqP5MKpBEGhSvK85oUBnn0iTyEjR9VoRhrfOFRmx05wvz - UMH3ojb7HA/EmnW7dVKltJsxSlznmSB36p1UDi5UTZoqv6y7BQ3DrzrDTTHD/CP+ - ddB24aanU7SRnLok5XdHb32GkS/b7LCb/rz749O/oE2tGpcJnB2NMXE4W4X9yOVU - Mv0Uxav8s82DPTKAkbTEJ6NL98VmrfrYBMMMMqWRIYRqmLP+iB+9bUDJEuwnnB+g - HQ+W9MliiKfpVx3LpcwaT68SNLSoSHt+hHcd0b8iccyvpxag7LExe36NEmg5Xvn7 - NfTcsCeRrbFW7NWmJnS0yZ5vNpf1K6Y9MVPLlNLOYL61QJAVf5EKQxxT66xvFjE6 - L068mFAn1w0vreXIf/Z7eS+A5nU3EBHigdNJO+EjrJ9Myr5fCJewE6tCptTUdkep - ahC2VvWj7d5JOmGVgdvO7/lb2N1Zv8Z7M3aj2tj1xLIeGh5/T/GaGBn56CZpoRtl - i76E08ehrXoKVZM+ej4awmipqkW3x3qDXFTP0Khdrr0FvFptT4/kJ7rNFnrHbNhC - 9fQsHMzxohtlq285hehFAoIBAQDldt9FiaWJr1YTcaaCopJO3VR39M+Ap+Nc3LCm - G320DMzBV310q8zjnbpQ72wCm8zOV8ZRoGLfSEI0cJBsw/w0fkPK3FqPdHosUQte - 4TvYqhz7qNV4fMqgqeFgwPXLzfKjeJwd1GSyBt6+LtL6vIVXsbiFa04wTi3Zal0S - aiKMt2l2qLC1RjIIQC/go3+Be5n4r/DsmPuIKXydsBJbmYxwIp8zGxE5SqEle66i - kg8CXm3TzMnydaSXW69s4/E4TsNdsdoUMEVFX9TR9QaYBGbYa4uBgblxnwapmoCG - 6/Qx72Wwn6jclnIJ6O4oCxRwt4+PFQoB1iggqYXvUVka7pdrAoIBAQDhen7s+u5/ - Gmkna0Z2gxwpUleEAeoqAPEab5HEZYvUfQFpmsCvavSLVH7I72HmXwxox41a634U - t19JGiPajRVy3d7NWJwT775Rr3JvIPOxJz7z0Erthvovt+usmybjM0GUAygcL3KX - NI7NIE/20+FrE1t5KydSDpnwGDjUOP3qke2a2LH3gkxye98zUiSP1beS9ZtND0t0 - tb15H6kE0RzKzoDr3g0H96uIYN3f/u/vd0QcoeUbeZ4zAfFT13FLGkTzM9hryi+c - u2GlNx8jlxjEdP1WQ3R16F2wiNUDXEO0rIJ39yJxq4Q+HitHs1II0ku8Zsl4j2ji - 4NjDWnVlGsf/AoIBACq3PkkLnq1K3pMwDPc0nFfiC99oy1f/6OtYieniJDxoRZWY - W9pkQPf+XLjEWS8DIdUnsdtwJ/e409o/OiOFOGXtO2GAM7uQy02EqME43JMw18Rh - YZvUX0QZW8NPKHKQHCMiSK2k5Dnf3Yc54Dyc65Wvl9H+b8e+Rfq7Oig7acMYQaG9 - NWgJcTbn/AGtbgcEo10QI6FnuStXzcLtC/3RT7twNoKk/0fTBLydkZzh47llKrzK - 2q0wCTvD7Zwvsq/wuZ2vhhFMCxknBc8v9dJGv356RrYNGCB2oh7gjByRwBurnRqM - HjsyJniBzBPYPC3fffH80KgW3dF2vWAXnKbO05ECggEBAN2KBfvB8WNhB8B7XphC - 6gCPrv09BF6Q3lQjGw3k2CbHTAP+0SOJ6wya0JeM3JwE96RbrT+P8ilYon6o+GFc - DZ728FrCnVfZTx3jTIRz0/xTmz7jgTswS9Fm3GCTcPn6+ov7LgXDeMuoEEbrYKYc - OHeEZXQAOnodbpLAudWKybEYAGUMVI/jrkkt/HfoVZQdYGk8eIKWbIUXrDq+KMta - yvdZPsKBQjx51EgDJP1Y91ZJ0NpxFzKPpChp2DuQivhHr0dlwVbnKTDNy3sRVb8a - TnN4nxGK5XqT/LnfN6w9kyiJ9wb4Axe8dgqyicPBJPEV7uQF+h5CicOhOFl5H69f - SPECggEAFQUHIFV+GqiBwoJv7PESfHWiCwmCNWsA5q1lRLhIiWHJ8veIXAizzYGj - B+AWnMIccF3KLPvX++7o7qY4kkhAjob6AoulRg49C/g3YtuuxPe3CI7MV12nOHKc - ltzRO/CZbcg7LMNoPDrsmVoxYsN91pAg7hH83MNLjEiHhsJMKW5MXcnV+ZEiSukK - gYEyN7T2sI7sGQYEYERfFybOyqNw9RjojHKFEAlfrVa+rDbpU837AATcd4+E+aav - iwVBz6BCvn+ozhvmu2ZkOminlve3Bfnr7eJ5p8NHBwB9ELscKBZK2SNcKlu3wtg4 - w6IEdTFRspEvNwyrTH9geMzYmqbIsA== + MIIJQgIBADANBgkqhkiG9w0BAQEFAASCCSwwggkoAgEAAoICAQDGyYF0ruBC4VC2 + Mq2geJf35xh3/DVzTM/zxuOdl6ygRnRJTknMWIZHp7QbRlJX7GHqGpzsGYpCi695 + 4KDM72CLqCwqpEGwNruY5T/FVLU7DHNGpmDwCXhL1vyi2x/+vwRSQvUK7f5IJb1U + cg1jm9lTdQ8fEW2NELlqnReOtDhmnErch50YKaeN2u72+GESsTZG+zWtbPimFCGu + oST4CCHyxsZLIkCgK87kShQEyJhQ1c14lyxz/HO6nEGuPm4PQtfeAlKFrfoQtkm6 + xhhRCfTOnOE81O/+cmhPX+E02X2nR++16bpK2iNunw5yyrrKSua91RZhFhO3fKn7 + eifIvQ8JfTiOwyX1Mf3RzhQG/+qIjfE5iEbUen563rgb+alqtwfMJ38ZlTX9tgtK + 0vCkGo9NfSEHWZNUpoTl307aGeL2ZPacPoBXZKDRdazKs5kJ6EfiHCplkcnS8dKX + 7UNyKRmAap3HzZ2itLQmoPnWW5j1bS/DOaL1l9VlKAHyknMwWsTJ2DtyOe0eYxBD + keAX1iHTxFVTIJ4trrTuetIGDB6UBxs5Gh/A6GtGcleuK3LREIQiFOG1eic8Lw52 + BPDveno95++TSGMChTZkjbJRMXeQz9ny5SCvZp4eavBcBaho6nn01VCftKQgjEQr + yA/BY3KgjxBeF76X++i0TUheYHq3pQIDAQABAoICAAi5ieTG6Ha5qxWgF6DyLLYC + TtTfUoeUZhEnQtC6ejwQ847PG0FB27zpNlK3Iqs+ElXDJBg0Y8OEHCgfgjc/Gg/v + H5HMjUnbER9GupE3j0+/k2SqWJlYdQ7xE+a1nYvMXaJwUrnPfx2yswKx5yFe0v+i + ejmCcI+PMxrOjnI35JIpBxvIQ6PD75keiZFN4/wnyn/Ri/DSwP67EKJjQLs2fNT0 + wU8aulboAqfgPMzb8rF453svjrwq0aKDCVqeM1hDS7RbCj0DSTUyvFLbWcW0HT+f + QOnZrQyQKsCxxDGSCn2NKaR56nBZbtIutCSOKNDIT6R30IhjfsCK0xluP7DuuVTn + u8jHZ/Lw3WT6YdkCC/XamQ78sz8YbsccvcIGL1IOoh41DsJ71A4De+L6awqj6d2v + n/YUx+qNrAQPC/qXU5FEb+KUOaHuCL31UjdRCjQSyFmu8OH5Z0IuYClO9c0gW3ej + QfLcMpASnQft/jbwhmK16SceEpssu7XSFbfSNDCeK+Au7jXXvl3joEr8sqfWTryR + LYRVEZH1nzpycl2Q6B7Lr8GAj3BEIpmYUiz+5Iz7n/Ocl7m0pLO0rsriL3p5nMcs + ZN3/J5vrelx+t8NJZ1idg7wUjHpRnFpvTIfdfZactZH1M22jVH2kaMsN78shIr7Z + NKCEPW3/jZPDQ1Qw58y5AoIBAQDlOsoUWwHbS2ZHCYYZ6Ef8CXpJHYpGnqfu6dEK + VOavL5HtxrrAhot/DYodATn5LdsU9iI6j+TBRgafgrcyhVSzXdpNDYCeCCroQAyl + P+p4MeC5YTKnqEOHeCEJ2hXbxL67umjykP5tAGSgNianUKO3MUm6qyJY3mpuotFj + t4zxstVxjk4xwROzSReN6VvNS8NbGZ5AkIiVaXiHk48jNOY7KwByRHh+RLk2juiK + UTLeIsVL193kF26WTFKzz+4OImk5OWmIS4bW901qsEeTMk5DEZf+YalFCs/iXjKF + BjlPkkA4U52lF8ps2RRBl97p5AuypFFFWTL6qu+5K+nkhKUZAoIBAQDeAJXjnbNQ + BZT4I1JpuUBwTQVkedDMch7aCsnxQ86I7Enu39w+5UKvgW5b4XOGGvzHpqrwNvXa + tTLjcS5o0ucL2dwAuQ2k31TxC1mHyzPsf9gFBr/qm3ej/qFM+F4qFDH5JpHRMxSR + LU4jX1Z9jk3IjDvQYgCAcQ9Q63BANoaMwIw8oitiysaM9oRe5NXGBURuegre8xGQ + mM1867z67vigG9+oWAvOiCYibq8AqS7QH4azP39H0w840SDkxPNpmyfjo/cIkKoQ + X4h8TbWHSDAWZZYzqLB9pSHfLZXZpvIaJLyqc8UnkvRbrUQuwB3nORlIMB5lc1L0 + TMnDHfb1hUxtAoIBAQC+/TiZtSna2PuifGDq/QndCNiGSsKHDHHHMbEl9o8tr2ja + wBWhi3h2p4VHnVn8LyS2Vk02leV20y15gaO5yDVGLRBR2soq4Ys+bv8aQCLPel9O + lZlq71s0Z/a/CqHjtg1kxG/kisIxd35frj6OcTW20K4phaTcfpQXEjl759bbGrGU + J4knE0OiwWeKqb//UPy3zqR4DvR96J6v91Xvc4fQDIgWrM3oBsJ6np9Jf1XulvHj + uzv5azyMpZLSlpA13p3lO9yd3q++M9ag6bb5AZULdIE7RYuPJu8GII3fH8xoa+Zu + B9z1TNkvz7582Fi2b3b/pMDCEwvSptGNavU5nPshAoIBABT5mS/YRojY/iozrCWx + CxWRMWE84S1v2tpfCpVxAqKNMK8RF44ZuyM552Nm+bNB+YAjjyMDDleLpzM2Dlx5 + ELtd6htnYJKjRzM/D0Z1PIKr0Y0oBhn0dmlYNYVAMiVnoxEXQ8+wM9FDpo5RA6r5 + NwArv7sAh2ubQwcyN3jZrfy+dIGGEPNUlU3cPMV6eXgsJzsI7N4DuTUgOsoCu+g5 + i+QUvhz5tKPe2TuHmaFGWtXvJJrq9OukVVvY5nK26QihUDOB2mVExcymOg8EiJIv + j9uN3ZMj7QjyATFE7VOTImy1691u8sG6cLv4suh33ltqWakoYrbTHpNWbfmxaxPn + dEECggEALNKGY0JSHzuwA8wL7XoiOawHJdfBx5BenpM8NpsKAFVACsVZcY4OzJHB + Dwdr4bhBcRaHrHdM1RrqIvnWnF2gaiNNifNZ9MwHberWhjqQMqdRiv7RD6wp/jPe + jeIFj/endqSWp3LhyayK8pHZDAEbx57sex0doiosyhCvqYWlcDe2XwvTxGkdpM8J + V9nqyQh7zNYvVZUby43Gl+Bdbxirolq5Jcx+Z2wxiXt2pL3JArGtnbwHFNP0oZio + 2RmFjD0ZqLQzYmWObQeYKMHcz/8kMNtNhgUax0ZXTGOGEt4bO6Iu2S/V/thVg/Wn + hnSzVDPZAGg6LcqDPW9csv6W+BCo9w== -----END PRIVATE KEY----- From e486d9cd5ba8fda35a07f117712efe26028732e3 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 10:20:47 +0200 Subject: [PATCH 16/64] split stack & demo --- .../applications/airflow-postgres.yaml | 0 .../argo-cd/applications/airflow.yaml | 3 +- .../argo-cd/applications/minio.yaml | 3 +- .../airflow/airflow-credentials.yaml | 17 +++++++++++ .../airflow/airflow-spark-clusterrole.yaml | 0 .../airflow-spark-clusterrolebinding.yaml | 0 .../argo-cd/manifests/airflow/airflow.yaml | 0 .../airflow/sealed-airflow-credentials.yaml | 24 ++++++++++++++++ .../argo-cd/manifests/minio/minio.yaml | 0 .../argo-cd/projects/airflow.yaml | 0 {stacks => demos}/argo-cd/projects/minio.yaml | 0 demos/demos-v2.yaml | 28 +++++++++++++++++++ .../argo-cd/applications/sealed-secrets.yaml | 3 +- .../airflow/sealed-airflow-credentials.yaml | 24 ---------------- stacks/stacks-v2.yaml | 12 -------- 15 files changed, 75 insertions(+), 39 deletions(-) rename {stacks => demos}/argo-cd/applications/airflow-postgres.yaml (100%) rename {stacks => demos}/argo-cd/applications/airflow.yaml (84%) rename {stacks => demos}/argo-cd/applications/minio.yaml (84%) create mode 100644 demos/argo-cd/manifests/airflow/airflow-credentials.yaml rename {stacks => demos}/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml (100%) rename {stacks => demos}/argo-cd/manifests/airflow/airflow-spark-clusterrolebinding.yaml (100%) rename {stacks => demos}/argo-cd/manifests/airflow/airflow.yaml (100%) create mode 100644 demos/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml rename {stacks => demos}/argo-cd/manifests/minio/minio.yaml (100%) rename {stacks => demos}/argo-cd/projects/airflow.yaml (100%) rename {stacks => demos}/argo-cd/projects/minio.yaml (100%) delete mode 100644 stacks/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml diff --git a/stacks/argo-cd/applications/airflow-postgres.yaml b/demos/argo-cd/applications/airflow-postgres.yaml similarity index 100% rename from stacks/argo-cd/applications/airflow-postgres.yaml rename to demos/argo-cd/applications/airflow-postgres.yaml diff --git a/stacks/argo-cd/applications/airflow.yaml b/demos/argo-cd/applications/airflow.yaml similarity index 84% rename from stacks/argo-cd/applications/airflow.yaml rename to demos/argo-cd/applications/airflow.yaml index 9d938bc9..9196c2cf 100644 --- a/stacks/argo-cd/applications/airflow.yaml +++ b/demos/argo-cd/applications/airflow.yaml @@ -9,7 +9,8 @@ spec: namespace: stackable-airflow source: repoURL: https://github.com/stackabletech/demos.git - targetRevision: "{{ demoReleaseVersion }}" + # TODO: adapt to release-25.3 + targetRevision: "spike/argocd-demo" path: stacks/argo-cd/manifests/airflow/ syncPolicy: syncOptions: diff --git a/stacks/argo-cd/applications/minio.yaml b/demos/argo-cd/applications/minio.yaml similarity index 84% rename from stacks/argo-cd/applications/minio.yaml rename to demos/argo-cd/applications/minio.yaml index 05d27e88..bfea03c9 100644 --- a/stacks/argo-cd/applications/minio.yaml +++ b/demos/argo-cd/applications/minio.yaml @@ -9,7 +9,8 @@ spec: namespace: minio source: repoURL: https://github.com/stackabletech/demos.git - targetRevision: "{{ demoReleaseVersion }}" + # TODO: adapt to release-25.3 + targetRevision: "spike/argocd-demo" path: stacks/argo-cd/manifests/minio/ syncPolicy: syncOptions: diff --git a/demos/argo-cd/manifests/airflow/airflow-credentials.yaml b/demos/argo-cd/manifests/airflow/airflow-credentials.yaml new file mode 100644 index 00000000..aed1fd28 --- /dev/null +++ b/demos/argo-cd/manifests/airflow/airflow-credentials.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: airflow-credentials + namespace: stackable-airflow +type: Opaque +stringData: + adminUser.username: admin + adminUser.firstname: Airflow + adminUser.lastname: Admin + adminUser.email: airflow@airflow.com + adminUser.password: adminadmin + connections.secretKey: airflowSecretKey + connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@postgresql-airflow/airflow + connections.celeryResultBackend: db+postgresql://airflow:airflow@postgresql-airflow/airflow + connections.celeryBrokerUrl: redis://:airflow@redis-airflow-master:6379/0 diff --git a/stacks/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml b/demos/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml similarity index 100% rename from stacks/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml rename to demos/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml diff --git a/stacks/argo-cd/manifests/airflow/airflow-spark-clusterrolebinding.yaml b/demos/argo-cd/manifests/airflow/airflow-spark-clusterrolebinding.yaml similarity index 100% rename from stacks/argo-cd/manifests/airflow/airflow-spark-clusterrolebinding.yaml rename to demos/argo-cd/manifests/airflow/airflow-spark-clusterrolebinding.yaml diff --git a/stacks/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd/manifests/airflow/airflow.yaml similarity index 100% rename from stacks/argo-cd/manifests/airflow/airflow.yaml rename to demos/argo-cd/manifests/airflow/airflow.yaml diff --git a/demos/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml b/demos/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml new file mode 100644 index 00000000..9f763340 --- /dev/null +++ b/demos/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml @@ -0,0 +1,24 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: airflow-credentials + namespace: stackable-airflow +spec: + encryptedData: + adminUser.email: AgBJq/dTwq0biM8b8bl2/daQSbWPRXSnOHOMnrJRjxM7A5KX68HI0Ifp4r47gSTZDQuKxNx5PaFuyU6W0tT9CvbFSPdhTXb6G5xLXITafrpd2he9LiTRzI0wzh/xa8sOChTNHK3rXBpU8n0Y7aYVDp7mibFk6CQKhMMq8s/OL5Scp2R4mbiajAtXMwQ2wt1fF3CAqR4QgxqGGr0cEugGf8zl/Z3GryUjJsqjNyhkjDxBN2E6oiP8Q1NCMV7AGGxdqHyi3PJH6hFddk07fTKu24oV1/IkdPZ7QgTXzhV2dk57KcSX8wB9gksWsCiFGlL4RnKwkmMbNzXq5m9zAk9YkN0QyMu0pRBI/NqhFTfhXIBcVsIbEa/37rGxV6oIaLqndneSG4fDTu0HP6ga98pM/nD2Nrzlj1gS+KwaBSGWq+CWs3HSah35treHxne3U2nuG98+nnIvUQUYI36b7MAP9xQ5Koykk4iU6LrVoxyck42k3VxeWNLc4W1axOBLGyjSxxUmZMWF4lmlz4Z045/SuBF8O4mmIHj/3cK5EEcAlme3lebNKht9/w3iaxDd+dG04CP8RMQGE7i3OdHpteilZMszXyjYz8m08u/wgiKG8aEkh8ad9fNeR4RnGP5CMpifrLZeCsismj4R9JN6B3lr3q2s1Ot/aedPsk4JMj0RgJAnLkSQHbYI3twJAOq4C/6HH+fSsripLIL4CjC39Hevp6Nku6ef + adminUser.firstname: AgC+TLraEdynGLvTTBkzXgBEvnK0c+97PsdcVTK3BIFD0L4APn3HM6V2WiUcyBH//YJqUiNvg5mnJZx3EA1EakPdzkXlitYNFy6XoMDTkrMZTgiD/VAiABBmPyVJesGz5xxlovqFGpbGWQbXjr+fGOINqSciQi1JMzN/jQ9EJQRnyxr7Xq3rmF4/NsWo8s+fEbQHx8SmNrS8mu9BFd+QSL5pTskZJkcaUuzduCXIheacJcRE+YwiufQVOFSo2AkgGcbwzLBRJuL6+u6MST3qr/xapsBuGSwHtEoVcI/NApTYsGZCeNN6H8e083BnRGCunfnw/R9/5Xg4uxdUEbGT16gP4H5/o8V22AmH2opSgPgxVSntAn6h5cxva2YUc5Bezn+pKFZmK0KCmC8tUSDusje3X0emMIs9+fC+ksj81UwSikC+hKbuZz+HuzS23UCy+w5R9X2SDGk802zxdLVhsXn/46dgaEqzOEa1pYJEyCRZUUv4KTdM0/XXfkid4r1uP0d6bZ7jlS9tSDQ/ajm5dAhl5W7nIYdRVAqM5KhvvY3mWpqIdscdr+bhpE46mJPstP6O4FM3/0OYPntmjxSVTFf0Uc8mHY4RRAM2sN/ViqcDGumn3Q8xlQtG4vMO2Tx+GAStzIu9AGsTFYcak3fI8kpZhs7mhYCXeigSusaIcsH5bfC7uekMrqhrKraY5hOU/xaxjydPIzN6 + adminUser.lastname: AgBzqzvmZLHUVDcG6f5cyBxUef4Ov0PNBGaqrYLKPMk4a681XpPlfJnNvMk87aV8Ts1xunoozWs4uM2Z6pQa6ZIYoR5oRkO4bLqK16frBaVypKVsKRPkol+bSyWPLGuI0izIKdAcHrtuaEffb6Sr98i6ColIqGLEYlzUalnPIuK/W7QqKXO91jHYnTv4m6LUMere1N/x1cmeZ3+W+BukWLkfiLko4wgCSauUCene9azY5ehtmlIAUo9xNC3oDA+rTGTI1SkG2Xe/ytI92sedCEMmVX7oU9PI0KGfckchc6azmOTsWN0hSnUdIHPPDn+9L0x77HwHeMfiVgO2Bai76lC5fO2b9Pw0OxID5EKY4jvIrtQ2uQANCtYupge6Z37njpzI070U/H65NDtCKAGmcuy4pFgdF60VrEPrfAN+uaVJXmJGu75QmaKqGWE0RAW9UcjiKD9YaQf7pN2tx6v2oga/OR/cDTR+LJYIcwAC2HI+v9t2cb3LqznnoixVwU83qOOqDelq/Pd1vV3Sf1R2Kj5E4UlYm/lEtSczKJyxX4Rim34kzMqZnmYP/56bs4NwXjJ8vmMHEeFkO5tL3NaVgeJ3yKvLG4yUtasXxlrSUgLdXXTu6wN7PuBUkzCN4maYHrDjJYApo1iIXhObtgRbDatosCUCJOSvE0BEEf9CKP++SkUdipWVk84QK3ryNYQOZ12Lc4rnkw== + adminUser.password: AgAXw2meNuLCnWIDeBdrUI1osShroov1kjvRJ5+mLtGHD9cOTfQlAJpGXN+QwVWXz/hsRl0zhMxI1EomJhXOOyNI67ptlaMnSV/Wtt5X92ISmHReR/1R2cXv0e/NQLbD4M3AmCpvSz0agYMBxtIFh6mVeMah2RRjVD74Xc3P3SrHjnY7CiLYe58JWVkxGAp6WO2JTpopUJI1Iagr9MHfkGzPp3ygFVxwiMjPIVCM+IeAbtHLNi2w6slrcKa3IJduLSaGOQEr/p8YcPm2Qx2TGHOpiM6Lw9api8JtN28mS5hArc7qj5kkwkfEiHW2exYMXVSdWy7glgEGFuTr7MRi0LpLqS72DT55E9uxAObnlWuiKVmk6UfExX2mMfze+Iz4sKXrqOFoLAIXx8+yjO3t5oDvJf9pxLNihl4rYyLg9v8+UkkGJLQZPOqjQoeuH9ighDkHdfkiUDdprnG9iZAQBFzKN7On+tMA09McwHkC6jV+RIv47zty1CMoEPQ2wEM+zDHU6DG2HvtHQYov1pCpXzyvMz48hcKiAwIjZQM1RZBL0tN5qd8Tp38MqtAvVt7PvllwAN2LKt9HdV5j82kyWseE6MPYkB1l8Q6ctUn5YY0kw+tWwgtmYbeRY9+kSIUphxntX+unzhCKCAY1oBOc/cdim3YQInGGe6lx3JGUQvz5LpP6+aVBPyy8HOPnUSyxdu2ZaBkG5SKhKvfH + adminUser.username: AgCVVTX2c+4moS0v6mXuNQ5v6EtxZ43hN0QYg8wgYG+GPR3qZ2Ep+FkUJq86q9dFcuX6EXXtS2h8BJr2sUrRW6QySFnhtmg7A1d4anslN3yAjLipwrreNT9cn1t4PsdIhU0q5Z2uPmMsnLF28D73AxWA59iiveIId2KSuB2lsaC8S8CeW3Df48LW3cnukHtOvc+j84YQgFdagujBtwAMCiym2pdBJG5GojTHQ/5P3AnV+iM5wzXaYIeF5UZ0wLs5WU3CqHLxsY3q7NrLKGzS0xTqqdc6H++tJQFZY5G3q4NcCx+jq9oRErE0ARz3ok2pOHPArUEbyas9+iibA9HJcCmrKguQ8QBvaIQh9L0YtHCtwvaMbR12ZmMfRtMW6pZnM/d2K648JFekuezGTOVEjLk+ZvVm45L4USZpB0MD5LXEy3m8cDdkn2wqTWvblfjgfi7opoLgpwLyf/BZuunrFWT08x9K1WmNQ7OkM4WxscLRyccFOtZQfoUtT8hW+K5/mxLjOQDpryWPCUeBDAMnBvbGG/0AABp7a+vWG9KVus2ZyfUHVFIHgJBsHXXDtONIkmRvTal5bjlRsVk4Y0O3DoJa+UjXYoIBSm1JwdUJMOnL1MoS7Ijk+iXKPMc/3fcbrnfqYxw3ymr485bL8fKh2n99MsCdR1yX/mTxw3f6bRZdN5O7IUTkkKVxx1b08FD1svSwNKHBZQ== + connections.celeryBrokerUrl: AgAD1Iu/Oggx5NbNY29aJweDDVHlnL2KnsSJwBufPSxsaQouGdIIMIsWr/y2z+qCtd+TBG7f/ZK6vvTlT8kGeq9PGguGTOs+jQObFsqOWoWMn+zbZl+H3SsXHE9ohlrq36n6b68vRrpR/zIvI/QcFg7FC8qhH7GhxzYoUfXbIVEg9ZUyg8W9GgznIM8pJD7v1blz16fXerC3D3wcY449wvQuygZucGyGNfCK9+y75/HQldYVg5lG0Gb914uT5hQ4lATlH7xnHQcxwXhT+30qphNcIJSNZW2D6S0kovtoZ8xk6P/xoO5uZNzNyFnRxIu/AhVi3T2L/dzlO9ztbxwjCu19xxDo16/vhXQ5awGcyJjHqZc8Pu4v5e7sJTElguAHooQraL/dcx9fJvVOxk+3GUMwixfzSR3R/BD2Na4l9mT3fwiW53UTYIcDm/jRwZ8EPZz6GtWVPtwoOkpe6y/OLoRtGNlJEW2fR2d4VwsrqnPV57716FuI5ETwvCSJcqaoMk1kPZBzWGjgf02owizSD7gKONBLX8G34981JsTu/vloPD9uPYQoQ9oFlKVtdmYKhL6u08ekiwLoojp51T/JB5I0RYLKSu4nR8O8SK49RH2CF/4viBs5cCezP8Cfq4qxSBHsd12PXHocUWK3v5e7xgQXi4wJzoLifNcS2FXwV+97DvHaYCM9KUogdL3K/MnPSfb6SGI5WHJkExK5Hr2WqKG+3zFIr+TfIf8V5QcrXaFEz6m0iK49x5hIvjgmnQ== + connections.celeryResultBackend: AgAEg3o3zzMI1fqVt119TiNZ2U9Uvg5ua1gSW1klHtI8P49n/ZTmYftFJcPkJfw8mjawxuSzwTvPWNirkcTyF0jpauyn2W3L7q2SwuMMmS96z4R0FBwa4a1AjP7Rovq9r4lPjtjn45IPX5VPDG2gBiscVhBFLzozFucgXVk6bLo3bu8/lXuf+5PXaTH7TtRfYuTipiY4jbvlCx5jR3omzgnLThOWaoplnfIQPQbhnLF4t9ydipLxV+Dr2ccGnT+yehHxbmROK5kRD8DgPe0oe1+qFS5zIRyG6KZBgI3+ru3wAkbREeuE/Bq9nDWmRS5JVDlOKaFqFtYU2h12LB/f+MKPhwPdvzCiWLmT+XU8WW25pjUXHDakXRm4RJI2J+24cTjue2jv9GbNF4+n+ywunUpNqEagJ3q0VFZ8o5evZWjw55qMnsE6eMpSoPygssD1sfQ5jf4W00onxjSEijR7STqi/6zBEZzKzPSij3fH5HeWmsd9MwnQ+1/J8VIighxKm2RSRn6ClRsXggMvzCCf/wg5zwYClMWage65fytKYH12lR4Dl2fNHFYexTIWvrL/36xuuKhP/Glpz6dJHKm5bHOjeNo+RMtKLTWvr3NVqMQMzK9yl0JdPJKByoqLy+xR5pKy9TacihXJXZhpEPd2IDK2/zxCI+71OIN0iNqL1KnfI5VMtFJu/jEmkgzD+Hqg4UlI3a2frgxtv5chMvIusHIxf/2fv3OoWVhfIjObD9nlnbwFQHhAfnMemeMhaxlnxQ34aI4gFZJ1m3hg + connections.secretKey: AgB9N48o16//pMoE7YPHGHQUDd6n5gQGJcsDK2kmsxM1fj4POpG8eRwxXpRlNwBcH8VhgJAWJKLWUrIy5cQ4DM77gDGcG1YOpHIXDMDzTtocLmLJmpxoFGrswv0lEB0dCZwSpOph3btBzCDuKNA/nzHD/iwoVkZiDao/CyC+wjRrudamIfC0aJJsnSo1NAzcmsP1IEaURF0LoWdAPXWCqzEHbJ6FjCdQe/5F2md30PHGt1p0x3/ro4i4gv54ZePWBAStZml/a5i219UBvGROJgBCAJYEb0wlQ5Aut+7PVzjgUfLgWpvGEl1XBXGjuR5NUOiEwcYok+QsonQcUGaNScMzzqrfWRIagAlcGnSzs4TsLUgA2opBlKClVdEqnyS1QEH062ZTrxoFJ0mGg6T1oh6rElGfzmOTMhU95nRIF3wtYh4Si20IDAbWkIJgk1j5+Vvg31KHOTQXR6WdXc1CqYuo3LVzjP4JeIvrsdyqIJC4dg2WAKDTrQ3qixDv9Yq+bT9jMHrdQPQBz1clb8g6OEqtuDeobkTJFU/vdlZRSz86/mtsujpBNHr2SPa5IwIr2ALBx61NXfalc+gp2lUDTQSZPRf/oHIyD/SbARFDCNGmf0rPO1yQ2dg85+kLLSE2+K6uPYPwqippSIyiazIzCi5FLa6B6IEqAxjdrPyBdrMu3Qgkzdhp23UaL7yRnXJkHnPtb+URhrbnqPVIDiNeF0Nt + connections.sqlalchemyDatabaseUri: AgBGW3Nva7kn/GxNKinig8hj/b0KSa6nESiU2zKPwW6ZDhwTA3Kf3jcJCfKE2ISSLY3L7qNeihVnYP3f6S00y+b7oixLxvYxUxCGaVvNON/xtK+dFYoYQfh3hXevLWYuf7Bg5et6Rr2hoDJbkq6Nix2uIEMDiRjWyll3ma3ajlVSxbt4xdboXb/OqRzlCLS433nAruu2woSvw2d4ypoDHYd0uGw/W4ushxOEIDVC61kJU3T9K7UqzUK4bTMljiYxrwMAb3cZ0dnPw2FU/fpT9SDomCWtjnktMkfbzgVWgD/oIh+2IZH0p3Y9GoIDJUhHlBFudDu/t+tQ91F2Xpy3gqmWFSHkfnCex699hx4dBqAeSlf2Frwfz0ZR0CM4aSIKlPH1eyumz8VA5z8AmK8fDcWCvwYeRdS67qPcu0SDKodAlhA0XR3CSj+357TmId8KHS+DZCisrxAuMJGMt6FbCbTRxY6lM4yJHrkP+L9Hv3474deDApZix4h64pqiH5P1hYIsiDM02EkdI5JwrRuZewcfVMDzGhve/u9nSa5dMADrDdjKZAys8hCUgi//ynfJzpOKi/WbfvNeGcYbQdvDX9dyXTbvsSYQ/+jq8DnU69RwQWF3lYIwZAWeoGAThX4jvSppuDY+FtS/1TsrymLQSpPH4lXjgSw7yomRvNx2x2n4Pa3mPpgQ2SR4qKQX4lLeE2t+R39xKQreA4FO3XItLFnZ8ddLj1mcmatGhSMmD6AWI2hmxxxJJ3Of4AnBDrCoV68fYdPk/HeY3C8JfHQnSKcG + template: + metadata: + creationTimestamp: null + name: airflow-credentials + namespace: stackable-airflow + type: Opaque diff --git a/stacks/argo-cd/manifests/minio/minio.yaml b/demos/argo-cd/manifests/minio/minio.yaml similarity index 100% rename from stacks/argo-cd/manifests/minio/minio.yaml rename to demos/argo-cd/manifests/minio/minio.yaml diff --git a/stacks/argo-cd/projects/airflow.yaml b/demos/argo-cd/projects/airflow.yaml similarity index 100% rename from stacks/argo-cd/projects/airflow.yaml rename to demos/argo-cd/projects/airflow.yaml diff --git a/stacks/argo-cd/projects/minio.yaml b/demos/argo-cd/projects/minio.yaml similarity index 100% rename from stacks/argo-cd/projects/minio.yaml rename to demos/argo-cd/projects/minio.yaml diff --git a/demos/demos-v2.yaml b/demos/demos-v2.yaml index bc4d2591..00fc8d22 100644 --- a/demos/demos-v2.yaml +++ b/demos/demos-v2.yaml @@ -1,5 +1,33 @@ --- demos: + argocd: + description: Deploy Stackable operators and Airflow via ArgoCD and activate a simple Airflow DAG. + stackableStack: argocd + labels: + - argocd + - airflow + - job-scheduling + manifests: + ################################ + # projects + ################################ + - plainYaml: demos/argo-cd/projects/airflow.yaml + - plainYaml: demos/argo-cd/projects/minio.yaml + ################################ + # prerequisites + ################################ + - plainYaml: demos/argo-cd/applications/airflow-postgres.yaml + - plainYaml: demos/argo-cd/applications/minio.yaml + ################################ + # products + ################################ + - plainYaml: demos/argo-cd/applications/airflow.yaml + supportedNamespaces: + - argo-cd + resourceRequests: + cpu: 2401m + memory: 9010Mi + pvc: 24Gi airflow-scheduled-job: description: Activate a simple Airflow DAG to run continuously at a set interval stackableStack: airflow diff --git a/stacks/argo-cd/applications/sealed-secrets.yaml b/stacks/argo-cd/applications/sealed-secrets.yaml index 145df9de..73fd722c 100644 --- a/stacks/argo-cd/applications/sealed-secrets.yaml +++ b/stacks/argo-cd/applications/sealed-secrets.yaml @@ -14,7 +14,8 @@ spec: valuesObject: secretName: sealed-secrets-key - repoURL: https://github.com/stackabletech/demos.git - targetRevision: "{{ demoReleaseVersion }}" + # TODO: adapt to release-25.3 + targetRevision: "spike/argocd-demo" path: stacks/argo-cd/secrets/ destination: server: https://kubernetes.default.svc diff --git a/stacks/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml b/stacks/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml deleted file mode 100644 index bc631bc7..00000000 --- a/stacks/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml +++ /dev/null @@ -1,24 +0,0 @@ ---- -apiVersion: bitnami.com/v1alpha1 -kind: SealedSecret -metadata: - creationTimestamp: null - name: airflow-credentials - namespace: stackable-airflow -spec: - encryptedData: - adminUser.email: AgCmhs/OLHZ0j4CnTdZvfj76m+y7e62IuAeJ/LksxaTOWbHlvTJ0p8dBsp+WX3RfiuRK8M7kJDundrSKYzbkw8MISjoHF4/v4pSu6qWLBBuDPaMGh3i1bpAJ9gKUti4SGLhp7dr/Lx1oFWnY5T69WMARQqTbhHnpH70LPkFWKRvlIkCN26PbuYsE7stKXaVweEP204sfUChK5qzafE4Lcm5l+1yGFfILBBKAbBGxLX2I3O4n4LuP/iv0Ne14wGnxss80Qapt+ue8Ce16EZiJPtY+ZCXa05fYgtitVr393pYZks/fO9as76ABjWFc2lnE5GaGV58gK9V/5VRd1TBfkK3Y8UZPxqHNULlkwVh4Vy2EHHBMzhtec0m6w72aWuWFecyUNkrwIuivZ8e9tiH6aulpYqgLshnbDZCQi8yQVDNx/xLv4GH0/VvDUEIB7O13CHRL4M+OH+GYgLpJYqhTTkJNvXdrmjyK6kWeiErw8yG77akVplDGSG6LschN0DGhJVoZLLwbjvvM9em6tHu3gRSXAYhfG80bB9PMwmU9Lei+cdq6Fp7RceqxR3F6lnigP2e5gCaEpc6YYzBKEnzgcdlR1iA57D+5lBK9SwvWkppH5/dbPRkzt5+kqWaExRq8VmEnwooiBJ/4Jx4i4FKWFnx3cnDF5OYXovIKPyYPZW1HVvifrjHHZAOe1jORxYpMOwPYOLraDIO83comkVEBzCVUOh8v - adminUser.firstname: AgB3NcEnKhwMsyFPOfSTfxKFkhIg1TyeixtdqW8RSq5I6zrMX9we431QLuOSfxQlaizsm3o670wjO+4yDNpOGB4/vXaSSpA+835tiMDymJ/URR7vb1vvLV/xx6azdn5JmmHWd+yEja47TxQ1ip5KDai1Oxc0BKbgzu2Ogalq1U/56riSKSe8MvRjJ9KY8IwBpoY1rKAjTFJWavUVVt5Ms0fdSB7sL69NoKVzUQEpucTemy6k8RQFabK+KwirQ2KSDcQ5J+df/ODx1/HZKG8j81Wla7dBJ0nwIeie0StAZdZ+j9QlsWQF4zCRaEPxn9p1vken7Na/EqQdYwIV62o3mo66p06t+ObdjxKRHEO5BiY2lYqY3XcgaETBpaDfl2PiMbneLngGpBfOQSKnVZC60svqB/BpvBlMCiwcr+0ZJczFM1f+ptmJGHLJirf1kTYKvm6ezstGussB848Ca/ayhzzt9cestW9UFS/SlAo7d+N+YUKv3O+8QPIb5AdN2nPN9kcQxdh2T+ogtg3yIKsb6makP8LC6Z0Hl8C1DuzSh3Inj5VxcmkRuB9LRYymfNu+6GjkVlAWk58FUdQM5+5syMDgiHdVIMmnoTjRNm3mCm3l6g6rG33ggaAI4X1ypyjnmMlxbpRJi/+KgsQNNByYYQG1Foo3MZ7JFQojABBFMVSCPumMyjR7i8kvYCTr7acrHS3ZSjvR+Msz - adminUser.lastname: AgCYaEJ973OSMSL0vBrQcHbFtmqQls6Ids1xcPVz3/uVpxZw89Phn67Xx6dPSNHqZIYHWbZ/gV24xOJd0XbtlFINUknGgjwqjwGJHm4Q+0F1nmCVmhrRqB+Jc8I7qqb5BT6XpsgQOa0vazDc+NVKlgm2DmbIKz8VvHrhUNRb1rd33RYd3lm0Kj8lSl4QZm8tIX+BZi3J9gSLDIMzGQftg5BxS8Wt8cbYsMW8lFeZ6Dg7Fmx9r0GCeAa7kZWy8UME0saeKj13tg/oYj79MNFCu/Q510Q+YbLcQAJ22ifPgIaPdN4YK03DYL9qiIeo4im8rlgfKqHaNkfNzmXXh758T4s74Nau9KLdG+qE2+j175B9nrYjBYlT2EDoJIF6m1iiOCAc93tk4FIc3fw5RNLkOsAYdEWoECWLOdDloQsDHa+DGwHnF6d9ZpzlATnKLMOJ6F3+RAW10NzD38bWMlUUJoGZTuhVOz9tQCyFs2/SqMC8/LauMrHx2TFbKEq2kpc4MRTYwUQe9L+zQFI2Oc5HXGulXDdteShL6ayT8sXGRIsNI0lCpNhHsUauGJ7sSkW4bklGMEPzRh48+Lua2/QQKOcLWxgPx53b492/7SvWExx7rs+POSZouLdwUlngXC2D1H5yvmnPXEnnTLVGwBNDRBa+BBefocVOmq6yZGX5trkhudTr/Nu/rZQkDGUtSfiMDzd6Uo73LQ== - adminUser.password: AgA7BxW4/FggUvHMVvnYxdP6Pmju3uSskl29Lc1ywr6AbtCSqiysp29Ravyr0muT+PlEb5zmUYrct+HaffstMOudDCYw5wSm50hhelRPFgqppJgoUvXwfeUMkx73rrkddfN2RnCepUZxmkJ8PcXA38+KPwTZM4zZmOHeq7lgKDlsjgin/O0MrAeochZGJ1JmsJ0B3QUxFit4aURq4Pko67ytrWMJBO2KQRLHUm7Swf3umqjCBWuqfqhbRtKrWEOhoAEHDzKOq4fUCaV5rf2uxpvNCC0KJY6zmhPcPgZAO+Wuc+W9iNmM4ROoTFCvDS/stXcneDR7oRObgtXN5gn/0xno3xhdypg0lqfm2bEoEiokwu2GVwrzPr5iNI2HlUVhXeo9bNwjJ1alWjt0faUvwtZRSbKI1AihBF0gelbXhKT9Iyya336TTuz0riW3sFeDixHDTRoVC3ZaHzvCTXLTLtw3xgmhxM1mCebU+pxS8DV4AAIy9MfkK4ZHkjpcCdVbV5D4Mt2A9Ltm1O7qHlKsdWF0+vMmllLn7dZzAmtAiCiI9srOUh0xj+bIOqqN2Ky8a2K73JPh3kXxO5njW9N0/lFFQ4D98VpgI/DDqoZwX97yzLcthLb54xr6OUtduvX6rDakgrRZT4OCSvlGUIWzvtTcoDwuBubSuOnLh6AFNd7xz/Ik+eepInSOCMpd3HIdp5KBB3TPm9eLFcLW - adminUser.username: AgAQWpaTaFTvYsZx0p9Am3iTwoUEZ/pWiETvhVSF0NANgKe17V5b1lQj+DamMVlFUGfGgV8Qcxdyw4Lc71yzGgwEB0fQERoy54Z2lwVpYLJm2qLcXuolkzFSyZEMBBulddkTGhF/zhDZWfpOnayhfAbQyyp/T3KUuZira220DN9HiXDb5BiId1+Gzu6xTW6WD5/c+7yFiD6SWFReGpIuqHofh4y02p8A5RED2jtcr/p7ltCZ/tqjIpnZcTP5K7wauEjDiJmDbLTcdAmzdSEOI4Q952+oK3xjPAIrhhivC3DAtQ0lgV2ye9859tx+CDEEkLkjewascVNwmX1yeYPxnh8ayJfabkEpNM8KOfmrDE/AZB82FeKChmvAcpQAu3fUuxBMyUit6RkNW73fvDf8hIyOmNSZdr0f21FPhJWFYnsF4Qfm3zOo/fXuG1LJz2WB2Qy7RuTmRGW1AE4VQOvzvVDCEOq5BsVfPdlGrSfo5oeI6ctCqTL4S+2CXS3VRUkIX61QdN54CU7ZeT4cAzZGpkUP6elfsrLv0KnQc4KvvzfBNu71j3xgjjYN41aGDNi8dqWaCVebYbuQtDqbcxLxUQNX+VqBdCflMcnCOC4kAEI+B2n/rUrGLWjKUIMhZvZIKJQd437j1fvJo2AHO8TbqtORspg3/H8OuwZpwcDsWl1dVQ1B0EczZweo3I5NkytTVxNEEXjsQA== - connections.celeryBrokerUrl: AgBEl833Xp1UogzJpkpJ1zhjlw6KKAqcfWzSqQ8LIPQ96GaIUfo6G8wx7jozw7yEF0vnAuTTdzdEZ7x8/PVwWQyqNJKiS9o6mIEdkhv9mQ+EG/wzpB8dnbQR4Xl758f+dPRqbIA0RYetmcD9NsiF08obXIIzZA+G95kR6x24WiixwJL+txMNdno0Qu92fRqasv6jV22CUju/kUTRFEov5IrzGOj7gLKQ7d2yrCGZd/W9SnP9sgX1ltz78C2yiEXX7aI0n066OjJUxE7DFTJCUWNrknDcJuSjGFrWTzf8GNydrPmkVp6g5JA3fGgUTm76osischHr/0/+cwzQzEyQKyb5H/fxcAd/5yqtXhD/xibPDoVHs484PQAmgy8M3PMG8fNQ9XyrXNB/ZuxcCnt+rqPkeP0jqEsdxog2Q3U1lXfH/FU2eJr5tLHb1zNISV7vNX28VRzLoefyYFtHkp1F2yVYThmuSCPj1cqkfOpt02rYsIDIYqwryxxOvUpTAH0uNbdoIWSvXTzX2ZARUVVm0bboDgfe4RxVQofxKjk1mNPHhEZ9jYd6GX1G1SCYuSVdrRN4fLSOxlyP+vPkyzOJtQ7WmugBZFTmZ/iUN38ce4HbDIajVKtMvDqD35kkN/q4NZ3d+/YjmZA9YvJ03Hd0XNznImr8R4wqOt5oZsK2+XaMxHTzPfxV2MpLT9Xgb+CAHiGKvCaheKdRNWODhFQQXAw8l6t4z8dvYNkZPAZAD6ehFUtlvpTbHVVqtEdiTA== - connections.celeryResultBackend: AgBywQulzF0AZOPR8ypHDnsQIicOWUri9rV2sLwR8kiD+J98eT6CWvblzUv8x3/I4gJWwHbwjmACv5Ke+MD1GxqVsJYl2zc/A78SkVvNT6V+kaUFqxXMEQ3EJN7q+BHUPFphxQDS2Ur5+ssON1hth56L+HAxZLmt4XC9Fjz20cO7Dcx+vpgXG7KVtdRmLvchs0PCaMnMZDANg/rgGC9lMcgi3BROhobGapbp3+iLjiIDtNJ4SwwQpQaA5KWKXqC8Dmfvtm18lyNAK5oWSrMQI6+eWe+ML56C8bMlz3jtlZsFO6upe7eOueD1WoK1+YI+NEfvw3HDChgMaQ559x3y/pIG94F7QNtltZbfTfXptStQru+H2rxmCUBYb/BaHv3oqZsOgQe95LubQNfrpH9tIIygGZcQDZGjZsnfjmu1ahYzqiKb6UEoPxZJskIMy5BAakWQf0FiVJQa7stt7/k/NosBrUfJbyPJNWA6xVwilV7K4Auz4MuBsMk1fRofgpw3N85SeipVmU3673GIs+L1IFy8UEYAtYpv14aFOvqJrE4YBpkufr+K/IvVpYfvIBL2zAF3CKm9SEzYAQnADwb3UgOCIm8w233vogjHo5oqL8wacUeZgzrZ+T1FIR9e1tXrYxk2VLmyv5d0VOWYe0AleUk+O3rVdLpi5yUJQjvKnExVXzMMLsApTJ3dy5OHwou99Po1mlg2D6xh439hDUlAcUQz4NfGYvJtYSgAb5cjMs3bEMUcAuadGIGidlIy74HQArfXosundworwKC8 - connections.secretKey: AgBpwR5AZJUjGxJwi7rS6dLH7WJKYza4wKjPxLF+RVskwtwtkGtYSpbRZ+vXxE6pn47eJQC0OpJzreiNqYZtSR1CHv6HyHZsnqCSnLSzlUBCPp9H51nfVQ57V+S967/jEVJ3V7446gqQqdJb3IS4uwbb+G8CJsjsZiEcwx/z/OK5vu+k7TiG5jiyksxXhC7EFzYKfc/CD8Lsr6LBimUEa2i1W/PhOoDcqKa7qvDMIM/iAEilvtHXjA+EU8kxplDYy2an1cadockyyMSoHvDmSOg6BU62bK8Xqx9o0X6uaSrTufeueymtBYb/wVHB0bTxm1UXb8QdGYfDePemnP0oRNwvETrupuD0HRgEutetyxLYEEezC98ZpRM377kirX9Zqb0/fZ9YVwECBL7hznl/sp8w1hE2O76orf3iAGNMftrFtOjEyzro+41zd+bz+sy1ivGluIWwovQbgondk25AoYRC0LhGDl7kQDVX3ND/XeH2NYErAuCEyjQ1MGHLJxQGGDxoCPaDXtLMuHvXlg3H71jibzT7rV+YikAExjfUVim0kuDtKqdBwAhvRqX+Si/kCRMrIUhRowWSqx0g54U9mwC6wfbSHV90RnYtvBPbFc144WbgV7irc5bfnekidrCj8MiIPKegUByqlB26ziyexEmiRN74BH2yCJ/M2yf1IWSad0uqb5d/48BcgEP8E2BOgeeaCJ6WOMeR80qvas07hN5e - connections.sqlalchemyDatabaseUri: AgAFm5FacxXw3q9rxKT+9eTpY3FPf7NSp3wdPUUzkBpQPsEa0JvafDeIZ5/+MC9WaFBWuSt0awrIh/FyHlNW4/KRhTWzFL4h2MrDAfCEdCaOgB4wrgYJF7tGz3968eNhBn0ALX7MU6dgiGawPGvX91FgPo4dwz/5Ob5+f5MhA/paH8uBXR8W96cTTeJlcJaqLV/xMzGCInurUpOwyrLWu8zjOHNdn/11P3n5gKVlwbi6w57AKAYtqux869oh9al4kIbgIEbRYRI42Q2Xa3v+RtmFCX3CelCs6uoWtzNgt+wUbQbfi48MuHrakiJf49yrqFHdc2vfj03O8jlKLGSGcSsy5Epbb1VBuTh04jVXAO03xb6Top3gqZvxY8/gWOWPs0SejeilvubNIwj40bhzRxPj+r7TnE2dmS+KnTN1fYR1fk39C6SNjSKVncMie8PH/xfq8WOVZSGhzlIwYNMs7/WpXYLVs58Wz+pVVRcbYf/CQjRNKy4ikEf+eHwMLvmsvHnslxr9S9Bwt2An7mFkkO0VvGubCYfOK/W1SoN9NBhFmJO6Es3vfTKuS1gqgcJAzd1oz0+hz6Kno8XRpZUzdUa+P8LPo4Y+mhif+y9OH0sXFH5rYGWvLbUxEgFfdUMWQRKpGRqJiGNKFBLo0kpRt/DqG3v9wcWHbYLl+/cRLCsC46FUsPW5L73T4AEtV3TLh6Q5hZiHCWTEyxW4WyeFD6Bd7FLPNbIA8gMD0oCP3ztd+odbSDecURUzhQifICH/SCJl4VH8jvr40Rd+TjrqEj5t - template: - metadata: - creationTimestamp: null - name: airflow-credentials - namespace: stackable-airflow - type: Opaque diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 35e45149..ec207d46 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -20,8 +20,6 @@ stacks: # projects ################################ - plainYaml: stacks/argo-cd/projects/stackable-operators.yaml - - plainYaml: stacks/argo-cd/projects/airflow.yaml - - plainYaml: stacks/argo-cd/projects/minio.yaml ################################ # operators ################################ @@ -35,12 +33,6 @@ stacks: # prerequisites ################################ - plainYaml: stacks/argo-cd/applications/sealed-secrets.yaml - - plainYaml: stacks/argo-cd/applications/airflow-postgres.yaml - - plainYaml: stacks/argo-cd/applications/minio.yaml - ################################ - # products - ################################ - - plainYaml: stacks/argo-cd/applications/airflow.yaml supportedNamespaces: - argo-cd resourceRequests: @@ -51,10 +43,6 @@ stacks: - name: stackableReleaseVersion description: Stackable release to be installed via Argo default: 25.3.0 - - name: demoReleaseVersion - description: The target revision, HEAD or e.g. release-25.3 - #default: release-25.3 - default: spike/argocd-demo - name: stackableOperatorNamespace description: Stackable namespace for the operators default: stackable-operators From 46f48b44b0e714503529f52940d603b1c9f3f432 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 10:31:52 +0200 Subject: [PATCH 17/64] install all operators --- .../applicationsets/commons-operator.yaml | 42 ------------------- .../applicationsets/listener-operator.yaml | 42 ------------------- .../applicationsets/secret-operator.yaml | 42 ------------------- .../applicationsets/spark-k8s-operator.yaml | 42 ------------------- ...operator.yaml => stackable-operators.yaml} | 36 ++++++++++++---- stacks/stacks-v2.yaml | 10 +---- 6 files changed, 28 insertions(+), 186 deletions(-) delete mode 100644 stacks/argo-cd/applicationsets/commons-operator.yaml delete mode 100644 stacks/argo-cd/applicationsets/listener-operator.yaml delete mode 100644 stacks/argo-cd/applicationsets/secret-operator.yaml delete mode 100644 stacks/argo-cd/applicationsets/spark-k8s-operator.yaml rename stacks/argo-cd/applicationsets/{airflow-operator.yaml => stackable-operators.yaml} (54%) diff --git a/stacks/argo-cd/applicationsets/commons-operator.yaml b/stacks/argo-cd/applicationsets/commons-operator.yaml deleted file mode 100644 index 19a2bbc6..00000000 --- a/stacks/argo-cd/applicationsets/commons-operator.yaml +++ /dev/null @@ -1,42 +0,0 @@ ---- -apiVersion: argoproj.io/v1alpha1 -kind: ApplicationSet -metadata: - name: commons-operator -spec: - # this interferes with stackablectl's templating and - # cannot really be used here properly but must be provided. - generators: - - list: - elements: - - cluster: development - template: - metadata: - name: commons-operator - spec: - project: stackable-operators - ignoreDifferences: - # mitigating: https://github.com/stackabletech/hdfs-operator/issues/626 - - group: "apiextensions.k8s.io" - kind: "CustomResourceDefinition" - jqPathExpressions: - - .spec.names.categories | select(. == []) - - .spec.names.shortNames | select(. == []) - - .spec.versions[].additionalPrinterColumns | select(. == []) - source: - repoURL: https://repo.stackable.tech/repository/helm-stable/ - targetRevision: "{{ stackableReleaseVersion }}" - chart: commons-operator - helm: - releaseName: commons-operator - destination: - server: https://kubernetes.default.svc - namespace: "{{ stackableOperatorNamespace }}" - syncPolicy: - syncOptions: - - CreateNamespace=true - - ServerSideApply=true - - RespectIgnoreDifferences=true - automated: - selfHeal: true - prune: true diff --git a/stacks/argo-cd/applicationsets/listener-operator.yaml b/stacks/argo-cd/applicationsets/listener-operator.yaml deleted file mode 100644 index 9053f49e..00000000 --- a/stacks/argo-cd/applicationsets/listener-operator.yaml +++ /dev/null @@ -1,42 +0,0 @@ ---- -apiVersion: argoproj.io/v1alpha1 -kind: ApplicationSet -metadata: - name: listener-operator -spec: - # this interferes with stackablectl's templating and - # cannot really be used here properly but must be provided. - generators: - - list: - elements: - - cluster: development - template: - metadata: - name: listener-operator - spec: - project: stackable-operators - ignoreDifferences: - # mitigating: https://github.com/stackabletech/hdfs-operator/issues/626 - - group: "apiextensions.k8s.io" - kind: "CustomResourceDefinition" - jqPathExpressions: - - .spec.names.categories | select(. == []) - - .spec.names.shortNames | select(. == []) - - .spec.versions[].additionalPrinterColumns | select(. == []) - source: - repoURL: https://repo.stackable.tech/repository/helm-stable/ - targetRevision: "{{ stackableReleaseVersion }}" - chart: listener-operator - helm: - releaseName: listener-operator - destination: - server: https://kubernetes.default.svc - namespace: "{{ stackableOperatorNamespace }}" - syncPolicy: - syncOptions: - - CreateNamespace=true - - ServerSideApply=true - - RespectIgnoreDifferences=true - automated: - selfHeal: true - prune: true diff --git a/stacks/argo-cd/applicationsets/secret-operator.yaml b/stacks/argo-cd/applicationsets/secret-operator.yaml deleted file mode 100644 index 7312c0bd..00000000 --- a/stacks/argo-cd/applicationsets/secret-operator.yaml +++ /dev/null @@ -1,42 +0,0 @@ ---- -apiVersion: argoproj.io/v1alpha1 -kind: ApplicationSet -metadata: - name: secret-operator -spec: - # this interferes with stackablectl's templating and - # cannot really be used here properly but must be provided. - generators: - - list: - elements: - - cluster: development - template: - metadata: - name: secret-operator - spec: - project: stackable-operators - ignoreDifferences: - # mitigating: https://github.com/stackabletech/hdfs-operator/issues/626 - - group: "apiextensions.k8s.io" - kind: "CustomResourceDefinition" - jqPathExpressions: - - .spec.names.categories | select(. == []) - - .spec.names.shortNames | select(. == []) - - .spec.versions[].additionalPrinterColumns | select(. == []) - source: - repoURL: https://repo.stackable.tech/repository/helm-stable/ - targetRevision: "{{ stackableReleaseVersion }}" - chart: secret-operator - helm: - releaseName: secret-operator - destination: - server: https://kubernetes.default.svc - namespace: "{{ stackableOperatorNamespace }}" - syncPolicy: - syncOptions: - - CreateNamespace=true - - ServerSideApply=true - - RespectIgnoreDifferences=true - automated: - selfHeal: true - prune: true diff --git a/stacks/argo-cd/applicationsets/spark-k8s-operator.yaml b/stacks/argo-cd/applicationsets/spark-k8s-operator.yaml deleted file mode 100644 index b5686b60..00000000 --- a/stacks/argo-cd/applicationsets/spark-k8s-operator.yaml +++ /dev/null @@ -1,42 +0,0 @@ ---- -apiVersion: argoproj.io/v1alpha1 -kind: ApplicationSet -metadata: - name: spark-k8s-operator -spec: - # this interferes with stackablectl's templating and - # cannot really be used here properly but must be provided. - generators: - - list: - elements: - - cluster: development - template: - metadata: - name: spark-k8s-operator - spec: - project: stackable-operators - ignoreDifferences: - # mitigating: https://github.com/stackabletech/hdfs-operator/issues/626 - - group: "apiextensions.k8s.io" - kind: "CustomResourceDefinition" - jqPathExpressions: - - .spec.names.categories | select(. == []) - - .spec.names.shortNames | select(. == []) - - .spec.versions[].additionalPrinterColumns | select(. == []) - source: - repoURL: https://repo.stackable.tech/repository/helm-stable/ - targetRevision: "{{ stackableReleaseVersion }}" - chart: spark-k8s-operator - helm: - releaseName: spark-k8s-operator - destination: - server: https://kubernetes.default.svc - namespace: "{{ stackableOperatorNamespace }}" - syncPolicy: - syncOptions: - - CreateNamespace=true - - ServerSideApply=true - - RespectIgnoreDifferences=true - automated: - selfHeal: true - prune: true diff --git a/stacks/argo-cd/applicationsets/airflow-operator.yaml b/stacks/argo-cd/applicationsets/stackable-operators.yaml similarity index 54% rename from stacks/argo-cd/applicationsets/airflow-operator.yaml rename to stacks/argo-cd/applicationsets/stackable-operators.yaml index 109ecc94..46a08902 100644 --- a/stacks/argo-cd/applicationsets/airflow-operator.yaml +++ b/stacks/argo-cd/applicationsets/stackable-operators.yaml @@ -2,17 +2,31 @@ apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet metadata: - name: airflow-operator + name: stackable-operators spec: - # this interferes with stackablectl's templating and - # cannot really be used here properly but must be provided. generators: - list: elements: - - cluster: development + # Left out as currently installed via stackablectl + # operator: commons + # operator: listener + # operator: secret + - operator: airflow + - operator: druid + - operator: hbase + - operator: hdfs + - operator: hive + - operator: kafka + - operator: nifi + - operator: opa + - operator: spark-k8s + - operator: superset + - operator: trino template: metadata: - name: airflow-operator + # {% raw %} + name: "{{ operator }}-operator" + # {% endraw %} spec: project: stackable-operators ignoreDifferences: @@ -25,13 +39,17 @@ spec: - .spec.versions[].additionalPrinterColumns | select(. == []) source: repoURL: https://repo.stackable.tech/repository/helm-stable/ - targetRevision: "{{ stackableReleaseVersion }}" - chart: airflow-operator + targetRevision: "25.3.0" + # {% raw %} + chart: "{{ operator }}-operator" + # {% endraw %} helm: - releaseName: airflow-operator + # {% raw %} + releaseName: "{{ operator }}-operator" + # {% endraw %} destination: server: https://kubernetes.default.svc - namespace: "{{ stackableOperatorNamespace }}" + namespace: stackable-operators syncPolicy: syncOptions: - CreateNamespace=true diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index ec207d46..6355b5bd 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -23,12 +23,7 @@ stacks: ################################ # operators ################################ - # currently deployed via stackablectl since it complains if no operators are deployed... - #- plainYaml: stacks/argo-cd/applicationsets/commons-operator.yaml - #- plainYaml: stacks/argo-cd/applicationsets/listener-operator.yaml - #- plainYaml: stacks/argo-cd/applicationsets/secret-operator.yaml - - plainYaml: stacks/argo-cd/applicationsets/airflow-operator.yaml - - plainYaml: stacks/argo-cd/applicationsets/spark-k8s-operator.yaml + - plainYaml: stacks/argo-cd/applicationsets/stackable-operators.yaml ################################ # prerequisites ################################ @@ -43,9 +38,6 @@ stacks: - name: stackableReleaseVersion description: Stackable release to be installed via Argo default: 25.3.0 - - name: stackableOperatorNamespace - description: Stackable namespace for the operators - default: stackable-operators - name: argocdAdminPassword description: Password of the ArgoCD admin user # generated via: `htpasswd -nbBC 10 "" adminadmin | tr -d ':\n'` From 5dca1f82cef9749447491d54b05b01422d4c16dc Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 10:34:36 +0200 Subject: [PATCH 18/64] fixes --- demos/argo-cd/applications/airflow.yaml | 2 +- demos/argo-cd/applications/minio.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/demos/argo-cd/applications/airflow.yaml b/demos/argo-cd/applications/airflow.yaml index 9196c2cf..edf5ac43 100644 --- a/demos/argo-cd/applications/airflow.yaml +++ b/demos/argo-cd/applications/airflow.yaml @@ -11,7 +11,7 @@ spec: repoURL: https://github.com/stackabletech/demos.git # TODO: adapt to release-25.3 targetRevision: "spike/argocd-demo" - path: stacks/argo-cd/manifests/airflow/ + path: demos/argo-cd/manifests/airflow/ syncPolicy: syncOptions: - CreateNamespace=true diff --git a/demos/argo-cd/applications/minio.yaml b/demos/argo-cd/applications/minio.yaml index bfea03c9..d3eaf0a5 100644 --- a/demos/argo-cd/applications/minio.yaml +++ b/demos/argo-cd/applications/minio.yaml @@ -11,7 +11,7 @@ spec: repoURL: https://github.com/stackabletech/demos.git # TODO: adapt to release-25.3 targetRevision: "spike/argocd-demo" - path: stacks/argo-cd/manifests/minio/ + path: demos/argo-cd/manifests/minio/ syncPolicy: syncOptions: - CreateNamespace=true From 95f9e5a6b2679cf245d34828907baf0369d3dbaa Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 13:41:28 +0200 Subject: [PATCH 19/64] use sealed secrets for minio / postgres --- .../applications/airflow-postgres.yaml | 35 ++++++++++--------- .../sealed-airflow-postgres-credentials.yaml | 17 +++++++++ .../airflow/airflow-credentials.yaml | 17 --------- demos/argo-cd/manifests/minio/minio.yaml | 15 -------- .../minio/sealed-minio-credentials.yaml | 17 +++++++++ 5 files changed, 53 insertions(+), 48 deletions(-) create mode 100644 demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml delete mode 100644 demos/argo-cd/manifests/airflow/airflow-credentials.yaml create mode 100644 demos/argo-cd/manifests/minio/sealed-minio-credentials.yaml diff --git a/demos/argo-cd/applications/airflow-postgres.yaml b/demos/argo-cd/applications/airflow-postgres.yaml index d59f9961..2d06423f 100644 --- a/demos/argo-cd/applications/airflow-postgres.yaml +++ b/demos/argo-cd/applications/airflow-postgres.yaml @@ -7,22 +7,25 @@ spec: destination: server: https://kubernetes.default.svc namespace: stackable-airflow - source: - repoURL: "registry-1.docker.io/bitnamicharts" - path: postgresql - # helm inspect chart oci://registry-1.docker.io/bitnamicharts/postgresql - targetRevision: 16.6.3 # 17.4.0 - chart: postgresql - helm: - # TODO this breaks naming as long as we use the airflow stack yaml which needs this svc name - releaseName: postgresql-airflow - valuesObject: - commonLabels: - stackable.tech/vendor: Stackable - auth: - username: airflow - password: airflow - database: airflow + sources: + - repoURL: "registry-1.docker.io/bitnamicharts" + path: postgresql + # helm inspect chart oci://registry-1.docker.io/bitnamicharts/postgresql + targetRevision: 16.6.3 # 17.4.0 + chart: postgresql + helm: + # TODO this breaks naming as long as we use the airflow stack yaml which needs this svc name + releaseName: postgresql-airflow + valuesObject: + commonLabels: + stackable.tech/vendor: Stackable + auth: + database: airflow + existingSecret: postgres-credentials + - repoURL: https://github.com/stackabletech/demos.git + # TODO: adapt to release-25.3 + targetRevision: "spike/argocd-demo" + path: demo/argo-cd/manifests/airflow-postgres syncPolicy: syncOptions: - CreateNamespace=true diff --git a/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml new file mode 100644 index 00000000..3af276ae --- /dev/null +++ b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: postgresql-credentials + namespace: stackable-airflow +spec: + encryptedData: + password: AgAFfYyDQzfNHmkIuwkQjm8Z25AdLgelTPx43vKNggCrfHBjcBoA/5FKzwrfQxQldVFmuQtn7PpVagu6aZZfl+JBmzLwjYFA6kQPKYtLoskYvZpocfCAsWQJIp5yusI1780lT7b4JTz7LDkzVdy5aK2wm6/Nz5y5NzpEZNxThdc8tnzz0VtZHCpMdgi9q3CmhEPeWn0pOroocbCTkBg9pMiOOTH4YuCv1ca0/xtSSbFbFvN0luXhLIBBPWx7GVlJZGggVtiHNvvvCYVAGXogSXKWTAJv9HQvsV9ruHIa94UeWXnAXV9fhqq1nD700Nhmy0uDjBjntvvNTsKqhhVwOBKuvDYkmPXzSA98A+okty3+3Mv9D/fbAic20636tpzfEIAVGvnMkAhVhY4jth7QpTi4MMk18IiVOm58kSJz/oXDEpRB3CN6a7aB0HHf1xTqxKuqQ2FuZku+kriMAdgrBxCFGJPAcfi9PoqPjRWAawKrW3B9T8TJVYxnXmDvGUIaEMli9++J4laTI2ClER3udi+ZmcVZ3YZYrTdr7dhh7vyMJVLYIde9vh01FGBQZ0EOCPDp3ch0TtLKAAnH5jI0CYPvOgyLo+vUJufDIrIgRY7jwTZD9wmulQaFxH2V8CumCH5LHqOVLbY49oQZtAVrqIWUVyEejKgPkHGJiPQ63jHmz976RTdddAoi0BZ9LrLY/bqY67tctJtm + postgres-password: AgB7V06AGcL09k9vpmHOT417G5QY/sCmyN7OONfzEup898UpjD58SailnyZRSGwkhxo5Mhz9PlVP7AaFnMf+ZY/kVxEgjZ0kjBROB32e7zKwsITnp6ix7WJ85TYItde7P8YgyphmP4xxGx1crkFNjNDwJP0HtmIt9j0RN6EX+jDUupP9l6fH0LHFnJVE9NU6o2xvJa9gQs+iJ9peLzL8Nc4uXJiXrrdXl5ptrYti5w/cJpkKzlVoWpAeph390eog5QPU0dardhc/9WnmVfomjkJE4axwZl/yHbmcQTR0FgxXh24ICHRFBjPv9PdFhs7LsXqffkkt7RILEflUBBrSU2lCusn2Jscg6F4DBDR/6aa4eDu1tsk0UL2PVQzErErDt16qKPLaAygdevSsRCjXEcYlMnj+eFFAHiplEVMNc32yc70L13wf1rnndFAfXLNI2v1Aq6OoLnZ2HHFtpNxW8FKE+5vOyxykuUtU/E/alR5o3OrgZBgvmYQQLUap+NMAXjbbk/ZnVIAhPANOIs7rM2b5RLJmrEmIZkOZDPYt8/aRZezPdVb2THo2H3N1daOI6liA3WjdzFPNsawFRkDh2cBfQ308GzM49PhlGV5DDu99qDXWhy1a6nMdexZRiyv1cZ+GoBT0AKollpx6xh5XqF19d4BIpRFZlxHQ5b5heIQDNE639g3QsSTFqW9nS0NBVQLlajOJFwQ3 + template: + metadata: + creationTimestamp: null + name: postgresql-credentials + namespace: stackable-airflow + type: Opaque diff --git a/demos/argo-cd/manifests/airflow/airflow-credentials.yaml b/demos/argo-cd/manifests/airflow/airflow-credentials.yaml deleted file mode 100644 index aed1fd28..00000000 --- a/demos/argo-cd/manifests/airflow/airflow-credentials.yaml +++ /dev/null @@ -1,17 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -metadata: - name: airflow-credentials - namespace: stackable-airflow -type: Opaque -stringData: - adminUser.username: admin - adminUser.firstname: Airflow - adminUser.lastname: Admin - adminUser.email: airflow@airflow.com - adminUser.password: adminadmin - connections.secretKey: airflowSecretKey - connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@postgresql-airflow/airflow - connections.celeryResultBackend: db+postgresql://airflow:airflow@postgresql-airflow/airflow - connections.celeryBrokerUrl: redis://:airflow@redis-airflow-master:6379/0 diff --git a/demos/argo-cd/manifests/minio/minio.yaml b/demos/argo-cd/manifests/minio/minio.yaml index 8681086f..687e37a7 100644 --- a/demos/argo-cd/manifests/minio/minio.yaml +++ b/demos/argo-cd/manifests/minio/minio.yaml @@ -5,21 +5,6 @@ kind: ServiceAccount metadata: name: "minio-sa" --- -# Source: minio/templates/secrets.yaml -apiVersion: v1 -kind: Secret -metadata: - name: minio - labels: - app: minio - chart: minio-5.4.0 - release: minio - heritage: Helm -type: Opaque -data: - rootUser: "YWRtaW4=" - rootPassword: "YWRtaW5hZG1pbg==" ---- # Source: minio/templates/configmap.yaml apiVersion: v1 kind: ConfigMap diff --git a/demos/argo-cd/manifests/minio/sealed-minio-credentials.yaml b/demos/argo-cd/manifests/minio/sealed-minio-credentials.yaml new file mode 100644 index 00000000..5a030480 --- /dev/null +++ b/demos/argo-cd/manifests/minio/sealed-minio-credentials.yaml @@ -0,0 +1,17 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: minio + namespace: minio +spec: + encryptedData: + rootPassword: AgBSBE4tAAJHV28O13Z+lg4CAzk/snvC+2WAM/ErBUeg4JiQZBIT+9o0MAIwM/IPcT7TpjrIpjx9UUiM5o8b/xjBIkxYBkpr8dNPE57IX+yVyWSxe5CPmBqazxEHU6cdlVT+s2NrydxyEsreSd6gs3+sHRd88AHQUhf3QJiD+hV2UE33jRe/so1if3OG9om5mNa/8EOqXeBnNBTCynaMDDuGyUcCUyzh5MEdaLG1YUu9flYCqlXNEsIkDR+1vgmbX1gcYdGcAIF7C71g8KMnGSt5dtAzxyuDhtDTCaq4+z3FbwEoJ4JQpmiDZzqtR5FDVUD8yS9b01Weg2cSlIEeNq83qN40wn8PkDQ6GEkl8Dou6yBpVklgnNJ+zgr+CxmFjrE6tGJnKQK1eOZmVCW43QFOOBvhyqXE0gpIlTO6rTzwfawHyWT1Wl83qCCcslb+/YylQx4tl/7fdFda5aNRN+1KQrfhkO9kc1zqeaJgO2nI31OpZOxu2U86WF9cQ/acFtOcF4/DDfYOu2dvvzyfzazfj6NIP0P2DjIpqIt7lTZ5t0SYgWB+cf5afsR8Y2TrO4K5B4Ys1Q9RTojqTuL/lyjuhNxYm2b/sIaQbBoJthntNrwaQO/KJ2nWngeRA+lFZTgbWigR6eQaLZXMIA+GioKKLiQxDiLeIvqfVMO6fwQWpeqznBruuQjVcB1aH18wLejD89ze9ztBI9JB + rootUser: AgAydeD6HzjMO94+dVqBLipjRRMLqHBDnAgqvDc+GUwT7dUuenMl2Qjx6D4G9Lk6HIuoqGpW1ZNVs65w+IEIETUGBR23I+E4drp2DFkGamXng+flkCJIvvvnmVoHYSDvO1mGEaqiF109vPOqj8vHtNHDjM89DdVWvdMkLQzM/XWK8pjIY/9g+K+a2te9yM3MzzlQBh7DPw5WdL5Nn97Ip5dUMA9D6fSLNidSqz24TLBbS9cLvRHERy9BLow7MqxyK0017iNH8TnpIzWphE+6iH9qPXGLNVlmeq8Ex4L74x3BIyas+yC2ErXPlvBNb/PG+L/788d4ZHW8JRNnQRhI2u8tvrf/Gg55/9a6hu2DQ+oqgT5jSJTPUE0meHIwAIVtLfxHozHYPD0IIdsgZgj89yfzaM3uIv02TnqAaDU113DVq5r7yZhh/THz9euw82/lPJYpuYwuSOuarddz18mOYWgQrhfBgOINxOr/OsFBl2HAfxnJyE2J4N50o4g9P5idr8+Px8P2bIKmrIbYI/YWVkHuGrslyXJFvdQvo3VJVXp7qGVJLOu3BTujxHc13CfMMDQGWXP3LvUFtDYH2fn1LHl0TlSlzyNzFdRir/b75tNKWRRuyOQ4qUJGSES1sDCmK7C8BA7d4oD0Sn0HZt7n4xp+3+b/8Rmicm/w8Uh1syTaxgW/2KQA9iXmHT8Un6gET71NF9E8mg== + template: + metadata: + creationTimestamp: null + name: minio + namespace: minio + type: Opaque From 74b893c63ac457ed8486b640daaf013107ebb7e4 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 13:44:27 +0200 Subject: [PATCH 20/64] add zookeeper --- stacks/argo-cd/applicationsets/stackable-operators.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/stacks/argo-cd/applicationsets/stackable-operators.yaml b/stacks/argo-cd/applicationsets/stackable-operators.yaml index 46a08902..4ef1670a 100644 --- a/stacks/argo-cd/applicationsets/stackable-operators.yaml +++ b/stacks/argo-cd/applicationsets/stackable-operators.yaml @@ -22,6 +22,7 @@ spec: - operator: spark-k8s - operator: superset - operator: trino + - operator: zookeeper template: metadata: # {% raw %} From 57f70cc0a631a0f2e3580abf6b3280fbdca73998 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 13:45:52 +0200 Subject: [PATCH 21/64] fix path --- demos/argo-cd/applications/airflow-postgres.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/argo-cd/applications/airflow-postgres.yaml b/demos/argo-cd/applications/airflow-postgres.yaml index 2d06423f..3c6c3cec 100644 --- a/demos/argo-cd/applications/airflow-postgres.yaml +++ b/demos/argo-cd/applications/airflow-postgres.yaml @@ -25,7 +25,7 @@ spec: - repoURL: https://github.com/stackabletech/demos.git # TODO: adapt to release-25.3 targetRevision: "spike/argocd-demo" - path: demo/argo-cd/manifests/airflow-postgres + path: demos/argo-cd/manifests/airflow-postgres syncPolicy: syncOptions: - CreateNamespace=true From cbdb400ed57925c68d0a39f2241c109db2325a91 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 13:46:08 +0200 Subject: [PATCH 22/64] fix path 2 --- demos/argo-cd/applications/airflow-postgres.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/argo-cd/applications/airflow-postgres.yaml b/demos/argo-cd/applications/airflow-postgres.yaml index 3c6c3cec..a7c06b18 100644 --- a/demos/argo-cd/applications/airflow-postgres.yaml +++ b/demos/argo-cd/applications/airflow-postgres.yaml @@ -25,7 +25,7 @@ spec: - repoURL: https://github.com/stackabletech/demos.git # TODO: adapt to release-25.3 targetRevision: "spike/argocd-demo" - path: demos/argo-cd/manifests/airflow-postgres + path: demos/argo-cd/manifests/airflow-postgres/ syncPolicy: syncOptions: - CreateNamespace=true From 0b436835690d3e02a50978043598dcb6d9923ae6 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 13:51:14 +0200 Subject: [PATCH 23/64] fix secret name --- demos/argo-cd/applications/airflow-postgres.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/argo-cd/applications/airflow-postgres.yaml b/demos/argo-cd/applications/airflow-postgres.yaml index a7c06b18..faf49c9e 100644 --- a/demos/argo-cd/applications/airflow-postgres.yaml +++ b/demos/argo-cd/applications/airflow-postgres.yaml @@ -21,7 +21,7 @@ spec: stackable.tech/vendor: Stackable auth: database: airflow - existingSecret: postgres-credentials + existingSecret: postgresql-credentials - repoURL: https://github.com/stackabletech/demos.git # TODO: adapt to release-25.3 targetRevision: "spike/argocd-demo" From 3fa06919cc001b8a092ab4843314797cd85bf389 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 13:59:10 +0200 Subject: [PATCH 24/64] fix credentials --- .../airflow-postgres/sealed-airflow-postgres-credentials.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml index 3af276ae..a0d76fe7 100644 --- a/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml +++ b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml @@ -7,8 +7,8 @@ metadata: namespace: stackable-airflow spec: encryptedData: - password: AgAFfYyDQzfNHmkIuwkQjm8Z25AdLgelTPx43vKNggCrfHBjcBoA/5FKzwrfQxQldVFmuQtn7PpVagu6aZZfl+JBmzLwjYFA6kQPKYtLoskYvZpocfCAsWQJIp5yusI1780lT7b4JTz7LDkzVdy5aK2wm6/Nz5y5NzpEZNxThdc8tnzz0VtZHCpMdgi9q3CmhEPeWn0pOroocbCTkBg9pMiOOTH4YuCv1ca0/xtSSbFbFvN0luXhLIBBPWx7GVlJZGggVtiHNvvvCYVAGXogSXKWTAJv9HQvsV9ruHIa94UeWXnAXV9fhqq1nD700Nhmy0uDjBjntvvNTsKqhhVwOBKuvDYkmPXzSA98A+okty3+3Mv9D/fbAic20636tpzfEIAVGvnMkAhVhY4jth7QpTi4MMk18IiVOm58kSJz/oXDEpRB3CN6a7aB0HHf1xTqxKuqQ2FuZku+kriMAdgrBxCFGJPAcfi9PoqPjRWAawKrW3B9T8TJVYxnXmDvGUIaEMli9++J4laTI2ClER3udi+ZmcVZ3YZYrTdr7dhh7vyMJVLYIde9vh01FGBQZ0EOCPDp3ch0TtLKAAnH5jI0CYPvOgyLo+vUJufDIrIgRY7jwTZD9wmulQaFxH2V8CumCH5LHqOVLbY49oQZtAVrqIWUVyEejKgPkHGJiPQ63jHmz976RTdddAoi0BZ9LrLY/bqY67tctJtm - postgres-password: AgB7V06AGcL09k9vpmHOT417G5QY/sCmyN7OONfzEup898UpjD58SailnyZRSGwkhxo5Mhz9PlVP7AaFnMf+ZY/kVxEgjZ0kjBROB32e7zKwsITnp6ix7WJ85TYItde7P8YgyphmP4xxGx1crkFNjNDwJP0HtmIt9j0RN6EX+jDUupP9l6fH0LHFnJVE9NU6o2xvJa9gQs+iJ9peLzL8Nc4uXJiXrrdXl5ptrYti5w/cJpkKzlVoWpAeph390eog5QPU0dardhc/9WnmVfomjkJE4axwZl/yHbmcQTR0FgxXh24ICHRFBjPv9PdFhs7LsXqffkkt7RILEflUBBrSU2lCusn2Jscg6F4DBDR/6aa4eDu1tsk0UL2PVQzErErDt16qKPLaAygdevSsRCjXEcYlMnj+eFFAHiplEVMNc32yc70L13wf1rnndFAfXLNI2v1Aq6OoLnZ2HHFtpNxW8FKE+5vOyxykuUtU/E/alR5o3OrgZBgvmYQQLUap+NMAXjbbk/ZnVIAhPANOIs7rM2b5RLJmrEmIZkOZDPYt8/aRZezPdVb2THo2H3N1daOI6liA3WjdzFPNsawFRkDh2cBfQ308GzM49PhlGV5DDu99qDXWhy1a6nMdexZRiyv1cZ+GoBT0AKollpx6xh5XqF19d4BIpRFZlxHQ5b5heIQDNE639g3QsSTFqW9nS0NBVQLlajOJFwQ3 + password: AgCt9AmvCSFrTO8mzuI2j3hmK8ue/f1Jqa/hXrIrH1Gjk5cMlELzHslhh/WVwNU5N37PhmAZLrepAoPXSis/pGt4BvwRav/+qTVW3Y7gYYtEKT5XAFBoqWmOy8xd+fwIXurzTLdDom09xaywm5uWnYjt8qk45HIpcP/eNXT1+QwSnmRx4OlHrEG6yUv/qKHFfVhngZuOnC51iH6Cpa3kna+8UYr1DZW2kyoLB+334fadOH6r0VW4Kl8FcZb15Re+fGjO5+/aQ3oBQiX+l/5GDB46C19buwvk8Ak4dTCSNXiQAHBUGXHAYR8L6iYRh1rwSr0f5LVZS5RRgco20nGMJWwTd2/hUr1prI/GOSH9kjDqQRJabxbeQKjIkYv9muEEH5qUGEN1j9m8rBUi/z9NQSx9apjB0YWX9nqXi6C1RmQ/GMYyEbWGgSXx+76UKOjmdAOnWPrriaQSR7YR8QjD5qBWYFzSbp/81O4Ua6BoFIQyfot7Owl4Z4B8Ce7T9kqFW5DBUf6uidP4hWdHvxmbxhhzwAqeotHEDCDJDINJ6zOg82ZSOaJHO6GRhTAMeERiv8WEVmUG4wgr6CaxXipShVjAdbHfnXPQoXYAhAAySQ9BgxvluAJxoY6cLQLAv34N23LND+bPdulY+ylLIdkR+eSp2uA7nPGTf5gofL1pQx4w3+ncGF6kc7lFRdLLM4kgRvUx8WYeDzqt + postgres-password: AgAFZH+PLIC4MCIE8dp+U/Y+DAu9B7Te2aZZjr7Tu1j7FjZSBXDSfu1XBC4d9a/Brwhlolw/HhBsKie6gQXWOxhIzI4wH9FN+mnkWwfnh9xXD2TU4XIRivARKZ10dpxsyRFTZzbKJNxDjnaTZxJMYQRQ24kX9WUHQYNjDIh8r2SVEjIeE5NMP7nPPJJVkqYqIeu5LyhDtyX80rsHQNI65d2Uot/9RIG7BEi4EF2MDHiEK0/O65pP+nH2VOJqxiHjp1P+GppE5MotFmLVOsL2q+z2nBSlNHT9ocBE17ozUPmWaIdvjmPQ5WkQxSHkqea/pD53geCB+Tr5jTOrP5KgulCFJOKIJ/77Ey9zEMlOl3cmqoD5k90S+AVJ+1WLVtdpHmkSJkdiwSAClxzH3LVgOcAxK5BzhCU3Z2AblzYmA4UiIxEHSdmdJleG2mBKyU1w5fjV77S6rn+vzjqzPsTF9l/siuvAxQn4wxHYGzerVe+N8FdPhTclTzmJ0usbbZZhfhvjLl85EW2LCAHpWDYx4D9ZU6RayhfxLqlTbFHj2stW8LCHogRkpktWT/Z9DoEy+9KQkdNxwV3PSR0wsV9A9ntGDBmdJafD3s0aDPsLbIxaONAOeAWxOpNY7POwBnH+uWLIrT1T3RNNrFaS8mjUU/1pMgZYBaMd9xMWLJ0q5JObNG3Oy6AWJ9gYupvZiv/nLprMgXwHyBFaILvSXLz9YldH template: metadata: creationTimestamp: null From d2b82f78722a95ed89be07ecc295845e116c84d7 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 14:08:25 +0200 Subject: [PATCH 25/64] attempt to fix secret --- .../airflow-postgres/sealed-airflow-postgres-credentials.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml index a0d76fe7..ef5896d3 100644 --- a/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml +++ b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml @@ -7,8 +7,8 @@ metadata: namespace: stackable-airflow spec: encryptedData: - password: AgCt9AmvCSFrTO8mzuI2j3hmK8ue/f1Jqa/hXrIrH1Gjk5cMlELzHslhh/WVwNU5N37PhmAZLrepAoPXSis/pGt4BvwRav/+qTVW3Y7gYYtEKT5XAFBoqWmOy8xd+fwIXurzTLdDom09xaywm5uWnYjt8qk45HIpcP/eNXT1+QwSnmRx4OlHrEG6yUv/qKHFfVhngZuOnC51iH6Cpa3kna+8UYr1DZW2kyoLB+334fadOH6r0VW4Kl8FcZb15Re+fGjO5+/aQ3oBQiX+l/5GDB46C19buwvk8Ak4dTCSNXiQAHBUGXHAYR8L6iYRh1rwSr0f5LVZS5RRgco20nGMJWwTd2/hUr1prI/GOSH9kjDqQRJabxbeQKjIkYv9muEEH5qUGEN1j9m8rBUi/z9NQSx9apjB0YWX9nqXi6C1RmQ/GMYyEbWGgSXx+76UKOjmdAOnWPrriaQSR7YR8QjD5qBWYFzSbp/81O4Ua6BoFIQyfot7Owl4Z4B8Ce7T9kqFW5DBUf6uidP4hWdHvxmbxhhzwAqeotHEDCDJDINJ6zOg82ZSOaJHO6GRhTAMeERiv8WEVmUG4wgr6CaxXipShVjAdbHfnXPQoXYAhAAySQ9BgxvluAJxoY6cLQLAv34N23LND+bPdulY+ylLIdkR+eSp2uA7nPGTf5gofL1pQx4w3+ncGF6kc7lFRdLLM4kgRvUx8WYeDzqt - postgres-password: AgAFZH+PLIC4MCIE8dp+U/Y+DAu9B7Te2aZZjr7Tu1j7FjZSBXDSfu1XBC4d9a/Brwhlolw/HhBsKie6gQXWOxhIzI4wH9FN+mnkWwfnh9xXD2TU4XIRivARKZ10dpxsyRFTZzbKJNxDjnaTZxJMYQRQ24kX9WUHQYNjDIh8r2SVEjIeE5NMP7nPPJJVkqYqIeu5LyhDtyX80rsHQNI65d2Uot/9RIG7BEi4EF2MDHiEK0/O65pP+nH2VOJqxiHjp1P+GppE5MotFmLVOsL2q+z2nBSlNHT9ocBE17ozUPmWaIdvjmPQ5WkQxSHkqea/pD53geCB+Tr5jTOrP5KgulCFJOKIJ/77Ey9zEMlOl3cmqoD5k90S+AVJ+1WLVtdpHmkSJkdiwSAClxzH3LVgOcAxK5BzhCU3Z2AblzYmA4UiIxEHSdmdJleG2mBKyU1w5fjV77S6rn+vzjqzPsTF9l/siuvAxQn4wxHYGzerVe+N8FdPhTclTzmJ0usbbZZhfhvjLl85EW2LCAHpWDYx4D9ZU6RayhfxLqlTbFHj2stW8LCHogRkpktWT/Z9DoEy+9KQkdNxwV3PSR0wsV9A9ntGDBmdJafD3s0aDPsLbIxaONAOeAWxOpNY7POwBnH+uWLIrT1T3RNNrFaS8mjUU/1pMgZYBaMd9xMWLJ0q5JObNG3Oy6AWJ9gYupvZiv/nLprMgXwHyBFaILvSXLz9YldH + password: AgAwuhroRPQeCx9j4lS/+pdIfnKzb9OiOutLfN80HnRK8+4nAACpwv+pXUyMDpp4IgY913//lvGHL9EYEUxN8V2LCI6ev+cd2MnbCL1zNE0RzFkeVEQgNlej+o+I/iS2KIHw8Z9q9vKmsfRFY1eOI+j86v5SA6sGSNFn4oRaYPapCZimqmi1BY9gV46mJC91hEJTu2HRZzr19JaZPb/CFudL6lX8oM5MmgKBJdlHCxX+G95h2zd2RiMpUhUpOSwESQ4cZFnRtTvTQiQaUBo72893ZGJGXP9AH+pqIp9p+UhAn56HV2StDaiC9weIz6RcfXCj/jFJiBWLzixqXGyQUhBepviE2DTY8Kguv/7c+bJ58Lp4qRMxVJrk/HxCJ+yiqDOfr+G6oxdhAJmQpqOp6u95ZOqPjvU0X9Pg3vwdw2KTdmxD6RVrsl3ZVv84Gey2HCkSe02JSO/9/JFJjA3fBnlzMyp+Bli/4/2ZjkDL0cxDdI7AOIPyEKitp1gJqJDGz7frKffKsWdHkrsRxPQhfk40BwbBG8z/V/gX6IEi9yhWTD1oP+r6tRA9ZxeJP4CNp6DC+9de/ACLt2UjxfKUP3No4fr+1ZuRWzwJa4ExTR3vWIVPLru8WAi4xoJvQclf6GIuwR3qmqj3oBDJJp1v2VAQgDc7U2oD3QdayfxPCuqJHt4uV2gsUFcfVVvePvoIvz4aTwRDtD79 + postgres-password: AgCo/nYBzH2rLBsSZjYX47OI/NCblDtXSvHvhH2IA8FKoZCcxh+ZV0Mg3YaZpaeSsT59FnwnbApFfVrCxXEFdubpWNvwWt+wE0UFZazUdst4x+OdUQlG2b4V6hs9pbt94h7G+OXDsgMblAACitqRaJufOaemgBh6ZU8bhU2flgMn02hFWaooilhQKoKWybDRNEr5oMzVRaxAAVoWhGzjoCRX0J73ZwCHUKvTZ/CH6vgoiKq4q2HpMEzxbZFTfOAsbWgEGxmk7230WQTIBn9ynERPAEgo5jRyug1cVtH3+t7dgH+HthkgI+m5tfgc7bEbBPtbaW53O2M31FZGMVTwlUl5AyqHZyUWU8DT2PlrUx/vQYFip8BtpBdIaKKWLQBv97YKaGSGLy1YoWuaTXAJaQhbcfeAa32+929P0DYD/8aVV/7HfbyIyrS4ZzJr5z4xa9942fxSnxwAHfNDGxrgSig1M6q2vNFpXppC/HP2TN9OGTh34HlsE4k5a4a66Ue94kA+zZtis2t1kWmrI3MrJ4p6wo6LOYqOVrb5g6D9HVwG+0nyPoWUtnDK6g/X07zQNAo8i6O36WspS05UGpAWX6idgVkPTBMq3TePKJweRhLFHUnwLyeylqQDJP0shhGrItvbfFDtmLBaxnEdAWoG3BU1qfw2gWMXR+zQqMvhXF5P8roZPL/FsYHgqH/WUTWiwW2YNBDFG8EPSOby8qo= template: metadata: creationTimestamp: null From a7a95fcbef28f9f59536dd0f96253baf3c9cdf47 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 14:30:11 +0200 Subject: [PATCH 26/64] seal minio connection --- .../applications/airflow-postgres.yaml | 12 ++++---- .../sealed-airflow-postgres-credentials.yaml | 4 +-- demos/argo-cd/manifests/airflow/airflow.yaml | 29 ++++++++++++++----- .../sealed-airflow-minio-connection.yaml | 16 ++++++++++ 4 files changed, 46 insertions(+), 15 deletions(-) create mode 100644 demos/argo-cd/manifests/airflow/sealed-airflow-minio-connection.yaml diff --git a/demos/argo-cd/applications/airflow-postgres.yaml b/demos/argo-cd/applications/airflow-postgres.yaml index faf49c9e..8a550d1d 100644 --- a/demos/argo-cd/applications/airflow-postgres.yaml +++ b/demos/argo-cd/applications/airflow-postgres.yaml @@ -20,12 +20,14 @@ spec: commonLabels: stackable.tech/vendor: Stackable auth: + username: airflow + password: airflow database: airflow - existingSecret: postgresql-credentials - - repoURL: https://github.com/stackabletech/demos.git - # TODO: adapt to release-25.3 - targetRevision: "spike/argocd-demo" - path: demos/argo-cd/manifests/airflow-postgres/ + # existingSecret: postgresql-credentials + # - repoURL: https://github.com/stackabletech/demos.git + # # TODO: adapt to release-25.3 + # targetRevision: "spike/argocd-demo" + # path: demos/argo-cd/manifests/airflow-postgres/ syncPolicy: syncOptions: - CreateNamespace=true diff --git a/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml index ef5896d3..9a4bbca6 100644 --- a/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml +++ b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml @@ -7,8 +7,8 @@ metadata: namespace: stackable-airflow spec: encryptedData: - password: AgAwuhroRPQeCx9j4lS/+pdIfnKzb9OiOutLfN80HnRK8+4nAACpwv+pXUyMDpp4IgY913//lvGHL9EYEUxN8V2LCI6ev+cd2MnbCL1zNE0RzFkeVEQgNlej+o+I/iS2KIHw8Z9q9vKmsfRFY1eOI+j86v5SA6sGSNFn4oRaYPapCZimqmi1BY9gV46mJC91hEJTu2HRZzr19JaZPb/CFudL6lX8oM5MmgKBJdlHCxX+G95h2zd2RiMpUhUpOSwESQ4cZFnRtTvTQiQaUBo72893ZGJGXP9AH+pqIp9p+UhAn56HV2StDaiC9weIz6RcfXCj/jFJiBWLzixqXGyQUhBepviE2DTY8Kguv/7c+bJ58Lp4qRMxVJrk/HxCJ+yiqDOfr+G6oxdhAJmQpqOp6u95ZOqPjvU0X9Pg3vwdw2KTdmxD6RVrsl3ZVv84Gey2HCkSe02JSO/9/JFJjA3fBnlzMyp+Bli/4/2ZjkDL0cxDdI7AOIPyEKitp1gJqJDGz7frKffKsWdHkrsRxPQhfk40BwbBG8z/V/gX6IEi9yhWTD1oP+r6tRA9ZxeJP4CNp6DC+9de/ACLt2UjxfKUP3No4fr+1ZuRWzwJa4ExTR3vWIVPLru8WAi4xoJvQclf6GIuwR3qmqj3oBDJJp1v2VAQgDc7U2oD3QdayfxPCuqJHt4uV2gsUFcfVVvePvoIvz4aTwRDtD79 - postgres-password: AgCo/nYBzH2rLBsSZjYX47OI/NCblDtXSvHvhH2IA8FKoZCcxh+ZV0Mg3YaZpaeSsT59FnwnbApFfVrCxXEFdubpWNvwWt+wE0UFZazUdst4x+OdUQlG2b4V6hs9pbt94h7G+OXDsgMblAACitqRaJufOaemgBh6ZU8bhU2flgMn02hFWaooilhQKoKWybDRNEr5oMzVRaxAAVoWhGzjoCRX0J73ZwCHUKvTZ/CH6vgoiKq4q2HpMEzxbZFTfOAsbWgEGxmk7230WQTIBn9ynERPAEgo5jRyug1cVtH3+t7dgH+HthkgI+m5tfgc7bEbBPtbaW53O2M31FZGMVTwlUl5AyqHZyUWU8DT2PlrUx/vQYFip8BtpBdIaKKWLQBv97YKaGSGLy1YoWuaTXAJaQhbcfeAa32+929P0DYD/8aVV/7HfbyIyrS4ZzJr5z4xa9942fxSnxwAHfNDGxrgSig1M6q2vNFpXppC/HP2TN9OGTh34HlsE4k5a4a66Ue94kA+zZtis2t1kWmrI3MrJ4p6wo6LOYqOVrb5g6D9HVwG+0nyPoWUtnDK6g/X07zQNAo8i6O36WspS05UGpAWX6idgVkPTBMq3TePKJweRhLFHUnwLyeylqQDJP0shhGrItvbfFDtmLBaxnEdAWoG3BU1qfw2gWMXR+zQqMvhXF5P8roZPL/FsYHgqH/WUTWiwW2YNBDFG8EPSOby8qo= + password: AgAiudvaW+Vk7SUbXFSNuHOqfLeZzfNTg2mVDv7+YaZicFhO8JjspjjGEzBsxZqiXEVAkx4Ui2IvLe+SeTAM3NNE62VpG6JTipzF1QEC8seh6CpvlitkCWcmu4izhUY/30qQVRmcuWDbiGOJMCQfd9fQNr9BGPNf4v0Bm4/JRBQlAzZHjNuZvlrQXDXh8VJpaNix4+h7AWKbhADUwex37Qm3reumYGyB2s9WDQ1wzpZxo4N6lxn4CMOmEBt6ROZF/ZJlhYH5FOfB5531Mv2ohs1F6aR1ms+KnJUJOw27LVH9rOqbLv22LN63KfgHHTlh7xY4Tg4sMDa2rbJE68BtYtdvFD4cTp6Q3G/6sWlfEbW4OTDghrW++Ah/y1Oyaz/2Fl6Gwg6iP8naKqqX6dznuuDeRFgZEgQGgaJYjbe/ql+9HdIjlbk07Oe04XK6y5COzncN+RP9QbgsvSjXhohjmOsPLAZdSgwZRh7wcwdxSxmvP6oZCm9bxBMmWQm3TNFp3j3X556sAN81zhjbG4uMqlzcFyfFUxzDstiDlUGISMgGCrzGZaRL2esJpZA4SH5awISsr+su8uzMV66byvj53CuTzFqFM/rdh4H/ZQPasdsXHjBJRF9f2Mbve4KI8A8Sf0ci1HTVRWkS3rtQfqNbtd+wLWU30A2NHbYLEuHtN3K3R6FV54LA4872JyMlNJkkbT9Etpa3eNZO + postgres-password: AgCWp/TsvOCQIpg29hQosIGxR2T1bCWP4fT7NfBz5k1jmsI7i3VQNDE7/mO4mfEZV+EJLuHGfeSggAq1wxbJclMm2CzgarEstOtrduG/M1Ak4o3JJCmHcMLbJMghVSMoBIOPoYfSLB6UQUyr8FsvutIpXRcet41pFN6G4n/YZ1xDBnfiE6d8EDnJeEvFB5zTkQ9cwsDMUf2PKEZMH4SY+pumDjqPZXEi4uW+6JqL79u4KlcUqSDmB3gQ3PSlovMm9ls3zDokbH1TchR+4ycG+qen8NpD5zplcjF5RsM3mx++5KjPykcO0bzxJbygUylU52oMPv6Yo129k4KI1JNeDSqug3ZxgmgQlEs5V5aAlZYrObd1PRJZaZO+9vlDeoFrc++4Zysu0ydCWApjZqOZdXPZgIe8yNXT41b2XcEZvUgZT7rrr6n6KrTD31X2z4GHWw+sdYfIDDGVyn9YHfUjKOhds18MDi4iKLPRhWrRq+w7Rhyrxve4MnZlWoNQ8/YwN2Qkhb8EWMFEb/+EcpElUTwC8yVSd9V0dDkwlEnENb079i404qzKs7R3hP67fxr6s8A2F6Hfdty8XV8HRbMV/b7fzMriO8R3KRwJZUR93EesqkfQQrTgppwSu4+llMCTeO4GQf2Y+wRBnXE5j2YZmwgL/+vkpiXeLn2Kfc+vFsUJS7ANthf7RRRwchmlJhPAWOhv9XFbA9DwYrYXUSU= template: metadata: creationTimestamp: null diff --git a/demos/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd/manifests/airflow/airflow.yaml index de8613d1..8aee92a5 100644 --- a/demos/argo-cd/manifests/airflow/airflow.yaml +++ b/demos/argo-cd/manifests/airflow/airflow.yaml @@ -43,6 +43,25 @@ spec: - name: minio-tls mountPath: /stackable/minio-tls webservers: + envOverrides: &envOverrides + AIRFLOW__CORE__DAGS_FOLDER: "/dags" + AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" + # Via sealed secrets, just kept for reference here + #AIRFLOW_CONN_MINIO: "aws://admin:adminadmin@/?endpoint_url=https%3A%2F%2Fminio.minio.svc.cluster.local%3A9000" + AWS_CA_BUNDLE: "/stackable/minio-tls/ca.crt" + AIRFLOW__LOGGING__REMOTE_LOGGING: "True" + AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER: s3://demo/airflow-task-logs/ + AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID: minio + podOverrides: &podOverrides + spec: + container: + - name: airflow + env: + - name: AIRFLOW_CONN_MINIO + valueFrom: + secretKeyRef: + name: airflow-minio-connection + key: airflow-minio-connection config: resources: cpu: @@ -53,19 +72,13 @@ spec: gracefulShutdownTimeout: 30s roleGroups: default: - envOverrides: &envOverrides - AIRFLOW__CORE__DAGS_FOLDER: "/dags" - AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" - AIRFLOW_CONN_MINIO: "aws://admin:adminadmin@/?endpoint_url=https%3A%2F%2Fminio.minio.svc.cluster.local%3A9000" - AWS_CA_BUNDLE: "/stackable/minio-tls/ca.crt" - AIRFLOW__LOGGING__REMOTE_LOGGING: "True" - AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER: s3://demo/airflow-task-logs/ - AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID: minio replicas: 1 kubernetesExecutors: envOverrides: *envOverrides + podOverrides: *podOverrides schedulers: envOverrides: *envOverrides + podOverrides: *podOverrides config: gracefulShutdownTimeout: 30s resources: diff --git a/demos/argo-cd/manifests/airflow/sealed-airflow-minio-connection.yaml b/demos/argo-cd/manifests/airflow/sealed-airflow-minio-connection.yaml new file mode 100644 index 00000000..8e9f4f8d --- /dev/null +++ b/demos/argo-cd/manifests/airflow/sealed-airflow-minio-connection.yaml @@ -0,0 +1,16 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + creationTimestamp: null + name: airflow-minio-connection + namespace: stackable-airflow +spec: + encryptedData: + airflow-minio-connection: AgAHQL1NIeZiWH1opQYqHINixw0muWaNsYKMhtgJqqdVMLeKIU/sckZ7Z6aokaEYXGVQRe+Tp+D9AaYaF5CdKeMQKQ3MmhxXYZh0EuAcNRyvz4u4K696KEIa4XpaT5k6oRbIwRUGCc4hiRVApmMOgGYl4iEwVQ/1Twd5ffEAohMsySgZQBn17zB+hKC3u2AcIrZ1btHvBrVhC4pil/kyytBwjTkMq1iAkF8lwt7gatLukD9ldLaVdYTe21R+nP/UT7DwHRnBlI1dzkfrMdplFud2XIEmSrFF7ihzc5r+Lsm2vsnjaV5OgzDjIPI3RRA7g8/06Lebpru/E9h6HB5RIu59ga72g02QIHh5HffGUCnbEsLTDDbBcuAQB2jSodotfsofVKOhZChOIJDchvohhrQ86wqlkQyjfNMq2Gh46Nc7ZjomlpL/p0afmak34GFoPYMCD7IJ2DafEsmKA8dYfjTI1kVzLhYuoeDB3f82tu8xL9cXCEUsw7Y7EosZjeITeLcV1oMm+jeHzNHGaLmCI9k1sY5bL6xjbG59sOvuEgI302tS9ua592u9uqOOKKjE4eZM7CAmEjgMmDPbeteXGnZcOqfj8FWmlp0f4gm9s+68KH5PDtkKdtBckNjDeLj3QhMC2gstXnPJR3Hnm/bFQ4i/6XpnkuOFCvQT/Baar5scw4vhJ0tN8TGeMd651YXU7N2MEB+2WtiO0StbzhcWIoiGmvN+SRLawPZ87Sqlt26/bs5+jnuEv4Fy1oXhxbZwuf/YsZjEaKFI+HbFVvwM0K2d00/z9DYAB44oY2PmrZwsk0KqP2hkuumq + template: + metadata: + creationTimestamp: null + name: airflow-minio-connection + namespace: stackable-airflow + type: Opaque From 450775e81ec06e3430defdbc8466d1a3230cab69 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 14:35:26 +0200 Subject: [PATCH 27/64] fix secret --- demos/argo-cd/applications/airflow-postgres.yaml | 12 +++++------- .../sealed-airflow-postgres-credentials.yaml | 4 ++-- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/demos/argo-cd/applications/airflow-postgres.yaml b/demos/argo-cd/applications/airflow-postgres.yaml index 8a550d1d..faf49c9e 100644 --- a/demos/argo-cd/applications/airflow-postgres.yaml +++ b/demos/argo-cd/applications/airflow-postgres.yaml @@ -20,14 +20,12 @@ spec: commonLabels: stackable.tech/vendor: Stackable auth: - username: airflow - password: airflow database: airflow - # existingSecret: postgresql-credentials - # - repoURL: https://github.com/stackabletech/demos.git - # # TODO: adapt to release-25.3 - # targetRevision: "spike/argocd-demo" - # path: demos/argo-cd/manifests/airflow-postgres/ + existingSecret: postgresql-credentials + - repoURL: https://github.com/stackabletech/demos.git + # TODO: adapt to release-25.3 + targetRevision: "spike/argocd-demo" + path: demos/argo-cd/manifests/airflow-postgres/ syncPolicy: syncOptions: - CreateNamespace=true diff --git a/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml index 9a4bbca6..bd47fe64 100644 --- a/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml +++ b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml @@ -7,8 +7,8 @@ metadata: namespace: stackable-airflow spec: encryptedData: - password: AgAiudvaW+Vk7SUbXFSNuHOqfLeZzfNTg2mVDv7+YaZicFhO8JjspjjGEzBsxZqiXEVAkx4Ui2IvLe+SeTAM3NNE62VpG6JTipzF1QEC8seh6CpvlitkCWcmu4izhUY/30qQVRmcuWDbiGOJMCQfd9fQNr9BGPNf4v0Bm4/JRBQlAzZHjNuZvlrQXDXh8VJpaNix4+h7AWKbhADUwex37Qm3reumYGyB2s9WDQ1wzpZxo4N6lxn4CMOmEBt6ROZF/ZJlhYH5FOfB5531Mv2ohs1F6aR1ms+KnJUJOw27LVH9rOqbLv22LN63KfgHHTlh7xY4Tg4sMDa2rbJE68BtYtdvFD4cTp6Q3G/6sWlfEbW4OTDghrW++Ah/y1Oyaz/2Fl6Gwg6iP8naKqqX6dznuuDeRFgZEgQGgaJYjbe/ql+9HdIjlbk07Oe04XK6y5COzncN+RP9QbgsvSjXhohjmOsPLAZdSgwZRh7wcwdxSxmvP6oZCm9bxBMmWQm3TNFp3j3X556sAN81zhjbG4uMqlzcFyfFUxzDstiDlUGISMgGCrzGZaRL2esJpZA4SH5awISsr+su8uzMV66byvj53CuTzFqFM/rdh4H/ZQPasdsXHjBJRF9f2Mbve4KI8A8Sf0ci1HTVRWkS3rtQfqNbtd+wLWU30A2NHbYLEuHtN3K3R6FV54LA4872JyMlNJkkbT9Etpa3eNZO - postgres-password: AgCWp/TsvOCQIpg29hQosIGxR2T1bCWP4fT7NfBz5k1jmsI7i3VQNDE7/mO4mfEZV+EJLuHGfeSggAq1wxbJclMm2CzgarEstOtrduG/M1Ak4o3JJCmHcMLbJMghVSMoBIOPoYfSLB6UQUyr8FsvutIpXRcet41pFN6G4n/YZ1xDBnfiE6d8EDnJeEvFB5zTkQ9cwsDMUf2PKEZMH4SY+pumDjqPZXEi4uW+6JqL79u4KlcUqSDmB3gQ3PSlovMm9ls3zDokbH1TchR+4ycG+qen8NpD5zplcjF5RsM3mx++5KjPykcO0bzxJbygUylU52oMPv6Yo129k4KI1JNeDSqug3ZxgmgQlEs5V5aAlZYrObd1PRJZaZO+9vlDeoFrc++4Zysu0ydCWApjZqOZdXPZgIe8yNXT41b2XcEZvUgZT7rrr6n6KrTD31X2z4GHWw+sdYfIDDGVyn9YHfUjKOhds18MDi4iKLPRhWrRq+w7Rhyrxve4MnZlWoNQ8/YwN2Qkhb8EWMFEb/+EcpElUTwC8yVSd9V0dDkwlEnENb079i404qzKs7R3hP67fxr6s8A2F6Hfdty8XV8HRbMV/b7fzMriO8R3KRwJZUR93EesqkfQQrTgppwSu4+llMCTeO4GQf2Y+wRBnXE5j2YZmwgL/+vkpiXeLn2Kfc+vFsUJS7ANthf7RRRwchmlJhPAWOhv9XFbA9DwYrYXUSU= + password: AgAW0KA8iN9DeYjbGhYkM9xP4NwWK78YhZR43WI20igr7ZZucfHhANCWdCcNZFaakS73B/XK4wgN/gg9N+XJQY8MsDsfYrvVNVmrEBP1Hs7ryMbbDEGWuc8WTB7oq57O7HEtbI/anitpKLQHvciTpfz7m2gxpxd/DmZdMgKVER2CHqIH6r1pkREQWDKx2rBhPForFd8nXs5J5ALgD32XMCUE7P/nqgk0yr5b3OxFUflweAFMaQcTRpwu3AEeoQSOFquPXDaqNclww8FOPrE2EeitPq0YV1O5smFHlazYMqr8wk+WQuRK48Hn3/8SVHZEmKjH6it85//+VVaVhNmuUQlaxSBTZqY4aXI47Vfn92i3ICmUp7pGcD2AfNJ2+Xp2krXUQW+LVyVHI/0JpJsiGAz9I/CHr5Gs23R6GmLedzZJR3SVIvEWkxd2Y6iXow5Yvz2v5UgRXugn5ai2gPBqwrGUf5msuCV83f8LgcAVgZbpL2izfyuXLVvrE1Snv1/IJbPpPxTI9gb5xOS7KHQbveiq+QL5rRcVemLKUkzKn9dXLynCxyUo/QYRDdl83kaOacwFZN38yke01wGe0aJ9G9935DCheDKijEXCTbjzbjr5LuQYMxEBNmNiAz1Iq64E3W0O9PG0Nyhqd445gA/XWqk6r9ExCDhQ+TyGp2ZNMgNtlDuzc2eb/bqw3LTyvDqiWINwCcwJKeRA + postgres-password: AgAL+Go2p0sE5fkrFiInfUqQ2XTvaAT9B/exQI+LXI1ESzfANIx6nBC3GZv7ywIzBojScr3n32QFnyjZXsP2nG03+XQVdHXrVFWaSYwvyBlFmv5Zgsc26OK3jsptO0OsSxVN6PW4Y+oeTJVjLSjzCnFpQRSaN1G6IXUuEC8PMgRSPkX417xkX76xBIsK5UhGpiE5URo/0su8tacR+ncSP+jKSXO8TokyBuDHyrAlbEROdZLqtJZJsS5RWuvr66Du6qW8d6MR4ox/+mx585/lMgsQq/jHjSTlxJYQJUAW18wvDzKc6o7vW3O5jTN6Q30va/0KEN+1e6pMOqLXKdvfmdR70UxuBTR2NfeBCXCl6mr/oWEKUn4smRsmSWWC6DpvGMaIslmH3IK0SyNFSn81xpnKR9O81cvLIaN65MAfr+oUqNCUPSkroHWyFEDjiMJmOLx8pqaH+QHP0ibB27CI97xDaYJbXK+eHnjfBa4f7ierYX6HBbgfuqL0E34VyArFbPOmDjsVBsMcKzpC3NPAAcy6WxVqiBHhHdTRnN5U3p883TFGxsK2xH/Bkls2sgXVNUWKt4bWiP8jpnYhuekVrljIBOTaDB2ikemguXrPgxwc/QVDcuoOCDB7d1/jRRUCsVPzmL524ik1M5hw1qShyIMdEy6ZCwTwTUsaGwfwWyyGkIDqC236osYbCyemRf/W1F72fX3yRpHGdRgU template: metadata: creationTimestamp: null From 9319fcf18f7cab1ed8cbe5ea865b7363e4e24a83 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 14:46:14 +0200 Subject: [PATCH 28/64] try fix postgres secret --- demos/argo-cd/applications/airflow-postgres.yaml | 1 + .../sealed-airflow-postgres-credentials.yaml | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/demos/argo-cd/applications/airflow-postgres.yaml b/demos/argo-cd/applications/airflow-postgres.yaml index faf49c9e..df12dbc8 100644 --- a/demos/argo-cd/applications/airflow-postgres.yaml +++ b/demos/argo-cd/applications/airflow-postgres.yaml @@ -21,6 +21,7 @@ spec: stackable.tech/vendor: Stackable auth: database: airflow + username: airflow existingSecret: postgresql-credentials - repoURL: https://github.com/stackabletech/demos.git # TODO: adapt to release-25.3 diff --git a/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml index bd47fe64..51bd342f 100644 --- a/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml +++ b/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml @@ -7,8 +7,10 @@ metadata: namespace: stackable-airflow spec: encryptedData: - password: AgAW0KA8iN9DeYjbGhYkM9xP4NwWK78YhZR43WI20igr7ZZucfHhANCWdCcNZFaakS73B/XK4wgN/gg9N+XJQY8MsDsfYrvVNVmrEBP1Hs7ryMbbDEGWuc8WTB7oq57O7HEtbI/anitpKLQHvciTpfz7m2gxpxd/DmZdMgKVER2CHqIH6r1pkREQWDKx2rBhPForFd8nXs5J5ALgD32XMCUE7P/nqgk0yr5b3OxFUflweAFMaQcTRpwu3AEeoQSOFquPXDaqNclww8FOPrE2EeitPq0YV1O5smFHlazYMqr8wk+WQuRK48Hn3/8SVHZEmKjH6it85//+VVaVhNmuUQlaxSBTZqY4aXI47Vfn92i3ICmUp7pGcD2AfNJ2+Xp2krXUQW+LVyVHI/0JpJsiGAz9I/CHr5Gs23R6GmLedzZJR3SVIvEWkxd2Y6iXow5Yvz2v5UgRXugn5ai2gPBqwrGUf5msuCV83f8LgcAVgZbpL2izfyuXLVvrE1Snv1/IJbPpPxTI9gb5xOS7KHQbveiq+QL5rRcVemLKUkzKn9dXLynCxyUo/QYRDdl83kaOacwFZN38yke01wGe0aJ9G9935DCheDKijEXCTbjzbjr5LuQYMxEBNmNiAz1Iq64E3W0O9PG0Nyhqd445gA/XWqk6r9ExCDhQ+TyGp2ZNMgNtlDuzc2eb/bqw3LTyvDqiWINwCcwJKeRA - postgres-password: AgAL+Go2p0sE5fkrFiInfUqQ2XTvaAT9B/exQI+LXI1ESzfANIx6nBC3GZv7ywIzBojScr3n32QFnyjZXsP2nG03+XQVdHXrVFWaSYwvyBlFmv5Zgsc26OK3jsptO0OsSxVN6PW4Y+oeTJVjLSjzCnFpQRSaN1G6IXUuEC8PMgRSPkX417xkX76xBIsK5UhGpiE5URo/0su8tacR+ncSP+jKSXO8TokyBuDHyrAlbEROdZLqtJZJsS5RWuvr66Du6qW8d6MR4ox/+mx585/lMgsQq/jHjSTlxJYQJUAW18wvDzKc6o7vW3O5jTN6Q30va/0KEN+1e6pMOqLXKdvfmdR70UxuBTR2NfeBCXCl6mr/oWEKUn4smRsmSWWC6DpvGMaIslmH3IK0SyNFSn81xpnKR9O81cvLIaN65MAfr+oUqNCUPSkroHWyFEDjiMJmOLx8pqaH+QHP0ibB27CI97xDaYJbXK+eHnjfBa4f7ierYX6HBbgfuqL0E34VyArFbPOmDjsVBsMcKzpC3NPAAcy6WxVqiBHhHdTRnN5U3p883TFGxsK2xH/Bkls2sgXVNUWKt4bWiP8jpnYhuekVrljIBOTaDB2ikemguXrPgxwc/QVDcuoOCDB7d1/jRRUCsVPzmL524ik1M5hw1qShyIMdEy6ZCwTwTUsaGwfwWyyGkIDqC236osYbCyemRf/W1F72fX3yRpHGdRgU + database: AgAlOcdpjYAaeiHbIwmGM3lb7okU4DROr+4bs7oQBPBY924XdamqlSw4Y7V8svvHRqD5Hz8SRx0SY2dkLaBQZBaXKV8qaMaVdqX0zK8zvoIbw/bQFXMTFpPq/kUD/kBdTwGJdAd/VpdY1m4Dcn247AhsQuNbqcZ8xibH2IW1e41f1/5nidHY+0a0O2qrSjAU8RxBAuBlLS6ZZiT2vmgahe6CUarJ3Dzr8InZRmtgwT7F8Zzli6cqgOPDsUg768KwDOIxpO+tnOT9b57SAbLUn5SOedFbAodWgQAg913URTiPOCrt5Loyp9B4tQXv1s74WFmgPlEkQmavkIyQg3R2Yki0QhkFysOFtp41ilRFTvgauxV3irryTv/frgA+XJDwfYexG2whgWH/VbT/HHIMoHBuRD864lpWZ/qDdLhFx7A5Eflli+YO+xuOQP4eFEsWeVYtcBNEi8Epy86LBiDCttCYWJ96XEmriqTpBebjxpaihxfCnPEbq5bkLv9RhO9vgzgYPgceLvPJ9GSpSJFv2s+ohvvJYyeAiuNLRkPR++tHFaofCVnuIfbeFgS1WVi/F4TrNsq2JlNfUnrzGTkc1CIkx/u02cE7K+Smfi/Dz30iDE7A+TLxDz7pguozMHX0eQCYDJ4ehehkArfH/XXe/ZH7Hb5MV6nMV32ZxkNi4x5iMdIsuLUdXJHWqSi0Ue9cZFBsP2rqmr5S + password: AgB/x1/DNC9uYWCTH1xu17qSr9GrnTcQ4LD5BCTu/5wUd/TK2rWM8zHce0mF97e6km+FyT+G96utiRThutNkAOpzT7xrZwkp45F0HqF0aD+NOYMzMn4duVLaK7JXLm6u7187Ci6kFMkmZ+Npb94wX+iUa+kgTQTkMP1HsdgQj4c8gj1ldpnoUDQt8/wsTF/gEukXD14E0BbEMkRTPh8PzkiirlLVWCff3YauQVRKXL2yWa79Fo47zkZ/WMJXyB8PVBQot2OIDL1ygddTBklBoP6ts2MOo3G7blmQWu5cgS/+IAw837pEWpDg5+UnUkM1+OQ+52PWLRXPbiSMFFlcbZIHwuASx7lRRm482q+kfJBb4oALd7EPHmjcBg1lV1iR74NetJ2wkMMcqnvkITxw8U8To1ZKseBpNH7mBWB9u2lvpjT5EWMOveVLCq6NwU2IcdgHhUaMByrxq5I3u1k1Y1hZvM0txQp8Mr/Zas/EEJ/w3sN+i9O/JWIonRx84Y9OJxIQEXManz7LzzzIxXikCvenm/8oQMfPHa7w5Scdb6sEQKsFXZ+aMQMcUTp7xuZbYzuX/7CmVdPHlFOhAXGAbRE2J739uWmyMVQkJ9cHSKzbCJdXDD3+oJB42Xz37tJg08xvMGBTPJjJoKnXfHIPS2yQISg1VgfyWNg/T2LZLuqt5htpUbQsWsL6K9/z7qtsama5pUiVTsmu + postgres-password: AgBpGdazaNMQ2j2jI1sK5D1PCFvhXn/kH7Xhj2P2GgHesCxml2WN8B99dtk2O81BDt7IYNqadhwdXe+pmvJU48qkbfcmgE+Kbjbbdo8bv9GxMQcYPPH0ahqIfFAxNgMBHnbQ0Zxy+a3vzNiXLeD4FAwU0Pxa7kSwED0/EiyfmZcvph2J5JTi4FwhU2ITycMwsjQnt7KVw1SGWMZkro6HoJ9wGSj7GRSeHX7HEMBCehf8TOMEo1Emy6IqYHAywdcNgfBEoLvgUUFElBxYLshCP2NTTCU2o374Xw2fiErjiR7Q075Mshpxl4bqGf2HgGT44Lsn5KDupPNnV7FPx0xTk3OmmE5Ja5gUAeLYU3bF8B7DSm61ltQnkL+FQ5sy3tjCTsH3wyYbg4zVoQlJuyRXJdG48Y0aHkQHwMpZlZEkB8IWGZZ9Kd9UKTTFnS3odca8OhdWYQ28ARz4Mj0Jio0dEO/63EyCJZiL4/Ck4rj8I49Wz832GjHVkMNFqthK6jN+HmlGcKXRKRMFLyIrvH7mUv1EuOsvh2+qtrsGGWSYwfETmb1J/iUp7X2+KhPnn6GP/poXLKeeDtiRfRSXTP70vRDllOMgSRTorRxS44FD/ac3KVA4RlpA+Ho+XU4QhYnNx8Q6haTQINY8Yypw/JRAPO4l379EFZtFdCSrKZ9XmRPsSZkZPZZJSVUe2ByGs/SScOR1xf1pwdPjWoOd + username: AgC716fxKUrMXCsRsWeiSTliHKP8NrwBr0TD7/w2F1HO93pqjSVU5/6hKOiFdNllbyPRudSHAeYWqJhQxxl5tS9kSaEp29zkxAUnmNwfLL3MKdCohxz2LZzC7saaOdjDkX2x8jcy68STPupsdg5bX40D4tO/s8qXnWWZJrW0ht/J+orgjbnfHk51JBMRBh6WD43yKLOBTHGO5U/TqRKg20kn+jDm2hcdec8axcjn2E57B3wrL8tvUcVRElO0XOcLHHBrav/BumRkwS4bMGqa+Vcarrr8U9MuQXVUzbCuwt326q/Kg862ozYY9uWAVtUy+Yfb4SAcEUC9Y+Ioo776H3YIr6q8P6RI+lFNmz4Q8AFDN9D5IRNjQ/D73llHw944dYRJl8CVjoJJYvRXv8z+4GzL7O+eF+smx4EhcY3sFxjkNCv+s+EDbUXvQR0ULqTEc5WZlBqp9TQEvzBwN3CVYO0k76FQYdjoU6xb/32gALe6zMxOLTliR9axqDvCL3OCj97/HJy8RLL2sYH82NZ1sGHqfTkTKYGk24Zfzd+a+p0PDbU14ZBuXKWBDkzNaVdpLvWhtdBczarfVp4Z2b6IutbT2C43fcmi7WYJOtV5em1sGhKOtbEy+wC+r5GKr6qRV58jm88cwtqZgm6IirDt/bzJDqhPndBzW5iQZzkko/1Tr4y/6oaTleTcmkDlSg2Cn8q/R7MNYUjT template: metadata: creationTimestamp: null From 84ee9a555b86b2f409285550a2d6550e4cc63fec Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 14:57:29 +0200 Subject: [PATCH 29/64] fix env override --- demos/argo-cd/manifests/airflow/airflow.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/demos/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd/manifests/airflow/airflow.yaml index 8aee92a5..b4ce9dd0 100644 --- a/demos/argo-cd/manifests/airflow/airflow.yaml +++ b/demos/argo-cd/manifests/airflow/airflow.yaml @@ -56,12 +56,12 @@ spec: spec: container: - name: airflow - env: - - name: AIRFLOW_CONN_MINIO - valueFrom: - secretKeyRef: - name: airflow-minio-connection - key: airflow-minio-connection + env: + - name: AIRFLOW_CONN_MINIO + valueFrom: + secretKeyRef: + name: airflow-minio-connection + key: airflow-minio-connection config: resources: cpu: From 4846f005b4f60569f0bb940f3e589d4930716bed Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 15:03:36 +0200 Subject: [PATCH 30/64] fix overrides --- demos/argo-cd/manifests/airflow/airflow.yaml | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/demos/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd/manifests/airflow/airflow.yaml index b4ce9dd0..29e6a071 100644 --- a/demos/argo-cd/manifests/airflow/airflow.yaml +++ b/demos/argo-cd/manifests/airflow/airflow.yaml @@ -54,7 +54,7 @@ spec: AIRFLOW__LOGGING__REMOTE_LOG_CONN_ID: minio podOverrides: &podOverrides spec: - container: + containers: - name: airflow env: - name: AIRFLOW_CONN_MINIO @@ -62,14 +62,6 @@ spec: secretKeyRef: name: airflow-minio-connection key: airflow-minio-connection - config: - resources: - cpu: - min: 400m - max: "1" - memory: - limit: 2Gi - gracefulShutdownTimeout: 30s roleGroups: default: replicas: 1 @@ -79,14 +71,6 @@ spec: schedulers: envOverrides: *envOverrides podOverrides: *podOverrides - config: - gracefulShutdownTimeout: 30s - resources: - cpu: - min: 400m - max: "1" - memory: - limit: 1Gi roleGroups: default: replicas: 1 From 8f7d766f3fa31f7edb133c4815ac95f6c904b021 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 15:16:51 +0200 Subject: [PATCH 31/64] fix container name --- demos/argo-cd/manifests/airflow/airflow.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/demos/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd/manifests/airflow/airflow.yaml index 29e6a071..3eca242e 100644 --- a/demos/argo-cd/manifests/airflow/airflow.yaml +++ b/demos/argo-cd/manifests/airflow/airflow.yaml @@ -67,7 +67,16 @@ spec: replicas: 1 kubernetesExecutors: envOverrides: *envOverrides - podOverrides: *podOverrides + podOverrides: &podOverrides + spec: + containers: + - name: base + env: + - name: AIRFLOW_CONN_MINIO + valueFrom: + secretKeyRef: + name: airflow-minio-connection + key: airflow-minio-connection schedulers: envOverrides: *envOverrides podOverrides: *podOverrides From 1c8b5a5ca36537ed52b8d110ddf2c0fdbc3de49b Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 15:24:28 +0200 Subject: [PATCH 32/64] fix overrides --- demos/argo-cd/manifests/airflow/airflow.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd/manifests/airflow/airflow.yaml index 3eca242e..75ace4c6 100644 --- a/demos/argo-cd/manifests/airflow/airflow.yaml +++ b/demos/argo-cd/manifests/airflow/airflow.yaml @@ -67,7 +67,7 @@ spec: replicas: 1 kubernetesExecutors: envOverrides: *envOverrides - podOverrides: &podOverrides + podOverrides: spec: containers: - name: base From 4fcc2a1468b0b9cd3aa0ba9ef18596793bcc2063 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 15:36:14 +0200 Subject: [PATCH 33/64] enable gitsync --- demos/argo-cd/dags/date_demo.py | 20 ++ demos/argo-cd/dags/pyspark_pi.py | 177 +++++++++++++ demos/argo-cd/dags/pyspark_pi.yaml | 36 +++ demos/argo-cd/manifests/airflow/airflow.yaml | 263 +------------------ 4 files changed, 238 insertions(+), 258 deletions(-) create mode 100644 demos/argo-cd/dags/date_demo.py create mode 100644 demos/argo-cd/dags/pyspark_pi.py create mode 100644 demos/argo-cd/dags/pyspark_pi.yaml diff --git a/demos/argo-cd/dags/date_demo.py b/demos/argo-cd/dags/date_demo.py new file mode 100644 index 00000000..704aac51 --- /dev/null +++ b/demos/argo-cd/dags/date_demo.py @@ -0,0 +1,20 @@ +"""Example DAG returning the current date""" +from datetime import datetime, timedelta + +from airflow import DAG +from airflow.operators.bash import BashOperator + +with DAG( + dag_id='date_demo', + schedule_interval='0-59 * * * *', + start_date=datetime(2021, 1, 1), + catchup=False, + dagrun_timeout=timedelta(minutes=5), + tags=['example'], + params={}, +) as dag: + + run_this = BashOperator( + task_id='run_every_minute', + bash_command='date', + ) diff --git a/demos/argo-cd/dags/pyspark_pi.py b/demos/argo-cd/dags/pyspark_pi.py new file mode 100644 index 00000000..16c81442 --- /dev/null +++ b/demos/argo-cd/dags/pyspark_pi.py @@ -0,0 +1,177 @@ +"""Example DAG demonstrating how to apply a Kubernetes Resource from Airflow running in-cluster""" +from datetime import datetime, timedelta +from airflow import DAG +from typing import TYPE_CHECKING, Optional, Sequence, Dict +from kubernetes import client +from airflow.exceptions import AirflowException +from airflow.sensors.base import BaseSensorOperator +from airflow.models import BaseOperator +from airflow.providers.cncf.kubernetes.hooks.kubernetes import KubernetesHook +import yaml +from airflow.utils import yaml +import os + +if TYPE_CHECKING: + from airflow.utils.context import Context + +class SparkKubernetesOperator(BaseOperator): + template_fields: Sequence[str] = ('application_file', 'namespace') + template_ext: Sequence[str] = ('.yaml', '.yml', '.json') + ui_color = '#f4a460' + + def __init__( + self, + *, + application_file: str, + namespace: Optional[str] = None, + kubernetes_conn_id: str = 'kubernetes_in_cluster', + api_group: str = 'spark.stackable.tech', + api_version: str = 'v1alpha1', + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.application_file = application_file + self.namespace = namespace + self.kubernetes_conn_id = kubernetes_conn_id + self.api_group = api_group + self.api_version = api_version + self.plural = "sparkapplications" + + def execute(self, context: 'Context'): + hook = KubernetesHook(conn_id=self.kubernetes_conn_id) + self.log.info("Creating SparkApplication...") + response = hook.create_custom_object( + group=self.api_group, + version=self.api_version, + plural=self.plural, + body=self.application_file, + namespace=self.namespace, + ) + return response + + +class SparkKubernetesSensor(BaseSensorOperator): + template_fields = ("application_name", "namespace") + # See https://github.com/stackabletech/spark-k8s-operator/pull/460/files#diff-d737837121132af6b60f50279a78464b05dcfd06c05d1d090f4198a5e962b5f6R371 + # Unknown is set immediately so it must be excluded from the failed states. + FAILURE_STATES = ("Failed") + SUCCESS_STATES = ("Succeeded") + + def __init__( + self, + *, + application_name: str, + attach_log: bool = False, + namespace: Optional[str] = None, + kubernetes_conn_id: str = 'kubernetes_in_cluster', + api_group: str = 'spark.stackable.tech', + api_version: str = 'v1alpha1', + poke_interval: float = 60, + **kwargs, + ) -> None: + super().__init__(**kwargs) + self.application_name = application_name + self.attach_log = attach_log + self.namespace = namespace + self.kubernetes_conn_id = kubernetes_conn_id + self.hook = KubernetesHook(conn_id=self.kubernetes_conn_id) + self.api_group = api_group + self.api_version = api_version + self.poke_interval = poke_interval + + def _log_driver(self, application_state: str, response: dict) -> None: + if not self.attach_log: + return + status_info = response["status"] + if "driverInfo" not in status_info: + return + driver_info = status_info["driverInfo"] + if "podName" not in driver_info: + return + driver_pod_name = driver_info["podName"] + namespace = response["metadata"]["namespace"] + log_method = self.log.error if application_state in self.FAILURE_STATES else self.log.info + try: + log = "" + for line in self.hook.get_pod_logs(driver_pod_name, namespace=namespace): + log += line.decode() + log_method(log) + except client.rest.ApiException as e: + self.log.warning( + "Could not read logs for pod %s. It may have been disposed.\n" + "Make sure timeToLiveSeconds is set on your SparkApplication spec.\n" + "underlying exception: %s", + driver_pod_name, + e, + ) + + def poke(self, context: Dict) -> bool: + self.log.info("Poking: %s", self.application_name) + response = self.hook.get_custom_object( + group=self.api_group, + version=self.api_version, + plural="sparkapplications", + name=self.application_name, + namespace=self.namespace, + ) + try: + application_state = response["status"]["phase"] + except KeyError: + self.log.debug(f"SparkApplication status could not be established: {response}") + return False + if self.attach_log and application_state in self.FAILURE_STATES + self.SUCCESS_STATES: + self._log_driver(application_state, response) + if application_state in self.FAILURE_STATES: + raise AirflowException(f"SparkApplication failed with state: {application_state}") + elif application_state in self.SUCCESS_STATES: + self.log.info("SparkApplication ended successfully") + return True + else: + self.log.info("SparkApplication is still in state: %s", application_state) + return False + +with DAG( + dag_id='sparkapp_dag', + schedule_interval=None, + start_date=datetime(2022, 1, 1), + catchup=False, + dagrun_timeout=timedelta(minutes=60), + tags=['example'], + params={"example_key": "example_value"}, +) as dag: + + def load_body_to_dict(body): + try: + body_dict = yaml.safe_load(body) + except yaml.YAMLError as e: + raise AirflowException(f"Exception when loading resource definition: {e}\n") + return body_dict + + yaml_path = os.path.join(os.environ.get('AIRFLOW__CORE__DAGS_FOLDER'), 'pyspark_pi.yaml') + + with open(yaml_path, 'r') as file: + crd = file.read() + with open('/run/secrets/kubernetes.io/serviceaccount/namespace', 'r') as file: + ns = file.read() + + document=load_body_to_dict(crd) + application_name='pyspark-pi-'+datetime.utcnow().strftime('%Y%m%d%H%M%S') + document.update({'metadata': {'name': application_name, 'namespace': ns}}) + + t1 = SparkKubernetesOperator( + task_id='spark_pi_submit', + namespace=ns, + application_file=document, + do_xcom_push=True, + dag=dag, + ) + + t2 = SparkKubernetesSensor( + task_id='spark_pi_monitor', + namespace=ns, + application_name="{{ task_instance.xcom_pull(task_ids='spark_pi_submit')['metadata']['name'] }}", + poke_interval=5, + dag=dag, + ) + + t1 >> t2 diff --git a/demos/argo-cd/dags/pyspark_pi.yaml b/demos/argo-cd/dags/pyspark_pi.yaml new file mode 100644 index 00000000..559f4822 --- /dev/null +++ b/demos/argo-cd/dags/pyspark_pi.yaml @@ -0,0 +1,36 @@ +--- +apiVersion: spark.stackable.tech/v1alpha1 +kind: SparkApplication +metadata: + name: pyspark-pi +spec: + version: "1.0" + sparkImage: + productVersion: 3.5.2 + mode: cluster + mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py + job: + config: + resources: + cpu: + min: 500m + max: 500m + memory: + limit: 512Mi + driver: + config: + resources: + cpu: + min: 1000m + max: 1200m + memory: + limit: 1024Mi + executor: + config: + resources: + cpu: + min: 500m + max: 1000m + memory: + limit: 1024Mi + replicas: 3 diff --git a/demos/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd/manifests/airflow/airflow.yaml index 75ace4c6..206555ae 100644 --- a/demos/argo-cd/manifests/airflow/airflow.yaml +++ b/demos/argo-cd/manifests/airflow/airflow.yaml @@ -1,5 +1,4 @@ --- -# {% raw %} apiVersion: airflow.stackable.tech/v1alpha1 kind: AirflowCluster metadata: @@ -12,10 +11,12 @@ spec: loadExamples: false exposeConfig: false credentialsSecret: airflow-credentials + dagsGitSync: + - repo: https://github.com/stackabletech/demos/ + branch: spike/argocd-demo + gitFolder: "demos/argo-cd/dags" + depth: 2 volumes: - - name: airflow-dags - configMap: - name: airflow-dags - name: minio-tls ephemeral: volumeClaimTemplate: @@ -30,16 +31,6 @@ spec: requests: storage: "1" storageClassName: secrets.stackable.tech - volumeMounts: - - name: airflow-dags - mountPath: /dags/date_demo.py - subPath: date_demo.py - - name: airflow-dags - mountPath: /dags/pyspark_pi.py - subPath: pyspark_pi.py - - name: airflow-dags - mountPath: /dags/pyspark_pi.yaml - subPath: pyspark_pi.yaml - name: minio-tls mountPath: /stackable/minio-tls webservers: @@ -83,247 +74,3 @@ spec: roleGroups: default: replicas: 1 ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: airflow-dags -data: - date_demo.py: | - """Example DAG returning the current date""" - from datetime import datetime, timedelta - - from airflow import DAG - from airflow.operators.bash import BashOperator - - with DAG( - dag_id='date_demo', - schedule_interval='0-59 * * * *', - start_date=datetime(2021, 1, 1), - catchup=False, - dagrun_timeout=timedelta(minutes=5), - tags=['example'], - params={}, - ) as dag: - - run_this = BashOperator( - task_id='run_every_minute', - bash_command='date', - ) - pyspark_pi.py: | - """Example DAG demonstrating how to apply a Kubernetes Resource from Airflow running in-cluster""" - from datetime import datetime, timedelta - from airflow import DAG - from typing import TYPE_CHECKING, Optional, Sequence, Dict - from kubernetes import client - from airflow.exceptions import AirflowException - from airflow.sensors.base import BaseSensorOperator - from airflow.models import BaseOperator - from airflow.providers.cncf.kubernetes.hooks.kubernetes import KubernetesHook - import yaml - from airflow.utils import yaml - import os - - if TYPE_CHECKING: - from airflow.utils.context import Context - - class SparkKubernetesOperator(BaseOperator): - template_fields: Sequence[str] = ('application_file', 'namespace') - template_ext: Sequence[str] = ('.yaml', '.yml', '.json') - ui_color = '#f4a460' - - def __init__( - self, - *, - application_file: str, - namespace: Optional[str] = None, - kubernetes_conn_id: str = 'kubernetes_in_cluster', - api_group: str = 'spark.stackable.tech', - api_version: str = 'v1alpha1', - **kwargs, - ) -> None: - super().__init__(**kwargs) - self.application_file = application_file - self.namespace = namespace - self.kubernetes_conn_id = kubernetes_conn_id - self.api_group = api_group - self.api_version = api_version - self.plural = "sparkapplications" - - def execute(self, context: 'Context'): - hook = KubernetesHook(conn_id=self.kubernetes_conn_id) - self.log.info("Creating SparkApplication...") - response = hook.create_custom_object( - group=self.api_group, - version=self.api_version, - plural=self.plural, - body=self.application_file, - namespace=self.namespace, - ) - return response - - - class SparkKubernetesSensor(BaseSensorOperator): - template_fields = ("application_name", "namespace") - # See https://github.com/stackabletech/spark-k8s-operator/pull/460/files#diff-d737837121132af6b60f50279a78464b05dcfd06c05d1d090f4198a5e962b5f6R371 - # Unknown is set immediately so it must be excluded from the failed states. - FAILURE_STATES = ("Failed") - SUCCESS_STATES = ("Succeeded") - - def __init__( - self, - *, - application_name: str, - attach_log: bool = False, - namespace: Optional[str] = None, - kubernetes_conn_id: str = 'kubernetes_in_cluster', - api_group: str = 'spark.stackable.tech', - api_version: str = 'v1alpha1', - poke_interval: float = 60, - **kwargs, - ) -> None: - super().__init__(**kwargs) - self.application_name = application_name - self.attach_log = attach_log - self.namespace = namespace - self.kubernetes_conn_id = kubernetes_conn_id - self.hook = KubernetesHook(conn_id=self.kubernetes_conn_id) - self.api_group = api_group - self.api_version = api_version - self.poke_interval = poke_interval - - def _log_driver(self, application_state: str, response: dict) -> None: - if not self.attach_log: - return - status_info = response["status"] - if "driverInfo" not in status_info: - return - driver_info = status_info["driverInfo"] - if "podName" not in driver_info: - return - driver_pod_name = driver_info["podName"] - namespace = response["metadata"]["namespace"] - log_method = self.log.error if application_state in self.FAILURE_STATES else self.log.info - try: - log = "" - for line in self.hook.get_pod_logs(driver_pod_name, namespace=namespace): - log += line.decode() - log_method(log) - except client.rest.ApiException as e: - self.log.warning( - "Could not read logs for pod %s. It may have been disposed.\n" - "Make sure timeToLiveSeconds is set on your SparkApplication spec.\n" - "underlying exception: %s", - driver_pod_name, - e, - ) - - def poke(self, context: Dict) -> bool: - self.log.info("Poking: %s", self.application_name) - response = self.hook.get_custom_object( - group=self.api_group, - version=self.api_version, - plural="sparkapplications", - name=self.application_name, - namespace=self.namespace, - ) - try: - application_state = response["status"]["phase"] - except KeyError: - self.log.debug(f"SparkApplication status could not be established: {response}") - return False - if self.attach_log and application_state in self.FAILURE_STATES + self.SUCCESS_STATES: - self._log_driver(application_state, response) - if application_state in self.FAILURE_STATES: - raise AirflowException(f"SparkApplication failed with state: {application_state}") - elif application_state in self.SUCCESS_STATES: - self.log.info("SparkApplication ended successfully") - return True - else: - self.log.info("SparkApplication is still in state: %s", application_state) - return False - - with DAG( - dag_id='sparkapp_dag', - schedule_interval=None, - start_date=datetime(2022, 1, 1), - catchup=False, - dagrun_timeout=timedelta(minutes=60), - tags=['example'], - params={"example_key": "example_value"}, - ) as dag: - - def load_body_to_dict(body): - try: - body_dict = yaml.safe_load(body) - except yaml.YAMLError as e: - raise AirflowException(f"Exception when loading resource definition: {e}\n") - return body_dict - - yaml_path = os.path.join(os.environ.get('AIRFLOW__CORE__DAGS_FOLDER'), 'pyspark_pi.yaml') - - with open(yaml_path, 'r') as file: - crd = file.read() - with open('/run/secrets/kubernetes.io/serviceaccount/namespace', 'r') as file: - ns = file.read() - - document=load_body_to_dict(crd) - application_name='pyspark-pi-'+datetime.utcnow().strftime('%Y%m%d%H%M%S') - document.update({'metadata': {'name': application_name, 'namespace': ns}}) - - t1 = SparkKubernetesOperator( - task_id='spark_pi_submit', - namespace=ns, - application_file=document, - do_xcom_push=True, - dag=dag, - ) - - t2 = SparkKubernetesSensor( - task_id='spark_pi_monitor', - namespace=ns, - application_name="{{ task_instance.xcom_pull(task_ids='spark_pi_submit')['metadata']['name'] }}", - poke_interval=5, - dag=dag, - ) - - t1 >> t2 - pyspark_pi.yaml: | - --- - apiVersion: spark.stackable.tech/v1alpha1 - kind: SparkApplication - metadata: - name: pyspark-pi - spec: - version: "1.0" - sparkImage: - productVersion: 3.5.2 - mode: cluster - mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py - job: - config: - resources: - cpu: - min: 500m - max: 500m - memory: - limit: 512Mi - driver: - config: - resources: - cpu: - min: 1000m - max: 1200m - memory: - limit: 1024Mi - executor: - config: - resources: - cpu: - min: 500m - max: 1000m - memory: - limit: 1024Mi - replicas: 3 - -# {% endraw %} From f52cb08de68f3cc86632c386a711729ba3a12603 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 15:39:43 +0200 Subject: [PATCH 34/64] fix git sync --- demos/argo-cd/manifests/airflow/airflow.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd/manifests/airflow/airflow.yaml index 206555ae..114021a4 100644 --- a/demos/argo-cd/manifests/airflow/airflow.yaml +++ b/demos/argo-cd/manifests/airflow/airflow.yaml @@ -31,11 +31,11 @@ spec: requests: storage: "1" storageClassName: secrets.stackable.tech + volumeMounts: - name: minio-tls mountPath: /stackable/minio-tls webservers: envOverrides: &envOverrides - AIRFLOW__CORE__DAGS_FOLDER: "/dags" AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" # Via sealed secrets, just kept for reference here #AIRFLOW_CONN_MINIO: "aws://admin:adminadmin@/?endpoint_url=https%3A%2F%2Fminio.minio.svc.cluster.local%3A9000" From 32011e805da338703f65b0492618010d7b0f3ab6 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 15:50:51 +0200 Subject: [PATCH 35/64] move yaml out of dags git sync --- demos/argo-cd/dags/pyspark_pi.py | 2 +- demos/argo-cd/{dags => manifests/spark-k8s}/pyspark_pi.yaml | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename demos/argo-cd/{dags => manifests/spark-k8s}/pyspark_pi.yaml (100%) diff --git a/demos/argo-cd/dags/pyspark_pi.py b/demos/argo-cd/dags/pyspark_pi.py index 16c81442..35bd4c38 100644 --- a/demos/argo-cd/dags/pyspark_pi.py +++ b/demos/argo-cd/dags/pyspark_pi.py @@ -147,7 +147,7 @@ def load_body_to_dict(body): raise AirflowException(f"Exception when loading resource definition: {e}\n") return body_dict - yaml_path = os.path.join(os.environ.get('AIRFLOW__CORE__DAGS_FOLDER'), 'pyspark_pi.yaml') + yaml_path = os.path.join(os.environ.get('AIRFLOW__CORE__DAGS_FOLDER'), '../manifests/spark-k8s/pyspark_pi.yaml') with open(yaml_path, 'r') as file: crd = file.read() diff --git a/demos/argo-cd/dags/pyspark_pi.yaml b/demos/argo-cd/manifests/spark-k8s/pyspark_pi.yaml similarity index 100% rename from demos/argo-cd/dags/pyspark_pi.yaml rename to demos/argo-cd/manifests/spark-k8s/pyspark_pi.yaml From 2efbab2b580bd777a1b885c2d165c0ce1bb6cd2e Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 15:55:35 +0200 Subject: [PATCH 36/64] set resources --- demos/argo-cd/manifests/airflow/airflow.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/demos/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd/manifests/airflow/airflow.yaml index 114021a4..58e45727 100644 --- a/demos/argo-cd/manifests/airflow/airflow.yaml +++ b/demos/argo-cd/manifests/airflow/airflow.yaml @@ -53,6 +53,14 @@ spec: secretKeyRef: name: airflow-minio-connection key: airflow-minio-connection + config: + gracefulShutdownTimeout: 30s + resources: + cpu: + min: 400m + max: "1" + memory: + limit: 1Gi roleGroups: default: replicas: 1 @@ -71,6 +79,14 @@ spec: schedulers: envOverrides: *envOverrides podOverrides: *podOverrides + config: + gracefulShutdownTimeout: 30s + resources: + cpu: + min: 400m + max: "1" + memory: + limit: 1Gi roleGroups: default: replicas: 1 From 945cbf28218fedb7e8f63b8ccf22a7702b7266d6 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Sun, 13 Apr 2025 16:28:41 +0200 Subject: [PATCH 37/64] linter --- demos/argo-cd/applications/airflow-postgres.yaml | 1 + demos/argo-cd/applications/airflow.yaml | 1 + demos/argo-cd/applications/minio.yaml | 1 + demos/argo-cd/manifests/airflow/airflow.yaml | 2 +- demos/argo-cd/projects/airflow.yaml | 1 + demos/argo-cd/projects/minio.yaml | 1 + stacks/argo-cd/applications/sealed-secrets.yaml | 1 + stacks/argo-cd/projects/stackable-operators.yaml | 1 + 8 files changed, 8 insertions(+), 1 deletion(-) diff --git a/demos/argo-cd/applications/airflow-postgres.yaml b/demos/argo-cd/applications/airflow-postgres.yaml index df12dbc8..7052b64e 100644 --- a/demos/argo-cd/applications/airflow-postgres.yaml +++ b/demos/argo-cd/applications/airflow-postgres.yaml @@ -1,3 +1,4 @@ +--- apiVersion: argoproj.io/v1alpha1 kind: Application metadata: diff --git a/demos/argo-cd/applications/airflow.yaml b/demos/argo-cd/applications/airflow.yaml index edf5ac43..82c1b71c 100644 --- a/demos/argo-cd/applications/airflow.yaml +++ b/demos/argo-cd/applications/airflow.yaml @@ -1,3 +1,4 @@ +--- apiVersion: argoproj.io/v1alpha1 kind: Application metadata: diff --git a/demos/argo-cd/applications/minio.yaml b/demos/argo-cd/applications/minio.yaml index d3eaf0a5..42ca5407 100644 --- a/demos/argo-cd/applications/minio.yaml +++ b/demos/argo-cd/applications/minio.yaml @@ -1,3 +1,4 @@ +--- apiVersion: argoproj.io/v1alpha1 kind: Application metadata: diff --git a/demos/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd/manifests/airflow/airflow.yaml index 58e45727..8ebbb9bf 100644 --- a/demos/argo-cd/manifests/airflow/airflow.yaml +++ b/demos/argo-cd/manifests/airflow/airflow.yaml @@ -38,7 +38,7 @@ spec: envOverrides: &envOverrides AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" # Via sealed secrets, just kept for reference here - #AIRFLOW_CONN_MINIO: "aws://admin:adminadmin@/?endpoint_url=https%3A%2F%2Fminio.minio.svc.cluster.local%3A9000" + # AIRFLOW_CONN_MINIO: "aws://admin:adminadmin@/?endpoint_url=https%3A%2F%2Fminio.minio.svc.cluster.local%3A9000" AWS_CA_BUNDLE: "/stackable/minio-tls/ca.crt" AIRFLOW__LOGGING__REMOTE_LOGGING: "True" AIRFLOW__LOGGING__REMOTE_BASE_LOG_FOLDER: s3://demo/airflow-task-logs/ diff --git a/demos/argo-cd/projects/airflow.yaml b/demos/argo-cd/projects/airflow.yaml index 3661434f..f518fd55 100644 --- a/demos/argo-cd/projects/airflow.yaml +++ b/demos/argo-cd/projects/airflow.yaml @@ -1,3 +1,4 @@ +--- apiVersion: argoproj.io/v1alpha1 kind: AppProject metadata: diff --git a/demos/argo-cd/projects/minio.yaml b/demos/argo-cd/projects/minio.yaml index ca58ee8a..38fee1df 100644 --- a/demos/argo-cd/projects/minio.yaml +++ b/demos/argo-cd/projects/minio.yaml @@ -1,3 +1,4 @@ +--- apiVersion: argoproj.io/v1alpha1 kind: AppProject metadata: diff --git a/stacks/argo-cd/applications/sealed-secrets.yaml b/stacks/argo-cd/applications/sealed-secrets.yaml index 73fd722c..450e7225 100644 --- a/stacks/argo-cd/applications/sealed-secrets.yaml +++ b/stacks/argo-cd/applications/sealed-secrets.yaml @@ -1,3 +1,4 @@ +--- apiVersion: argoproj.io/v1alpha1 kind: Application metadata: diff --git a/stacks/argo-cd/projects/stackable-operators.yaml b/stacks/argo-cd/projects/stackable-operators.yaml index 3ee37fc2..4acfc6cd 100644 --- a/stacks/argo-cd/projects/stackable-operators.yaml +++ b/stacks/argo-cd/projects/stackable-operators.yaml @@ -1,3 +1,4 @@ +--- apiVersion: argoproj.io/v1alpha1 kind: AppProject metadata: From 5e3498b7fe44fba88e49ead92ee65b2b51f6bcd6 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 14 Apr 2025 12:04:27 +0200 Subject: [PATCH 38/64] install all operators via argo --- demos/demos-v2.yaml | 1 + .../argo-cd/applicationsets/stackable-operators.yaml | 7 +++---- stacks/stacks-v2.yaml | 10 ++-------- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/demos/demos-v2.yaml b/demos/demos-v2.yaml index 00fc8d22..82c4e96b 100644 --- a/demos/demos-v2.yaml +++ b/demos/demos-v2.yaml @@ -5,6 +5,7 @@ demos: stackableStack: argocd labels: - argocd + - git ops - airflow - job-scheduling manifests: diff --git a/stacks/argo-cd/applicationsets/stackable-operators.yaml b/stacks/argo-cd/applicationsets/stackable-operators.yaml index 4ef1670a..b6138cae 100644 --- a/stacks/argo-cd/applicationsets/stackable-operators.yaml +++ b/stacks/argo-cd/applicationsets/stackable-operators.yaml @@ -7,10 +7,9 @@ spec: generators: - list: elements: - # Left out as currently installed via stackablectl - # operator: commons - # operator: listener - # operator: secret + - operator: commons + - operator: listener + - operator: secret - operator: airflow - operator: druid - operator: hbase diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 6355b5bd..67d44096 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -3,17 +3,11 @@ stacks: argocd: description: Deploying Stackable Demos with ArgoCD stackableRelease: dev - # TODO: We actually want to deploy all operators via ArgoCD, but we currently *have to* install - # operators with stackablectl. Therefore we install the internal operators via stackablectl. - # stackableOperators: [] - stackableOperators: - - listener - - commons - - secret + stackableOperators: ["argo-deploys-operators"] labels: - argocd - sealed secrets - - airflow + - git ops manifests: - helmChart: stacks/_templates/argo-cd.yaml ################################ From 0c30a99bc83626754762e3f49613bb0f886075ae Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 14 Apr 2025 12:06:29 +0200 Subject: [PATCH 39/64] improve comments and labels --- demos/demos-v2.yaml | 3 +++ stacks/stacks-v2.yaml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/demos/demos-v2.yaml b/demos/demos-v2.yaml index 82c4e96b..327f944d 100644 --- a/demos/demos-v2.yaml +++ b/demos/demos-v2.yaml @@ -8,6 +8,9 @@ demos: - git ops - airflow - job-scheduling + - git sync + - kubernetes executor + - sealed secrets manifests: ################################ # projects diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 67d44096..e387f2d8 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -1,7 +1,7 @@ --- stacks: argocd: - description: Deploying Stackable Demos with ArgoCD + description: Deploying Stackable Operators with ArgoCD stackableRelease: dev stackableOperators: ["argo-deploys-operators"] labels: From 33a375e81b37f3ce8ea9304f6bd0f8f8d6eaa3be Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Thu, 24 Apr 2025 10:46:40 +0200 Subject: [PATCH 40/64] remove airflowdb from clusterrole --- .../manifests/airflow/airflow-spark-clusterrole.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/demos/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml b/demos/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml index 66abed2f..d13657fa 100644 --- a/demos/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml +++ b/demos/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml @@ -12,14 +12,6 @@ rules: - create - get - list - - apiGroups: - - airflow.stackable.tech - resources: - - airflowdbs - verbs: - - create - - get - - list - apiGroups: - apps resources: From a6596b3dcfa877e55069cb65f163e2cc3a170e4d Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 19 May 2025 11:38:50 +0200 Subject: [PATCH 41/64] use 25.3 release --- .../applicationsets/stackable-operators.yaml | 69 ++++++++++++------- stacks/stacks-v2.yaml | 7 +- 2 files changed, 48 insertions(+), 28 deletions(-) diff --git a/stacks/argo-cd/applicationsets/stackable-operators.yaml b/stacks/argo-cd/applicationsets/stackable-operators.yaml index b6138cae..23a7fcf3 100644 --- a/stacks/argo-cd/applicationsets/stackable-operators.yaml +++ b/stacks/argo-cd/applicationsets/stackable-operators.yaml @@ -5,28 +5,47 @@ metadata: name: stackable-operators spec: generators: - - list: - elements: - - operator: commons - - operator: listener - - operator: secret - - operator: airflow - - operator: druid - - operator: hbase - - operator: hdfs - - operator: hive - - operator: kafka - - operator: nifi - - operator: opa - - operator: spark-k8s - - operator: superset - - operator: trino - - operator: zookeeper + - matrix: + generators: + - list: + elements: + - operator: commons + - operator: listener + - operator: secret + - operator: airflow + - operator: druid + - operator: hbase + - operator: hdfs + - operator: hive + - operator: kafka + - operator: nifi + - operator: opa + - operator: spark-k8s + - operator: superset + - operator: trino + - operator: zookeeper + - list: + elements: + - cluster: demo + server: https://kubernetes.default.svc + targetRevision: "25.3.0" + repoUrl: https://repo.stackable.tech/repository/helm-stable/ + # - cluster: development + # server: https://kubernetes-development.default.svc + # targetRevision: "0.0.0-dev" + # repoUrl: https://repo.stackable.tech/repository/helm-dev/ + # - cluster: staging + # server: https://kubernetes-staging.default.svc + # targetRevision: "25.3.0" + # repoUrl: https://repo.stackable.tech/repository/helm-stable/ + # - cluster: production + # server: https://kubernetes-production.default.svc + # targetRevision: "24.11.1" + # repoUrl: https://repo.stackable.tech/repository/helm-stable/ + # {% raw %} template: metadata: - # {% raw %} name: "{{ operator }}-operator" - # {% endraw %} spec: project: stackable-operators ignoreDifferences: @@ -38,17 +57,14 @@ spec: - .spec.names.shortNames | select(. == []) - .spec.versions[].additionalPrinterColumns | select(. == []) source: - repoURL: https://repo.stackable.tech/repository/helm-stable/ - targetRevision: "25.3.0" - # {% raw %} + repoURL: "{{ repoUrl }}" + targetRevision: "{{ targetRevision }}" chart: "{{ operator }}-operator" - # {% endraw %} helm: - # {% raw %} releaseName: "{{ operator }}-operator" - # {% endraw %} + destination: - server: https://kubernetes.default.svc + server: "{{ server }}" namespace: stackable-operators syncPolicy: syncOptions: @@ -58,3 +74,4 @@ spec: automated: selfHeal: true prune: true + # {% endraw %} diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index e387f2d8..8660ed99 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -9,13 +9,16 @@ stacks: - sealed secrets - git ops manifests: + ################################ + # bootstrap argo + ################################ - helmChart: stacks/_templates/argo-cd.yaml ################################ - # projects + # project - operators ################################ - plainYaml: stacks/argo-cd/projects/stackable-operators.yaml ################################ - # operators + # applicationset - operators ################################ - plainYaml: stacks/argo-cd/applicationsets/stackable-operators.yaml ################################ From 41c9cb5908a75dc9fbc51b8a0b540077ab0b107c Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 21 Jul 2025 17:48:25 +0200 Subject: [PATCH 42/64] bump airflow version, adapt listenerclass, parameterize git sync repo --- demos/argo-cd/manifests/airflow/airflow.yaml | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/demos/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd/manifests/airflow/airflow.yaml index 8ebbb9bf..6f4badd6 100644 --- a/demos/argo-cd/manifests/airflow/airflow.yaml +++ b/demos/argo-cd/manifests/airflow/airflow.yaml @@ -5,15 +5,14 @@ metadata: name: airflow spec: image: - productVersion: 2.10.4 + productVersion: 3.0.1 clusterConfig: - listenerClass: external-unstable loadExamples: false exposeConfig: false credentialsSecret: airflow-credentials dagsGitSync: - - repo: https://github.com/stackabletech/demos/ - branch: spike/argocd-demo + - repo: "{{ customGitUrl }}" + branch: "{{ customGitBranch }}" gitFolder: "demos/argo-cd/dags" depth: 2 volumes: @@ -35,6 +34,8 @@ spec: - name: minio-tls mountPath: /stackable/minio-tls webservers: + roleConfig: + listenerClass: external-unstable envOverrides: &envOverrides AIRFLOW_CONN_KUBERNETES_IN_CLUSTER: "kubernetes://?__extra__=%7B%22extra__kubernetes__in_cluster%22%3A+true%2C+%22extra__kubernetes__kube_config%22%3A+%22%22%2C+%22extra__kubernetes__kube_config_path%22%3A+%22%22%2C+%22extra__kubernetes__namespace%22%3A+%22%22%7D" # Via sealed secrets, just kept for reference here @@ -63,6 +64,11 @@ spec: limit: 1Gi roleGroups: default: + # NOTE: In order to properly show ArgoCD in action - syncing changes via Git - do the following: + # - Fork this repository + # - Use the demo parameters `customGitUrl` and `customGitBranch` to point the + # the repository to your fork and branch + # - Change the replicas here and push that to the provided `customGitBranch` in your fork replicas: 1 kubernetesExecutors: envOverrides: *envOverrides From bb9e227aa7626f2b63c55e735255351b9698d558 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 21 Jul 2025 17:52:49 +0200 Subject: [PATCH 43/64] revert templating - manged by argo --- demos/argo-cd/manifests/airflow/airflow.yaml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/demos/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd/manifests/airflow/airflow.yaml index 6f4badd6..d84d8267 100644 --- a/demos/argo-cd/manifests/airflow/airflow.yaml +++ b/demos/argo-cd/manifests/airflow/airflow.yaml @@ -10,9 +10,15 @@ spec: loadExamples: false exposeConfig: false credentialsSecret: airflow-credentials + # NOTE: This cannot be parameterized via stackablectl since it is synced via ArgoCD + # + # In terms of forking the repository, this should not be a problem since the DAGs are the same. + # You can still change this locally and point to the fork in case of any changes to DAGs. + # + # TODO(@maltesander): Adapt branch to main / 25.7.0 dagsGitSync: - - repo: "{{ customGitUrl }}" - branch: "{{ customGitBranch }}" + - repo: https://github.com/stackabletech/demos/ + branch: spike/argocd-demo gitFolder: "demos/argo-cd/dags" depth: 2 volumes: From 83e4ca0a4dbe9aceeb589efe8e42df8dd3cfae2a Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 21 Jul 2025 17:53:06 +0200 Subject: [PATCH 44/64] adapt sealed secrets version --- stacks/argo-cd/applications/sealed-secrets.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/argo-cd/applications/sealed-secrets.yaml b/stacks/argo-cd/applications/sealed-secrets.yaml index 450e7225..0bb704f1 100644 --- a/stacks/argo-cd/applications/sealed-secrets.yaml +++ b/stacks/argo-cd/applications/sealed-secrets.yaml @@ -8,7 +8,7 @@ spec: sources: - repoURL: "registry-1.docker.io/bitnamicharts" path: sealed-secrets - targetRevision: 2.5.9 # 0.29.0 + targetRevision: 2.5.16 # 0.30.0 chart: sealed-secrets helm: releaseName: sealed-secrets-controller From 304ec52b602ded7c82418f9f49ee1216f159760a Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 21 Jul 2025 17:53:44 +0200 Subject: [PATCH 45/64] customize repo, add opensearch as comment --- .../applicationsets/stackable-operators.yaml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/stacks/argo-cd/applicationsets/stackable-operators.yaml b/stacks/argo-cd/applicationsets/stackable-operators.yaml index 23a7fcf3..b31931b8 100644 --- a/stacks/argo-cd/applicationsets/stackable-operators.yaml +++ b/stacks/argo-cd/applicationsets/stackable-operators.yaml @@ -20,6 +20,8 @@ spec: - operator: kafka - operator: nifi - operator: opa + # TODO: enable Opensearch operator + # - opensearch - operator: spark-k8s - operator: superset - operator: trino @@ -28,20 +30,12 @@ spec: elements: - cluster: demo server: https://kubernetes.default.svc - targetRevision: "25.3.0" - repoUrl: https://repo.stackable.tech/repository/helm-stable/ + targetRevision: "{{ stackableReleaseVersion }}" + repoUrl: "{{ stackableRepositoryUrl }}" # - cluster: development # server: https://kubernetes-development.default.svc # targetRevision: "0.0.0-dev" # repoUrl: https://repo.stackable.tech/repository/helm-dev/ - # - cluster: staging - # server: https://kubernetes-staging.default.svc - # targetRevision: "25.3.0" - # repoUrl: https://repo.stackable.tech/repository/helm-stable/ - # - cluster: production - # server: https://kubernetes-production.default.svc - # targetRevision: "24.11.1" - # repoUrl: https://repo.stackable.tech/repository/helm-stable/ # {% raw %} template: metadata: From 0057f8ea62d2c620999fe449a0ab11d4fdf4fad7 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 21 Jul 2025 17:54:09 +0200 Subject: [PATCH 46/64] template repo urls --- demos/argo-cd/applications/airflow-postgres.yaml | 5 ++--- demos/argo-cd/applications/airflow.yaml | 5 ++--- demos/argo-cd/applications/minio.yaml | 5 ++--- 3 files changed, 6 insertions(+), 9 deletions(-) diff --git a/demos/argo-cd/applications/airflow-postgres.yaml b/demos/argo-cd/applications/airflow-postgres.yaml index 7052b64e..93361499 100644 --- a/demos/argo-cd/applications/airflow-postgres.yaml +++ b/demos/argo-cd/applications/airflow-postgres.yaml @@ -24,9 +24,8 @@ spec: database: airflow username: airflow existingSecret: postgresql-credentials - - repoURL: https://github.com/stackabletech/demos.git - # TODO: adapt to release-25.3 - targetRevision: "spike/argocd-demo" + - repoURL: "{{ customGitUrl }}" + targetRevision: "{{ customGitBranch }}" path: demos/argo-cd/manifests/airflow-postgres/ syncPolicy: syncOptions: diff --git a/demos/argo-cd/applications/airflow.yaml b/demos/argo-cd/applications/airflow.yaml index 82c1b71c..9070e058 100644 --- a/demos/argo-cd/applications/airflow.yaml +++ b/demos/argo-cd/applications/airflow.yaml @@ -9,9 +9,8 @@ spec: server: https://kubernetes.default.svc namespace: stackable-airflow source: - repoURL: https://github.com/stackabletech/demos.git - # TODO: adapt to release-25.3 - targetRevision: "spike/argocd-demo" + repoURL: "{{ customGitUrl }}" + targetRevision: "{{ customGitBranch }}" path: demos/argo-cd/manifests/airflow/ syncPolicy: syncOptions: diff --git a/demos/argo-cd/applications/minio.yaml b/demos/argo-cd/applications/minio.yaml index 42ca5407..7a2c1f4f 100644 --- a/demos/argo-cd/applications/minio.yaml +++ b/demos/argo-cd/applications/minio.yaml @@ -9,9 +9,8 @@ spec: server: https://kubernetes.default.svc namespace: minio source: - repoURL: https://github.com/stackabletech/demos.git - # TODO: adapt to release-25.3 - targetRevision: "spike/argocd-demo" + repoURL: "{{ customGitUrl }}" + targetRevision: "{{ customGitBranch }}" path: demos/argo-cd/manifests/minio/ syncPolicy: syncOptions: From 8a52bd91022d925a030c093789cdf70cfb47fd19 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 21 Jul 2025 17:54:48 +0200 Subject: [PATCH 47/64] add parameters, improve descrition --- demos/demos-v2.yaml | 7 +++++++ stacks/stacks-v2.yaml | 13 +++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/demos/demos-v2.yaml b/demos/demos-v2.yaml index 327f944d..c8bd8285 100644 --- a/demos/demos-v2.yaml +++ b/demos/demos-v2.yaml @@ -32,6 +32,13 @@ demos: cpu: 2401m memory: 9010Mi pvc: 24Gi + parameters: + - name: customGitUrl + description: The URL to the Git repository (this can be adapted in case of forking the repository) to demonstrate syncing via Argo and Git. + default: https://github.com/stackabletech/demos/ + - name: customGitBranch + description: The Branch in the Git repository (this can be adapted in case of forking the repository) to demonstrate syncing via Argo and Git. + default: spike/argocd-demo airflow-scheduled-job: description: Activate a simple Airflow DAG to run continuously at a set interval stackableStack: airflow diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 7e773915..316ed0b4 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -2,11 +2,12 @@ stacks: argocd: description: Deploying Stackable Operators with ArgoCD - stackableRelease: dev - stackableOperators: ["argo-deploys-operators"] + stackableRelease: dev # This does not matter -> managed by ArgoCD + stackableOperators: ["argo-deploys-operators"] # This does not matter -> managed by ArgoCD labels: - argocd - sealed secrets + - stackable operators - git ops manifests: ################################ @@ -33,10 +34,14 @@ stacks: pvc: 20Gi parameters: - name: stackableReleaseVersion - description: Stackable release to be installed via Argo + description: The Stackable release to be installed via Argo. This replaces the `stackableRelease` parameter. default: 25.3.0 + - name: stackableRepositoryUrl + description: The Stackable repository URL. This is usually required to test dev versions. + #default: https://repo.stackable.tech/repository/helm-dev/ + default: https://repo.stackable.tech/repository/helm-stable/ - name: argocdAdminPassword - description: Password of the ArgoCD admin user + description: Password of the ArgoCD admin user. # generated via: `htpasswd -nbBC 10 "" adminadmin | tr -d ':\n'` default: $2y$10$HhJC3pGHTlk8RyBoS39N/.wC72mdWxV2X8QS1wROUwCFxl.2tGfky monitoring: From e0e9b0aac4acf37e331f9cebdd798d390ec1d5ea Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 21 Jul 2025 19:12:33 +0200 Subject: [PATCH 48/64] bump argocd helm chart v8.1.4 --- stacks/_templates/argo-cd.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/_templates/argo-cd.yaml b/stacks/_templates/argo-cd.yaml index 1c72ab9c..b972f912 100644 --- a/stacks/_templates/argo-cd.yaml +++ b/stacks/_templates/argo-cd.yaml @@ -4,7 +4,7 @@ name: argo-cd repo: name: argo-cd url: https://argoproj.github.io/argo-helm -version: v7.8.23 +version: v8.1.4 options: configs: secret: From be5d6e620fcd3cb9c31edbfc421b235b675b8375 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 21 Jul 2025 19:14:55 +0200 Subject: [PATCH 49/64] switch to 0.0.0-dev operators --- .../argo-cd/applicationsets/stackable-operators.yaml | 10 ++-------- stacks/stacks-v2.yaml | 8 ++------ 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/stacks/argo-cd/applicationsets/stackable-operators.yaml b/stacks/argo-cd/applicationsets/stackable-operators.yaml index b31931b8..816d2119 100644 --- a/stacks/argo-cd/applicationsets/stackable-operators.yaml +++ b/stacks/argo-cd/applicationsets/stackable-operators.yaml @@ -31,11 +31,6 @@ spec: - cluster: demo server: https://kubernetes.default.svc targetRevision: "{{ stackableReleaseVersion }}" - repoUrl: "{{ stackableRepositoryUrl }}" - # - cluster: development - # server: https://kubernetes-development.default.svc - # targetRevision: "0.0.0-dev" - # repoUrl: https://repo.stackable.tech/repository/helm-dev/ # {% raw %} template: metadata: @@ -51,12 +46,11 @@ spec: - .spec.names.shortNames | select(. == []) - .spec.versions[].additionalPrinterColumns | select(. == []) source: - repoURL: "{{ repoUrl }}" + repoURL: "oci.stackable.tech" targetRevision: "{{ targetRevision }}" - chart: "{{ operator }}-operator" + chart: "sdp-charts/{{ operator }}-operator" helm: releaseName: "{{ operator }}-operator" - destination: server: "{{ server }}" namespace: stackable-operators diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 316ed0b4..17b2ad08 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -34,12 +34,8 @@ stacks: pvc: 20Gi parameters: - name: stackableReleaseVersion - description: The Stackable release to be installed via Argo. This replaces the `stackableRelease` parameter. - default: 25.3.0 - - name: stackableRepositoryUrl - description: The Stackable repository URL. This is usually required to test dev versions. - #default: https://repo.stackable.tech/repository/helm-dev/ - default: https://repo.stackable.tech/repository/helm-stable/ + description: The Stackable release to be installed via Argo. This replaces the demo `stackableRelease` parameter. + default: 0.0.0-dev - name: argocdAdminPassword description: Password of the ArgoCD admin user. # generated via: `htpasswd -nbBC 10 "" adminadmin | tr -d ':\n'` From 35099931bf726d21b1bc62a4504111490912e3eb Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 21 Jul 2025 19:19:45 +0200 Subject: [PATCH 50/64] attempt to fix dags for airflow 3 --- demos/argo-cd/dags/date_demo.py | 2 +- demos/argo-cd/dags/pyspark_pi.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/demos/argo-cd/dags/date_demo.py b/demos/argo-cd/dags/date_demo.py index 704aac51..45a5e211 100644 --- a/demos/argo-cd/dags/date_demo.py +++ b/demos/argo-cd/dags/date_demo.py @@ -6,7 +6,7 @@ with DAG( dag_id='date_demo', - schedule_interval='0-59 * * * *', + schedule='0-59 * * * *', start_date=datetime(2021, 1, 1), catchup=False, dagrun_timeout=timedelta(minutes=5), diff --git a/demos/argo-cd/dags/pyspark_pi.py b/demos/argo-cd/dags/pyspark_pi.py index 35bd4c38..ebc1290c 100644 --- a/demos/argo-cd/dags/pyspark_pi.py +++ b/demos/argo-cd/dags/pyspark_pi.py @@ -132,7 +132,7 @@ def poke(self, context: Dict) -> bool: with DAG( dag_id='sparkapp_dag', - schedule_interval=None, + schedule=None, start_date=datetime(2022, 1, 1), catchup=False, dagrun_timeout=timedelta(minutes=60), @@ -147,7 +147,7 @@ def load_body_to_dict(body): raise AirflowException(f"Exception when loading resource definition: {e}\n") return body_dict - yaml_path = os.path.join(os.environ.get('AIRFLOW__CORE__DAGS_FOLDER'), '../manifests/spark-k8s/pyspark_pi.yaml') + yaml_path = os.path.join(os.environ.get('AIRFLOW__CORE__DAGS_FOLDER'), 'pyspark_pi.yaml') with open(yaml_path, 'r') as file: crd = file.read() From f364191ec3d0e45ef03b2ebc06ff50b362c7979e Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 21 Jul 2025 19:19:55 +0200 Subject: [PATCH 51/64] fix scope --- demos/argo-cd/manifests/airflow/airflow.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/demos/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd/manifests/airflow/airflow.yaml index d84d8267..39a2bc1c 100644 --- a/demos/argo-cd/manifests/airflow/airflow.yaml +++ b/demos/argo-cd/manifests/airflow/airflow.yaml @@ -28,7 +28,7 @@ spec: metadata: annotations: secrets.stackable.tech/class: tls - secrets.stackable.tech/scope: pod,node + secrets.stackable.tech/scope: pod spec: accessModes: - ReadWriteOnce From d9eb2c794615670ea26f6135ac6beaad6689fa63 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 21 Jul 2025 19:38:05 +0200 Subject: [PATCH 52/64] attempt to fix dag --- demos/argo-cd/manifests/spark-k8s/pyspark_pi.yaml | 2 +- stacks/airflow/airflow.yaml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/demos/argo-cd/manifests/spark-k8s/pyspark_pi.yaml b/demos/argo-cd/manifests/spark-k8s/pyspark_pi.yaml index 559f4822..d5d84da3 100644 --- a/demos/argo-cd/manifests/spark-k8s/pyspark_pi.yaml +++ b/demos/argo-cd/manifests/spark-k8s/pyspark_pi.yaml @@ -6,7 +6,7 @@ metadata: spec: version: "1.0" sparkImage: - productVersion: 3.5.2 + productVersion: 3.5.6 mode: cluster mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py job: diff --git a/stacks/airflow/airflow.yaml b/stacks/airflow/airflow.yaml index 1a02b1ef..e3b8cf36 100644 --- a/stacks/airflow/airflow.yaml +++ b/stacks/airflow/airflow.yaml @@ -238,7 +238,7 @@ data: raise AirflowException(f"Exception when loading resource definition: {e}\n") return body_dict - yaml_path = os.path.join(os.environ.get('AIRFLOW__CORE__DAGS_FOLDER'), 'pyspark_pi.yaml') + yaml_path = os.path.join(os.environ.get('AIRFLOW__CORE__DAGS_FOLDER'), '../manifests/spark-k8s/pyspark_pi.yaml') with open(yaml_path, 'r') as file: crd = file.read() @@ -303,6 +303,8 @@ data: memory: limit: 1024Mi replicas: 3 + + # {% endraw %} --- apiVersion: v1 From 24ff3e87fd61321bfa0e6737d0625ce8aa59c55c Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Mon, 21 Jul 2025 19:47:58 +0200 Subject: [PATCH 53/64] change path in correct airflow file... --- demos/argo-cd/dags/pyspark_pi.py | 2 +- stacks/airflow/airflow.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/demos/argo-cd/dags/pyspark_pi.py b/demos/argo-cd/dags/pyspark_pi.py index ebc1290c..9c408527 100644 --- a/demos/argo-cd/dags/pyspark_pi.py +++ b/demos/argo-cd/dags/pyspark_pi.py @@ -147,7 +147,7 @@ def load_body_to_dict(body): raise AirflowException(f"Exception when loading resource definition: {e}\n") return body_dict - yaml_path = os.path.join(os.environ.get('AIRFLOW__CORE__DAGS_FOLDER'), 'pyspark_pi.yaml') + yaml_path = os.path.join(os.environ.get('AIRFLOW__CORE__DAGS_FOLDER'), '../manifests/spark-k8s/pyspark_pi.yaml') with open(yaml_path, 'r') as file: crd = file.read() diff --git a/stacks/airflow/airflow.yaml b/stacks/airflow/airflow.yaml index e3b8cf36..dab3f072 100644 --- a/stacks/airflow/airflow.yaml +++ b/stacks/airflow/airflow.yaml @@ -238,7 +238,7 @@ data: raise AirflowException(f"Exception when loading resource definition: {e}\n") return body_dict - yaml_path = os.path.join(os.environ.get('AIRFLOW__CORE__DAGS_FOLDER'), '../manifests/spark-k8s/pyspark_pi.yaml') + yaml_path = os.path.join(os.environ.get('AIRFLOW__CORE__DAGS_FOLDER'), 'pyspark_pi.yaml') with open(yaml_path, 'r') as file: crd = file.read() From 0b9761f487a145d2c29e30ac2d83eb4316d4df8c Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 22 Jul 2025 12:34:09 +0200 Subject: [PATCH 54/64] use airflow 2.10.5 --- demos/argo-cd/manifests/airflow/airflow.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/demos/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd/manifests/airflow/airflow.yaml index 39a2bc1c..dbc8f311 100644 --- a/demos/argo-cd/manifests/airflow/airflow.yaml +++ b/demos/argo-cd/manifests/airflow/airflow.yaml @@ -5,12 +5,15 @@ metadata: name: airflow spec: image: - productVersion: 3.0.1 + # Currently does not work with the kubernetes executor S3 logging + # (and its still marked experimental as of release 25.7) + # productVersion: 3.0.1 + productVersion: 2.10.5 clusterConfig: loadExamples: false exposeConfig: false credentialsSecret: airflow-credentials - # NOTE: This cannot be parameterized via stackablectl since it is synced via ArgoCD + # NOTE: This cannot be parameterized via stackablectl since it is synced via ArgoCD. # # In terms of forking the repository, this should not be a problem since the DAGs are the same. # You can still change this locally and point to the fork in case of any changes to DAGs. From 59c25dcf618d033961f79c15e4b9ecb073e2a60b Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Tue, 22 Jul 2025 12:36:26 +0200 Subject: [PATCH 55/64] deploy sealed secrets before operators --- stacks/stacks-v2.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index 17b2ad08..aabda5d2 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -15,6 +15,10 @@ stacks: ################################ - helmChart: stacks/_templates/argo-cd.yaml ################################ + # prerequisites + ################################ + - plainYaml: stacks/argo-cd/applications/sealed-secrets.yaml + ################################ # project - operators ################################ - plainYaml: stacks/argo-cd/projects/stackable-operators.yaml @@ -22,10 +26,6 @@ stacks: # applicationset - operators ################################ - plainYaml: stacks/argo-cd/applicationsets/stackable-operators.yaml - ################################ - # prerequisites - ################################ - - plainYaml: stacks/argo-cd/applications/sealed-secrets.yaml supportedNamespaces: - argo-cd resourceRequests: From 6b538b1910065f1073c9cc402972aba87c3221c2 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 23 Jul 2025 15:34:30 +0200 Subject: [PATCH 56/64] change demo name to argo-cd-git-ops --- .../applications/airflow-postgres.yaml | 2 +- .../applications/airflow.yaml | 2 +- .../applications/minio.yaml | 2 +- .../{argo-cd => argo-cd-git-ops}/dags/date_demo.py | 0 .../dags/pyspark_pi.py | 0 .../sealed-airflow-postgres-credentials.yaml | 0 .../airflow/airflow-spark-clusterrole.yaml | 0 .../airflow/airflow-spark-clusterrolebinding.yaml | 0 .../manifests/airflow/airflow.yaml | 2 +- .../airflow/sealed-airflow-credentials.yaml | 0 .../airflow/sealed-airflow-minio-connection.yaml | 0 .../manifests/minio/minio.yaml | 0 .../manifests/minio/sealed-minio-credentials.yaml | 0 .../manifests/spark-k8s/pyspark_pi.yaml | 0 .../projects/airflow.yaml | 0 .../projects/minio.yaml | 0 demos/demos-v2.yaml | 14 +++++++------- .../applications/sealed-secrets.yaml | 2 +- .../applicationsets/stackable-operators.yaml | 0 .../projects/stackable-operators.yaml | 0 .../secrets/sealed-secrets-key.yaml | 0 stacks/stacks-v2.yaml | 8 ++++---- 22 files changed, 16 insertions(+), 16 deletions(-) rename demos/{argo-cd => argo-cd-git-ops}/applications/airflow-postgres.yaml (94%) rename demos/{argo-cd => argo-cd-git-ops}/applications/airflow.yaml (88%) rename demos/{argo-cd => argo-cd-git-ops}/applications/minio.yaml (88%) rename demos/{argo-cd => argo-cd-git-ops}/dags/date_demo.py (100%) rename demos/{argo-cd => argo-cd-git-ops}/dags/pyspark_pi.py (100%) rename demos/{argo-cd => argo-cd-git-ops}/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml (100%) rename demos/{argo-cd => argo-cd-git-ops}/manifests/airflow/airflow-spark-clusterrole.yaml (100%) rename demos/{argo-cd => argo-cd-git-ops}/manifests/airflow/airflow-spark-clusterrolebinding.yaml (100%) rename demos/{argo-cd => argo-cd-git-ops}/manifests/airflow/airflow.yaml (98%) rename demos/{argo-cd => argo-cd-git-ops}/manifests/airflow/sealed-airflow-credentials.yaml (100%) rename demos/{argo-cd => argo-cd-git-ops}/manifests/airflow/sealed-airflow-minio-connection.yaml (100%) rename demos/{argo-cd => argo-cd-git-ops}/manifests/minio/minio.yaml (100%) rename demos/{argo-cd => argo-cd-git-ops}/manifests/minio/sealed-minio-credentials.yaml (100%) rename demos/{argo-cd => argo-cd-git-ops}/manifests/spark-k8s/pyspark_pi.yaml (100%) rename demos/{argo-cd => argo-cd-git-ops}/projects/airflow.yaml (100%) rename demos/{argo-cd => argo-cd-git-ops}/projects/minio.yaml (100%) rename stacks/{argo-cd => argo-cd-git-ops}/applications/sealed-secrets.yaml (94%) rename stacks/{argo-cd => argo-cd-git-ops}/applicationsets/stackable-operators.yaml (100%) rename stacks/{argo-cd => argo-cd-git-ops}/projects/stackable-operators.yaml (100%) rename stacks/{argo-cd => argo-cd-git-ops}/secrets/sealed-secrets-key.yaml (100%) diff --git a/demos/argo-cd/applications/airflow-postgres.yaml b/demos/argo-cd-git-ops/applications/airflow-postgres.yaml similarity index 94% rename from demos/argo-cd/applications/airflow-postgres.yaml rename to demos/argo-cd-git-ops/applications/airflow-postgres.yaml index 93361499..a2f4daf1 100644 --- a/demos/argo-cd/applications/airflow-postgres.yaml +++ b/demos/argo-cd-git-ops/applications/airflow-postgres.yaml @@ -26,7 +26,7 @@ spec: existingSecret: postgresql-credentials - repoURL: "{{ customGitUrl }}" targetRevision: "{{ customGitBranch }}" - path: demos/argo-cd/manifests/airflow-postgres/ + path: demos/argo-cd-git-ops/manifests/airflow-postgres/ syncPolicy: syncOptions: - CreateNamespace=true diff --git a/demos/argo-cd/applications/airflow.yaml b/demos/argo-cd-git-ops/applications/airflow.yaml similarity index 88% rename from demos/argo-cd/applications/airflow.yaml rename to demos/argo-cd-git-ops/applications/airflow.yaml index 9070e058..276ede5e 100644 --- a/demos/argo-cd/applications/airflow.yaml +++ b/demos/argo-cd-git-ops/applications/airflow.yaml @@ -11,7 +11,7 @@ spec: source: repoURL: "{{ customGitUrl }}" targetRevision: "{{ customGitBranch }}" - path: demos/argo-cd/manifests/airflow/ + path: demos/argo-cd-git-ops/manifests/airflow/ syncPolicy: syncOptions: - CreateNamespace=true diff --git a/demos/argo-cd/applications/minio.yaml b/demos/argo-cd-git-ops/applications/minio.yaml similarity index 88% rename from demos/argo-cd/applications/minio.yaml rename to demos/argo-cd-git-ops/applications/minio.yaml index 7a2c1f4f..12e220fa 100644 --- a/demos/argo-cd/applications/minio.yaml +++ b/demos/argo-cd-git-ops/applications/minio.yaml @@ -11,7 +11,7 @@ spec: source: repoURL: "{{ customGitUrl }}" targetRevision: "{{ customGitBranch }}" - path: demos/argo-cd/manifests/minio/ + path: demos/argo-cd-git-ops/manifests/minio/ syncPolicy: syncOptions: - CreateNamespace=true diff --git a/demos/argo-cd/dags/date_demo.py b/demos/argo-cd-git-ops/dags/date_demo.py similarity index 100% rename from demos/argo-cd/dags/date_demo.py rename to demos/argo-cd-git-ops/dags/date_demo.py diff --git a/demos/argo-cd/dags/pyspark_pi.py b/demos/argo-cd-git-ops/dags/pyspark_pi.py similarity index 100% rename from demos/argo-cd/dags/pyspark_pi.py rename to demos/argo-cd-git-ops/dags/pyspark_pi.py diff --git a/demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml b/demos/argo-cd-git-ops/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml similarity index 100% rename from demos/argo-cd/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml rename to demos/argo-cd-git-ops/manifests/airflow-postgres/sealed-airflow-postgres-credentials.yaml diff --git a/demos/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml b/demos/argo-cd-git-ops/manifests/airflow/airflow-spark-clusterrole.yaml similarity index 100% rename from demos/argo-cd/manifests/airflow/airflow-spark-clusterrole.yaml rename to demos/argo-cd-git-ops/manifests/airflow/airflow-spark-clusterrole.yaml diff --git a/demos/argo-cd/manifests/airflow/airflow-spark-clusterrolebinding.yaml b/demos/argo-cd-git-ops/manifests/airflow/airflow-spark-clusterrolebinding.yaml similarity index 100% rename from demos/argo-cd/manifests/airflow/airflow-spark-clusterrolebinding.yaml rename to demos/argo-cd-git-ops/manifests/airflow/airflow-spark-clusterrolebinding.yaml diff --git a/demos/argo-cd/manifests/airflow/airflow.yaml b/demos/argo-cd-git-ops/manifests/airflow/airflow.yaml similarity index 98% rename from demos/argo-cd/manifests/airflow/airflow.yaml rename to demos/argo-cd-git-ops/manifests/airflow/airflow.yaml index dbc8f311..8a5be9c2 100644 --- a/demos/argo-cd/manifests/airflow/airflow.yaml +++ b/demos/argo-cd-git-ops/manifests/airflow/airflow.yaml @@ -22,7 +22,7 @@ spec: dagsGitSync: - repo: https://github.com/stackabletech/demos/ branch: spike/argocd-demo - gitFolder: "demos/argo-cd/dags" + gitFolder: "demos/argo-cd-git-ops/dags" depth: 2 volumes: - name: minio-tls diff --git a/demos/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml b/demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-credentials.yaml similarity index 100% rename from demos/argo-cd/manifests/airflow/sealed-airflow-credentials.yaml rename to demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-credentials.yaml diff --git a/demos/argo-cd/manifests/airflow/sealed-airflow-minio-connection.yaml b/demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-minio-connection.yaml similarity index 100% rename from demos/argo-cd/manifests/airflow/sealed-airflow-minio-connection.yaml rename to demos/argo-cd-git-ops/manifests/airflow/sealed-airflow-minio-connection.yaml diff --git a/demos/argo-cd/manifests/minio/minio.yaml b/demos/argo-cd-git-ops/manifests/minio/minio.yaml similarity index 100% rename from demos/argo-cd/manifests/minio/minio.yaml rename to demos/argo-cd-git-ops/manifests/minio/minio.yaml diff --git a/demos/argo-cd/manifests/minio/sealed-minio-credentials.yaml b/demos/argo-cd-git-ops/manifests/minio/sealed-minio-credentials.yaml similarity index 100% rename from demos/argo-cd/manifests/minio/sealed-minio-credentials.yaml rename to demos/argo-cd-git-ops/manifests/minio/sealed-minio-credentials.yaml diff --git a/demos/argo-cd/manifests/spark-k8s/pyspark_pi.yaml b/demos/argo-cd-git-ops/manifests/spark-k8s/pyspark_pi.yaml similarity index 100% rename from demos/argo-cd/manifests/spark-k8s/pyspark_pi.yaml rename to demos/argo-cd-git-ops/manifests/spark-k8s/pyspark_pi.yaml diff --git a/demos/argo-cd/projects/airflow.yaml b/demos/argo-cd-git-ops/projects/airflow.yaml similarity index 100% rename from demos/argo-cd/projects/airflow.yaml rename to demos/argo-cd-git-ops/projects/airflow.yaml diff --git a/demos/argo-cd/projects/minio.yaml b/demos/argo-cd-git-ops/projects/minio.yaml similarity index 100% rename from demos/argo-cd/projects/minio.yaml rename to demos/argo-cd-git-ops/projects/minio.yaml diff --git a/demos/demos-v2.yaml b/demos/demos-v2.yaml index 66622b5c..683a174d 100644 --- a/demos/demos-v2.yaml +++ b/demos/demos-v2.yaml @@ -1,8 +1,8 @@ --- demos: - argocd: + argo-cd-git-ops: description: Deploy Stackable operators and Airflow via ArgoCD and activate a simple Airflow DAG. - stackableStack: argocd + stackableStack: argo-cd-git-ops labels: - argocd - git ops @@ -15,17 +15,17 @@ demos: ################################ # projects ################################ - - plainYaml: demos/argo-cd/projects/airflow.yaml - - plainYaml: demos/argo-cd/projects/minio.yaml + - plainYaml: demos/argo-cd-git-ops/projects/airflow.yaml + - plainYaml: demos/argo-cd-git-ops/projects/minio.yaml ################################ # prerequisites ################################ - - plainYaml: demos/argo-cd/applications/airflow-postgres.yaml - - plainYaml: demos/argo-cd/applications/minio.yaml + - plainYaml: demos/argo-cd-git-ops/applications/airflow-postgres.yaml + - plainYaml: demos/argo-cd-git-ops/applications/minio.yaml ################################ # products ################################ - - plainYaml: demos/argo-cd/applications/airflow.yaml + - plainYaml: demos/argo-cd-git-ops/applications/airflow.yaml supportedNamespaces: - argo-cd resourceRequests: diff --git a/stacks/argo-cd/applications/sealed-secrets.yaml b/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml similarity index 94% rename from stacks/argo-cd/applications/sealed-secrets.yaml rename to stacks/argo-cd-git-ops/applications/sealed-secrets.yaml index 0bb704f1..84284847 100644 --- a/stacks/argo-cd/applications/sealed-secrets.yaml +++ b/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml @@ -17,7 +17,7 @@ spec: - repoURL: https://github.com/stackabletech/demos.git # TODO: adapt to release-25.3 targetRevision: "spike/argocd-demo" - path: stacks/argo-cd/secrets/ + path: stacks/argo-cd-git-ops/secrets/ destination: server: https://kubernetes.default.svc namespace: kube-system diff --git a/stacks/argo-cd/applicationsets/stackable-operators.yaml b/stacks/argo-cd-git-ops/applicationsets/stackable-operators.yaml similarity index 100% rename from stacks/argo-cd/applicationsets/stackable-operators.yaml rename to stacks/argo-cd-git-ops/applicationsets/stackable-operators.yaml diff --git a/stacks/argo-cd/projects/stackable-operators.yaml b/stacks/argo-cd-git-ops/projects/stackable-operators.yaml similarity index 100% rename from stacks/argo-cd/projects/stackable-operators.yaml rename to stacks/argo-cd-git-ops/projects/stackable-operators.yaml diff --git a/stacks/argo-cd/secrets/sealed-secrets-key.yaml b/stacks/argo-cd-git-ops/secrets/sealed-secrets-key.yaml similarity index 100% rename from stacks/argo-cd/secrets/sealed-secrets-key.yaml rename to stacks/argo-cd-git-ops/secrets/sealed-secrets-key.yaml diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index adea23df..9bf264b4 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -1,6 +1,6 @@ --- stacks: - argocd: + argo-cd-git-ops: description: Deploying Stackable Operators with ArgoCD stackableRelease: dev # This does not matter -> managed by ArgoCD stackableOperators: ["argo-deploys-operators"] # This does not matter -> managed by ArgoCD @@ -17,15 +17,15 @@ stacks: ################################ # prerequisites ################################ - - plainYaml: stacks/argo-cd/applications/sealed-secrets.yaml + - plainYaml: stacks/argo-cd-git-ops/applications/sealed-secrets.yaml ################################ # project - operators ################################ - - plainYaml: stacks/argo-cd/projects/stackable-operators.yaml + - plainYaml: stacks/argo-cd-git-ops/projects/stackable-operators.yaml ################################ # applicationset - operators ################################ - - plainYaml: stacks/argo-cd/applicationsets/stackable-operators.yaml + - plainYaml: stacks/argo-cd-git-ops/applicationsets/stackable-operators.yaml supportedNamespaces: - argo-cd resourceRequests: From 68778e105c4087844d7cf2232ccb94b2c2d8f9b5 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 23 Jul 2025 15:46:09 +0200 Subject: [PATCH 57/64] parameterize sealed secrets repo / target revision --- stacks/argo-cd-git-ops/applications/sealed-secrets.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml b/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml index 84284847..1415daf0 100644 --- a/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml +++ b/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml @@ -14,9 +14,8 @@ spec: releaseName: sealed-secrets-controller valuesObject: secretName: sealed-secrets-key - - repoURL: https://github.com/stackabletech/demos.git - # TODO: adapt to release-25.3 - targetRevision: "spike/argocd-demo" + - repoURL: "{{ customGitUrl }}" + targetRevision: "{{ customGitBranch }}" path: stacks/argo-cd-git-ops/secrets/ destination: server: https://kubernetes.default.svc From f63e6a126b2080cf0eaa5876cf02401c3670da8e Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 23 Jul 2025 15:46:16 +0200 Subject: [PATCH 58/64] wip - docs --- docs/modules/demos/pages/argo-cd-git-ops.adoc | 283 ++++++++++++++++++ docs/modules/demos/partials/demos.adoc | 1 + 2 files changed, 284 insertions(+) create mode 100644 docs/modules/demos/pages/argo-cd-git-ops.adoc diff --git a/docs/modules/demos/pages/argo-cd-git-ops.adoc b/docs/modules/demos/pages/argo-cd-git-ops.adoc new file mode 100644 index 00000000..065233c4 --- /dev/null +++ b/docs/modules/demos/pages/argo-cd-git-ops.adoc @@ -0,0 +1,283 @@ += argo-cd-git-ops +:description: Deploy Stackable operators and products with ArgoCD. Sync manifests and secrets from Git. + +:k8s-cpu: https://kubernetes.io/docs/tasks/debug/debug-cluster/resource-metrics-pipeline/#cpu +:argo-cd: https://argoproj.github.io/cd/ +:argo-cd-application: https://argo-cd.readthedocs.io/en/stable/operator-manual/declarative-setup/#applications +:argo-cd-applicationset: https://argo-cd.readthedocs.io/en/stable/user-guide/application-set/ +:sealed-secrets: https://github.com/bitnami-labs/sealed-secrets +:stackable-demo-repository: https://github.com/stackabletech/demos/ +:airflow: https://airflow.apache.org/ + +This demo shows how to utilize GitOps and Infrastructure as Code (IaC) with Stackable and {argo-cd}[ArgoCD]. +All products and manifests are synced and deployed via ArgoCD (except ArgoCD itself, which is bootstrapped via `stackablectl`). + +The key points to show are: +* GitOps: How to deploy my changes from a Git repository to my cluster +* Secrets: How to safely deploy credentials, access keys and store them in a Git repository +* Although not included in this Demo, it assumes / tries to imitate a possible + multicluster setup for development, staging and production. +* Interaction with Stackable products (e.g. Airflow and DAGs via gitsync) + +Install this demo on an existing Kubernetes cluster: + +[source,console] +---- +$ `stackablectl` demo install argo-cd-git-ops --namespace argo-cd +---- + +WARNING: This demo should not be run alongside other demos. + +NOTE: ArgoCD will be deployed in the `argo-cd` namespace by `stackablectl`. ArgoCD itself will create other namespaces for the deployed products. + +[#system-requirements] +== System requirements + +To run this demo, your system needs at least: + +* 20 {k8s-cpu}[cpu units] (core/hyperthread) +* 20GiB memory +* 20GiB disk storage + +== Overview + +This demo consists of multiple parts: + +* Bootstrapping via `stackablectl` +** Install a {sealed-secrets}[Sealed Secrets] controller via `stackablectl` to handle sensitive data like credentials or secret keys. +** Install {argo-cd}[Argo CD] via `stackablectl`. +* After the initial bootstrapping, ArgoCD takes over the deployment: +** Install all Stackable operators using an `ApplicationSet`. +** Spin up requirements like Minio and Postgres as `Application`. +** Deploy Stackable Airflow manifests in their respective `Projects`. +** DAGs are synced via Airflow gitsync, not ArgoCD. +** Airflow uses the Kubernetes executor and write its logs to S3 / Minio. +** DAGs can be started manually in the Airflow webserver UI + +TODO: arch overview image + +== ArgoCD UI + +ArgoCD will be the first product that is deployed in this Demo. Once the pods are ready, you can port-forward the argocd-server in order to access the Web UI. + +[source,console] +---- +kubectl --namespace argo-cd port-forward service/argocd-server 8443:https +---- + +In your browser, go to `https://localhost:8443` and login with username `admin` and password `adminadmin`. + +NOTE: There will be an initial warning from the Browser that this site is insecure due to self-signed certificates. This can be ignored in this case. + +TODO: screenshots + +== Sealed Secrets + +When managing all resources and configs via Git, deploying sensitive properties like certificates or credentials via Git becomes a problem. + +There are multiple solutions like Hashicorp or Bitwarden, which heavily depend on the infrastructure already available. + +For the sake of this demo, {sealed-secrets}[Bitnami's Sealed Secrets] are utilized. +Sensitive data is encrypted as a `SealedSecret` before commiting to the Git repository, synced via ArgoCD and decrypted by the Sealed Secrets controller into a standard Kubernetes `Secret`. + +This way, everything will be stored and managed in Git. + +== Stackable operators + +The Stackable operators are deployed via ArgoCD using the Stackable Helm charts and an ArgoCD {argo-cd-applicationset}[`ApplicationSet`]. +`ApplicationSets` allow templating, which is required to e.g. manage and deploy to multi cluster environments (e.g. development - staging - production), +using different versions and Git sources (repository & branch) as well as the possibility to deploy to different clusters. + +NOTE: This demo does not use a multi cluster environment for the sake of simplicity. + +[source,console] +---- +--- +apiVersion: argoproj.io/v1alpha1 +kind: ApplicationSet +metadata: + name: stackable-operators +spec: + generators: + - matrix: + generators: + - list: + elements: + - operator: commons + - operator: listener + - operator: secret + - operator: airflow + - operator: druid + - operator: hbase + - operator: hdfs + - operator: hive + - operator: kafka + - operator: nifi + - operator: opa + - operator: spark-k8s + - operator: superset + - operator: trino + - operator: zookeeper + - list: + elements: + - cluster: demo + server: https://kubernetes.default.svc + targetRevision: 25.7.0 + ########################################################################################### + # The following definitions are not used in this Demo, it is shown for completeness + # for multi cluster setups + ########################################################################################### + + ########################################################################################### + # Development cluster: Checking newest Stackable developments for nightly 0.0.0-dev builds + ########################################################################################### + # - cluster: development + # server: https://kubernetes-development.default.svc + # targetRevision: 0.0.0-dev + ########################################################################################### + # Staging cluster: Checking compatibility for upgrades from 25.3.0 to 25.7.0 + ########################################################################################### + # - cluster: staging + # server: https://kubernetes-staging.default.svc + # targetRevision: 25.7.0 + ########################################################################################### + # Production cluster: Currently running release 25.3.0 and awaiting upgrade to 25.7.0 + ########################################################################################### + # - cluster: production + # server: https://kubernetes-production.default.svc + # targetRevision: 25.3.0 +# [...] +---- + +The `matrix.generators.list[].elements[]` will create a union of parameters that may be used in the `ApplicationSet` template as follows: + +[source,console] +---- +# [...] +template: + metadata: + name: "{{ operator }}-operator" + spec: + project: "stackable-operators" + ignoreDifferences: + # mitigating: https://github.com/stackabletech/hdfs-operator/issues/626 + - group: "apiextensions.k8s.io" + kind: "CustomResourceDefinition" + jqPathExpressions: + - .spec.names.categories | select(. == []) + - .spec.names.shortNames | select(. == []) + - .spec.versions[].additionalPrinterColumns | select(. == []) + source: + repoURL: "oci.stackable.tech" + targetRevision: "{{ targetRevision }}" + chart: "sdp-charts/{{ operator }}-operator" + helm: + releaseName: "{{ operator }}-operator" + destination: + server: "{{ server }}" + namespace: "stackable-operators" + syncPolicy: + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + automated: + selfHeal: true + prune: true +---- + +The templated version for e.g. the parameters `operator=zookeeper`, `server=https://kubernetes.default.svc` and `targetRevision=25.7.0` will result in the following template: + +[source,console] +---- +# [...] +template: + metadata: + name: "zookeeper-operator" + spec: + project: "stackable-operators" + ignoreDifferences: + - group: "apiextensions.k8s.io" + kind: "CustomResourceDefinition" + jqPathExpressions: + - .spec.names.categories | select(. == []) + - .spec.names.shortNames | select(. == []) + - .spec.versions[].additionalPrinterColumns | select(. == []) + source: + repoURL: "oci.stackable.tech" + targetRevision: "25.7.0" + chart: "sdp-charts/zookeeper-operator" + helm: + releaseName: "zookeeper-operator" + destination: + server: "https://kubernetes.default.svc" + namespace: "stackable-operators" + syncPolicy: + syncOptions: + - CreateNamespace=true + - ServerSideApply=true + - RespectIgnoreDifferences=true + automated: + selfHeal: true + prune: true +---- + +This allows control over which releases and versions are deployed to which cluster. + +Now with ArgoCD deployed, the Sealed Secrets controller and Stackable operators up and running, you can inspect Airflow as the first Stackable product. + +== Airflow + +The Airflow Web UI is reachable via Nodeport or easier, using a port-forward: + +[source,console] +---- +kubectl --namespace stackable-airflow port-forward service/airflow-webserver 8080 +---- + +In your browser, go to `https://localhost:8080` and login with username `admin` and password `adminadmin`. + +TODO: Screenshots + +TODO: Similar to the ArgoCD {argo-cd-application}[`Application`] + +=== Starting DAGs + +TODO: screenshots? + +=== Checking logs + +TODO: even important here? + +== How to interact with ArgoCD and the Git repository + +Since this Demo is hosted in the {stackable-demo-repository}[Stackable Demo repository], where merging etc. requires approval, the recommendation is to fork the {stackable-demo-repository}[Stackable Demo repository]. + +Once forked, you can install this demo using `stackablectl` parameters to customize the forked repository: + +[source,console] +---- +stackablectl demo install argo-cd-git-ops --namespace argo-cd --parameters customGitUrl= --parameters customGitBranch= +---- + +This way, ArgoCD is instructed to pull the Stackable manifests from the forked repository, where your changes can be properly synced via ArgoCD. + +=== Increase Airflow webserver replicas + +Assuming your working directory ist the root of the forked demo repository, try to increase the `spec.webservers.roleGroups..replicas` in the folder `demos/argo-cd-git-ops/manifests/airflow/airflow.yaml`. +Once this is pushed / merged, ArgoCD should sync the changes and you should see more webserver pods. + +=== Add new Airflow DAGs + +Similar to ArgoCD, after adding a new DAG to the folder `demos/argo-cd-git-ops/dags`, Airflow should pick up the new DAG via gitsync. + + + + + + + + + + + diff --git a/docs/modules/demos/partials/demos.adoc b/docs/modules/demos/partials/demos.adoc index ebd54e62..c02bf24e 100644 --- a/docs/modules/demos/partials/demos.adoc +++ b/docs/modules/demos/partials/demos.adoc @@ -1,4 +1,5 @@ * xref:airflow-scheduled-job.adoc[] +* xref:argo-cd-git-ops.adoc[] * xref:data-lakehouse-iceberg-trino-spark.adoc[] * xref:end-to-end-security.adoc[] * xref:hbase-hdfs-load-cycling-data.adoc[] From 7d747c2c144cb66f64c0aeb57afdcdca127f3738 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 23 Jul 2025 16:12:36 +0200 Subject: [PATCH 59/64] small fixes --- docs/modules/demos/pages/argo-cd-git-ops.adoc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/modules/demos/pages/argo-cd-git-ops.adoc b/docs/modules/demos/pages/argo-cd-git-ops.adoc index 065233c4..0c97ce87 100644 --- a/docs/modules/demos/pages/argo-cd-git-ops.adoc +++ b/docs/modules/demos/pages/argo-cd-git-ops.adoc @@ -23,7 +23,7 @@ Install this demo on an existing Kubernetes cluster: [source,console] ---- -$ `stackablectl` demo install argo-cd-git-ops --namespace argo-cd +$ stackablectl demo install argo-cd-git-ops --namespace argo-cd ---- WARNING: This demo should not be run alongside other demos. @@ -235,7 +235,7 @@ The Airflow Web UI is reachable via Nodeport or easier, using a port-forward: kubectl --namespace stackable-airflow port-forward service/airflow-webserver 8080 ---- -In your browser, go to `https://localhost:8080` and login with username `admin` and password `adminadmin`. +In your browser, go to `http://localhost:8080` and login with username `admin` and password `adminadmin`. TODO: Screenshots @@ -269,7 +269,7 @@ Once this is pushed / merged, ArgoCD should sync the changes and you should see === Add new Airflow DAGs -Similar to ArgoCD, after adding a new DAG to the folder `demos/argo-cd-git-ops/dags`, Airflow should pick up the new DAG via gitsync. +Similar to ArgoCD, after adding a new DAG to the folder `demos/argo-cd-git-ops/dags`, Airflow should pick up the new DAG via gitsync and display it in the UI. From 957d7e4bbbb92f5045e77f7f3a7446c4930d0cdf Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 23 Jul 2025 16:35:49 +0200 Subject: [PATCH 60/64] revert sealed secret paramterization - demo parameters not picked up in stack --- docs/modules/demos/pages/argo-cd-git-ops.adoc | 28 +++++++++++++++++-- .../applications/sealed-secrets.yaml | 7 +++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/docs/modules/demos/pages/argo-cd-git-ops.adoc b/docs/modules/demos/pages/argo-cd-git-ops.adoc index 0c97ce87..f736801b 100644 --- a/docs/modules/demos/pages/argo-cd-git-ops.adoc +++ b/docs/modules/demos/pages/argo-cd-git-ops.adoc @@ -100,7 +100,19 @@ metadata: spec: generators: - matrix: - generators: + generators:licationSet`. +** Spin up requirements like Minio and Postgres as `Application`. +** Deploy Stackable Airflow manifests in their respective `Projects`. +** DAGs are synced via Airflow gitsync, not ArgoCD. +** Airflow uses the Kubernetes executor and write its logs to S3 / Minio. +** DAGs can be started manually in the Airflow webserver UI + +TODO: arch overview image + +== ArgoCD UI + +ArgoCD will be the first product that is deployed in this Demo. Once the pods are ready, you can port-forward the argocd-server in order to access the Web UI. + - list: elements: - operator: commons @@ -108,7 +120,19 @@ spec: - operator: secret - operator: airflow - operator: druid - - operator: hbase + - operator: hbaselicationSet`. +** Spin up requirements like Minio and Postgres as `Application`. +** Deploy Stackable Airflow manifests in their respective `Projects`. +** DAGs are synced via Airflow gitsync, not ArgoCD. +** Airflow uses the Kubernetes executor and write its logs to S3 / Minio. +** DAGs can be started manually in the Airflow webserver UI + +TODO: arch overview image + +== ArgoCD UI + +ArgoCD will be the first product that is deployed in this Demo. Once the pods are ready, you can port-forward the argocd-server in order to access the Web UI. + - operator: hdfs - operator: hive - operator: kafka diff --git a/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml b/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml index 1415daf0..1522ae34 100644 --- a/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml +++ b/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml @@ -14,8 +14,11 @@ spec: releaseName: sealed-secrets-controller valuesObject: secretName: sealed-secrets-key - - repoURL: "{{ customGitUrl }}" - targetRevision: "{{ customGitBranch }}" + # TODO: This should be parameterized with customGitUrl and customGitBranch + # Currently these parameters are defined in the demo-v2.yaml and are + # not picked up in the stack - What to do? + - repoURL: https://github.com/stackabletech/demos/ + targetRevision: spike/argocd-demo path: stacks/argo-cd-git-ops/secrets/ destination: server: https://kubernetes.default.svc From c4ae6157d422427f2d09111cf109d71d6c44d6d5 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 23 Jul 2025 17:15:35 +0200 Subject: [PATCH 61/64] docs fixes --- docs/modules/demos/pages/argo-cd-git-ops.adoc | 59 ++++++------------- .../applications/sealed-secrets.yaml | 3 +- 2 files changed, 19 insertions(+), 43 deletions(-) diff --git a/docs/modules/demos/pages/argo-cd-git-ops.adoc b/docs/modules/demos/pages/argo-cd-git-ops.adoc index f736801b..9af8415b 100644 --- a/docs/modules/demos/pages/argo-cd-git-ops.adoc +++ b/docs/modules/demos/pages/argo-cd-git-ops.adoc @@ -56,6 +56,17 @@ This demo consists of multiple parts: TODO: arch overview image +== Sealed Secrets + +When managing all resources and configs via Git, deploying sensitive properties like certificates or credentials via Git becomes a problem. + +There are multiple solutions like Hashicorp or Bitwarden, which heavily depend on the infrastructure already available. + +For the sake of this demo, {sealed-secrets}[Bitnami's Sealed Secrets] are utilized. +Sensitive data is encrypted as a `SealedSecret` before commiting to the Git repository, synced via ArgoCD and decrypted by the Sealed Secrets controller into a standard Kubernetes `Secret`. + +This way, everything will be stored and managed in Git. + == ArgoCD UI ArgoCD will be the first product that is deployed in this Demo. Once the pods are ready, you can port-forward the argocd-server in order to access the Web UI. @@ -67,21 +78,10 @@ kubectl --namespace argo-cd port-forward service/argocd-server 8443:https In your browser, go to `https://localhost:8443` and login with username `admin` and password `adminadmin`. -NOTE: There will be an initial warning from the Browser that this site is insecure due to self-signed certificates. This can be ignored in this case. +NOTE: There will be an initial warning from the Browser, that this site is insecure due to self-signed certificates. This can be ignored in this case. TODO: screenshots -== Sealed Secrets - -When managing all resources and configs via Git, deploying sensitive properties like certificates or credentials via Git becomes a problem. - -There are multiple solutions like Hashicorp or Bitwarden, which heavily depend on the infrastructure already available. - -For the sake of this demo, {sealed-secrets}[Bitnami's Sealed Secrets] are utilized. -Sensitive data is encrypted as a `SealedSecret` before commiting to the Git repository, synced via ArgoCD and decrypted by the Sealed Secrets controller into a standard Kubernetes `Secret`. - -This way, everything will be stored and managed in Git. - == Stackable operators The Stackable operators are deployed via ArgoCD using the Stackable Helm charts and an ArgoCD {argo-cd-applicationset}[`ApplicationSet`]. @@ -90,8 +90,7 @@ using different versions and Git sources (repository & branch) as well as the po NOTE: This demo does not use a multi cluster environment for the sake of simplicity. -[source,console] ----- +[source,yaml] --- apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet @@ -100,19 +99,7 @@ metadata: spec: generators: - matrix: - generators:licationSet`. -** Spin up requirements like Minio and Postgres as `Application`. -** Deploy Stackable Airflow manifests in their respective `Projects`. -** DAGs are synced via Airflow gitsync, not ArgoCD. -** Airflow uses the Kubernetes executor and write its logs to S3 / Minio. -** DAGs can be started manually in the Airflow webserver UI - -TODO: arch overview image - -== ArgoCD UI - -ArgoCD will be the first product that is deployed in this Demo. Once the pods are ready, you can port-forward the argocd-server in order to access the Web UI. - + generators: - list: elements: - operator: commons @@ -120,19 +107,7 @@ ArgoCD will be the first product that is deployed in this Demo. Once the pods ar - operator: secret - operator: airflow - operator: druid - - operator: hbaselicationSet`. -** Spin up requirements like Minio and Postgres as `Application`. -** Deploy Stackable Airflow manifests in their respective `Projects`. -** DAGs are synced via Airflow gitsync, not ArgoCD. -** Airflow uses the Kubernetes executor and write its logs to S3 / Minio. -** DAGs can be started manually in the Airflow webserver UI - -TODO: arch overview image - -== ArgoCD UI - -ArgoCD will be the first product that is deployed in this Demo. Once the pods are ready, you can port-forward the argocd-server in order to access the Web UI. - + - operator: hbase - operator: hdfs - operator: hive - operator: kafka @@ -175,7 +150,7 @@ ArgoCD will be the first product that is deployed in this Demo. Once the pods ar The `matrix.generators.list[].elements[]` will create a union of parameters that may be used in the `ApplicationSet` template as follows: -[source,console] +[source,yaml] ---- # [...] template: @@ -212,7 +187,7 @@ template: The templated version for e.g. the parameters `operator=zookeeper`, `server=https://kubernetes.default.svc` and `targetRevision=25.7.0` will result in the following template: -[source,console] +[source,yaml] ---- # [...] template: diff --git a/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml b/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml index 1522ae34..b05275db 100644 --- a/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml +++ b/stacks/argo-cd-git-ops/applications/sealed-secrets.yaml @@ -14,9 +14,10 @@ spec: releaseName: sealed-secrets-controller valuesObject: secretName: sealed-secrets-key - # TODO: This should be parameterized with customGitUrl and customGitBranch + # TODO: This should be parameterized with customGitUrl and customGitBranch parameters. # Currently these parameters are defined in the demo-v2.yaml and are # not picked up in the stack - What to do? + # TODO: adapt to main / 25.7 - repoURL: https://github.com/stackabletech/demos/ targetRevision: spike/argocd-demo path: stacks/argo-cd-git-ops/secrets/ From 01ba8c1e877fdf35231a2d947c79be408b543980 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 23 Jul 2025 17:16:23 +0200 Subject: [PATCH 62/64] doc fixes 2 --- docs/modules/demos/pages/argo-cd-git-ops.adoc | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/modules/demos/pages/argo-cd-git-ops.adoc b/docs/modules/demos/pages/argo-cd-git-ops.adoc index 9af8415b..2881578e 100644 --- a/docs/modules/demos/pages/argo-cd-git-ops.adoc +++ b/docs/modules/demos/pages/argo-cd-git-ops.adoc @@ -91,6 +91,7 @@ using different versions and Git sources (repository & branch) as well as the po NOTE: This demo does not use a multi cluster environment for the sake of simplicity. [source,yaml] +---- --- apiVersion: argoproj.io/v1alpha1 kind: ApplicationSet From 4428e486044970dc8a8317ee77e0b29e021cbfc8 Mon Sep 17 00:00:00 2001 From: Malte Sander Date: Wed, 23 Jul 2025 17:17:19 +0200 Subject: [PATCH 63/64] doc fixes 3 --- docs/modules/demos/pages/argo-cd-git-ops.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/demos/pages/argo-cd-git-ops.adoc b/docs/modules/demos/pages/argo-cd-git-ops.adoc index 2881578e..1f3cff66 100644 --- a/docs/modules/demos/pages/argo-cd-git-ops.adoc +++ b/docs/modules/demos/pages/argo-cd-git-ops.adoc @@ -13,6 +13,7 @@ This demo shows how to utilize GitOps and Infrastructure as Code (IaC) with Stac All products and manifests are synced and deployed via ArgoCD (except ArgoCD itself, which is bootstrapped via `stackablectl`). The key points to show are: + * GitOps: How to deploy my changes from a Git repository to my cluster * Secrets: How to safely deploy credentials, access keys and store them in a Git repository * Although not included in this Demo, it assumes / tries to imitate a possible @@ -160,7 +161,6 @@ template: spec: project: "stackable-operators" ignoreDifferences: - # mitigating: https://github.com/stackabletech/hdfs-operator/issues/626 - group: "apiextensions.k8s.io" kind: "CustomResourceDefinition" jqPathExpressions: From fcd3bf765545a9af0180a080d5d9fea72490608d Mon Sep 17 00:00:00 2001 From: maltesander Date: Thu, 24 Jul 2025 14:52:07 +0200 Subject: [PATCH 64/64] fix overview parts --- docs/modules/demos/pages/argo-cd-git-ops.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/demos/pages/argo-cd-git-ops.adoc b/docs/modules/demos/pages/argo-cd-git-ops.adoc index 1f3cff66..dd643daf 100644 --- a/docs/modules/demos/pages/argo-cd-git-ops.adoc +++ b/docs/modules/demos/pages/argo-cd-git-ops.adoc @@ -45,9 +45,9 @@ To run this demo, your system needs at least: This demo consists of multiple parts: * Bootstrapping via `stackablectl` -** Install a {sealed-secrets}[Sealed Secrets] controller via `stackablectl` to handle sensitive data like credentials or secret keys. ** Install {argo-cd}[Argo CD] via `stackablectl`. * After the initial bootstrapping, ArgoCD takes over the deployment: +** Install a {sealed-secrets}[Sealed Secrets] controller to handle sensitive data like credentials or secret keys. ** Install all Stackable operators using an `ApplicationSet`. ** Spin up requirements like Minio and Postgres as `Application`. ** Deploy Stackable Airflow manifests in their respective `Projects`.