Airflow demo with spark-k8s (#119)

## Description

Add an airflow job executing and monitoring a spark-k8s job to the demo.

Author: adwk67

demos/airflow-scheduled-job/01-airflow-spark-clusterrole.yaml

@@ -0,0 +1,36 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: airflow-spark-clusterrole
+rules:
+- apiGroups:
+  - spark.stackable.tech
+  resources:
+  - sparkapplications
+  verbs:
+  - create
+  - get
+  - list
+- apiGroups:
+  - airflow.stackable.tech
+  resources:
+  - airflowdbs
+  verbs:
+  - create
+  - get
+  - list
+- apiGroups:
+  - apps
+  resources:
+  - statefulsets
+  verbs:
+  - get
+  - watch
+  - list
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - list

demos/airflow-scheduled-job/02-airflow-spark-clusterrolebinding.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: airflow-spark-clusterrole-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: airflow-spark-clusterrole
+subjects:
+- apiGroup: rbac.authorization.k8s.io
+  kind: Group
+  name: system:serviceaccounts

demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml

@@ -0,0 +1,26 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: start-pyspark-job
+spec:
+  template:
+    spec:
+      containers:
+        - name: start-pyspark-job
+          image: docker.stackable.tech/stackable/tools:0.2.0-stackable0.3.0
+          # N.B. it is possible for the scheduler to report that a DAG exists, only for the worker task to fail if a pod is unexpectedly
+          # restarted. Additionally, the db-init job takes a few minutes to complete before the cluster is deployed. The wait/watch steps
+          # below are not "water-tight" but add a layer of stability by at least ensuring that the db is initialized and ready and that
+          # all pods are reachable (albeit independent of each other).
+          command: ["bash", "-c", "
+          kubectl wait airflowdb/airflow --for jsonpath='{.status.condition}'=Ready --timeout 600s
+          && kubectl rollout status --watch statefulset/airflow-webserver-default
+          && kubectl rollout status --watch statefulset/airflow-worker-default
+          && kubectl rollout status --watch statefulset/airflow-scheduler-default
+          && curl -i -s --user airflow:airflow http://airflow-webserver-default:8080/api/v1/dags/sparkapp_dag
+          && curl -i -s --user airflow:airflow -H 'Content-Type:application/json' -XPATCH http://airflow-webserver-default:8080/api/v1/dags/sparkapp_dag -d '{\"is_paused\": false}'
+          && curl -i -s --user airflow:airflow -H 'Content-Type:application/json' -XPOST http://airflow-webserver-default:8080/api/v1/dags/sparkapp_dag/dagRuns -d '{}'
+          "]
+      restartPolicy: OnFailure
+  backoffLimit: 20 # give some time for the Airflow cluster to be available

demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml

@@ -0,0 +1,26 @@
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: start-date-job
+spec:
+  template:
+    spec:
+      containers:
+        - name: start-date-job
+          image: docker.stackable.tech/stackable/tools:0.2.0-stackable0.3.0
+          # N.B. it is possible for the scheduler to report that a DAG exists, only for the worker task to fail if a pod is unexpectedly
+          # restarted. Additionally, the db-init job takes a few minutes to complete before the cluster is deployed. The wait/watch steps
+          # below are not "water-tight" but add a layer of stability by at least ensuring that the db is initialized and ready and that
+          # all pods are reachable (albeit independent of each other).
+          command: ["bash", "-c", "
+          kubectl wait airflowdb/airflow --for jsonpath='{.status.condition}'=Ready --timeout 600s
+          && kubectl rollout status --watch statefulset/airflow-webserver-default
+          && kubectl rollout status --watch statefulset/airflow-worker-default
+          && kubectl rollout status --watch statefulset/airflow-scheduler-default
+          && curl -i -s --user airflow:airflow http://airflow-webserver-default:8080/api/v1/dags/date_demo
+          && curl -i -s --user airflow:airflow -H 'Content-Type:application/json' -XPATCH http://airflow-webserver-default:8080/api/v1/dags/date_demo -d '{\"is_paused\": false}'
+          && curl -i -s --user airflow:airflow -H 'Content-Type:application/json' -XPOST http://airflow-webserver-default:8080/api/v1/dags/date_demo/dagRuns -d '{}'
+          "]
+      restartPolicy: OnFailure
+  backoffLimit: 20 # give some time for the Airflow cluster to be available

demos/airflow-scheduled-job/enable-and-run-dag.yaml

@@ -7,7 +7,10 @@ demos:
       - airflow
       - job-scheduling
     manifests:
-      - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/airflow-scheduled-job/enable-and-run-dag.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/airflow-scheduled-job/01-airflow-spark-clusterrole.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/airflow-scheduled-job/02-airflow-spark-clusterrolebinding.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/stackablectl/main/demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml
   hbase-hdfs-load-cycling-data:
     description: Copy data from S3 bucket to an HBase table
     stackableStack: hdfs-hbase