Merge branch 'main' of https://github.com/stackabletech/demos into bump/nifi-2.2.0-git-registry-client

Author: dervoeti

demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml

@@ -68,8 +68,8 @@ data:
 
     organization = "stackabletech"
     repository = "demos"
-    branch = "bump/nifi-2.2.0-git-registry-client"
-    version = "bump/nifi-2.2.0-git-registry-client"
+    branch = "release-25.3"
+    version = "release-25.3"
     directory = "demos/data-lakehouse-iceberg-trino-spark"
     flow_name = "LakehouseKafkaIngest"
 

demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml

@@ -77,8 +77,8 @@ data:
 
     organization = "stackabletech"
     repository = "demos"
-    branch = "bump/nifi-2.2.0-git-registry-client"
-    version = "bump/nifi-2.2.0-git-registry-client"
+    branch = "release-25.3"
+    version = "release-25.3"
     directory = "demos/nifi-kafka-druid-earthquake-data"
     flow_name = "IngestEarthquakesToKafka"
 

demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml

@@ -77,8 +77,8 @@ data:
 
     organization = "stackabletech"
     repository = "demos"
-    branch = "bump/nifi-2.2.0-git-registry-client"
-    version = "bump/nifi-2.2.0-git-registry-client"
+    branch = "release-25.3"
+    version = "release-25.3"
     directory = "demos/nifi-kafka-druid-water-level-data"
     flow_name = "IngestWaterLevelsToKafka"
 

demos/signal-processing/create-nifi-ingestion-job.yaml

@@ -88,8 +88,8 @@ data:
 
     organization = "stackabletech"
     repository = "demos"
-    branch = "bump/nifi-2.2.0-git-registry-client"
-    version = "bump/nifi-2.2.0-git-registry-client"
+    branch = "release-25.3"
+    version = "release-25.3"
     directory = "demos/signal-processing"
     flow_name = "DownloadAndWriteToDB"
 

demos/trino-taxi-data/load-test-data.yaml

@@ -10,13 +10,60 @@ spec:
         - name: load-ny-taxi-data
           image: "bitnami/minio:2024-debian-12"
           # yamllint disable-line rule:line-length
-          command: ["bash", "-c", "cd /tmp && for month in 2020-01 2020-02 2020-03 2020-04 2020-05 2020-06 2020-07 2020-08 2020-09 2020-10 2020-11 2020-12 2021-01 2021-02 2021-03 2021-04 2021-05 2021-06 2021-07 2021-08 2021-09 2021-10 2021-11 2021-12 2022-01 2022-02 2022-03 2022-04; do curl -O https://repo.stackable.tech/repository/misc/ny-taxi-data/yellow_tripdata_$month.parquet && mc --insecure alias set minio http://minio:9000/ $(cat /minio-s3-credentials/accessKey) $(cat /minio-s3-credentials/secretKey) && mc cp yellow_tripdata_$month.parquet minio/demo/ny-taxi-data/raw/; done"]
+          command:
+            - bash
+            - -ce
+            - |
+              # Copy the CA cert from the "tls" SecretClass
+              cp -v /etc/minio/mc/original_certs/ca.crt /.mc/certs/CAs/public.crt
+
+              MINIO_ENDPOINT="https://minio.default.svc.cluster.local:9000/"
+              MINIO_ACCESS_KEY=$(cat /minio-s3-credentials/accessKey)
+              MINIO_SECRET_KEY=$(cat /minio-s3-credentials/secretKey)
+
+              cd /tmp
+              for month in \
+              2020-01 2020-02 2020-03 2020-04 2020-05 2020-06 2020-07 2020-08 2020-09 2020-10 \
+              2020-11 2020-12 2021-01 2021-02 2021-03 2021-04 2021-05 2021-06 2021-07 2021-08 \
+              2021-09 2021-10 2021-11 2021-12 2022-01 2022-02 2022-03 2022-04; do
+                curl -O "https://repo.stackable.tech/repository/misc/ny-taxi-data/yellow_tripdata_$month.parquet"
+                mc alias set minio "$MINIO_ENDPOINT" "$MINIO_ACCESS_KEY" "$MINIO_SECRET_KEY"
+                mc cp "yellow_tripdata_$month.parquet" minio/demo/ny-taxi-data/raw/
+              done
           volumeMounts:
             - name: minio-s3-credentials
               mountPath: /minio-s3-credentials
+            # Mount the certificate generated by the secret-operator
+            - name: tls
+              mountPath: /etc/minio/mc/original_certs
+            # On startup, we will rename the certs and move them here:
+            - mountPath: /.mc/certs/CAs
+              name: certs
+
       volumes:
         - name: minio-s3-credentials
           secret:
             secretName: minio-s3-credentials
+        # Request a TLS certificate from the secret-operator
+        - name: tls
+          ephemeral:
+            volumeClaimTemplate:
+              metadata:
+                annotations:
+                  secrets.stackable.tech/class: tls
+                  secrets.stackable.tech/scope: |-
+                    service=minio
+              spec:
+                storageClassName: secrets.stackable.tech
+                accessModes:
+                  - ReadWriteOnce
+                resources:
+                  requests:
+                    storage: 1
+        # Create an in-memory emptyDir to copy the certs to (to avoid permission errors)
+        - name: certs
+          emptyDir:
+            sizeLimit: 5Mi
+            medium: Memory
       restartPolicy: OnFailure
   backoffLimit: 50

stacks/_templates/minio-tls/rendered-chart.yaml

@@ -135,7 +135,7 @@ data:
     }
     
     # Try connecting to MinIO instance
-    scheme=http
+    scheme=https
     connectToMinio $scheme
     
     
@@ -223,7 +223,7 @@ data:
     }
     
     # Try connecting to MinIO instance
-    scheme=http
+    scheme=https
     connectToMinio $scheme
     
     
@@ -291,7 +291,7 @@ data:
     }
     
     # Try connecting to MinIO instance
-    scheme=http
+    scheme=https
     connectToMinio $scheme
     
     
@@ -372,7 +372,7 @@ data:
     }
     
     # Try connecting to MinIO instance
-    scheme=http
+    scheme=https
     connectToMinio $scheme
     
     
@@ -418,7 +418,7 @@ data:
     }
     
     # Try connecting to MinIO instance
-    scheme=http
+    scheme=https
     connectToMinio $scheme
 ---
 # Source: minio/templates/pvc.yaml
@@ -452,7 +452,7 @@ spec:
   type: NodePort
   externalTrafficPolicy: "Cluster"
   ports:
-    - name: http
+    - name: https
       port: 9001
       protocol: TCP
       targetPort: 9001
@@ -475,7 +475,7 @@ spec:
   type: NodePort
   externalTrafficPolicy: "Cluster"
   ports:
-    - name: http
+    - name: https
       port: 9000
       protocol: TCP
       targetPort: 9000
@@ -514,7 +514,7 @@ spec:
         stackable.tech/vendor: Stackable
       annotations:
         checksum/secrets: fa63e34a92c817c84057e2d452fa683e66462a57b0529388fb96a57e05f38e57
-        checksum/config: 2b1e6b6d0485236a84032ab7e9eeee4a7bac29d2b63d3b0260bde76e84626730
+        checksum/config: ebea49cc4c1bfbd1b156a58bf770a776ff87fe199f642d31c2816b5515112e72
     spec:
       securityContext:
 
@@ -549,9 +549,9 @@ spec:
             - mountPath: /etc/minio/certs
               name: certs
           ports:
-            - name: http
+            - name: https
               containerPort: 9000
-            - name: http-console
+            - name: https-console
               containerPort: 9001
           env:
             - name: MINIO_ROOT_USER
@@ -579,7 +579,7 @@ spec:
         - name: minio-user
           secret:
             secretName: minio        
-          
+
         - ephemeral:
             volumeClaimTemplate:
               metadata:
@@ -633,12 +633,38 @@ spec:
                   name: minio
               - secret:
                   name: minio
+        - ephemeral:
+            volumeClaimTemplate:
+              metadata:
+                annotations:
+                  secrets.stackable.tech/class: tls
+                  secrets.stackable.tech/scope: service=minio
+              spec:
+                accessModes:
+                - ReadWriteOnce
+                resources:
+                  requests:
+                    storage: 1
+                storageClassName: secrets.stackable.tech
+          name: tls
+        - emptyDir:
+            medium: Memory
+            sizeLimit: 5Mi
+          name: certs
       serviceAccountName: minio-sa
       containers:
         - name: minio-make-bucket
           image: "quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z"
           imagePullPolicy: IfNotPresent
-          command: [ "/bin/sh", "/config/initialize" ]
+          command:
+            - "/bin/sh"
+            - "-ce"
+            - |
+              # Copy the CA cert from the "tls" SecretClass
+              # mkdir -p /etc/minio/mc/certs/CAs
+              cp -v /etc/minio/mc/original_certs/ca.crt /etc/minio/mc/certs/CAs/public.crt
+
+              . /config/initialize
           env:
             - name: MINIO_ENDPOINT
               value: minio
@@ -651,13 +677,25 @@ spec:
               mountPath: /tmp
             - name: minio-configuration
               mountPath: /config
+            - name: tls
+              mountPath: /etc/minio/mc/original_certs
+            - name: certs
+              mountPath: /etc/minio/mc/certs/CAs
           resources:
             requests:
               memory: 128Mi
         - name: minio-make-user
           image: "quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z"
           imagePullPolicy: IfNotPresent
-          command: [ "/bin/sh", "/config/add-user" ]
+          command:
+            - "/bin/sh"
+            - "-ce"
+            - |
+              # Copy the CA cert from the "tls" SecretClass
+              # mkdir -p /etc/minio/mc/certs/CAs
+              cp -v /etc/minio/mc/original_certs/ca.crt /etc/minio/mc/certs/CAs/public.crt
+
+              . /config/add-user
           env:
             - name: MINIO_ENDPOINT
               value: minio
@@ -670,6 +708,10 @@ spec:
               mountPath: /tmp
             - name: minio-configuration
               mountPath: /config
+            - name: tls
+              mountPath: /etc/minio/mc/original_certs
+            - name: certs
+              mountPath: /etc/minio/mc/certs/CAs
           resources:
             requests:
               memory: 128Mi

stacks/_templates/minio-tls/values.yaml

@@ -20,6 +20,8 @@ service:
 consoleService:
   type: NodePort
   nodePort: null
+tls:
+  enabled: true
 extraVolumes:
   # Request a TLS certificate from the secret-operator
   - name: tls
@@ -49,3 +51,35 @@ extraVolumeMounts:
   # On startup, we will rename the certs and move them here:
   - mountPath: /etc/minio/certs
     name: certs
+
+customCommandJob:
+  extraVolumes:
+    # Request a TLS certificate from the secret-operator
+    - name: tls
+      ephemeral:
+        volumeClaimTemplate:
+          metadata:
+            annotations:
+              secrets.stackable.tech/class: tls
+              secrets.stackable.tech/scope: |-
+                service=minio
+          spec:
+            storageClassName: secrets.stackable.tech
+            accessModes:
+              - ReadWriteOnce
+            resources:
+              requests:
+                storage: 1
+    # Create an in-memory emptyDir to copy the certs to (to avoid permission errors)
+    - name: certs
+      emptyDir:
+        sizeLimit: 5Mi
+        medium: Memory
+  # WARNING: this is currently only used by the custom-scripts job container. Other containers do not mount these.
+  extraVolumeMounts:
+    # Mount the certificate generated by the secret-operator
+    - name: tls
+      mountPath: /etc/minio/mc/original_certs
+    # On startup, we will rename the certs and move them here:
+    - mountPath: /etc/minio/mc/certs/CAs
+      name: certs