From ee78446df15d7ed89e450ac3bcf35c7088024b6e Mon Sep 17 00:00:00 2001 From: dervoeti Date: Wed, 19 Mar 2025 14:47:38 +0100 Subject: [PATCH 1/3] fix: Use Minio with TLS for data-lakehouse demo --- .pre-commit-config.yaml | 2 +- .../load-test-data.yaml | 32 +- .../minio-distributed-tls/README.md | 16 + .../minio-distributed-tls/rendered-chart.yaml | 714 ++++++++++++++++++ .../minio-distributed-tls/values.yaml | 87 +++ .../s3-connection.yaml | 7 +- stacks/stacks-v2.yaml | 4 +- 7 files changed, 857 insertions(+), 5 deletions(-) create mode 100644 stacks/_templates/minio-distributed-tls/README.md create mode 100644 stacks/_templates/minio-distributed-tls/rendered-chart.yaml create mode 100644 stacks/_templates/minio-distributed-tls/values.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f9a2a2bd..572a91e4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ default_language_version: node: system -exclude: '^stacks/_templates/minio-tls/rendered-chart\.yaml$' +exclude: '^stacks/_templates/minio(-distributed)?-tls/rendered-chart\.yaml$' repos: - repo: https://github.com/pre-commit/pre-commit-hooks diff --git a/demos/data-lakehouse-iceberg-trino-spark/load-test-data.yaml b/demos/data-lakehouse-iceberg-trino-spark/load-test-data.yaml index b5fefaaa..0821b6ef 100644 --- a/demos/data-lakehouse-iceberg-trino-spark/load-test-data.yaml +++ b/demos/data-lakehouse-iceberg-trino-spark/load-test-data.yaml @@ -16,7 +16,10 @@ spec: - pipefail - -c - | - mc --insecure alias set minio http://minio:9000/ $(cat /minio-s3-credentials/accessKey) $(cat /minio-s3-credentials/secretKey) + # Copy the CA cert from the "tls" SecretClass + cp -v /etc/minio/mc/original_certs/ca.crt /.mc/certs/CAs/public.crt + + mc alias set minio https://minio.default.svc.cluster.local:9000/ $(cat /minio-s3-credentials/accessKey) $(cat /minio-s3-credentials/secretKey) cd /tmp curl -sO https://repo.stackable.tech/repository/misc/datasets/open-postcode-geo/open-postcode-geo.csv @@ -69,9 +72,36 @@ spec: volumeMounts: - name: minio-s3-credentials mountPath: /minio-s3-credentials + # Mount the certificate generated by the secret-operator + - name: tls + mountPath: /etc/minio/mc/original_certs + # On startup, we will rename the certs and move them here: + - mountPath: /.mc/certs/CAs + name: certs volumes: - name: minio-s3-credentials secret: secretName: minio-s3-credentials + # Request a TLS certificate from the secret-operator + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: |- + service=minio,pod + spec: + storageClassName: secrets.stackable.tech + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + # Create an in-memory emptyDir to copy the certs to (to avoid permission errors) + - name: certs + emptyDir: + sizeLimit: 5Mi + medium: Memory restartPolicy: OnFailure backoffLimit: 50 diff --git a/stacks/_templates/minio-distributed-tls/README.md b/stacks/_templates/minio-distributed-tls/README.md new file mode 100644 index 00000000..a007da87 --- /dev/null +++ b/stacks/_templates/minio-distributed-tls/README.md @@ -0,0 +1,16 @@ +# MinIO with TLS from secret-operator + +MinIO has a severe limitation whereby the TLS certificates must be named `public.crt` +and `private.key`. This goes against Kubernetes naming of `tls.crt` and `tls.key`. + +The upstream minio chart is also too limited: + +- No way to add initContainers (to rename cert files in a shared volume). +- No way to edit the container command (to rename cert files before starting minio). + +Therefore, we will render the upstream chart here, and then apply the necessary +customizations on top. + +```yaml +helm template minio minio/minio -f values.yaml > rendered-chart.yaml +``` diff --git a/stacks/_templates/minio-distributed-tls/rendered-chart.yaml b/stacks/_templates/minio-distributed-tls/rendered-chart.yaml new file mode 100644 index 00000000..81108ba7 --- /dev/null +++ b/stacks/_templates/minio-distributed-tls/rendered-chart.yaml @@ -0,0 +1,714 @@ +--- +# Source: minio/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: "minio-sa" +--- +# Source: minio/templates/secrets.yaml +apiVersion: v1 +kind: Secret +metadata: + name: minio + labels: + app: minio + chart: minio-5.4.0 + release: minio + heritage: Helm +type: Opaque +data: + rootUser: "YWRtaW4=" + rootPassword: "YWRtaW5hZG1pbg==" +--- +# Source: minio/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: minio + labels: + app: minio + chart: minio-5.4.0 + release: minio + heritage: Helm +data: + initialize: |- + #!/bin/sh + set -e # Have script exit in the event of a failed command. + MC_CONFIG_DIR="/etc/minio/mc/" + MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" + + # connectToMinio + # Use a check-sleep-check loop to wait for MinIO service to be available + connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 + LIMIT=29 # Allow 30 attempts + set -e # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) + SECRET=$(cat /config/rootPassword) + set +e # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" + $MC_COMMAND + STATUS=$? + until [ $STATUS = 0 ]; do + ATTEMPTS=$(expr $ATTEMPTS + 1) + echo \"Failed attempts: $ATTEMPTS\" + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 + fi + sleep 2 # 1 second intervals between attempts + $MC_COMMAND + STATUS=$? + done + set -e # reset `e` as active + return 0 + } + + # checkBucketExists ($bucket) + # Check if the bucket exists, by using the exit code of `mc ls` + checkBucketExists() { + BUCKET=$1 + CMD=$(${MC} stat myminio/$BUCKET >/dev/null 2>&1) + return $? + } + + # createBucket ($bucket, $policy, $purge) + # Ensure bucket exists, purging if asked to + createBucket() { + BUCKET=$1 + POLICY=$2 + PURGE=$3 + VERSIONING=$4 + OBJECTLOCKING=$5 + + # Purge the bucket, if set & exists + # Since PURGE is user input, check explicitly for `true` + if [ $PURGE = true ]; then + if checkBucketExists $BUCKET; then + echo "Purging bucket '$BUCKET'." + set +e # don't exit if this fails + ${MC} rm -r --force myminio/$BUCKET + set -e # reset `e` as active + else + echo "Bucket '$BUCKET' does not exist, skipping purge." + fi + fi + + # Create the bucket if it does not exist and set objectlocking if enabled (NOTE: versioning will be not changed if OBJECTLOCKING is set because it enables versioning to the Buckets created) + if ! checkBucketExists $BUCKET; then + if [ ! -z $OBJECTLOCKING ]; then + if [ $OBJECTLOCKING = true ]; then + echo "Creating bucket with OBJECTLOCKING '$BUCKET'" + ${MC} mb --with-lock myminio/$BUCKET + elif [ $OBJECTLOCKING = false ]; then + echo "Creating bucket '$BUCKET'" + ${MC} mb myminio/$BUCKET + fi + elif [ -z $OBJECTLOCKING ]; then + echo "Creating bucket '$BUCKET'" + ${MC} mb myminio/$BUCKET + else + echo "Bucket '$BUCKET' already exists." + fi + fi + + # set versioning for bucket if objectlocking is disabled or not set + if [ $OBJECTLOCKING = false ]; then + if [ ! -z $VERSIONING ]; then + if [ $VERSIONING = true ]; then + echo "Enabling versioning for '$BUCKET'" + ${MC} version enable myminio/$BUCKET + elif [ $VERSIONING = false ]; then + echo "Suspending versioning for '$BUCKET'" + ${MC} version suspend myminio/$BUCKET + fi + fi + else + echo "Bucket '$BUCKET' versioning unchanged." + fi + + # At this point, the bucket should exist, skip checking for existence + # Set policy on the bucket + echo "Setting policy of bucket '$BUCKET' to '$POLICY'." + ${MC} anonymous set $POLICY myminio/$BUCKET + } + + # Try connecting to MinIO instance + scheme=https + connectToMinio $scheme + + + + # Create the buckets + createBucket staging "public" false false false + createBucket lakehouse "public" false false false + + add-user: |- + #!/bin/sh + set -e ; # Have script exit in the event of a failed command. + MC_CONFIG_DIR="/etc/minio/mc/" + MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" + + # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. + # Special characters for example : ',",<,>,{,} + MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_tmp" + + # connectToMinio + # Use a check-sleep-check loop to wait for MinIO service to be available + connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 1 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 + } + + # checkUserExists () + # Check if the user exists, by using the exit code of `mc admin user info` + checkUserExists() { + CMD=$(${MC} admin user info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) + return $? + } + + # createUser ($policy) + createUser() { + POLICY=$1 + #check accessKey_and_secretKey_tmp file + if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then + echo "credentials file does not exist" + return 1 + fi + if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then + echo "credentials file is invalid" + rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP + return 1 + fi + USER=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) + # Create the user if it does not exist + if ! checkUserExists ; then + echo "Creating user '$USER'" + cat $MINIO_ACCESSKEY_SECRETKEY_TMP | ${MC} admin user add myminio + else + echo "User '$USER' already exists." + fi + #clean up credentials files. + rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP + + # set policy for user + if [ ! -z $POLICY -a $POLICY != " " ] ; then + echo "Adding policy '$POLICY' for '$USER'" + set +e ; # policy already attach errors out, allow it. + ${MC} admin policy attach myminio $POLICY --user=$USER + set -e + else + echo "User '$USER' has no policy attached." + fi + } + + # Try connecting to MinIO instance + scheme=https + connectToMinio $scheme + + + + # Create the users + echo console > $MINIO_ACCESSKEY_SECRETKEY_TMP + echo console123 >> $MINIO_ACCESSKEY_SECRETKEY_TMP + createUser consoleAdmin + + add-policy: |- + #!/bin/sh + set -e ; # Have script exit in the event of a failed command. + MC_CONFIG_DIR="/etc/minio/mc/" + MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" + + # connectToMinio + # Use a check-sleep-check loop to wait for MinIO service to be available + connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 1 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 + } + + # checkPolicyExists ($policy) + # Check if the policy exists, by using the exit code of `mc admin policy info` + checkPolicyExists() { + POLICY=$1 + CMD=$(${MC} admin policy info myminio $POLICY > /dev/null 2>&1) + return $? + } + + # createPolicy($name, $filename) + createPolicy () { + NAME=$1 + FILENAME=$2 + + # Create the name if it does not exist + echo "Checking policy: $NAME (in /config/$FILENAME.json)" + if ! checkPolicyExists $NAME ; then + echo "Creating policy '$NAME'" + else + echo "Policy '$NAME' already exists." + fi + ${MC} admin policy create myminio $NAME /config/$FILENAME.json + + } + + # Try connecting to MinIO instance + scheme=https + connectToMinio $scheme + + + + add-svcacct: |- + #!/bin/sh + set -e ; # Have script exit in the event of a failed command. + MC_CONFIG_DIR="/etc/minio/mc/" + MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" + + # AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. + # Special characters for example : ',",<,>,{,} + MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_svcacct_tmp" + + # connectToMinio + # Use a check-sleep-check loop to wait for MinIO service to be available + connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 2 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 + } + + # checkSvcacctExists () + # Check if the svcacct exists, by using the exit code of `mc admin user svcacct info` + checkSvcacctExists() { + CMD=$(${MC} admin user svcacct info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) + return $? + } + + # createSvcacct ($user) + createSvcacct () { + USER=$1 + FILENAME=$2 + #check accessKey_and_secretKey_tmp file + if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then + echo "credentials file does not exist" + return 1 + fi + if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then + echo "credentials file is invalid" + rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP + return 1 + fi + SVCACCT=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) + # Create the svcacct if it does not exist + if ! checkSvcacctExists ; then + echo "Creating svcacct '$SVCACCT'" + # Check if policy file is define + if [ -z $FILENAME ]; then + ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) myminio $USER + else + ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --policy /config/$FILENAME.json myminio $USER + fi + else + echo "Svcacct '$SVCACCT' already exists." + fi + #clean up credentials files. + rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP + } + + # Try connecting to MinIO instance + scheme=https + connectToMinio $scheme + + + + custom-command: |- + #!/bin/sh + set -e ; # Have script exit in the event of a failed command. + MC_CONFIG_DIR="/etc/minio/mc/" + MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" + + # connectToMinio + # Use a check-sleep-check loop to wait for MinIO service to be available + connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 1 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 + } + + # runCommand ($@) + # Run custom mc command + runCommand() { + ${MC} "$@" + return $? + } + + # Try connecting to MinIO instance + scheme=https + connectToMinio $scheme +--- +# Source: minio/templates/console-service.yaml +apiVersion: v1 +kind: Service +metadata: + name: minio-console + labels: + app: minio + chart: minio-5.4.0 + release: minio + heritage: Helm +spec: + type: NodePort + externalTrafficPolicy: "Cluster" + ports: + - name: https + port: 9001 + protocol: TCP + targetPort: 9001 + selector: + app: minio + release: minio +--- +# Source: minio/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: minio + labels: + app: minio + chart: minio-5.4.0 + release: minio + heritage: Helm + monitoring: "true" +spec: + type: NodePort + externalTrafficPolicy: "Cluster" + ports: + - name: https + port: 9000 + protocol: TCP + targetPort: 9000 + selector: + app: minio + release: minio +--- +# Source: minio/templates/statefulset.yaml +apiVersion: v1 +kind: Service +metadata: + name: minio-svc + labels: + app: minio + chart: minio-5.4.0 + release: minio + heritage: Helm +spec: + publishNotReadyAddresses: true + clusterIP: None + ports: + - name: https + port: 9000 + protocol: TCP + targetPort: 9000 + selector: + app: minio + release: minio +--- +# Source: minio/templates/statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: minio + labels: + app: minio + chart: minio-5.4.0 + release: minio + heritage: Helm + stackable.tech/vendor: Stackable +spec: + updateStrategy: + type: RollingUpdate + podManagementPolicy: "Parallel" + serviceName: minio-svc + replicas: 5 + selector: + matchLabels: + app: minio + release: minio + template: + metadata: + name: minio + labels: + app: minio + release: minio + stackable.tech/vendor: Stackable + annotations: + checksum/secrets: fa63e34a92c817c84057e2d452fa683e66462a57b0529388fb96a57e05f38e57 + checksum/config: 66f252598ba8542f924dd76a5da7f64ae6a943b95798d752c65d32214320cbb4 + spec: + securityContext: + fsGroup: 1000 + fsGroupChangePolicy: OnRootMismatch + runAsGroup: 1000 + runAsUser: 1000 + serviceAccountName: minio-sa + containers: + - name: minio + image: quay.io/minio/minio:RELEASE.2024-12-18T13-15-44Z + imagePullPolicy: IfNotPresent + command: + - "/bin/sh" + - "-ce" + - | + # minio requires the TLS key pair to be specially named + mkdir -p /etc/minio/certs/CAs + cp -v /etc/minio/original_certs/tls.crt /etc/minio/certs/public.crt + cp -v /etc/minio/original_certs/tls.key /etc/minio/certs/private.key + cp -v /etc/minio/original_certs/ca.crt /etc/minio/certs/CAs/ca.crt + + /usr/bin/docker-entrypoint.sh minio server https://minio-{0...4}.minio-svc.default.svc.cluster.local/export -S /etc/minio/certs/ --address :9000 --console-address :9001 + volumeMounts: + - name: export + mountPath: /export + - mountPath: /etc/minio/original_certs + name: tls + - mountPath: /etc/minio/certs + name: certs + ports: + - name: https + containerPort: 9000 + - name: https-console + containerPort: 9001 + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: minio + key: rootUser + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: minio + key: rootPassword + - name: MINIO_PROMETHEUS_AUTH_TYPE + value: "public" + resources: + requests: + cpu: 500m + memory: 2Gi + securityContext: + readOnlyRootFilesystem: false + volumes: + - name: minio-user + secret: + secretName: minio + - ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: service=minio,pod + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech + name: tls + - emptyDir: + medium: Memory + sizeLimit: 5Mi + name: certs + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: export + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 250Gi +--- +# Source: minio/templates/post-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: minio-post-job + labels: + app: minio-post-job + chart: minio-5.4.0 + release: minio + heritage: Helm + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation +spec: + template: + metadata: + labels: + app: minio-job + release: minio + stackable.tech/vendor: Stackable + spec: + restartPolicy: OnFailure + volumes: + - name: etc-path + emptyDir: {} + - name: tmp + emptyDir: {} + - name: minio-configuration + projected: + sources: + - configMap: + name: minio + - secret: + name: minio + - ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: service=minio,pod + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + storageClassName: secrets.stackable.tech + name: tls + - emptyDir: + medium: Memory + sizeLimit: 5Mi + name: certs + serviceAccountName: minio-sa + containers: + - name: minio-make-bucket + image: "quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z" + imagePullPolicy: IfNotPresent + command: + - "/bin/sh" + - "-ce" + - | + # Copy the CA cert from the "tls" SecretClass + cp -v /etc/minio/mc/original_certs/ca.crt /etc/minio/mc/certs/CAs/public.crt + . /config/initialize + env: + - name: MINIO_ENDPOINT + value: minio + - name: MINIO_PORT + value: "9000" + volumeMounts: + - name: etc-path + mountPath: /etc/minio/mc + - name: tmp + mountPath: /tmp + - name: minio-configuration + mountPath: /config + - name: tls + mountPath: /etc/minio/mc/original_certs + - name: certs + mountPath: /etc/minio/mc/certs/CAs + resources: + requests: + memory: 128Mi + - name: minio-make-user + image: "quay.io/minio/mc:RELEASE.2024-11-21T17-21-54Z" + imagePullPolicy: IfNotPresent + command: [ "/bin/sh", "/config/add-user" ] + env: + - name: MINIO_ENDPOINT + value: minio + - name: MINIO_PORT + value: "9000" + volumeMounts: + - name: etc-path + mountPath: /etc/minio/mc + - name: tmp + mountPath: /tmp + - name: minio-configuration + mountPath: /config + - name: tls + mountPath: /etc/minio/mc/original_certs + - name: certs + mountPath: /etc/minio/mc/certs/CAs + resources: + requests: + memory: 128Mi diff --git a/stacks/_templates/minio-distributed-tls/values.yaml b/stacks/_templates/minio-distributed-tls/values.yaml new file mode 100644 index 00000000..35e3ce1d --- /dev/null +++ b/stacks/_templates/minio-distributed-tls/values.yaml @@ -0,0 +1,87 @@ +additionalLabels: + stackable.tech/vendor: Stackable +podLabels: + stackable.tech/vendor: Stackable +rootUser: admin +rootPassword: adminadmin +mode: distributed +replicas: 5 +persistence: + size: 250Gi +buckets: + - name: staging + policy: public + - name: lakehouse + policy: public +resources: + requests: + cpu: 500m + memory: 2Gi +service: + type: NodePort + nodePort: null +consoleService: + type: NodePort + nodePort: null +tls: + enabled: true +extraVolumes: + # Request a TLS certificate from the secret-operator + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: |- + service=minio,pod + spec: + storageClassName: secrets.stackable.tech + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1 + # Create an in-memory emptyDir to copy the certs to (to avoid permission errors) + - name: certs + emptyDir: + sizeLimit: 5Mi + medium: Memory +extraVolumeMounts: + # Mount the certificate generated by the secret-operator + - name: tls + mountPath: /etc/minio/original_certs + # On startup, we will rename the certs and move them here: + - mountPath: /etc/minio/certs + name: certs +customCommandJob: + extraVolumes: + # Request a TLS certificate from the secret-operator + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + secrets.stackable.tech/scope: |- + service=minio,pod + spec: + storageClassName: secrets.stackable.tech + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1 + # Create an in-memory emptyDir to copy the certs to (to avoid permission errors) + - name: certs + emptyDir: + sizeLimit: 5Mi + medium: Memory + # WARNING: this is currently only used by the custom-scripts job container. Other containers do not mount these. + extraVolumeMounts: + # Mount the certificate generated by the secret-operator + - name: tls + mountPath: /etc/minio/mc/original_certs + # On startup, we will rename the certs and move them here: + - mountPath: /etc/minio/mc/certs/CAs + name: certs \ No newline at end of file diff --git a/stacks/data-lakehouse-iceberg-trino-spark/s3-connection.yaml b/stacks/data-lakehouse-iceberg-trino-spark/s3-connection.yaml index 7e026539..411d7a4d 100644 --- a/stacks/data-lakehouse-iceberg-trino-spark/s3-connection.yaml +++ b/stacks/data-lakehouse-iceberg-trino-spark/s3-connection.yaml @@ -4,11 +4,16 @@ kind: S3Connection metadata: name: minio spec: - host: minio + host: minio.default.svc.cluster.local port: 9000 accessStyle: Path credentials: secretClass: minio-s3-credentials + tls: + verification: + server: + caCert: + secretClass: tls --- apiVersion: secrets.stackable.tech/v1alpha1 kind: SecretClass diff --git a/stacks/stacks-v2.yaml b/stacks/stacks-v2.yaml index b4bf0069..8594690c 100644 --- a/stacks/stacks-v2.yaml +++ b/stacks/stacks-v2.yaml @@ -135,7 +135,7 @@ stacks: - minio - s3 manifests: - - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/minio-distributed.yaml + - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/minio-distributed-tls/rendered-chart.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-hive.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-hive-iceberg.yaml - helmChart: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/_templates/postgresql-superset.yaml @@ -146,7 +146,7 @@ stacks: - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/data-lakehouse-iceberg-trino-spark/kafka.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/data-lakehouse-iceberg-trino-spark/nifi.yaml - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/stacks/nifi-kafka-druid-superset-s3/superset.yaml # Reuse - supportedNamespaces: [] + supportedNamespaces: ["default"] # until namespace can be templates, the minio FQDN hard-codes the namespace resourceRequests: cpu: "71" memory: 160Gi From 8eabd5eba89e68e4ec46328eb0e47828e7c455da Mon Sep 17 00:00:00 2001 From: dervoeti Date: Wed, 19 Mar 2025 21:15:30 +0100 Subject: [PATCH 2/3] fix: trino resources / hive downgrade / volume name --- .../create-spark-ingestion-job.yaml | 6 +++--- .../minio-distributed-tls/rendered-chart.yaml | 10 +++++----- stacks/_templates/minio-distributed-tls/values.yaml | 6 +++--- .../hive-metastores.yaml | 4 ++-- stacks/data-lakehouse-iceberg-trino-spark/trino.yaml | 4 ++-- 5 files changed, 15 insertions(+), 15 deletions(-) diff --git a/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml b/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml index 6a7c3028..50e1d820 100644 --- a/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml +++ b/demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml @@ -67,7 +67,7 @@ data: - name: script configMap: name: write-iceberg-table-script - - name: tls + - name: spark-tls ephemeral: volumeClaimTemplate: metadata: @@ -101,7 +101,7 @@ data: volumeMounts: - name: script mountPath: /stackable/spark/jobs - - name: tls + - name: spark-tls mountPath: /stackable/tls executor: replicas: 4 @@ -115,7 +115,7 @@ data: volumeMounts: - name: script mountPath: /stackable/spark/jobs - - name: tls + - name: spark-tls mountPath: /stackable/tls --- apiVersion: v1 diff --git a/stacks/_templates/minio-distributed-tls/rendered-chart.yaml b/stacks/_templates/minio-distributed-tls/rendered-chart.yaml index 81108ba7..4a5b2cfc 100644 --- a/stacks/_templates/minio-distributed-tls/rendered-chart.yaml +++ b/stacks/_templates/minio-distributed-tls/rendered-chart.yaml @@ -546,7 +546,7 @@ spec: - name: export mountPath: /export - mountPath: /etc/minio/original_certs - name: tls + name: minio-tls - mountPath: /etc/minio/certs name: certs ports: @@ -590,7 +590,7 @@ spec: requests: storage: "1" storageClassName: secrets.stackable.tech - name: tls + name: minio-tls - emptyDir: medium: Memory sizeLimit: 5Mi @@ -653,7 +653,7 @@ spec: requests: storage: "1" storageClassName: secrets.stackable.tech - name: tls + name: minio-tls - emptyDir: medium: Memory sizeLimit: 5Mi @@ -682,7 +682,7 @@ spec: mountPath: /tmp - name: minio-configuration mountPath: /config - - name: tls + - name: minio-tls mountPath: /etc/minio/mc/original_certs - name: certs mountPath: /etc/minio/mc/certs/CAs @@ -705,7 +705,7 @@ spec: mountPath: /tmp - name: minio-configuration mountPath: /config - - name: tls + - name: minio-tls mountPath: /etc/minio/mc/original_certs - name: certs mountPath: /etc/minio/mc/certs/CAs diff --git a/stacks/_templates/minio-distributed-tls/values.yaml b/stacks/_templates/minio-distributed-tls/values.yaml index 35e3ce1d..3fc2a1df 100644 --- a/stacks/_templates/minio-distributed-tls/values.yaml +++ b/stacks/_templates/minio-distributed-tls/values.yaml @@ -49,7 +49,7 @@ extraVolumes: medium: Memory extraVolumeMounts: # Mount the certificate generated by the secret-operator - - name: tls + - name: minio-tls mountPath: /etc/minio/original_certs # On startup, we will rename the certs and move them here: - mountPath: /etc/minio/certs @@ -57,7 +57,7 @@ extraVolumeMounts: customCommandJob: extraVolumes: # Request a TLS certificate from the secret-operator - - name: tls + - name: minio-tls ephemeral: volumeClaimTemplate: metadata: @@ -80,7 +80,7 @@ customCommandJob: # WARNING: this is currently only used by the custom-scripts job container. Other containers do not mount these. extraVolumeMounts: # Mount the certificate generated by the secret-operator - - name: tls + - name: minio-tls mountPath: /etc/minio/mc/original_certs # On startup, we will rename the certs and move them here: - mountPath: /etc/minio/mc/certs/CAs diff --git a/stacks/data-lakehouse-iceberg-trino-spark/hive-metastores.yaml b/stacks/data-lakehouse-iceberg-trino-spark/hive-metastores.yaml index becb2662..2c485fa1 100644 --- a/stacks/data-lakehouse-iceberg-trino-spark/hive-metastores.yaml +++ b/stacks/data-lakehouse-iceberg-trino-spark/hive-metastores.yaml @@ -5,7 +5,7 @@ metadata: name: hive spec: image: - productVersion: 4.0.1 + productVersion: 4.0.0 clusterConfig: database: connString: jdbc:postgresql://postgresql-hive:5432/hive @@ -24,7 +24,7 @@ metadata: name: hive-iceberg spec: image: - productVersion: 4.0.1 + productVersion: 4.0.0 clusterConfig: database: connString: jdbc:postgresql://postgresql-hive-iceberg:5432/hive diff --git a/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml b/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml index 6f102420..b5328b35 100644 --- a/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml +++ b/stacks/data-lakehouse-iceberg-trino-spark/trino.yaml @@ -31,13 +31,13 @@ spec: replicas: 1 workers: config: - queryMaxMemoryPerNode: 8GB + queryMaxMemoryPerNode: 10GB resources: cpu: min: "2" max: "6" memory: - limit: '16Gi' + limit: '20Gi' roleGroups: default: replicas: 4 From 56aa9b8a68c89eae6363549e176632d1e6e7c7b7 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Thu, 20 Mar 2025 11:38:57 +0100 Subject: [PATCH 3/3] chore: newline at end of file --- stacks/_templates/minio-distributed-tls/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stacks/_templates/minio-distributed-tls/values.yaml b/stacks/_templates/minio-distributed-tls/values.yaml index 3fc2a1df..95da9110 100644 --- a/stacks/_templates/minio-distributed-tls/values.yaml +++ b/stacks/_templates/minio-distributed-tls/values.yaml @@ -84,4 +84,4 @@ customCommandJob: mountPath: /etc/minio/mc/original_certs # On startup, we will rename the certs and move them here: - mountPath: /etc/minio/mc/certs/CAs - name: certs \ No newline at end of file + name: certs