Merge pull request #36 from lenaxia/majorversion300

[v3.1.0] Major Version - PVC Rendering Fix

charts/local-ai/templates/_helpers.tpl

@@ -23,6 +23,7 @@ If release name contains chart name it will be used as a full name.
 {{- end }}
 {{- end }}
 
+
 {{/*
 Create chart name and version as used by the chart label.
 */}}
@@ -42,3 +43,9 @@ app.kubernetes.io/managed-by: {{ .Release.Service }}
 app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 {{- end }}
 {{- end }}
+
+# Add defaults for global.labels and global.annotations
+{{- define "local-ai.annotations" -}}
+  {}
+{{- end -}}
+

charts/local-ai/templates/_pvc.yaml

@@ -0,0 +1,31 @@
+{{- define "local-ai.pvc" -}}
+{{- $rootContext := .rootContext -}}
+{{- $pvcObject := .object -}}
+
+{{- $labels := merge
+  ($pvcObject.labels | default dict)
+  (include "local-ai.labels" $rootContext | fromYaml)
+-}}
+
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: {{ $pvcObject.name }}
+  {{- with $labels }}
+  labels: {{- toYaml . | nindent 4 -}}
+  {{- end }}
+  {{- with $pvcObject.annotations }}
+  annotations: {{- toYaml . | nindent 4 -}}
+  {{- end }}
+spec:
+  accessModes:
+    - {{ required "accessMode is required for PVC" $pvcObject.accessMode | quote }}
+  resources:
+    requests:
+      storage: {{ required "size is required for PVC" $pvcObject.size | quote }}
+  {{- if $pvcObject.storageClass }}
+  storageClassName: {{ if (eq "-" $pvcObject.storageClass) }}""{{- else }}{{ $pvcObject.storageClass | quote }}{{- end }}
+  {{- end }}
+{{- end -}}
+

charts/local-ai/templates/deployment.yaml

@@ -1,4 +1,6 @@
+# yamllint disable rule:line-length
 {{- $urls := "" -}}
+{{- $rootPersistence := .Values.persistence }}
 {{- range $idx, $model := .Values.models.list }}
 {{- $urls = printf "%s%s %s," $urls $model.url ($model.basicAuth | default "") }}
 {{- end }}
@@ -27,11 +29,52 @@ spec:
         checksum/config-prompt-templates: {{ include (print $.Template.BasePath "/configmap-prompt-templates.yaml") . | sha256sum }}
         {{- end }}
     spec:
-    {{- with .Values.deployment.imagePullSecrets }}
+      {{- with .Values.deployment.imagePullSecrets }}
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
-    {{- end }}
+      {{- end }}
       initContainers:
+        # Additional initContainers from values.yaml
+        {{- if .Values.initContainers }}
+        {{- range .Values.initContainers }}
+        - name: {{ .name }}
+          image: {{ .image }}
+          imagePullPolicy: {{ .imagePullPolicy }}
+          {{- if .command }}
+          command:
+            {{- toYaml .command | nindent 12 }}
+          {{- end }}
+          {{- if .args }}
+          args:
+            {{- toYaml .args | nindent 6 }}
+          {{- end }}
+          {{- if .env }}
+          env:
+            {{- toYaml .env | nindent 6 }}
+          {{- end }}
+          {{- if .resources }}
+          resources:
+            {{- toYaml .resources | nindent 6 }}
+          {{- end }}
+          {{- if or .volumeMounts $rootPersistence }}
+          volumeMounts:
+          {{- if .volumeMounts }}
+            {{- toYaml .volumeMounts | nindent 6 }}
+          {{- end }}
+          {{- range $key, $pvc := $rootPersistence }}
+          {{- if $pvc.enabled }}
+            - name: {{ $key }}
+              mountPath: {{ $pvc.globalMount | default (print "/" $key) }}
+          {{- end }}
+          {{- end }}
+          {{- end }}
+          {{- if .securityContext }}
+          securityContext:
+            {{- toYaml .securityContext | nindent 6 }}
+          {{- end }}
+        {{- end }}
+        {{- end }}
+
         {{- if .Values.promptTemplates }}
         - name: prompt-templates
           image: {{ .Values.deployment.prompt_templates.image }}
@@ -43,8 +86,12 @@ spec:
           volumeMounts:
             - mountPath: /prompt-templates
               name: prompt-templates
-            - mountPath: /models
-              name: models
+            {{- range $key, $pvc := $rootPersistence }}
+              {{- if $pvc.enabled }}
+            - name: {{ $key }}
+              mountPath: {{ $pvc.globalMount | default (print "/" $key) }}
+              {{- end }}
+            {{- end }}
         {{- end }}
         - name: download-model
           image: {{ .Values.deployment.download_model.image }}
@@ -55,74 +102,163 @@ spec:
               MODEL_DIR={{ .Values.deployment.modelsPath }}
               FORCE_DOWNLOAD={{ .Values.models.forceDownload }}
               URLS="{{ $urls }}"
+              LOCK_DIR=/tmp/model-download-locks
 
               mkdir -p "$MODEL_DIR"
+              mkdir -p "$LOCK_DIR"
+              mkdir -p "/tmp/generated/images"
+              mkdir -p "/tmp/generated/audio"
+              rm -rf "/models/lost+found"
+
+              validate_url() {
+                  local url=$1
+                  local regex='^(https?|ftp)://[a-zA-Z0-9.-]+(:[a-zA-Z0-9.-]+)?(/[a-zA-Z0-9.-]*)*$'
+                  if [[ $url =~ $regex ]]; then
+                      return 0 # URL is valid
+                  else
+                      return 1 # URL is invalid
+                  fi
+              }
+
+              echo "List of URLs:"
+              echo "$URLS"
 
-              # Split urls on commas
               echo "$URLS" | awk -F, '{for (i=1; i<=NF; i++) print $i}' | while read -r line; do
                   url=$(echo "$line" | awk '{print $1}')
                   auth=$(echo "$line" | awk '{print $2}')
+                  full_filename=$(basename "$url" .bin)
+                  short_filename=$(echo "$full_filename" | cut -c1-20)
+                  hash=$(echo "$full_filename" | sha256sum | cut -c1-12)
+                  filename="${short_filename}_${hash}"
+                  lockfile="$LOCK_DIR/$filename.lock"
 
-                  if [ -n "$url" ]; then
-                      filename=$(basename "$url" .bin)
+                  # Validate URL
+                  if ! validate_url "$url"; then
+                      echo "Invalid URL: $url. Skipping download."
+                      continue
+                  fi
 
-                      if [ "$FORCE_DOWNLOAD" = false ] && [ -f "$MODEL_DIR/$filename" ]; then
-                          echo "File $filename already exists. Skipping download."
-                          continue
-                      fi
+                  if [ -e "$MODEL_DIR/$filename" ]; then
+                      echo "File $filename already exists. Skipping download."
+                      continue
+                  fi
 
-                      rm -f "$MODEL_DIR/$filename"
+                  if [ -e "$lockfile" ]; then
+                      echo "Another pod is downloading $filename. Waiting for download to complete."
+                      while [ -e "$lockfile" ]; do sleep 1; done
+                      continue
+                  fi
 
-                      echo "Downloading $filename"
+                  touch "$lockfile"
 
-                      if [ -n "$auth" ]; then
-                          wget --header "Authorization: Basic $auth" "$url" -O "$MODEL_DIR/$filename"
-                      else
-                          wget "$url" -O "$MODEL_DIR/$filename"
-                      fi
+                  echo "Downloading $filename"
+                  if [ -n "$auth" ]; then
+                      wget --header "Authorization: Basic $auth" "$url" -O "$MODEL_DIR/$filename"
+                  else
+                      wget "$url" -O "$MODEL_DIR/$filename"
+                  fi
 
-                      if [ "$?" -ne 0 ]; then
-                          echo "Download failed."
-                      else
-                          echo "Download completed."
-                      fi
+                  if [ "$?" -ne 0 ]; then
+                      echo "Download failed."
+                      rm -f "$lockfile"
+                      exit 1
+                  else
+                      echo "Download completed."
+                      rm -f "$lockfile"
                   fi
               done
+
           volumeMounts:
-          - mountPath: {{ .Values.deployment.modelsPath }}
-            name: models
+            {{- range $key, $pvc := $rootPersistence }}
+            {{- if $pvc.enabled }}
+            - name: {{ $key }}
+              mountPath: {{ $pvc.globalMount | default (print "/" $key) }}
+            {{- end }}
+            {{- end }}
+
       containers:
+        # Sidecar containers from values.yaml
+        {{- range .Values.sidecarContainers }}
+        - name: {{ .name }}
+          image: {{ .image }}
+          imagePullPolicy: {{ .imagePullPolicy }}
+          {{- if .command }}
+          command:
+            {{- toYaml .command | nindent 12 }}
+          {{- end }}
+          {{- if .args }}
+          args:
+            {{- toYaml .args | nindent 6 }}
+          {{- end }}
+          {{- if .env }}
+          env:
+            {{- toYaml .env | nindent 6 }}
+          {{- end }}
+          {{- if .ports }}
+          ports:
+            {{- toYaml .ports | nindent 6 }}
+          {{- end }}
+          {{- if .resources }}
+          resources:
+            {{- toYaml .resources | nindent 6 }}
+          {{- end }}
+          {{- if or .volumeMounts $rootPersistence }}
+          volumeMounts:
+          {{- if .volumeMounts }}
+            {{- toYaml .volumeMounts | nindent 6 }}
+          {{- end }}
+          {{- range $key, $pvc := $rootPersistence }}
+          {{- if $pvc.enabled }}
+            - name: {{ $key }}
+              mountPath: {{ $pvc.globalMount | default (print "/" $key) }}
+          {{- end }}
+          {{- end }}
+          {{- end }}
+          {{- if .livenessProbe }}
+          livenessProbe:
+            {{- toYaml .livenessProbe | nindent 6 }}
+          {{- end }}
+          {{- if .readinessProbe }}
+          readinessProbe:
+            {{- toYaml .readinessProbe | nindent 6 }}
+          {{- end }}
+          {{- if .securityContext }}
+          securityContext:
+            {{- toYaml .securityContext | nindent 6 }}
+          {{- end }}
+        {{- end }}
         - name: {{ template "local-ai.fullname" . }}
-          image: {{ .Values.deployment.image }}
+          image: "{{ .Values.deployment.image.repository }}:{{ .Values.deployment.image.tag }}"
           imagePullPolicy: {{ .Values.deployment.pullPolicy }}
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
           env:
-          {{- range $key, $value := .Values.deployment.env }}
-          - name: {{ $key | upper }}
-            value: {{ quote $value }}
-          {{- end }}
-          - name: MODELS_PATH
-            value: {{ .Values.deployment.modelsPath }}
+            {{- range $key, $value := .Values.deployment.env }}
+            - name: {{ $key | upper }}
+              value: {{ quote $value }}
+            {{- end }}
+            - name: MODELS_PATH
+              value: {{ .Values.deployment.modelsPath }}
           volumeMounts:
-          - mountPath: {{ .Values.deployment.modelsPath }}
-            name: models
+            {{- range $key, $pvc := $rootPersistence}}
+              {{- if $pvc.enabled }}
+            - name: {{ $key }}
+              mountPath: {{ $pvc.globalMount | default (print "/" $key) }}
+              {{- end }}
+            {{- end }}
+      {{- if $rootPersistence}}
       volumes:
-      {{- if .Values.models.persistence.pvc.enabled }}
-      - name: models
-        persistentVolumeClaim:
-          claimName: {{ template "local-ai.fullname" . }}
-      {{- else if .Values.models.persistence.hostPath.enabled }}
-      - name: models
-        hostPath:
-          path: {{ .Values.models.persistence.hostPath.path }}
-      {{- else }}
-      - name: models
-        emptyDir: {}
+        {{- range $key, $pvc := $rootPersistence}}
+          {{- if $pvc.enabled }}
+        - name: {{ $key }}
+          persistentVolumeClaim:
+            claimName: {{ printf "%s-%s" (include "local-ai.fullname" $) $key }}
+          {{- end }}
+        {{- end }}
+        - name: prompt-templates
+          configMap:
+            name: {{ template "local-ai.fullname" . }}-prompt-templates
       {{- end }}
-      - name: prompt-templates
-        configMap:
-          name: {{ template "local-ai.fullname" . }}-prompt-templates
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}

charts/local-ai/templates/pvcs.yaml

@@ -0,0 +1,9 @@
+{{- range $key, $pvc := .Values.persistence -}}
+  {{- if $pvc.enabled -}}
+    {{- $pvcObject := dict "name" (printf "%s-%s" (include "local-ai.fullname" $) $key) "accessMode" $pvc.accessModes "size" $pvc.size "storageClass" $pvc.storageClass "annotations" $pvc.annotations "labels" $pvc.labels -}}
+
+    {{- /* Include the PVC template */ -}}
+    {{- include "local-ai.pvc" (dict "rootContext" $ "object" $pvcObject) | nindent 0 -}}
+  {{- end -}}
+{{- end -}}
+