Fix failing cloudnative pg backup by enabling internal backups

CHANGELOG.md

@@ -9,6 +9,7 @@ See [keep a changelog] for information about writing changes to this log.
 ## [Unreleased]
 
 * Changed nodeSelectorTerm on vllm deployments, for better upgradeability
+* Added internal backups for PostgreSQL databases below LiteLLM and OpenWeb UI
 
 ## [0.1.1] - 2026-01-14
 

applications/argo-cd-resources/templates/projects/template-controller.yaml

@@ -0,0 +1,25 @@
+apiVersion: argoproj.io/v1alpha1
+kind: AppProject
+metadata:
+  name: template-controller
+spec:
+  destinations:
+  - name: in-cluster
+    namespace: template-controller
+    server: https://kubernetes.default.svc
+  clusterResourceWhitelist:
+  - group: ''
+    kind: Namespace
+  - group: apiextensions.k8s.io
+    kind: CustomResourceDefinition
+  - group: rbac.authorization.k8s.io
+    kind: ClusterRole
+  - group: rbac.authorization.k8s.io
+    kind: ClusterRoleBinding
+  namespaceResourceWhitelist:
+  - group: '*'
+    kind: '*'
+  orphanedResources:
+    warn: false
+  sourceRepos:
+  - {{ .Values.repoUrl }} # git repository

applications/argo-cd-resources/values.yaml

@@ -33,6 +33,11 @@ apps:
     namespace: kube-system
     automated: true
 
+  - name: template-controller
+    project: template-controller
+    namespace: template-controller
+    automated: true
+
   ## Storage
   - name: cloudnative-pg-operator
     project: cloudnative-pg-operator

applications/litellm/cloudnative-pg-values.yaml

@@ -4,7 +4,7 @@ cloudnative-pg:
   # -- Override the full name of the chart
   fullnameOverride: cloudnative-pg-cluster
   # -- Override the namespace of the chart
-  namespaceOverride: "litellm"
+  namespaceOverride: ""
 
   ###
   # -- Type of the CNPG database. Available types:
@@ -372,11 +372,11 @@ cloudnative-pg:
 
   backups:
     # -- You need to configure backups manually, so backups are disabled by default.
-    enabled: false
+    enabled: true
 
     # -- Overrides the provider specific default endpoint. Defaults to:
     # S3: https://s3.<region>.amazonaws.com"
-    endpointURL: https://hel1.your-objectstorage.com  # Leave empty if using the default S3 endpoint
+    endpointURL: http://rook-ceph-rgw-ceph-objectstore.rook-ceph.svc  # Leave empty if using the default S3 endpoint
     # -- Specifies a CA bundle to validate a privately signed certificate.
     endpointCA:
       # -- Creates a secret with the given value if true, otherwise uses an existing secret.
@@ -393,9 +393,9 @@ cloudnative-pg:
     # -- One of `s3`, `azure` or `google`
     provider: s3
     s3:
-      region: hel1
-      bucket: kom1-cluster-backups
-      path: /litellm/
+      region:
+      bucket: litellm-backup-bucket
+      path: /
       accessKey: ""
       secretKey: ""
       # -- Use the role based authentication without providing explicitly the keys
@@ -418,7 +418,7 @@ cloudnative-pg:
       # -- Whether to create a secret for the backup credentials
       create: false
       # -- Name of the backup credentials secret
-      name: hetzner-s3-backup-credentials
+      name: litellm-backup-bucket-transformed
 
     wal:
       # -- WAL compression method. One of `` (for no compression), `gzip`, `bzip2` or `snappy`.

applications/litellm/templates/backup-bucket-transformer.yaml

@@ -0,0 +1,51 @@
+apiVersion: templates.kluctl.io/v1alpha1
+kind: ObjectTemplate
+metadata:
+  name: {{ .Release.Name }}-backup-bucket-transformer
+spec:
+  serviceAccountName: {{ .Release.Name }}-backup-bucket-transformer
+  prune: true
+  matrix:
+    - name: secret
+      object:
+        ref:
+          apiVersion: v1
+          kind: Secret
+          name: {{ .Release.Name }}-backup-bucket
+  templates:
+    - object:
+        apiVersion: v1
+        kind: Secret
+        metadata:
+          name: {{ .Release.Name }}-backup-bucket-transformed
+        data:
+          ACCESS_KEY_ID: {{ "'{{ matrix.secret.data.AWS_ACCESS_KEY_ID }}'" }}
+          ACCESS_SECRET_KEY: {{ "'{{ matrix.secret.data.AWS_SECRET_ACCESS_KEY }}'" }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ .Release.Name }}-backup-bucket-transformer
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames: ["{{ .Release.Name }}-backup-bucket", "{{ .Release.Name }}-backup-bucket-transformed"]
+    verbs: ["*"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ .Release.Name }}-backup-bucket-transformer
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ .Release.Name }}-backup-bucket-transformer
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Release.Name }}-backup-bucket-transformer
+    namespace: {{ .Release.Namespace }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Release.Name }}-backup-bucket-transformer

applications/litellm/templates/backup-bucket.yaml

@@ -0,0 +1,7 @@
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: {{ .Release.Name }}-backup-bucket
+spec:
+  bucketName: {{ .Release.Name }}-backup-bucket
+  storageClassName: s3-bucket-retain

applications/openwebui/cloudnative-pg-values.yaml

@@ -375,7 +375,7 @@ cloudnative-pg:
 
     # -- Overrides the provider specific default endpoint. Defaults to:
     # S3: https://s3.<region>.amazonaws.com"
-    endpointURL: https://hel1.your-objectstorage.com  # Leave empty if using the default S3 endpoint
+    endpointURL: http://rook-ceph-rgw-ceph-objectstore.rook-ceph.svc  # Leave empty if using the default S3 endpoint
     # -- Specifies a CA bundle to validate a privately signed certificate.
     endpointCA:
       # -- Creates a secret with the given value if true, otherwise uses an existing secret.
@@ -392,9 +392,9 @@ cloudnative-pg:
     # -- One of `s3`, `azure` or `google`
     provider: s3
     s3:
-      region: hel1
-      bucket: kom1-cluster-backups
-      path: /openwebui/
+      region:
+      bucket: openwebui-backup-bucket
+      path: /
       accessKey: ""
       secretKey: ""
       # -- Use the role based authentication without providing explicitly the keys
@@ -417,7 +417,7 @@ cloudnative-pg:
       # -- Whether to create a secret for the backup credentials
       create: false
       # -- Name of the backup credentials secret
-      name: hetzner-s3-backup-credentials
+      name: openwebui-backup-bucket-transformed
 
     wal:
       # -- WAL compression method. One of `` (for no compression), `gzip`, `bzip2` or `snappy`.

applications/openwebui/templates/backup-bucket-transformer.yaml

@@ -0,0 +1,51 @@
+apiVersion: templates.kluctl.io/v1alpha1
+kind: ObjectTemplate
+metadata:
+  name: {{ .Release.Name }}-backup-bucket-transformer
+spec:
+  serviceAccountName: {{ .Release.Name }}-backup-bucket-transformer
+  prune: true
+  matrix:
+    - name: secret
+      object:
+        ref:
+          apiVersion: v1
+          kind: Secret
+          name: {{ .Release.Name }}-backup-bucket
+  templates:
+    - object:
+        apiVersion: v1
+        kind: Secret
+        metadata:
+          name: {{ .Release.Name }}-backup-bucket-transformed
+        data:
+          ACCESS_KEY_ID: {{ "'{{ matrix.secret.data.AWS_ACCESS_KEY_ID }}'" }}
+          ACCESS_SECRET_KEY: {{ "'{{ matrix.secret.data.AWS_SECRET_ACCESS_KEY }}'" }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ .Release.Name }}-backup-bucket-transformer
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames: ["{{ .Release.Name }}-backup-bucket", "{{ .Release.Name }}-backup-bucket-transformed"]
+    verbs: ["*"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ .Release.Name }}-backup-bucket-transformer
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ .Release.Name }}-backup-bucket-transformer
+subjects:
+  - kind: ServiceAccount
+    name: {{ .Release.Name }}-backup-bucket-transformer
+    namespace: {{ .Release.Namespace }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Release.Name }}-backup-bucket-transformer

applications/openwebui/templates/backup-bucket.yaml

@@ -0,0 +1,7 @@
+apiVersion: objectbucket.io/v1alpha1
+kind: ObjectBucketClaim
+metadata:
+  name: {{ .Release.Name }}-backup-bucket
+spec:
+  bucketName: {{ .Release.Name }}-backup-bucket
+  storageClassName: s3-bucket-retain

applications/template-controller/Chart.yaml

@@ -0,0 +1,7 @@
+apiVersion: v2
+name: template-controller
+version: 0.0.0
+dependencies:
+  - name: template-controller
+    version: 0.9.4
+    repository: oci://ghcr.io/kluctl/charts

applications/template-controller/values.yaml

@@ -0,0 +1,78 @@
+# Default values for flux-kluctl-controller.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+image:
+  repository: ghcr.io/kluctl/template-controller
+  pullPolicy: IfNotPresent
+  # Overrides the image tag whose default is the chart appVersion.
+  tag: ""
+
+# -- If set, install and upgrade CRDs through helm chart.
+installCRDs: true
+
+crds:
+  annotations: {}
+
+imagePullSecrets: []
+nameOverride: ""
+fullnameOverride: ""
+
+serviceAccount:
+  # Specifies whether a service account should be created
+  create: true
+  # Annotations to add to the service account
+  annotations: {}
+  # The name of the service account to use.
+  # If not set and create is true, a name is generated using the fullname template
+  name: ""
+
+podAnnotations:
+  prometheus.io/scrape: "true"
+  prometheus.io/port: "8080"
+
+podSecurityContext:
+  fsGroup: 1337
+
+securityContext:
+  capabilities:
+    drop:
+      - ALL
+  readOnlyRootFilesystem: true
+  runAsNonRoot: true
+  runAsUser: 1337
+  seccompProfile:
+    type: RuntimeDefault
+
+# Additional environment variables
+env: []
+  # - name: HTTPS_PROXY
+  #   value: https://proxy.example.org
+  # - name: NO_PROXY
+  #   value: 10.96.0.1
+
+service:
+  type: ClusterIP
+  prometheus:
+    port: 8080
+  health:
+    port: 8081
+
+resources: {}
+  # We usually recommend not to specify default resources and to leave this as a conscious
+  # choice for the user. This also increases chances charts run on environments with little
+  # resources, such as Minikube. If you do want to specify resources, uncomment the following
+  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
+  # limits:
+  #   cpu: 1000m
+  #   memory: 1Gi
+  # requests:
+  #   cpu: 100m
+#   memory: 64Mi
+
+nodeSelector: {}
+
+tolerations: []
+
+affinity: {}
+