Merge pull request #2472 from GiganticMinecraft/add-proxmox-backup-so…

…lution

デバッグサーバにproxmox-backup-clientを使用した定期バックアップの仕組みを構築する

seichi-onp-k8s/manifests/seichi-kubernetes/apps/root/projects.yaml

@@ -141,6 +141,18 @@ spec:
   clusterResourceWhitelist:
     - group: "*"
       kind: "*"
+  syncWindows:
+    # 朝4時〜実行しているバックアップの際に kubectl patch で replicas に干渉しているので
+    # selfHeal の実行される時間を制限しておく (デバッグでの実行も想定し長めに指定)
+    - kind: allow
+      schedule: "00 7 * * *"  # 7:00 から
+      duration: 1h           # 1時間 selfHeal を有効化
+      applications: ["*"]
+      manualSync: false
+    - kind: deny
+      schedule: "00 8 * * *"  # 毎日 8:00 から
+      duration: 23h            # 2時間 selfHeal を無効化
+      applications: ["*"]
 ---
 apiVersion: argoproj.io/v1alpha1
 kind: AppProject

seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-minecraft/mcserver--debug-s1/argo-workflows-backup.yaml

@@ -0,0 +1,129 @@
+apiVersion: argoproj.io/v1alpha1
+kind: CronWorkflow
+metadata:
+  name: argo-workflows-backup
+  namespace: seichi-debug-minecraft
+spec:
+  schedule: "0 4 * * *"
+  timezone: Asia/Tokyo
+  concurrencyPolicy: "Forbid"
+  workflowSpec:
+    ttlStrategy:
+      secondsAfterCompletion: 86400
+    serviceAccountName: mcserver--debug-s1-workflow-sa
+    entrypoint: stop-backup-and-boot
+    templates:
+      - name: stop-backup-and-boot
+        steps:
+          - - name: patch-statefulset-to-0
+              template: patch-statefulset-to-0
+          - - name: wait-for-scale-to-0
+              template: wait-for-scale-to-0
+          - - name: run-backup
+              template: run-backup
+          - - name: patch-statefulset-to-1
+              template: patch-statefulset-to-1
+          - - name: wait-for-scale-to-1
+              template: wait-for-scale-to-1
+
+      - name: patch-statefulset-to-0
+        script:
+          image: bitnami/kubectl:1.32.1
+          command: ["/bin/sh", "-c"]
+          source: |
+            kubectl patch statefulset mcserver--debug-s1 -n seichi-debug-minecraft --type='merge' -p '{"spec": {"replicas": 0}}'
+
+      - name: wait-for-scale-to-0
+        script:
+          image: bitnami/kubectl:1.32.1
+          command: ["/bin/sh", "-c"]
+          source: |
+            echo "Waiting for StatefulSet to scale down..."
+            while [ "$(kubectl get statefulset mcserver--debug-s1 -n seichi-debug-minecraft -o jsonpath='{.status.availableReplicas}')" != "0" ]; do
+              echo "Still scaling down..."
+              sleep 5
+            done
+            echo "Scale-down confirmed!"
+
+      - name: run-backup
+        script:
+          image: debian:12
+          command: ["/bin/sh", "-c"]
+          source: |
+            set -e
+            echo "Updating package lists..."
+            apt update
+            echo "Installing curl..."
+            apt install -y curl
+            echo "Adding Proxmox Backup Client repository..."
+            echo "deb http://download.proxmox.com/debian/pbs-client bookworm main" > /etc/apt/sources.list.d/pbs-client.list
+            curl -fsSL https://enterprise.proxmox.com/debian/proxmox-release-bookworm.gpg -o /etc/apt/trusted.gpg.d/proxmox-release-bookworm.gpg
+            echo "Updating package lists..."
+            apt update
+            echo "Installing proxmox-backup-client version 3.3.2-1..."
+            apt install -y proxmox-backup-client=3.3.2-1
+            echo "Running backup..."
+            proxmox-backup-client backup "${BACKUP_NAME}" \
+              --repository "${PBS_USER}@${PBS_HOST}:${PBS_DATASTORE}" \
+              --backup-id "${BACKUP_ID}"
+          env:
+            - name: BACKUP_NAME
+              value: "data.pxar:/data"
+            # proxmox-backup-server側でどのサーバのバックアップかを識別するためにサーバごとに異なるbackup-idを指定する
+            - name: BACKUP_ID
+              value: "mcserver--debug-s1"
+            - name: PBS_USER
+              valueFrom:
+                secretKeyRef:
+                  name: pbs-credentials
+                  key: user
+            - name: PBS_HOST
+              valueFrom:
+                secretKeyRef:
+                  name: pbs-credentials
+                  key: host
+            - name: PBS_DATASTORE
+              valueFrom:
+                secretKeyRef:
+                  name: pbs-credentials
+                  key: datastore
+            - name: PBS_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: pbs-credentials
+                  key: password
+            - name: PBS_FINGERPRINT
+              valueFrom:
+                secretKeyRef:
+                  name: pbs-credentials
+                  key: fingerprint
+          volumeMounts:
+            - name: backup-target-volume
+              mountPath: /data
+
+      # FIXME ここにMariaDBのメンテ完了を待つ処理を入れる必要がある
+
+      - name: patch-statefulset-to-1
+        script:
+          image: bitnami/kubectl:1.32.1
+          command: ["/bin/sh", "-c"]
+          source: |
+            kubectl patch statefulset mcserver--debug-s1 -n seichi-debug-minecraft --type='merge' -p '{"spec": {"replicas": 1}}'
+
+      - name: wait-for-scale-to-1
+        activeDeadlineSeconds: 300 # 5分待っても上がってこない場合は諦める
+        script:
+          image: bitnami/kubectl:1.32.1
+          command: ["/bin/sh", "-c"]
+          source: |
+            echo "Waiting for StatefulSet to scale up..."
+            while [ "$(kubectl get statefulset mcserver--debug-s1 -n seichi-debug-minecraft -o jsonpath='{.status.availableReplicas}')" != "1" ]; do
+              echo "Still scaling up..."
+              sleep 5
+            done
+            echo "Scale-up confirmed!"
+
+    volumes:
+      - name: backup-target-volume
+        persistentVolumeClaim:
+          claimName: minecraft-server-data-mcserver--debug-s1-0

seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-minecraft/mcserver--debug-s1/stateful-set.yaml

@@ -417,16 +417,12 @@ spec:
         # mod-downloaderからプラグインをinitContainerでダウンロードしてMinecraftに受け渡すためのvolume
         - name: mod-downloader-volume
           emptyDir: {}
-
-        # ワールドデータ周りが保管されると SeichiAssist によるマイグレーションで詰まるので一旦コメントアウト
-        # # サーバーデータが格納されているディレクトリはNAS上の特定のiSCSIパスで公開されたLUNドライブを直接マウントする
-        # # このボリュームに保存されたデータのライフサイクルに関する管理(バックアップリストアほか)はkubernetes上で管理されずNASに一任される
-        # - name: minecraft-server-data
-        #   iscsi:
-        #     targetPortal: 192.168.16.240
-        #     iqn: iqn.2000-01.com.synology:seichi-cloud.k8s-static-pv--minecraft-server-data-mcserver--debug-s1-0
-        #     lun: 1
-        #     readOnly: false
-        #     fsType: ext4
-        #     chapAuthDiscovery: false
-        #     chapAuthSession: false
+  volumeClaimTemplates:
+    - metadata:
+        name: minecraft-server-data
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: 50Gi

seichi-onp-k8s/manifests/seichi-kubernetes/apps/seichi-debug-minecraft/mcserver--debug-s1/workflow-sa.yaml

@@ -0,0 +1,35 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: mcserver--debug-s1-workflow-sa
+  namespace: seichi-debug-minecraft
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: mcserver--debug-s1-workflow-role
+  namespace: seichi-debug-minecraft
+rules:
+  - apiGroups: ["apps"]
+    resources: ["statefulsets"]
+    verbs: ["get", "patch"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get"]
+  - apiGroups: ["argoproj.io"]
+    resources: ["workflowtaskresults"]
+    verbs: ["create", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: mcserver--debug-s1-workflow-binding
+  namespace: seichi-debug-minecraft
+subjects:
+  - kind: ServiceAccount
+    name: mcserver--debug-s1-workflow-sa
+    namespace: seichi-debug-minecraft
+roleRef:
+  kind: Role
+  name: mcserver--debug-s1-workflow-role
+  apiGroup: rbac.authorization.k8s.io

terraform/main.tf

@@ -341,3 +341,37 @@ variable "argo_events_github_access_token" {
 }
 
 # endregion
+
+# region env variables for proxmox-backup-client
+
+variable "proxmox_backup_client__user" {
+  description = "proxmox-backup-client user name"
+  type        = string
+  sensitive   = true
+}
+
+variable "proxmox_backup_client__host" {
+  description = "proxmox-backup-client host"
+  type        = string
+  sensitive   = true
+}
+
+variable "proxmox_backup_client__datastore" {
+  description = "proxmox-backup-client datastore"
+  type        = string
+  sensitive   = true
+}
+
+variable "proxmox_backup_client__password" {
+  description = "proxmox-backup-client password"
+  type        = string
+  sensitive   = true
+}
+
+variable "proxmox_backup_client__fingerprint" {
+  description = "proxmox-backup-client fingerprint at target server"
+  type        = string
+  sensitive   = true
+}
+
+# endregion

terraform/onp_cluster_minecraft_secrets.tf

@@ -159,3 +159,22 @@ resource "helm_release" "onp_minecraft_debug_minio_secrets" {
   }
 
 }
+
+resource "kubernetes_secret" "onp_minecraft_debug_pbs_credentials" {
+  depends_on = [kubernetes_namespace.onp_seichi_debug_minecraft]
+
+  metadata {
+    name      = "pbs-credentials"
+    namespace = "seichi-debug-minecraft"
+  }
+
+  data = {
+    user        = var.proxmox_backup_client__user
+    host        = var.proxmox_backup_client__host
+    datastore   = var.proxmox_backup_client__datastore
+    password    = var.proxmox_backup_client__password
+    fingerprint = var.proxmox_backup_client__fingerprint
+  }
+
+  type = "Opaque"
+}