feat: Create a backup mechanism using Buildah image

A backup of workspace is done using Buildah and storing a content of the
workspace PVC into a container image. The image is later stored in a
registry and can be used to recover data.

A prototype script was updated and stored under project-backup
directory and is build alongside the controller.

The backup job calls the script and execute following steps:
- mount a volume with workspace data
- build container image using buildah
- push image to registry configured by the operator admin

Signed-off-by: Ales Raszka <[email protected]>

Author: Allda

.github/workflows/next-build.yml

@@ -73,6 +73,17 @@ jobs:
           quay.io/devfile/project-clone:sha-${{ steps.git-sha.outputs.sha }}
         file: ./project-clone/Dockerfile
 
+    - name: Build and push
+      uses: docker/build-push-action@0a97817b6ade9f46837855d676c4cca3a2471fc9 #v4.2.1
+      with:
+        context: ./project-backup
+        push: true
+        platforms: linux/amd64, linux/arm64, linux/ppc64le, linux/s390x
+        tags: |
+          quay.io/devfile/project-backup:next
+          quay.io/devfile/project-backup:sha-${{ steps.git-sha.outputs.sha }}
+        file: ./project-backup/Dockerfile
+
   build-next-olm-imgs:
     runs-on: ubuntu-latest
     needs: build-next-imgs
@@ -147,6 +158,7 @@ jobs:
           export TAG="sha-${{ needs.build-next-imgs.outputs.git-sha }}"
           export DEFAULT_DWO_IMG="quay.io/devfile/devworkspace-controller:$TAG"
           export PROJECT_CLONE_IMG="quay.io/devfile/project-clone:$TAG"
+          export PROJECT_BACKUP_IMG="quay.io/devfile/project-backup:$TAG"
 
           # Next builds are not rolled out unless the version is incremented. We want to use semver
           # prerelease tags to make sure each new build increments on the previous one, e.g.

.github/workflows/pr.yml

@@ -130,3 +130,6 @@ jobs:
     -
       name: Check if project-clone dockerimage build is working
       run: docker build -f ./project-clone/Dockerfile .
+    -
+      name: Check if project-backup containerimage build is working
+      run: docker build -f ./project-backup/Containerfile project-backup/

Makefile

@@ -26,6 +26,7 @@ export DWO_IMG ?= quay.io/devfile/devworkspace-controller:next
 export DWO_BUNDLE_IMG ?= quay.io/devfile/devworkspace-operator-bundle:next
 export DWO_INDEX_IMG ?= quay.io/devfile/devworkspace-operator-index:next
 export PROJECT_CLONE_IMG ?= quay.io/devfile/project-clone:next
+export PROJECT_BACKUP_IMG ?= quay.io/devfile/project-clone:next
 export PULL_POLICY ?= Always
 export DEFAULT_ROUTING ?= basic
 export KUBECONFIG ?= ${HOME}/.kube/config
@@ -128,6 +129,7 @@ _print_vars:
 	@echo "    DWO_BUNDLE_IMG=$(DWO_BUNDLE_IMG)"
 	@echo "    DWO_INDEX_IMG=$(DWO_INDEX_IMG)"
 	@echo "    PROJECT_CLONE_IMG=$(PROJECT_CLONE_IMG)"
+	@echo "    PROJECT_BACKUP_IMG=$(PROJECT_BACKUP_IMG)"
 	@echo "    PULL_POLICY=$(PULL_POLICY)"
 	@echo "    ROUTING_SUFFIX=$(ROUTING_SUFFIX)"
 	@echo "    DEFAULT_ROUTING=$(DEFAULT_ROUTING)"
@@ -369,6 +371,7 @@ help: Makefile
 	@echo 'Supported environment variables:'
 	@echo '    DWO_IMG                    - Image used for controller'
 	@echo '    PROJECT_CLONE_IMG          - Image used for project-clone init container'
+	@echo '    PROJECT_BACKUP_IMG         - Image used for project-backup workspace backup container'
 	@echo '    NAMESPACE                  - Namespace to use for deploying controller'
 	@echo '    KUBECONFIG                 - Kubeconfig which should be used for accessing to the cluster. Currently is: $(KUBECONFIG)'
 	@echo '    ROUTING_SUFFIX             - Cluster routing suffix (e.g. $$(minikube ip).nip.io, apps-crc.testing)'

apis/controller/v1alpha1/devworkspaceoperatorconfig_types.go

@@ -82,6 +82,10 @@ type BackupCronJobConfig struct {
 	// +kubebuilder:default:="0 2 * * *"
 	// +kubebuilder:validation:Optional
 	Schedule string `json:"schedule,omitempty"`
+	// A registry where backup images are stored. Images are stored
+	// in {registry}/backup-${DEVWORKSPACE_NAMESPACE}-${DEVWORKSPACE_NAME}
+	// +kubebuilder:validation:Optional
+	Registry string `json:"registry,omitempty"`
 }
 
 type RoutingConfig struct {

build/scripts/generate_deployment.sh

@@ -31,7 +31,7 @@
 set -e
 
 # List of environment variables that will be replaced by envsubst
-SUBST_VARS='$NAMESPACE $DWO_IMG $PROJECT_CLONE_IMG $ROUTING_SUFFIX $DEFAULT_ROUTING $PULL_POLICY'
+SUBST_VARS='$NAMESPACE $DWO_IMG $PROJECT_CLONE_IMG $PROJECT_BACKUP_IMG $ROUTING_SUFFIX $DEFAULT_ROUTING $PULL_POLICY'
 
 SCRIPT_DIR=$(cd "$(dirname "$0")"; pwd)
 DEPLOY_DIR="$SCRIPT_DIR/../../deploy/"
@@ -58,6 +58,11 @@ Arguments:
       '--use-defaults' is passed; otherwise, the value of the PROJECT_CLONE_IMG
       environment variable is used. If unspecifed, the default value of
       'quay.io/devfile/project-clone:next' is used.
+  --project-backup-image
+      Image to use for the project backup workspace. Used only when
+      '--use-defaults' is passed; otherwise, the value of the PROJECT_BACKUP_IMG
+      environment variable is used. If unspecifed, the default value of
+      'quay.io/devfile/project-backup:next' is used.
   --split-yaml
       Parse output file combined.yaml into a yaml file for each record
       in combined yaml. Files are output to the 'objects' subdirectory
@@ -96,6 +101,10 @@ while [[ "$#" -gt 0 ]]; do
       PROJECT_CLONE_IMG=$2
       shift
       ;;
+      --project-backup-image)
+      PROJECT_BACKUP_IMG=$2
+      shift
+      ;;
       --split-yamls)
       SPLIT_YAMLS=true
       ;;
@@ -118,6 +127,7 @@ if $USE_DEFAULT_ENV; then
   export NAMESPACE=devworkspace-controller
   export DWO_IMG=${DEFAULT_DWO_IMG:-"quay.io/devfile/devworkspace-controller:next"}
   export PROJECT_CLONE_IMG=${PROJECT_CLONE_IMG:-"quay.io/devfile/project-clone:next"}
+  export PROJECT_BACKUP_IMG=${PROJECT_BACKUP_IMG:-"quay.io/devfile/project-backup:next"}
   export PULL_POLICY=Always
   export DEFAULT_ROUTING=basic
   export DEVWORKSPACE_API_VERSION=a6ec0a38307b63a29fad2eea945cc69bee97a683

controllers/backupcronjob/backupcronjob_controller.go

@@ -22,6 +22,7 @@ import (
 
 	dw "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
 	controllerv1alpha1 "github.com/devfile/devworkspace-operator/apis/controller/v1alpha1"
+	"github.com/devfile/devworkspace-operator/internal/images"
 	"github.com/devfile/devworkspace-operator/pkg/conditions"
 	"github.com/devfile/devworkspace-operator/pkg/config"
 	"github.com/devfile/devworkspace-operator/pkg/constants"
@@ -137,11 +138,10 @@ func (r *BackupCronJobReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Complete(r)
 }
 
-// +kubebuilder:rbac:groups=batch,resources=cronjobs,verbs=get;list;create;update;patch;delete
-// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list
-// +kubebuilder:rbac:groups=controller.devfile.io,resources=devworkspaceoperatorconfigs,verbs=get;list;watch
-// +kubebuilder:rbac:groups=workspace.devfile.io,resources=devworkspaces,verbs=get;list
 // +kubebuilder:rbac:groups="",resources=persistentvolumeclaims,verbs=get;list
+// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;create;update;patch;delete
+// +kubebuilder:rbac:groups=controller.devfile.io,resources=devworkspaceoperatorconfigs,verbs=get;list;update;patch;watch
+// +kubebuilder:rbac:groups=workspace.devfile.io,resources=devworkspaces,verbs=get;list
 
 // Reconcile is the main reconciliation loop for the BackupCronJob controller.
 func (r *BackupCronJobReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
@@ -180,7 +180,7 @@ func (r *BackupCronJobReconciler) isBackupEnabled(config *controllerv1alpha1.Dev
 }
 
 // startCron starts the cron scheduler with the backup job according to the provided configuration.
-func (r *BackupCronJobReconciler) startCron(ctx context.Context, backUpConfig *controllerv1alpha1.BackupCronJobConfig, logger logr.Logger) {
+func (r *BackupCronJobReconciler) startCron(ctx context.Context, req ctrl.Request, backUpConfig *controllerv1alpha1.BackupCronJobConfig, logger logr.Logger) {
 	log := logger.WithName("backup cron")
 	log.Info("Starting backup cron scheduler")
 
@@ -198,7 +198,7 @@ func (r *BackupCronJobReconciler) startCron(ctx context.Context, backUpConfig *c
 		taskLog := logger.WithName("cronTask")
 
 		taskLog.Info("Starting DevWorkspace backup job")
-		if err := r.executeBackupSync(ctx, logger); err != nil {
+		if err := r.executeBackupSync(ctx, req, backUpConfig, logger); err != nil {
 			taskLog.Error(err, "Failed to execute backup job for DevWorkspaces")
 		}
 		taskLog.Info("DevWorkspace backup job finished")
@@ -231,7 +231,7 @@ func (r *BackupCronJobReconciler) stopCron(logger logr.Logger) {
 
 // executeBackupSync executes the backup job for all DevWorkspaces in the cluster that
 // have been stopped in the last N minutes.
-func (r *BackupCronJobReconciler) executeBackupSync(ctx context.Context, logger logr.Logger) error {
+func (r *BackupCronJobReconciler) executeBackupSync(ctx context.Context, req ctrl.Request, backUpConfig *controllerv1alpha1.BackupCronJobConfig, logger logr.Logger) error {
 	log := logger.WithName("executeBackupSync")
 	log.Info("Executing backup sync for all DevWorkspaces")
 	devWorkspaces := &dw.DevWorkspaceList{}
@@ -248,7 +248,7 @@ func (r *BackupCronJobReconciler) executeBackupSync(ctx context.Context, logger
 		dwID := dw.Status.DevWorkspaceId
 		log.Info("Found DevWorkspace", "namespace", dw.Namespace, "devworkspace", dw.Name, "id", dwID)
 
-		if err := r.createBackupJob(&dw, ctx, logger); err != nil {
+		if err := r.createBackupJob(&dw, ctx, req, backUpConfig, logger); err != nil {
 			log.Error(err, "Failed to create backup Job for DevWorkspace", "id", dwID)
 			continue
 		}
@@ -285,8 +285,12 @@ func (r *BackupCronJobReconciler) wasStoppedInTimeRange(workspace *dw.DevWorkspa
 	return false
 }
 
+func ptrInt64(i int64) *int64 { return &i }
+func ptrInt32(i int32) *int32 { return &i }
+func ptrBool(b bool) *bool    { return &b }
+
 // createBackupJob creates a Kubernetes Job to back up the workspace's PVC data.
-func (r *BackupCronJobReconciler) createBackupJob(workspace *dw.DevWorkspace, ctx context.Context, logger logr.Logger) error {
+func (r *BackupCronJobReconciler) createBackupJob(workspace *dw.DevWorkspace, ctx context.Context, req ctrl.Request, backUpConfig *controllerv1alpha1.BackupCronJobConfig, logger logr.Logger) error {
 	log := logger.WithName("createBackupJob")
 	dwID := workspace.Status.DevWorkspaceId
 
@@ -307,38 +311,47 @@ func (r *BackupCronJobReconciler) createBackupJob(workspace *dw.DevWorkspace, ct
 			Template: corev1.PodTemplateSpec{
 				Spec: corev1.PodSpec{
 					RestartPolicy: corev1.RestartPolicyNever,
+					SecurityContext: &corev1.PodSecurityContext{
+						FSGroup: ptrInt64(0),
+					},
 					Containers: []corev1.Container{
 						{
-							Name:    "backup",
-							Image:   defaultBackupImage,
-							Command: []string{"/bin/sh", "-c"},
+							Name: "backup-workspace",
 							Env: []corev1.EnvVar{
-								{
-									Name:  "DEVWORKSPACE_NAME",
-									Value: workspace.Name,
-								},
-								{
-									Name:  "DEVWORKSPACE_NAMESPACE",
-									Value: workspace.Namespace,
-								},
-								{
-									Name:  "WORKSPACE_ID",
-									Value: dwID,
-								},
+								{Name: "DEVWORKSPACE_NAME", Value: workspace.Name},
+								{Name: "DEVWORKSPACE_NAMESPACE", Value: workspace.Namespace},
+								{Name: "WORKSPACE_ID", Value: dwID},
 								{
 									Name:  "BACKUP_SOURCE_PATH",
 									Value: "/workspace/" + dwID + "/" + constants.DefaultProjectsSourcesRoot,
 								},
+								{Name: "STORAGE_DRIVER", Value: "overlay"},
+								{Name: "BUILDAH_ISOLATION", Value: "chroot"},
+								{Name: "DEVWORKSPACE_BACKUP_REGISTRY", Value: backUpConfig.Registry},
+								{Name: "BUILDAH_PUSH_OPTIONS", Value: "--tls-verify=false"},
 							},
-							// TODO: Replace the following command with actual backup logic
+							Image: images.GetProjectBackupImage(),
+							// Image: "localhost:5001/workspace-backup-image:v3",
 							Args: []string{
-								"echo \"Starting backup for workspace $WORKSPACE_ID\" && ls -l \"$BACKUP_SOURCE_PATH\" && sleep 1 && echo Backup completed for workspace " + dwID,
+								"/workspace-recovery.sh",
+								"--backup",
 							},
 							VolumeMounts: []corev1.VolumeMount{
 								{
 									MountPath: "/workspace",
 									Name:      "workspace-data",
 								},
+								{
+									MountPath: "/var/lib/containers",
+									Name:      "build-storage",
+								},
+							},
+							SecurityContext: &corev1.SecurityContext{
+								RunAsUser: ptrInt64(0),
+								Capabilities: &corev1.Capabilities{
+									Add: []corev1.Capability{"SYS_ADMIN", "SYS_CHROOT"},
+								},
+								AllowPrivilegeEscalation: ptrBool(false),
 							},
 						},
 					},
@@ -351,6 +364,12 @@ func (r *BackupCronJobReconciler) createBackupJob(workspace *dw.DevWorkspace, ct
 								},
 							},
 						},
+						{
+							Name: "build-storage",
+							VolumeSource: corev1.VolumeSource{
+								EmptyDir: &corev1.EmptyDirVolumeSource{},
+							},
+						},
 					},
 				},
 			},