feat(restore): create namespace if targeted namespace doesn't exist (#…

…140)

Adding support to create destination namespace if namespace-mapping is used for restore


Signed-off-by: mayank <[email protected]>

README.md

@@ -234,8 +234,6 @@ Once the backup is completed you should see the backup marked as `Completed`.
 - _If backup name ends with "-20190513104034" format then it is considered as part of scheduled backup_
 
 #### Creating a restore for remote backup
-Before restoring from remote backup, make sure that you have created the namespace in your destination cluster.
-
 To restore data from remote backup, run the following command:
 
 ```
@@ -265,6 +263,8 @@ To restore in different namespace, run the following command:
 velero restore create --from-backup backup_name --restore-volumes=true --namespace-mappings source_ns:destination_ns
 ```
 
+Plugin will create the destination_ns, if it doesn't exist.
+
 **Once restore for remote backup is completed, You need to set targetip in relevant replica. Refer [Setting targetip in replica](#setting-targetip-in-replica).**
 
 #### Setting targetip in replica
@@ -338,8 +338,6 @@ During the first backup iteration of a schedule, full data of the volume will be
 - _If backup name ends with "-20190513104034" format then it is considered as part of scheduled backup_
 
 #### Creating a restore from scheduled remote backup
-Before restoring from remote backup, make sure that you have created the namespace in your destination cluster.
-
 Backups generated by schedule are incremental backups. The first backup of the schedule includes a snapshot of all volume data, and the subsequent backups include the snapshot of modified data from the previous backup. In the older version of velero-plugin(<2.2.0) we need to create restore for all the backup, from base backup to the required backup, Refer [Restoring the scheduled backup without restoreAllIncrementalSnapshots](#restoring-the-scheduled-backup-without-restoreallincrementalsnapshots).
 
 You can automate this process by setting the config parameter `restoreAllIncrementalSnapshots` to `"true"` in volumesnapshotlocation.
@@ -374,7 +372,7 @@ Above command will create the cstor volume and restore all the snapshots backed
 
 Here, base backup means the first backup created by schedule. To restore from scheduled backups, base-backup must be available.
 
-You can restore the scheduled remote backup to a different namespace using the `--namespace-mappings` argument while creating a restore.
+You can restore the scheduled remote backup to a different namespace using the `--namespace-mappings` argument while creating a restore. Plugin will create the destination namespace, if it doesn't exist.
 
 Once restore for remote scheduled backup is completed, You need to set targetip in relevant replica. Refer [Setting targetip in replica](#setting-targetip-in-replica).
 

changelogs/unreleased/140-mynktl

@@ -0,0 +1 @@
+Adding support to create destination namespace, for restore, if it doesn't exist

pkg/cstor/pvc_operation.go

@@ -26,13 +26,19 @@ import (
 	v1 "k8s.io/api/core/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
-// PVCWaitCount control time limit for createPVC
-var PVCWaitCount = 100
+const (
+	// PVCWaitCount control time limit for createPVC
+	PVCWaitCount = 100
 
-// PVCCheckInterval defines amount of delay for PVC bound check
-var PVCCheckInterval = 5 * time.Second
+	// NamespaceCreateTimeout defines timeout for namespace creation
+	NamespaceCreateTimeout = 5 * time.Minute
+
+	// PVCCheckInterval defines amount of delay for PVC bound check
+	PVCCheckInterval = 5 * time.Second
+)
 
 // backupPVC perform backup for given volume's PVC
 func (p *Plugin) backupPVC(volumeID string) error {
@@ -102,6 +108,12 @@ func (p *Plugin) createPVC(volumeID, snapName string) (*Volume, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	err = p.EnsureNamespaceOrCreate(targetedNs)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error verifying namespace")
+	}
+
 	pvc.Namespace = targetedNs
 
 	newVol, err := p.getVolumeFromPVC(*pvc)
@@ -300,3 +312,65 @@ func (p *Plugin) removePVCAnnotationKey(pvc *v1.PersistentVolumeClaim, annotatio
 		Update(pvc)
 	return err
 }
+
+// EnsureNamespaceOrCreate ensure that given namespace exists and ready
+// - If namespace exists and ready to use then it will return nil
+// - If namespace is in terminating state then function will wait for ns removal and re-create it
+// - If namespace doesn't exist then function will create it
+func (p *Plugin) EnsureNamespaceOrCreate(ns string) error {
+	checkNs := func(namespace string) (bool, error) {
+		var isNsReady bool
+
+		err := wait.PollImmediate(time.Second, NamespaceCreateTimeout, func() (bool, error) {
+			p.Log.Debugf("Checking namespace=%s", namespace)
+
+			obj, err := p.K8sClient.CoreV1().Namespaces().Get(namespace, metav1.GetOptions{})
+			if err != nil {
+				if !k8serrors.IsNotFound(err) {
+					return false, err
+				}
+
+				// namespace doesn't exist
+				return true, nil
+			}
+
+			if obj.GetDeletionTimestamp() != nil || obj.Status.Phase == v1.NamespaceTerminating {
+				// will wait till namespace get deleted
+				return false, nil
+			}
+
+			if obj.Status.Phase == v1.NamespaceActive {
+				isNsReady = true
+			}
+
+			return isNsReady, nil
+		})
+
+		return isNsReady, err
+	}
+
+	isReady, err := checkNs(ns)
+	if err != nil {
+		return errors.Wrapf(err, "failed to check namespace")
+	}
+
+	if isReady {
+		return nil
+	}
+
+	// namespace doesn't exist, create it
+	nsObj := &v1.Namespace{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: ns,
+		},
+	}
+
+	p.Log.Infof("Creating namespace=%s", ns)
+
+	_, err = p.K8sClient.CoreV1().Namespaces().Create(nsObj)
+	if err != nil {
+		return errors.Wrapf(err, "failed to create namespace")
+	}
+
+	return nil
+}

tests/app/application_setup.go

@@ -45,6 +45,18 @@ func CreateNamespace(ns string) error {
 	return err
 }
 
+// DestroyNamespace destory the given namespace
+func DestroyNamespace(ns string) error {
+	err := k8s.Client.CoreV1().Namespaces().Delete(ns, &metav1.DeleteOptions{})
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			return nil
+		}
+		return k8s.Client.WaitForNamespaceCleanup(ns)
+	}
+	return nil
+}
+
 // DeployApplication deploy application
 func DeployApplication(appYaml, ns string) error {
 	var p corev1.Pod

tests/k8s/status.go

@@ -157,6 +157,30 @@ func (k *KubeClient) WaitForDeploymentCleanup(labelSelector, ns string) error {
 	}
 }
 
+// WaitForNamespaceCleanup wait for cleanup of the given namespace
+func (k *KubeClient) WaitForNamespaceCleanup(ns string) error {
+	dumpLog := 0
+	for {
+		_, err := k.CoreV1().Namespaces().Get(ns, metav1.GetOptions{})
+
+		if k8serrors.IsNotFound(err) {
+			return nil
+		}
+
+		if err != nil {
+			return err
+		}
+
+		if dumpLog > 6 {
+			fmt.Printf("Waiting for cleanup of namespace %s\n", ns)
+			dumpLog = 0
+		}
+
+		dumpLog++
+		time.Sleep(5 * time.Second)
+	}
+}
+
 // GetPodList return list of pod for given label and namespace
 func (k *KubeClient) GetPodList(ns, label string) (*corev1.PodList, error) {
 	return k.CoreV1().Pods(ns).List(metav1.ListOptions{

tests/sanity/backup_test.go

@@ -58,9 +58,6 @@ var _ = BeforeSuite(func() {
 	err = app.CreateNamespace(AppNs)
 	Expect(err).NotTo(HaveOccurred())
 
-	err = app.CreateNamespace(TargetedNs)
-	Expect(err).NotTo(HaveOccurred())
-
 	err = k8s.Client.CreateStorageClass(openebs.SCYaml)
 	Expect(err).NotTo(HaveOccurred())
 
@@ -134,6 +131,7 @@ var _ = Describe("Backup/Restore Test", func() {
 			By("Destroying Application and Volume")
 			err = app.DestroyApplication(app.BusyboxYaml, AppNs)
 			Expect(err).NotTo(HaveOccurred(), "Failed to destroy application in namespace=%s", AppNs)
+
 			err = openebs.Client.DeleteVolume(openebs.PVCYaml, AppNs)
 			Expect(err).NotTo(HaveOccurred(), "Failed to delete volume for namespace=%s", AppNs)
 		})
@@ -211,8 +209,12 @@ var _ = Describe("Backup/Restore Test", func() {
 			By("Destroying Application and Volume")
 			err = app.DestroyApplication(app.BusyboxYaml, TargetedNs)
 			Expect(err).NotTo(HaveOccurred(), "Failed to destroy application in namespace=%s", TargetedNs)
+
 			err = openebs.Client.DeleteVolume(openebs.PVCYaml, TargetedNs)
 			Expect(err).NotTo(HaveOccurred(), "Failed to delete volume for namespace=%s", TargetedNs)
+
+			err = app.DestroyNamespace(TargetedNs)
+			Expect(err).NotTo(HaveOccurred(), "Failed to delete namespace=%s", TargetedNs)
 		})
 
 		It("Restore from non-scheduled backup to different Namespace", func() {