From 7ffa5b9dbab11645072a9fd28e26a52884475b1e Mon Sep 17 00:00:00 2001 From: Matthieu Patou Date: Tue, 17 Dec 2024 14:00:49 -0800 Subject: [PATCH] Support filesystem backup as well as blob store backup --- api/v1beta1/foundationdbbackup_types.go | 26 ++++++- api/v1beta1/foundationdbbackup_types_test.go | 33 +++++++++ api/v1beta1/foundationdbrestore_types.go | 17 ++++- api/v1beta1/foundationdbrestore_types_test.go | 67 ++++++++++++++----- api/v1beta2/foundationdbbackup_types.go | 36 ++++++++-- api/v1beta2/foundationdbrestore_types.go | 25 ++++++- api/v1beta2/foundationdbrestore_types_test.go | 33 +++++++++ ....foundationdb.org_foundationdbbackups.yaml | 24 +++++++ ...foundationdb.org_foundationdbrestores.yaml | 24 +++++++ fdbclient/admin_client.go | 6 ++ 10 files changed, 262 insertions(+), 29 deletions(-) diff --git a/api/v1beta1/foundationdbbackup_types.go b/api/v1beta1/foundationdbbackup_types.go index 962747786..8d47f9989 100644 --- a/api/v1beta1/foundationdbbackup_types.go +++ b/api/v1beta1/foundationdbbackup_types.go @@ -112,6 +112,8 @@ type FoundationDBBackupSpec struct { // This is the configuration of the target blobstore for this backup. BlobStoreConfiguration *BlobStoreConfiguration `json:"blobStoreConfiguration,omitempty"` + // This is the configuration of the target filesystem for this backup. + FSConfiguration *FSConfiguration `json:"fsConfiguration,omitempty"` } // FoundationDBBackupStatus describes the current status of the backup for a cluster. @@ -209,6 +211,19 @@ type BlobStoreConfiguration struct { URLParameters []URLParamater `json:"urlParameters,omitempty"` } +// FSConfiguration describes the blob store configuration. +type FSConfiguration struct { + // The name for the backup. + // If empty defaults to .metadata.name. + // +kubebuilder:validation:MaxLength=1024 + BackupName string `json:"backupName,omitempty"` + + // The url to use with the backup destination. + // +kubebuilder:validation:MaxLength=255 + // +kubebuilder:validation:Required + URL string `json:"url"` +} + // ShouldRun determines whether a backup should be running. func (backup *FoundationDBBackup) ShouldRun() bool { return backup.Spec.BackupState == "" || backup.Spec.BackupState == BackupStateRunning || backup.Spec.BackupState == BackupStatePaused @@ -236,7 +251,7 @@ func (backup *FoundationDBBackup) Bucket() string { // BackupName gets the name of the backup in the destination. // This will fill in a default value if the backup name in the spec is empty. func (backup *FoundationDBBackup) BackupName() string { - if backup.Spec.BackupName == "" && (backup.Spec.BlobStoreConfiguration == nil || backup.Spec.BlobStoreConfiguration.BackupName == "") { + if backup.Spec.BackupName == "" && (backup.Spec.BlobStoreConfiguration == nil || backup.Spec.BlobStoreConfiguration.BackupName == "") && (backup.Spec.FSConfiguration == nil || backup.Spec.FSConfiguration.BackupName == "") { return backup.ObjectMeta.Name } @@ -244,6 +259,10 @@ func (backup *FoundationDBBackup) BackupName() string { return backup.Spec.BlobStoreConfiguration.BackupName } + if backup.Spec.FSConfiguration != nil && backup.Spec.FSConfiguration.BackupName != "" { + return backup.Spec.FSConfiguration.BackupName + } + return backup.Spec.BackupName } @@ -253,6 +272,11 @@ func (backup *FoundationDBBackup) BackupURL() string { return backup.Spec.BlobStoreConfiguration.getURL(backup.BackupName(), backup.Bucket()) } + // mpatou: should we use file:// ? + if backup.Spec.FSConfiguration != nil { + return fmt.Sprintf("%s/%s", backup.Spec.FSConfiguration.URL, backup.BackupName()) + } + return fmt.Sprintf("blobstore://%s/%s?bucket=%s", backup.Spec.AccountName, backup.BackupName(), backup.Bucket()) } diff --git a/api/v1beta1/foundationdbbackup_types_test.go b/api/v1beta1/foundationdbbackup_types_test.go index cc5a8c277..c1bbfe743 100644 --- a/api/v1beta1/foundationdbbackup_types_test.go +++ b/api/v1beta1/foundationdbbackup_types_test.go @@ -381,6 +381,39 @@ var _ = Describe("[api] FoundationDBBackup", func() { }, }, "blobstore://account@account/mybackup?bucket=fdb-backups&secure_connection=0"), + Entry("A backup with a fs config and a backup name", + FoundationDBBackup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mybackup", + }, + Spec: FoundationDBBackupSpec{ + FSConfiguration: &FSConfiguration{ + URL: "/some/path", + BackupName: "backup", + }, + }, + }, + "/some/path/backup"), + Entry("A backup with a fs config and no specific name", + FoundationDBBackup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mybackup", + }, + Spec: FoundationDBBackupSpec{ + FSConfiguration: &FSConfiguration{ + URL: "/some/path", + }, + }, + }, + "/some/path/mybackup"), + Entry("A backup with just a metadata name will return a generic URL based on the backup name", + FoundationDBBackup{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mybackup", + }, + Spec: FoundationDBBackupSpec{}, + }, + "blobstore:///mybackup?bucket=fdb-backups"), ) }) }) diff --git a/api/v1beta1/foundationdbrestore_types.go b/api/v1beta1/foundationdbrestore_types.go index 0f0b70e5d..66f6c7c03 100644 --- a/api/v1beta1/foundationdbrestore_types.go +++ b/api/v1beta1/foundationdbrestore_types.go @@ -21,6 +21,7 @@ package v1beta1 import ( + "fmt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -63,6 +64,8 @@ type FoundationDBRestoreSpec struct { // This is the configuration of the target blobstore for this backup. BlobStoreConfiguration *BlobStoreConfiguration `json:"blobStoreConfiguration,omitempty"` + // This is the configuration of the target filesystem for this backup. + FSConfiguration *FSConfiguration `json:"fsConfiguration,omitempty"` // CustomParameters defines additional parameters to pass to the backup // agents. @@ -93,11 +96,16 @@ type FoundationDBKeyRange struct { // BackupName gets the name of the backup for the source backup. // This will fill in a default value if the backup name in the spec is empty. func (restore *FoundationDBRestore) BackupName() string { - if restore.Spec.BlobStoreConfiguration == nil || restore.Spec.BlobStoreConfiguration.BackupName == "" { - return restore.ObjectMeta.Name + + if restore.Spec.BlobStoreConfiguration != nil && restore.Spec.BlobStoreConfiguration.BackupName != "" { + return restore.Spec.BlobStoreConfiguration.BackupName + } + + if restore.Spec.FSConfiguration != nil && restore.Spec.FSConfiguration.BackupName != "" { + return restore.Spec.FSConfiguration.BackupName } - return restore.Spec.BlobStoreConfiguration.BackupName + return restore.ObjectMeta.Name } // BackupURL gets the destination url of the backup. @@ -105,6 +113,9 @@ func (restore *FoundationDBRestore) BackupURL() string { if restore.Spec.BlobStoreConfiguration != nil { return restore.Spec.BlobStoreConfiguration.getURL(restore.BackupName(), restore.Spec.BlobStoreConfiguration.BucketName()) } + if restore.Spec.FSConfiguration != nil { + return fmt.Sprintf("%s/%s", restore.Spec.FSConfiguration.URL, restore.BackupName()) + } return restore.Spec.BackupURL } diff --git a/api/v1beta1/foundationdbrestore_types_test.go b/api/v1beta1/foundationdbrestore_types_test.go index 4e33ae30f..141350149 100644 --- a/api/v1beta1/foundationdbrestore_types_test.go +++ b/api/v1beta1/foundationdbrestore_types_test.go @@ -1,21 +1,21 @@ /* - * foundationdbbrestore_types_test.go - * - * This source file is part of the FoundationDB open source project - * - * Copyright 2020-2021 Apple Inc. and the FoundationDB project authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. +* foundationdbbrestore_types_test.go +* +* This source file is part of the FoundationDB open source project +* +* Copyright 2020-2021 Apple Inc. and the FoundationDB project authors +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. */ package v1beta1 @@ -126,6 +126,39 @@ var _ = Describe("[api] FoundationDBRestore", func() { }, }, "blobstore://account@account/mybackup?bucket=fdb-backups&secure_connection=0"), + Entry("A restore with a fs config and a backup name", + FoundationDBRestore{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mybackup", + }, + Spec: FoundationDBRestoreSpec{ + FSConfiguration: &FSConfiguration{ + URL: "/some/path", + BackupName: "backup", + }, + }, + }, + "/some/path/backup"), + Entry("A restore with a fs config and no specific name", + FoundationDBRestore{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mybackup", + }, + Spec: FoundationDBRestoreSpec{ + FSConfiguration: &FSConfiguration{ + URL: "/some/path", + }, + }, + }, + "/some/path/mybackup"), + Entry("A restore with just a metadata name will return an empty name", + FoundationDBRestore{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mybackup", + }, + Spec: FoundationDBRestoreSpec{}, + }, + ""), ) }) }) diff --git a/api/v1beta2/foundationdbbackup_types.go b/api/v1beta2/foundationdbbackup_types.go index b4b7312c0..23de36338 100644 --- a/api/v1beta2/foundationdbbackup_types.go +++ b/api/v1beta2/foundationdbbackup_types.go @@ -95,6 +95,8 @@ type FoundationDBBackupSpec struct { // This is the configuration of the target blobstore for this backup. BlobStoreConfiguration *BlobStoreConfiguration `json:"blobStoreConfiguration,omitempty"` + // This is the configuration of the target filesystem for this backup. + FSConfiguration *FSConfiguration `json:"fsConfiguration,omitempty"` // MainContainer defines customization for the foundationdb container. MainContainer ContainerOverrides `json:"mainContainer,omitempty"` @@ -211,6 +213,19 @@ type BlobStoreConfiguration struct { URLParameters []URLParameter `json:"urlParameters,omitempty"` } +// FSConfiguration describes the blob store configuration. +type FSConfiguration struct { + // The name for the backup. + // If empty defaults to .metadata.name. + // +kubebuilder:validation:MaxLength=1024 + BackupName string `json:"backupName,omitempty"` + + // The url to use with the backup destination. + // +kubebuilder:validation:MaxLength=255 + // +kubebuilder:validation:Required + URL string `json:"url"` +} + // ShouldRun determines whether a backup should be running. func (backup *FoundationDBBackup) ShouldRun() bool { return backup.Spec.BackupState == "" || backup.Spec.BackupState == BackupStateRunning || backup.Spec.BackupState == BackupStatePaused @@ -224,7 +239,7 @@ func (backup *FoundationDBBackup) ShouldBePaused() bool { // Bucket gets the bucket this backup will use. // This will fill in a default value if the bucket in the spec is empty. func (backup *FoundationDBBackup) Bucket() string { - if backup.Spec.BlobStoreConfiguration.Bucket == "" { + if backup.Spec.BlobStoreConfiguration == nil || backup.Spec.BlobStoreConfiguration.Bucket == "" { return "fdb-backups" } @@ -234,16 +249,27 @@ func (backup *FoundationDBBackup) Bucket() string { // BackupName gets the name of the backup in the destination. // This will fill in a default value if the backup name in the spec is empty. func (backup *FoundationDBBackup) BackupName() string { - if backup.Spec.BlobStoreConfiguration.BackupName == "" { - return backup.ObjectMeta.Name + if backup.Spec.BlobStoreConfiguration != nil && backup.Spec.BlobStoreConfiguration.BackupName != "" { + return backup.Spec.BlobStoreConfiguration.BackupName + } + + if backup.Spec.FSConfiguration != nil && backup.Spec.FSConfiguration.BackupName != "" { + return backup.Spec.FSConfiguration.BackupName } - return backup.Spec.BlobStoreConfiguration.BackupName + return backup.ObjectMeta.Name } // BackupURL gets the destination url of the backup. func (backup *FoundationDBBackup) BackupURL() string { - return backup.Spec.BlobStoreConfiguration.getURL(backup.BackupName(), backup.Bucket()) + if backup.Spec.BlobStoreConfiguration != nil { + return backup.Spec.BlobStoreConfiguration.getURL(backup.BackupName(), backup.Bucket()) + } + // mpatou: should we use file:// ? + if backup.Spec.FSConfiguration != nil { + return fmt.Sprintf("%s/%s", backup.Spec.FSConfiguration.URL, backup.BackupName()) + } + return "" } // SnapshotPeriodSeconds gets the period between snapshots for a backup. diff --git a/api/v1beta2/foundationdbrestore_types.go b/api/v1beta2/foundationdbrestore_types.go index 40c3e2107..c50d91628 100644 --- a/api/v1beta2/foundationdbrestore_types.go +++ b/api/v1beta2/foundationdbrestore_types.go @@ -17,6 +17,7 @@ limitations under the License. package v1beta2 import ( + "fmt" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -55,6 +56,8 @@ type FoundationDBRestoreSpec struct { // This is the configuration of the target blobstore for this backup. BlobStoreConfiguration *BlobStoreConfiguration `json:"blobStoreConfiguration,omitempty"` + // This is the configuration of the target filesystem for this backup. + FSConfiguration *FSConfiguration `json:"fsConfiguration,omitempty"` // CustomParameters defines additional parameters to pass to the backup // agents. @@ -85,16 +88,32 @@ type FoundationDBKeyRange struct { // BackupName gets the name of the backup for the source backup. // This will fill in a default value if the backup name in the spec is empty. func (restore *FoundationDBRestore) BackupName() string { - if restore.Spec.BlobStoreConfiguration == nil || restore.Spec.BlobStoreConfiguration.BackupName == "" { + if (restore.Spec.BlobStoreConfiguration == nil || restore.Spec.BlobStoreConfiguration.BackupName == "") && (restore.Spec.FSConfiguration == nil || restore.Spec.FSConfiguration.BackupName == "") { return restore.ObjectMeta.Name } - return restore.Spec.BlobStoreConfiguration.BackupName + if restore.Spec.BlobStoreConfiguration != nil && restore.Spec.BlobStoreConfiguration.BackupName != "" { + return restore.Spec.BlobStoreConfiguration.BackupName + } + + if restore.Spec.FSConfiguration != nil && restore.Spec.FSConfiguration.BackupName != "" { + return restore.Spec.FSConfiguration.BackupName + } + + return "" } // BackupURL gets the destination url of the backup. func (restore *FoundationDBRestore) BackupURL() string { - return restore.Spec.BlobStoreConfiguration.getURL(restore.BackupName(), restore.Spec.BlobStoreConfiguration.BucketName()) + if restore.Spec.BlobStoreConfiguration != nil { + return restore.Spec.BlobStoreConfiguration.getURL(restore.BackupName(), restore.Spec.BlobStoreConfiguration.BucketName()) + } + + if restore.Spec.FSConfiguration != nil { + return fmt.Sprintf("%s/%s", restore.Spec.FSConfiguration.URL, restore.BackupName()) + } + + return "" } func init() { diff --git a/api/v1beta2/foundationdbrestore_types_test.go b/api/v1beta2/foundationdbrestore_types_test.go index 9753bcab2..10070d943 100644 --- a/api/v1beta2/foundationdbrestore_types_test.go +++ b/api/v1beta2/foundationdbrestore_types_test.go @@ -116,6 +116,39 @@ var _ = Describe("[api] FoundationDBRestore", func() { }, }, "blobstore://account@account:80/mybackup?bucket=fdb-backups&secure_connection=0"), + Entry("A restore with a fs config and a backup name", + FoundationDBRestore{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mybackup", + }, + Spec: FoundationDBRestoreSpec{ + FSConfiguration: &FSConfiguration{ + URL: "/some/path", + BackupName: "backup", + }, + }, + }, + "/some/path/backup"), + Entry("A restore with a fs config and no specific name", + FoundationDBRestore{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mybackup", + }, + Spec: FoundationDBRestoreSpec{ + FSConfiguration: &FSConfiguration{ + URL: "/some/path", + }, + }, + }, + "/some/path/mybackup"), + Entry("A restore with just a metadata name will return an empty name", + FoundationDBRestore{ + ObjectMeta: metav1.ObjectMeta{ + Name: "mybackup", + }, + Spec: FoundationDBRestoreSpec{}, + }, + ""), ) }) }) diff --git a/config/crd/bases/apps.foundationdb.org_foundationdbbackups.yaml b/config/crd/bases/apps.foundationdb.org_foundationdbbackups.yaml index 29abf12dd..35ea6a822 100644 --- a/config/crd/bases/apps.foundationdb.org_foundationdbbackups.yaml +++ b/config/crd/bases/apps.foundationdb.org_foundationdbbackups.yaml @@ -96,6 +96,18 @@ spec: required: - accountName type: object + fsConfiguration: + properties: + backupName: + maxLength: 1024 + type: string + url: + maxLength: 255 + minLength: 1 + type: string + required: + - url + type: object bucket: type: string clusterName: @@ -3452,6 +3464,18 @@ spec: required: - accountName type: object + fsConfiguration: + properties: + backupName: + maxLength: 1024 + type: string + url: + maxLength: 255 + minLength: 1 + type: string + required: + - url + type: object clusterName: type: string customParameters: diff --git a/config/crd/bases/apps.foundationdb.org_foundationdbrestores.yaml b/config/crd/bases/apps.foundationdb.org_foundationdbrestores.yaml index bd2256b5a..44e5ec0ab 100644 --- a/config/crd/bases/apps.foundationdb.org_foundationdbrestores.yaml +++ b/config/crd/bases/apps.foundationdb.org_foundationdbrestores.yaml @@ -56,6 +56,18 @@ spec: required: - accountName type: object + fsConfiguration: + properties: + backupName: + maxLength: 1024 + type: string + url: + maxLength: 255 + minLength: 1 + type: string + required: + - url + type: object customParameters: items: maxLength: 100 @@ -128,6 +140,18 @@ spec: required: - accountName type: object + fsConfiguration: + properties: + backupName: + maxLength: 1024 + type: string + url: + maxLength: 255 + minLength: 1 + type: string + required: + - url + type: object customParameters: items: maxLength: 100 diff --git a/fdbclient/admin_client.go b/fdbclient/admin_client.go index 2f1f7097b..babcd2399 100644 --- a/fdbclient/admin_client.go +++ b/fdbclient/admin_client.go @@ -556,6 +556,9 @@ func (client *cliAdminClient) GetProtocolVersion(version string) (string, error) } func (client *cliAdminClient) StartBackup(url string, snapshotPeriodSeconds int) error { + if url == "" { + return fmt.Errorf("URL is required to start a backup, check the configuration") + } _, err := client.runCommand(cliCommand{ binary: fdbbackupStr, args: []string{ @@ -646,6 +649,9 @@ func (client *cliAdminClient) GetBackupStatus() (*fdbv1beta2.FoundationDBLiveBac // StartRestore starts a new restore. func (client *cliAdminClient) StartRestore(url string, keyRanges []fdbv1beta2.FoundationDBKeyRange) error { + if url == "" { + return fmt.Errorf("URL is required to start a restore, check the configuration") + } args := []string{ "start", "-r",