Merge pull request #209 from danisla/gcs-snapshot

Support for GCS Snapshots

README.md

@@ -176,7 +176,7 @@ If changes are required to the cluster, say the replica count of the data nodes
 
 # Snapshot
 
-Elasticsearch can snapshot it's indexes for easy backup / recovery of the cluster. Currently there's an integration to Amazon S3 as the backup repository for snapshots. The `upmcenterprises` docker images include the [S3 Plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-s3.html) which enables this feature in AWS. 
+Elasticsearch can snapshot it's indexes for easy backup / recovery of the cluster. Currently there's an integration to Amazon S3 or Google Cloud Storage as the backup repository for snapshots. The `upmcenterprises` docker images include the [S3 Plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-s3.html) and the [GCS Plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-gcs.html) which enables this feature in AWS and GCP. 
 
 ## Schedule
 
@@ -221,6 +221,21 @@ To enable the snapshots create a bucket in S3, then apply the following IAM perm
 }
 ```
 
+## GCP Setup
+
+To enable snapshots with GCS on GKE, create a bucket in GCS and bind the `storage.admin` role to the cluster service account replacing `${BUCKET}` with your bucket name:
+
+```
+gsutil mb gs://${BUCKET}
+
+SA_EMAIL=$(kubectl run shell --rm --restart=Never -it --image google/cloud-sdk --command /usr/bin/curl -- -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/email)
+
+PROJECT=$(gcloud config get-value project)
+
+gcloud projects add-iam-policy-binding ${PROJECT} \
+  --role roles/storage.admin --member serviceAccount:${SA_EMAIL}
+```
+
 ## Snapshot Authentication
 If you are using an elasticsearch image that requires authentication for the snapshot url, you can specify basic auth credentials.
 

pkg/apis/elasticsearchoperator/v1/cluster.go

@@ -133,7 +133,10 @@ type Snapshot struct {
 	// Enabled determines if snapshots are enabled
 	SchedulerEnabled bool `json:"scheduler-enabled"`
 
-	// BucketName defines the AWS S3 bucket to store snapshots
+	// RepoType defines the type of Elasticsearch Repository, s3 or gcs
+	RepoType string `json:"type"`
+
+	// BucketName defines the AWS S3 or GCS bucket to store snapshots
 	BucketName string `json:"bucket-name"`
 
 	// CronSchedule defines how to run the snapshots
@@ -204,7 +207,8 @@ type Cerebro struct {
 
 // Scheduler stores info about how to snapshot the cluster
 type Scheduler struct {
-	S3bucketName string
+	RepoType     string
+	BucketName   string
 	CronSchedule string
 	Enabled      bool
 	Auth         SchedulerAuthentication

pkg/processor/processor.go

@@ -165,6 +165,7 @@ func (p *Processor) refreshClusters() error {
 					KeepSecretsOnDelete: cluster.Spec.KeepSecretsOnDelete,
 					Snapshot: myspec.Snapshot{
 						SchedulerEnabled: cluster.Spec.Snapshot.SchedulerEnabled,
+						RepoType:         cluster.Spec.Snapshot.RepoType,
 						BucketName:       cluster.Spec.Snapshot.BucketName,
 						CronSchedule:     cluster.Spec.Snapshot.CronSchedule,
 					},
@@ -175,7 +176,8 @@ func (p *Processor) refreshClusters() error {
 						VolumeReclaimPolicy:    cluster.Spec.Storage.VolumeReclaimPolicy,
 					},
 					Scheduler: myspec.Scheduler{
-						S3bucketName: cluster.Spec.Snapshot.BucketName,
+						RepoType:     cluster.Spec.Snapshot.RepoType,
+						BucketName:   cluster.Spec.Snapshot.BucketName,
 						CronSchedule: cluster.Spec.Snapshot.CronSchedule,
 						Enabled:      cluster.Spec.Snapshot.SchedulerEnabled,
 						Auth: myspec.SchedulerAuthentication{
@@ -210,6 +212,7 @@ func (p *Processor) refreshClusters() error {
 				},
 			},
 			Scheduler: snapshot.New(
+				cluster.Spec.Snapshot.RepoType,
 				cluster.Spec.Snapshot.BucketName,
 				cluster.Spec.Snapshot.CronSchedule,
 				cluster.Spec.Snapshot.SchedulerEnabled,

pkg/snapshot/scheduler.go

@@ -53,17 +53,22 @@ type Scheduler struct {
 }
 
 // New creates an instance of Scheduler
-func New(bucketName, cronSchedule string, enabled, useSSL bool, userName, password, image,
+func New(repoType, bucketName, cronSchedule string, enabled, useSSL bool, userName, password, image,
 	elasticURL, clusterName, namespace string, kc kubernetes.Interface) *Scheduler {
 
+	if repoType == "" {
+		repoType = "s3"
+	}
+
 	if image == "" {
 		image = defaultCronImage
 	}
 
 	return &Scheduler{
 		Kclient: kc,
 		CRD: enterprisesv1.Scheduler{
-			S3bucketName: bucketName,
+			RepoType:     repoType,
+			BucketName:   bucketName,
 			CronSchedule: cronSchedule,
 			ElasticURL:   elasticURL,
 			Auth: enterprisesv1.SchedulerAuthentication{
@@ -214,7 +219,8 @@ func (s *Scheduler) CreateCronJob(namespace, clusterName, action, cronSchedule s
 										},
 										Args: []string{
 											fmt.Sprintf("--action=%s", action),
-											fmt.Sprintf("--s3-bucket-name=%s", s.CRD.S3bucketName),
+											fmt.Sprintf("--repo-type=%s", s.CRD.RepoType),
+											fmt.Sprintf("--bucket-name=%s", s.CRD.BucketName),
 											fmt.Sprintf("--elastic-url=%s", s.CRD.ElasticURL),
 											fmt.Sprintf("--auth-username=%s", s.CRD.Auth.UserName),
 											fmt.Sprintf("--auth-password=%s", s.CRD.Auth.Password),