Merge pull request #96 from upmc-enterprises/kibana

Added support for deploying Kibana automatically

README.md

@@ -48,6 +48,8 @@ Following parameters are available to customize the elastic cluster:
     - storage-class: Name of an existing StorageClass object to use (zones can be [])
 - instrumentation
   - statsd-host: Sets the statsd host to send metrics to if enabled
+- kibana: Deploy kibana to cluster and automatically reference certs from secret
+  - image: Image to use (Note: Using [custom image](https://github.com/upmc-enterprises/kibana-docker) since upstream has x-pack installed and causes issues)
 
 ## Certs secret
 

example/example-es-cluster.json

@@ -5,13 +5,16 @@
     "name": "example-es-cluster"
   },
   "spec": {
+    "kibana": {
+      "image": "upmcenterprises/kibana:5.3.1"
+    },
     "client-node-replicas": 3,
     "master-node-replicas": 2,
     "data-node-replicas": 3,
     "network-host": "0.0.0.0",
     "zones": ["us-east-1c", "us-east-1d", "us-east-1e"],
     "data-volume-size": "10Gi",
-    "java-options": "-Xms1024m -Xmx1024m",
+    "java-options": "-Xms256m -Xmx256m",
     "snapshot": {
       "scheduler-enabled": false,
       "bucket-name": "elasticsnapshots99",

pkg/k8sutil/deployments.go

@@ -35,10 +35,10 @@ import (
 	"k8s.io/client-go/pkg/apis/extensions/v1beta1"
 )
 
-// DeleteClientDeployment deletes the client or master deployment
-func (k *K8sutil) DeleteClientDeployment(clusterName, namespace string) error {
+// DeleteDeployment deletes a deployment
+func (k *K8sutil) DeleteDeployment(clusterName, namespace, deploymentType string) error {
 
-	labelSelector := "component=elasticsearch" + "-" + clusterName + ",role=client"
+	labelSelector := fmt.Sprintf("component=elasticsearch-%s,role=%s", clusterName, deploymentType)
 
 	// Get list of deployments
 	deployments, err := k.Kclient.ExtensionsV1beta1().Deployments(namespace).List(metav1.ListOptions{LabelSelector: labelSelector})
@@ -87,7 +87,7 @@ func (k *K8sutil) DeleteClientDeployment(clusterName, namespace string) error {
 	return nil
 }
 
-// CreateClientDeployment creates the client or master deployment
+// CreateClientDeployment creates the client deployment
 func (k *K8sutil) CreateClientDeployment(baseImage string, replicas *int32, javaOptions string,
 	resources myspec.Resources, imagePullSecrets []myspec.ImagePullSecrets, clusterName, statsdEndpoint, networkHost, namespace string) error {
 
@@ -271,3 +271,104 @@ func (k *K8sutil) CreateClientDeployment(baseImage string, replicas *int32, java
 
 	return nil
 }
+
+// CreateKibanaDeployment creates a deployment of Kibana
+func (k *K8sutil) CreateKibanaDeployment(baseImage, clusterName, namespace string, imagePullSecrets []myspec.ImagePullSecrets) error {
+
+	replicaCount := int32(1)
+
+	component := fmt.Sprintf("elasticsearch-%s", clusterName)
+	elasticHTTPEndpoint := fmt.Sprintf("https://%s:9200", component)
+	deploymentName := fmt.Sprintf("%s-%s", kibanaDeploymentName, clusterName)
+
+	// Check if deployment exists
+	deployment, err := k.Kclient.ExtensionsV1beta1().Deployments(namespace).Get(deploymentName, metav1.GetOptions{})
+
+	if len(deployment.Name) == 0 {
+		logrus.Infof("%s not found, creating...", deploymentName)
+
+		deployment := &v1beta1.Deployment{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: deploymentName,
+				Labels: map[string]string{
+					"component": component,
+					"role":      "kibana",
+					"name":      deploymentName,
+				},
+			},
+			Spec: v1beta1.DeploymentSpec{
+				Replicas: &replicaCount,
+				Template: v1.PodTemplateSpec{
+					ObjectMeta: metav1.ObjectMeta{
+						Labels: map[string]string{
+							"component": component,
+							"role":      "kibana",
+							"name":      deploymentName,
+						},
+					},
+					Spec: v1.PodSpec{
+						Containers: []v1.Container{
+							v1.Container{
+								Name:            deploymentName,
+								Image:           baseImage,
+								ImagePullPolicy: "Always",
+								Env: []v1.EnvVar{
+									v1.EnvVar{
+										Name:  "ELASTICSEARCH_URL",
+										Value: elasticHTTPEndpoint,
+									},
+									v1.EnvVar{
+										Name:  "ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES",
+										Value: "/elasticsearch/config/certs/ca.pem",
+									},
+									v1.EnvVar{
+										Name:  "NODE_DATA",
+										Value: "false",
+									},
+								},
+								Ports: []v1.ContainerPort{
+									v1.ContainerPort{
+										Name:          "http",
+										ContainerPort: 5601,
+										Protocol:      v1.ProtocolTCP,
+									},
+								},
+								VolumeMounts: []v1.VolumeMount{
+									v1.VolumeMount{
+										Name:      fmt.Sprintf("%s-%s", secretName, clusterName),
+										MountPath: "/elasticsearch/config/certs",
+									},
+								},
+							},
+						},
+						Volumes: []v1.Volume{
+							v1.Volume{
+								Name: fmt.Sprintf("%s-%s", secretName, clusterName),
+								VolumeSource: v1.VolumeSource{
+									Secret: &v1.SecretVolumeSource{
+										SecretName: fmt.Sprintf("%s-%s", secretName, clusterName),
+									},
+								},
+							},
+						},
+						ImagePullSecrets: TemplateImagePullSecrets(imagePullSecrets),
+					},
+				},
+			},
+		}
+
+		_, err := k.Kclient.ExtensionsV1beta1().Deployments(namespace).Create(deployment)
+
+		if err != nil {
+			logrus.Error("Could not create kibana deployment: ", err)
+			return err
+		}
+	} else {
+		if err != nil {
+			logrus.Error("Could not get kibana deployment! ", err)
+			return err
+		}
+	}
+
+	return nil
+}

pkg/k8sutil/k8sutil.go

@@ -64,6 +64,8 @@ const (
 	masterDeploymentName = "es-master"
 	dataDeploymentName   = "es-data"
 
+	kibanaDeploymentName = "kibana"
+
 	secretName = "es-certs"
 )
 

pkg/processor/processor.go

@@ -178,6 +178,9 @@ func (p *Processor) refreshClusters() error {
 				Instrumentation: myspec.Instrumentation{
 					StatsdHost: cluster.Spec.Instrumentation.StatsdHost,
 				},
+				Kibana: myspec.Kibana{
+					Image: cluster.Spec.Kibana.Image,
+				},
 			},
 		}
 	}
@@ -274,6 +277,11 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster)
 			c.Spec.DataDiskSize, c.Spec.Resources, c.Spec.ImagePullSecrets, c.ObjectMeta.Name, c.Spec.Instrumentation.StatsdHost, c.Spec.NetworkHost, c.ObjectMeta.Namespace, c.Spec.JavaOptions)
 	}
 
+	// Deploy Kibana
+	if c.Spec.Kibana.Image != "" {
+		p.k8sclient.CreateKibanaDeployment(c.Spec.Kibana.Image, c.ObjectMeta.Name, c.ObjectMeta.Namespace, c.Spec.ImagePullSecrets)
+	}
+
 	// Setup CronSchedule
 	p.clusters[fmt.Sprintf("%s-%s", c.ObjectMeta.Name, c.ObjectMeta.Namespace)].Spec.Scheduler.Init()
 
@@ -283,11 +291,16 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster)
 func (p *Processor) deleteElasticSearchCluster(c *myspec.ElasticsearchCluster) error {
 	logrus.Println("--------> ElasticSearch Cluster deleted...removing all components...")
 
-	err := p.k8sclient.DeleteClientDeployment(c.ObjectMeta.Name, c.ObjectMeta.Namespace)
+	err := p.k8sclient.DeleteDeployment(c.ObjectMeta.Name, c.ObjectMeta.Namespace, "client")
 	if err != nil {
 		logrus.Error("Could not delete client deployment:", err)
 	}
 
+	err = p.k8sclient.DeleteDeployment(c.ObjectMeta.Name, c.ObjectMeta.Namespace, "kibana")
+	if err != nil {
+		logrus.Error("Could not delete kibana deployment:", err)
+	}
+
 	err = p.k8sclient.DeleteStatefulSet("master", c.ObjectMeta.Name, c.ObjectMeta.Namespace)
 	if err != nil {
 		logrus.Error("Could not delete master deployment:", err)

pkg/spec/cluster.go

@@ -90,6 +90,9 @@ type ClusterSpec struct {
 	//NodePort
 	NodePort int32 `json:"nodePort"`
 
+	// Kibana
+	Kibana Kibana `json:"kibana"`
+
 	Scheduler *snapshot.Scheduler
 }
 
@@ -154,6 +157,12 @@ type Instrumentation struct {
 	StatsdHost string `json:"statsd-host"`
 }
 
+// Kibana properties if wanting operator to deploy for user
+type Kibana struct {
+	// Defines the image to use for deploying kibana
+	Image string `json:"image"`
+}
+
 type ElasticsearchClusterCopy ElasticsearchCluster
 
 func (c *ElasticsearchCluster) UnmarshalJSON(data []byte) error {