Merge pull request #59 from upmc-enterprises/statsd

Enable support for specifying statsd host for instrumentation

.travis.yml

@@ -1,7 +1,7 @@
 language: go
 
 go:
-  - 1.7
+  - 1.8
 
 sudo: required
 

README.md

@@ -43,6 +43,8 @@ Following parameters are available to customize the elastic cluster:
 - [storage](https://kubernetes.io/docs/user-guide/persistent-volumes/)
   - type: Defines the type of storage to provision based upon cloud (e.g. `gp2`)
   - storage-class-provisioner: Defines which type of provisioner to use (e.g. `kubernetes.io/aws-ebs`)
+- instrumentation
+  - statsd-host: Sets the statsd host to send metrics to if enabled
 
 ## Certs secret
 

example/example-es-cluster-minikube.json

@@ -5,6 +5,7 @@
     "name": "example-es-cluster"
   },
   "spec": {
+    "elastic-search-image": "upmcenterprises/docker-elasticsearch-kubernetes:5.3.1",
     "client-node-replicas": 1,
     "master-node-replicas": 1,
     "data-node-replicas": 3,

pkg/k8sutil/k8sutil.go

@@ -274,7 +274,7 @@ func (k *K8sutil) CreateKubernetesThirdPartyResource() error {
 // DeleteServices creates the discovery service
 func (k *K8sutil) DeleteServices(clusterName string) {
 
-	fullDiscoveryServiceName := discoveryServiceName + "-" + clusterName
+	fullDiscoveryServiceName := fmt.Sprintf("%s-%s", discoveryServiceName, clusterName)
 	err := k.Kclient.Services(namespace).Delete(fullDiscoveryServiceName, &v1.DeleteOptions{})
 	if err != nil {
 		logrus.Error("Could not delete service "+fullDiscoveryServiceName+":", err)
@@ -303,7 +303,7 @@ func (k *K8sutil) DeleteServices(clusterName string) {
 // CreateDiscoveryService creates the discovery service
 func (k *K8sutil) CreateDiscoveryService(clusterName string) error {
 
-	fullDiscoveryServiceName := discoveryServiceName + "-" + clusterName
+	fullDiscoveryServiceName := fmt.Sprintf("%s-%s", discoveryServiceName, clusterName)
 	component := "elasticsearch" + "-" + clusterName
 	// Check if service exists
 	svc, err := k.Kclient.Services(namespace).Get(fullDiscoveryServiceName)
@@ -541,9 +541,12 @@ func (k *K8sutil) DeleteStatefulSet(clusterName string) error {
 }
 
 // CreateClientMasterDeployment creates the client or master deployment
-func (k *K8sutil) CreateClientMasterDeployment(deploymentType, baseImage string, replicas *int32, javaOptions string, resources myspec.Resources, imagePullSecrets []myspec.ImagePullSecrets, clusterName string) error {
+func (k *K8sutil) CreateClientMasterDeployment(deploymentType, baseImage string, replicas *int32, javaOptions string,
+	resources myspec.Resources, imagePullSecrets []myspec.ImagePullSecrets, clusterName, statsdEndpoint string) error {
+
+	component := fmt.Sprintf("elasticsearch-%s", clusterName)
+	discoveryServiceNameCluster := fmt.Sprintf("%s-%s", discoveryServiceName, clusterName)
 
-	component := "elasticsearch" + "-" + clusterName
 	var deploymentName, role, isNodeMaster, httpEnable string
 
 	if deploymentType == "client" {
@@ -635,6 +638,14 @@ func (k *K8sutil) CreateClientMasterDeployment(deploymentType, baseImage string,
 										Name:  "ES_JAVA_OPTS",
 										Value: javaOptions,
 									},
+									v1.EnvVar{
+										Name:  "STATSD_HOST",
+										Value: statsdEndpoint,
+									},
+									v1.EnvVar{
+										Name:  "DISCOVERY_SERVICE",
+										Value: discoveryServiceNameCluster,
+									},
 								},
 								Ports: []v1.ContainerPort{
 									v1.ContainerPort{
@@ -731,10 +742,12 @@ func TemplateImagePullSecrets(ips []myspec.ImagePullSecrets) []v1.LocalObjectRef
 }
 
 // CreateDataNodeDeployment creates the data node deployment
-func (k *K8sutil) CreateDataNodeDeployment(replicas *int32, baseImage, storageClass string, dataDiskSize string, resources myspec.Resources, imagePullSecrets []myspec.ImagePullSecrets, clusterName string) error {
-	fullDataDeploymentName := dataDeploymentName + "-" + clusterName
-	component := "elasticsearch" + "-" + clusterName
+func (k *K8sutil) CreateDataNodeDeployment(replicas *int32, baseImage, storageClass string, dataDiskSize string, resources myspec.Resources,
+	imagePullSecrets []myspec.ImagePullSecrets, clusterName, statsdEndpoint string) error {
 
+	fullDataDeploymentName := fmt.Sprintf("%s-%s", dataDeploymentName, clusterName)
+	component := fmt.Sprintf("elasticsearch-%s", clusterName)
+	discoveryServiceNameCluster := fmt.Sprintf("%s-%s", discoveryServiceName, clusterName)
 	statefulSetName := fmt.Sprintf("%s-%s", fullDataDeploymentName, storageClass)
 
 	// Check if StatefulSet exists
@@ -813,6 +826,14 @@ func (k *K8sutil) CreateDataNodeDeployment(replicas *int32, baseImage, storageCl
 										Name:  "ES_JAVA_OPTS",
 										Value: "-Xms1024m -Xmx1024m",
 									},
+									v1.EnvVar{
+										Name:  "STATSD_HOST",
+										Value: statsdEndpoint,
+									},
+									v1.EnvVar{
+										Name:  "DISCOVERY_SERVICE",
+										Value: discoveryServiceNameCluster,
+									},
 								},
 								Ports: []v1.ContainerPort{
 									v1.ContainerPort{

pkg/processor/processor.go

@@ -144,6 +144,9 @@ func (p *Processor) refreshClusters() error {
 						CPU:    cluster.Spec.Resources.Requests.CPU,
 					},
 				},
+				Instrumentation: myspec.Instrumentation{
+					StatsdHost: cluster.Spec.Instrumentation.StatsdHost,
+				},
 			},
 		}
 	}
@@ -180,8 +183,8 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster)
 	p.k8sclient.CreateDataService(c.Metadata.Name)
 	p.k8sclient.CreateClientService(c.Metadata.Name)
 
-	p.k8sclient.CreateClientMasterDeployment("client", baseImage, &c.Spec.ClientNodeReplicas, c.Spec.JavaOptions, c.Spec.Resources, c.Spec.ImagePullSecrets, c.Metadata.Name)
-	p.k8sclient.CreateClientMasterDeployment("master", baseImage, &c.Spec.MasterNodeReplicas, c.Spec.JavaOptions, c.Spec.Resources, c.Spec.ImagePullSecrets, c.Metadata.Name)
+	p.k8sclient.CreateClientMasterDeployment("client", baseImage, &c.Spec.ClientNodeReplicas, c.Spec.JavaOptions, c.Spec.Resources, c.Spec.ImagePullSecrets, c.Metadata.Name, c.Spec.Instrumentation.StatsdHost)
+	p.k8sclient.CreateClientMasterDeployment("master", baseImage, &c.Spec.MasterNodeReplicas, c.Spec.JavaOptions, c.Spec.Resources, c.Spec.ImagePullSecrets, c.Metadata.Name, c.Spec.Instrumentation.StatsdHost)
 
 	zoneCount := 0
 	if len(c.Spec.Zones) != 0 {
@@ -195,13 +198,13 @@ func (p *Processor) processElasticSearchCluster(c *myspec.ElasticsearchCluster)
 		zoneDistribution := p.calculateZoneDistribution(c.Spec.DataNodeReplicas, zoneCount)
 
 		for index, count := range zoneDistribution {
-			p.k8sclient.CreateDataNodeDeployment(&count, baseImage, c.Spec.Zones[index], c.Spec.DataDiskSize, c.Spec.Resources, c.Spec.ImagePullSecrets, c.Metadata.Name)
+			p.k8sclient.CreateDataNodeDeployment(&count, baseImage, c.Spec.Zones[index], c.Spec.DataDiskSize, c.Spec.Resources, c.Spec.ImagePullSecrets, c.Metadata.Name, c.Spec.Instrumentation.StatsdHost)
 		}
 	} else {
 		// No zones defined, rely on current provisioning logic which may break. Other strategy is to use emptyDir?
 		// NOTE: Issue with dynamic PV provisioning (https://github.com/kubernetes/kubernetes/issues/34583)
 		p.k8sclient.CreateStorageClass("standard", c.Spec.Storage.StorageClassProvisoner, c.Spec.Storage.StorageType, c.Metadata.Name)
-		p.k8sclient.CreateDataNodeDeployment(func() *int32 { i := int32(c.Spec.DataNodeReplicas); return &i }(), baseImage, "standard", c.Spec.DataDiskSize, c.Spec.Resources, c.Spec.ImagePullSecrets, c.Metadata.Name)
+		p.k8sclient.CreateDataNodeDeployment(func() *int32 { i := int32(c.Spec.DataNodeReplicas); return &i }(), baseImage, "standard", c.Spec.DataDiskSize, c.Spec.Resources, c.Spec.ImagePullSecrets, c.Metadata.Name, c.Spec.Instrumentation.StatsdHost)
 	}
 
 	// Setup CronSchedule

pkg/spec/cluster.go

@@ -86,6 +86,9 @@ type ClusterSpec struct {
 	// Resources defines memory / cpu constraints
 	Resources Resources `json:"resources"`
 
+	// Instrumentation defines metrics for the cluster
+	Instrumentation Instrumentation `json:"instrumentation"`
+
 	Scheduler *snapshot.Scheduler
 }
 
@@ -142,6 +145,11 @@ type MemoryCPU struct {
 	CPU string `json:"cpu"`
 }
 
+// Instrumentation handles all metrics for the cluster
+type Instrumentation struct {
+	StatsdHost string `json:"statsd-host"`
+}
+
 // Required to satisfy Object interface
 func (e *ElasticsearchCluster) GetObjectKind() unversioned.ObjectKind {
 	return &e.TypeMeta