From 6680f945c1742a18c10e566d9ee677f975945038 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse Date: Wed, 3 Dec 2014 22:44:45 +0100 Subject: [PATCH 1/4] Build using travis. --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..f603aaa --- /dev/null +++ b/.travis.yml @@ -0,0 +1,4 @@ +before_install: + - sudo add-apt-repository -y ppa:ubuntu-cloud-archive/icehouse-staging + +install: bin/build From ad3bfaed00984679d684aeb7aa966083dc2bbe7d Mon Sep 17 00:00:00 2001 From: Giulio Eulisse Date: Wed, 3 Dec 2014 23:55:06 +0100 Subject: [PATCH 2/4] Move to mesos 0.20.1 and ES 1.4.1 --- pom.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index ad7f54c..c776ef8 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ ${elasticsearch.version}-${mesos.version}-${elasticsearch-mesos.version} - 0.19.0-1 + 0.20.1-1 1 @@ -17,8 +17,8 @@ 1.7 2.10.4 - 1.2.1 - 0.19.0 + 1.4.1 + 0.20.1 1.0.9 From 72b20330a45e49ba7707001c2de0cfbeceedf886 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse Date: Mon, 8 Dec 2014 11:37:08 +0100 Subject: [PATCH 3/4] Offers selection based on attributes as well. --- .../ElasticSearchScheduler.scala | 23 ++++++++++++++++--- .../scala/mesosphere/elasticsearch/Main.scala | 9 +++++++- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/src/main/scala/mesosphere/elasticsearch/ElasticSearchScheduler.scala b/src/main/scala/mesosphere/elasticsearch/ElasticSearchScheduler.scala index f0ea139..92e23c9 100644 --- a/src/main/scala/mesosphere/elasticsearch/ElasticSearchScheduler.scala +++ b/src/main/scala/mesosphere/elasticsearch/ElasticSearchScheduler.scala @@ -20,7 +20,8 @@ class ElasticSearchScheduler(masterUrl: String, confServerHostName: String, confServerPort: Int, resources: mutable.Map[String, Float], - numberOfHwNodes: Int) + numberOfHwNodes: Int, + requestedAttributes: mutable.Map[String, String]) extends Scheduler with Runnable with Logger { val initialized = new CountDownLatch(1) @@ -141,6 +142,22 @@ class ElasticSearchScheduler(masterUrl: String, // Check if offer is reasonable def isOfferGood(offer: Offer) = { + // First of all check if we have all the requested + // attributes in the offer. + // To do so we make sure the intersection of the + // offered and required attributes is actually + // equal to the required attributes themselves. + val offeredAttributes = offer.getAttributesList.asScala.toList.map { + k => (k.getName, k.getText.getValue) + }.toSet + + val requiredAttrs = requestedAttributes.toSet + val hasAllAttributes = offeredAttributes.intersect(requiredAttrs) == requiredAttrs + + debug("attributes offered: " + offeredAttributes) + debug("attributes required: " + requiredAttrs) + + // Make a list of offered resources val offeredRes = offer.getResourcesList.asScala.toList.map { k => (k.getName, k.getScalar.getValue) @@ -170,8 +187,8 @@ class ElasticSearchScheduler(masterUrl: String, }.size // don't start the same framework multiple times on the same host and - // make sure we got all resources we asked for - taskSet.forall(_.hostname != offer.getHostname) && offersTooSmall == 0 + // make sure we got all resources and attributes we asked for + taskSet.forall(_.hostname != offer.getHostname) && offersTooSmall == 0 && hasAllAttributes } def reregistered(driver: SchedulerDriver, masterInfo: MasterInfo) { diff --git a/src/main/scala/mesosphere/elasticsearch/Main.scala b/src/main/scala/mesosphere/elasticsearch/Main.scala index fa32f3d..1330bdd 100644 --- a/src/main/scala/mesosphere/elasticsearch/Main.scala +++ b/src/main/scala/mesosphere/elasticsearch/Main.scala @@ -43,6 +43,12 @@ object Main extends App with Logger { case (k, v) => k.replaceAllLiterally("resource.", "") -> v.toString.toFloat } + val requestedAttributes = mesosConf.filter { + _._1.startsWith("attribute.") + }.map { + case (k, v) => k.replaceAllLiterally("attribute.", "") -> v.toString + } + //TODO load the ElasticSearch log-properties file BasicConfigurator.configure() getRootLogger.setLevel(Level.INFO) @@ -55,7 +61,8 @@ object Main extends App with Logger { confServerHostName, confServerPort, resources, - numberOfHwNodes) + numberOfHwNodes, + requestedAttributes) val schedThred = new Thread(scheduler) schedThred.start() From b83c01b7a9aa41f349d4fadbcab02290194a6810 Mon Sep 17 00:00:00 2001 From: Giulio Eulisse Date: Mon, 8 Dec 2014 14:05:35 +0100 Subject: [PATCH 4/4] Drop -f option since not supported by new elasticsearch. --- .../scala/mesosphere/elasticsearch/ElasticSearchScheduler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/scala/mesosphere/elasticsearch/ElasticSearchScheduler.scala b/src/main/scala/mesosphere/elasticsearch/ElasticSearchScheduler.scala index 92e23c9..283d61b 100644 --- a/src/main/scala/mesosphere/elasticsearch/ElasticSearchScheduler.scala +++ b/src/main/scala/mesosphere/elasticsearch/ElasticSearchScheduler.scala @@ -90,7 +90,7 @@ class ElasticSearchScheduler(masterUrl: String, s"&& rm logging.yml " + s"&& curl -sSfLO http://${confServerHostName}:${confServerPort}/logging.yml " + s"&& cd .. " + - s"&& bin/elasticsearch -f") + s"&& bin/elasticsearch")