From 5310c8d070687746eda95ef23359f5278c3ca568 Mon Sep 17 00:00:00 2001 From: Ali Rasim Kocal Date: Thu, 7 Jun 2018 13:14:35 +0200 Subject: [PATCH 1/3] Rewrite Dockerfile and merge both images Also refactored deploy script. --- .dockerignore | 2 ++ Dockerfile | 33 +++++++++++++++++++-- Dockerfile-gearpump | 12 -------- bootstrap.sh | 6 ++++ deployer/src/app.py | 60 +++++++++++++++++++++------------------ local-deploy.sh | 5 ++-- local-push-to-gearpump.sh | 3 +- 7 files changed, 74 insertions(+), 47 deletions(-) create mode 100644 .dockerignore delete mode 100644 Dockerfile-gearpump create mode 100644 bootstrap.sh diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9a715f5 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,2 @@ +.dockerignore +Dockerfile \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 364ea41..f12a16c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,12 +2,39 @@ FROM jamesdbloom/docker-java8-maven RUN apt-get update -qq && apt-get install -y build-essential python-setuptools +ADD pom.xml /app/pom.xml +ADD src /app/src + +WORKDIR /app + +RUN mvn clean install -DskipTests + + + + +FROM ubuntu:xenial +RUN apt-get update -qq && apt-get upgrade -y + +RUN apt-get install -y curl unzip openjdk-8-jre-headless python-setuptools + +RUN curl --location --retry 3 --insecure https://github.com/gearpump/gearpump/releases/download/0.8.0/gearpump-2.11-0.8.0.zip -o tmp.zip && unzip -q tmp.zip && rm tmp.zip && chmod +x gearpump-2.11-0.8.0/bin/* + +ADD gear.conf gearpump-2.11-0.8.0/conf/gear.conf + +RUN sed -i -e "s/gearpump.root.logger=RollingFileAppender/gearpump.root.logger=RollingFileAppender,console/g" gearpump-2.11-0.8.0/conf/log4j.properties +RUN sed -i -e "s/gearpump.application.logger=ApplicationLogAppender/gearpump.application.logger=ApplicationLogAppender,console/g" gearpump-2.11-0.8.0/conf/log4j.properties + +EXPOSE 8090 + RUN easy_install poster RUN easy_install requests RUN easy_install pip +RUN pip install kafka-python ADD . /app +COPY --from=0 /app/ /app +RUN chmod +x /app/bootstrap.sh -WORKDIR /app - -RUN mvn clean install -DskipTests +WORKDIR /app/ +# CMD ["bash", "bootstrap.sh"] +CMD bash \ No newline at end of file diff --git a/Dockerfile-gearpump b/Dockerfile-gearpump deleted file mode 100644 index d54c5a6..0000000 --- a/Dockerfile-gearpump +++ /dev/null @@ -1,12 +0,0 @@ -FROM errordeveloper/oracle-jre - -RUN curl --location --retry 3 --insecure https://github.com/gearpump/gearpump/releases/download/0.8.0/gearpump-2.11-0.8.0.zip -o tmp.zip && unzip -q tmp.zip && rm tmp.zip && chmod +x gearpump-2.11-0.8.0/bin/* - -ADD gear.conf gearpump-2.11-0.8.0/conf/gear.conf - -RUN sed -i -e "s/gearpump.root.logger=RollingFileAppender/gearpump.root.logger=RollingFileAppender,console/g" gearpump-2.11-0.8.0/conf/log4j.properties -RUN sed -i -e "s/gearpump.application.logger=ApplicationLogAppender/gearpump.application.logger=ApplicationLogAppender,console/g" gearpump-2.11-0.8.0/conf/log4j.properties - -EXPOSE 8090 - -ENTRYPOINT gearpump-2.11-0.8.0/bin/local & gearpump-2.11-0.8.0/bin/services diff --git a/bootstrap.sh b/bootstrap.sh new file mode 100644 index 0000000..1f72dda --- /dev/null +++ b/bootstrap.sh @@ -0,0 +1,6 @@ +source /app/target/classes/resources/version.properties && +export RULE_ENGINE_PACKAGE_NAME=gearpump-rule-engine-${VERSION}-jar-with-dependencies.jar && + +/app/wait-for-it.sh localhost:8090 -t 300 -- /app/local-deploy.sh & + +/gearpump-2.11-0.8.0/bin/local & /gearpump-2.11-0.8.0/bin/services diff --git a/deployer/src/app.py b/deployer/src/app.py index 6b8fb4b..ee8c554 100644 --- a/deployer/src/app.py +++ b/deployer/src/app.py @@ -1,4 +1,4 @@ -# Copyright (c) 2015 Intel Corporation +# Copyright (c) 2015-2018 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -11,46 +11,52 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +"""Deploy gearpump application locally - -import argparse +Application is a jar located at $RULE_ENGINE_PACKAGE_NAME, +Gearpump must run locally, port number is read from $GEARPUMP, +which stores a whole URL. +""" import os -import time - -import cloudfoundry_bridge from gearpump_api import GearpumpApi - from kafka import KafkaConsumer -def main(): - - parser = argparse.ArgumentParser() - parser.add_argument("--local", action="store_true") - args = parser.parse_args() - - cloud_bridge = cloudfoundry_bridge.CloudfoundryBridge() - - config = cloud_bridge.build_config(local=args.local) - - gearpump_api = GearpumpApi(uri=cloud_bridge.gearpump_dashboard_url, credentials=cloud_bridge.gearpump_credentials) +import cloudfoundry_bridge - rule_engine_jar_name = os.environ['RULE_ENGINE_PACKAGE_NAME'] +def wait_for_frontend(): + """Wait for OISP frontend hearbeat.""" + kafka_server = os.environ["KAFKA"] + heartbeat_topic = os.environ["KAFKA_HEARTBEAT_TOPIC"] + consumer = KafkaConsumer(heartbeat_topic, bootstrap_servers=kafka_server, + auto_offset_reset='latest') - consumer = KafkaConsumer(config['kafka_heartbeat_topic'], bootstrap_servers=config['kafka_servers'], auto_offset_reset='latest') - for message in consumer: + # Frontend heartbeat message is dashboard for historical reasons if message.value == "dashboard": + break - print 'Submitting application - %s into gearpump ...' % rule_engine_jar_name + print("Frontend is up") - gearpump_api.submit_app(filename=rule_engine_jar_name, app_name=config['application_name'], gearpump_app_config=config, force=True) - break +def main(): + """Deploy app to local gearpump instance.""" + rule_engine_jar_name = os.environ['RULE_ENGINE_PACKAGE_NAME'] + + # Cloudfoundry needs frontend + wait_for_frontend() + cloud_bridge = cloudfoundry_bridge.CloudfoundryBridge() + config = cloud_bridge.build_config(local=True) + + # We are only interested in port number because we deploy locally + gearpump_port = os.environ["GEARPUMP"].split(":")[1] + gearpump_api = GearpumpApi(uri="http://localhost:{}".format(gearpump_port), + credentials=cloud_bridge.gearpump_credentials) + print("Submitting application '{}' into gearpump ...".format(rule_engine_jar_name)) + gearpump_api.submit_app(filename=rule_engine_jar_name, + app_name=config['application_name'], + gearpump_app_config=config, force=True) - if not args.local: - while True: - time.sleep(60) if __name__ == "__main__": main() diff --git a/local-deploy.sh b/local-deploy.sh index 41eafc9..1dbac11 100755 --- a/local-deploy.sh +++ b/local-deploy.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (c) 2016 Intel Corporation +# Copyright (c) 2016-2018 Intel Corporation # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,8 +14,7 @@ # limitations under the License. # -mvn clean install && source target/classes/resources/version.properties && export RULE_ENGINE_PACKAGE_NAME=gearpump-rule-engine-${VERSION}-jar-with-dependencies.jar && cd deployer && -python src/app.py --local \ No newline at end of file +python src/app.py diff --git a/local-push-to-gearpump.sh b/local-push-to-gearpump.sh index 845c004..2fd8476 100755 --- a/local-push-to-gearpump.sh +++ b/local-push-to-gearpump.sh @@ -17,5 +17,4 @@ source target/classes/resources/version.properties && export RULE_ENGINE_PACKAGE_NAME=gearpump-rule-engine-${VERSION}-jar-with-dependencies.jar && cd deployer && -pip install kafka-python && -python src/app.py --local \ No newline at end of file +python src/app.py --local From b55e11a672fb07580f8d7524f53543ec29318644 Mon Sep 17 00:00:00 2001 From: Ali Rasim Kocal Date: Wed, 20 Jun 2018 13:14:24 +0200 Subject: [PATCH 2/3] Update deployer script to wait for Kafka brokers --- deployer/src/app.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/deployer/src/app.py b/deployer/src/app.py index ee8c554..417271c 100644 --- a/deployer/src/app.py +++ b/deployer/src/app.py @@ -18,22 +18,31 @@ which stores a whole URL. """ import os +import time -from gearpump_api import GearpumpApi -from kafka import KafkaConsumer +import kafka +from gearpump_api import GearpumpApi import cloudfoundry_bridge +KAFKA_BROKER_TIMEOUT = 300 + def wait_for_frontend(): """Wait for OISP frontend hearbeat.""" kafka_server = os.environ["KAFKA"] heartbeat_topic = os.environ["KAFKA_HEARTBEAT_TOPIC"] - consumer = KafkaConsumer(heartbeat_topic, bootstrap_servers=kafka_server, - auto_offset_reset='latest') + + for _ in range(KAFKA_BROKER_TIMEOUT): + try: + consumer = kafka.KafkaConsumer(heartbeat_topic, bootstrap_servers=kafka_server, + auto_offset_reset='latest') + except kafka.errors.NoBrokersAvailable: + time.sleep(1) for message in consumer: # Frontend heartbeat message is dashboard for historical reasons if message.value == "dashboard": + print("Time, dashboard message ts:", time.time(), message.timestamp) break print("Frontend is up") From c92fc7fd76fc037c912483456e59e2812482cb50 Mon Sep 17 00:00:00 2001 From: Ali Rasim Kocal Date: Fri, 28 Sep 2018 11:05:11 +0200 Subject: [PATCH 3/3] Update deployer script, minor bugfixes Script used to reconnect to kafka even after succesful connection. Gearpump app name was not normalized correctly if dashboard adress contained explicit port number. --- deployer/src/app.py | 6 +++++- deployer/src/vcap.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/deployer/src/app.py b/deployer/src/app.py index 417271c..fe9a670 100644 --- a/deployer/src/app.py +++ b/deployer/src/app.py @@ -31,14 +31,18 @@ def wait_for_frontend(): """Wait for OISP frontend hearbeat.""" kafka_server = os.environ["KAFKA"] heartbeat_topic = os.environ["KAFKA_HEARTBEAT_TOPIC"] - + print("Connecting to kafka at {}, topic: {} ".format(kafka_server, heartbeat_topic)) for _ in range(KAFKA_BROKER_TIMEOUT): try: consumer = kafka.KafkaConsumer(heartbeat_topic, bootstrap_servers=kafka_server, auto_offset_reset='latest') + break except kafka.errors.NoBrokersAvailable: + print("No kafka brokers available, trying again") time.sleep(1) + print("Connected to kafka") + for message in consumer: # Frontend heartbeat message is dashboard for historical reasons if message.value == "dashboard": diff --git a/deployer/src/vcap.py b/deployer/src/vcap.py index fc3a194..701ce45 100644 --- a/deployer/src/vcap.py +++ b/deployer/src/vcap.py @@ -66,7 +66,7 @@ def __parse_dashboard_url(self, ups): .replace("http://", "") \ .replace("https://", "") \ .replace("-", "_") \ - .replace(".", "_") + .replace(".", "_").split(":")[0] def __gather_topics_names(self, ups): kafka_ups = self.__get_ups_by_name(ups, 'kafka-ups')