From ac3641f9eab7d4db7471ffb0ce642b40a7521e5d Mon Sep 17 00:00:00 2001 From: David Quintanel Date: Fri, 18 Oct 2024 10:14:14 +0200 Subject: [PATCH 01/12] master and builder --- docker/build_kraken_deb11.sh | 16 +++++++++ docker/debian11/Dockerfile-builder-kraken | 10 ++++++ docker/debian11/Dockerfile-master | 42 +++++++++++++++++++++++ source/kraken/CMakeLists.txt | 2 +- source/monitor/monitor_kraken/app.py | 8 +++-- 5 files changed, 75 insertions(+), 3 deletions(-) create mode 100644 docker/build_kraken_deb11.sh create mode 100644 docker/debian11/Dockerfile-builder-kraken create mode 100644 docker/debian11/Dockerfile-master diff --git a/docker/build_kraken_deb11.sh b/docker/build_kraken_deb11.sh new file mode 100644 index 0000000000..36cb7926cd --- /dev/null +++ b/docker/build_kraken_deb11.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +cd /navitia/navitia/ +mkdir -p docker_build && cd docker_build +cmake -DCMAKE_BUILD_TYPE=Release ../source +make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) protobuf_files +make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) kraken + +# Build libkeepalive https://libkeepalive.sourceforge.net/ +# libkeepalive is a library to keep tcp connection alive. The reason of doing so is that aws LoadBalancer, to which +# kraken is connected, check the aliveness continuously at regular interval of non-customisable 350s +# (https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html). If nothing happens during +# this interval, the connection will be dropped. It takes Kraken, by default, 7200s(/proc/sys/net/ipv4/tcp_keepalive_time) +# to be aware of the disconnection. By using libkeepalive, we can override tcp_keepalive_time, +# which is not customisable via docker-container. +(mkdir -p build_libkeepalive && cp -rf ../docker/libkeepalive-0.3/. build_libkeepalive/ && cd build_libkeepalive/ && make ) diff --git a/docker/debian11/Dockerfile-builder-kraken b/docker/debian11/Dockerfile-builder-kraken new file mode 100644 index 0000000000..957817b3e8 --- /dev/null +++ b/docker/debian11/Dockerfile-builder-kraken @@ -0,0 +1,10 @@ +FROM 162230498103.dkr.ecr.eu-west-1.amazonaws.com/mutable-debian11_dev:latest + +RUN apt-get update && apt-get install -y git wget + +RUN git config --global --add safe.directory /navitia/navitia + +COPY docker/build_kraken_deb11.sh /build_navitia.sh +RUN chmod +x /build_navitia.sh + +ENTRYPOINT ["bash", "/build_navitia.sh"] diff --git a/docker/debian11/Dockerfile-master b/docker/debian11/Dockerfile-master new file mode 100644 index 0000000000..9b8eab2ba9 --- /dev/null +++ b/docker/debian11/Dockerfile-master @@ -0,0 +1,42 @@ +FROM 162230498103.dkr.ecr.eu-west-1.amazonaws.com/mutable-debian11_dev:latest + +RUN echo "deb http://archive.debian.org/debian/ bullseye main" > /etc/apt/sources.list.d/archive.list + +# update package list from providers +RUN apt-get update --force-yes --fix-missing || exit 0 + +# install postgresql-client for tyr-beat +# netcat for kraken +# curl for jormun +RUN apt-get install -y --force-yes ca-certificates \ + postgresql-client \ + netcat \ + dh-python \ + curl \ + wget + +COPY docker/ca-certificates/*.crt /usr/local/share/ca-certificates/ +RUN update-ca-certificates + +# install a more recent version of CMake +RUN apt-get remove --yes cmake \ + && cd /tmp \ + && wget https://cmake.org/files/v3.18/cmake-3.18.6-Linux-x86_64.tar.gz \ + && tar xf cmake-3.18.6-Linux-x86_64.tar.gz --strip-components=1 --directory=/usr/local \ + && rm -f cmake-3.18.6-Linux-x86_64.tar.gz \ + && cd - + +# Python 'requests' package handle its own CA certificate list +# Let's force it to use the OS's list +ENV REQUESTS_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt + +# install rustup +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo + +ENV PATH=/usr/local/cargo/bin:${PATH} +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --profile minimal \ + && chmod -R a+wrX ${RUSTUP_HOME} ${CARGO_HOME} \ + && rustup --version \ + && cargo --version \ + && rustc --version diff --git a/source/kraken/CMakeLists.txt b/source/kraken/CMakeLists.txt index 6a96920024..09a4b850cb 100644 --- a/source/kraken/CMakeLists.txt +++ b/source/kraken/CMakeLists.txt @@ -36,7 +36,7 @@ target_link_libraries(fill_disruption_from_database add_executable(kraken kraken_zmq.cpp) target_link_libraries(kraken workers ${NAVITIA_ALLOCATOR} ${Boost_THREAD_LIBRARY}) -add_dependencies(kraken protobuf_files) +add_dependencies(kraken protobuf_files prometheus-cpp::pull) install(TARGETS kraken DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) diff --git a/source/monitor/monitor_kraken/app.py b/source/monitor/monitor_kraken/app.py index b70b7da0ed..5fd51295ef 100644 --- a/source/monitor/monitor_kraken/app.py +++ b/source/monitor/monitor_kraken/app.py @@ -29,7 +29,6 @@ from flask import Flask, request import json -from ConfigParser import ConfigParser import zmq import os @@ -37,6 +36,11 @@ from monitor_kraken import response_pb2 from monitor_kraken import type_pb2 +try: + import ConfigParser +except ImportError: + import configparser as ConfigParser # type: ignore + app = Flask(__name__) # config used in monitor() for several kraken app.config.from_object('monitor_kraken.default_settings') @@ -217,4 +221,4 @@ def request_kraken_zmq_status(zmq_socket, zmq_timeout_in_ms): if __name__ == '__main__': - app.run(host=os.environ.get("MONITOR_HOST", "127.0.0.1")) + app.run(host=os.environ.get("MONITOR_HOST", "0.0.0.0")) From 6741e03b82e1fa639edfcf7f2f905534089e1a7a Mon Sep 17 00:00:00 2001 From: David Quintanel Date: Fri, 18 Oct 2024 10:15:44 +0200 Subject: [PATCH 02/12] kraken --- docker/debian11/Dockerfile-kraken | 51 +++++++++++++++++++++++++++++++ docker/run_kraken_deb11.sh | 3 ++ 2 files changed, 54 insertions(+) create mode 100644 docker/debian11/Dockerfile-kraken create mode 100644 docker/run_kraken_deb11.sh diff --git a/docker/debian11/Dockerfile-kraken b/docker/debian11/Dockerfile-kraken new file mode 100644 index 0000000000..8712d6f2c7 --- /dev/null +++ b/docker/debian11/Dockerfile-kraken @@ -0,0 +1,51 @@ +FROM debian:bullseye-slim + +WORKDIR /srv/kraken + +COPY ./docker/ca-certificates/*.crt /usr/local/share/ca-certificates/ +COPY ./source/monitor /srv/monitor_kraken + +RUN apt-get update --fix-missing \ + && apt-get install -y ca-certificates \ + && update-ca-certificates \ + && apt-get install -y libgoogle-perftools4 \ + python3.9 \ + python3-pip \ + libboost-all-dev \ + libprotobuf-dev \ + liblog4cplus-2.0.5 \ + libzmq3-dev \ + libpqxx-dev \ + netcat \ + && (cd /srv/monitor_kraken && pip3 install --no-cache-dir -r requirements.txt && python3 setup.py install) \ + && apt-get purge -y \ + && apt-get purge -y python3-pip \ + && apt-get autoremove -y + +RUN ln -s /usr/local/lib/python*/dist-packages/monitor_kraken/app.py monitor_kraken + +COPY docker/run_kraken_deb11.sh /srv/kraken/run_kraken.sh + +RUN chmod +x /srv/kraken/run_kraken.sh + +COPY ./docker_build/kraken/kraken /usr/bin/kraken +RUN chmod +x /usr/bin/kraken + +RUN mkdir /libkeepalive +COPY ./docker_build/build_libkeepalive/libkeepalive.so /libkeepalive/. + +# User timeout = 300 + 30 * 6 = 480s +ENV KEEPIDLE=300 +ENV KEEPCNT=6 +ENV KEEPINTVL=30 +# if you are willing to use settings from os, set KEEPALIVE to off +# ENV KEEPALIVE=Off + +ENV KRAKEN_GENERAL_zmq_socket=tcp://0.0.0.0:30000 +ENV KRAKEN_GENERAL_log_level=INFO +HEALTHCHECK CMD nc -z localhost 30000 || exit 1 +EXPOSE 30000 +EXPOSE 5000 + + +ENTRYPOINT ["bash", "/srv/kraken/run_kraken.sh"] diff --git a/docker/run_kraken_deb11.sh b/docker/run_kraken_deb11.sh new file mode 100644 index 0000000000..51a59f0e7c --- /dev/null +++ b/docker/run_kraken_deb11.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +python3 monitor_kraken & LD_PRELOAD=/libkeepalive/libkeepalive.so /usr/bin/kraken From c54ae6177bc32f29060965800f6943b348e35cba Mon Sep 17 00:00:00 2001 From: David Quintanel Date: Fri, 18 Oct 2024 10:15:59 +0200 Subject: [PATCH 03/12] tyr-beat --- docker/debian11/Dockerfile-tyr-beat | 13 ++++++------- docker/run_tyr_beat.sh | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/docker/debian11/Dockerfile-tyr-beat b/docker/debian11/Dockerfile-tyr-beat index 9ea3a6bf3f..4a58290269 100644 --- a/docker/debian11/Dockerfile-tyr-beat +++ b/docker/debian11/Dockerfile-tyr-beat @@ -5,22 +5,21 @@ WORKDIR /usr/share/tyr/ COPY ./source/navitiacommon ./navitiacommon COPY ./source/tyr ./tyr COPY ./docker/ca-certificates/*.crt /usr/local/share/ca-certificates/ -COPY ./docker/run_tyr_beat.sh ./run.sh +COPY ./docker/run_tyr_beat.sh /run.sh COPY ./source/sql/alembic /usr/share/navitia/ed/alembic COPY ./source/sql/requirements.txt /tmp/requirements.txt COPY ./source/cities /usr/share/navitia/cities COPY ./docker/templates/* /templates/ +RUN chmod +x /run.sh + RUN apt-get update --fix-missing \ - && apt-get install -y libpq5 python3.9-dev python3-pip git ca-certificates postgresql-client gettext-base jq \ + && apt-get install -y --force-yes libgeos-dev libpq5 python3.9-dev python3-pip git ca-certificates postgresql-client gettext-base jq \ && update-ca-certificates \ && (cd navitiacommon && python3 setup.py install) \ && (cd tyr && python3 setup.py install && pip3 install --no-cache-dir -U -r requirements.txt)\ && pip3 install --no-cache-dir -U -r /tmp/requirements.txt \ - && rm -rf navitiacommon tyr \ - && apt-get purge -y \ - python3-pip \ - git \ + && apt-get purge -y git \ && apt-get autoremove -y -ENTRYPOINT ["bash", "/usr/share/tyr/run.sh"] +ENTRYPOINT ["bash", "/run.sh"] diff --git a/docker/run_tyr_beat.sh b/docker/run_tyr_beat.sh index d3a959df77..302f0eb4b0 100644 --- a/docker/run_tyr_beat.sh +++ b/docker/run_tyr_beat.sh @@ -63,4 +63,4 @@ while read var ; do done < <(env) -exec celery beat -A tyr.tasks +exec celery -A tyr.tasks beat From 0a48445a98caa9cf7690689b3e741a49d90bef9a Mon Sep 17 00:00:00 2001 From: David Quintanel Date: Fri, 18 Oct 2024 10:17:20 +0200 Subject: [PATCH 04/12] tyr-web --- docker/debian11/Dockerfile-tyr-web | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/docker/debian11/Dockerfile-tyr-web b/docker/debian11/Dockerfile-tyr-web index d55cce6153..9a8e8b8972 100644 --- a/docker/debian11/Dockerfile-tyr-web +++ b/docker/debian11/Dockerfile-tyr-web @@ -7,14 +7,14 @@ COPY ./source/tyr ./tyr COPY ./docker/ca-certificates/*.crt /usr/local/share/ca-certificates/ COPY ./docker/run_tyr_web.sh /usr/src/app/run.sh +RUN apt-get update --force-yes --fix-missing || exit 0 -RUN apt-get update --fix-missing \ - && apt-get install -y curl libpq5 python3.9-dev python3-pip git ca-certificates libgeos-c1v5 postgresql-client \ +RUN apt-get install -y --force-yes curl libpq5 python3.9-dev python3-pip git ca-certificates libgeos-c1v5 postgresql-client \ && update-ca-certificates \ && (cd navitiacommon && python3 setup.py install) \ && (cd tyr && python3 setup.py install && pip3 install --no-cache-dir -U -r requirements.txt)\ - && pip3 install --no-cache-dir uwsgi==2.0.21 \ - && rm -rf navitiacommon tyr \ + && pip3 install --no-cache-dir uwsgi==2.0.22 \ + && pip3 install --no-cache-dir -r /usr/share/tyr/requirements.txt \ && chmod +x /usr/src/app/run.sh \ && ln -sf /usr/share/tyr/migrations migrations \ && ln -s /usr/bin/python3.9 /usr/bin/python \ @@ -24,9 +24,6 @@ RUN apt-get update --fix-missing \ git \ && apt-get autoremove -y - - # Python 'requests' package handle its own CA certificate list - # Let's force it to use the OS's list - ENV REQUESTS_CA_BUNDLE /etc/ssl/certs +ENV REQUESTS_CA_BUNDLE=/etc/ssl/certs ENTRYPOINT ["bash", "/usr/src/app/run.sh" ] From 517847db89ce7eac2c11c87f810d2696baedd0b6 Mon Sep 17 00:00:00 2001 From: David Quintanel Date: Fri, 18 Oct 2024 10:17:37 +0200 Subject: [PATCH 05/12] tyr-worker --- docker/build_kraken_deb11.sh | 8 ++++ docker/debian11/Dockerfile-tyr-worker | 66 +++++++++++++++++++++++++++ docker/run_tyr_worker.sh | 2 +- source/tyr/tyr/binarisation.py | 2 +- source/tyr/tyr/helper.py | 2 +- 5 files changed, 77 insertions(+), 3 deletions(-) create mode 100644 docker/debian11/Dockerfile-tyr-worker diff --git a/docker/build_kraken_deb11.sh b/docker/build_kraken_deb11.sh index 36cb7926cd..54de523acd 100644 --- a/docker/build_kraken_deb11.sh +++ b/docker/build_kraken_deb11.sh @@ -5,6 +5,14 @@ mkdir -p docker_build && cd docker_build cmake -DCMAKE_BUILD_TYPE=Release ../source make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) protobuf_files make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) kraken +make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) cities +make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) ed2nav +make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) fare2ed +make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) fusio2ed +make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) geopal2ed +make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) gtfs2ed +make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) osm2ed +make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) poi2ed # Build libkeepalive https://libkeepalive.sourceforge.net/ # libkeepalive is a library to keep tcp connection alive. The reason of doing so is that aws LoadBalancer, to which diff --git a/docker/debian11/Dockerfile-tyr-worker b/docker/debian11/Dockerfile-tyr-worker new file mode 100644 index 0000000000..e26410cae3 --- /dev/null +++ b/docker/debian11/Dockerfile-tyr-worker @@ -0,0 +1,66 @@ +# Compilation of rust binaries from tartare-tools +FROM rust:1.82-bullseye AS tartare-tools + +ARG TARTARE_TOOLS_VERSION="v0.47.1" +ARG GITHUB_TOKEN +RUN git config --global url."https://x-access-token:${GITHUB_TOKEN}@github.com/hove-io/".insteadOf "ssh://git@github.com/hove-io/" +RUN git clone -b ${TARTARE_TOOLS_VERSION} --depth 1 https://x-access-token:${GITHUB_TOKEN}@github.com/hove-io/tartare-tools + +RUN cd tartare-tools \ + && cargo build --release -p enrich-ntfs-with-addresses \ + && cargo build --release -p split-trip-geometries \ + && cp target/release/enrich-ntfs-with-addresses /usr/bin/ \ + && cp target/release/split-trip-geometries /usr/bin/ \ + && cd .. \ + && rm -rf tartare-tools + +FROM debian:bullseye-slim + +WORKDIR /usr/src/app/ + +COPY ./source/navitiacommon ./navitiacommon +COPY ./source/tyr ./tyr +COPY ./docker/run_tyr_worker.sh /usr/src/app/run.sh +RUN chmod +x /usr/src/app/run.sh + +COPY mimirsbrunn7_bullseye-*.deb cosmogony2cities_*.deb mimirsbrunn-config-*.deb ./ + +RUN apt-get update --fix-missing \ + && apt-get install -y --force-yes unzip curl libgeos-dev libpq5 python3.9 python3-pip git ca-certificates postgresql-client gettext-base jq \ + libboost-all-dev \ + libprotobuf-dev \ + liblog4cplus-2.0.5 \ + libzmq3-dev \ + libpqxx-dev \ + libgoogle-perftools4 \ + libosmpbf-dev \ + libproj-dev \ + ./mimirsbrunn7_bullseye-*.deb ./cosmogony2cities_*.deb ./mimirsbrunn-config-*.deb \ + && update-ca-certificates \ + && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \ + && unzip awscliv2.zip && ./aws/install \ + && (cd navitiacommon && python3 setup.py install) \ + && (cd tyr && python3 setup.py install && pip3 install --no-cache-dir -U -r requirements.txt)\ + && rm -rf ./mimirsbrunn7_bullseye-*.deb ./cosmogony2cities*.deb ./mimirsbrunn-config-*.deb \ + && apt-get purge -y git python3-pip \ + && apt-get autoremove -y + +# binaries from tartare-tools +COPY --from=tartare-tools /usr/bin/enrich-ntfs-with-addresses /usr/bin/ +COPY --from=tartare-tools /usr/bin/split-trip-geometries /usr/bin/ + +# binaries from navitia +COPY ./docker_build/cities/cities /usr/bin/cities +COPY ./docker_build/ed/ed2nav /usr/bin/ed2nav +COPY ./docker_build/ed/fare2ed /usr/bin/fare2ed +COPY ./docker_build/ed/fusio2ed /usr/bin/fusio2ed +COPY ./docker_build/ed/geopal2ed /usr/bin/geopal2ed +COPY ./docker_build/ed/gtfs2ed /usr/bin/gtfs2ed +COPY ./docker_build/ed/osm2ed /usr/bin/osm2ed +COPY ./docker_build/ed/poi2ed /usr/bin/poi2ed + +# TODO change the user to remove this ugly C_FORCE_ROOT +ENV C_FORCE_ROOT=1 +ENV TYR_WORKER_N_PROC=1 + +ENTRYPOINT ["bash", "/usr/src/app/run.sh"] diff --git a/docker/run_tyr_worker.sh b/docker/run_tyr_worker.sh index 8b4df8212b..34f076e7e3 100644 --- a/docker/run_tyr_worker.sh +++ b/docker/run_tyr_worker.sh @@ -1,3 +1,3 @@ #!/bin/bash -celery worker -A tyr.tasks -O fair -c $TYR_WORKER_N_PROC +celery -A tyr.tasks worker -O fair -c $TYR_WORKER_N_PROC diff --git a/source/tyr/tyr/binarisation.py b/source/tyr/tyr/binarisation.py index c532f21f57..e8801e41b8 100644 --- a/source/tyr/tyr/binarisation.py +++ b/source/tyr/tyr/binarisation.py @@ -184,7 +184,7 @@ def wrapper(*args, **kwargs): logging.debug('args: %s -- kwargs: %s', args, kwargs) job = models.Job.query.get(job_id) logger = get_instance_logger(job.instance, task_id=job_id) - task = args[func.func_code.co_varnames.index('self')] + task = args[func.__code__.co_varnames.index('self')] try: lock = redis.lock('tyr.lock|' + job.instance.name, timeout=self.timeout) locked = lock.acquire(blocking=False) diff --git a/source/tyr/tyr/helper.py b/source/tyr/tyr/helper.py index bdeae1ffe2..c9e8d1d609 100644 --- a/source/tyr/tyr/helper.py +++ b/source/tyr/tyr/helper.py @@ -337,7 +337,7 @@ def get_named_arg(arg_name, func, args, kwargs): if kwargs and arg_name in kwargs: return kwargs[arg_name] else: - idx = func.func_code.co_varnames.index(arg_name) + idx = func.__code__.co_varnames.index(arg_name) if args and idx < len(args): return args[idx] else: From c58d0f5d6655dfebab82e57e4d9c75e184355ccb Mon Sep 17 00:00:00 2001 From: David Quintanel Date: Mon, 21 Oct 2024 17:15:45 +0200 Subject: [PATCH 06/12] mock-kraken --- docker/build_kraken_deb11.sh | 8 ++++++++ docker/debian11/Dockerfile-mock-kraken | 24 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 docker/debian11/Dockerfile-mock-kraken diff --git a/docker/build_kraken_deb11.sh b/docker/build_kraken_deb11.sh index 54de523acd..ab494b55d2 100644 --- a/docker/build_kraken_deb11.sh +++ b/docker/build_kraken_deb11.sh @@ -3,8 +3,16 @@ cd /navitia/navitia/ mkdir -p docker_build && cd docker_build cmake -DCMAKE_BUILD_TYPE=Release ../source + +# kraken make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) protobuf_files make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) kraken + +# mocks for integration tests +make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) departure_board_test +make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) main_routing_test + +# some binaries for tyr-worker make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) cities make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) ed2nav make -j$(($(grep -c '^processor' /proc/cpuinfo)+1)) fare2ed diff --git a/docker/debian11/Dockerfile-mock-kraken b/docker/debian11/Dockerfile-mock-kraken new file mode 100644 index 0000000000..7e9d67a1f7 --- /dev/null +++ b/docker/debian11/Dockerfile-mock-kraken @@ -0,0 +1,24 @@ +FROM debian:bullseye-slim + +WORKDIR /srv/kraken + +RUN apt-get update --fix-missing \ + && apt-get install -y libgoogle-perftools4 \ + libboost-all-dev \ + libprotobuf-dev \ + liblog4cplus-2.0.5 \ + libzmq3-dev \ + libpqxx-dev \ + netcat \ + && apt-get purge -y \ + && apt-get autoremove -y + +COPY ./docker_build/tests/mock-kraken/departure_board_test /srv/kraken/departure_board_test +RUN chmod +x /srv/kraken/departure_board_test + +COPY ./docker_build/tests/mock-kraken/main_routing_test /srv/kraken/main_routing_test +RUN chmod +x /srv/kraken/main_routing_test + +EXPOSE 30000 +ENV KRAKEN_GENERAL_zmq_socket=tcp://*:30000 +ENV KRAKEN_GENERAL_log_level=INFO From a3d3b4eda81f4386152d525017f4382bbd501c9b Mon Sep 17 00:00:00 2001 From: David Quintanel Date: Wed, 30 Oct 2024 17:28:18 +0100 Subject: [PATCH 07/12] instances-configurator --- .../debian11/Dockerfile-instances-configurator | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 docker/debian11/Dockerfile-instances-configurator diff --git a/docker/debian11/Dockerfile-instances-configurator b/docker/debian11/Dockerfile-instances-configurator new file mode 100644 index 0000000000..42278f3931 --- /dev/null +++ b/docker/debian11/Dockerfile-instances-configurator @@ -0,0 +1,17 @@ +FROM debian:bullseye-slim + + +COPY ./source/sql/alembic /usr/share/navitia/ed/alembic +COPY ./source/sql/requirements.txt /tmp/requirements.txt +COPY ./source/cities /usr/share/navitia/cities +COPY ./docker/templates/* /templates/ + +COPY docker/instances_configuration.sh / +RUN chmod +x /instances_configuration.sh + +RUN apt-get update --fix-missing \ + && apt-get install -y --force-yes libpq5 python3.9-dev python3-pip postgresql-client gettext-base \ + && pip3 install --no-cache-dir -U -r /tmp/requirements.txt \ + && apt-get autoremove -y + +ENTRYPOINT ["/bin/bash","/instances_configuration.sh"] From ce7d4082ccabc68f41a2cc5a690d3ab523f1ca80 Mon Sep 17 00:00:00 2001 From: David Quintanel Date: Wed, 30 Oct 2024 17:54:42 +0100 Subject: [PATCH 08/12] eitri --- docker/debian11/Dockerfile-eitri | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 docker/debian11/Dockerfile-eitri diff --git a/docker/debian11/Dockerfile-eitri b/docker/debian11/Dockerfile-eitri new file mode 100644 index 0000000000..c2f38f7984 --- /dev/null +++ b/docker/debian11/Dockerfile-eitri @@ -0,0 +1,27 @@ +FROM debian:bullseye-slim + +COPY source/ /navitia/source/ + +RUN apt-get update --fix-missing \ + && apt-get install -y --force-yes libpq5 python3.9 python3-pip git zip jq curl \ + libboost-all-dev \ + libprotobuf-dev \ + liblog4cplus-2.0.5 \ + libzmq3-dev \ + libpqxx-dev \ + libgoogle-perftools4 \ + libosmpbf-dev \ + libproj-dev \ + && pip3 install --no-cache-dir -U -r /navitia/source/eitri/requirements.txt\ + && apt-get autoremove -y + +# binaries from navitia +COPY ./docker_build/ed/ed2nav /usr/bin/ed2nav +COPY ./docker_build/ed/fare2ed /usr/bin/fare2ed +COPY ./docker_build/ed/fusio2ed /usr/bin/fusio2ed +COPY ./docker_build/ed/geopal2ed /usr/bin/geopal2ed +COPY ./docker_build/ed/gtfs2ed /usr/bin/gtfs2ed +COPY ./docker_build/ed/osm2ed /usr/bin/osm2ed +COPY ./docker_build/ed/poi2ed /usr/bin/poi2ed + +CMD [""] From bb689bc632d7d31a8a9e663e9e96976718b62536 Mon Sep 17 00:00:00 2001 From: David Quintanel Date: Fri, 18 Oct 2024 10:18:34 +0200 Subject: [PATCH 09/12] CI --- .github/workflows/dockers_builder.yml | 181 +++++++++----------------- 1 file changed, 61 insertions(+), 120 deletions(-) diff --git a/.github/workflows/dockers_builder.yml b/.github/workflows/dockers_builder.yml index 49466b09b1..60794f66bf 100644 --- a/.github/workflows/dockers_builder.yml +++ b/.github/workflows/dockers_builder.yml @@ -8,7 +8,6 @@ on: - '*' env: - front_debian8_components: 'tyr-web instances-configurator' backend_debian8_components: 'kraken tyr-beat tyr-worker mock-kraken eitri' @@ -17,7 +16,6 @@ env: PRD_ECR_REGISTRY: "162230498103.dkr.ecr.eu-west-1.amazonaws.com" jobs: - common_variables: name: Common variables runs-on: [self-hosted, corefront, sandbox] @@ -55,84 +53,12 @@ jobs: echo '{"text":":warning: Github Actions: workflow dockers_builder common_variables failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}} - debian11_images: - runs-on: [self-hosted, corefront, sandbox] - name: Build debian11 images - needs: common_variables - steps: - - name: force chown to avoid errors - run: sudo chown -R $USER:$USER . - - - name: Git config - run: git config --global --add safe.directory /__w/navitia/navitia - - - name: Generate github private access token - id: ci-core-app-token - uses: getsentry/action-github-app-token@v2.0.0 - with: - app_id: ${{ secrets.CI_CORE_APP_ID }} - private_key: ${{ secrets.CI_CORE_APP_PEM }} - - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: recursive - token: ${{ steps.ci-core-app-token.outputs.token }} - - - name: Login to Amazon ECR - uses: aws-actions/amazon-ecr-login@v1 - env: - AWS_REGION: eu-west-1 - with: - # 162230498103 : shared - # 051314639660 : corefront sbx - registries: "162230498103,051314639660" - - - name: Create builder docker - run: | - docker build -f docker/debian11/Dockerfile-builder -t navitia/builder_debian11 . - - - name: Build navitia - run: | - docker run -v `pwd`:/navitia/navitia/ navitia/builder_debian11 - - - name: Create navitia images - run: | - echo "********* Building Jormungandr ***************" - docker build -t navitia/jormungandr_debian11 -f docker/debian11/Dockerfile-jormungandr . - - - name: Push jormun dev image on SBX ECR - if: github.ref == 'refs/heads/dev' - run: | - jormun_tag=${SBX_ECR_REGISTRY_FRONT}/jormungandr:no_config_dev - docker tag navitia/jormungandr_debian11 ${jormun_tag} - docker push ${jormun_tag} - - - name: Push jormun release image on SBX ECR - if: startsWith(github.ref, 'refs/tags/') - run: | - jormun_tag=${SBX_ECR_REGISTRY_FRONT}/jormungandr:no_config_${{ needs.common_variables.outputs.RELEASE_TAG }} - docker tag navitia/jormungandr_debian11 ${jormun_tag} - docker push ${jormun_tag} - - - - name: failure notification - if: failure() - run: | - echo '{"text":":warning: Github Actions: workflow dockers_builder debian11_images failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}} - - - name: clean up workspace - if: ${{ always() }} - run: | - # some files are created by a docker container - sudo chown -R $USER:$USER . - rm -rf ./* - rm -rf ./.??* - - debian8_back_images: + debian11_back_images: runs-on: [self-hosted, kraken, sandbox] - name: Build debian8 back images + name: Build debian11 back images needs: common_variables + env: + backend_debian11_components: 'kraken tyr-beat tyr-worker mock-kraken eitri' steps: - name: force chown to avoid errors run: sudo chown -R $USER:$USER . @@ -158,7 +84,7 @@ jobs: with: repo: 'hove-io/cosmogony2cities' version: 'tags/v0.1.1' - file: 'cosmogony2cities-debian8.deb' + file: 'cosmogony2cities-debian11.deb' target: 'cosmogony2cities_.deb' token: ${{ steps.ci-core-app-token.outputs.token }} @@ -167,8 +93,8 @@ jobs: with: repo: 'hove-io/mimirsbrunn' version: 'tags/v3.1.0' - file: 'mimirsbrunn7_jessie-3.1.0.deb' - target: 'mimirsbrunn7_jessie-.deb' + file: 'mimirsbrunn7_bullseye-3.1.0.deb' + target: 'mimirsbrunn7_bullseye-.deb' token: ${{ steps.ci-core-app-token.outputs.token }} - name: Download mimir-config package @@ -187,28 +113,28 @@ jobs: with: # 162230498103 : shared # 110444322584 : kraken sbx + # 051314639660: corefront sbx registries: "162230498103,110444322584,051314639660" - - name: Create master docker + - name: Create builder docker image run: | - docker build -f docker/debian8/Dockerfile-master -t navitia/master . - docker build -f docker/debian8/Dockerfile-builder -t navitia/builder . + docker build -f docker/debian11/Dockerfile-master -t navitia/master_debian11 . + docker build -f docker/debian11/Dockerfile-builder-kraken -t navitia/builder_debian11 . - name: Build packages in master docker - # Will build navitia-*.deb packages in current folder - run: docker run -v `pwd`:/build/navitia/ navitia/builder + run: docker run -v `pwd`:/navitia/navitia/ navitia/builder_debian11 - name: Create navitia images run: | - for component in ${{env.backend_debian8_components}}; do + for component in ${{env.backend_debian11_components}}; do echo "********* Building $component ***************" - docker build -t navitia/$component --build-arg GITHUB_TOKEN=${{ steps.ci-core-app-token.outputs.token }} -f docker/debian8/Dockerfile-${component} . + docker build -t navitia/$component --build-arg GITHUB_TOKEN=${{ steps.ci-core-app-token.outputs.token }} -f docker/debian11/Dockerfile-${component} . done - name: Push dev images on SBX ECR if: github.ref == 'refs/heads/dev' run: | - for component in ${{env.backend_debian8_components}}; do + for component in ${{env.backend_debian11_components}}; do component_tag=${SBX_ECR_REGISTRY_BACKEND}/${component}:dev docker tag navitia/$component ${component_tag} docker push ${component_tag} @@ -232,14 +158,11 @@ jobs: docker tag navitia/tyr-worker ${tyr_worker_tag} docker push ${tyr_worker_tag} - - - - name: failure notification if: failure() run: | sudo apt update && sudo apt install -y httpie - echo '{"text":":warning: Github Actions: workflow dockers_builder debian8_images failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}} + echo '{"text":":warning: Github Actions: workflow dockers_builder debian11_images failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}} - name: clean up workspace if: ${{ always() }} @@ -249,9 +172,9 @@ jobs: rm -rf ./* rm -rf ./.??* - debian8_front_images: + debian11_front_images: runs-on: [self-hosted, corefront, sandbox] - name: Build debian8 front images + name: Build debian11 front images needs: common_variables steps: - name: force chown to avoid errors @@ -273,7 +196,6 @@ jobs: submodules: recursive token: ${{ steps.ci-core-app-token.outputs.token }} - - name: Login to Amazon ECR uses: aws-actions/amazon-ecr-login@v1 env: @@ -281,34 +203,56 @@ jobs: with: # 162230498103 : shared # 051314639660 : corefront sbx - registries: "162230498103,110444322584,051314639660" + registries: "162230498103,051314639660" - - name: Create master docker + - name: Create builder docker run: | - docker build -f docker/debian8/Dockerfile-master -t navitia/master . - docker build -f docker/debian8/Dockerfile-builder -t navitia/builder . + docker build -f docker/debian11/Dockerfile-builder -t navitia/builder . - - name: Build packages in master docker - # Will build navitia-*.deb packages in current folder - run: docker run -v `pwd`:/build/navitia/ navitia/builder + - name: Build navitia + run: | + docker run -v `pwd`:/navitia/navitia/ navitia/builder - - name: Create navitia images + - name: Create docker image Jormungandr run: | - for component in ${{env.front_debian8_components}}; do - echo "********* Building $component ***************" - docker build -t navitia/$component -f docker/debian8/Dockerfile-${component} . - done + echo "********* Building Jormungandr ***************" + docker build -t navitia/jormungandr_debian11 -f docker/debian11/Dockerfile-jormungandr . - - name: Push dev images on SBX ECR + - name: Create images for Tyr-web + run: docker build -t navitia/tyr-web -f docker/debian11/Dockerfile-tyr-web . + + - name: Create images for Instance configurator + run: docker build -t navitia/instances-configurator -f docker/debian11/Dockerfile-instances-configurator . + + - name: Push Jormungandr dev image on SBX ECR if: github.ref == 'refs/heads/dev' run: | - for component in ${{env.front_debian8_components}}; do - component_tag=${SBX_ECR_REGISTRY_FRONT}/${component}:dev - docker tag navitia/$component ${component_tag} - docker push ${component_tag} - done + jormun_tag=${SBX_ECR_REGISTRY_FRONT}/jormungandr:no_config_dev + docker tag navitia/jormungandr_debian11 ${jormun_tag} + docker push ${jormun_tag} - - name: Push release images on PRD ECR + - name: Push Jormungandr release image on SBX ECR + if: startsWith(github.ref, 'refs/tags/') + run: | + jormun_tag=${SBX_ECR_REGISTRY_FRONT}/jormungandr:no_config_${{ needs.common_variables.outputs.RELEASE_TAG }} + docker tag navitia/jormungandr_debian11 ${jormun_tag} + docker push ${jormun_tag} + + - name: Push Tyr-Web dev image on SBX ECR + if: github.ref == 'refs/heads/dev' + run: | + component_tag=${SBX_ECR_REGISTRY_FRONT}/tyr-web:dev + docker tag navitia/tyr-web ${component_tag} + docker push ${component_tag} + + - name: Push Instances configurator dev image on SBX ECR + if: github.ref == 'refs/heads/dev' + run: | + component_tag=${SBX_ECR_REGISTRY_FRONT}/instances-configurator:dev + docker tag navitia/instances-configurator ${component_tag} + docker push ${component_tag} + + - name: Push Try-web release image on PRD ECR if: startsWith(github.ref, 'refs/tags/') run: | # Tyr-web @@ -319,8 +263,7 @@ jobs: - name: failure notification if: failure() run: | - sudo apt update && sudo apt install -y httpie - echo '{"text":":warning: Github Actions: workflow dockers_builder debian8_images failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}} + echo '{"text":":warning: Github Actions: workflow dockers_builder debian11_images failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}} - name: clean up workspace if: ${{ always() }} @@ -330,12 +273,10 @@ jobs: rm -rf ./* rm -rf ./.??* - - publish_aws: runs-on: [self-hosted, corefront, sandbox] name: Aws Dispatch (Dev) - needs: [debian8_front_images, debian8_back_images, debian11_images, common_variables] + needs: [debian11_front_images, debian11_back_images, common_variables] steps: - name: Generate token for aws images From 83896689a50c1ac569424deabc74931ca9646685 Mon Sep 17 00:00:00 2001 From: David Quintanel Date: Wed, 30 Oct 2024 17:57:03 +0100 Subject: [PATCH 10/12] delete unused workflows --- .github/workflows/dockers_builder_dev.yml | 359 ----------------- .github/workflows/dockers_builder_release.yml | 365 ------------------ 2 files changed, 724 deletions(-) delete mode 100644 .github/workflows/dockers_builder_dev.yml delete mode 100644 .github/workflows/dockers_builder_release.yml diff --git a/.github/workflows/dockers_builder_dev.yml b/.github/workflows/dockers_builder_dev.yml deleted file mode 100644 index 7f242b42e3..0000000000 --- a/.github/workflows/dockers_builder_dev.yml +++ /dev/null @@ -1,359 +0,0 @@ -name: Build Navitia Dockers (Dev) - -on: - push: - branches: - - _dev - -env: - jessie_version: debian8 - bullseye_version: debian11 - front_jessie_navitia_components: 'tyr-web instances-configurator' - backend_jessie_navitia_components: 'kraken tyr-beat tyr-worker mock-kraken eitri' - - SBX_ECR_REGISTRY_BACKEND: "110444322584.dkr.ecr.eu-west-1.amazonaws.com" - SBX_ECR_REGISTRY_FRONT: "051314639660.dkr.ecr.eu-west-1.amazonaws.com" - TAG: 'dev' - -jobs: - front_bullseye_builder: - runs-on: [self-hosted, corefront, sandbox] - name: Front Bullseye builder (Dev) - steps: - - name: force chown to avoid errors - run: sudo chown -R $USER:$USER . - - name: Git config - run: git config --global --add safe.directory /__w/navitia/navitia - - name: Install dependencies - run: | - sudo apt update && sudo apt install -y httpie - - - name: Generate github private access token - id: ci-core-app-token - uses: getsentry/action-github-app-token@v2.0.0 - with: - app_id: ${{ secrets.CI_CORE_APP_ID }} - private_key: ${{ secrets.CI_CORE_APP_PEM }} - - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: recursive - token: ${{ steps.ci-core-app-token.outputs.token }} - # we need entire history for tags - fetch-depth: 0 - -# - name: Restore ccache -# uses: hendrikmuhs/ccache-action@v1.2 -# with: -# key: build_dockers -# max-size: 2000M - - - name: Login to Amazon ECR SBX - uses: aws-actions/amazon-ecr-login@v1 - env: - AWS_REGION: eu-west-1 - - - name: Login to Amazon ECR Shared - uses: aws-actions/amazon-ecr-login@v1 - env: - AWS_REGION: eu-west-1 - with: - registries: "162230498103" - - - name: Create builder docker - run: | - docker build -f docker/${{env.bullseye_version}}/Dockerfile-builder -t navitia/builder . - - - name: Build navitia - run: | - docker run -v `pwd`:/navitia/navitia/ navitia/builder - - - name: Create navitia images - run: | - echo "********* Building Jormungandr ***************" - docker build -t ${SBX_ECR_REGISTRY_FRONT}/jormungandr:no_config_${TAG} -f docker/${{env.bullseye_version}}/Dockerfile-jormungandr . - - - name: Push images on SBX ECR - run: | - echo "********* Push Jormungandr ***************" - docker push ${SBX_ECR_REGISTRY_FRONT}/jormungandr:no_config_${TAG} - - - name: failure notification - if: failure() - run: | - echo '{"text":":warning: Github Actions: workflow dockers_builder_dev Job front_bullseye_builder is failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}} - - - name: clean up workspace - if: ${{ always() }} - run: | - # some files are created by a docker container - sudo chown -R $USER:$USER . - rm -rf ./* - rm -rf ./.??* - - front_jessie_builder: - runs-on: [self-hosted, corefront, sandbox] - name: Front Jessie builder (Dev) - steps: - - name: force chown to avoid errors - run: sudo chown -R $USER:$USER . - - name: Git config - run: git config --global --add safe.directory /__w/navitia/navitia - - name: Install dependencies - run: | - sudo apt update && sudo apt install -y httpie - - name: Generate github private access token - id: ci-core-app-token - uses: getsentry/action-github-app-token@v2.0.0 - with: - app_id: ${{ secrets.CI_CORE_APP_ID }} - private_key: ${{ secrets.CI_CORE_APP_PEM }} - - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: recursive - token: ${{ steps.ci-core-app-token.outputs.token }} - # we need entire history for tags - fetch-depth: 0 - -# - name: Restore ccache -# uses: hendrikmuhs/ccache-action@v1.2 -# with: -# key: build_dockers -# max-size: 2000M - - - name: Login to Amazon ECR SBX - uses: aws-actions/amazon-ecr-login@v1 - env: - AWS_REGION: eu-west-1 - - - name: Login to Amazon ECR Shared - uses: aws-actions/amazon-ecr-login@v1 - env: - AWS_REGION: eu-west-1 - with: - registries: "162230498103" - - - name: Create master docker - run: | - docker build -f docker/${{env.jessie_version}}/Dockerfile-master -t navitia/master . - docker build -f docker/${{env.jessie_version}}/Dockerfile-builder -t navitia/builder . - - - name: Build packages in master docker - # Will build navitia-*.deb packages in current folder - run: docker run -v `pwd`:/build/navitia/ navitia/builder - - - name: Create navitia images - run: | - for component in ${{env.front_jessie_navitia_components}}; do - echo "********* Building $component ***************" - docker build -t ${SBX_ECR_REGISTRY_FRONT}/$component:${TAG} -f docker/${{env.jessie_version}}/Dockerfile-${component} . - done - - - name: Push images on SBX ECR - run: | - for component in ${{env.front_jessie_navitia_components}}; do - echo "********* Building $component ***************" - docker push ${SBX_ECR_REGISTRY_FRONT}/$component:${TAG} - done - - - name: failure notification - if: failure() - run: | - echo '{"text":":warning: Github Actions: workflow dockers_builder_dev Job front_jessie_builder is failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}} - - - name: clean up workspace - if: ${{ always() }} - run: | - # some files are created by a docker container - sudo chown -R $USER:$USER . - rm -rf ./* - rm -rf ./.??* - - backend_jessie_builder: - runs-on: [self-hosted, kraken, sandbox] - name: Backend Jessie builder (Dev) - steps: - - name: force chown to avoid errors - run: sudo chown -R $USER:$USER . - - name: Git config - run: git config --global --add safe.directory /__w/navitia/navitia - - name: Install dependencies - run: | - sudo apt update && sudo apt install -y zip httpie curl python3 python3-pip - - name: Generate github private access token - id: ci-core-app-token - uses: getsentry/action-github-app-token@v2.0.0 - with: - app_id: ${{ secrets.CI_CORE_APP_ID }} - private_key: ${{ secrets.CI_CORE_APP_PEM }} - - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: recursive - token: ${{ steps.ci-core-app-token.outputs.token }} - # we need entire history for tags - fetch-depth: 0 - - - name: Checkout core_team_ci_tools - uses: actions/checkout@v3 - with: - repository : 'hove-io/core_team_ci_tools' - path: core_team_ci_tools - token: ${{ steps.ci-core-app-token.outputs.token }} - - - name: Setup core_team_ci_tools python environment - run: | - pip install -r core_team_ci_tools/github_artifacts/requirements.txt --user - - - name: Download mimirsbrunn package - run: | - mimirsbrunn_package="debian-package-master.zip" - python3 core_team_ci_tools/github_artifacts/github_artifacts.py \ - -o hove-io \ - -r mimirsbrunn \ - -t ${{ steps.ci-core-app-token.outputs.token }} \ - -w main7.yml \ - -a debian-package-master.zip \ - --output-dir . - unzip -qo $mimirsbrunn_package - rm -f $mimirsbrunn_package - - - name: Download cosmogony2cities from release - uses: dsaltares/fetch-gh-release-asset@master - with: - repo: 'hove-io/cosmogony2cities' - version: 'tags/v0.1.1' - file: 'cosmogony2cities-${{env.jessie_version}}.deb' - target: 'cosmogony2cities_0.1.0-1_amd64.deb' - token: ${{ steps.ci-core-app-token.outputs.token }} - - - name: Download mimir-config package - run: | - conf_mimir_package="debian-package-release.zip" - python3 core_team_ci_tools/github_artifacts/github_artifacts.py \ - -o hove-io \ - -r mimirsbrunn-config \ - -t ${{ steps.ci-core-app-token.outputs.token }} \ - -w release.yml \ - -a $conf_mimir_package \ - -b main \ - --skip-check-artifacts-size \ - --output-dir . - unzip -qo $conf_mimir_package - rm -f $conf_mimir_package - -# - name: Restore ccache -# uses: hendrikmuhs/ccache-action@v1.2 -# with: -# key: build_dockers -# max-size: 2000M - - - name: Login to Amazon ECR SBX - uses: aws-actions/amazon-ecr-login@v1 - env: - AWS_REGION: eu-west-1 - - - name: Login to Amazon ECR Shared - uses: aws-actions/amazon-ecr-login@v1 - env: - AWS_REGION: eu-west-1 - with: - registries: "162230498103" - - - name: Create master docker - run: | - docker build -f docker/${{env.jessie_version}}/Dockerfile-master -t navitia/master . - docker build -f docker/${{env.jessie_version}}/Dockerfile-builder -t navitia/builder . - - - name: Build packages in master docker - # Will build navitia-*.deb packages in current folder - run: docker run -v `pwd`:/build/navitia/ navitia/builder - - - name: Create navitia images - run: | - for component in ${{env.backend_jessie_navitia_components}}; do - echo "********* Building $component ***************" - docker build -t ${SBX_ECR_REGISTRY_BACKEND}/$component:${TAG} --build-arg GITHUB_TOKEN=${{ steps.ci-core-app-token.outputs.token }} -f docker/${{env.jessie_version}}/Dockerfile-${component} . - done - - - name: Push images on SBX ECR - run: | - for component in ${{env.backend_jessie_navitia_components}}; do - echo "********* Building $component ***************" - docker push ${SBX_ECR_REGISTRY_BACKEND}/$component:${TAG} - done - - - name: failure notification - if: failure() - run: | - echo '{"text":":warning: Github Actions: workflow dockers_builder_dev Job backend_jessie_builder is failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}} - - - name: clean up workspace - if: ${{ always() }} - run: | - # some files are created by a docker container - sudo chown -R $USER:$USER . - rm -rf ./* - rm -rf ./.??* - - publish_aws: - runs-on: [self-hosted, corefront, sandbox] - name: Aws Dispatch (Dev) - needs: [front_jessie_builder, front_bullseye_builder, backend_jessie_builder] - steps: - - name: Install dependencies - run: | - sudo apt update && sudo apt install -y httpie - - - name: Generate token for aws images - id: app-token - uses: getsentry/action-github-app-token@v2.0.0 - with: - app_id: ${{ secrets.GA_OS_WORKFLOW_TRIGGER_APP_ID }} - private_key: ${{ secrets.GA_OS_WORKFLOW_TRIGGER_APP_PEM }} - - - name: Aws Dispatch Frontend - uses: peter-evans/repository-dispatch@v2 - with: - token: ${{ steps.app-token.outputs.token }} - repository: hove-io/corefront-aws-assets - event-type: build-trigger - client-payload: '{"branch": "dev", "tag": "dev"}' - - - name: failure notification - if: failure() - run: | - echo '{"text":":warning: Github Actions: dockers_builder_dev the job publish_aws is failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}} - - - run_artemis: - runs-on: [self-hosted, corefront, sandbox] - needs: [front_jessie_builder, front_bullseye_builder, backend_jessie_builder, publish_aws] - name: Run artemis Dispatch (Dev) - steps: - - name: Install dependencies - run: | - sudo apt update && sudo apt install -y httpie - - - name: Generate github private access token - id: ci-core-app-token - uses: getsentry/action-github-app-token@v2.0.0 - with: - app_id: ${{ secrets.CI_CORE_APP_ID }} - private_key: ${{ secrets.CI_CORE_APP_PEM }} - - - name: Run artemis on push to dev - uses: peter-evans/repository-dispatch@v2 - with: - token: ${{ steps.ci-core-app-token.outputs.token }} - repository: hove-io/artemis - event-type: run_artemis_ng - - - name: failure notification - if: failure() - run: | - echo '{"text":":warning: Github Actions: workflow dockers_builder_dev Job run_artemis is failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}} diff --git a/.github/workflows/dockers_builder_release.yml b/.github/workflows/dockers_builder_release.yml deleted file mode 100644 index a7a1dcd899..0000000000 --- a/.github/workflows/dockers_builder_release.yml +++ /dev/null @@ -1,365 +0,0 @@ -name: Build Navitia Dockers (Release) - -on: - push: - tags: - - 'toto' - -env: - jessie_version: debian8 - bullseye_version: debian11 - SBX_ECR_REGISTRY_FRONT: "051314639660.dkr.ecr.eu-west-1.amazonaws.com" - PRD_ECR_REGISTRY: "162230498103.dkr.ecr.eu-west-1.amazonaws.com" - REGION: "eu-west-1" - -jobs: - common_variables: - name: Common variables - runs-on: [self-hosted, corefront, sandbox] - outputs: - NAVITIA_TAG: ${{ steps.choose_navitia_tag.outputs.navitia_tag }} - steps: - - name: force chown to avoid errors - run: sudo chown -R $USER:$USER . - - - name: Install dependencies - run: | - sudo apt update && sudo apt install -y --force-yes httpie - - - name: Generate github private access token - id: ci-core-app-token - uses: getsentry/action-github-app-token@v2.0.0 - with: - app_id: ${{ secrets.CI_CORE_APP_ID }} - private_key: ${{ secrets.CI_CORE_APP_PEM }} - - - name: Checkout navitia - id: checkout_navitia - uses: actions/checkout@v3 - with: - submodules: recursive - token: ${{ steps.ci-core-app-token.outputs.token }} - # we need entire history for tags - fetch-depth: 0 - - - name: Choose navitia tag - id: choose_navitia_tag - run: | - version=$(git describe --tags) - echo "navitia_tag=$version" >> $GITHUB_OUTPUT - - - name: failure notification - if: failure() - run: | - echo '{"text":":warning: Github Actions: workflow dockers_builder_release Job init common variables failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}} - - front_bullseye_builder: - runs-on: [self-hosted, corefront, sandbox] - name: Front Bullseye builder (Release) - needs: common_variables - steps: - - name: force chown to avoid errors - run: sudo chown -R $USER:$USER . - - name: Install dependencies - run: | - sudo apt update && sudo apt install -y httpie - - - name: Generate github private access token - id: ci-core-app-token - uses: getsentry/action-github-app-token@v2.0.0 - with: - app_id: ${{ secrets.CI_CORE_APP_ID }} - private_key: ${{ secrets.CI_CORE_APP_PEM }} - - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: recursive - token: ${{ steps.ci-core-app-token.outputs.token }} - # we need entire history for tags - fetch-depth: 0 - - # - name: Restore ccache - # uses: hendrikmuhs/ccache-action@v1.2 - # with: - # key: build_dockers - # max-size: 2000M - - - name: Login to Amazon ECR SBX - uses: aws-actions/amazon-ecr-login@v1 - env: - AWS_REGION: eu-west-1 - - - name: Login to Amazon ECR Shared - uses: aws-actions/amazon-ecr-login@v1 - env: - AWS_REGION: eu-west-1 - with: - registries: "162230498103" - - - name: Create builder docker - run: | - docker build -f docker/${{env.bullseye_version}}/Dockerfile-builder -t navitia/builder . - - - name: Build navitia - run: | - docker run -v `pwd`:/navitia/navitia/ navitia/builder - - - name: Create navitia images - run: | - echo "********* Building Jormungandr ***************" - docker build -t ${SBX_ECR_REGISTRY_FRONT}/jormungandr:no_config_${{ needs.common_variables.outputs.NAVITIA_TAG }} -f docker/${{env.bullseye_version}}/Dockerfile-jormungandr . - - - name: Push images on SBX ECR - run: | - echo "********* Push Jormungandr ***************" - docker push ${SBX_ECR_REGISTRY_FRONT}/jormungandr:no_config_${{ needs.common_variables.outputs.NAVITIA_TAG }} - - - name: failure notification - if: failure() - run: | - echo '{"text":":warning: Github Actions: workflow dockers_builder_release Job front_bullseye_builder is failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}} - - front_jessie_builder: - runs-on: [self-hosted, corefront, sandbox] - name: Front Jessie builder (Release) - needs: common_variables - steps: - - name: force chown to avoid errors - run: sudo chown -R $USER:$USER . - - name: Install dependencies - run: | - sudo apt update && sudo apt install -y httpie - - name: Generate github private access token - id: ci-core-app-token - uses: getsentry/action-github-app-token@v2.0.0 - with: - app_id: ${{ secrets.CI_CORE_APP_ID }} - private_key: ${{ secrets.CI_CORE_APP_PEM }} - - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: recursive - token: ${{ steps.ci-core-app-token.outputs.token }} - # we need entire history for tags - fetch-depth: 0 - - # - name: Restore ccache - # uses: hendrikmuhs/ccache-action@v1.2 - # with: - # key: build_dockers - # max-size: 2000M - - - name: Login to Amazon ECR Shared - uses: aws-actions/amazon-ecr-login@v1 - env: - AWS_REGION: eu-west-1 - with: - registries: "162230498103" - - - name: Create master docker - run: | - docker build -f docker/${{env.jessie_version}}/Dockerfile-master -t navitia/master . - docker build -f docker/${{env.jessie_version}}/Dockerfile-builder -t navitia/builder . - - - name: Build packages in master docker - # Will build navitia-*.deb packages in current folder - run: docker run -v `pwd`:/build/navitia/ navitia/builder - - - name: Create navitia images - run: | - # Tyr-web - component_tag=${PRD_ECR_REGISTRY}/navitia-tyr-web-tyr-web:${{ needs.common_variables.outputs.NAVITIA_TAG }} - docker build -t ${component_tag} -f docker/${{env.jessie_version}}/Dockerfile-tyr-web . - - - name: Push images on PRD ECR - run: | - # Tyr-web - component_tag=${PRD_ECR_REGISTRY}/navitia-tyr-web-tyr-web:${{ needs.common_variables.outputs.NAVITIA_TAG }} - docker push ${component_tag} - - - name: failure notification - if: failure() - run: | - echo '{"text":":warning: Github Actions: workflow dockers_builder_release Job front_jessie_builder is failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}} - - - backend_jessie_builder: - runs-on: [self-hosted, kraken, sandbox] - name: Backend Jessie builder (Release) - needs: common_variables - steps: - - name: force chown to avoid errors - run: sudo chown -R $USER:$USER . - - name: Install dependencies - run: | - sudo apt update && sudo apt install -y zip httpie curl python3 python3-pip - - name: Generate github private access token - id: ci-core-app-token - uses: getsentry/action-github-app-token@v2.0.0 - with: - app_id: ${{ secrets.CI_CORE_APP_ID }} - private_key: ${{ secrets.CI_CORE_APP_PEM }} - - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: recursive - token: ${{ steps.ci-core-app-token.outputs.token }} - # we need entire history for tags - fetch-depth: 0 - - - name: Checkout core_team_ci_tools - uses: actions/checkout@v3 - with: - repository : 'hove-io/core_team_ci_tools' - path: core_team_ci_tools - token: ${{ steps.ci-core-app-token.outputs.token }} - - - name: Setup core_team_ci_tools python environment - run: | - pip install -r core_team_ci_tools/github_artifacts/requirements.txt --user - - - name: Download mimirsbrunn package - run: | - mimirsbrunn_package="debian-package-release.zip" - python3 core_team_ci_tools/github_artifacts/github_artifacts.py \ - -o hove-io \ - -r mimirsbrunn \ - -t ${{ steps.ci-core-app-token.outputs.token }} \ - -w release7.yml \ - -a $mimirsbrunn_package \ - --output-dir . - unzip -qo $mimirsbrunn_package - rm -f $mimirsbrunn_package - - - name: Download cosmogony2cities from release - uses: dsaltares/fetch-gh-release-asset@master - with: - repo: 'hove-io/cosmogony2cities' - version: 'tags/v0.1.1' - file: 'cosmogony2cities-${{env.jessie_version}}.deb' - target: 'cosmogony2cities_0.1.0-1_amd64.deb' - token: ${{ steps.ci-core-app-token.outputs.token }} - - - name: Download mimir-config package - run: | - conf_mimir_package="debian-package-release.zip" - python3 core_team_ci_tools/github_artifacts/github_artifacts.py \ - -o hove-io \ - -r mimirsbrunn-config \ - -t ${{ steps.ci-core-app-token.outputs.token }} \ - -w release.yml \ - -a $conf_mimir_package \ - -b main \ - --skip-check-artifacts-size \ - --output-dir . - unzip -qo $conf_mimir_package - rm -f $conf_mimir_package - - # - name: Restore ccache - # uses: hendrikmuhs/ccache-action@v1.2 - # with: - # key: build_dockers - # max-size: 2000M - - - name: Login to Amazon ECR Shared - uses: aws-actions/amazon-ecr-login@v1 - env: - AWS_REGION: eu-west-1 - with: - registries: "162230498103" - - - name: Create master docker - run: | - docker build -f docker/${{env.jessie_version}}/Dockerfile-master -t navitia/master . - docker build -f docker/${{env.jessie_version}}/Dockerfile-builder -t navitia/builder . - - - name: Build packages in master docker - # Will build navitia-*.deb packages in current folder - run: docker run -v `pwd`:/build/navitia/ navitia/builder - - - name: Create images - run: | - # Kraken - component_tag=${PRD_ECR_REGISTRY}/navitia-kraken-kraken:${{ needs.common_variables.outputs.NAVITIA_TAG }} - docker build -t ${component_tag} -f docker/${{env.jessie_version}}/Dockerfile-kraken . - - # Tyr-beat - component_tag=${PRD_ECR_REGISTRY}/navitia-tyr-tyrbeat:${{ needs.common_variables.outputs.NAVITIA_TAG }} - docker build -t ${component_tag} -f docker/${{env.jessie_version}}/Dockerfile-tyr-beat . - - # Tyr-worker - component_tag=${PRD_ECR_REGISTRY}/navitia-tyr-tyrworker:${{ needs.common_variables.outputs.NAVITIA_TAG }} - docker build -t ${component_tag} --build-arg GITHUB_TOKEN=${{ steps.ci-core-app-token.outputs.token }} -f docker/${{env.jessie_version}}/Dockerfile-tyr-worker . - - - name: Push on PRD ECR - run: | - # Kraken - component_tag=${PRD_ECR_REGISTRY}/navitia-kraken-kraken:${{ needs.common_variables.outputs.NAVITIA_TAG }} - docker push ${component_tag} - - # Tyr-beat - component_tag=${PRD_ECR_REGISTRY}/navitia-tyr-tyrbeat:${{ needs.common_variables.outputs.NAVITIA_TAG }} - docker push ${component_tag} - - # Tyr-worker - component_tag=${PRD_ECR_REGISTRY}/navitia-tyr-tyrworker:${{ needs.common_variables.outputs.NAVITIA_TAG }} - docker push ${component_tag} - - - name: failure notification - if: failure() - run: | - echo '{"text":":warning: Github Actions: workflow dockers_builder_release Job backend_jessie_builder is failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}} - - name: failure notification - if: success() - run: | - echo '{"text":":octopus: Github Actions: workflow build_dockers succeeded. New navitia ${{ needs.common_variables.outputs.NAVITIA_TAG }} images available."}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}} - - publish_aws: - runs-on: [self-hosted, corefront, sandbox] - name: Aws Dispatch (Release) - needs: [front_jessie_builder, front_bullseye_builder, backend_jessie_builder, common_variables] - steps: - - name: force chown to avoid errors - run: sudo chown -R $USER:$USER . - - - name: Install dependencies - run: | - sudo apt update && sudo apt install -y httpie - - - name: Generate token for aws images - id: app-token - uses: getsentry/action-github-app-token@v2.0.0 - with: - app_id: ${{ secrets.GA_OS_WORKFLOW_TRIGGER_APP_ID }} - private_key: ${{ secrets.GA_OS_WORKFLOW_TRIGGER_APP_PEM }} - - - name: Generate github private access token - id: ci-core-app-token - uses: getsentry/action-github-app-token@v2.0.0 - with: - app_id: ${{ secrets.CI_CORE_APP_ID }} - private_key: ${{ secrets.CI_CORE_APP_PEM }} - - - name: Checkout - uses: actions/checkout@v3 - with: - submodules: recursive - token: ${{ steps.ci-core-app-token.outputs.token }} - # we need entire history for tags - fetch-depth: 0 - - - name: Aws Dispatch Frontend - uses: peter-evans/repository-dispatch@v2 - with: - token: ${{ steps.app-token.outputs.token }} - repository: hove-io/corefront-aws-assets - event-type: build-trigger - client-payload: '{"branch": "release", "tag": "${{ needs.common_variables.outputs.NAVITIA_TAG }}"}' - - - name: failure notification - if: failure() - run: | - echo '{"text":":warning: Github Actions: dockers_builder_release the job publish_aws is failed !"}' | http --json POST ${{secrets.SLACK_NAVITIA_TEAM_URL}} From ea52b2b20a5d5280d9c6b044c2559ffefa8a87c8 Mon Sep 17 00:00:00 2001 From: David Quintanel Date: Wed, 30 Oct 2024 17:59:07 +0100 Subject: [PATCH 11/12] remove debian11/Dockerfile_master --- .github/workflows/dockers_builder.yml | 1 - docker/debian11/Dockerfile-master | 42 --------------------------- 2 files changed, 43 deletions(-) delete mode 100644 docker/debian11/Dockerfile-master diff --git a/.github/workflows/dockers_builder.yml b/.github/workflows/dockers_builder.yml index 60794f66bf..0aa32c9d79 100644 --- a/.github/workflows/dockers_builder.yml +++ b/.github/workflows/dockers_builder.yml @@ -118,7 +118,6 @@ jobs: - name: Create builder docker image run: | - docker build -f docker/debian11/Dockerfile-master -t navitia/master_debian11 . docker build -f docker/debian11/Dockerfile-builder-kraken -t navitia/builder_debian11 . - name: Build packages in master docker diff --git a/docker/debian11/Dockerfile-master b/docker/debian11/Dockerfile-master deleted file mode 100644 index 9b8eab2ba9..0000000000 --- a/docker/debian11/Dockerfile-master +++ /dev/null @@ -1,42 +0,0 @@ -FROM 162230498103.dkr.ecr.eu-west-1.amazonaws.com/mutable-debian11_dev:latest - -RUN echo "deb http://archive.debian.org/debian/ bullseye main" > /etc/apt/sources.list.d/archive.list - -# update package list from providers -RUN apt-get update --force-yes --fix-missing || exit 0 - -# install postgresql-client for tyr-beat -# netcat for kraken -# curl for jormun -RUN apt-get install -y --force-yes ca-certificates \ - postgresql-client \ - netcat \ - dh-python \ - curl \ - wget - -COPY docker/ca-certificates/*.crt /usr/local/share/ca-certificates/ -RUN update-ca-certificates - -# install a more recent version of CMake -RUN apt-get remove --yes cmake \ - && cd /tmp \ - && wget https://cmake.org/files/v3.18/cmake-3.18.6-Linux-x86_64.tar.gz \ - && tar xf cmake-3.18.6-Linux-x86_64.tar.gz --strip-components=1 --directory=/usr/local \ - && rm -f cmake-3.18.6-Linux-x86_64.tar.gz \ - && cd - - -# Python 'requests' package handle its own CA certificate list -# Let's force it to use the OS's list -ENV REQUESTS_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt - -# install rustup -ENV RUSTUP_HOME=/usr/local/rustup \ - CARGO_HOME=/usr/local/cargo - -ENV PATH=/usr/local/cargo/bin:${PATH} -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --profile minimal \ - && chmod -R a+wrX ${RUSTUP_HOME} ${CARGO_HOME} \ - && rustup --version \ - && cargo --version \ - && rustc --version From b7acc53ee9e4a310ddcc2708c0cdb4398542ff5f Mon Sep 17 00:00:00 2001 From: David Quintanel Date: Wed, 30 Oct 2024 18:01:50 +0100 Subject: [PATCH 12/12] delete debian8/10 docker images --- .github/workflows/dockers_builder.yml | 3 - .github/workflows/workflow.yml | 12 +--- docker/debian10/Dockerfile-eitri | 17 ----- .../Dockerfile-instances-configurator | 17 ----- docker/debian10/Dockerfile-jormungandr | 49 ------------- docker/debian10/Dockerfile-kraken | 54 --------------- docker/debian10/Dockerfile-master | 18 ----- docker/debian10/Dockerfile-mock-kraken | 20 ------ docker/debian10/Dockerfile-tyr-beat | 27 -------- docker/debian10/Dockerfile-tyr-web | 23 ------- docker/debian10/Dockerfile-tyr-worker | 38 ---------- docker/debian8/Dockerfile-builder | 6 -- docker/debian8/Dockerfile-eitri | 14 ---- .../debian8/Dockerfile-instances-configurator | 15 ---- docker/debian8/Dockerfile-jormungandr | 54 --------------- docker/debian8/Dockerfile-kraken | 57 --------------- docker/debian8/Dockerfile-master | 41 ----------- docker/debian8/Dockerfile-mock-kraken | 20 ------ docker/debian8/Dockerfile-tyr-beat | 38 ---------- docker/debian8/Dockerfile-tyr-web | 33 --------- docker/debian8/Dockerfile-tyr-worker | 69 ------------------- 21 files changed, 1 insertion(+), 624 deletions(-) delete mode 100644 docker/debian10/Dockerfile-eitri delete mode 100644 docker/debian10/Dockerfile-instances-configurator delete mode 100644 docker/debian10/Dockerfile-jormungandr delete mode 100644 docker/debian10/Dockerfile-kraken delete mode 100644 docker/debian10/Dockerfile-master delete mode 100644 docker/debian10/Dockerfile-mock-kraken delete mode 100644 docker/debian10/Dockerfile-tyr-beat delete mode 100644 docker/debian10/Dockerfile-tyr-web delete mode 100644 docker/debian10/Dockerfile-tyr-worker delete mode 100755 docker/debian8/Dockerfile-builder delete mode 100644 docker/debian8/Dockerfile-eitri delete mode 100644 docker/debian8/Dockerfile-instances-configurator delete mode 100644 docker/debian8/Dockerfile-jormungandr delete mode 100644 docker/debian8/Dockerfile-kraken delete mode 100755 docker/debian8/Dockerfile-master delete mode 100644 docker/debian8/Dockerfile-mock-kraken delete mode 100644 docker/debian8/Dockerfile-tyr-beat delete mode 100644 docker/debian8/Dockerfile-tyr-web delete mode 100644 docker/debian8/Dockerfile-tyr-worker diff --git a/.github/workflows/dockers_builder.yml b/.github/workflows/dockers_builder.yml index 0aa32c9d79..428d710da6 100644 --- a/.github/workflows/dockers_builder.yml +++ b/.github/workflows/dockers_builder.yml @@ -8,9 +8,6 @@ on: - '*' env: - front_debian8_components: 'tyr-web instances-configurator' - backend_debian8_components: 'kraken tyr-beat tyr-worker mock-kraken eitri' - SBX_ECR_REGISTRY_BACKEND: "110444322584.dkr.ecr.eu-west-1.amazonaws.com" SBX_ECR_REGISTRY_FRONT: "051314639660.dkr.ecr.eu-west-1.amazonaws.com" PRD_ECR_REGISTRY: "162230498103.dkr.ecr.eu-west-1.amazonaws.com" diff --git a/.github/workflows/workflow.yml b/.github/workflows/workflow.yml index 9398456106..53d10d19fa 100644 --- a/.github/workflows/workflow.yml +++ b/.github/workflows/workflow.yml @@ -119,7 +119,7 @@ jobs: strategy: fail-fast: false matrix: - os: [{docker_image: mutable-debian8_dev , python_version: python3.6}, {docker_image: mutable-debian11_dev, python_version: python3.9}] + os: [{docker_image: mutable-debian11_dev, python_version: python3.9}] container: image: 162230498103.dkr.ecr.eu-west-1.amazonaws.com/${{ matrix.os.docker_image }}:latest @@ -159,16 +159,6 @@ jobs: key: ${{matrix.os.docker_image}}-ci max-size: 2000M save: ${{ github.event_name == 'push' }} - - name: Ensure CMake version is recent enough - if: ${{ matrix.os.docker_image == 'mutable-debian8_dev'}} - run: | - apt remove --yes cmake - mkdir local_bin - cd local_bin - wget https://cmake.org/files/v3.18/cmake-3.18.6-Linux-x86_64.tar.gz - tar xf cmake-3.18.6-Linux-x86_64.tar.gz - echo "`pwd`/cmake-3.18.6-Linux-x86_64/bin" >> $GITHUB_PATH - cd .. - name: configure for Release run: mkdir build && cd ./build && cmake -DSTRIP_SYMBOLS=ON ../source - name: run diff --git a/docker/debian10/Dockerfile-eitri b/docker/debian10/Dockerfile-eitri deleted file mode 100644 index d0c9e06e4c..0000000000 --- a/docker/debian10/Dockerfile-eitri +++ /dev/null @@ -1,17 +0,0 @@ -FROM navitia/master - -# copy package from context inside the docker -COPY navitia-ed_*.deb / -COPY source/ /navitia/source/ - -# install dependencies -RUN apt-get update \ - && apt-get install -y python3 python3-pip libpq-dev jq git zip curl \ - && apt-get install -y /navitia-ed_*.deb \ - && apt-get clean \ - && rm -rf /navitia-ed_*.deb - -# install eitri requirements -RUN pip3 install --no-cache-dir -r /navitia/source/eitri/requirements.txt - -CMD [""] diff --git a/docker/debian10/Dockerfile-instances-configurator b/docker/debian10/Dockerfile-instances-configurator deleted file mode 100644 index 2ff8189314..0000000000 --- a/docker/debian10/Dockerfile-instances-configurator +++ /dev/null @@ -1,17 +0,0 @@ -FROM navitia/master - -COPY source/sql/alembic /usr/share/navitia/ed/alembic -COPY source/sql/requirements.txt /tmp/requirements.txt -COPY source/cities /usr/share/navitia/cities -COPY docker/templates/* /templates/ - -RUN apt-get update \ - && apt install -y python python-pip git libpq-dev postgresql-client gettext-base \ - && apt-get clean - -RUN pip install -r /tmp/requirements.txt - -COPY docker/instances_configuration.sh / -RUN chmod +x /instances_configuration.sh - -ENTRYPOINT ["/bin/bash","/instances_configuration.sh"] diff --git a/docker/debian10/Dockerfile-jormungandr b/docker/debian10/Dockerfile-jormungandr deleted file mode 100644 index 3ac767aaf7..0000000000 --- a/docker/debian10/Dockerfile-jormungandr +++ /dev/null @@ -1,49 +0,0 @@ -FROM navitia/master - -COPY navitia-common_*.deb navitia-jormungandr_*.deb ./ - -RUN apt-get update \ - && apt install -y curl python python-pip git libgeos-c1v5 libpq5 apache2 ./navitia-common_*.deb ./navitia-jormungandr_*.deb \ - && apt-get clean \ - && apt-get autoremove \ - && rm -rf /var/lib/apt/lists/* \ - && rm -rf navitia-common_*.deb navitia-jormungandr_*.deb - -# install jormungandr requirements -RUN pip install --no-cache-dir -r /usr/share/jormungandr/requirements.txt \ - && pip install uwsgi - - -# Add apache config -RUN rm /etc/apache2/sites-available/000-default.conf -COPY docker/apache/jormungandr.default.conf /etc/apache2/sites-available/000-default.conf -COPY docker/apache/apache2.conf /etc/apache2/apache2.conf -COPY docker/apache/envvars /etc/apache2/envvars -COPY docker/apache/logformat.conf /etc/apache2/conf-available/logformat.conf - -# jormungandr wsgi -COPY docker/jormungandr.wsgi /usr/src/app/jormungandr.wsgi -COPY docker/run_jormungandr.sh /usr/src/app/run.sh - -# Enable site and modes -RUN a2ensite 000-default.conf \ - && a2enmod rewrite \ - && a2enmod proxy \ - && a2enmod proxy_balancer \ - && a2enmod proxy_http \ - && a2enconf logformat.conf - -# Redirect apache log output to docker log collector -RUN ln -sf /proc/self/fd/1 /var/log/apache2/access.log \ - && ln -sf /proc/self/fd/2 /var/log/apache2/error.log - -WORKDIR /usr/src/app/ - -ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp -ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION=2 - -HEALTHCHECK CMD curl -f http://localhost/v1 || exit 1 - -EXPOSE 80 - -ENTRYPOINT [ "/usr/src/app/run.sh" ] diff --git a/docker/debian10/Dockerfile-kraken b/docker/debian10/Dockerfile-kraken deleted file mode 100644 index dd75ea1b84..0000000000 --- a/docker/debian10/Dockerfile-kraken +++ /dev/null @@ -1,54 +0,0 @@ -FROM navitia/master - -# copy package from context inside the docker -COPY navitia-kraken_*.deb navitia-monitor-kraken_*.deb ./ - -RUN apt-get update \ - && apt install -y ./navitia-kraken_*.deb \ - && apt install -y curl python python-pip \ - && apt install -y ./navitia-monitor-kraken_*.deb \ - && apt install -y netcat --autoremove \ - && apt-get clean \ - && rm -rf navitia-kraken_*.deb \ - && rm -rf navitia-monitor-kraken_*.deb \ - && pip install --no-cache-dir -r /usr/share/monitor_kraken/requirements.txt - -COPY docker/run_kraken.sh /usr/src/app/run_kraken.sh -RUN chmod +x /usr/src/app/run_kraken.sh - -WORKDIR / -RUN ln -s /usr/lib/python*/dist-packages/monitor_kraken/app.py monitor_kraken - -# Build libkeepalive https://libkeepalive.sourceforge.net/ -# libkeepalive is a library to keep tcp connection alive. The reason of doing so is that aws LoadBalancer, to which -# kraken is connected, check the aliveness continuously at regular interval of non-customisable 350s -# (https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html). If nothing happens during -# this interval, the connection will be dropped. It takes Kraken, by default, 7200s(/proc/sys/net/ipv4/tcp_keepalive_time) -# to be aware of the disconnection. By using libkeepalive, we can override tcp_keepalive_time, -# which is not customisable via docker-container. - -RUN apt install -y wget build-essential -WORKDIR /tmp/libkeepalive -COPY libkeepalive-0.3.tar.gz /tmp/libkeepalive/libkeepalive-0.3.tar.gz -RUN tar -xf libkeepalive-0.3.tar.gz && \ - cd libkeepalive-0.3 && \ - make && \ - mkdir /libkeepalive && \ - cp libkeepalive.so /libkeepalive && \ - rm -rf /tmp/libkeepalive -RUN apt purge -y wget build-essential -# User timeout = 300 + 30 * 6 = 480s -ENV KEEPIDLE=300 -ENV KEEPCNT=6 -ENV KEEPINTVL=30 -# if you are willing to use settings from os, set KEEPALIVE to off -# ENV KEEPALIVE=Off - -EXPOSE 30000 -ENV KRAKEN_GENERAL_zmq_socket=tcp://*:30000 -ENV KRAKEN_GENERAL_log_level=INFO -HEALTHCHECK CMD nc -z localhost 30000 || exit 1 - -ENV MONITOR_HOST=0.0.0.0 - -ENTRYPOINT ["bash", "/usr/src/app/run_kraken.sh"] diff --git a/docker/debian10/Dockerfile-master b/docker/debian10/Dockerfile-master deleted file mode 100644 index 99403d40bf..0000000000 --- a/docker/debian10/Dockerfile-master +++ /dev/null @@ -1,18 +0,0 @@ -FROM debian:10-slim - -# update package list from providers -RUN apt-get update && \ - apt-get upgrade -y && \ - apt-get install -y ca-certificates dh-python && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - - COPY docker/ca-certificates/*.crt /usr/local/share/ca-certificates/ - COPY docker/build_kraken.sh /build_kraken.sh - RUN chmod +x /build_kraken.sh - RUN update-ca-certificates - # Python 'requests' package handle its own CA certificate list - # Let's force it to use the OS's list - ENV REQUESTS_CA_BUNDLE /etc/ssl/certs - -CMD ["/build_kraken.sh"] diff --git a/docker/debian10/Dockerfile-mock-kraken b/docker/debian10/Dockerfile-mock-kraken deleted file mode 100644 index 9d765b741a..0000000000 --- a/docker/debian10/Dockerfile-mock-kraken +++ /dev/null @@ -1,20 +0,0 @@ -FROM navitia/master - -# copy package from context inside the docker -COPY mock-kraken_*.deb ./ - -RUN apt-get update \ - && apt install -y ./mock-kraken_*.deb \ - && apt-get clean \ - && rm -rf mock-kraken_*.deb - -RUN mkdir -p /srv/kraken \ - && ln -s /usr/bin/main_routing_test /srv/kraken/main_routing_test \ - && ln -s /usr/bin/departure_board_test /srv/kraken/departure_board_test - -WORKDIR /srv/kraken -EXPOSE 30000 -ENV KRAKEN_GENERAL_zmq_socket=tcp://*:30000 -ENV KRAKEN_GENERAL_log_level=INFO - -CMD [""] diff --git a/docker/debian10/Dockerfile-tyr-beat b/docker/debian10/Dockerfile-tyr-beat deleted file mode 100644 index 61f4bdd1f9..0000000000 --- a/docker/debian10/Dockerfile-tyr-beat +++ /dev/null @@ -1,27 +0,0 @@ -FROM navitia/master - -# used for data migrations -COPY source/sql/alembic /usr/share/navitia/ed/alembic -COPY source/sql/requirements.txt /tmp/requirements.txt -COPY source/cities /usr/share/navitia/cities -COPY docker/templates/* /templates/ - -COPY navitia-common_*.deb navitia-tyr_*.deb ./ -COPY docker/run_tyr_beat.sh / - -RUN apt-get update \ - && apt install -y python python-pip git postgresql-client libgeos-c1v5 libpq5 gettext-base jq python-pip ./navitia-common_*.deb ./navitia-tyr_*.deb \ - && apt-get clean \ - && rm -rf navitia-common_*.deb navitia-tyr_*.deb - -# install ed migrations requirements -RUN pip install -r /tmp/requirements.txt - -# install tyr requirements -RUN pip install --no-cache-dir -r /usr/share/tyr/requirements.txt - -RUN chmod +x /run_tyr_beat.sh - -WORKDIR /usr/share/tyr/ - -ENTRYPOINT ["bash", "/run_tyr_beat.sh"] diff --git a/docker/debian10/Dockerfile-tyr-web b/docker/debian10/Dockerfile-tyr-web deleted file mode 100644 index 358f51aee6..0000000000 --- a/docker/debian10/Dockerfile-tyr-web +++ /dev/null @@ -1,23 +0,0 @@ -FROM navitia/master - -COPY navitia-common_*.deb navitia-tyr_*.deb ./ - -RUN apt-get update \ - && apt install -y curl python python-pip postgresql-client git libgeos-c1v5 ./navitia-common_*.deb ./navitia-tyr_*.deb \ - && apt-get clean \ - && rm -rf navitia-common_*.deb navitia-tyr_*.deb - -# install tyr requirements -RUN pip install --no-cache-dir -r /usr/share/tyr/requirements.txt \ - && pip install uwsgi - -WORKDIR /usr/src/app/ - -RUN cp /usr/bin/manage_tyr.py /usr/src/app/manage_tyr.py - -COPY docker/run_tyr_web.sh /usr/src/app/run.sh -RUN chmod +x /usr/src/app/run.sh - -RUN ln -s /usr/share/tyr/migrations migrations - -ENTRYPOINT ["bash", "/usr/src/app/run.sh" ] diff --git a/docker/debian10/Dockerfile-tyr-worker b/docker/debian10/Dockerfile-tyr-worker deleted file mode 100644 index f905364a18..0000000000 --- a/docker/debian10/Dockerfile-tyr-worker +++ /dev/null @@ -1,38 +0,0 @@ -FROM navitia/master - -COPY navitia-common_*deb navitia-tyr_*.deb navitia-ed_*.deb navitia-cities_*.deb mimirsbrunn7_buster-*.deb cosmogony2cities_*.deb ./ - -RUN apt-get update \ - && apt install -y python python-pip git libgeos-c1v5 libpq5 \ - ./navitia-common_*.deb ./navitia-tyr_*.deb \ - ./navitia-ed_*.deb ./navitia-cities_*.deb \ - ./mimirsbrunn7_buster-*.deb ./cosmogony2cities*.deb \ - && apt-get clean \ - && rm -rf ./navitia-common_*.deb ./navitia-tyr_*.deb \ - ./navitia-ed_*.deb ./navitia-cities_*.deb \ - ./mimirsbrunn7_buster-*.deb ./cosmogony2cities*.deb - - - -RUN apt install -y --force-yes unzip -RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" -RUN unzip awscliv2.zip -RUN ./aws/install - -# install tyr requirements -RUN pip install --no-cache-dir -r /usr/share/tyr/requirements.txt - -EXPOSE 5000 - -WORKDIR /usr/src/app/ -RUN cp /usr/bin/manage_tyr.py /usr/src/app/manage_tyr.py - -# TODO change the user to remove this ugly C_FORCE_ROOT -ENV C_FORCE_ROOT=1 -ENV TYR_WORKER_N_PROC=1 - -COPY docker/run_tyr_worker.sh /usr/src/app/ - -RUN chmod +x /usr/src/app/run_tyr_worker.sh - -ENTRYPOINT ["bash", "/usr/src/app/run_tyr_worker.sh"] diff --git a/docker/debian8/Dockerfile-builder b/docker/debian8/Dockerfile-builder deleted file mode 100755 index 66b2b23530..0000000000 --- a/docker/debian8/Dockerfile-builder +++ /dev/null @@ -1,6 +0,0 @@ -FROM navitia/master - -COPY docker/build_kraken.sh /build_kraken.sh -RUN chmod +x /build_kraken.sh - -ENTRYPOINT ["bash", "/build_kraken.sh"] diff --git a/docker/debian8/Dockerfile-eitri b/docker/debian8/Dockerfile-eitri deleted file mode 100644 index 490ee35a8d..0000000000 --- a/docker/debian8/Dockerfile-eitri +++ /dev/null @@ -1,14 +0,0 @@ -FROM navitia/master - -# copy package from context inside the docker -COPY navitia-ed_*.deb / -COPY source/ /navitia/source/ - -# install navitia-ed package -RUN dpkg -i ./navitia-ed_*.deb || exit 0 -RUN apt-get install -f -y --force-yes - -RUN rm navitia-ed_*.deb - -# install eitri requirements -RUN pip install --no-cache-dir -r /navitia/source/eitri/requirements.txt diff --git a/docker/debian8/Dockerfile-instances-configurator b/docker/debian8/Dockerfile-instances-configurator deleted file mode 100644 index 5a5d9117a8..0000000000 --- a/docker/debian8/Dockerfile-instances-configurator +++ /dev/null @@ -1,15 +0,0 @@ -FROM navitia/master - - -COPY source/sql/alembic /usr/share/navitia/ed/alembic -COPY source/sql/requirements.txt /tmp/requirements.txt -RUN pip install -r /tmp/requirements.txt - -COPY source/cities /usr/share/navitia/cities - -COPY docker/instances_configuration.sh / -COPY docker/templates/* /templates/ - - - -ENTRYPOINT [ "sh", "-x", "/instances_configuration.sh" ] diff --git a/docker/debian8/Dockerfile-jormungandr b/docker/debian8/Dockerfile-jormungandr deleted file mode 100644 index 96c9eec03b..0000000000 --- a/docker/debian8/Dockerfile-jormungandr +++ /dev/null @@ -1,54 +0,0 @@ -FROM navitia/master - -COPY navitia-common_*.deb ./ -COPY navitia-jormungandr_*.deb ./ - -# install navitia-common package -RUN dpkg -i ./navitia-common_*.deb || exit 0 -RUN apt-get install -f -y --force-yes - -# install navitia-jormun package -RUN dpkg -i ./navitia-jormungandr_*.deb || exit 0 -RUN apt-get install -f -y --force-yes - -RUN apt-get install -y --force-yes apache2 \ - && apt-get clean \ - && apt-get autoremove - -# install jormungandr requirements -RUN pip install --no-cache-dir -r /usr/share/jormungandr/requirements.txt \ - && pip install uwsgi - -# Add apache config -RUN rm /etc/apache2/sites-available/000-default.conf -COPY docker/apache/jormungandr.default.conf /etc/apache2/sites-available/000-default.conf -COPY docker/apache/apache2.conf /etc/apache2/apache2.conf -COPY docker/apache/envvars /etc/apache2/envvars -COPY docker/apache/logformat.conf /etc/apache2/conf-available/logformat.conf - -# Enable site and modes -RUN a2ensite 000-default.conf \ - && a2enmod rewrite \ - && a2enmod proxy \ - && a2enmod proxy_balancer \ - && a2enmod proxy_http \ - && a2enconf logformat.conf - -# Redirect apache log output to docker log collector -RUN ln -sf /proc/self/fd/1 /var/log/apache2/access.log \ - && ln -sf /proc/self/fd/2 /var/log/apache2/error.log - -# jormungandr wsgi -COPY docker/jormungandr.wsgi /usr/src/app/jormungandr.wsgi -COPY docker/run_jormungandr.sh /usr/src/app/run.sh - -WORKDIR /usr/src/app/ - -HEALTHCHECK CMD curl -f http://localhost/v1 || exit 1 - -ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp -ENV PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION=2 - -EXPOSE 80 - -ENTRYPOINT [ "/usr/src/app/run.sh" ] diff --git a/docker/debian8/Dockerfile-kraken b/docker/debian8/Dockerfile-kraken deleted file mode 100644 index 76941d3cf0..0000000000 --- a/docker/debian8/Dockerfile-kraken +++ /dev/null @@ -1,57 +0,0 @@ -FROM navitia/master - - -# copy package from context inside the docker -COPY navitia-kraken_*.deb navitia-monitor-kraken_*.deb ./ - -# install navitia-kraken package -RUN dpkg -i ./navitia-kraken_*.deb ./navitia-monitor-kraken_*.deb -RUN apt-get install -f -y --force-yes - -RUN rm navitia-kraken_*.deb \ - navitia-monitor-kraken_*.deb - -# put the binary in /srv/kraken -RUN mkdir -p /srv/kraken -RUN cp /usr/bin/kraken /srv/kraken/kraken - -RUN pip install --no-cache-dir -r /usr/share/monitor_kraken/requirements.txt - -COPY docker/run_kraken.sh /usr/src/app/run_kraken.sh -RUN chmod +x /usr/src/app/run_kraken.sh - -WORKDIR / -RUN ln -s /usr/lib/python*/dist-packages/monitor_kraken/app.py monitor_kraken - -EXPOSE 30000 - -ENV KRAKEN_GENERAL_zmq_socket=tcp://*:30000 -ENV KRAKEN_GENERAL_log_level=INFO -HEALTHCHECK CMD nc -z localhost 30000 || exit 1 - -ENV MONITOR_HOST=0.0.0.0 - -# Build libkeepalive https://libkeepalive.sourceforge.net/ -# libkeepalive is a library to keep tcp connection alive. The reason of doing so is that aws LoadBalancer, to which -# kraken is connected, check the aliveness continuously at regular interval of non-customisable 350s -# (https://docs.aws.amazon.com/elasticloadbalancing/latest/network/network-load-balancers.html). If nothing happens during -# this interval, the connection will be dropped. It takes Kraken, by default, 7200s(/proc/sys/net/ipv4/tcp_keepalive_time) -# to be aware of the disconnection. By using libkeepalive, we can override tcp_keepalive_time, -# which is not customisable via docker-container. - -WORKDIR /tmp/libkeepalive -COPY docker/libkeepalive-0.3/ /tmp/libkeepalive/ -RUN cd /tmp/libkeepalive/ && \ - make && \ - mkdir /libkeepalive && \ - cp /tmp/libkeepalive/libkeepalive.so /libkeepalive && \ - rm -rf /tmp/libkeepalive - -# User timeout = 300 + 30 * 6 = 480s -ENV KEEPIDLE=300 -ENV KEEPCNT=6 -ENV KEEPINTVL=30 -# if you are willing to use settings from os, set KEEPALIVE to off -# ENV KEEPALIVE=Off - -ENTRYPOINT ["bash", "/usr/src/app/run_kraken.sh"] diff --git a/docker/debian8/Dockerfile-master b/docker/debian8/Dockerfile-master deleted file mode 100755 index d628fdac3e..0000000000 --- a/docker/debian8/Dockerfile-master +++ /dev/null @@ -1,41 +0,0 @@ -FROM 162230498103.dkr.ecr.eu-west-1.amazonaws.com/mutable-debian8_dev:latest - -RUN echo "deb http://archive.debian.org/debian/ jessie main" > /etc/apt/sources.list.d/archive.list - -# update package list from providers -RUN apt-get update --force-yes --fix-missing || exit 0 - -# install postgresql-client for tyr-beat -# netcat for kraken -# curl for jormun -RUN apt-get install -y --force-yes ca-certificates \ - postgresql-client \ - netcat \ - dh-python \ - curl - -COPY docker/ca-certificates/*.crt /usr/local/share/ca-certificates/ -RUN update-ca-certificates - -# install a more recent version of CMake -RUN apt-get remove --yes cmake \ - && cd /tmp \ - && wget https://cmake.org/files/v3.18/cmake-3.18.6-Linux-x86_64.tar.gz \ - && tar xf cmake-3.18.6-Linux-x86_64.tar.gz --strip-components=1 --directory=/usr/local \ - && rm -f cmake-3.18.6-Linux-x86_64.tar.gz \ - && cd - - -# Python 'requests' package handle its own CA certificate list -# Let's force it to use the OS's list -ENV REQUESTS_CA_BUNDLE /etc/ssl/certs/ca-certificates.crt - -# install rustup -ENV RUSTUP_HOME=/usr/local/rustup \ - CARGO_HOME=/usr/local/cargo - -ENV PATH=/usr/local/cargo/bin:${PATH} -RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --no-modify-path --profile minimal \ - && chmod -R a+wrX ${RUSTUP_HOME} ${CARGO_HOME} \ - && rustup --version \ - && cargo --version \ - && rustc --version diff --git a/docker/debian8/Dockerfile-mock-kraken b/docker/debian8/Dockerfile-mock-kraken deleted file mode 100644 index 647b45ccc6..0000000000 --- a/docker/debian8/Dockerfile-mock-kraken +++ /dev/null @@ -1,20 +0,0 @@ -FROM navitia/master - -# copy package from context inside the docker -COPY mock-kraken_*.deb ./ - -# install mock-kraken package -RUN dpkg -i ./mock-kraken_*.deb -RUN apt-get install -f -y --force-yes - -RUN rm mock-kraken_*.deb - -RUN mkdir -p /srv/kraken -RUN cp /usr/bin/main_routing_test /srv/kraken/ -RUN cp /usr/bin/departure_board_test /srv/kraken/ -WORKDIR /srv/kraken -EXPOSE 30000 -ENV KRAKEN_GENERAL_zmq_socket=tcp://*:30000 -ENV KRAKEN_GENERAL_log_level=INFO - - diff --git a/docker/debian8/Dockerfile-tyr-beat b/docker/debian8/Dockerfile-tyr-beat deleted file mode 100644 index 01b026b324..0000000000 --- a/docker/debian8/Dockerfile-tyr-beat +++ /dev/null @@ -1,38 +0,0 @@ -FROM navitia/master - -COPY navitia-common_*.deb navitia-tyr_*.deb ./ - -RUN apt-get update \ - && apt install -y --force-yes jq - -# used for data migrations -COPY source/sql/alembic /usr/share/navitia/ed/alembic -COPY source/sql/requirements.txt /tmp/requirements.txt -COPY source/cities /usr/share/navitia/cities -COPY docker/templates/* /templates/ - -# install navitia-common package -RUN dpkg -i ./navitia-common_*.deb || exit 0 - -# install navitia-tyr package -RUN dpkg -i ./navitia-tyr_*.deb || exit 0 - -RUN apt-get install -f -y --force-yes - -RUN rm navitia-common_*.deb \ - navitia-tyr_*.deb - -# install ed migrations requirements -RUN pip install -r /tmp/requirements.txt - -# install tyr requirements -RUN pip install --no-cache-dir -r /usr/share/tyr/requirements.txt - -# install postgresql-client -RUN apt-get install -y --force-yes postgresql-client - -COPY docker/run_tyr_beat.sh / -RUN chmod +x /run_tyr_beat.sh - -WORKDIR /usr/share/tyr/ -ENTRYPOINT ["bash", "/run_tyr_beat.sh"] diff --git a/docker/debian8/Dockerfile-tyr-web b/docker/debian8/Dockerfile-tyr-web deleted file mode 100644 index 6a49c6ba0f..0000000000 --- a/docker/debian8/Dockerfile-tyr-web +++ /dev/null @@ -1,33 +0,0 @@ -FROM navitia/master - -COPY navitia-common_*.deb navitia-tyr_*.deb ./ - -# install navitia-common package -RUN dpkg -i ./navitia-common_*.deb || exit 0 - -# install navitia-tyr package -RUN dpkg -i ./navitia-tyr_*.deb || exit 0 - -RUN apt-get install -f -y --force-yes - -RUN rm navitia-common_*.deb \ - navitia-tyr_*.deb - -# install postgresql-client -RUN apt-get install -y --force-yes postgresql-client - -# install tyr requirements -RUN pip install --no-cache-dir -r /usr/share/tyr/requirements.txt - -# install uwsgi -RUN pip install uwsgi==2.0.22 - -WORKDIR /usr/src/app/ -RUN cp /usr/bin/manage_tyr.py /usr/src/app/manage_tyr.py - -COPY docker/run_tyr_web.sh /usr/src/app/run.sh -RUN chmod +x /usr/src/app/run.sh - -RUN ln -s /usr/share/tyr/migrations migrations - -ENTRYPOINT ["bash", "/usr/src/app/run.sh" ] diff --git a/docker/debian8/Dockerfile-tyr-worker b/docker/debian8/Dockerfile-tyr-worker deleted file mode 100644 index 7e5b56100d..0000000000 --- a/docker/debian8/Dockerfile-tyr-worker +++ /dev/null @@ -1,69 +0,0 @@ -FROM navitia/master - -# Install some binaries from tartare-tools -ENV TARTARE_TOOLS_VERSION="v0.47.1" -ARG GITHUB_TOKEN -RUN git config --global url."https://x-access-token:${GITHUB_TOKEN}@github.com/hove-io/".insteadOf "ssh://git@github.com/hove-io/" -RUN git clone -b ${TARTARE_TOOLS_VERSION} --depth 1 https://x-access-token:${GITHUB_TOKEN}@github.com/hove-io/tartare-tools - -# rustc 1.80.0 broke the compilation -RUN rustup install 1.79.0 - -RUN cd tartare-tools \ - && cargo +1.79.0 build --release -p enrich-ntfs-with-addresses \ - && cargo +1.79.0 build --release -p split-trip-geometries \ - && cp target/release/enrich-ntfs-with-addresses /usr/bin/ \ - && cp target/release/split-trip-geometries /usr/bin/ \ - && cd .. \ - && rm -rf tartare-tools - -COPY navitia-common_*deb navitia-tyr_*.deb navitia-ed_*.deb navitia-cities_*.deb mimirsbrunn7_jessie-*.deb cosmogony2cities_*.deb mimirsbrunn-config-*.deb ./ - -# install navitia-common package -RUN dpkg -i ./navitia-common_*.deb || exit 0 - -# install navitia-tyr package -RUN dpkg -i ./navitia-tyr_*.deb || exit 0 - -# install navitia-ed package -RUN dpkg -i ./navitia-ed_*.deb || exit 0 - -# install navitia-cities package -RUN dpkg -i ./navitia-cities_*.deb || exit 0 - -# install navitia-mimirsbrunn package -RUN dpkg -i ./mimirsbrunn7_jessie-*.deb \ - # install cosmogony2cities package - ./cosmogony2cities*.deb \ - # install conf mimir package - ./mimirsbrunn-config-*.deb - -RUN apt-get install -f -y --force-yes - -RUN rm navitia-common_*deb \ - navitia-tyr_*.deb \ - navitia-cities_*.deb \ - navitia-ed_*.deb \ - mimirsbrunn-config-*.deb \ - mimirsbrunn7_jessie-*.deb \ - cosmogony2cities_*.deb - -# install tyr requirements -RUN pip install --no-cache-dir -r /usr/share/tyr/requirements.txt - -RUN apt install -y --force-yes unzip -RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" -RUN unzip awscliv2.zip -RUN ./aws/install - - -EXPOSE 5000 - - -WORKDIR /usr/src/app/ -RUN cp /usr/bin/manage_tyr.py /usr/src/app/manage_tyr.py - -# TODO change the user to remove this ugly C_FORCE_ROOT -ENV C_FORCE_ROOT=1 -ENV TYR_WORKER_N_PROC=1 -ENTRYPOINT ["celery", "worker", "-A", "tyr.tasks", "-O", "fair"]