diff --git a/.gitignore b/.gitignore index d967199..dccbf22 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ -environments/db.env -environments/keycloak.env +environments/*.env +qiita_logs/*log +qiita_logs/*.pid \ No newline at end of file diff --git a/Images/nginx/Dockerfile b/Images/nginx/Dockerfile new file mode 100644 index 0000000..2d7b039 --- /dev/null +++ b/Images/nginx/Dockerfile @@ -0,0 +1,83 @@ +FROM ubuntu:22.04 + +ARG MINIFORGE_VERSION=24.1.2-0 +ARG MODZIP_VERSION=1.3.0 +ARG NGINX_VERSION=1.26.0 + +ENV CONDA_DIR=/opt/conda +ENV PATH=${CONDA_DIR}/bin:${PATH} + +RUN apt-get -y update +RUN apt-get -y install \ + git \ + wget \ + libpcre2-dev \ + libxslt-dev \ + libgd-dev \ + libssl-dev +RUN apt-get -y install build-essential +# install miniforge3 for "conda" +# see https://github.com/conda-forge/miniforge-images/blob/master/ubuntu/Dockerfile +RUN wget https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-x86_64.sh -O /tmp/miniforge3.sh && \ + /bin/bash /tmp/miniforge3.sh -b -p ${CONDA_DIR} && \ + echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> /etc/skel/.bashrc && \ + echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> ~/.bashrc \ + conda init +RUN conda create --quiet --yes -n nginx + +SHELL ["conda", "run", "-n", "nginx", "/bin/bash", "-c"] + +RUN wget https://github.com/evanmiller/mod_zip/archive/refs/tags/${MODZIP_VERSION}.tar.gz -O /usr/local/src/mod_zip-${MODZIP_VERSION}.tar.gz +RUN cd /usr/local/src/ && tar xzvf mod_zip-${MODZIP_VERSION}.tar.gz +RUN wget https://github.com/nginx/nginx/archive/refs/tags/release-${NGINX_VERSION}.tar.gz -O /usr/local/src/nginx-${NGINX_VERSION}.tar.gz +RUN cd /usr/local/src/ && tar xzvf nginx-${NGINX_VERSION}.tar.gz +# fix include for the iconv header +RUN sed "s|^#include |#include \"/usr/include/iconv.h\"|" -i /usr/local/src/mod_zip-${MODZIP_VERSION}/ngx_http_zip_file.c +# ensure runtime library paths are correct and openssl headers can be found at compile time +RUN cd /usr/local/src/nginx-release-${NGINX_VERSION} && ./auto/configure \ + --http-log-path=var/log/nginx/access.log \ + --error-log-path=var/log/nginx/error.log \ + --pid-path=var/run/nginx/nginx.pid \ + --lock-path=var/run/nginx/nginx.lock \ + --http-client-body-temp-path=var/tmp/nginx/client \ + --http-proxy-temp-path=var/tmp/nginx/proxy \ + --http-fastcgi-temp-path=var/tmp/nginx/fastcgi \ + --http-scgi-temp-path=var/tmp/nginx/scgi \ + --http-uwsgi-temp-path=var/tmp/nginx/uwsgi \ + --sbin-path=sbin/nginx \ + --conf-path=etc/nginx/nginx.conf \ + --modules-path=lib/nginx/modules \ + --with-threads \ + --with-http_ssl_module \ + --with-http_v2_module \ + --with-http_realip_module \ + --with-http_addition_module \ + --with-http_sub_module \ + --with-http_gunzip_module \ + --with-http_gzip_static_module \ + --with-http_auth_request_module \ + --with-http_secure_link_module \ + --with-http_stub_status_module \ + --with-http_xslt_module=dynamic \ + --with-stream=dynamic \ + --with-http_image_filter_module=dynamic \ + --with-pcre \ + --with-pcre-jit \ + --with-cc-opt=" -I $CONDA_DIR/envs/qiita/include/openssl " \ + --with-ld-opt="" \ + --prefix=/usr/local \ + --add-module=/usr/local/src/mod_zip-${MODZIP_VERSION} \ + --with-ld-opt=" -Wl,-rpath,$CONDA_DIR/envs/qiita/lib " +RUN cd /usr/local/src/nginx-release-${NGINX_VERSION} && make +RUN cd /usr/local/src/nginx-release-${NGINX_VERSION} && make install + +COPY nginx_qiita.conf . +COPY start_nginx.sh . + +RUN chmod 777 nginx_qiita.conf +RUN chmod 777 start_nginx.sh + +RUN mkdir /var/log/nginx + +#ENTRYPOINT ["/bin/bash", "-l", "-c" ] +ENTRYPOINT ["conda", "run", "-n", "nginx", "./start_nginx.sh"] \ No newline at end of file diff --git a/Images/nginx/nginx_qiita.conf b/Images/nginx/nginx_qiita.conf new file mode 100644 index 0000000..426c3e2 --- /dev/null +++ b/Images/nginx/nginx_qiita.conf @@ -0,0 +1,95 @@ +user nobody nogroup; +daemon off; +# error_log /var/log/nginx/error_log warn; +error_log /qiita_logs/nginx_error_log warn; + +events { + worker_connections 1024; +} + +http { + client_max_body_size 7M; # increase maximum body size from default 1M to match https://github.com/qiita-spots/qiita/blob/ac62aba5333f537c32e213855edc39c273aa9871/qiita_pet/static/vendor/js/resumable-uploader.js#L51 (which is 3M). Note that resumable-uploader.js's last chunk can be max. twice as large as chunk size, see: https://github.com/23/resumable.js/issues/51 + + # ports to redirect for mainqiita + upstream mainqiita { + server qiita:21174; + server qiita_worker:21175; + server qiita_worker:21176; + server qiita_worker:21177; + } + + # define variables for the actions that shall be taken for websocket handshake + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + # listening to 8080 and redirecting to https + #server { + # listen 8383; + # server_name localhost; + # return 301 https://$server_name$request_uri; + #} + + server { + listen 8383 ssl; + server_name _; + merge_slashes off; + + access_log /qiita_logs/nginx_access_log; + + ssl_certificate /qiita/qiita_core/support_files/ci_server.crt; + ssl_certificate_key /qiita/qiita_core/support_files/ci_server.key; + + ssl_session_timeout 5m; + + # no catche + expires off; + + port_in_redirect off; + + # download configuration, based on: + # https://groups.google.com/forum/#!topic/python-tornado/sgadmx8Hd_s + + # protected location for working diretory + location /protected-working_dir/ { + internal; + + # CHANGE ME: This should match the WORKING_DIR in your qiita + # config. E.g., + alias /qiita/qiita_db/support_files/test_data/working_dir/; + } + + # protected location + location /protected/ { + internal; + + # CHANGE ME: This should match the BASE_DATA_DIR in your qiita + # config. E.g., + alias /qiita/qiita_db/support_files/test_data/; + } + + # enables communiction through websockets. + # Currently, only endpoints /consumer/, /analysis/selected/socket/, and /study/list/socket/ use websockets + # not needed for our local docker setup + # location ~ ^/(consumer|analysis/selected/socket|study/list/socket)/ { + # proxy_pass $scheme://mainqiita; + # proxy_set_header Host $http_host; + # proxy_redirect http:// https://; + # proxy_http_version 1.1; + # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # proxy_set_header Upgrade $http_upgrade; + # proxy_set_header Connection $connection_upgrade; + # proxy_set_header X-Forwarded-Host $http_host; + # } + + location / { + proxy_pass $scheme://mainqiita; + proxy_redirect off; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Accept-Encoding identity; + } + } +} diff --git a/Images/nginx/start_nginx.sh b/Images/nginx/start_nginx.sh new file mode 100644 index 0000000..728f7b7 --- /dev/null +++ b/Images/nginx/start_nginx.sh @@ -0,0 +1,4 @@ +#!/bin/bash +mkdir -p /opt/conda/envs/nginx/var/run/nginx/ /usr/local/var/tmp/nginx/ + +nginx -c /nginx_qiita.conf \ No newline at end of file diff --git a/Images/qiita/Dockerfile b/Images/qiita/Dockerfile new file mode 100644 index 0000000..678a60e --- /dev/null +++ b/Images/qiita/Dockerfile @@ -0,0 +1,79 @@ +FROM ubuntu:24.04 + +ARG MINIFORGE_VERSION=24.1.2-0 +ARG MODZIP_VERSION=1.3.0 +ARG NGINX_VERSION=1.26.0 + +ENV CONDA_DIR=/opt/conda +ENV PATH=${CONDA_DIR}/bin:${PATH} + +RUN apt-get -y update +# install following packages for nginx compilation: libpcre2-dev, libxslt-dev and libgd-dev +RUN apt-get -y --fix-missing install \ + git \ + wget \ + libpq-dev \ + python3-dev \ + gcc \ + libpcre2-dev \ + libxslt-dev \ + libgd-dev \ + postgresql-client +RUN apt-get -y install build-essential +# install miniforge3 for "conda" +# see https://github.com/conda-forge/miniforge-images/blob/master/ubuntu/Dockerfile +RUN wget https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-x86_64.sh -O /tmp/miniforge3.sh && \ + /bin/bash /tmp/miniforge3.sh -b -p ${CONDA_DIR} && \ + echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> /etc/skel/.bashrc && \ + echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> ~/.bashrc \ + conda init + +# create conda env for qiita with all necessary dependencies (conda and pip) +RUN conda create --quiet --yes -n qiita python=3.9 pip libgfortran numpy cython anaconda::redis +# TODO: Redis container +# Make RUN commands use the new environment: +# append --format docker to the build command, see https://github.com/containers/podman/issues/8477 +SHELL ["conda", "run", "-n", "qiita", "/bin/bash", "-c"] + +RUN pip install -U pip +RUN pip install \ + sphinx \ + sphinx-bootstrap-theme \ + nose-timer \ + Click \ + coverage \ + psycopg2-binary + + +# Clone the Qiita Repo +# RUN git clone -b master https://github.com/qiita-spots/qiita.git +RUN git clone -b auth_oidc https://github.com/jlab/qiita.git + +# We need to install necessary dependencies +# as well as some extra dependencies for psycopg2 to work +RUN git clone https://github.com/psycopg/psycopg2.git +RUN export PATH=/usr/lib/postgresql/14.11/bin/:$PATH +RUN pip install -e psycopg2/. + +# Install pip packaages for Qiita +RUN pip install -e qiita --no-binary redbiom + + +# Copy modified config file to the container + COPY config_qiita_oidc.cfg /qiita/ + RUN chmod 755 /qiita/config_qiita_oidc.cfg + +# Copy Bash Script to run Qiita to the container + COPY start_qiita.sh . + RUN chmod 755 start_qiita.sh + + COPY start_qiita_worker.sh . + RUN chmod 755 start_qiita_worker.sh + +COPY supervisor_foreground.conf . +RUN chmod 755 supervisor_foreground.conf + +# I will leave this ENTRYPOINT here as a comment in case debugging +# is necessary +# SHELL ["/bin/bash"] +ENTRYPOINT ["conda", "run", "-n", "qiita"] \ No newline at end of file diff --git a/Images/qiita/config_qiita_oidc.cfg b/Images/qiita/config_qiita_oidc.cfg new file mode 100644 index 0000000..17e466f --- /dev/null +++ b/Images/qiita/config_qiita_oidc.cfg @@ -0,0 +1,265 @@ +# WARNING!!!! DO NOT MODIFY THIS FILE +# IF YOU NEED TO PROVIDE YOUR OWN CONFIGURATION, COPY THIS FILE TO A NEW +# LOCATION AND EDIT THE COPY + +# ----------------------------------------------------------------------------- +# Copyright (c) 2014--, The Qiita Development Team. +# +# Distributed under the terms of the BSD 3-clause License. +# +# The full license is in the file LICENSE, distributed with this software. +# ----------------------------------------------------------------------------- + +# ------------------------------ Main settings -------------------------------- +[main] +# Change to FALSE in a production system +TEST_ENVIRONMENT = TRUE + +# Absolute path to the directory where log files are saved. If not given, no +# log file will be created +LOG_DIR = /qiita_logs/ + +# Whether studies require admin approval to be made available +REQUIRE_APPROVAL = True + +# Base URL: DO NOT ADD TRAILING SLASH +BASE_URL = https://localhost:8383 + +# Download path files +UPLOAD_DATA_DIR = /qiita/qiita_db/support_files/test_data/uploads/ + +# Working directory path +WORKING_DIR = /qiita/qiita_db/support_files/test_data/working_dir/ + +# Maximum upload size (in Gb) +MAX_UPLOAD_SIZE = 100 + +# Path to the base directory where the data files are going to be stored +BASE_DATA_DIR = /qiita/qiita_db/support_files/test_data/ + +# Valid upload extension, comma separated. Empty for no uploads +VALID_UPLOAD_EXTENSION = fastq,fastq.gz,txt,tsv,sff,fna,qual + +# The script used to start the qiita environment, if any +# used to spawn private CLI to a cluster +QIITA_ENV = source activate qiita + +# Script used for launching private Qiita tasks +PRIVATE_LAUNCHER = qiita-private-launcher + +# Script used for launching plugins +PLUGIN_LAUNCHER = qiita-plugin-launcher + +# Plugins configuration directory +PLUGIN_DIR = /qiita/plugins/ + +# Webserver certificate file paths +CERTIFICATE_FILE = +KEY_FILE = + +# The value used to secure cookies used for user sessions. A suitable value can +# be generated with: +# +# python -c "from base64 import b64encode;\ +# from uuid import uuid4;\ +# print b64encode(uuid4().bytes + uuid4().bytes)" +COOKIE_SECRET = SECRET + +# The value used to secure JWTs for delegated permission artifact download. +JWT_SECRET = SUPER_SECRET + +# Address a user should write to when asking for help +HELP_EMAIL = foo@bar.com + +# The email address, Qiita sends internal notifications to a sys admin +SYSADMIN_EMAIL = jeff@bar.com + +# ----------------------------- SMTP settings ----------------------------- +[smtp] +# The hostname to connect to +# Google: smtp.google.com +HOST = localhost + +# The port to connect to the database +# Google: 587 +PORT = 25 + +# SSL needed (True or False) +# Google: True +SSL = False + +# The user name to connect with +USER = + +# The user password to connect with +PASSWORD = + +# The email to have messages sent from +EMAIL = example@domain.com + +# ----------------------------- Redis settings -------------------------------- +[redis] +HOST = redis +PORT = 7777 +PASSWORD = +# The redis database you will use, redis has a max of 16. +# Qiita should have its own database +DB = 13 + +# ----------------------------- Postgres settings ----------------------------- +[postgres] +# The user name to connect to the database +USER = postgres + +# The administrator user, which can be used to create/drop environments +ADMIN_USER = postgres + +# The database to connect to +DATABASE = qiita_test + +# The host where the database lives on +HOST = qiita-db + +# The port to connect to the database +PORT = 5432 + +# The password to use to connect to the database +PASSWORD = postgres + +# The postgres password for the admin_user +ADMIN_PASSWORD = postgres + +# ----------------------------- Job Scheduler Settings ----------------------------- +[job_scheduler] +# The email address of the submitter of jobs +JOB_SCHEDULER_JOB_OWNER = user@somewhere.org + +# The number of seconds to wait between successive calls +JOB_SCHEDULER__POLLING_VALUE = 15 + +# Hard upper-limit on concurrently running validator jobs +JOB_SCHEDULER_PROCESSING_QUEUE_COUNT = 2 + +# ----------------------------- EBI settings ----------------------------- +[ebi] +# The user to use when submitting to EBI +EBI_SEQ_XFER_USER = Webin-41528 + +# Password for the above user +EBI_SEQ_XFER_PASS = + +# URL of EBI's FASP site +EBI_SEQ_XFER_URL = webin.ebi.ac.uk + +# URL of EBI's HTTPS dropbox +# live submission URL +#EBI_DROPBOX_URL = https://www.ebi.ac.uk/ena/submit/drop-box/submit/ +# testing URL +EBI_DROPBOX_URL = https://www-test.ebi.ac.uk/ena/submit/drop-box/submit/ + +# The name of the sequencing center to use when doing EBI submissions +EBI_CENTER_NAME = qiita-test + +# This string (with an underscore) will be prefixed to your EBI submission and +# study aliases +EBI_ORGANIZATION_PREFIX = example_organization + +# ----------------------------- VAMPS settings ----------------------------- +[vamps] +# general info to submit to vamps +USER = user +PASSWORD = password +URL = https://vamps.mbl.edu/mobe_workshop/getfile.php + +# ----------------------------- Portal settings ----------------------------- +[portal] + +# Portal the site is working under +PORTAL = QIITA + +# Portal subdirectory +PORTAL_DIR = + +# Full path to portal styling config file +PORTAL_FP = + +# The center latitude of the world map, shown on the Stats map. +# Defaults to 40.01027 (Boulder, CO, USA) +STATS_MAP_CENTER_LATITUDE = + +# The center longitude of the world map, shown on the Stats map. +# Defaults to -105.24827 (Boulder, CO, USA) +STATS_MAP_CENTER_LONGITUDE = + +# ----------------------------- iframes settings --------------------------- +[iframe] +# The real world QIIMP will always need to be accessed with https because Qiita +# runs on https too +QIIMP = https://localhost:8898/ + + +# --------------------- External Identity Provider settings -------------------- +# user authentication happens per default within Qiita, i.e. when a user logs in, +# the stored password hash and email address is compared against what a user +# just provided. You might however, use an external identity provider (IdP) to +# authenticate the user like +# google: https://developers.google.com/identity/protocols/oauth2 or +# github: https://docs.github.com/en/apps/oauth-apps/building-oauth-apps/authorizing-oauth-apps or +# self hosted keycloak: https://www.keycloak.org/ +# Thus, you don't have to deal with user verification, reset passwords, ... +# Authorization (i.e. if the authorized user is allowed to use Qiita or which +# user level he/she gets assigned is an independent process. You can even use +# multiple independent external identity providers! +# Qiita currently only support the "open ID connect" protocol with the implicit flow. +# Each identity provider comes as its own config section [oidc_foo] and needs +# to specify the following five fields: +# +# Typical identity provider manage multiple "realms" and specific "clients" per realm +# You need to contact your IdP and register Qiita as a new "client". The IdP will +# provide you with the correct values. +# +# The authorization protocol requires three steps to obtain user information: +# 1) you identify as the correct client and ask the IdP for a request code +# You have to forward the user to the login page of your IdP. To let the IdP +# know how to come back to Qiita, you need to provide a redirect URL +# 2) you exchange the code for a user token +# 3) you obtain information about the user for the obtaines user token +# Typically, each step is implemented as a separate URL endpoint +# +# To activate IdP: comment out the following config section + +# [oidc_localkeycloak] +# +## client ID for Qiita as registered at your Identity Provider of choice +# CLIENT_ID = qiita +# +## client secret to verify Qiita as the correct client. Not all IdPs require +## a client secret! +# CLIENT_SECRET = supersecretString + +# +## redirect URL (end point in your Qiita instance), to which the IdP redirects +## after user types in his/her credentials. If you don't want to change code in +## qiita_pet/webserver.py the URL must follow the pattern: +## base_URL/auth/login_OIDC/foo where foo is the name of this config section +## without the oidc_ prefix! +# REDIRECT_ENDPOINT = /auth/login_OIDC/localkeycloak +# +## The URL of the well-known json document, specifying how API end points +## like 'authorize', 'token' or 'userinfo' are defined. See e.g. +## https://swagger.io/docs/specification/authentication/ +## openid-connect-discovery/ +# WELLKNOWN_URI = http://localhost:9999/realms/qiita_realm/.well-known/openid-configuration +# +## a speaking label for the Identity Provider. Section name is used if empty. +# LABEL = localkeycloak +# +## The scope, i.e. fields about a user, which Qiita requests from the +## Identity Provider, e.g. "profile email eduperson_orcid". +## Will be automatically extended by the scope "openid", to enable the +## "authorize_code" OIDC flow. +# SCOPE = openid +# +##Optional. Name of a file in qiita_pet/static/img that shall be +##displayed for login through Service Provider, instead of a plain button +# LOGO = diff --git a/Images/qiita/start_qiita.sh b/Images/qiita/start_qiita.sh new file mode 100644 index 0000000..7cf8e42 --- /dev/null +++ b/Images/qiita/start_qiita.sh @@ -0,0 +1,11 @@ +#!/bin/bash +#sleep 300000 +export QIITA_CONFIG_FP="/qiita/config_qiita_oidc.cfg" + +if [ "$( psql -h localhost -U postgres -XtAc "SELECT 1 FROM postgres WHERE datname='qiita_test'" )" = '1' ] +then + qiita pet webserver --no-build-docs start --port 21174 --master +else + qiita-env make --no-load-ontologies + qiita pet webserver --no-build-docs start --port 21174 --master +fi diff --git a/Images/qiita/start_qiita_worker.sh b/Images/qiita/start_qiita_worker.sh new file mode 100755 index 0000000..8316b9c --- /dev/null +++ b/Images/qiita/start_qiita_worker.sh @@ -0,0 +1,7 @@ +#!/bin/bash + + +export QIITA_CONFIG_FP="/qiita/config_qiita_oidc.cfg" + +qiita pet webserver --no-build-docs start --port 21175 + diff --git a/Images/qiita/supervisor_foreground.conf b/Images/qiita/supervisor_foreground.conf new file mode 100644 index 0000000..1016fe2 --- /dev/null +++ b/Images/qiita/supervisor_foreground.conf @@ -0,0 +1,34 @@ +[supervisorctl] +serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL for a unix socket + +[supervisord] +logfile=/qiita_logs/supervisord.log ; main log file; default $CWD/supervisord.log +loglevel=debug ; log level; default info; others: debug,warn,trace +pidfile=/qiita_logs/supervisord.pid ; supervisord pidfile; default supervisord.pid +nodaemon=true + +[include] +files=/home/travis/miniconda3/envs/qiita/bin + +[group:qiita_all] +programs=qiita_workers_0,qiita_workers_1,qiita_workers_2,qiita_master ; each refers to 'x' in [program:x] definitions + +[program:qiita_master] +command=qiita pet webserver --no-build-docs start --port 21174 --master +process_name=%(program_name)s ; process_name expr (default %(program_name)s) +numprocs=1 ; number of processes copies to start (def 1) + +[program:qiita_workers_0] +command=qiita pet webserver --no-build-docs start --port 21175 +process_name=%(program_name)s ; process_name expr (default %(program_name)s) +numprocs=1 ; number of processes copies to start (def 1) + +[program:qiita_workers_1] +command=qiita pet webserver --no-build-docs start --port 21176 +process_name=%(program_name)s ; process_name expr (default %(program_name)s) +numprocs=1 ; number of processes copies to start (def 1) + +[program:qiita_workers_2] +command=qiita pet webserver --no-build-docs start --port 21177 +process_name=%(program_name)s ; process_name expr (default %(program_name)s) +numprocs=1 ; number of processes copies to start (def 1) diff --git a/README.md b/README.md new file mode 100644 index 0000000..1946766 --- /dev/null +++ b/README.md @@ -0,0 +1,41 @@ +**IMPORTANT: Have docker installed!** +**THIS VERSION CURRENTLY ONLY WORKS WITH DOCKER, NOT WITH PODMAN** +**FOR TESTING ON LOCAL MACHINES** + +### Hopefully "foolproof" instructions: +0. Log files will be mounted at qiita_logs on your local machine in this repo directory. Otherwise, change the file path to your desired path in the compose file as well as in the qiita, nginx and supervisord conf. +1. Clone repository +2. Move into Image Folder `cd Images/qiita` +3. Build docker image `docker build . -f qiita/Dockerfile -t local-qiita` +4. Build the nginx Image the same way as the qiita image, only in the nginx folder, using the image tag `local-nginx_qiita`. +5. Move to folder containing compose file `cd ../..` +6. Copy the `qiita_db.env.example` and the `qiita.env.example` files, configure them to your needs, and delete the `.example` from the file names. +7. Run docker compose `docker compose up qiita-db redis nginx qiita` +8. Open `http://localhost:8383` +9. To stop: Run `docker compose down qiita nginx qiita-db redis` + - Use `docker compose down qiita nginx qiita-db --volumes`if you wish to remove all associated volumes as well. + +Extras: +- If you want to remove a specific volume `docker volume rm ` +- If you want to access a container `docker ps`to fetch the ID and `docker exec -it bash` + +### IF YOU WANT TO USE LOCAL KEYCLOAK: + +1. Clone repository +2. Run `docker compose up keycloak keycloakdb` +3. Open `http://localhost:8080`, login admin pw admin +4. Configure Qiita as a service, create a user +5. Edit `config_qiita_oidc.cfg` to fit your local Keycloak configuration, remove # from necessary oidc block. +6. Open a new terminal, move into Image Folder `cd Images/qiita` +7. Build docker image for qiita and nginx according to steps 3 and 4 from the instructions above. +8. Move to folder containing compose file `cd ../..` +9. Copy the `qiita_db.env.example` and the `qiita.env.example` files, configure them to your needs, and delete the `.example` from the file names. +10. Run docker compose `docker compose up qiita qiita-db redis nginx` +11. Open `http://localhost:8383` + +### IF YOU WANT TO RUN MULTIPLE INSTANCES WITHOUT SUPERVISORD + +1. Perform all the steps listed in the keycloak instructions until you arrive at step 10 +2. Check the amount of replicas you desire for your run in the compose file. +3. Run docker compose with `docker compose up qiita qiita-db redis qiita_worker nginx` +4. Open `http://localhost:8383` diff --git a/compose.yaml b/compose.yaml index e581619..8bb1cdd 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,52 +1,159 @@ -version: '3' - services: - keycloak-db: + qiita-db: image: postgres:15 - container_name: keycloak-db - hostname: keycloak-db - restart: unless-stopped + container_name: qiita-db + hostname: qiita-db + restart: no env_file: - - ./environments/db.env + - ./environments/qiita_db.env environment: - POSTGRES_DB=postgres - POSTGRES_USER=postgres - - KEYCLOAK_DB_NAME=keycloak - - KEYCLOAK_DB_USER=keycloak volumes: - - './environments/db-init.sh:/docker-entrypoint-initdb.d/db-init.sh' + - './environments/qiita-db-init.sh:/docker-entrypoint-initdb.d/qiita-db-init.sh' - 'postgres-data:/var/lib/postgresql/data' + networks: + - qiita-net - keycloak: - image: quay.io/keycloak/keycloak:24.0.2 - command: ['start'] - ports: - - "8282:8282" # as the HAproxy of BCF if configured to forward requests here - restart: unless-stopped + qiita: + image: local-qiita:latest + build: # image wird hier direkt gebaut + context: ./Images/qiita + dockerfile: Dockerfile + command: ['./start_qiita.sh'] # executes bash script inside the container + # entrypoint: /bin/bash + # stdin_open: true + # tty: true + # ports: + # - "21174:21174" # wihtout nginx + # - 127.0.0.1:8383:8383 #damit bur ich dran komme + restart: no + depends_on: + - qiita-db + - redis + env_file: + - './environments/qiita.env' + environment: + - QIITA_ROOTCA_CERT=/qiita/qiita_core/support_files/ci_rootca.crt + - QIITA_CONFIG_FP=/config_qiita_oidc.cfg + volumes: + - qiita-data:/qiita + - ./qiita_logs:/qiita_logs + - ./Images/qiita/config_qiita_oidc.cfg:/qiita/config_qiita_oidc.cfg + # - ./Images/qiita/start_qiita.sh:/qiita/start_qiita.sh + networks: + - qiita-net + # deploy: + # replicas: 3 + qiita_worker: + image: local-qiita:latest + build: # image wird hier direkt gebaut + context: ./Images/qiita + dockerfile: Dockerfile + command: ['./start_qiita_worker.sh'] # executes bash script inside the container + # entrypoint: /bin/bash + # stdin_open: true + # tty: true + # ports: + # - "21174:21174" # wihtout nginx + # - 127.0.0.1:8383:8383 #damit bur ich dran komme + restart: no depends_on: - - keycloak-db + - qiita-db + - redis + - qiita env_file: - - './environments/keycloak.env' + - './environments/qiita.env' environment: - - KEYCLOAK_ADMIN=admin - - KC_DB=postgres - - KC_DB_URL_HOST=keycloak-db - - KC_DB_URL_PORT=5432 - - KC_DB_URL_DATABASE=keycloak - - KC_DB_USERNAME=keycloak - - KC_DB_SCHEMA=public - - KC_HTTP_HOST=0.0.0.0 - - KC_HTTP_ENABLED=false - - KC_HTTP_PORT=8282 - - KC_PROXY=edge - - KC_PROXY_HEADERS=forwarded - - KC_PROXY_ADDRESS_FORWARDING=true - - KC_HOSTNAME_URL=https://keycloak.jlab.bio - - KC_HOSTNAME_ADMIN_URL=https://keycloak.jlab.bio - - KC_LOG_LEVEL=WARN - #- KC_HOSTNAME_DEBUG=true - #- JAVA_OPTS_APPEND="-Djava.net.preferIPv4Stack=true" + - QIITA_ROOTCA_CERT=/qiita/qiita_core/support_files/ci_rootca.crt + - QIITA_CONFIG_FP=/config_qiita_oidc.cfg + volumes: + - qiita-data:/qiita + - ./qiita_logs:/qiita_logs + - ./Images/qiita/config_qiita_oidc.cfg:/qiita/config_qiita_oidc.cfg + # - ./Images/qiita/start_qiita_worker.sh:/qiita/start_qiita_worker.sh + networks: + - qiita-net + deploy: + replicas: 3 + redis: + image: redis:latest + restart: no + command: > + sh -c "redis-server --port 7777 && + redis-server --port 6379" + volumes: + - qiita-data:/qiita + - ./qiita_logs:/qiita_logs + networks: + - qiita-net + keycloak: # from https://stackoverflow.com/questions/78071458/keycloak-docker-compose + image: quay.io/keycloak/keycloak:24.0.2 + container_name: keycloak + environment: + KC_DB: postgres + KC_DB_URL: jdbc:postgresql://keycloakdb:5432/keycloak + KC_DB_USERNAME: keycloak + KC_DB_PASSWORD: password + + KC_HOSTNAME: keycloak + KC_HOSTNAME_PORT: 9999 + KC_HOSTNAME_STRICT: false + KC_HOSTNAME_STRICT_HTTPS: false + + KC_LOG_LEVEL: info + KC_METRICS_ENABLED: false + KC_HEALTH_ENABLED: true + KEYCLOAK_ADMIN: admin + KEYCLOAK_ADMIN_PASSWORD: admin + command: ["start-dev", "--http-port=9999"] + depends_on: + - keycloakdb + ports: + - 127.0.0.1:9999:9999 + networks: + - qiita-net + + keycloakdb: + image: postgres:15 + volumes: + - keycloak-postgres-data:/var/lib/postgresql/data + environment: + POSTGRES_DB: keycloak + POSTGRES_USER: keycloak + POSTGRES_PASSWORD: password + networks: + - qiita-net + + nginx: + image: local-nginx_qiita:latest + build: + context: ./Images/nginx + dockerfile: Dockerfile + ports: + - "8383:8383" + command: ['./start_nginx.sh'] + # stdin_open: true + # tty: true + restart: no + depends_on: + - qiita + volumes: + - qiita-data:/qiita + - ./nginx_logs:/qiita_logs + - ./Images/nginx/nginx_qiita.conf:/nginx_qiita.conf + networks: + - qiita-net +networks: + qiita-net: + name: qiita-net + volumes: postgres-data: + name: qiita-postgres-data + keycloak-postgres-data: name: keycloak-postgres-data + qiita-data: + name: qiita-data + diff --git a/environments/qiita-db-init.sh b/environments/qiita-db-init.sh new file mode 100644 index 0000000..3534aa4 --- /dev/null +++ b/environments/qiita-db-init.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +set -e +set -u + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" "$POSTGRES_DB" <<-EOSQL + ALTER DATABASE "$POSTGRES_DB" OWNER TO "$POSTGRES_USER" +EOSQL diff --git a/environments/qiita.env.example b/environments/qiita.env.example new file mode 100644 index 0000000..74ee17a --- /dev/null +++ b/environments/qiita.env.example @@ -0,0 +1,2 @@ +# This is a place for additional configurations to your Qiita installation +# which are not addressed in the Qiita image itself \ No newline at end of file diff --git a/environments/qiita_db.env.example b/environments/qiita_db.env.example new file mode 100644 index 0000000..f8ac7cf --- /dev/null +++ b/environments/qiita_db.env.example @@ -0,0 +1 @@ +POSTGRES_PASSWORD=supersecretpassword \ No newline at end of file diff --git a/keycloak_compose.yaml b/keycloak_compose.yaml new file mode 100644 index 0000000..e581619 --- /dev/null +++ b/keycloak_compose.yaml @@ -0,0 +1,52 @@ +version: '3' + +services: + keycloak-db: + image: postgres:15 + container_name: keycloak-db + hostname: keycloak-db + restart: unless-stopped + env_file: + - ./environments/db.env + environment: + - POSTGRES_DB=postgres + - POSTGRES_USER=postgres + - KEYCLOAK_DB_NAME=keycloak + - KEYCLOAK_DB_USER=keycloak + volumes: + - './environments/db-init.sh:/docker-entrypoint-initdb.d/db-init.sh' + - 'postgres-data:/var/lib/postgresql/data' + + keycloak: + image: quay.io/keycloak/keycloak:24.0.2 + command: ['start'] + ports: + - "8282:8282" # as the HAproxy of BCF if configured to forward requests here + restart: unless-stopped + depends_on: + - keycloak-db + env_file: + - './environments/keycloak.env' + environment: + - KEYCLOAK_ADMIN=admin + - KC_DB=postgres + - KC_DB_URL_HOST=keycloak-db + - KC_DB_URL_PORT=5432 + - KC_DB_URL_DATABASE=keycloak + - KC_DB_USERNAME=keycloak + - KC_DB_SCHEMA=public + - KC_HTTP_HOST=0.0.0.0 + - KC_HTTP_ENABLED=false + - KC_HTTP_PORT=8282 + - KC_PROXY=edge + - KC_PROXY_HEADERS=forwarded + - KC_PROXY_ADDRESS_FORWARDING=true + - KC_HOSTNAME_URL=https://keycloak.jlab.bio + - KC_HOSTNAME_ADMIN_URL=https://keycloak.jlab.bio + - KC_LOG_LEVEL=WARN + #- KC_HOSTNAME_DEBUG=true + #- JAVA_OPTS_APPEND="-Djava.net.preferIPv4Stack=true" + +volumes: + postgres-data: + name: keycloak-postgres-data diff --git a/qiita_logs/placeholder.txt b/qiita_logs/placeholder.txt new file mode 100644 index 0000000..e9f85fa --- /dev/null +++ b/qiita_logs/placeholder.txt @@ -0,0 +1 @@ +# placeholder so we can push this folder to git \ No newline at end of file