Improvement database dependencies

.github/workflows/docker_build_bots.yaml

@@ -10,15 +10,16 @@ on:
     - 'docker/entrypoint.sh'
     - 'docker/gunicorn_conf.py'
     branches:
-    - main
+    - master
+    - awake
 
 permissions:
   packages: write
 
 jobs:
   build:
     name: build docker
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
@@ -35,4 +36,5 @@ jobs:
           file: docker/Dockerfile.bots
           push: true
           tags: |
+            ghcr.io/${{ env.GHCR_OWNER }}/taranis-ng-bots:latest
             ghcr.io/${{ env.GHCR_OWNER }}/taranis-ng-bots:${{ github.ref_name }}

.github/workflows/docker_build_collectors.yaml

@@ -10,15 +10,16 @@ on:
     - 'docker/entrypoint.sh'
     - 'docker/gunicorn_conf.py'
     branches:
-    - main
+    - master
+    - awake
 
 permissions:
   packages: write
 
 jobs:
   build:
     name: build docker
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
@@ -35,4 +36,5 @@ jobs:
           file: docker/Dockerfile.collectors
           push: true
           tags: |
+            ghcr.io/${{ env.GHCR_OWNER }}/taranis-ng-collectors:latest
             ghcr.io/${{ env.GHCR_OWNER }}/taranis-ng-collectors:${{ github.ref_name }}

.github/workflows/docker_build_core.yaml

@@ -10,15 +10,62 @@ on:
     - 'docker/entrypoint.sh'
     - 'docker/gunicorn_conf.py'
     branches:
-    - main
+    - master
+    - awake
+    tags:
+    - '**'
 
 permissions:
   packages: write
+  contents: write
 
 jobs:
-  build:
+  test:
+    name: pytest
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: test
+        working-directory: src/core
+        run: |
+          python -m pip install --upgrade -r ../../dev_requirements.txt
+          pip install -e .
+          echo "Starting Pytest"
+          pytest
+
+  build_wheel:
+    name: build wheel
+    runs-on: ubuntu-22.04
+    needs: test
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+      - name: build
+        working-directory: src/core
+        run: |
+          python -m pip install --upgrade pip setuptools wheel build
+          python -m build
+      - name: Release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          files: |
+            src/core/dist/taranis_ng_core-*.whl
+            src/core/dist/taranis_ng_core-*.tar.gz
+      - name: Release to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN_CORE }}
+
+  build_docker:
     name: build docker
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
@@ -35,4 +82,5 @@ jobs:
           file: docker/Dockerfile.core
           push: true
           tags: |
+            ghcr.io/${{ env.GHCR_OWNER }}/taranis-ng-core:latest
             ghcr.io/${{ env.GHCR_OWNER }}/taranis-ng-core:${{ github.ref_name }}

.github/workflows/docker_build_gui.yaml

@@ -7,15 +7,16 @@ on:
     - 'docker/Dockerfile.gui'
     - 'VERSION.md'
     branches:
-    - main
+    - master
+    - awake
 
 permissions:
   packages: write
 
 jobs:
   build:
     name: build docker
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
@@ -32,4 +33,5 @@ jobs:
           file: docker/Dockerfile.gui
           push: true
           tags: |
+            ghcr.io/${{ env.GHCR_OWNER }}/taranis-ng-gui:latest
             ghcr.io/${{ env.GHCR_OWNER }}/taranis-ng-gui:${{ github.ref_name }}

.github/workflows/docker_build_presenters.yaml

@@ -10,15 +10,16 @@ on:
     - 'docker/entrypoint.sh'
     - 'docker/gunicorn_conf.py'
     branches:
-    - main
+    - master
+    - awake
 
 permissions:
   packages: write
 
 jobs:
   build:
     name: build docker
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
@@ -35,4 +36,5 @@ jobs:
           file: docker/Dockerfile.presenters
           push: true
           tags: |
+            ghcr.io/${{ env.GHCR_OWNER }}/taranis-ng-presenters:latest
             ghcr.io/${{ env.GHCR_OWNER }}/taranis-ng-presenters:${{ github.ref_name }}

.github/workflows/docker_build_publishers.yaml

@@ -10,15 +10,16 @@ on:
     - 'docker/entrypoint.sh'
     - 'docker/gunicorn_conf.py'
     branches:
-    - main
+    - master
+    - awake
 
 permissions:
   packages: write
 
 jobs:
   build:
     name: build docker
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
@@ -35,4 +36,5 @@ jobs:
           file: docker/Dockerfile.publishers
           push: true
           tags: |
+            ghcr.io/${{ env.GHCR_OWNER }}/taranis-ng-publishers:latest
             ghcr.io/${{ env.GHCR_OWNER }}/taranis-ng-publishers:${{ github.ref_name }}

.github/workflows/docker_build_shared.yaml

@@ -25,7 +25,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip setuptools wheel build
           python -m build
-      - name: Release
+      - name: Release to GitHub
         uses: softprops/action-gh-release@v1
         if: startsWith(github.ref, 'refs/tags/')
         with:

.github/workflows/linting.yaml

@@ -7,12 +7,12 @@ on:
 jobs:
   lint:
     name: lint & check formatting
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       matrix:
         python-version: ['3.8', '3.10']
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - name: lint with black
         uses: rickstaa/action-black@v1
         with:

.gitignore

@@ -1,11 +1,11 @@
 # built and cached files
 venv/
+.envrc
 __pycache__/
 *.pyc
 node_modules/
 dist/
 build/
-.envrc
 *.egg-info/
 .installed.cfg
 *.egg
@@ -20,6 +20,7 @@ build/
 
 # sensitive data not to be commited
 .env.local
+docker/.env
 .env.*.local
 src/.env
 *.pem

dev_requirements.txt

@@ -0,0 +1,8 @@
+flake8
+black
+watchdog
+pytest
+pre-commit
+build
+wheel
+setuptools

docker/.env → docker/.env.sample

@@ -1,5 +1,6 @@
 # General
 COMPOSE_PROJECT_NAME=taranis-ng
+DOCKER_IMAGE_NAMESPACE=skcert
 HTTP_PROXY=
 HTTPS_PROXY=
 
@@ -9,7 +10,8 @@ TZ=Europe/Bratislava
 # Default passwords. CHANGE THESE FOR PRODUCTION!
 POSTGRES_PASSWORD=supersecret
 JWT_SECRET_KEY=supersecret
-COLLECTOR_PRESENTER_PUBLISHER_API_KEY=supersecret
+API_KEY=supersecret
+KEYCLOAK_PASSWORD=supersecret
 
 # Paths
 CVE_UPDATE_FILE=/data/cve_dictionary.xml
@@ -19,13 +21,21 @@ CPE_UPDATE_FILE=/data/cpe_dictionary.xml
 TARANIS_NG_HOSTNAME=localhost
 TARANIS_NG_HTTPS_PORT=4443
 TARANIS_NG_HTTP_PORT=8080
-TARANIS_NG_HTTPS_URI=https://localhost:4443
+
+DB_URL=localhost
+DB_DATABASE=taranis
+DB_USER=taranis
+DB_PASSWORD=supersecret
+
+TARANIS_NG_HTTPS_URI=https://taranis:4443
+KEYCLOAK_DOMAIN=keycloak.local
 TRAEFIK_MANAGEMENT_PORT=127.0.0.1:8081
 
 # Versions
 REDIS_TAG=6-alpine
 POSTGRES_TAG=13-alpine
-TARANIS_NG_TAG=v22.05.1
+TARANIS_NG_TAG=latest
+KEYCLOAK_TAG=16.1.0
 
 # Limits
 DB_SHARED_BUFFERS=64MB

docker/Dockerfile.bots

@@ -1,4 +1,4 @@
-FROM python:3.7-alpine3.14 AS build_shared
+FROM python:3-slim AS build_shared
 
 WORKDIR /build_shared/
 
@@ -7,38 +7,28 @@ COPY ./src/shared/. .
 RUN python -m build
 
 
-
-FROM python:3.7-alpine3.14 AS production
-
+FROM python:3-slim
 WORKDIR /app/
 
 # upgrade pip
-RUN python -m pip install --upgrade pip
+RUN python -m pip install --upgrade pip wheel
 
-# install "shared" package from build_shared stage
-# TODO: somehow squash the following two layers into one to conserve space
+# install dependencies
+RUN apt-get update && apt-get upgrade -y
+COPY ./src/bots/setup.cfg ./src/bots/setup.py /app/
 
 COPY --from=build_shared /build_shared/dist/taranis_ng_shared-*.whl custom_packages/
 RUN pip install --no-cache-dir ./custom_packages/taranis_ng_shared-*.whl && rm -rf ./custom_packages/
 
-# install dependencies
-COPY ./src/bots/requirements.txt /app/requirements.txt
-RUN \
-    apk add --no-cache --virtual .build-deps \
-    gcc \
-    g++ \
-    make \
-    musl-dev \
-    python3-dev \
-    libffi-dev && \
-    pip install --no-cache-dir -r /app/requirements.txt && \
-    apk --purge del .build-deps
-
-COPY ./docker/start.sh /start.sh
-RUN chmod +x /start.sh
-
-COPY ./docker/prestart.sh /app/prestart.sh
-RUN chmod +x /app/prestart.sh
+RUN apt-get install --no-install-recommends -y \
+      build-essential \
+      python3-dev \
+      git \
+      cmake && \
+    pip install -e /app/ && \
+    apt-get purge -y build-essential python3-dev && \
+    apt-get autoremove -y --purge &&\
+    apt-get clean
 
 COPY ./docker/entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
@@ -51,10 +41,8 @@ EXPOSE 80
 
 # setup environment variables
 ENV PYTHONPATH=/app
-ENV MODULE_NAME run
-ENV VARIABLE_NAME app
+ENV APP_MODULE run:app
 ENV GUNICORN_CMD_ARGS --timeout 120
-ENV WORKERS_PER_CORE 8
+ENV WORKERS_PER_CORE 4
 
 ENTRYPOINT ["/entrypoint.sh"]
-CMD ["/start.sh"]